from __future__ import print_function from argparse import ArgumentParser from hashlib import md5 from datetime import datetime import sqlite3 import time import json import os strptime = datetime.strptime ARG_DATE_FMT = '%Y-%m-%d %H:%M:%S' simple_date = lambda s: strptime(s, ARG_DATE_FMT) def find_hash(hsh, db=None): cursor = db.cursor() cursor.execute("select * from file where hash = ?", (hsh, ), ) return cursor.fetchone() def insert_file(hsh, filename, timestamp, db=None): cursor = db.cursor() cursor.execute(""" insert into file(hash, filename, timestamp) values(?, ?, ?) """, (hsh, filename, int(time.mktime(timestamp.timetuple())), ) ) def main(): parser = ArgumentParser() subparsers = parser.add_subparsers(dest='entity') dbparser = subparsers.add_parser('db') dbparser.add_argument('action') dbparser.add_argument('-from-date', dest='from_date', type=simple_date, ) dbparser.add_argument('--dry-run', dest='dry_run', action='store_const', const=True, default=False ) args = parser.parse_args() action = args.action with open('/etc/fourier-config.json', 'r') as fp: config = json.loads(fp.read()) device_id = config['device_id'] dbpath = '/var/fourier/{}/files.db'.format(device_id) if action == 'stats': conn = sqlite3.connect(dbpath) cursor = conn.cursor() cursor.execute("select count(*), count(uploaded) from file") total, uploaded, = cursor.fetchone() print("total: {}".format(total)) print("uploaded: {}".format(uploaded)) print("pending: {}".format(total - uploaded)) elif action == 'index-files': counter = 0 already_indexed = 0 conn = sqlite3.connect(dbpath) path = os.path.join('/var/fourier', device_id) for folder, folders, files in os.walk(path): for file in files: if file.endswith('.mp3'): filename = os.path.join(folder, file) ahash = md5() with open(filename, 'rb') as fp: while True: data = fp.read(4096) if data: ahash.update(data) else: break thehash = ahash.hexdigest() exists = find_hash(thehash, db=conn) if not exists: if args.from_date: dt = datetime.strptime( file[:19], '%Y-%m-%dT%H-%M-%S' ) if dt >= args.from_date: counter += 1 insert_file(thehash, filename, dt, db=conn) print(dt) else: counter += 1 insert_file(thehash, filename, dt, db=conn) print(dt) else: already_indexed += 1 print('already indexed: {}'.format(filename)) if not args.dry_run: conn.commit() else: conn.rollback() print('\n[WARNING] DRY RUN FINISHED') print('----------------------------------') print('total files indexed: {}'.format(counter)) print('total files in existence: {}'.format(already_indexed)) print('----------------------------------') if __name__ == '__main__': main()