From c5890ab1202cb859dd1aecca3e6b55f1627dfc14 Mon Sep 17 00:00:00 2001 From: Emory P Date: Sun, 29 Dec 2013 14:30:12 -0500 Subject: [PATCH] Changed db to use materialized paths for all folder and track operations Changed CLI to use flask-script (much cleaner and smaller) --- api/albums_songs.py | 5 +- api/browse.py | 16 ++- api/media.py | 4 +- cli.py | 281 ++++++++++++++++++-------------------------- db.py | 34 +++--- folder.py | 2 + main.py | 3 +- managers/folder.py | 47 ++++---- scanner.py | 91 +++++++------- web.py | 51 ++++---- 10 files changed, 253 insertions(+), 281 deletions(-) diff --git a/api/albums_songs.py b/api/albums_songs.py index 8bdfe42..b286cda 100755 --- a/api/albums_songs.py +++ b/api/albums_songs.py @@ -30,7 +30,8 @@ def rand_songs(): if genre: query = query.filter(Track.genre == genre) if fid: - query = query.filter(Track.root_folder_id == fid) + f = Folder.query.get(fid) + query = query.filter(Track.path.like(f.path + '%')) count = query.count() if not count: @@ -82,6 +83,8 @@ def album_list(): elif ltype == 'alphabeticalByName': query = query.order_by(Folder.name) elif ltype == 'alphabeticalByArtist': + # this is a mess because who knows how your file structure is set up + # with the database changes it's more difficult to get the parent of a dir parent = aliased(Folder) query = query.join(parent, Folder.parent).order_by(parent.name).order_by(Folder.name) else: diff --git a/api/browse.py b/api/browse.py index 2fbe0f9..e9accc5 100755 --- a/api/browse.py +++ b/api/browse.py @@ -2,7 +2,7 @@ from flask import request from web import app -from db import Folder, Artist, Album, Track +from db import Folder, Artist, Album, Track, func from api import get_entity import uuid, time, string import os.path @@ -14,7 +14,7 @@ def list_folders(): 'musicFolder': [ { 'id': str(f.id), 'name': f.name - } for f in Folder.query.filter(Folder.root == True).order_by(Folder.name).all() ] + } for f in Folder.query.filter(Folder.root == True).order_by(Folder.path).all() ] } }) @@ -51,10 +51,10 @@ def list_indexes(): artists = [] childs = [] for f in folder: - artists += f.children + artists += f.get_children() childs += f.tracks else: - artists = folder.children + artists = folder.get_children() childs = folder.tracks indexes = {} @@ -95,10 +95,14 @@ def show_directory(): directory = { 'id': str(res.id), 'name': res.name, - 'child': [ f.as_subsonic_child(request.user) for f in sorted(res.children, key = lambda c: c.name.lower()) ] + [ t.as_subsonic_child(request.user) for t in sorted(res.tracks, key = lambda t: t.sort_key()) ] + 'child': [ f.as_subsonic_child(request.user) for f in res.get_children() ] + [ t.as_subsonic_child(request.user) for t in sorted(res.tracks, key = lambda t: t.sort_key()) ] } if not res.root: - directory['parent'] = str(res.parent_id) + parent = Folder.query.with_entities(Folder.id) \ + .filter(Folder.path.like(res.path[:len(res.path)-len(res.name)-1])) \ + .order_by(func.length(Folder.path).desc()).first() + if parent: + directory['parent'] = str(parent.id) return request.formatter({ 'directory': directory }) diff --git a/api/media.py b/api/media.py index 7c5a469..1186ea3 100755 --- a/api/media.py +++ b/api/media.py @@ -49,7 +49,7 @@ def stream_media(): if redirect and xsendfile: response.headers['X-Accel-Charset'] = 'utf-8' response.headers['X-Accel-Redirect'] = redirect + xsendfile.encode('UTF8') - app.logger.debug('X-Accel-Redirect: ' + redirect + xsendfile.encode('UTF8')) + app.logger.debug('X-Accel-Redirect: ' + redirect + xsendfile) return response status, res = get_entity(request, Track) @@ -127,7 +127,7 @@ def stream_media(): encoder = transcoder[pipe_index+1:] transcoder = None - app.logger.warn('decoder' + str(decoder) + '\nencoder' + str(encoder)) + app.logger.debug('decoder' + str(decoder) + '\nencoder' + str(encoder)) try: if transcoder: diff --git a/cli.py b/cli.py index 075bb71..19237bd 100755 --- a/cli.py +++ b/cli.py @@ -1,203 +1,146 @@ # coding: utf-8 -import sys, cmd, argparse, getpass + import config +config.check() -class CLIParser(argparse.ArgumentParser): - def error(self, message): - self.print_usage(sys.stderr) - raise RuntimeError(message) +from web import app +import db +from flask.ext.script import Manager, Command, Option, prompt_pass +import os.path +from managers.folder import FolderManager +from managers.user import UserManager +from scanner import Scanner -class CLI(cmd.Cmd): - prompt = "supysonic> " +manager = Manager(app) - def _make_do(self, command): - def method(obj, line): - try: - args = getattr(obj, command + '_parser').parse_args(line.split()) - except RuntimeError, e: - print >>sys.stderr, e - return +@manager.command +def folder_list(): + "Lists all Folders to Scan" + print 'Name\t\tPath\n----\t\t----' + print '\n'.join('{0: <16}{1}'.format(f.name, f.path) for f in db.Folder.query.filter(db.Folder.root == True)) - if hasattr(obj.__class__, command + '_subparsers'): - try: - func = getattr(obj, '{}_{}'.format(command, args.action)) - except AttributeError: - return obj.default(line) - return func(** { key: vars(args)[key] for key in vars(args) if key != 'action' }) - else: - try: - func = getattr(obj, command) - except AttributeError: - return obj.default(line) - return func(**vars(args)) - return method +@manager.command +def folder_add(name, path): + "Add a folder to the Library" + ret = FolderManager.add(path) + if ret != FolderManager.SUCCESS: + print FolderManager.error_str(ret) + else: + print "Folder '{}' added".format(name) - def __init__(self): - cmd.Cmd.__init__(self) +@manager.command +def folder_delete(path): + "Delete folder from Library" - # Generate do_* and help_* methods - for parser_name in filter(lambda attr: attr.endswith('_parser') and '_' not in attr[:-7], dir(self.__class__)): - command = parser_name[:-7] + s = Scanner(db.session) - if not hasattr(self.__class__, 'do_' + command): - setattr(self.__class__, 'do_' + command, self._make_do(command)) + ret = FolderManager.delete_by_name(path, s) + if ret != FolderManager.SUCCESS: + print FolderManager.error_str(ret) + else: + print "Deleted folder" + path - if hasattr(self.__class__, 'do_' + command) and not hasattr(self.__class__, 'help_' + command): - setattr(self.__class__, 'help_' + command, getattr(self.__class__, parser_name).print_help) - if hasattr(self.__class__, command + '_subparsers'): - for action, subparser in getattr(self.__class__, command + '_subparsers').choices.iteritems(): - setattr(self, 'help_{} {}'.format(command, action), subparser.print_help) +@manager.command +def folder_scan(): + s = Scanner(db.session) - def do_EOF(self, line): - return True + folders = db.Folder.query.filter(db.Folder.root == True) - do_exit = do_EOF + if folders: + for folder in folders: + print "Scanning: " + folder.path + FolderManager.scan(folder.id, s) - def default(self, line): - print 'Unknown command %s' % line.split()[0] - self.do_help(None) + added, deleted = s.stats() - def postloop(self): - print + print "\a" + print "Scanning done" + print 'Added: %i artists, %i albums, %i tracks' % (added[0], added[1], added[2]) + print 'Deleted: %i artists, %i albums, %i tracks' % (deleted[0], deleted[1], deleted[2]) - def completedefault(self, text, line, begidx, endidx): - command = line.split()[0] - parsers = getattr(self.__class__, command + '_subparsers', None) - if not parsers: - return [] +@manager.command +def folder_prune(): + s = Scanner(db.session) - num_words = len(line[len(command):begidx].split()) - if num_words == 0: - return [ a for a in parsers.choices.keys() if a.startswith(text) ] - return [] + folders = db.Folder.query.filter(db.Folder.root == True) - folder_parser = CLIParser(prog = 'folder', add_help = False) - folder_subparsers = folder_parser.add_subparsers(dest = 'action') - folder_subparsers.add_parser('list', help = 'Lists folders', add_help = False) - folder_add_parser = folder_subparsers.add_parser('add', help = 'Adds a folder', add_help = False) - folder_add_parser.add_argument('name', help = 'Name of the folder to add') - folder_add_parser.add_argument('path', help = 'Path to the directory pointed by the folder') - folder_del_parser = folder_subparsers.add_parser('delete', help = 'Deletes a folder', add_help = False) - folder_del_parser.add_argument('name', help = 'Name of the folder to delete') - folder_scan_parser = folder_subparsers.add_parser('scan', help = 'Run a scan on specified folders', add_help = False) - folder_scan_parser.add_argument('folders', metavar = 'folder', nargs = '*', help = 'Folder(s) to be scanned. If ommitted, all folders are scanned') + if folders: + for folder in folders: + print "Pruning: " + folder.path + FolderManager.prune(folder.id, s) - def folder_list(self): - print 'Name\t\tPath\n----\t\t----' - print '\n'.join('{0: <16}{1}'.format(f.name, f.path) for f in db.Folder.query.filter(db.Folder.root == True)) + added, deleted = s.stats() - def folder_add(self, name, path): - ret = FolderManager.add(name, path) - if ret != FolderManager.SUCCESS: - print FolderManager.error_str(ret) - else: - print "Folder '{}' added".format(name) + print "\a" + print "Pruning done" + print 'Added: %i artists, %i albums, %i tracks' % (added[0], added[1], added[2]) + print 'Deleted: %i artists, %i albums, %i tracks' % (deleted[0], deleted[1], deleted[2]) - def folder_delete(self, name): - ret = FolderManager.delete_by_name(name) - if ret != FolderManager.SUCCESS: - print FolderManager.error_str(ret) - else: - print "Deleted folder '{}'".format(name) +@manager.command +def user_list(): + print 'Name\t\tAdmin\tEmail\n----\t\t-----\t-----' + print '\n'.join('{0: <16}{1}\t{2}'.format(u.name, '*' if u.admin else '', u.mail) for u in db.User.query.all()) - def folder_scan(self, folders): - s = Scanner(db.session) - if folders: - folders = map(lambda n: db.Folder.query.filter(db.Folder.name == n and db.Folder.root == True).first() or n, folders) - print folders - if any(map(lambda f: isinstance(f, basestring), folders)): - print "No such folder(s): " + ' '.join(f for f in folders if isinstance(f, basestring)) - for folder in filter(lambda f: isinstance(f, db.Folder), folders): - FolderManager.scan(folder.id, s) - else: - for folder in db.Folder.query.filter(db.Folder.root == True): - FolderManager.scan(folder.id, s) +@manager.command +def user_add(name, admin=False, email=None): + password = prompt_pass("Please enter a password") + if password: + status = UserManager.add(name, password, email, admin) + if status != UserManager.SUCCESS: + print >>sys.stderr, UserManager.error_str(status) - added, deleted = s.stats() - db.session.commit() +@manager.command +def user_delete(name): + user = db.User.query.filter(db.User.name == name).first() + if not user: + print >>sys.stderr, 'No such user' + else: + db.session.delete(user) + db.session.commit() + print "User '{}' deleted".format(name) - print "Scanning done" - print 'Added: %i artists, %i albums, %i tracks' % (added[0], added[1], added[2]) - print 'Deleted: %i artists, %i albums, %i tracks' % (deleted[0], deleted[1], deleted[2]) +@manager.command +def user_setadmin(name, off): + user = db.User.query.filter(db.User.name == name).first() + if not user: + print >>sys.stderr, 'No such user' + else: + user.admin = not off + db.session.commit() + print "{0} '{1}' admin rights".format('Revoked' if off else 'Granted', name) - user_parser = CLIParser(prog = 'user', add_help = False) - user_subparsers = user_parser.add_subparsers(dest = 'action') - user_subparsers.add_parser('list', help = 'List users', add_help = False) - user_add_parser = user_subparsers.add_parser('add', help = 'Adds a user', add_help = False) - user_add_parser.add_argument('name', help = 'Name/login of the user to add') - user_add_parser.add_argument('-a', '--admin', action = 'store_true', help = 'Give admin rights to the new user') - user_add_parser.add_argument('-p', '--password', help = "Specifies the user's password") - user_add_parser.add_argument('-e', '--email', default = '', help = "Sets the user's email address") - user_del_parser = user_subparsers.add_parser('delete', help = 'Deletes a user', add_help = False) - user_del_parser.add_argument('name', help = 'Name/login of the user to delete') - user_admin_parser = user_subparsers.add_parser('setadmin', help = 'Enable/disable admin rights for a user', add_help = False) - user_admin_parser.add_argument('name', help = 'Name/login of the user to grant/revoke admin rights') - user_admin_parser.add_argument('--off', action = 'store_true', help = 'Revoke admin rights if present, grant them otherwise') - user_pass_parser = user_subparsers.add_parser('changepass', help = "Changes a user's password", add_help = False) - user_pass_parser.add_argument('name', help = 'Name/login of the user to which change the password') - user_pass_parser.add_argument('password', nargs = '?', help = 'New password') +@manager.command +def user_changepass(name, password): + if not password: + password = getpass.getpass() + confirm = getpass.getpass('Confirm password: ') + if password != confirm: + print >>sys.stderr, "Passwords don't match" + return + status = UserManager.change_password2(name, password) + if status != UserManager.SUCCESS: + print >>sys.stderr, UserManager.error_str(status) + else: + print "Successfully changed '{}' password".format(name) - def user_list(self): - print 'Name\t\tAdmin\tEmail\n----\t\t-----\t-----' - print '\n'.join('{0: <16}{1}\t{2}'.format(u.name, '*' if u.admin else '', u.mail) for u in db.User.query.all()) +@manager.command +def init_db(): + db.init_db() - def user_add(self, name, admin, password, email): - if not password: - password = getpass.getpass() - confirm = getpass.getpass('Confirm password: ') - if password != confirm: - print >>sys.stderr, "Passwords don't match" - return - status = UserManager.add(name, password, email, admin) - if status != UserManager.SUCCESS: - print >>sys.stderr, UserManager.error_str(status) +@manager.command +def recreate_db(): + db.recreate_db() - def user_delete(self, name): - user = db.User.query.filter(db.User.name == name).first() - if not user: - print >>sys.stderr, 'No such user' - else: - db.session.delete(user) - db.session.commit() - print "User '{}' deleted".format(name) - - def user_setadmin(self, name, off): - user = db.User.query.filter(db.User.name == name).first() - if not user: - print >>sys.stderr, 'No such user' - else: - user.admin = not off - db.session.commit() - print "{0} '{1}' admin rights".format('Revoked' if off else 'Granted', name) - - def user_changepass(self, name, password): - if not password: - password = getpass.getpass() - confirm = getpass.getpass('Confirm password: ') - if password != confirm: - print >>sys.stderr, "Passwords don't match" - return - status = UserManager.change_password2(name, password) - if status != UserManager.SUCCESS: - print >>sys.stderr, UserManager.error_str(status) - else: - print "Successfully changed '{}' password".format(name) if __name__ == "__main__": - if not config.check(): - sys.exit(1) + import config - import db - db.init_db() + if not config.check(): + sys.exit(1) - from managers.folder import FolderManager - from managers.user import UserManager - from scanner import Scanner - - if len(sys.argv) > 1: - CLI().onecmd(' '.join(sys.argv[1:])) - else: - CLI().cmdloop() + if not os.path.exists(config.get('base', 'cache_dir')): + os.makedirs(config.get('base', 'cache_dir')) + manager.run() diff --git a/db.py b/db.py index 587dd40..3fa4155 100755 --- a/db.py +++ b/db.py @@ -4,9 +4,11 @@ import config from flask.ext.sqlalchemy import SQLAlchemy from sqlalchemy.types import TypeDecorator, BINARY +from sqlalchemy.ext.hybrid import * from sqlalchemy.dialects.postgresql import UUID as pgUUID import uuid, datetime, time +import mimetypes import os.path database = SQLAlchemy() @@ -82,7 +84,7 @@ class User(database.Model): lastfm_session = Column(String(32), nullable = True) lastfm_status = Column(Boolean, default = True) # True: ok/unlinked, False: invalid session - last_play_id = Column(UUID, ForeignKey('track.id'), nullable = True) + last_play_id = Column(UUID, ForeignKey('track.id', ondelete = 'SET NULL'), nullable = True) last_play = relationship('Track') last_play_date = Column(DateTime, nullable = True) @@ -115,14 +117,17 @@ class Folder(database.Model): id = UUID.gen_id_column() root = Column(Boolean, default = False) - name = Column(String(256)) path = Column(String(4096)) # should be unique, but mysql don't like such large columns created = Column(DateTime, default = now) has_cover_art = Column(Boolean, default = False) last_scan = Column(Integer, default = 0) - parent_id = Column(UUID, ForeignKey('folder.id'), nullable = True) - children = relationship('Folder', backref = backref('parent', remote_side = [ id ])) + @hybrid_property + def name(self): + return self.path[self.path.rfind(os.sep) + 1:] + + def get_children(self): + return Folder.query.filter(Folder.path.like(self.path + '/%%')).filter(~Folder.path.like(self.path + '/%%/%%')) def as_subsonic_child(self, user): info = { @@ -133,8 +138,12 @@ class Folder(database.Model): 'created': self.created.isoformat() } if not self.root: - info['parent'] = str(self.parent_id) - info['artist'] = self.parent.name + parent = session.query(Folder) \ + .filter(Folder.path.like(self.path[:len(self.path)-len(self.name)-1])) \ + .order_by(func.length(Folder.path).desc()).first() + if(parent): + info['parent'] = str(parent.id) + info['artist'] = parent.name if self.has_cover_art: info['coverArt'] = str(self.id) @@ -176,7 +185,7 @@ class Album(database.Model): id = UUID.gen_id_column() name = Column(String(255)) artist_id = Column(UUID, ForeignKey('artist.id')) - tracks = relationship('Track', backref = 'album') + tracks = relationship('Track', backref = 'album', cascade="delete") def as_subsonic_album(self, user): info = { @@ -214,17 +223,14 @@ class Track(database.Model): bitrate = Column(Integer) path = Column(String(4096)) # should be unique, but mysql don't like such large columns - content_type = Column(String(32)) created = Column(DateTime, default = now) last_modification = Column(Integer) play_count = Column(Integer, default = 0) last_play = Column(DateTime, nullable = True) - root_folder_id = Column(UUID, ForeignKey('folder.id')) - root_folder = relationship('Folder', primaryjoin = Folder.id == root_folder_id) - folder_id = Column(UUID, ForeignKey('folder.id')) - folder = relationship('Folder', primaryjoin = Folder.id == folder_id, backref = 'tracks') + folder_id = Column(UUID, ForeignKey('folder.id', ondelete="CASCADE")) + folder = relationship('Folder', backref = 'tracks') def as_subsonic_child(self, user): info = { @@ -236,11 +242,11 @@ class Track(database.Model): 'artist': self.album.artist.name, 'track': self.number, 'size': os.path.getsize(self.path), - 'contentType': self.content_type, + 'contentType': mimetypes.guess_type(self.path), 'suffix': self.suffix(), 'duration': self.duration, 'bitRate': self.bitrate, - 'path': self.path[len(self.root_folder.path) + 1:], + 'path': self.path, 'isVideo': False, 'discNumber': self.disc, 'created': self.created.isoformat(), diff --git a/folder.py b/folder.py index bf3aae6..d9e3ec6 100755 --- a/folder.py +++ b/folder.py @@ -9,6 +9,8 @@ from db import session, Folder from managers.user import UserManager from managers.folder import FolderManager +import scanner + @app.before_request def check_admin(): if not request.path.startswith('/folder'): diff --git a/main.py b/main.py index 9c6020f..0f887ec 100755 --- a/main.py +++ b/main.py @@ -10,8 +10,7 @@ if __name__ == '__main__': if not os.path.exists(config.get('base', 'cache_dir')): os.makedirs(config.get('base', 'cache_dir')) - import db from web import app - app.run(host = '0.0.0.0', debug = True) + app.run(host = '0.0.0.0') diff --git a/managers/folder.py b/managers/folder.py index 30bad65..ba27ef5 100755 --- a/managers/folder.py +++ b/managers/folder.py @@ -30,8 +30,8 @@ class FolderManager: return FolderManager.SUCCESS, folder @staticmethod - def add(name, path): - if Folder.query.filter(Folder.name == name and Folder.root == True).first(): + def add(path): + if Folder.query.filter(Folder.path == path and Folder.root == True).first(): return FolderManager.NAME_EXISTS path = os.path.abspath(path) @@ -41,14 +41,14 @@ class FolderManager: if folder: return FolderManager.PATH_EXISTS - folder = Folder(root = True, name = name, path = path) + folder = Folder(root = True, path = path) session.add(folder) session.commit() return FolderManager.SUCCESS @staticmethod - def delete(uid): + def delete(uid, scanner): status, folder = FolderManager.get(uid) if status != FolderManager.SUCCESS: return status @@ -56,34 +56,26 @@ class FolderManager: if not folder.root: return FolderManager.NO_SUCH_FOLDER - # delete associated tracks and prune empty albums/artists - for artist in Artist.query.all(): - for album in artist.albums[:]: - for track in filter(lambda t: t.root_folder.id == folder.id, album.tracks): - album.tracks.remove(track) - session.delete(track) - if len(album.tracks) == 0: - artist.albums.remove(album) - session.delete(album) - if len(artist.albums) == 0: - session.delete(artist) + session.delete(folder) - def cleanup_folder(folder): - for f in folder.children: - cleanup_folder(f) - session.delete(folder) + paths = session.query(Folder.path.like(folder.path + os.sep + '%')).delete() + #for f in paths: + #if not any (p.path in f.path for p in paths) and not f.root: + #app.logger.debug('Deleting path with no parent: ' + f.path) + #self.__session.delete(f) + + scanner.prune(folder) - cleanup_folder(folder) session.commit() return FolderManager.SUCCESS @staticmethod - def delete_by_name(name): - folder = Folder.query.filter(Folder.name == name and Folder.root == True).first() + def delete_by_name(path, scanner): + folder = Folder.query.filter(Folder.path == path and Folder.root == True).first() if not folder: return FolderManager.NO_SUCH_FOLDER - return FolderManager.delete(folder.id) + return FolderManager.delete(folder.id, scanner) @staticmethod def scan(uid, scanner): @@ -95,6 +87,15 @@ class FolderManager: scanner.prune(folder) return FolderManager.SUCCESS + @staticmethod + def prune(uid, scanner): + status, folder = FolderManager.get(uid) + if status != FolderManager.SUCCESS: + return status + + scanner.prune(folder) + return FolderManager.SUCCESS + @staticmethod def error_str(err): if err == FolderManager.SUCCESS: diff --git a/scanner.py b/scanner.py index 9353beb..2387470 100755 --- a/scanner.py +++ b/scanner.py @@ -3,10 +3,11 @@ import os, os.path import time import mutagen -import config, db +import config import math import sys, traceback from web import app +import db class Scanner: def __init__(self, session): @@ -14,6 +15,7 @@ class Scanner: self.__tracks = db.Track.query.all() self.__tracks = {x.path: x for x in self.__tracks} + self.__tracktimes = {x.path: x.last_modification for x in self.__tracks.values()} self.__artists = db.Artist.query.all() self.__artists = {x.name.lower(): x for x in self.__artists} @@ -33,52 +35,59 @@ class Scanner: extensions = config.get('base', 'scanner_extensions') self.__extensions = map(str.lower, extensions.split()) if extensions else None - def scan(self, folder): - print "scanning", folder.path + def scan(self, root_folder): + print "scanning", root_folder.path valid = [x.lower() for x in config.get('base','filetypes').split(',')] valid = tuple(valid) print "valid filetypes: ",valid - for root, subfolders, files in os.walk(folder.path, topdown=False): + for root, subfolders, files in os.walk(root_folder.path, topdown=False): + if(root not in self.__folders): + app.logger.debug('Adding folder (empty): ' + root) + self.__folders[root] = db.Folder(path = root) + for f in files: if f.lower().endswith(valid): try: - self.__scan_file(os.path.join(root, f), folder) + path = os.path.join(root, f) + self.__scan_file(path, root) except: app.logger.error('Problem adding file: ' + os.path.join(root,f)) app.logger.error(traceback.print_exc()) sys.exit(0) self.__session.rollback() - print "\a" + self.__session.add_all(self.__folders.values()) self.__session.add_all(self.__tracks.values()) + root_folder.last_scan = int(time.time()) self.__session.commit() - folder.last_scan = int(time.time()) def prune(self, folder): - for path, root_folder_id, track_id in self.__session.query(db.Track.path, db.Track.root_folder_id, db.Track.id): - if root_folder_id == folder.id and not os.path.exists(path): - app.logger.debug('Removed invalid path: ' + path) - self.__remove_track(self.__session.merge(db.Track(id = track_id))) + # check for invalid paths still in database + #app.logger.debug('Checking for invalid paths...') + #for path in self.__tracks.keys(): + #if not os.path.exists(path.encode('utf-8')): + #app.logger.debug('Removed invalid path: ' + path) + #self.__remove_track(self.__tracks[path]) - self.__session.commit() - - for album in [ album for artist in self.__artists.values() for album in artist.albums if len(album.tracks) == 0 ]: + app.logger.debug('Checking for empty albums...') + for album in db.Album.query.filter(~db.Album.id.in_(self.__session.query(db.Track.album_id).distinct())): + app.logger.debug(album.name + ' Removed') album.artist.albums.remove(album) self.__session.delete(album) self.__deleted_albums += 1 - self.__session.commit() - + app.logger.debug('Checking for artists with no albums...') for artist in [ a for a in self.__artists.values() if len(a.albums) == 0 ]: self.__session.delete(artist) self.__deleted_artists += 1 self.__session.commit() + app.logger.debug('Cleaning up folder...') self.__cleanup_folder(folder) - def __scan_file(self, path, folder): + def __scan_file(self, path, root): curmtime = int(math.floor(os.path.getmtime(path))) if path in self.__tracks: @@ -88,7 +97,7 @@ class Scanner: if not tr.last_modification: tr.last_modification = curmtime - if curmtime <= tr.last_modification: + if curmtime <= self.__tracktimes[path]: app.logger.debug('\tFile not modified') return False @@ -107,7 +116,7 @@ class Scanner: app.logger.debug('\tProblem reading tag') return False - tr = db.Track(path = path, root_folder = folder, folder = self.__find_folder(path, folder)) + tr = db.Track(path = path, folder = self.__find_folder(root)) self.__tracks[path] = tr self.__added_tracks += 1 @@ -137,7 +146,7 @@ class Scanner: else: #Flair! sys.stdout.write('\033[K') - sys.stdout.write('%s\r' % artist) + sys.stdout.write('%s\r' % artist.encode('utf-8')) sys.stdout.flush() ar = db.Artist(name = artist) self.__artists[artist] = ar @@ -150,26 +159,14 @@ class Scanner: self.__added_albums += 1 return db.Album(name = album, artist = ar) - def __find_folder(self, path, folder): + def __find_folder(self, path): - path = os.path.dirname(path) if path in self.__folders: return self.__folders[path] - # must find parent directory to create new one - full_path = folder.path - path = path[len(folder.path) + 1:] - - for name in path.split(os.sep): - full_path = os.path.join(full_path, name) - - if full_path in self.__folders: - folder = self.__folders[full_path] - else: - folder = db.Folder(root = False, name = name, path = full_path, parent = folder) - self.__folders[full_path] = folder - - return folder + app.logger.debug('Adding folder: ' + path) + self.__folders[path] = db.Folder(path = path) + return self.__folders[path] def __try_load_tag(self, path): try: @@ -201,11 +198,23 @@ class Scanner: self.__deleted_tracks += 1 def __cleanup_folder(self, folder): - for f in folder.children: - self.__cleanup_folder(f) - if len(folder.children) == 0 and len(folder.tracks) == 0 and not folder.root: - folder.parent = None - self.__session.delete(folder) + + + # Get all subfolders of folder + all_descendants = self.__session.query(db.Folder).filter(db.Folder.path.like(folder.path + os.sep + '%')) + + app.logger.debug('Checking for empty paths') + + # Delete folder if there is no track in a subfolder + for d in all_descendants: + if any(d.path in k for k in self.__tracks.keys()): + continue; + else: + app.logger.debug('Deleting path with no tracks: ' + d.path) + self.__session.delete(d) + + self.__session.commit() + return def stats(self): return (self.__added_artists, self.__added_albums, self.__added_tracks), (self.__deleted_artists, self.__deleted_albums, self.__deleted_tracks) diff --git a/web.py b/web.py index 683cb7b..d46c334 100755 --- a/web.py +++ b/web.py @@ -1,32 +1,37 @@ # coding: utf-8 from flask import Flask, request, session, flash, render_template, redirect, url_for -import config - -app = Flask(__name__) -app.secret_key = '?9huDM\\H' - -if(config.get('base', 'accel-redirect')): - app.use_x_sendfile = True - -if config.get('base', 'debug'): - app.debug = True - app.config['SQLALCHEMY_ECHO'] = True - -if config.get('base', 'log_file'): - import logging - from logging.handlers import TimedRotatingFileHandler - handler = TimedRotatingFileHandler(config.get('base', 'log_file'), when = 'midnight', encoding = 'UTF-8') - handler.setLevel(logging.DEBUG) - app.logger.addHandler(handler) - -app.config['SQLALCHEMY_DATABASE_URI'] = config.get('base', 'database_uri') - import db -db.database.init_app(app) +def create_app(): + app = Flask(__name__) + app.secret_key = '?9huDM\\H' + + import config + if(config.get('base', 'accel-redirect')): + app.use_x_sendfile = True + + if config.get('base', 'debug'): + app.debug = True + app.config['SQLALCHEMY_ECHO'] = True + + if config.get('base', 'log_file'): + import logging + from logging.handlers import TimedRotatingFileHandler + handler = TimedRotatingFileHandler(config.get('base', 'log_file'), when = 'midnight', encoding = 'UTF-8') + handler.setLevel(logging.DEBUG) + app.logger.addHandler(handler) + + app.config['SQLALCHEMY_DATABASE_URI'] = config.get('base', 'database_uri') + + db.database.init_app(app) + + return app + +app = create_app() + with app.app_context(): - db.init_db() + db.init_db() from managers.user import UserManager