github/supysonic
1
0
Fork 0
mirror of https://github.com/spl0k/supysonic.git synced 2024-09-19 10:51:04 +00:00
Code Issues Releases Wiki Activity

Merge branch 'scanner_daemon'

Browse Source
This commit is contained in:
spl0k 2015-04-05 16:15:41 +02:00
commit 8b9fa416f8
10 changed files with 427 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
README.md
View File

@ -29,6 +29,7 @@ or as a WSGI application (on Apache for instance). But first:
* simplejson (`apt-get install python-simplejson`)
* [requests](http://docs.python-requests.org/) >= 1.0.0 (`pip install requests`)
* [mutagen](https://code.google.com/p/mutagen/) (`apt-get install python-mutagen`)
* [watchdog](https://github.com/gorakhargosh/watchdog) (`pip install watchdog`)
### Configuration
@ -41,16 +42,21 @@ Available settings are:
* **database_uri**: required, a Storm [database URI](https://storm.canonical.com/Manual#Databases).
I personally use SQLite (`sqlite:////var/supysonic/supysonic.db`), but it might not be the brightest idea for large libraries.
Note that to use PostgreSQL you'll need *psycopg2* version 2.4 (not 2.5!) or [patch storm](https://bugs.launchpad.net/storm/+bug/1170063).
* **cache_dir**: path to a cache folder. Mostly used for resized cover art images. Defaults to `<system temp dir>/supysonic`.
* **log_file**: path and base name of a rolling log file.
* **scanner_extensions**: space-separated list of file extensions the scanner is restricted to. If omitted, files will be scanned
regardless of their extension
* Section **webapp**
* **cache_dir**: path to a cache folder. Mostly used for resized cover art images. Defaults to `<system temp dir>/supysonic`.
* **log_file**: path and base name of a rolling log file.
* **log_level**: logging level. Possible values are *DEBUG*, *INFO*, *WARNING*, *ERROR* or *CRITICAL*.
* Section **lastfm**:
* **api_key**: Last.FM [API key](http://www.last.fm/api/accounts) to enable scrobbling
* **secret**: Last.FM API secret matching the key.
* Section **transcoding**: see [Transcoding](https://github.com/spl0k/supysonic/wiki/Transcoding)
* Section **mimetypes**: extension to content-type mappings. Designed to help the system guess types, to help clients relying on
the content-type. See [the list of common types](https://en.wikipedia.org/wiki/Internet_media_type#List_of_common_media_types).
* Section **daemon**
* **log_file**: path and base name of a rolling log file.
* **log_level**: logging level. Possible values are *DEBUG*, *INFO*, *WARNING*, *ERROR* or *CRITICAL*.
### Database initialization

11
bin/supysonic-cli
View File

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python
# coding: utf-8
# This file is part of Supysonic.
@ -141,18 +141,19 @@ class CLI(cmd.Cmd):
print "Scanning '{0}': {1}% ({2}/{3})".format(self.__name, (scanned * 100) / total, scanned, total)
self.__last_display = time.time()
s = Scanner(self.__store)
scanner = Scanner(self.__store)
if folders:
folders = map(lambda n: self.__store.find(Folder, Folder.name == n, Folder.root == True).one() or n, folders)
if any(map(lambda f: isinstance(f, basestring), folders)):
print "No such folder(s): " + ' '.join(f for f in folders if isinstance(f, basestring))
for folder in filter(lambda f: isinstance(f, Folder), folders):
FolderManager.scan(self.__store, folder.id, s, TimedProgressDisplay(folder.name))
scanner.scan(folder, TimedProgressDisplay(folder.name))
else:
for folder in self.__store.find(Folder, Folder.root == True):
FolderManager.scan(self.__store, folder.id, s, TimedProgressDisplay(folder.name))
scanner.scan(folder, TimedProgressDisplay(folder.name))
added, deleted = s.stats()
scanner.finish()
added, deleted = scanner.stats()
self.__store.commit()
print "Scanning done"

34
bin/supysonic-watcher Executable file
View File

@ -0,0 +1,34 @@
#!/usr/bin/env python
# coding: utf-8
# This file is part of Supysonic.
#
# Supysonic is a Python implementation of the Subsonic server API.
# Copyright (C) 2014 Alban 'spl0k' Féron
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from daemon.runner import DaemonRunner
from supysonic.watcher import SupysonicWatcher
watcher = SupysonicWatcher()
watcher.stdin_path = '/dev/null'
watcher.stdout_path = '/dev/tty'
watcher.stderr_path = '/dev/tty'
watcher.pidfile_path = '/tmp/supysonic-watcher.pid'
watcher.pidfile_timeout = 5
daemon_runner = DaemonRunner(watcher)
daemon_runner.do_action()

2
setup.py
View File

@ -26,6 +26,6 @@ setup(name='supysonic',
""",
packages=['supysonic', 'supysonic.api', 'supysonic.frontend',
'supysonic.managers'],
scripts=['bin/supysonic-cli'],
scripts=['bin/supysonic-cli', 'bin/supysonic-watcher'],
package_data={'supysonic': ['templates/*.html']}
)

2
supysonic/api/media.py
View File

@ -151,7 +151,7 @@ def cover_art():
if size > im.size[0] and size > im.size[1]:
return send_file(os.path.join(res.path, 'cover.jpg'))
size_path = os.path.join(config.get('base', 'cache_dir'), str(size))
size_path = os.path.join(config.get('webapp', 'cache_dir'), str(size))
path = os.path.join(size_path, str(res.id))
if os.path.exists(path):
return send_file(path)

10
supysonic/frontend/folder.py
View File

@ -84,17 +84,19 @@ def del_folder(id):
@app.route('/folder/scan')
@app.route('/folder/scan/<id>')
def scan_folder(id = None):
s = Scanner(store)
scanner = Scanner(store)
if id is None:
for folder in store.find(Folder, Folder.root == True):
FolderManager.scan(store, folder.id, s)
scanner.scan(folder)
else:
status = FolderManager.scan(store, id, s)
status, folder = FolderManager.get(store, id)
if status != FolderManager.SUCCESS:
flash(FolderManager.error_str(status))
return redirect(url_for('folder_index'))
scanner.scan(folder)
added, deleted = s.stats()
scanner.finish()
added, deleted = scanner.stats()
store.commit()
flash('Added: %i artists, %i albums, %i tracks' % (added[0], added[1], added[2]))

33
supysonic/managers/folder.py
View File

@ -20,6 +20,7 @@
import os.path, uuid
from supysonic.db import Folder, Artist, Album, Track
from supysonic.scanner import Scanner
class FolderManager:
SUCCESS = 0
@ -77,24 +78,11 @@ class FolderManager:
if not folder.root:
return FolderManager.NO_SUCH_FOLDER
# delete associated tracks and prune empty albums/artists
potentially_removed_albums = set()
scanner = Scanner(store)
for track in store.find(Track, Track.root_folder_id == folder.id):
potentially_removed_albums.add(track.album)
store.remove(track)
potentially_removed_artists = set()
for album in filter(lambda album: album.tracks.count() == 0, potentially_removed_albums):
potentially_removed_artists.add(album.artist)
store.remove(album)
for artist in filter(lambda artist: artist.albums.count() == 0, potentially_removed_artists):
store.remove(artist)
def cleanup_folder(folder):
for f in folder.children:
cleanup_folder(f)
store.remove(folder)
cleanup_folder(folder)
scanner.remove_file(track.path)
scanner.finish()
store.remove(folder)
store.commit()
return FolderManager.SUCCESS
@ -106,17 +94,6 @@ class FolderManager:
return FolderManager.NO_SUCH_FOLDER
return FolderManager.delete(store, folder.id)
@staticmethod
def scan(store, uid, scanner, progress_callback = None):
status, folder = FolderManager.get(store, uid)
if status != FolderManager.SUCCESS:
return status
scanner.scan(folder, progress_callback)
scanner.prune(folder)
scanner.check_cover_art(folder)
return FolderManager.SUCCESS
@staticmethod
def error_str(err):
if err == FolderManager.SUCCESS:

135
supysonic/scanner.py
View File

@ -21,6 +21,7 @@
import os, os.path
import time, mimetypes
import mutagen
from storm.expr import Like, SQL
from supysonic import config
from supysonic.db import Folder, Artist, Album, Track
@ -41,43 +42,59 @@ class Scanner:
extensions = config.get('base', 'scanner_extensions')
self.__extensions = map(str.lower, extensions.split()) if extensions else None
self.__folders_to_check = set()
self.__artists_to_check = set()
self.__albums_to_check = set()
def __del__(self):
if self.__folders_to_check or self.__artists_to_check or self.__albums_to_check:
raise Exception("There's still something to check. Did you run Scanner.finish()?")
def scan(self, folder, progress_callback = None):
# Scan new/updated files
files = [ os.path.join(root, f) for root, _, fs in os.walk(folder.path) for f in fs if self.__is_valid_path(os.path.join(root, f)) ]
total = len(files)
current = 0
for path in files:
self.__scan_file(path, folder)
self.scan_file(path)
current += 1
if progress_callback:
progress_callback(current, total)
# Remove files that have been deleted
for track in [ t for t in self.__store.find(Track, Track.root_folder_id == folder.id) if not self.__is_valid_path(t.path) ]:
self.remove_file(track.path)
# Update cover art info
folders = [ folder ]
while folders:
f = folders.pop()
f.has_cover_art = os.path.isfile(os.path.join(f.path, 'cover.jpg'))
folders += f.children
folder.last_scan = int(time.time())
self.__store.flush()
def prune(self, folder):
for track in [ t for t in self.__store.find(Track, Track.root_folder_id == folder.id) if not self.__is_valid_path(t.path) ]:
self.__store.remove(track)
self.__deleted_tracks += 1
# TODO execute the conditional part on SQL
for album in [ a for a in self.__store.find(Album) if a.tracks.count() == 0 ]:
def finish(self):
for album in [ a for a in self.__albums_to_check if not a.tracks.count() ]:
self.__artists_to_check.add(album.artist)
self.__store.remove(album)
self.__deleted_albums += 1
self.__albums_to_check.clear()
# TODO execute the conditional part on SQL
for artist in [ a for a in self.__store.find(Artist) if a.albums.count() == 0 ]:
for artist in [ a for a in self.__artists_to_check if not a.albums.count() ]:
self.__store.remove(artist)
self.__deleted_artists += 1
self.__artists_to_check.clear()
self.__cleanup_folder(folder)
self.__store.flush()
while self.__folders_to_check:
folder = self.__folders_to_check.pop()
if folder.root:
continue
def check_cover_art(self, folder):
folder.has_cover_art = os.path.isfile(os.path.join(folder.path, 'cover.jpg'))
for f in folder.children:
self.check_cover_art(f)
if not folder.tracks.count() and not folder.children.count():
self.__folders_to_check.add(folder.parent)
self.__store.remove(folder)
def __is_valid_path(self, path):
if not os.path.exists(path):
@ -86,7 +103,7 @@ class Scanner:
return True
return os.path.splitext(path)[1][1:].lower() in self.__extensions
def __scan_file(self, path, folder):
def scan_file(self, path):
tr = self.__store.find(Track, Track.path == path).one()
add = False
if tr:
@ -95,8 +112,7 @@ class Scanner:
tag = self.__try_load_tag(path)
if not tag:
self.__store.remove(tr)
self.__deleted_tracks += 1
self.remove_file(path)
return
else:
tag = self.__try_load_tag(path)
@ -121,17 +137,46 @@ class Scanner:
if add:
tralbum = self.__find_album(self.__try_read_tag(tag, 'artist', ''), self.__try_read_tag(tag, 'album', ''))
trfolder = self.__find_folder(path, folder)
trroot = self.__find_root_folder(path)
trfolder = self.__find_folder(path)
# Set the references at the very last as searching for them will cause the added track to be flushed, even if
# it is incomplete, causing not null constraints errors.
tr.album = tralbum
tr.folder = trfolder
tr.root_folder = folder
tr.root_folder = trroot
self.__store.add(tr)
self.__added_tracks += 1
def remove_file(self, path):
tr = self.__store.find(Track, Track.path == path).one()
if not tr:
return
self.__folders_to_check.add(tr.folder)
self.__albums_to_check.add(tr.album)
self.__store.remove(tr)
self.__deleted_tracks += 1
def move_file(self, src_path, dst_path):
tr = self.__store.find(Track, Track.path == src_path).one()
if not tr:
return
self.__folders_to_check.add(tr.folder)
tr_dst = self.__store.find(Track, Track.path == dst_path).one()
if tr_dst:
tr.root_folder = tr_dst.root_folder
tr.folder = tr_dst.folder
self.remove_file(dst_path)
else:
root = self.__find_root_folder(dst_path)
folder = self.__find_folder(dst_path)
tr.root_folder = root
tr.folder = folder
tr.path = dst_path
def __find_album(self, artist, album):
ar = self.__find_artist(artist)
al = ar.albums.find(name = album).one()
@ -160,26 +205,40 @@ class Scanner:
return ar
def __find_folder(self, path, folder):
def __find_root_folder(self, path):
path = os.path.dirname(path)
fold = self.__store.find(Folder, Folder.path == path).one()
if fold:
return fold
folders = self.__store.find(Folder, Like(path, SQL("folder.path||'%'")), Folder.root == True)
count = folders.count()
if count > 1:
raise Exception("Found multiple root folders for '{}'.".format(path))
elif count == 0:
raise Exception("Couldn't find the root folder for '{}'.\nDon't scan files that aren't located in a defined music folder".format(path))
return folders.one()
def __find_folder(self, path):
path = os.path.dirname(path)
folders = self.__store.find(Folder, Folder.path == path)
count = folders.count()
if count > 1:
raise Exception("Found multiple folders for '{}'.".format(path))
elif count == 1:
return folders.one()
folder = self.__store.find(Folder, Like(path, SQL("folder.path||'%'"))).order_by(Folder.path).last()
full_path = folder.path
path = path[len(folder.path) + 1:]
for name in path.split(os.sep):
full_path = os.path.join(full_path, name)
fold = self.__store.find(Folder, Folder.path == full_path).one()
if not fold:
fold = Folder()
fold.root = False
fold.name = name
fold.path = full_path
fold.parent = folder
self.__store.add(fold)
fold = Folder()
fold.root = False
fold.name = name
fold.path = full_path
fold.parent = folder
self.__store.add(fold)
folder = fold
@ -202,12 +261,6 @@ class Scanner:
except:
return default
def __cleanup_folder(self, folder):
for f in folder.children:
self.__cleanup_folder(f)
if folder.children.count() == 0 and folder.tracks.count() == 0 and not folder.root:
self.__store.remove(folder)
def stats(self):
return (self.__added_artists, self.__added_albums, self.__added_tracks), (self.__deleted_artists, self.__deleted_albums, self.__deleted_tracks)

259
supysonic/watcher.py Normal file
View File

@ -0,0 +1,259 @@
# coding: utf-8
# This file is part of Supysonic.
#
# Supysonic is a Python implementation of the Subsonic server API.
# Copyright (C) 2014 Alban 'spl0k' Féron
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import time
import logging
from signal import signal, SIGTERM
from threading import Thread, Condition, Timer
from logging.handlers import TimedRotatingFileHandler
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
from supysonic import config, db
from supysonic.scanner import Scanner
OP_SCAN = 1
OP_REMOVE = 2
OP_MOVE = 4
class SupysonicWatcherEventHandler(PatternMatchingEventHandler):
def __init__(self, queue, logger):
extensions = config.get('base', 'scanner_extensions')
patterns = map(lambda e: "*." + e.lower(), extensions.split()) if extensions else None
super(SupysonicWatcherEventHandler, self).__init__(patterns = patterns, ignore_directories = True)
self.__queue = queue
self.__logger = logger
def dispatch(self, event):
try:
super(SupysonicWatcherEventHandler, self).dispatch(event)
except Exception, e:
self.__logger.critical(e)
def on_created(self, event):
self.__logger.debug("File created: '%s'", event.src_path)
self.__queue.put(event.src_path, OP_SCAN)
def on_deleted(self, event):
self.__logger.debug("File deleted: '%s'", event.src_path)
self.__queue.put(event.src_path, OP_REMOVE)
def on_modified(self, event):
self.__logger.debug("File modified: '%s'", event.src_path)
self.__queue.put(event.src_path, OP_SCAN)
def on_moved(self, event):
self.__logger.debug("File moved: '%s' -> '%s'", event.src_path, event.dest_path)
self.__queue.put(event.dest_path, OP_MOVE, src_path = event.src_path)
class Event(object):
def __init__(self, path, operation, **kwargs):
if operation & (OP_SCAN | OP_REMOVE) == (OP_SCAN | OP_REMOVE):
raise Exception("Flags SCAN and REMOVE both set")
self.__path = path
self.__time = time.time()
self.__op = operation
self.__src = kwargs.get("src_path")
def set(self, operation, **kwargs):
if operation & (OP_SCAN | OP_REMOVE) == (OP_SCAN | OP_REMOVE):
raise Exception("Flags SCAN and REMOVE both set")
self.__time = time.time()
if operation & OP_SCAN:
self.__op &= ~OP_REMOVE
if operation & OP_REMOVE:
self.__op &= ~OP_SCAN
self.__op |= operation
src_path = kwargs.get("src_path")
if src_path:
self.__src = src_path
@property
def path(self):
return self.__path
@property
def time(self):
return self.__time
@property
def operation(self):
return self.__op
@property
def src_path(self):
return self.__src
class ScannerProcessingQueue(Thread):
def __init__(self, logger):
super(ScannerProcessingQueue, self).__init__()
self.__logger = logger
self.__cond = Condition()
self.__timer = None
self.__queue = {}
self.__running = True
def run(self):
try:
self.__run()
except Exception, e:
self.__logger.critical(e)
def __run(self):
while self.__running:
time.sleep(0.1)
with self.__cond:
self.__cond.wait()
if not self.__queue:
continue
self.__logger.debug("Instantiating scanner")
store = db.get_store(config.get('base', 'database_uri'))
scanner = Scanner(store)
item = self.__next_item()
while item:
if item.operation & OP_MOVE:
self.__logger.info("Moving: '%s' -> '%s'", item.src_path, item.path)
scanner.move_file(item.src_path, item.path)
if item.operation & OP_SCAN:
self.__logger.info("Scanning: '%s'", item.path)
scanner.scan_file(item.path)
if item.operation & OP_REMOVE:
self.__logger.info("Removing: '%s'", item.path)
scanner.remove_file(item.path)
item = self.__next_item()
scanner.finish()
store.commit()
store.close()
self.__logger.debug("Freeing scanner")
del scanner
def stop(self):
self.__running = False
with self.__cond:
self.__cond.notify()
def put(self, path, operation, **kwargs):
if not self.__running:
raise RuntimeError("Trying to put an item in a stopped queue")
with self.__cond:
if path in self.__queue:
event = self.__queue[path]
event.set(operation, **kwargs)
else:
event = Event(path, operation, **kwargs)
self.__queue[path] = event
if operation & OP_MOVE and kwargs["src_path"] in self.__queue:
previous = self.__queue[kwargs["src_path"]]
event.set(previous.operation, src_path = previous.src_path)
del self.__queue[kwargs["src_path"]]
if self.__timer:
self.__timer.cancel()
self.__timer = Timer(5, self.__wakeup)
self.__timer.start()
def __wakeup(self):
with self.__cond:
self.__cond.notify()
self.__timer = None
def __next_item(self):
with self.__cond:
if not self.__queue:
return None
next = min(self.__queue.iteritems(), key = lambda i: i[1].time)
if not self.__running or next[1].time + 5 <= time.time():
del self.__queue[next[0]]
return next[1]
return None
class SupysonicWatcher(object):
def run(self):
if not config.check():
return
logger = logging.getLogger(__name__)
if config.get('daemon', 'log_file'):
log_handler = TimedRotatingFileHandler(config.get('daemon', 'log_file'), when = 'midnight')
else:
log_handler = logging.NullHandler()
log_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
logger.addHandler(log_handler)
if config.get('daemon', 'log_level'):
mapping = {
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'ERROR': logging.ERROR,
'CRTICAL': logging.CRITICAL
}
logger.setLevel(mapping.get(config.get('daemon', 'log_level').upper(), logging.NOTSET))
store = db.get_store(config.get('base', 'database_uri'))
folders = store.find(db.Folder, db.Folder.root == True)
if not folders.count():
logger.info("No folder set. Exiting.")
store.close()
return
queue = ScannerProcessingQueue(logger)
handler = SupysonicWatcherEventHandler(queue, logger)
observer = Observer()
for folder in folders:
logger.info("Starting watcher for %s", folder.path)
observer.schedule(handler, folder.path, recursive = True)
store.close()
signal(SIGTERM, self.__terminate)
self.__running = True
queue.start()
observer.start()
while self.__running:
time.sleep(2)
logger.info("Stopping watcher")
observer.stop()
observer.join()
queue.stop()
queue.join()
def stop(self):
self.__running = False
def __terminate(self, signum, frame):
self.stop()

18
supysonic/web.py
View File

@ -45,19 +45,27 @@ def create_application():
if not config.check():
return None
if not os.path.exists(config.get('base', 'cache_dir')):
os.makedirs(config.get('base', 'cache_dir'))
if not os.path.exists(config.get('webapp', 'cache_dir')):
os.makedirs(config.get('webapp', 'cache_dir'))
app = Flask(__name__)
app.secret_key = '?9huDM\\H'
app.teardown_appcontext(teardown_db)
if config.get('base', 'log_file'):
if config.get('webapp', 'log_file'):
import logging
from logging.handlers import TimedRotatingFileHandler
handler = TimedRotatingFileHandler(config.get('base', 'log_file'), when = 'midnight')
handler.setLevel(logging.WARNING)
handler = TimedRotatingFileHandler(config.get('webapp', 'log_file'), when = 'midnight')
if config.get('webapp', 'log_level'):
mapping = {
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'ERROR': logging.ERROR,
'CRTICAL': logging.CRITICAL
}
handler.setLevel(mapping.get(config.get('webapp', 'log_level').upper(), logging.NOTSET))
app.logger.addHandler(handler)
from supysonic import frontend