mirror of
https://github.com/spl0k/supysonic.git
synced 2024-12-22 17:06:17 +00:00
Merge branch 'scanner_daemon'
This commit is contained in:
commit
8b9fa416f8
10
README.md
10
README.md
@ -29,6 +29,7 @@ or as a WSGI application (on Apache for instance). But first:
|
|||||||
* simplejson (`apt-get install python-simplejson`)
|
* simplejson (`apt-get install python-simplejson`)
|
||||||
* [requests](http://docs.python-requests.org/) >= 1.0.0 (`pip install requests`)
|
* [requests](http://docs.python-requests.org/) >= 1.0.0 (`pip install requests`)
|
||||||
* [mutagen](https://code.google.com/p/mutagen/) (`apt-get install python-mutagen`)
|
* [mutagen](https://code.google.com/p/mutagen/) (`apt-get install python-mutagen`)
|
||||||
|
* [watchdog](https://github.com/gorakhargosh/watchdog) (`pip install watchdog`)
|
||||||
|
|
||||||
### Configuration
|
### Configuration
|
||||||
|
|
||||||
@ -41,16 +42,21 @@ Available settings are:
|
|||||||
* **database_uri**: required, a Storm [database URI](https://storm.canonical.com/Manual#Databases).
|
* **database_uri**: required, a Storm [database URI](https://storm.canonical.com/Manual#Databases).
|
||||||
I personally use SQLite (`sqlite:////var/supysonic/supysonic.db`), but it might not be the brightest idea for large libraries.
|
I personally use SQLite (`sqlite:////var/supysonic/supysonic.db`), but it might not be the brightest idea for large libraries.
|
||||||
Note that to use PostgreSQL you'll need *psycopg2* version 2.4 (not 2.5!) or [patch storm](https://bugs.launchpad.net/storm/+bug/1170063).
|
Note that to use PostgreSQL you'll need *psycopg2* version 2.4 (not 2.5!) or [patch storm](https://bugs.launchpad.net/storm/+bug/1170063).
|
||||||
* **cache_dir**: path to a cache folder. Mostly used for resized cover art images. Defaults to `<system temp dir>/supysonic`.
|
|
||||||
* **log_file**: path and base name of a rolling log file.
|
|
||||||
* **scanner_extensions**: space-separated list of file extensions the scanner is restricted to. If omitted, files will be scanned
|
* **scanner_extensions**: space-separated list of file extensions the scanner is restricted to. If omitted, files will be scanned
|
||||||
regardless of their extension
|
regardless of their extension
|
||||||
|
* Section **webapp**
|
||||||
|
* **cache_dir**: path to a cache folder. Mostly used for resized cover art images. Defaults to `<system temp dir>/supysonic`.
|
||||||
|
* **log_file**: path and base name of a rolling log file.
|
||||||
|
* **log_level**: logging level. Possible values are *DEBUG*, *INFO*, *WARNING*, *ERROR* or *CRITICAL*.
|
||||||
* Section **lastfm**:
|
* Section **lastfm**:
|
||||||
* **api_key**: Last.FM [API key](http://www.last.fm/api/accounts) to enable scrobbling
|
* **api_key**: Last.FM [API key](http://www.last.fm/api/accounts) to enable scrobbling
|
||||||
* **secret**: Last.FM API secret matching the key.
|
* **secret**: Last.FM API secret matching the key.
|
||||||
* Section **transcoding**: see [Transcoding](https://github.com/spl0k/supysonic/wiki/Transcoding)
|
* Section **transcoding**: see [Transcoding](https://github.com/spl0k/supysonic/wiki/Transcoding)
|
||||||
* Section **mimetypes**: extension to content-type mappings. Designed to help the system guess types, to help clients relying on
|
* Section **mimetypes**: extension to content-type mappings. Designed to help the system guess types, to help clients relying on
|
||||||
the content-type. See [the list of common types](https://en.wikipedia.org/wiki/Internet_media_type#List_of_common_media_types).
|
the content-type. See [the list of common types](https://en.wikipedia.org/wiki/Internet_media_type#List_of_common_media_types).
|
||||||
|
* Section **daemon**
|
||||||
|
* **log_file**: path and base name of a rolling log file.
|
||||||
|
* **log_level**: logging level. Possible values are *DEBUG*, *INFO*, *WARNING*, *ERROR* or *CRITICAL*.
|
||||||
|
|
||||||
### Database initialization
|
### Database initialization
|
||||||
|
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
#!/usr/bin/python
|
#!/usr/bin/env python
|
||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
|
|
||||||
# This file is part of Supysonic.
|
# This file is part of Supysonic.
|
||||||
@ -141,18 +141,19 @@ class CLI(cmd.Cmd):
|
|||||||
print "Scanning '{0}': {1}% ({2}/{3})".format(self.__name, (scanned * 100) / total, scanned, total)
|
print "Scanning '{0}': {1}% ({2}/{3})".format(self.__name, (scanned * 100) / total, scanned, total)
|
||||||
self.__last_display = time.time()
|
self.__last_display = time.time()
|
||||||
|
|
||||||
s = Scanner(self.__store)
|
scanner = Scanner(self.__store)
|
||||||
if folders:
|
if folders:
|
||||||
folders = map(lambda n: self.__store.find(Folder, Folder.name == n, Folder.root == True).one() or n, folders)
|
folders = map(lambda n: self.__store.find(Folder, Folder.name == n, Folder.root == True).one() or n, folders)
|
||||||
if any(map(lambda f: isinstance(f, basestring), folders)):
|
if any(map(lambda f: isinstance(f, basestring), folders)):
|
||||||
print "No such folder(s): " + ' '.join(f for f in folders if isinstance(f, basestring))
|
print "No such folder(s): " + ' '.join(f for f in folders if isinstance(f, basestring))
|
||||||
for folder in filter(lambda f: isinstance(f, Folder), folders):
|
for folder in filter(lambda f: isinstance(f, Folder), folders):
|
||||||
FolderManager.scan(self.__store, folder.id, s, TimedProgressDisplay(folder.name))
|
scanner.scan(folder, TimedProgressDisplay(folder.name))
|
||||||
else:
|
else:
|
||||||
for folder in self.__store.find(Folder, Folder.root == True):
|
for folder in self.__store.find(Folder, Folder.root == True):
|
||||||
FolderManager.scan(self.__store, folder.id, s, TimedProgressDisplay(folder.name))
|
scanner.scan(folder, TimedProgressDisplay(folder.name))
|
||||||
|
|
||||||
added, deleted = s.stats()
|
scanner.finish()
|
||||||
|
added, deleted = scanner.stats()
|
||||||
self.__store.commit()
|
self.__store.commit()
|
||||||
|
|
||||||
print "Scanning done"
|
print "Scanning done"
|
||||||
|
34
bin/supysonic-watcher
Executable file
34
bin/supysonic-watcher
Executable file
@ -0,0 +1,34 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
# coding: utf-8
|
||||||
|
|
||||||
|
# This file is part of Supysonic.
|
||||||
|
#
|
||||||
|
# Supysonic is a Python implementation of the Subsonic server API.
|
||||||
|
# Copyright (C) 2014 Alban 'spl0k' Féron
|
||||||
|
#
|
||||||
|
# This program is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
from daemon.runner import DaemonRunner
|
||||||
|
from supysonic.watcher import SupysonicWatcher
|
||||||
|
|
||||||
|
watcher = SupysonicWatcher()
|
||||||
|
watcher.stdin_path = '/dev/null'
|
||||||
|
watcher.stdout_path = '/dev/tty'
|
||||||
|
watcher.stderr_path = '/dev/tty'
|
||||||
|
watcher.pidfile_path = '/tmp/supysonic-watcher.pid'
|
||||||
|
watcher.pidfile_timeout = 5
|
||||||
|
|
||||||
|
daemon_runner = DaemonRunner(watcher)
|
||||||
|
daemon_runner.do_action()
|
||||||
|
|
2
setup.py
2
setup.py
@ -26,6 +26,6 @@ setup(name='supysonic',
|
|||||||
""",
|
""",
|
||||||
packages=['supysonic', 'supysonic.api', 'supysonic.frontend',
|
packages=['supysonic', 'supysonic.api', 'supysonic.frontend',
|
||||||
'supysonic.managers'],
|
'supysonic.managers'],
|
||||||
scripts=['bin/supysonic-cli'],
|
scripts=['bin/supysonic-cli', 'bin/supysonic-watcher'],
|
||||||
package_data={'supysonic': ['templates/*.html']}
|
package_data={'supysonic': ['templates/*.html']}
|
||||||
)
|
)
|
||||||
|
@ -151,7 +151,7 @@ def cover_art():
|
|||||||
if size > im.size[0] and size > im.size[1]:
|
if size > im.size[0] and size > im.size[1]:
|
||||||
return send_file(os.path.join(res.path, 'cover.jpg'))
|
return send_file(os.path.join(res.path, 'cover.jpg'))
|
||||||
|
|
||||||
size_path = os.path.join(config.get('base', 'cache_dir'), str(size))
|
size_path = os.path.join(config.get('webapp', 'cache_dir'), str(size))
|
||||||
path = os.path.join(size_path, str(res.id))
|
path = os.path.join(size_path, str(res.id))
|
||||||
if os.path.exists(path):
|
if os.path.exists(path):
|
||||||
return send_file(path)
|
return send_file(path)
|
||||||
|
@ -84,17 +84,19 @@ def del_folder(id):
|
|||||||
@app.route('/folder/scan')
|
@app.route('/folder/scan')
|
||||||
@app.route('/folder/scan/<id>')
|
@app.route('/folder/scan/<id>')
|
||||||
def scan_folder(id = None):
|
def scan_folder(id = None):
|
||||||
s = Scanner(store)
|
scanner = Scanner(store)
|
||||||
if id is None:
|
if id is None:
|
||||||
for folder in store.find(Folder, Folder.root == True):
|
for folder in store.find(Folder, Folder.root == True):
|
||||||
FolderManager.scan(store, folder.id, s)
|
scanner.scan(folder)
|
||||||
else:
|
else:
|
||||||
status = FolderManager.scan(store, id, s)
|
status, folder = FolderManager.get(store, id)
|
||||||
if status != FolderManager.SUCCESS:
|
if status != FolderManager.SUCCESS:
|
||||||
flash(FolderManager.error_str(status))
|
flash(FolderManager.error_str(status))
|
||||||
return redirect(url_for('folder_index'))
|
return redirect(url_for('folder_index'))
|
||||||
|
scanner.scan(folder)
|
||||||
|
|
||||||
added, deleted = s.stats()
|
scanner.finish()
|
||||||
|
added, deleted = scanner.stats()
|
||||||
store.commit()
|
store.commit()
|
||||||
|
|
||||||
flash('Added: %i artists, %i albums, %i tracks' % (added[0], added[1], added[2]))
|
flash('Added: %i artists, %i albums, %i tracks' % (added[0], added[1], added[2]))
|
||||||
|
@ -20,6 +20,7 @@
|
|||||||
|
|
||||||
import os.path, uuid
|
import os.path, uuid
|
||||||
from supysonic.db import Folder, Artist, Album, Track
|
from supysonic.db import Folder, Artist, Album, Track
|
||||||
|
from supysonic.scanner import Scanner
|
||||||
|
|
||||||
class FolderManager:
|
class FolderManager:
|
||||||
SUCCESS = 0
|
SUCCESS = 0
|
||||||
@ -77,24 +78,11 @@ class FolderManager:
|
|||||||
if not folder.root:
|
if not folder.root:
|
||||||
return FolderManager.NO_SUCH_FOLDER
|
return FolderManager.NO_SUCH_FOLDER
|
||||||
|
|
||||||
# delete associated tracks and prune empty albums/artists
|
scanner = Scanner(store)
|
||||||
potentially_removed_albums = set()
|
|
||||||
for track in store.find(Track, Track.root_folder_id == folder.id):
|
for track in store.find(Track, Track.root_folder_id == folder.id):
|
||||||
potentially_removed_albums.add(track.album)
|
scanner.remove_file(track.path)
|
||||||
store.remove(track)
|
scanner.finish()
|
||||||
potentially_removed_artists = set()
|
store.remove(folder)
|
||||||
for album in filter(lambda album: album.tracks.count() == 0, potentially_removed_albums):
|
|
||||||
potentially_removed_artists.add(album.artist)
|
|
||||||
store.remove(album)
|
|
||||||
for artist in filter(lambda artist: artist.albums.count() == 0, potentially_removed_artists):
|
|
||||||
store.remove(artist)
|
|
||||||
|
|
||||||
def cleanup_folder(folder):
|
|
||||||
for f in folder.children:
|
|
||||||
cleanup_folder(f)
|
|
||||||
store.remove(folder)
|
|
||||||
|
|
||||||
cleanup_folder(folder)
|
|
||||||
store.commit()
|
store.commit()
|
||||||
|
|
||||||
return FolderManager.SUCCESS
|
return FolderManager.SUCCESS
|
||||||
@ -106,17 +94,6 @@ class FolderManager:
|
|||||||
return FolderManager.NO_SUCH_FOLDER
|
return FolderManager.NO_SUCH_FOLDER
|
||||||
return FolderManager.delete(store, folder.id)
|
return FolderManager.delete(store, folder.id)
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def scan(store, uid, scanner, progress_callback = None):
|
|
||||||
status, folder = FolderManager.get(store, uid)
|
|
||||||
if status != FolderManager.SUCCESS:
|
|
||||||
return status
|
|
||||||
|
|
||||||
scanner.scan(folder, progress_callback)
|
|
||||||
scanner.prune(folder)
|
|
||||||
scanner.check_cover_art(folder)
|
|
||||||
return FolderManager.SUCCESS
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def error_str(err):
|
def error_str(err):
|
||||||
if err == FolderManager.SUCCESS:
|
if err == FolderManager.SUCCESS:
|
||||||
|
@ -21,6 +21,7 @@
|
|||||||
import os, os.path
|
import os, os.path
|
||||||
import time, mimetypes
|
import time, mimetypes
|
||||||
import mutagen
|
import mutagen
|
||||||
|
from storm.expr import Like, SQL
|
||||||
from supysonic import config
|
from supysonic import config
|
||||||
from supysonic.db import Folder, Artist, Album, Track
|
from supysonic.db import Folder, Artist, Album, Track
|
||||||
|
|
||||||
@ -41,43 +42,59 @@ class Scanner:
|
|||||||
extensions = config.get('base', 'scanner_extensions')
|
extensions = config.get('base', 'scanner_extensions')
|
||||||
self.__extensions = map(str.lower, extensions.split()) if extensions else None
|
self.__extensions = map(str.lower, extensions.split()) if extensions else None
|
||||||
|
|
||||||
|
self.__folders_to_check = set()
|
||||||
|
self.__artists_to_check = set()
|
||||||
|
self.__albums_to_check = set()
|
||||||
|
|
||||||
|
def __del__(self):
|
||||||
|
if self.__folders_to_check or self.__artists_to_check or self.__albums_to_check:
|
||||||
|
raise Exception("There's still something to check. Did you run Scanner.finish()?")
|
||||||
|
|
||||||
def scan(self, folder, progress_callback = None):
|
def scan(self, folder, progress_callback = None):
|
||||||
|
# Scan new/updated files
|
||||||
files = [ os.path.join(root, f) for root, _, fs in os.walk(folder.path) for f in fs if self.__is_valid_path(os.path.join(root, f)) ]
|
files = [ os.path.join(root, f) for root, _, fs in os.walk(folder.path) for f in fs if self.__is_valid_path(os.path.join(root, f)) ]
|
||||||
total = len(files)
|
total = len(files)
|
||||||
current = 0
|
current = 0
|
||||||
|
|
||||||
for path in files:
|
for path in files:
|
||||||
self.__scan_file(path, folder)
|
self.scan_file(path)
|
||||||
current += 1
|
current += 1
|
||||||
if progress_callback:
|
if progress_callback:
|
||||||
progress_callback(current, total)
|
progress_callback(current, total)
|
||||||
|
|
||||||
|
# Remove files that have been deleted
|
||||||
|
for track in [ t for t in self.__store.find(Track, Track.root_folder_id == folder.id) if not self.__is_valid_path(t.path) ]:
|
||||||
|
self.remove_file(track.path)
|
||||||
|
|
||||||
|
# Update cover art info
|
||||||
|
folders = [ folder ]
|
||||||
|
while folders:
|
||||||
|
f = folders.pop()
|
||||||
|
f.has_cover_art = os.path.isfile(os.path.join(f.path, 'cover.jpg'))
|
||||||
|
folders += f.children
|
||||||
|
|
||||||
folder.last_scan = int(time.time())
|
folder.last_scan = int(time.time())
|
||||||
|
|
||||||
self.__store.flush()
|
def finish(self):
|
||||||
|
for album in [ a for a in self.__albums_to_check if not a.tracks.count() ]:
|
||||||
def prune(self, folder):
|
self.__artists_to_check.add(album.artist)
|
||||||
for track in [ t for t in self.__store.find(Track, Track.root_folder_id == folder.id) if not self.__is_valid_path(t.path) ]:
|
|
||||||
self.__store.remove(track)
|
|
||||||
self.__deleted_tracks += 1
|
|
||||||
|
|
||||||
# TODO execute the conditional part on SQL
|
|
||||||
for album in [ a for a in self.__store.find(Album) if a.tracks.count() == 0 ]:
|
|
||||||
self.__store.remove(album)
|
self.__store.remove(album)
|
||||||
self.__deleted_albums += 1
|
self.__deleted_albums += 1
|
||||||
|
self.__albums_to_check.clear()
|
||||||
|
|
||||||
# TODO execute the conditional part on SQL
|
for artist in [ a for a in self.__artists_to_check if not a.albums.count() ]:
|
||||||
for artist in [ a for a in self.__store.find(Artist) if a.albums.count() == 0 ]:
|
|
||||||
self.__store.remove(artist)
|
self.__store.remove(artist)
|
||||||
self.__deleted_artists += 1
|
self.__deleted_artists += 1
|
||||||
|
self.__artists_to_check.clear()
|
||||||
|
|
||||||
self.__cleanup_folder(folder)
|
while self.__folders_to_check:
|
||||||
self.__store.flush()
|
folder = self.__folders_to_check.pop()
|
||||||
|
if folder.root:
|
||||||
|
continue
|
||||||
|
|
||||||
def check_cover_art(self, folder):
|
if not folder.tracks.count() and not folder.children.count():
|
||||||
folder.has_cover_art = os.path.isfile(os.path.join(folder.path, 'cover.jpg'))
|
self.__folders_to_check.add(folder.parent)
|
||||||
for f in folder.children:
|
self.__store.remove(folder)
|
||||||
self.check_cover_art(f)
|
|
||||||
|
|
||||||
def __is_valid_path(self, path):
|
def __is_valid_path(self, path):
|
||||||
if not os.path.exists(path):
|
if not os.path.exists(path):
|
||||||
@ -86,7 +103,7 @@ class Scanner:
|
|||||||
return True
|
return True
|
||||||
return os.path.splitext(path)[1][1:].lower() in self.__extensions
|
return os.path.splitext(path)[1][1:].lower() in self.__extensions
|
||||||
|
|
||||||
def __scan_file(self, path, folder):
|
def scan_file(self, path):
|
||||||
tr = self.__store.find(Track, Track.path == path).one()
|
tr = self.__store.find(Track, Track.path == path).one()
|
||||||
add = False
|
add = False
|
||||||
if tr:
|
if tr:
|
||||||
@ -95,8 +112,7 @@ class Scanner:
|
|||||||
|
|
||||||
tag = self.__try_load_tag(path)
|
tag = self.__try_load_tag(path)
|
||||||
if not tag:
|
if not tag:
|
||||||
self.__store.remove(tr)
|
self.remove_file(path)
|
||||||
self.__deleted_tracks += 1
|
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
tag = self.__try_load_tag(path)
|
tag = self.__try_load_tag(path)
|
||||||
@ -121,17 +137,46 @@ class Scanner:
|
|||||||
|
|
||||||
if add:
|
if add:
|
||||||
tralbum = self.__find_album(self.__try_read_tag(tag, 'artist', ''), self.__try_read_tag(tag, 'album', ''))
|
tralbum = self.__find_album(self.__try_read_tag(tag, 'artist', ''), self.__try_read_tag(tag, 'album', ''))
|
||||||
trfolder = self.__find_folder(path, folder)
|
trroot = self.__find_root_folder(path)
|
||||||
|
trfolder = self.__find_folder(path)
|
||||||
|
|
||||||
# Set the references at the very last as searching for them will cause the added track to be flushed, even if
|
# Set the references at the very last as searching for them will cause the added track to be flushed, even if
|
||||||
# it is incomplete, causing not null constraints errors.
|
# it is incomplete, causing not null constraints errors.
|
||||||
tr.album = tralbum
|
tr.album = tralbum
|
||||||
tr.folder = trfolder
|
tr.folder = trfolder
|
||||||
tr.root_folder = folder
|
tr.root_folder = trroot
|
||||||
|
|
||||||
self.__store.add(tr)
|
self.__store.add(tr)
|
||||||
self.__added_tracks += 1
|
self.__added_tracks += 1
|
||||||
|
|
||||||
|
def remove_file(self, path):
|
||||||
|
tr = self.__store.find(Track, Track.path == path).one()
|
||||||
|
if not tr:
|
||||||
|
return
|
||||||
|
|
||||||
|
self.__folders_to_check.add(tr.folder)
|
||||||
|
self.__albums_to_check.add(tr.album)
|
||||||
|
self.__store.remove(tr)
|
||||||
|
self.__deleted_tracks += 1
|
||||||
|
|
||||||
|
def move_file(self, src_path, dst_path):
|
||||||
|
tr = self.__store.find(Track, Track.path == src_path).one()
|
||||||
|
if not tr:
|
||||||
|
return
|
||||||
|
|
||||||
|
self.__folders_to_check.add(tr.folder)
|
||||||
|
tr_dst = self.__store.find(Track, Track.path == dst_path).one()
|
||||||
|
if tr_dst:
|
||||||
|
tr.root_folder = tr_dst.root_folder
|
||||||
|
tr.folder = tr_dst.folder
|
||||||
|
self.remove_file(dst_path)
|
||||||
|
else:
|
||||||
|
root = self.__find_root_folder(dst_path)
|
||||||
|
folder = self.__find_folder(dst_path)
|
||||||
|
tr.root_folder = root
|
||||||
|
tr.folder = folder
|
||||||
|
tr.path = dst_path
|
||||||
|
|
||||||
def __find_album(self, artist, album):
|
def __find_album(self, artist, album):
|
||||||
ar = self.__find_artist(artist)
|
ar = self.__find_artist(artist)
|
||||||
al = ar.albums.find(name = album).one()
|
al = ar.albums.find(name = album).one()
|
||||||
@ -160,26 +205,40 @@ class Scanner:
|
|||||||
|
|
||||||
return ar
|
return ar
|
||||||
|
|
||||||
def __find_folder(self, path, folder):
|
def __find_root_folder(self, path):
|
||||||
path = os.path.dirname(path)
|
path = os.path.dirname(path)
|
||||||
fold = self.__store.find(Folder, Folder.path == path).one()
|
folders = self.__store.find(Folder, Like(path, SQL("folder.path||'%'")), Folder.root == True)
|
||||||
if fold:
|
count = folders.count()
|
||||||
return fold
|
if count > 1:
|
||||||
|
raise Exception("Found multiple root folders for '{}'.".format(path))
|
||||||
|
elif count == 0:
|
||||||
|
raise Exception("Couldn't find the root folder for '{}'.\nDon't scan files that aren't located in a defined music folder".format(path))
|
||||||
|
return folders.one()
|
||||||
|
|
||||||
|
def __find_folder(self, path):
|
||||||
|
path = os.path.dirname(path)
|
||||||
|
folders = self.__store.find(Folder, Folder.path == path)
|
||||||
|
count = folders.count()
|
||||||
|
if count > 1:
|
||||||
|
raise Exception("Found multiple folders for '{}'.".format(path))
|
||||||
|
elif count == 1:
|
||||||
|
return folders.one()
|
||||||
|
|
||||||
|
folder = self.__store.find(Folder, Like(path, SQL("folder.path||'%'"))).order_by(Folder.path).last()
|
||||||
|
|
||||||
full_path = folder.path
|
full_path = folder.path
|
||||||
path = path[len(folder.path) + 1:]
|
path = path[len(folder.path) + 1:]
|
||||||
|
|
||||||
for name in path.split(os.sep):
|
for name in path.split(os.sep):
|
||||||
full_path = os.path.join(full_path, name)
|
full_path = os.path.join(full_path, name)
|
||||||
fold = self.__store.find(Folder, Folder.path == full_path).one()
|
|
||||||
if not fold:
|
|
||||||
fold = Folder()
|
|
||||||
fold.root = False
|
|
||||||
fold.name = name
|
|
||||||
fold.path = full_path
|
|
||||||
fold.parent = folder
|
|
||||||
|
|
||||||
self.__store.add(fold)
|
fold = Folder()
|
||||||
|
fold.root = False
|
||||||
|
fold.name = name
|
||||||
|
fold.path = full_path
|
||||||
|
fold.parent = folder
|
||||||
|
|
||||||
|
self.__store.add(fold)
|
||||||
|
|
||||||
folder = fold
|
folder = fold
|
||||||
|
|
||||||
@ -202,12 +261,6 @@ class Scanner:
|
|||||||
except:
|
except:
|
||||||
return default
|
return default
|
||||||
|
|
||||||
def __cleanup_folder(self, folder):
|
|
||||||
for f in folder.children:
|
|
||||||
self.__cleanup_folder(f)
|
|
||||||
if folder.children.count() == 0 and folder.tracks.count() == 0 and not folder.root:
|
|
||||||
self.__store.remove(folder)
|
|
||||||
|
|
||||||
def stats(self):
|
def stats(self):
|
||||||
return (self.__added_artists, self.__added_albums, self.__added_tracks), (self.__deleted_artists, self.__deleted_albums, self.__deleted_tracks)
|
return (self.__added_artists, self.__added_albums, self.__added_tracks), (self.__deleted_artists, self.__deleted_albums, self.__deleted_tracks)
|
||||||
|
|
||||||
|
259
supysonic/watcher.py
Normal file
259
supysonic/watcher.py
Normal file
@ -0,0 +1,259 @@
|
|||||||
|
# coding: utf-8
|
||||||
|
|
||||||
|
# This file is part of Supysonic.
|
||||||
|
#
|
||||||
|
# Supysonic is a Python implementation of the Subsonic server API.
|
||||||
|
# Copyright (C) 2014 Alban 'spl0k' Féron
|
||||||
|
#
|
||||||
|
# This program is free software: you can redistribute it and/or modify
|
||||||
|
# it under the terms of the GNU Affero General Public License as published by
|
||||||
|
# the Free Software Foundation, either version 3 of the License, or
|
||||||
|
# (at your option) any later version.
|
||||||
|
#
|
||||||
|
# This program is distributed in the hope that it will be useful,
|
||||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
# GNU Affero General Public License for more details.
|
||||||
|
#
|
||||||
|
# You should have received a copy of the GNU Affero General Public License
|
||||||
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
from signal import signal, SIGTERM
|
||||||
|
from threading import Thread, Condition, Timer
|
||||||
|
from logging.handlers import TimedRotatingFileHandler
|
||||||
|
from watchdog.observers import Observer
|
||||||
|
from watchdog.events import PatternMatchingEventHandler
|
||||||
|
|
||||||
|
from supysonic import config, db
|
||||||
|
from supysonic.scanner import Scanner
|
||||||
|
|
||||||
|
OP_SCAN = 1
|
||||||
|
OP_REMOVE = 2
|
||||||
|
OP_MOVE = 4
|
||||||
|
|
||||||
|
class SupysonicWatcherEventHandler(PatternMatchingEventHandler):
|
||||||
|
def __init__(self, queue, logger):
|
||||||
|
extensions = config.get('base', 'scanner_extensions')
|
||||||
|
patterns = map(lambda e: "*." + e.lower(), extensions.split()) if extensions else None
|
||||||
|
super(SupysonicWatcherEventHandler, self).__init__(patterns = patterns, ignore_directories = True)
|
||||||
|
|
||||||
|
self.__queue = queue
|
||||||
|
self.__logger = logger
|
||||||
|
|
||||||
|
def dispatch(self, event):
|
||||||
|
try:
|
||||||
|
super(SupysonicWatcherEventHandler, self).dispatch(event)
|
||||||
|
except Exception, e:
|
||||||
|
self.__logger.critical(e)
|
||||||
|
|
||||||
|
def on_created(self, event):
|
||||||
|
self.__logger.debug("File created: '%s'", event.src_path)
|
||||||
|
self.__queue.put(event.src_path, OP_SCAN)
|
||||||
|
|
||||||
|
def on_deleted(self, event):
|
||||||
|
self.__logger.debug("File deleted: '%s'", event.src_path)
|
||||||
|
self.__queue.put(event.src_path, OP_REMOVE)
|
||||||
|
|
||||||
|
def on_modified(self, event):
|
||||||
|
self.__logger.debug("File modified: '%s'", event.src_path)
|
||||||
|
self.__queue.put(event.src_path, OP_SCAN)
|
||||||
|
|
||||||
|
def on_moved(self, event):
|
||||||
|
self.__logger.debug("File moved: '%s' -> '%s'", event.src_path, event.dest_path)
|
||||||
|
self.__queue.put(event.dest_path, OP_MOVE, src_path = event.src_path)
|
||||||
|
|
||||||
|
class Event(object):
|
||||||
|
def __init__(self, path, operation, **kwargs):
|
||||||
|
if operation & (OP_SCAN | OP_REMOVE) == (OP_SCAN | OP_REMOVE):
|
||||||
|
raise Exception("Flags SCAN and REMOVE both set")
|
||||||
|
|
||||||
|
self.__path = path
|
||||||
|
self.__time = time.time()
|
||||||
|
self.__op = operation
|
||||||
|
self.__src = kwargs.get("src_path")
|
||||||
|
|
||||||
|
def set(self, operation, **kwargs):
|
||||||
|
if operation & (OP_SCAN | OP_REMOVE) == (OP_SCAN | OP_REMOVE):
|
||||||
|
raise Exception("Flags SCAN and REMOVE both set")
|
||||||
|
|
||||||
|
self.__time = time.time()
|
||||||
|
if operation & OP_SCAN:
|
||||||
|
self.__op &= ~OP_REMOVE
|
||||||
|
if operation & OP_REMOVE:
|
||||||
|
self.__op &= ~OP_SCAN
|
||||||
|
self.__op |= operation
|
||||||
|
|
||||||
|
src_path = kwargs.get("src_path")
|
||||||
|
if src_path:
|
||||||
|
self.__src = src_path
|
||||||
|
|
||||||
|
@property
|
||||||
|
def path(self):
|
||||||
|
return self.__path
|
||||||
|
|
||||||
|
@property
|
||||||
|
def time(self):
|
||||||
|
return self.__time
|
||||||
|
|
||||||
|
@property
|
||||||
|
def operation(self):
|
||||||
|
return self.__op
|
||||||
|
|
||||||
|
@property
|
||||||
|
def src_path(self):
|
||||||
|
return self.__src
|
||||||
|
|
||||||
|
class ScannerProcessingQueue(Thread):
|
||||||
|
def __init__(self, logger):
|
||||||
|
super(ScannerProcessingQueue, self).__init__()
|
||||||
|
|
||||||
|
self.__logger = logger
|
||||||
|
self.__cond = Condition()
|
||||||
|
self.__timer = None
|
||||||
|
self.__queue = {}
|
||||||
|
self.__running = True
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
try:
|
||||||
|
self.__run()
|
||||||
|
except Exception, e:
|
||||||
|
self.__logger.critical(e)
|
||||||
|
|
||||||
|
def __run(self):
|
||||||
|
while self.__running:
|
||||||
|
time.sleep(0.1)
|
||||||
|
|
||||||
|
with self.__cond:
|
||||||
|
self.__cond.wait()
|
||||||
|
|
||||||
|
if not self.__queue:
|
||||||
|
continue
|
||||||
|
|
||||||
|
self.__logger.debug("Instantiating scanner")
|
||||||
|
store = db.get_store(config.get('base', 'database_uri'))
|
||||||
|
scanner = Scanner(store)
|
||||||
|
|
||||||
|
item = self.__next_item()
|
||||||
|
while item:
|
||||||
|
if item.operation & OP_MOVE:
|
||||||
|
self.__logger.info("Moving: '%s' -> '%s'", item.src_path, item.path)
|
||||||
|
scanner.move_file(item.src_path, item.path)
|
||||||
|
if item.operation & OP_SCAN:
|
||||||
|
self.__logger.info("Scanning: '%s'", item.path)
|
||||||
|
scanner.scan_file(item.path)
|
||||||
|
if item.operation & OP_REMOVE:
|
||||||
|
self.__logger.info("Removing: '%s'", item.path)
|
||||||
|
scanner.remove_file(item.path)
|
||||||
|
item = self.__next_item()
|
||||||
|
|
||||||
|
scanner.finish()
|
||||||
|
store.commit()
|
||||||
|
store.close()
|
||||||
|
self.__logger.debug("Freeing scanner")
|
||||||
|
del scanner
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
self.__running = False
|
||||||
|
with self.__cond:
|
||||||
|
self.__cond.notify()
|
||||||
|
|
||||||
|
def put(self, path, operation, **kwargs):
|
||||||
|
if not self.__running:
|
||||||
|
raise RuntimeError("Trying to put an item in a stopped queue")
|
||||||
|
|
||||||
|
with self.__cond:
|
||||||
|
if path in self.__queue:
|
||||||
|
event = self.__queue[path]
|
||||||
|
event.set(operation, **kwargs)
|
||||||
|
else:
|
||||||
|
event = Event(path, operation, **kwargs)
|
||||||
|
self.__queue[path] = event
|
||||||
|
|
||||||
|
if operation & OP_MOVE and kwargs["src_path"] in self.__queue:
|
||||||
|
previous = self.__queue[kwargs["src_path"]]
|
||||||
|
event.set(previous.operation, src_path = previous.src_path)
|
||||||
|
del self.__queue[kwargs["src_path"]]
|
||||||
|
|
||||||
|
if self.__timer:
|
||||||
|
self.__timer.cancel()
|
||||||
|
self.__timer = Timer(5, self.__wakeup)
|
||||||
|
self.__timer.start()
|
||||||
|
|
||||||
|
def __wakeup(self):
|
||||||
|
with self.__cond:
|
||||||
|
self.__cond.notify()
|
||||||
|
self.__timer = None
|
||||||
|
|
||||||
|
def __next_item(self):
|
||||||
|
with self.__cond:
|
||||||
|
if not self.__queue:
|
||||||
|
return None
|
||||||
|
|
||||||
|
next = min(self.__queue.iteritems(), key = lambda i: i[1].time)
|
||||||
|
if not self.__running or next[1].time + 5 <= time.time():
|
||||||
|
del self.__queue[next[0]]
|
||||||
|
return next[1]
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
class SupysonicWatcher(object):
|
||||||
|
def run(self):
|
||||||
|
if not config.check():
|
||||||
|
return
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
if config.get('daemon', 'log_file'):
|
||||||
|
log_handler = TimedRotatingFileHandler(config.get('daemon', 'log_file'), when = 'midnight')
|
||||||
|
else:
|
||||||
|
log_handler = logging.NullHandler()
|
||||||
|
log_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
|
||||||
|
logger.addHandler(log_handler)
|
||||||
|
if config.get('daemon', 'log_level'):
|
||||||
|
mapping = {
|
||||||
|
'DEBUG': logging.DEBUG,
|
||||||
|
'INFO': logging.INFO,
|
||||||
|
'WARNING': logging.WARNING,
|
||||||
|
'ERROR': logging.ERROR,
|
||||||
|
'CRTICAL': logging.CRITICAL
|
||||||
|
}
|
||||||
|
logger.setLevel(mapping.get(config.get('daemon', 'log_level').upper(), logging.NOTSET))
|
||||||
|
|
||||||
|
store = db.get_store(config.get('base', 'database_uri'))
|
||||||
|
folders = store.find(db.Folder, db.Folder.root == True)
|
||||||
|
|
||||||
|
if not folders.count():
|
||||||
|
logger.info("No folder set. Exiting.")
|
||||||
|
store.close()
|
||||||
|
return
|
||||||
|
|
||||||
|
queue = ScannerProcessingQueue(logger)
|
||||||
|
handler = SupysonicWatcherEventHandler(queue, logger)
|
||||||
|
observer = Observer()
|
||||||
|
|
||||||
|
for folder in folders:
|
||||||
|
logger.info("Starting watcher for %s", folder.path)
|
||||||
|
observer.schedule(handler, folder.path, recursive = True)
|
||||||
|
|
||||||
|
store.close()
|
||||||
|
signal(SIGTERM, self.__terminate)
|
||||||
|
|
||||||
|
self.__running = True
|
||||||
|
queue.start()
|
||||||
|
observer.start()
|
||||||
|
while self.__running:
|
||||||
|
time.sleep(2)
|
||||||
|
|
||||||
|
logger.info("Stopping watcher")
|
||||||
|
observer.stop()
|
||||||
|
observer.join()
|
||||||
|
queue.stop()
|
||||||
|
queue.join()
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
self.__running = False
|
||||||
|
|
||||||
|
def __terminate(self, signum, frame):
|
||||||
|
self.stop()
|
||||||
|
|
@ -45,19 +45,27 @@ def create_application():
|
|||||||
if not config.check():
|
if not config.check():
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if not os.path.exists(config.get('base', 'cache_dir')):
|
if not os.path.exists(config.get('webapp', 'cache_dir')):
|
||||||
os.makedirs(config.get('base', 'cache_dir'))
|
os.makedirs(config.get('webapp', 'cache_dir'))
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
app.secret_key = '?9huDM\\H'
|
app.secret_key = '?9huDM\\H'
|
||||||
|
|
||||||
app.teardown_appcontext(teardown_db)
|
app.teardown_appcontext(teardown_db)
|
||||||
|
|
||||||
if config.get('base', 'log_file'):
|
if config.get('webapp', 'log_file'):
|
||||||
import logging
|
import logging
|
||||||
from logging.handlers import TimedRotatingFileHandler
|
from logging.handlers import TimedRotatingFileHandler
|
||||||
handler = TimedRotatingFileHandler(config.get('base', 'log_file'), when = 'midnight')
|
handler = TimedRotatingFileHandler(config.get('webapp', 'log_file'), when = 'midnight')
|
||||||
handler.setLevel(logging.WARNING)
|
if config.get('webapp', 'log_level'):
|
||||||
|
mapping = {
|
||||||
|
'DEBUG': logging.DEBUG,
|
||||||
|
'INFO': logging.INFO,
|
||||||
|
'WARNING': logging.WARNING,
|
||||||
|
'ERROR': logging.ERROR,
|
||||||
|
'CRTICAL': logging.CRITICAL
|
||||||
|
}
|
||||||
|
handler.setLevel(mapping.get(config.get('webapp', 'log_level').upper(), logging.NOTSET))
|
||||||
app.logger.addHandler(handler)
|
app.logger.addHandler(handler)
|
||||||
|
|
||||||
from supysonic import frontend
|
from supysonic import frontend
|
||||||
|
Loading…
Reference in New Issue
Block a user