made more debug friendly, added accel-redirect support, added

supervisord compatible uwsgi ini file, vastly improved scanning speed,
ignored logs in directory, fixed cover art searching algorithm to find
more results

.gitignore

@@ -1,3 +1,4 @@
 *.pyc
 *.swp
 start_server.sh
+*.log

api/browse.py

@@ -5,6 +5,7 @@ from web import app
 from db import Folder, Artist, Album, Track
 from api import get_entity
 import uuid, time, string
+import os.path
 
 @app.route('/rest/getMusicFolders.view', methods = [ 'GET', 'POST' ])
 def list_folders():
@@ -89,6 +90,8 @@ def show_directory():
 	if not status:
 		return res
 
+        res.tracks = [t for t in res.tracks if os.path.isfile(t.path)]
+
 	directory = {
 		'id': str(res.id),
 		'name': res.name,

api/media.py

@@ -6,12 +6,28 @@ from PIL import Image
 import subprocess
 import shlex
 import mutagen
+import fnmatch
+import mimetypes
 
 import config, scanner
 from web import app
 from db import Track, Folder, User, now, session
 from api import get_entity
 
+from flask import g
+
+def after_this_request(func):
+	if not hasattr(g, 'call_after_request'):
+		g.call_after_request = []
+	g.call_after_request.append(func)
+	return func
+
+@app.after_request
+def per_request_callbacks(response):
+	for func in getattr(g, 'call_after_request', ()):
+		response = func(response)
+	return response
+
 def prepare_transcoding_cmdline(base_cmdline, input_file, input_format, output_format, output_bitrate):
 	if not base_cmdline:
 		return None
@@ -27,7 +43,7 @@ def stream_media():
 	status, res = get_entity(request, Track)
 
 	if not status:
-            return res
+		return res
 
 	maxBitRate, format, timeOffset, size, estimateContentLength = map(request.args.get, [ 'maxBitRate', 'format', 'timeOffset', 'size', 'estimateContentLength' ])
 	if format:
@@ -35,9 +51,9 @@ def stream_media():
 
 	do_transcoding = False
 	src_suffix = res.suffix()
-	dst_suffix = res.suffix()
+	dst_suffix = src_suffix
 	dst_bitrate = res.bitrate
-	dst_mimetype = res.content_type
+	dst_mimetype = mimetypes.guess_type('a.' + src_suffix)
 
 	if format != 'raw': # That's from API 1.9.0 but whatever
 		if maxBitRate:
@@ -53,17 +69,16 @@ def stream_media():
 		if format and format != src_suffix:
 			do_transcoding = True
 			dst_suffix = format
-			dst_mimetype = scanner.get_mime(dst_suffix)
+			dst_mimetype = mimetypes.guess_type(dst_suffix)
 
 	if not format and src_suffix == 'flac':
 		dst_suffix = 'ogg'
 		dst_bitrate = 320
-		dst_mimetype = scanner.get_mime(dst_suffix)
+		dst_mimetype = 'audio/ogg'
 		do_transcoding = True
 
-	app.logger.debug('Serving file: ' + res.path)
 	duration = mutagen.File(res.path).info.length
-	app.logger.debug('\tDuration of file: ' + str(duration))
+	app.logger.debug('Serving file: ' + res.path + '\n\tDuration of file: ' + str(duration))
 
 	if do_transcoding:
 		transcoder = config.get('transcoding', 'transcoder_{}_{}'.format(src_suffix, dst_suffix))
@@ -82,9 +97,7 @@ def stream_media():
 		encoder = map(lambda s: s.decode('UTF8'), shlex.split(encoder.encode('utf8')))
 		transcoder = map(lambda s: s.decode('UTF8'), shlex.split(transcoder.encode('utf8')))
 
-		app.logger.debug(decoder)
+		app.logger.debug(str( decoder ) + '\n' + str( encoder ) + '\n' + str(transcoder))
-		app.logger.debug(encoder)
-		app.logger.debug(transcoder)
 
 		if '|' in transcoder:
 			pipe_index = transcoder.index('|')
@@ -92,8 +105,7 @@ def stream_media():
 			encoder = transcoder[pipe_index+1:]
 			transcoder = None
 
-		app.logger.warn('decoder' + str(decoder))
+		app.logger.warn('decoder' + str(decoder) + '\nencoder' + str(encoder))
-		app.logger.warn('encoder' + str(encoder))
 
 		try:
 			if transcoder:
@@ -109,8 +121,14 @@ def stream_media():
 
 	else:
 		app.logger.warn('no transcode')
-		response = send_file(res.path, mimetype = dst_mimetype)
+		response = send_file(res.path)
+		response.headers['Content-Type'] = dst_mimetype
+		response.headers['Accept-Ranges'] = 'bytes'
 		response.headers['X-Content-Duration'] = str(duration)
+		redirect = config.get('base', 'accel-redirect')
+		if(redirect):
+			response.headers['X-Accel-Redirect'] = redirect + res.path
+			app.logger.debug('X-Accel-Redirect: ' + response.headers['X-Accel-Redirect'])
 
 	res.play_count = res.play_count + 1
 	res.last_play = now()
@@ -130,13 +148,35 @@ def download_media():
 
 @app.route('/rest/getCoverArt.view', methods = [ 'GET', 'POST' ])
 def cover_art():
+	@after_this_request
+	def add_header(response):
+		if 'X-Sendfile' in response.headers:
+			redirect = response.headers['X-Sendfile'] or ''
+			xsendfile = config.get('base', 'accel-redirect')
+			if redirect and xsendfile:
+				response.headers['X-Accel-Redirect'] =  xsendfile + redirect
+				app.logger.debug('X-Accel-Redirect: ' + xsendfile + redirect)
+		return response
+
 	status, res = get_entity(request, Folder)
+
 	if not status:
 		return res
 
-	if not res.has_cover_art or not os.path.isfile(os.path.join(res.path, 'cover.jpg')):
+	app.logger.debug('Cover Art Check: ' + res.path + '/*.jp*g')
+
+	coverfile = os.listdir(res.path)
+	coverfile = fnmatch.filter(coverfile, '*.jp*g')
+	app.logger.debug('Found Images: ' + str(coverfile))
+
+	if not coverfile:
+		app.logger.debug('No Art Found!')
+		res.has_cover_art = False
+		session.commit()
 		return request.error_formatter(70, 'Cover art not found')
 
+	coverfile = coverfile[0]
+
 	size = request.args.get('size')
 	if size:
 		try:
@@ -144,20 +184,25 @@ def cover_art():
 		except:
 			return request.error_formatter(0, 'Invalid size value')
 	else:
-		return send_file(os.path.join(res.path, 'cover.jpg'))
+		app.logger.debug('Serving cover art: ' + res.path + coverfile)
+		return send_file(os.path.join(res.path, coverfile))
 
-	im = Image.open(os.path.join(res.path, 'cover.jpg'))
+	im = Image.open(os.path.join(res.path, coverfile))
 	if size > im.size[0] and size > im.size[1]:
-		return send_file(os.path.join(res.path, 'cover.jpg'))
+		app.logger.debug('Serving cover art: ' + res.path + coverfile)
+		return send_file(os.path.join(res.path, coverfile))
 
 	size_path = os.path.join(config.get('base', 'cache_dir'), str(size))
 	path = os.path.join(size_path, str(res.id))
 	if os.path.exists(path):
+		app.logger.debug('Serving cover art: ' + path)
 		return send_file(path)
 	if not os.path.exists(size_path):
 		os.makedirs(size_path)
 
 	im.thumbnail([size, size], Image.ANTIALIAS)
 	im.save(path, 'JPEG')
+
+	app.logger.debug('Serving cover art: ' + path)
 	return send_file(path)
 

db.py

@@ -13,44 +13,13 @@ from sqlalchemy.dialects.postgresql import UUID as pgUUID
 import uuid, datetime, time
 import os.path
 
-def _unique(session, cls, hashfunc, queryfunc, constructor, arg, kw):
+Base = declarative_base()
-	cache = getattr(session, '_unique_cache', None)
-	if cache is None:
-		session._unique_cache = cache = {}
 
-	key = (cls, hashfunc(*arg, **kw))
+engine = create_engine(config.get('base', 'database_uri'), convert_unicode = True, echo = True)
-	if key in cache:
-		return cache[key]
-	else:
-		with session.no_autoflush:
-			q = session.query(cls)
-		q = queryfunc(q, *arg, **kw)
-		obj = q.first()
-		if not obj:
-			obj = constructor(*arg, **kw)
-			session.add(obj)
-		cache[key] = obj
-		return obj
 
-class UniqueMixin(object):
+session = scoped_session(sessionmaker(autoflush = False, bind = engine))
-	@classmethod
-	def unique_hash(cls, *arg, **kw):
-		raise NotImplementedError()
 
-	@classmethod
+Base.query = session.query_property()
-	def unique_filter(cls, query, *arg, **kw):
-		raise NotImplementedError()
-
-	@classmethod
-	def as_unique(cls, session, *arg, **kw):
-		return _unique(
-				session,
-				cls,
-				cls.unique_hash,
-				cls.unique_filter,
-				cls,
-				arg, kw
-				)
 
 class UUID(TypeDecorator):
 	"""Platform-somewhat-independent UUID type
@@ -96,11 +65,6 @@ class UUID(TypeDecorator):
 def now():
 	return datetime.datetime.now().replace(microsecond = 0)
 
-engine = create_engine(config.get('base', 'database_uri'), convert_unicode = True)
-session = scoped_session(sessionmaker(autoflush = False, bind = engine))
-
-Base = declarative_base()
-Base.query = session.query_property()
 
 class User(Base):
 	__tablename__ = 'user'
@@ -177,21 +141,13 @@ class Folder(Base):
 
 		return info
 
-class Artist(UniqueMixin, Base):
+class Artist(Base):
 	__tablename__ = 'artist'
 
 	id = UUID.gen_id_column()
-	name = Column(String(256), unique = True, nullable=False)
+	name = Column(String(255), nullable=False)
 	albums = relationship('Album', backref = 'artist')
 
-	@classmethod
-	def unique_hash(cls, name):
-		return name
-
-	@classmethod
-	def unique_filter(cls, query, name):
-		return query.filter(Artist.name == name)
-
 	def as_subsonic_artist(self, user):
 		info = {
 			'id': str(self.id),
@@ -210,7 +166,7 @@ class Album(Base):
 	__tablename__ = 'album'
 
 	id = UUID.gen_id_column()
-	name = Column(String(256))
+	name = Column(String(255))
 	artist_id = Column(UUID, ForeignKey('artist.id'))
 	tracks = relationship('Track', backref = 'album')
 

managers/folder.py

@@ -93,7 +93,6 @@ class FolderManager:
 
 		scanner.scan(folder)
 		scanner.prune(folder)
-		scanner.check_cover_art(folder)
 		return FolderManager.SUCCESS
 
 	@staticmethod

scanner.py

@@ -1,24 +1,29 @@
 # coding: utf-8
 
-import sys
 import os, os.path
-import time, mimetypes
+import time
 import mutagen
 import config, db
 import math
+import sys, traceback
 from web import app
 
-def get_mime(ext):
+from profilehooks import profile
-	return mimetypes.guess_type('dummy.' + ext, False)[0] or config.get('mimetypes', ext) or 'application/octet-stream'
 
 class Scanner:
 	def __init__(self, session):
 		self.__session = session
+
 		self.__tracks  = db.Track.query.all()
 		self.__tracks = {x.path: x for x in self.__tracks}
 
 		self.__artists = db.Artist.query.all()
+		self.__artists = {x.name.lower(): x for x in self.__artists}
+
 		self.__folders = db.Folder.query.all()
+		self.__folders = {x.path: x for x in self.__folders}
+
+		self.__playlists = db.Playlist.query.all()
 
 		self.__added_artists = 0
 		self.__added_albums  = 0
@@ -27,74 +32,59 @@ class Scanner:
 		self.__deleted_albums  = 0
 		self.__deleted_tracks  = 0
 
-		extensions = config.get('base', 'scanner_extensions')
-		self.__extensions = map(str.lower, extensions.split()) if extensions else None
-
 	def scan(self, folder):
 		print "scanning", folder.path
 		valid = [x.lower() for x in config.get('base','filetypes').split(',')]
+		valid = tuple(valid)
 		print "valid filetypes: ",valid
 
-		n = 0
 		for root, subfolders, files in os.walk(folder.path, topdown=False):
 			for f in files:
-				suffix = os.path.splitext(f)[1][1:].lower()
+				if f.lower().endswith(valid):
-				n += 1
-				if n == 1000:
-					app.logger.debug('commit db')
-					self.__session.commit()
-					n = 0
-
-				if suffix in valid:
 					try:
-						app.logger.debug('Scanning File: ' + os.path.join(root, f))
 						self.__scan_file(os.path.join(root, f), folder)
-						self.__session.flush()
 					except:
 						app.logger.error('Problem adding file: ' + os.path.join(root,f))
-						app.logger.error(sys.exc_info())
+						app.logger.error(traceback.print_exc())
+						sys.exit(0)
 						self.__session.rollback()
 
-
+		print "\a"
-
+		self.__session.add_all(self.__tracks.values())
 		self.__session.commit()
 		folder.last_scan = int(time.time())
 
 	def prune(self, folder):
-		for k, track in self.__tracks.iteritems():
+		for path, root_folder_id, track_id in self.__session.query(db.Track.path, db.Track.root_folder_id, db.Track.id):
-			if track.root_folder.id == folder.id and not self.__is_valid_path(k):
+			if root_folder_id == folder.id and not os.path.exists(path):
-				app.debug('Removed invalid path: ' + k)
+				app.logger.debug('Removed invalid path: ' + path)
-				self.__remove_track(track)
+				self.__remove_track(self.__session.merge(db.Track(id = track_id)))
 
-		for album in [ album for artist in self.__artists for album in artist.albums if len(album.tracks) == 0 ]:
+		self.__session.commit()
+
+		for album in [ album for artist in self.__artists.values() for album in artist.albums if len(album.tracks) == 0 ]:
 			album.artist.albums.remove(album)
 			self.__session.delete(album)
 			self.__deleted_albums += 1
 
-		for artist in [ a for a in self.__artists if len(a.albums) == 0 ]:
+		self.__session.commit()
+
+		for artist in [ a for a in self.__artists.values() if len(a.albums) == 0 ]:
 			self.__session.delete(artist)
 			self.__deleted_artists += 1
 
+		self.__session.commit()
+
 		self.__cleanup_folder(folder)
 
-	def check_cover_art(self, folder):
+	@profile
-		folder.has_cover_art = os.path.isfile(os.path.join(folder.path, 'cover.jpg'))
-		for f in folder.children:
-			self.check_cover_art(f)
-
-	def __is_valid_path(self, path):
-		if not os.path.exists(path):
-			return False
-		if not self.__extensions:
-			return True
-		return os.path.splitext(path)[1][1:].lower() in self.__extensions
-
 	def __scan_file(self, path, folder):
 		curmtime = int(math.floor(os.path.getmtime(path)))
 
 		if path in self.__tracks:
 			tr = self.__tracks[path]
 
+			app.logger.debug('Existing File: ' + path)
 			if not tr.last_modification:
 				tr.last_modification = curmtime
 
@@ -111,16 +101,16 @@ class Scanner:
 				self.__remove_track(tr)
 				return False
 		else:
-			app.logger.debug('\tReading tag')
+			app.logger.debug('Scanning File: ' + path + '\n\tReading tag')
 			tag = self.__try_load_tag(path)
 			if not tag:
 				app.logger.debug('\tProblem reading tag')
 				return False
 
 			tr = db.Track(path = path, root_folder = folder, folder = self.__find_folder(path, folder))
+
 			self.__tracks[path] = tr
 			self.__added_tracks += 1
-			print "Added ", path
 
 		tr.last_modification = curmtime
 		tr.disc     = self.__try_read_tag(tag, 'discnumber',  1, lambda x: int(x[0].split('/')[0]))
@@ -129,49 +119,55 @@ class Scanner:
 		tr.year     = self.__try_read_tag(tag, 'date', None, lambda x: int(x[0].split('-')[0]))
 		tr.genre    = self.__try_read_tag(tag, 'genre')
 		tr.duration = int(tag.info.length)
+
+		# TODO: use album artist if available, then artist, then unknown
 		tr.album    = self.__find_album(self.__try_read_tag(tag, 'artist', 'Unknown'), self.__try_read_tag(tag, 'album', 'Unknown'))
+
 		tr.bitrate  = (tag.info.bitrate if hasattr(tag.info, 'bitrate') else int(os.path.getsize(path) * 8 / tag.info.length)) / 1000
-		tr.content_type = get_mime(os.path.splitext(path)[1][1:])
 
 		return True
 
 	def __find_album(self, artist, album):
-		ar = self.__find_artist(artist)
+		# TODO : DB specific issues with single column name primary key
+		#		for instance, case sensitivity and trailing spaces
+		artist = artist.rstrip()
 
-		al = filter(lambda a: a.name == album, ar.albums)
+		if artist in self.__artists:
-		if al:
+			ar = self.__artists[artist]
-			return al[0]
+		else:
+			#Flair!
+			sys.stdout.write('\033[K')
+			sys.stdout.write('%s\r' % artist)
+			sys.stdout.flush()
+			ar = db.Artist(name = artist)
+			self.__artists[artist] = ar
+			self.__added_artists += 1
 
-		al = db.Album(name = album, artist = ar)
+		al = {a.name: a for a in ar.albums}
-		self.__added_albums += 1
+		if album in al:
-
+			return al[album]
-		return al
+		else:
-
+			self.__added_albums += 1
-	def __find_artist(self, artist):
+			return db.Album(name = album, artist = ar)
-		ar = db.Artist.as_unique(self.__session, name = artist)
-
-		self.__artists.append(ar)
-		self.__added_artists += 1
-
-		return ar
 
 	def __find_folder(self, path, folder):
-		path = os.path.dirname(path)
-		fold = filter(lambda f: f.path == path, self.__folders)
-		if fold:
-			return fold[0]
 
+		path = os.path.dirname(path)
+		if path in self.__folders:
+			return self.__folders[path]
+
+		# must find parent directory to create new one
 		full_path = folder.path
 		path = path[len(folder.path) + 1:]
 
 		for name in path.split(os.sep):
 			full_path = os.path.join(full_path, name)
-			fold = filter(lambda f: f.path == full_path, self.__folders)
+
-			if fold:
+			if full_path in self.__folders:
-				folder = fold[0]
+				folder = self.__folders[full_path]
 			else:
 				folder = db.Folder(root = False, name = name, path = full_path, parent = folder)
-				self.__folders.append(folder)
+				self.__folders[full_path] = folder
 
 		return folder
 
@@ -193,12 +189,12 @@ class Scanner:
 			return default
 
 	def __remove_track(self, track):
-		track.album.tracks.remove(track)
-		track.folder.tracks.remove(track)
 		# As we don't have a track -> playlists relationship, SQLAlchemy doesn't know it has to remove tracks
 		# from playlists as well, so let's help it
-		for playlist in db.Playlist.query.filter(db.Playlist.tracks.contains(track)):
+		for playlist in self.__playlists:
-			playlist.tracks.remove(track)
+			if track in playlist.tracks:
+				playlist.tracks.remove(track)
+
 		self.__session.delete(track)
 		self.__deleted_tracks += 1
 

supysonic.ini

@@ -0,0 +1,10 @@
+[uwsgi]
+
+    socket = /tmp/supysonic.sock
+    wsgi-file = /home/emory/supysonic/main.wsgi
+    master = true
+    processes = 4
+    threads = 2
+    uid = 1000
+    vacuum = true
+    chmod-socket = 666

web.py

@@ -6,13 +6,16 @@ import config
 app = Flask(__name__)
 app.secret_key = '?9huDM\\H'
 
+if(config.get('base', 'accel-redirect')):
+    app.use_x_sendfile = True
+
 if config.get('base', 'debug'):
     app.debug = True
 
 if config.get('base', 'log_file'):
 	import logging
 	from logging.handlers import TimedRotatingFileHandler
-	handler = TimedRotatingFileHandler(config.get('base', 'log_file'), when = 'midnight')
+	handler = TimedRotatingFileHandler(config.get('base', 'log_file'), when = 'midnight', encoding = 'UTF-8')
 	handler.setLevel(logging.DEBUG)
 	app.logger.addHandler(handler)