From 2c4ec6c0e8969257b01fd6ba18e2162c643d04db Mon Sep 17 00:00:00 2001 From: Taizo Simpson Date: Wed, 10 Oct 2018 12:58:47 -0400 Subject: [PATCH 1/6] Whether or not a track has embeded art is tracked just like any other metadata, to reduce disk r/w and mem usage and a lot of other things. WARNING: This needs migrations before being merged to master --- supysonic/db.py | 12 ++++++++---- supysonic/scanner.py | 1 + supysonic/schema/mysql.sql | 1 + supysonic/schema/postgres.sql | 1 + supysonic/schema/sqlite.sql | 1 + tests/base/test_db.py | 2 ++ 6 files changed, 14 insertions(+), 4 deletions(-) diff --git a/supysonic/db.py b/supysonic/db.py index 1db68a3..969f6cf 100644 --- a/supysonic/db.py +++ b/supysonic/db.py @@ -105,7 +105,7 @@ class Folder(PathMixin, db.Entity): info['coverArt'] = str(self.id) else: for track in self.tracks: - if track.extract_cover_art(): + if track.has_art: info['coverArt'] = str(track.id) break @@ -216,6 +216,7 @@ class Track(PathMixin, db.Entity): year = Optional(int) genre = Optional(str, nullable = True) duration = Required(int) + has_art = Required(bool, default=False) album = Required(Album, column = 'album_id') artist = Required(Artist, column = 'artist_id') @@ -266,7 +267,7 @@ class Track(PathMixin, db.Entity): info['year'] = self.year if self.genre: info['genre'] = self.genre - if self.extract_cover_art(): + if self.has_art: info['coverArt'] = str(self.id) elif self.folder.cover_art: info['coverArt'] = str(self.folder.id) @@ -304,8 +305,11 @@ class Track(PathMixin, db.Entity): return (self.album.artist.name + self.album.name + ("%02i" % self.disc) + ("%02i" % self.number) + self.title).lower() def extract_cover_art(self): - if os.path.exists(self.path): - metadata = mutagen.File(self.path) + return Track._extract_cover_art(self.path) + + def _extract_cover_art(path): + if os.path.exists(path): + metadata = mutagen.File(path) data = None if metadata: if isinstance(metadata.tags, mutagen.id3.ID3Tags) and len(metadata.tags.getall('APIC')) > 0: diff --git a/supysonic/scanner.py b/supysonic/scanner.py index 99b5527..ae5061b 100644 --- a/supysonic/scanner.py +++ b/supysonic/scanner.py @@ -140,6 +140,7 @@ class Scanner: trdict['year'] = self.__try_read_tag(tag, 'date', None, lambda x: int(x[0].split('-')[0])) trdict['genre'] = self.__try_read_tag(tag, 'genre') trdict['duration'] = int(tag.info.length) + trdict['has_art'] = bool(Track._extract_cover_art(path)) trdict['bitrate'] = (tag.info.bitrate if hasattr(tag.info, 'bitrate') else int(os.path.getsize(path) * 8 / tag.info.length)) // 1000 trdict['content_type'] = mimetypes.guess_type(path, False)[0] or 'application/octet-stream' diff --git a/supysonic/schema/mysql.sql b/supysonic/schema/mysql.sql index 9787df2..68571b4 100644 --- a/supysonic/schema/mysql.sql +++ b/supysonic/schema/mysql.sql @@ -29,6 +29,7 @@ CREATE TABLE IF NOT EXISTS track ( year INTEGER, genre VARCHAR(256), duration INTEGER NOT NULL, + has_art BOOLEAN NOT NULL, album_id BINARY(16) NOT NULL REFERENCES album, artist_id BINARY(16) NOT NULL REFERENCES artist, bitrate INTEGER NOT NULL, diff --git a/supysonic/schema/postgres.sql b/supysonic/schema/postgres.sql index 9339f55..5f447c1 100644 --- a/supysonic/schema/postgres.sql +++ b/supysonic/schema/postgres.sql @@ -29,6 +29,7 @@ CREATE TABLE IF NOT EXISTS track ( year INTEGER, genre VARCHAR(256), duration INTEGER NOT NULL, + has_art BOOLEAN NOT NULL, album_id UUID NOT NULL REFERENCES album, artist_id UUID NOT NULL REFERENCES artist, bitrate INTEGER NOT NULL, diff --git a/supysonic/schema/sqlite.sql b/supysonic/schema/sqlite.sql index bf8e567..c7781b4 100644 --- a/supysonic/schema/sqlite.sql +++ b/supysonic/schema/sqlite.sql @@ -31,6 +31,7 @@ CREATE TABLE IF NOT EXISTS track ( year INTEGER, genre VARCHAR(256), duration INTEGER NOT NULL, + has_art BOOLEAN NOT NULL, album_id CHAR(36) NOT NULL REFERENCES album, artist_id CHAR(36) NOT NULL REFERENCES artist, bitrate INTEGER NOT NULL, diff --git a/tests/base/test_db.py b/tests/base/test_db.py index 8365196..c3bc600 100644 --- a/tests/base/test_db.py +++ b/tests/base/test_db.py @@ -71,6 +71,7 @@ class DbTestCase(unittest.TestCase): disc = 1, number = 1, duration = 3, + has_art = True, bitrate = 320, path = 'tests/assets/formats/silence.ogg', content_type = 'audio/ogg', @@ -106,6 +107,7 @@ class DbTestCase(unittest.TestCase): disc = 1, number = 1, duration = 5, + has_art = True, bitrate = 96, path = 'tests/assets/formats/silence.flac', content_type = 'audio/flac', From 6efb8e8c2d5b231ea4b1f418669ee1890c3e76bc Mon Sep 17 00:00:00 2001 From: Taizo Simpson Date: Fri, 12 Oct 2018 13:53:56 -0400 Subject: [PATCH 2/6] Added some migrations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WARNING: Still not ready for production — schema version needs incrementing, and testing still required --- supysonic/schema/migration/mysql/20181010.py | 24 +++++++++++++++++++ .../schema/migration/postgres/20181010.py | 21 ++++++++++++++++ supysonic/schema/migration/sqlite/20181010.py | 18 ++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 supysonic/schema/migration/mysql/20181010.py create mode 100644 supysonic/schema/migration/postgres/20181010.py create mode 100644 supysonic/schema/migration/sqlite/20181010.py diff --git a/supysonic/schema/migration/mysql/20181010.py b/supysonic/schema/migration/mysql/20181010.py new file mode 100644 index 0000000..cc42071 --- /dev/null +++ b/supysonic/schema/migration/mysql/20181010.py @@ -0,0 +1,24 @@ +import argparse +from supysonic import db +try: + import MySQLdb as provider +except ImportError: + import pymysql as provider + +parser = argparse.ArgumentParser() +parser.add_argument('username') +parser.add_argument('password') +parser.add_argument('database') +parser.add_argument('-H', '--host', default = 'localhost', help = 'default: localhost') +args = parser.parse_args() + +with provider.connect(host = args.host, user = args.username, passwd = args.password, db = args.database) as conn: + c = conn.cursor() + c.execute('ALTER TABLE track ADD COLUMN has_art BOOLEAN NOT NULL DEFAULT false') + + art = dict() + c.execute('SELECT path FROM track') + for row in c: + art[row[0]] = bool(db.Track._extract_cover_art(row[0].encode('utf-8'))) + c.executemany('UPDATE track SET has_art=? WHERE path=?', [ (a, p) for p, a in art.items() ]) + conn.commit() diff --git a/supysonic/schema/migration/postgres/20181010.py b/supysonic/schema/migration/postgres/20181010.py new file mode 100644 index 0000000..8716e26 --- /dev/null +++ b/supysonic/schema/migration/postgres/20181010.py @@ -0,0 +1,21 @@ +import argparse +import psycopg2 +from supysonic import db + +parser = argparse.ArgumentParser() +parser.add_argument('username') +parser.add_argument('password') +parser.add_argument('database') +parser.add_argument('-H', '--host', default = 'localhost', help = 'default: localhost') +args = parser.parse_args() + +with psycopg2.connect(host = args.host, user = args.username, password = args.password, dbname = args.database) as conn: + c = conn.cursor() + c.execute('ALTER TABLE track ADD COLUMN has_art BOOLEAN NOT NULL DEFAULT false') + + art = dict() + c.execute('SELECT path FROM track') + for row in c.fetchall(): + art[row[0]] = bool(db.Track._extract_cover_art(row[0].encode('utf-8'))) + c.executemany('UPDATE track SET has_art=%s WHERE path=%s', [ (a, p) for p, a in art.items() ]) + conn.commit() diff --git a/supysonic/schema/migration/sqlite/20181010.py b/supysonic/schema/migration/sqlite/20181010.py new file mode 100644 index 0000000..c3b7103 --- /dev/null +++ b/supysonic/schema/migration/sqlite/20181010.py @@ -0,0 +1,18 @@ +import argparse +import sqlite3 +from supysonic import db + +parser = argparse.ArgumentParser() +parser.add_argument('dbfile', help = 'Path to the SQLite database file') +args = parser.parse_args() + +with sqlite3.connect(args.dbfile) as conn: + c = conn.cursor() + c.execute('ALTER TABLE track ADD COLUMN has_art BOOLEAN NOT NULL DEFAULT false') + + art = dict() + for row in c.execute('SELECT path FROM track'): + art[row[0]] = bool(db.Track._extract_cover_art(row[0].encode('utf-8'))) + c.executemany('UPDATE track SET has_art=? WHERE path=?', [ (a, p) for p, a in art.items() ]) + conn.commit() + conn.execute('VACUUM') From 066658f167c02c20a2403d02306511b86bf4aed3 Mon Sep 17 00:00:00 2001 From: Taizo Simpson Date: Fri, 12 Oct 2018 19:07:48 -0400 Subject: [PATCH 3/6] Consider embeded art when serializing albums, add relevant test --- supysonic/db.py | 4 ++++ tests/base/test_db.py | 17 ++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/supysonic/db.py b/supysonic/db.py index 969f6cf..6c488b6 100644 --- a/supysonic/db.py +++ b/supysonic/db.py @@ -190,6 +190,10 @@ class Album(db.Entity): track_with_cover = self.tracks.select(lambda t: t.folder.cover_art is not None).first() if track_with_cover is not None: info['coverArt'] = str(track_with_cover.folder.id) + else: + track_with_cover = self.tracks.select(lambda t: t.has_art).first() + if track_with_cover is not None: + info['coverArt'] = str(track_with_cover.id) try: starred = StarredAlbum[user.id, self.id] diff --git a/tests/base/test_db.py b/tests/base/test_db.py index c3bc600..e99c7a5 100644 --- a/tests/base/test_db.py +++ b/tests/base/test_db.py @@ -97,9 +97,9 @@ class DbTestCase(unittest.TestCase): return track1, track2 - def create_track_in(self, folder, root): - artist = db.Artist(name = 'Snazzy Artist') - album = db.Album(artist = artist, name = 'Rockin\' Album') + def create_track_in(self, folder, root, artist = None, album = None, has_art = True): + artist = artist or db.Artist(name = 'Snazzy Artist') + album = album or db.Album(artist = artist, name = 'Rockin\' Album') return db.Track( title = 'Nifty Number', album = album, @@ -107,7 +107,7 @@ class DbTestCase(unittest.TestCase): disc = 1, number = 1, duration = 5, - has_art = True, + has_art = has_art, bitrate = 96, path = 'tests/assets/formats/silence.flac', content_type = 'audio/flac', @@ -234,7 +234,8 @@ class DbTestCase(unittest.TestCase): # No tracks, shouldn't be stored under normal circumstances self.assertRaises(ValueError, album.as_subsonic_album, user) - self.create_some_tracks(artist, album) + root_folder, folder_art, folder_noart = self.create_some_folders() + track1 = self.create_track_in(root_folder, folder_noart, artist = artist, album = album) album_dict = album.as_subsonic_album(user) self.assertIsInstance(album_dict, dict) @@ -246,11 +247,13 @@ class DbTestCase(unittest.TestCase): self.assertIn('duration', album_dict) self.assertIn('created', album_dict) self.assertIn('starred', album_dict) + self.assertIn('coverArt', album_dict) self.assertEqual(album_dict['name'], album.name) self.assertEqual(album_dict['artist'], artist.name) self.assertEqual(album_dict['artistId'], str(artist.id)) - self.assertEqual(album_dict['songCount'], 2) - self.assertEqual(album_dict['duration'], 8) + self.assertEqual(album_dict['songCount'], 1) + self.assertEqual(album_dict['duration'], 5) + self.assertEqual(album_dict['coverArt'], str(track1.id)) self.assertRegex(album_dict['created'], date_regex) self.assertRegex(album_dict['starred'], date_regex) From 753b4d9df86d1b0b840fa7ae6fc7d202289bc788 Mon Sep 17 00:00:00 2001 From: Taizo Simpson Date: Fri, 12 Oct 2018 19:13:12 -0400 Subject: [PATCH 4/6] Updated schema version --- supysonic/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supysonic/db.py b/supysonic/db.py index 6c488b6..a13c90f 100644 --- a/supysonic/db.py +++ b/supysonic/db.py @@ -31,7 +31,7 @@ try: except ImportError: from urlparse import urlparse, parse_qsl -SCHEMA_VERSION = '20180829' +SCHEMA_VERSION = '20181010' def now(): return datetime.now().replace(microsecond = 0) From c89395b2206b621ccfe16e47b7f588a5b73d58a4 Mon Sep 17 00:00:00 2001 From: Taizo Simpson Date: Fri, 12 Oct 2018 19:36:21 -0400 Subject: [PATCH 5/6] Annotated method as static so python2 stops complaining --- supysonic/db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/supysonic/db.py b/supysonic/db.py index a13c90f..bc65e84 100644 --- a/supysonic/db.py +++ b/supysonic/db.py @@ -311,6 +311,7 @@ class Track(PathMixin, db.Entity): def extract_cover_art(self): return Track._extract_cover_art(self.path) + @staticmethod def _extract_cover_art(path): if os.path.exists(path): metadata = mutagen.File(path) From f7dc6292fd93b53e4561839493705d7d4e85dfd6 Mon Sep 17 00:00:00 2001 From: Taizo Simpson Date: Sun, 14 Oct 2018 14:55:36 -0400 Subject: [PATCH 6/6] Switched from python to sql migration --- supysonic/schema/migration/mysql/20181010.py | 24 ------------------- supysonic/schema/migration/mysql/20181010.sql | 6 +++++ .../schema/migration/postgres/20181010.py | 21 ---------------- .../schema/migration/postgres/20181010.sql | 6 +++++ supysonic/schema/migration/sqlite/20181010.py | 18 -------------- .../schema/migration/sqlite/20181010.sql | 3 +++ supysonic/schema/mysql.sql | 2 +- supysonic/schema/postgres.sql | 2 +- supysonic/schema/sqlite.sql | 2 +- 9 files changed, 18 insertions(+), 66 deletions(-) delete mode 100644 supysonic/schema/migration/mysql/20181010.py create mode 100644 supysonic/schema/migration/mysql/20181010.sql delete mode 100644 supysonic/schema/migration/postgres/20181010.py create mode 100644 supysonic/schema/migration/postgres/20181010.sql delete mode 100644 supysonic/schema/migration/sqlite/20181010.py create mode 100644 supysonic/schema/migration/sqlite/20181010.sql diff --git a/supysonic/schema/migration/mysql/20181010.py b/supysonic/schema/migration/mysql/20181010.py deleted file mode 100644 index cc42071..0000000 --- a/supysonic/schema/migration/mysql/20181010.py +++ /dev/null @@ -1,24 +0,0 @@ -import argparse -from supysonic import db -try: - import MySQLdb as provider -except ImportError: - import pymysql as provider - -parser = argparse.ArgumentParser() -parser.add_argument('username') -parser.add_argument('password') -parser.add_argument('database') -parser.add_argument('-H', '--host', default = 'localhost', help = 'default: localhost') -args = parser.parse_args() - -with provider.connect(host = args.host, user = args.username, passwd = args.password, db = args.database) as conn: - c = conn.cursor() - c.execute('ALTER TABLE track ADD COLUMN has_art BOOLEAN NOT NULL DEFAULT false') - - art = dict() - c.execute('SELECT path FROM track') - for row in c: - art[row[0]] = bool(db.Track._extract_cover_art(row[0].encode('utf-8'))) - c.executemany('UPDATE track SET has_art=? WHERE path=?', [ (a, p) for p, a in art.items() ]) - conn.commit() diff --git a/supysonic/schema/migration/mysql/20181010.sql b/supysonic/schema/migration/mysql/20181010.sql new file mode 100644 index 0000000..df807de --- /dev/null +++ b/supysonic/schema/migration/mysql/20181010.sql @@ -0,0 +1,6 @@ +START TRANSACTION; + +ALTER TABLE track ADD has_art BOOLEAN DEFAULT false NOT NULL; + +COMMIT; + diff --git a/supysonic/schema/migration/postgres/20181010.py b/supysonic/schema/migration/postgres/20181010.py deleted file mode 100644 index 8716e26..0000000 --- a/supysonic/schema/migration/postgres/20181010.py +++ /dev/null @@ -1,21 +0,0 @@ -import argparse -import psycopg2 -from supysonic import db - -parser = argparse.ArgumentParser() -parser.add_argument('username') -parser.add_argument('password') -parser.add_argument('database') -parser.add_argument('-H', '--host', default = 'localhost', help = 'default: localhost') -args = parser.parse_args() - -with psycopg2.connect(host = args.host, user = args.username, password = args.password, dbname = args.database) as conn: - c = conn.cursor() - c.execute('ALTER TABLE track ADD COLUMN has_art BOOLEAN NOT NULL DEFAULT false') - - art = dict() - c.execute('SELECT path FROM track') - for row in c.fetchall(): - art[row[0]] = bool(db.Track._extract_cover_art(row[0].encode('utf-8'))) - c.executemany('UPDATE track SET has_art=%s WHERE path=%s', [ (a, p) for p, a in art.items() ]) - conn.commit() diff --git a/supysonic/schema/migration/postgres/20181010.sql b/supysonic/schema/migration/postgres/20181010.sql new file mode 100644 index 0000000..df807de --- /dev/null +++ b/supysonic/schema/migration/postgres/20181010.sql @@ -0,0 +1,6 @@ +START TRANSACTION; + +ALTER TABLE track ADD has_art BOOLEAN DEFAULT false NOT NULL; + +COMMIT; + diff --git a/supysonic/schema/migration/sqlite/20181010.py b/supysonic/schema/migration/sqlite/20181010.py deleted file mode 100644 index c3b7103..0000000 --- a/supysonic/schema/migration/sqlite/20181010.py +++ /dev/null @@ -1,18 +0,0 @@ -import argparse -import sqlite3 -from supysonic import db - -parser = argparse.ArgumentParser() -parser.add_argument('dbfile', help = 'Path to the SQLite database file') -args = parser.parse_args() - -with sqlite3.connect(args.dbfile) as conn: - c = conn.cursor() - c.execute('ALTER TABLE track ADD COLUMN has_art BOOLEAN NOT NULL DEFAULT false') - - art = dict() - for row in c.execute('SELECT path FROM track'): - art[row[0]] = bool(db.Track._extract_cover_art(row[0].encode('utf-8'))) - c.executemany('UPDATE track SET has_art=? WHERE path=?', [ (a, p) for p, a in art.items() ]) - conn.commit() - conn.execute('VACUUM') diff --git a/supysonic/schema/migration/sqlite/20181010.sql b/supysonic/schema/migration/sqlite/20181010.sql new file mode 100644 index 0000000..f4013c9 --- /dev/null +++ b/supysonic/schema/migration/sqlite/20181010.sql @@ -0,0 +1,3 @@ +ALTER TABLE track ADD has_art BOOLEAN DEFAULT false NOT NULL; + +COMMIT; diff --git a/supysonic/schema/mysql.sql b/supysonic/schema/mysql.sql index 68571b4..5c48f03 100644 --- a/supysonic/schema/mysql.sql +++ b/supysonic/schema/mysql.sql @@ -29,7 +29,7 @@ CREATE TABLE IF NOT EXISTS track ( year INTEGER, genre VARCHAR(256), duration INTEGER NOT NULL, - has_art BOOLEAN NOT NULL, + has_art BOOLEAN NOT NULL DEFAULT false, album_id BINARY(16) NOT NULL REFERENCES album, artist_id BINARY(16) NOT NULL REFERENCES artist, bitrate INTEGER NOT NULL, diff --git a/supysonic/schema/postgres.sql b/supysonic/schema/postgres.sql index 5f447c1..66cbb60 100644 --- a/supysonic/schema/postgres.sql +++ b/supysonic/schema/postgres.sql @@ -29,7 +29,7 @@ CREATE TABLE IF NOT EXISTS track ( year INTEGER, genre VARCHAR(256), duration INTEGER NOT NULL, - has_art BOOLEAN NOT NULL, + has_art BOOLEAN NOT NULL DEFAULT false, album_id UUID NOT NULL REFERENCES album, artist_id UUID NOT NULL REFERENCES artist, bitrate INTEGER NOT NULL, diff --git a/supysonic/schema/sqlite.sql b/supysonic/schema/sqlite.sql index c7781b4..169ab58 100644 --- a/supysonic/schema/sqlite.sql +++ b/supysonic/schema/sqlite.sql @@ -31,7 +31,7 @@ CREATE TABLE IF NOT EXISTS track ( year INTEGER, genre VARCHAR(256), duration INTEGER NOT NULL, - has_art BOOLEAN NOT NULL, + has_art BOOLEAN NOT NULL DEFAULT false, album_id CHAR(36) NOT NULL REFERENCES album, artist_id CHAR(36) NOT NULL REFERENCES artist, bitrate INTEGER NOT NULL,