Provided migration scripts for existing storm databases

README.md

@@ -69,8 +69,9 @@ You may also need a database specific package:
 
 ### Configuration
 
-Supysonic looks for two files for its configuration: `/etc/supysonic` and
+Supysonic looks for four files for its configuration: `/etc/supysonic`,
-`~/.supysonic`, merging values from the two files.
+`~/.supysonic`, `~/.config/supysonic/supysonic.conf` and `supysonic.conf` in
+the current folder, merging values from all files.
 
 Configuration files must respect a structure similar to Windows INI file, with
 `[section]` headers and using a `KEY = VALUE` or `KEY: VALUE` syntax.
@@ -383,3 +384,7 @@ the case migration scripts will be provided in the `schema/migration`
 folder, prefixed by the date of commit that introduced the changes. Those
 scripts shouldn't be used when initializing a new database, only when
 upgrading from a previous schema.
+There could be both SQL scripts or Python scripts. The Python scripts require
+arguments that are explained when the script is invoked with the `-h` flag.
+If a migration script isn't provided for a specific database engine, it simply
+means that no migration is needed for this engine.

schema/migration/20171230.mysql.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+#
+# This file is part of Supysonic.
+# Supysonic is a Python implementation of the Subsonic server API.
+#
+# Copyright (C) 2017 Alban 'spl0k' Féron
+#
+# Distributed under terms of the GNU AGPLv3 license.
+
+# Converts ids from hex-encoded strings to binary data
+
+import argparse
+try:
+    import MySQLdb as provider
+except ImportError:
+    import pymysql as provider
+
+from uuid import UUID
+from warnings import filterwarnings
+
+parser = argparse.ArgumentParser()
+parser.add_argument('username')
+parser.add_argument('password')
+parser.add_argument('database')
+parser.add_argument('-H', '--host', default = 'localhost', help = 'default: localhost')
+args = parser.parse_args()
+
+def process_table(connection, table, fields):
+    to_update = { field: set() for field in fields }
+
+    c = connection.cursor()
+    c.execute('SELECT {1} FROM {0}'.format(table, ','.join(fields)))
+    for row in c:
+        for field, value in zip(fields, row):
+            if value is None or not isinstance(value, basestring):
+                continue
+            to_update[field].add(value)
+
+    for field, values in to_update.iteritems():
+        sql = 'UPDATE {0} SET {1}=%s WHERE {1}=%s'.format(table, field)
+        c.executemany(sql, map(lambda v: (UUID(v).bytes, v), values))
+        sql = 'ALTER TABLE {0} MODIFY {1} BINARY(16)'.format(table, field)
+        c.execute(sql)
+
+    connection.commit()
+
+filterwarnings('ignore', category = provider.Warning)
+conn =  provider.connect(host = args.host, user = args.username, passwd = args.password, db = args.database)
+conn.cursor().execute('SET FOREIGN_KEY_CHECKS = 0')
+
+process_table(conn, 'folder', ('id', 'parent_id'))
+process_table(conn, 'artist', ('id',))
+process_table(conn, 'album', ('id', 'artist_id'))
+process_table(conn, 'track', ('id', 'album_id', 'artist_id', 'root_folder_id', 'folder_id'))
+process_table(conn, 'user', ('id', 'last_play_id'))
+process_table(conn, 'client_prefs', ('user_id',))
+process_table(conn, 'starred_folder', ('user_id', 'starred_id'))
+process_table(conn, 'starred_artist', ('user_id', 'starred_id'))
+process_table(conn, 'starred_album', ('user_id', 'starred_id'))
+process_table(conn, 'starred_track', ('user_id', 'starred_id'))
+process_table(conn, 'rating_folder', ('user_id', 'rated_id'))
+process_table(conn, 'rating_track', ('user_id', 'rated_id'))
+process_table(conn, 'chat_message', ('id', 'user_id'))
+process_table(conn, 'playlist', ('id', 'user_id'))
+
+conn.cursor().execute('SET FOREIGN_KEY_CHECKS = 1')
+conn.close()
+

schema/migration/20171230.sqlite.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+#
+# This file is part of Supysonic.
+# Supysonic is a Python implementation of the Subsonic server API.
+#
+# Copyright (C) 2017 Alban 'spl0k' Féron
+#
+# Distributed under terms of the GNU AGPLv3 license.
+
+# Converts ids from hex-encoded strings to binary data
+
+import argparse
+import sqlite3
+
+from uuid import UUID
+
+parser = argparse.ArgumentParser()
+parser.add_argument('dbfile', help = 'Path to the SQLite database file')
+args = parser.parse_args()
+
+def process_table(connection, table, fields):
+    to_update = { field: set() for field in fields }
+
+    c = connection.cursor()
+    for row in c.execute('SELECT {1} FROM {0}'.format(table, ','.join(fields))):
+        for field, value in zip(fields, row):
+            if value is None or not isinstance(value, basestring):
+                continue
+            to_update[field].add(value)
+
+    for field, values in to_update.iteritems():
+        sql = 'UPDATE {0} SET {1}=? WHERE {1}=?'.format(table, field)
+        c.executemany(sql, map(lambda v: (buffer(UUID(v).bytes), v), values))
+
+    connection.commit()
+
+with sqlite3.connect(args.dbfile) as conn:
+    conn.cursor().execute('PRAGMA foreign_keys = OFF')
+
+    process_table(conn, 'folder', ('id', 'parent_id'))
+    process_table(conn, 'artist', ('id',))
+    process_table(conn, 'album', ('id', 'artist_id'))
+    process_table(conn, 'track', ('id', 'album_id', 'artist_id', 'root_folder_id', 'folder_id'))
+    process_table(conn, 'user', ('id', 'last_play_id'))
+    process_table(conn, 'client_prefs', ('user_id',))
+    process_table(conn, 'starred_folder', ('user_id', 'starred_id'))
+    process_table(conn, 'starred_artist', ('user_id', 'starred_id'))
+    process_table(conn, 'starred_album', ('user_id', 'starred_id'))
+    process_table(conn, 'starred_track', ('user_id', 'starred_id'))
+    process_table(conn, 'rating_folder', ('user_id', 'rated_id'))
+    process_table(conn, 'rating_track', ('user_id', 'rated_id'))
+    process_table(conn, 'chat_message', ('id', 'user_id'))
+    process_table(conn, 'playlist', ('id', 'user_id'))
+

schema/mysql.sql

@@ -1,35 +1,35 @@
 CREATE TABLE folder (
-	id CHAR(36) PRIMARY KEY,
+	id BINARY(16) PRIMARY KEY,
 	root BOOLEAN NOT NULL,
 	name VARCHAR(256) NOT NULL,
 	path VARCHAR(4096) NOT NULL,
 	created DATETIME NOT NULL,
 	has_cover_art BOOLEAN NOT NULL,
 	last_scan INTEGER NOT NULL,
-	parent_id CHAR(36) REFERENCES folder
+	parent_id BINARY(16) REFERENCES folder
 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
 
 CREATE TABLE artist (
-	id CHAR(36) PRIMARY KEY,
+	id BINARY(16) PRIMARY KEY,
 	name VARCHAR(256) NOT NULL
 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
 
 CREATE TABLE album (
-	id CHAR(36) PRIMARY KEY,
+	id BINARY(16) PRIMARY KEY,
 	name VARCHAR(256) NOT NULL,
-	artist_id CHAR(36) NOT NULL REFERENCES artist
+	artist_id BINARY(16) NOT NULL REFERENCES artist
 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
 
 CREATE TABLE track (
-	id CHAR(36) PRIMARY KEY,
+	id BINARY(16) PRIMARY KEY,
 	disc INTEGER NOT NULL,
 	number INTEGER NOT NULL,
 	title VARCHAR(256) NOT NULL,
 	year INTEGER,
 	genre VARCHAR(256),
 	duration INTEGER NOT NULL,
-	album_id CHAR(36) NOT NULL REFERENCES album,
+	album_id BINARY(16) NOT NULL REFERENCES album,
-	artist_id CHAR(36) NOT NULL REFERENCES artist,
+	artist_id BINARY(16) NOT NULL REFERENCES artist,
 	bitrate INTEGER NOT NULL,
 	path VARCHAR(4096) NOT NULL,
 	content_type VARCHAR(32) NOT NULL,
@@ -37,12 +37,12 @@ CREATE TABLE track (
 	last_modification INTEGER NOT NULL,
 	play_count INTEGER NOT NULL,
 	last_play DATETIME,
-	root_folder_id CHAR(36) NOT NULL REFERENCES folder,
+	root_folder_id BINARY(16) NOT NULL REFERENCES folder,
-	folder_id CHAR(36) NOT NULL REFERENCES folder
+	folder_id BINARY(16) NOT NULL REFERENCES folder
 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
 
 CREATE TABLE user (
-	id CHAR(36) PRIMARY KEY,
+	id BINARY(16) PRIMARY KEY,
 	name VARCHAR(64) NOT NULL,
 	mail VARCHAR(256),
 	password CHAR(40) NOT NULL,
@@ -50,12 +50,12 @@ CREATE TABLE user (
 	admin BOOLEAN NOT NULL,
 	lastfm_session CHAR(32),
 	lastfm_status BOOLEAN NOT NULL,
-	last_play_id CHAR(36) REFERENCES track,
+	last_play_id BINARY(16) REFERENCES track,
 	last_play_date DATETIME
 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
 
 CREATE TABLE client_prefs (
-	user_id CHAR(36) NOT NULL,
+	user_id BINARY(16) NOT NULL,
 	client_name VARCHAR(32) NOT NULL,
 	format VARCHAR(8),
 	bitrate INTEGER,
@@ -63,57 +63,57 @@ CREATE TABLE client_prefs (
 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
 
 CREATE TABLE starred_folder (
-	user_id CHAR(36) NOT NULL REFERENCES user,
+	user_id BINARY(16) NOT NULL REFERENCES user,
-	starred_id CHAR(36) NOT NULL REFERENCES folder,
+	starred_id BINARY(16) NOT NULL REFERENCES folder,
 	date DATETIME NOT NULL,
 	PRIMARY KEY (user_id, starred_id)
 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
 
 CREATE TABLE starred_artist (
-	user_id CHAR(36) NOT NULL REFERENCES user,
+	user_id BINARY(16) NOT NULL REFERENCES user,
-	starred_id CHAR(36) NOT NULL REFERENCES artist,
+	starred_id BINARY(16) NOT NULL REFERENCES artist,
 	date DATETIME NOT NULL,
 	PRIMARY KEY (user_id, starred_id)
 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
 
 CREATE TABLE starred_album (
-	user_id CHAR(36) NOT NULL REFERENCES user,
+	user_id BINARY(16) NOT NULL REFERENCES user,
-	starred_id CHAR(36) NOT NULL REFERENCES album,
+	starred_id BINARY(16) NOT NULL REFERENCES album,
 	date DATETIME NOT NULL,
 	PRIMARY KEY (user_id, starred_id)
 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
 
 CREATE TABLE starred_track (
-	user_id CHAR(36) NOT NULL REFERENCES user,
+	user_id BINARY(16) NOT NULL REFERENCES user,
-	starred_id CHAR(36) NOT NULL REFERENCES track,
+	starred_id BINARY(16) NOT NULL REFERENCES track,
 	date DATETIME NOT NULL,
 	PRIMARY KEY (user_id, starred_id)
 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
 
 CREATE TABLE rating_folder (
-	user_id CHAR(36) NOT NULL REFERENCES user,
+	user_id BINARY(16) NOT NULL REFERENCES user,
-	rated_id CHAR(36) NOT NULL REFERENCES folder,
+	rated_id BINARY(16) NOT NULL REFERENCES folder,
 	rating INTEGER NOT NULL CHECK(rating BETWEEN 1 AND 5),
 	PRIMARY KEY (user_id, rated_id)
 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
 
 CREATE TABLE rating_track (
-	user_id CHAR(36) NOT NULL REFERENCES user,
+	user_id BINARY(16) NOT NULL REFERENCES user,
-	rated_id CHAR(36) NOT NULL REFERENCES track,
+	rated_id BINARY(16) NOT NULL REFERENCES track,
 	rating INTEGER NOT NULL CHECK(rating BETWEEN 1 AND 5),
 	PRIMARY KEY (user_id, rated_id)
 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
 
 CREATE TABLE chat_message (
-	id CHAR(36) PRIMARY KEY,
+	id BINARY(16) PRIMARY KEY,
-	user_id CHAR(36) NOT NULL REFERENCES user,
+	user_id BINARY(16) NOT NULL REFERENCES user,
 	time INTEGER NOT NULL,
 	message VARCHAR(512) NOT NULL
 ) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;
 
 CREATE TABLE playlist (
-	id CHAR(36) PRIMARY KEY,
+	id BINARY(16) PRIMARY KEY,
-	user_id CHAR(36) NOT NULL REFERENCES user,
+	user_id BINARY(16) NOT NULL REFERENCES user,
 	name VARCHAR(256) NOT NULL,
 	comment VARCHAR(256),
 	public BOOLEAN NOT NULL,

supysonic/db.py

@@ -23,7 +23,7 @@ import mimetypes
 import os.path
 
 from datetime import datetime
-from pony.orm import Database, Required, Optional, Set, PrimaryKey
+from pony.orm import Database, Required, Optional, Set, PrimaryKey, LongStr
 from pony.orm import ObjectNotFound
 from pony.orm import min, max, avg, sum
 from urlparse import urlparse
@@ -40,7 +40,7 @@ class Folder(db.Entity):
     id = PrimaryKey(UUID, default = uuid4)
     root = Required(bool, default = False)
     name = Required(str)
-    path = Required(str, unique = True)
+    path = Required(str, 4096) # unique
     created = Required(datetime, precision = 0, default = now)
     has_cover_art = Required(bool, default = False)
     last_scan = Required(int, default = 0)
@@ -88,7 +88,7 @@ class Artist(db.Entity):
     _table_ = 'artist'
 
     id = PrimaryKey(UUID, default = uuid4)
-    name = Required(str, unique = True)
+    name = Required(str) # unique
     albums = Set(lambda: Album)
     tracks = Set(lambda: Track)
 
@@ -161,7 +161,7 @@ class Track(db.Entity):
 
     bitrate = Required(int)
 
-    path = Required(str, unique = True)
+    path = Required(str, 4096) # unique
     content_type = Required(str)
     created = Required(datetime, precision = 0, default = now)
     last_modification = Required(int)
@@ -244,12 +244,12 @@ class User(db.Entity):
     _table_ = 'user'
 
     id = PrimaryKey(UUID, default = uuid4)
-    name = Required(str, unique = True)
+    name = Required(str, 64) # unique
     mail = Optional(str)
-    password = Required(str)
+    password = Required(str, 40)
-    salt = Required(str)
+    salt = Required(str, 6)
     admin = Required(bool, default = False)
-    lastfm_session = Optional(str, nullable = True)
+    lastfm_session = Optional(str, 32, nullable = True)
     lastfm_status = Required(bool, default = True) # True: ok/unlinked, False: invalid session
 
     last_play = Optional(Track, column = 'last_play_id')
@@ -288,9 +288,9 @@ class ClientPrefs(db.Entity):
     _table_ = 'client_prefs'
 
     user = Required(User, column = 'user_id')
-    client_name = Required(str)
+    client_name = Required(str, 32)
     PrimaryKey(user, client_name)
-    format = Optional(str)
+    format = Optional(str, 8)
     bitrate = Optional(int)
 
 class StarredFolder(db.Entity):
@@ -333,7 +333,7 @@ class RatingFolder(db.Entity):
     _table_ = 'rating_folder'
     user = Required(User, column = 'user_id')
     rated = Required(Folder, column = 'rated_id')
-    rating = Required(int)
+    rating = Required(int, min = 1, max = 5)
 
     PrimaryKey(user, rated)
 
@@ -341,7 +341,7 @@ class RatingTrack(db.Entity):
     _table_ = 'rating_track'
     user = Required(User, column = 'user_id')
     rated = Required(Track, column = 'rated_id')
-    rating = Required(int)
+    rating = Required(int, min = 1, max = 5)
 
     PrimaryKey(user, rated)
 
@@ -351,7 +351,7 @@ class ChatMessage(db.Entity):
     id = PrimaryKey(UUID, default = uuid4)
     user = Required(User, column = 'user_id')
     time = Required(int, default = lambda: int(time.time()))
-    message = Required(str)
+    message = Required(str, 512)
 
     def responsize(self):
         return {
@@ -369,7 +369,7 @@ class Playlist(db.Entity):
     comment = Optional(str)
     public = Required(bool, default = False)
     created = Required(datetime, precision = 0, default = now)
-    tracks = Optional(str)
+    tracks = Optional(LongStr)
 
     def as_subsonic_playlist(self, user):
         tracks = self.get_tracks()