From 3005e529a2ac8552791b5d508af1d7b9dda0f09e Mon Sep 17 00:00:00 2001 From: spl0k Date: Wed, 21 Feb 2018 21:55:00 +0100 Subject: [PATCH 1/2] Fixed MySQL schema using broken UTF-8 --- schema/migration/20180221.mysql.sql | 60 +++++++++++++++++++++++++++++ schema/mysql.sql | 28 +++++++------- 2 files changed, 74 insertions(+), 14 deletions(-) create mode 100644 schema/migration/20180221.mysql.sql diff --git a/schema/migration/20180221.mysql.sql b/schema/migration/20180221.mysql.sql new file mode 100644 index 0000000..dbdd772 --- /dev/null +++ b/schema/migration/20180221.mysql.sql @@ -0,0 +1,60 @@ +ALTER TABLE folder DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE artist DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE album DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE track DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE user DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE client_prefs DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE starred_folder DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE starred_artist DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE starred_album DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE starred_track DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE rating_folder DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE rating_track DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE chat_message DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE playlist DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + +ALTER TABLE folder CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE artist CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE album CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE track CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE user CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE client_prefs CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE starred_folder CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE starred_artist CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE starred_album CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE starred_track CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE rating_folder CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE rating_track CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE chat_message CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; +ALTER TABLE playlist CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + +REPAIR TABLE folder; +REPAIR TABLE artist; +REPAIR TABLE album; +REPAIR TABLE track; +REPAIR TABLE user; +REPAIR TABLE client_prefs; +REPAIR TABLE starred_folder; +REPAIR TABLE starred_artist; +REPAIR TABLE starred_album; +REPAIR TABLE starred_track; +REPAIR TABLE rating_folder; +REPAIR TABLE rating_track; +REPAIR TABLE chat_message; +REPAIR TABLE playlist; + +OPTIMIZE TABLE folder; +OPTIMIZE TABLE artist; +OPTIMIZE TABLE album; +OPTIMIZE TABLE track; +OPTIMIZE TABLE user; +OPTIMIZE TABLE client_prefs; +OPTIMIZE TABLE starred_folder; +OPTIMIZE TABLE starred_artist; +OPTIMIZE TABLE starred_album; +OPTIMIZE TABLE starred_track; +OPTIMIZE TABLE rating_folder; +OPTIMIZE TABLE rating_track; +OPTIMIZE TABLE chat_message; +OPTIMIZE TABLE playlist; + diff --git a/schema/mysql.sql b/schema/mysql.sql index 12e4bb5..b19f455 100644 --- a/schema/mysql.sql +++ b/schema/mysql.sql @@ -7,18 +7,18 @@ CREATE TABLE folder ( has_cover_art BOOLEAN NOT NULL, last_scan INTEGER NOT NULL, parent_id BINARY(16) REFERENCES folder -) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE artist ( id BINARY(16) PRIMARY KEY, name VARCHAR(256) NOT NULL -) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE album ( id BINARY(16) PRIMARY KEY, name VARCHAR(256) NOT NULL, artist_id BINARY(16) NOT NULL REFERENCES artist -) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE track ( id BINARY(16) PRIMARY KEY, @@ -39,7 +39,7 @@ CREATE TABLE track ( last_play DATETIME, root_folder_id BINARY(16) NOT NULL REFERENCES folder, folder_id BINARY(16) NOT NULL REFERENCES folder -) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE user ( id BINARY(16) PRIMARY KEY, @@ -52,7 +52,7 @@ CREATE TABLE user ( lastfm_status BOOLEAN NOT NULL, last_play_id BINARY(16) REFERENCES track, last_play_date DATETIME -) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE client_prefs ( user_id BINARY(16) NOT NULL, @@ -60,56 +60,56 @@ CREATE TABLE client_prefs ( format VARCHAR(8), bitrate INTEGER, PRIMARY KEY (user_id, client_name) -) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE starred_folder ( user_id BINARY(16) NOT NULL REFERENCES user, starred_id BINARY(16) NOT NULL REFERENCES folder, date DATETIME NOT NULL, PRIMARY KEY (user_id, starred_id) -) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE starred_artist ( user_id BINARY(16) NOT NULL REFERENCES user, starred_id BINARY(16) NOT NULL REFERENCES artist, date DATETIME NOT NULL, PRIMARY KEY (user_id, starred_id) -) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE starred_album ( user_id BINARY(16) NOT NULL REFERENCES user, starred_id BINARY(16) NOT NULL REFERENCES album, date DATETIME NOT NULL, PRIMARY KEY (user_id, starred_id) -) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE starred_track ( user_id BINARY(16) NOT NULL REFERENCES user, starred_id BINARY(16) NOT NULL REFERENCES track, date DATETIME NOT NULL, PRIMARY KEY (user_id, starred_id) -) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE rating_folder ( user_id BINARY(16) NOT NULL REFERENCES user, rated_id BINARY(16) NOT NULL REFERENCES folder, rating INTEGER NOT NULL CHECK(rating BETWEEN 1 AND 5), PRIMARY KEY (user_id, rated_id) -) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE rating_track ( user_id BINARY(16) NOT NULL REFERENCES user, rated_id BINARY(16) NOT NULL REFERENCES track, rating INTEGER NOT NULL CHECK(rating BETWEEN 1 AND 5), PRIMARY KEY (user_id, rated_id) -) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE chat_message ( id BINARY(16) PRIMARY KEY, user_id BINARY(16) NOT NULL REFERENCES user, time INTEGER NOT NULL, message VARCHAR(512) NOT NULL -) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; CREATE TABLE playlist ( id BINARY(16) PRIMARY KEY, @@ -119,5 +119,5 @@ CREATE TABLE playlist ( public BOOLEAN NOT NULL, created DATETIME NOT NULL, tracks TEXT -) DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; From ba61281ba136ecf76bb8927afb79b9c8d640ff7a Mon Sep 17 00:00:00 2001 From: spl0k Date: Wed, 21 Feb 2018 22:13:35 +0100 Subject: [PATCH 2/2] Allow extra db connection args. Set utf8mb4 as the default charset for MySQL --- docs/configuration.md | 11 +++++++++-- supysonic/db.py | 13 ++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 913dbb8..9e33ca6 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,7 +27,12 @@ URI is: driver://username:password@host:port/database -Supported drivers are `sqlite`, `mysql` and `postgres` (or `postgresql`) +If the connection needs some additional parameters, they can be provided as a +query string, such as: + + driver://username:password@host:port/database?param1=value1¶m2=value2 + +Supported drivers are `sqlite`, `mysql` and `postgres` (or `postgresql`). As SQLite connects to local files, the format is slightly different. The "file" portion of the URI is the filename of the database. For a relative path, it @@ -43,8 +48,10 @@ database_uri = sqlite:////home/user/supysonic.db database_uri = sqlite:///C:\Users\user\supysonic.db ``` -A MySQL-compatible database require either `MySQLdb` or `pymysql` to be +A MySQL-compatible database requires either `MySQLdb` or `pymysql` to be installed. PostgreSQL needs `psycopg2`. +Note that for MySQL if no character set is defined on the URI it defaults to +`utf8mb4` regardless of what's set on your MySQL installation. If `database_uri` isn't provided, it defaults to a SQLite database stored in `/tmp/supysonic/supysonic.db`. diff --git a/supysonic/db.py b/supysonic/db.py index 2ceebf2..154a056 100644 --- a/supysonic/db.py +++ b/supysonic/db.py @@ -31,9 +31,9 @@ from uuid import UUID, uuid4 from .py23 import dict, strtype try: - from urllib.parse import urlparse + from urllib.parse import urlparse, parse_qsl except ImportError: - from urlparse import urlparse + from urlparse import urlparse, parse_qsl def now(): return datetime.now().replace(microsecond = 0) @@ -443,6 +443,8 @@ def parse_uri(database_uri): raise TypeError('Expecting a string') uri = urlparse(database_uri) + args = dict(parse_qsl(uri.query)) + if uri.scheme == 'sqlite': path = uri.path if not path: @@ -450,11 +452,12 @@ def parse_uri(database_uri): elif path[0] == '/': path = path[1:] - return dict(provider = 'sqlite', filename = path) + return dict(provider = 'sqlite', filename = path, **args) elif uri.scheme in ('postgres', 'postgresql'): - return dict(provider = 'postgres', user = uri.username, password = uri.password, host = uri.hostname, database = uri.path[1:]) + return dict(provider = 'postgres', user = uri.username, password = uri.password, host = uri.hostname, dbname = uri.path[1:], **args) elif uri.scheme == 'mysql': - return dict(provider = 'mysql', user = uri.username, passwd = uri.password, host = uri.hostname, db = uri.path[1:]) + args.setdefault('charset', 'utf8mb4') + return dict(provider = 'mysql', user = uri.username, passwd = uri.password, host = uri.hostname, db = uri.path[1:], **args) return dict() def init_database(database_uri, create_tables = False):