add sqlite2mysql.sh and sqlite2mysql.py
This commit is contained in:
lian
parent
524f227b02
commit
24d8e4131f
2 changed files with 197 additions and 0 deletions
79
scripts/sqlite2mysql.py
Normal file
79
scripts/sqlite2mysql.py
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
"""Lifted from:
|
||||
http://stackoverflow.com/questions/18671/quick-easy-way-to-migrate-sqlite3-to-mysql
|
||||
|
||||
Run like so:
|
||||
|
||||
sqlite3 <your db>.db .dump | python sqlite2mysql.py > <your db>.sql
|
||||
|
||||
Then you can import the .sql file into MySql
|
||||
|
||||
Note - you need to add foreign key constrains manually since sqlite doesn't actually support them
|
||||
"""
|
||||
import re
|
||||
import fileinput
|
||||
|
||||
def this_line_is_useless(line):
|
||||
useless_es = [
|
||||
'BEGIN TRANSACTION',
|
||||
'COMMIT',
|
||||
'sqlite_sequence',
|
||||
'CREATE UNIQUE INDEX',
|
||||
'PRAGMA',
|
||||
]
|
||||
for useless in useless_es:
|
||||
if re.search(useless, line):
|
||||
return True
|
||||
|
||||
def has_primary_key(line):
|
||||
return bool(re.search(r'PRIMARY KEY', line))
|
||||
|
||||
for line in fileinput.input():
|
||||
searching_for_end = False
|
||||
if this_line_is_useless(line): continue
|
||||
|
||||
# this line was necessary because ''); was getting
|
||||
# converted (inappropriately) to \');
|
||||
if re.match(r".*, ''\);", line):
|
||||
line = re.sub(r"''\);", r'``);', line)
|
||||
|
||||
if re.match(r'^CREATE TABLE.*', line):
|
||||
searching_for_end = True
|
||||
|
||||
m = re.search('CREATE TABLE [`"]?(\w*)[`"]?(.*)', line)
|
||||
if m:
|
||||
name, sub = m.groups()
|
||||
sub = sub.replace('"','`')
|
||||
line = line % dict(name=name, sub=sub)
|
||||
else:
|
||||
m = re.search('INSERT INTO "(\w*)"(.*)', line)
|
||||
if m:
|
||||
name, sub = m.groups()
|
||||
line = 'INSERT INTO `%s`%s\n' % m.groups()
|
||||
line = line.replace('"', r'\"')
|
||||
line = line.replace('"', "'")
|
||||
# line = re.sub(r"([^'])'t'(.)", r"\1THIS_IS_TRUE\2", line)
|
||||
# line = line.replace('THIS_IS_TRUE', '1')
|
||||
# line = re.sub(r"([^'])'f'(.)", r"\1THIS_IS_FALSE\2", line)
|
||||
# line = line.replace('THIS_IS_FALSE', '0')
|
||||
|
||||
# Add auto_increment if it's not there since sqlite auto_increments ALL
|
||||
# primary keys
|
||||
if searching_for_end:
|
||||
# replace " and ' with ` because mysql doesn't like quotes in CREATE commands
|
||||
line = line.replace('"', '`').replace("'", '`')
|
||||
|
||||
# And now we convert it back (see above)
|
||||
if re.match(r".*, ``\);", line):
|
||||
line = re.sub(r'``\);', r"'');", line)
|
||||
|
||||
if searching_for_end and re.match(r'.*\);', line):
|
||||
searching_for_end = False
|
||||
|
||||
if re.match(r"CREATE INDEX", line):
|
||||
line = re.sub('"', '`', line)
|
||||
|
||||
line = line.replace('"', '`')
|
||||
line = line.replace('AUTOINCREMENT', 'AUTO_INCREMENT')
|
||||
print(line)
|
||||
118
scripts/sqlite2mysql.sh
Executable file
118
scripts/sqlite2mysql.sh
Executable file
|
|
@ -0,0 +1,118 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# This shell script and corresponding sqlite2mysql.py are used to
|
||||
# migrate Seafile data from SQLite to MySQL.
|
||||
#
|
||||
# Setup:
|
||||
#
|
||||
# 1. Move this file and sqlite2mysql.py to the top directory of your Seafile
|
||||
# installation path (e.g. /data/haiwen).
|
||||
# 2. Run: ./sqlite2mysql.sh
|
||||
# 3. Three files(ccnet-db.sql, seafile-db.sql, seahub-db.sql) are created.
|
||||
# 4. Loads these files to MySQL
|
||||
# (mysql> source ccnet-db.sql)
|
||||
#
|
||||
|
||||
CCNET_DB='ccnet-db.sql'
|
||||
SEAFILE_DB='seafile-db.sql'
|
||||
SEAHUB_DB='seahub-db.sql'
|
||||
|
||||
########## ccnet
|
||||
seafile_path=$(pwd)
|
||||
if [ -f "${seafile_path}/conf/ccnet.conf" ]; then
|
||||
USER_MGR_DB=${seafile_path}/ccnet/PeerMgr/usermgr.db
|
||||
GRP_MGR_DB=${seafile_path}/ccnet/GroupMgr/groupmgr.db
|
||||
else
|
||||
echo "${seafile_path}/conf/ccnet.conf does not exists."
|
||||
read -p "Please provide your ccnet.conf path(e.g. /data/haiwen/conf/ccnet.conf): " ccnet_conf_path
|
||||
if [ -f ${ccnet_conf_path} ]; then
|
||||
USER_MGR_DB=$(dirname $(dirname "${ccnet_conf_path}"))/ccnet/PeerMgr/usermgr.db
|
||||
GRP_MGR_DB=$(dirname $(dirname "${ccnet_conf_path}"))/ccnet/GroupMgr/groupmgr.db
|
||||
else
|
||||
echo "${ccnet_conf_path} does not exists, quit."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
rm -rf ${CCNET_DB}
|
||||
|
||||
echo "sqlite3 ${USER_MGR_DB} .dump | python sqlite2mysql.py > ${CCNET_DB}"
|
||||
sqlite3 ${USER_MGR_DB} .dump | python sqlite2mysql.py > ${CCNET_DB}
|
||||
echo "sqlite3 ${GRP_MGR_DB} .dump | python sqlite2mysql.py >> ${CCNET_DB}"
|
||||
sqlite3 ${GRP_MGR_DB} .dump | python sqlite2mysql.py >> ${CCNET_DB}
|
||||
|
||||
# change ctime from INTEGER to BIGINT in EmailUser table
|
||||
sed 's/ctime INTEGER/ctime BIGINT/g' ${CCNET_DB} > ${CCNET_DB}.tmp && mv ${CCNET_DB}.tmp ${CCNET_DB}
|
||||
|
||||
# change email in UserRole from TEXT to VARCHAR(255)
|
||||
sed 's/email TEXT, role TEXT/email VARCHAR(255), role TEXT/g' ${CCNET_DB} > ${CCNET_DB}.tmp && mv ${CCNET_DB}.tmp ${CCNET_DB}
|
||||
|
||||
########## seafile
|
||||
rm -rf ${SEAFILE_DB}
|
||||
|
||||
if [ -f "${seafile_path}/seafile-data/seafile.db" ]; then
|
||||
echo "sqlite3 ${seafile_path}/seafile-data/seafile.db .dump | python sqlite2mysql.py > ${SEAFILE_DB}"
|
||||
sqlite3 ${seafile_path}/seafile-data/seafile.db .dump | python sqlite2mysql.py > ${SEAFILE_DB}
|
||||
else
|
||||
echo "${seafile_path}/seafile-data/seafile.db does not exists."
|
||||
read -p "Please provide your seafile.db path(e.g. /data/haiwen/seafile-data/seafile.db): " seafile_db_path
|
||||
if [ -f ${seafile_db_path} ];then
|
||||
echo "sqlite3 ${seafile_db_path} .dump | python sqlite2mysql.py > ${SEAFILE_DB}"
|
||||
sqlite3 ${seafile_db_path} .dump | python sqlite2mysql.py > ${SEAFILE_DB}
|
||||
else
|
||||
echo "${seafile_db_path} does not exists, quit."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# change owner_id in RepoOwner from TEXT to VARCHAR(255)
|
||||
sed 's/owner_id TEXT/owner_id VARCHAR(255)/g' ${SEAFILE_DB} > ${SEAFILE_DB}.tmp && mv ${SEAFILE_DB}.tmp ${SEAFILE_DB}
|
||||
|
||||
# change user_name in RepoGroup from TEXT to VARCHAR(255)
|
||||
sed 's/user_name TEXT/user_name VARCHAR(255)/g' ${SEAFILE_DB} > ${SEAFILE_DB}.tmp && mv ${SEAFILE_DB}.tmp ${SEAFILE_DB}
|
||||
|
||||
########## seahub
|
||||
rm -rf ${SEAHUB_DB}
|
||||
|
||||
if [ -f "${seafile_path}/seahub.db" ]; then
|
||||
echo "sqlite3 ${seafile_path}/seahub.db .dump | tr -d '\n' | sed 's/;/;\n/g' | python sqlite2mysql.py > ${SEAHUB_DB}"
|
||||
sqlite3 ${seafile_path}/seahub.db .dump | tr -d '\n' | sed 's/;/;\n/g' | python sqlite2mysql.py > ${SEAHUB_DB}
|
||||
else
|
||||
echo "${seafile_path}/seahub.db does not exists."
|
||||
read -p "Please prove your seahub.db path(e.g. /data/haiwen/seahub.db): " seahub_db_path
|
||||
if [ -f ${seahub_db_path} ]; then
|
||||
echo "sqlite3 ${seahub_db_path} .dump | tr -d '\n' | sed 's/;/;\n/g' | python sqlite2mysql.py > ${SEAHUB_DB}"
|
||||
sqlite3 ${seahub_db_path} .dump | tr -d '\n' | sed 's/;/;\n/g' | python sqlite2mysql.py > ${SEAHUB_DB}
|
||||
else
|
||||
echo "${seahub_db_path} does not exists, quit."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# change username from VARCHAR(256) to VARCHAR(255) in wiki_personalwiki
|
||||
sed 's/varchar(256) NOT NULL UNIQUE/varchar(255) NOT NULL UNIQUE/g' ${SEAHUB_DB} > ${SEAHUB_DB}.tmp && mv ${SEAHUB_DB}.tmp ${SEAHUB_DB}
|
||||
|
||||
# remove unique from contacts_contact
|
||||
sed 's/, UNIQUE (`user_email`, `contact_email`)//g' ${SEAHUB_DB} > ${SEAHUB_DB}.tmp && mv ${SEAHUB_DB}.tmp ${SEAHUB_DB}
|
||||
|
||||
# remove base_dirfileslastmodifiedinfo records to avoid json string parsing issue between sqlite and mysql
|
||||
sed '/INSERT INTO `base_dirfileslastmodifiedinfo`/d' ${SEAHUB_DB} > ${SEAHUB_DB}.tmp && mv ${SEAHUB_DB}.tmp ${SEAHUB_DB}
|
||||
|
||||
# remove notifications_usernotification records to avoid json string parsing issue between sqlite and mysql
|
||||
sed '/INSERT INTO `notifications_usernotification`/d' ${SEAHUB_DB} > ${SEAHUB_DB}.tmp && mv ${SEAHUB_DB}.tmp ${SEAHUB_DB}
|
||||
|
||||
|
||||
########## common logic
|
||||
|
||||
# add ENGIN=INNODB to create table statment
|
||||
for sql_file in $CCNET_DB $SEAFILE_DB $SEAHUB_DB
|
||||
do
|
||||
sed -r 's/(CREATE TABLE.*);/\1 ENGINE=INNODB;/g' $sql_file > $sql_file.tmp && mv $sql_file.tmp $sql_file
|
||||
done
|
||||
|
||||
# remove COLLATE NOCASE if possible
|
||||
for sql_file in $CCNET_DB $SEAFILE_DB $SEAHUB_DB
|
||||
do
|
||||
sed 's/COLLATE NOCASE//g' $sql_file > $sql_file.tmp && mv $sql_file.tmp $sql_file
|
||||
done
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue