Skip to content

S
slapos.core
Commit c0912336 authored by Jérome Perrin's avatar Jérome Perrin Committed by Thomas Gambier
Browse files
Options

Remove old slapproxy tables when migrating to new version 

Some tools are using directly slapproxy sqlite database and they stopped working when table number was increased in nexedi/slapos.core!76 .

We made a quick improvement to the tools so that they can autodetect the current table names instead of hardcoding ( nexedi/slapos@b8c895cd ) but this failed, because slapproxy remove did not remove the old tables after migration. This branch is about removing the table, to quickly repair these tools.

The long term plan is (hopefully) to improve these tools so that they don't use the private tables like this.

/reviewed-on nexedi/slapos.core!105
parents 3343cbea d0ce721b
Expand all Show whitespace changes
Inline Side-by-side
Showing with 463 additions and 99 deletions
slapos/proxy/schema.sql
View file @ c0912336
--version:12 --version:13
CREATE TABLE IF NOT EXISTS software%(version)s ( CREATE TABLE IF NOT EXISTS software%(version)s (
url VARCHAR(255), url VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT '%(computer)s', computer_reference VARCHAR(255) DEFAULT '%(computer)s',
......
slapos/proxy/views.py
View file @ c0912336
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
from lxml import etree from lxml import etree
import random import random
import string import string
from datetime import datetime
from slapos.slap.slap import Computer, ComputerPartition, \ from slapos.slap.slap import Computer, ComputerPartition, \
SoftwareRelease, SoftwareInstance, NotFoundError SoftwareRelease, SoftwareInstance, NotFoundError
from slapos.proxy.db_version import DB_VERSION from slapos.proxy.db_version import DB_VERSION
...@@ -176,6 +177,22 @@ def _upgradeDatabaseIfNeeded(): ...@@ -176,6 +177,22 @@ def _upgradeDatabaseIfNeeded():
if current_schema_version == DB_VERSION: if current_schema_version == DB_VERSION:
return return
previous_table_list = _getTableList()
# first, make a backup of current database
if current_schema_version != '-1':
backup_file_name = "{}-backup-{}to{}-{}.sql".format(
app.config['DATABASE_URI'],
current_schema_version,
DB_VERSION,
datetime.now().isoformat())
app.logger.info(
'Old schema detected: Creating a backup of current tables at %s',
backup_file_name
)
with open(backup_file_name, 'w') as f:
for line in g.db.iterdump():
f.write('%s\n' % line)
with app.open_resource('schema.sql', 'r') as f: with app.open_resource('schema.sql', 'r') as f:
schema = f.read() % dict(version=DB_VERSION, computer=app.config['computer_id']) schema = f.read() % dict(version=DB_VERSION, computer=app.config['computer_id'])
g.db.cursor().executescript(schema) g.db.cursor().executescript(schema)
...@@ -186,14 +203,15 @@ def _upgradeDatabaseIfNeeded(): ...@@ -186,14 +203,15 @@ def _upgradeDatabaseIfNeeded():
# Migrate all data to new tables # Migrate all data to new tables
app.logger.info('Old schema detected: Migrating old tables...') app.logger.info('Old schema detected: Migrating old tables...')
app.logger.info('Note that old tables are not alterated.')
for table in ('software', 'computer', 'partition', 'slave', 'partition_network'): for table in ('software', 'computer', 'partition', 'slave', 'partition_network'):
for row in execute_db(table, 'SELECT * from %s', db_version=current_schema_version): for row in execute_db(table, 'SELECT * from %s', db_version=current_schema_version):
columns = ', '.join(row.keys()) columns = ', '.join(row.keys())
placeholders = ':'+', :'.join(row.keys()) placeholders = ':'+', :'.join(row.keys())
query = 'INSERT INTO %s (%s) VALUES (%s)' % ('%s', columns, placeholders) query = 'INSERT INTO %s (%s) VALUES (%s)' % ('%s', columns, placeholders)
execute_db(table, query, row) execute_db(table, query, row)
# then drop old tables
for previous_table in previous_table_list:
g.db.execute("DROP table %s" % previous_table)
g.db.commit() g.db.commit()
is_schema_already_executed = False is_schema_already_executed = False
......
slapos/tests/test_cli.py
View file @ c0912336
...@@ -98,7 +98,8 @@ class TestCliProxyShow(CliMixin): ...@@ -98,7 +98,8 @@ class TestCliProxyShow(CliMixin):
# load database # load database
schema = bytes2str(pkg_resources.resource_string( schema = bytes2str(pkg_resources.resource_string(
'slapos.tests.test_slapproxy', 'database_dump_version_current.sql')) 'slapos.tests',
os.path.join('test_slapproxy', 'database_dump_version_current.sql')))
db = sqlite_connect(self.db_file.name) db = sqlite_connect(self.db_file.name)
db.cursor().executescript(schema) db.cursor().executescript(schema)
db.commit() db.commit()
...@@ -124,7 +125,7 @@ class TestCliProxyShow(CliMixin): ...@@ -124,7 +125,7 @@ class TestCliProxyShow(CliMixin):
# installed softwares are listed # installed softwares are listed
logger.info.assert_any_call( logger.info.assert_any_call(
' /srv/slapgrid/slappart8/srv/runner/project/slapos/software/erp5/software.cfg available slaprunner 287375f0cba269902ba1bc50242839d7 ' ) ' /srv/slapgrid/slappart8/srv/runner/project/slapos/software/erp5/software.cfg slaprunner available 287375f0cba269902ba1bc50242839d7 ')
# instance parameters are listed # instance parameters are listed
# _ parameter is json formatted # _ parameter is json formatted
......
slapos/tests/test_slapproxy/database_dump_version_11.sql 0 → 100644
View file @ c0912336
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE software11 (
url VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
CONSTRAINT uniq PRIMARY KEY (url, computer_reference)
);
INSERT INTO "software11" VALUES('/srv/slapgrid//srv//runner/project//slapos/software.cfg','computer');
CREATE TABLE computer11 (
reference VARCHAR(255) DEFAULT 'computer',
address VARCHAR(255),
netmask VARCHAR(255),
CONSTRAINT uniq PRIMARY KEY (reference)
);
INSERT INTO "computer11" VALUES('computer','127.0.0.1','255.255.255.255');
CREATE TABLE partition11 (
reference VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
slap_state VARCHAR(255) DEFAULT 'free',
software_release VARCHAR(255),
xml TEXT,
connection_xml TEXT,
slave_instance_list TEXT,
software_type VARCHAR(255),
partition_reference VARCHAR(255), -- name of the instance
requested_by VARCHAR(255), -- only used for debugging,
-- slapproxy does not support proper scope
requested_state VARCHAR(255) NOT NULL DEFAULT 'started',
CONSTRAINT uniq PRIMARY KEY (reference, computer_reference)
);
INSERT INTO "partition11" VALUES('slappart0','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="json">{
"site-id": "erp5"
}
}</parameter>
</instance>
',NULL,NULL,'production','slapos',NULL,'started');
INSERT INTO "partition11" VALUES('slappart1','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">mysql://127.0.0.1:45678/erp5</parameter>
</instance>
',NULL,'mariadb','MariaDB DataBase','slappart0','started');
INSERT INTO "partition11" VALUES('slappart2','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="cloudooo-json"></parameter>
</instance>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">cloudooo://127.0.0.1:23000/</parameter>
</instance>
',NULL,'cloudooo','Cloudooo','slappart0','started');
INSERT INTO "partition11" VALUES('slappart3','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">memcached://127.0.0.1:11000/</parameter>
</instance>
',NULL,'memcached','Memcached','slappart0','started');
INSERT INTO "partition11" VALUES('slappart4','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">memcached://127.0.0.1:13301/</parameter>
</instance>
',NULL,'kumofs','KumoFS','slappart0','started');
INSERT INTO "partition11" VALUES('slappart5','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="kumofs-url">memcached://127.0.0.1:13301/</parameter>
<parameter id="memcached-url">memcached://127.0.0.1:11000/</parameter>
<parameter id="cloudooo-url">cloudooo://127.0.0.1:23000/</parameter>
</instance>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">https://[fc00::1]:10001</parameter>
</instance>
',NULL,'tidstorage','TidStorage','slappart0','started');
INSERT INTO "partition11" VALUES('slappart6','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
INSERT INTO "partition11" VALUES('slappart7','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
INSERT INTO "partition11" VALUES('slappart8','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
INSERT INTO "partition11" VALUES('slappart9','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
CREATE TABLE slave11 (
reference VARCHAR(255), -- unique slave reference
computer_reference VARCHAR(255) DEFAULT 'computer',
connection_xml TEXT,
hosted_by VARCHAR(255),
asked_by VARCHAR(255) -- only used for debugging,
-- slapproxy does not support proper scope
);
CREATE TABLE partition_network11 (
partition_reference VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
reference VARCHAR(255),
address VARCHAR(255),
netmask VARCHAR(255)
);
INSERT INTO "partition_network11" VALUES('slappart0','computer','slappart0','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network11" VALUES('slappart0','computer','slappart0','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network11" VALUES('slappart1','computer','slappart1','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network11" VALUES('slappart1','computer','slappart1','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network11" VALUES('slappart2','computer','slappart2','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network11" VALUES('slappart2','computer','slappart2','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network11" VALUES('slappart3','computer','slappart3','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network11" VALUES('slappart3','computer','slappart3','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network11" VALUES('slappart4','computer','slappart4','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network11" VALUES('slappart4','computer','slappart4','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network11" VALUES('slappart5','computer','slappart5','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network11" VALUES('slappart5','computer','slappart5','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network11" VALUES('slappart6','computer','slappart6','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network11" VALUES('slappart6','computer','slappart6','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network11" VALUES('slappart7','computer','slappart7','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network11" VALUES('slappart7','computer','slappart7','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network11" VALUES('slappart8','computer','slappart8','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network11" VALUES('slappart8','computer','slappart8','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network11" VALUES('slappart9','computer','slappart9','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network11" VALUES('slappart9','computer','slappart9','fc00::1','ffff:ffff:ffff::');
CREATE TABLE forwarded_partition_request11 (
partition_reference VARCHAR(255), -- a.k.a source_instance_id
master_url VARCHAR(255)
);
COMMIT;
slapos/tests/test_slapproxy/database_dump_version_12.sql 0 → 100644
View file @ c0912336
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE software12 (
url VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
requested_state VARCHAR(255) DEFAULT 'available',
CONSTRAINT uniq PRIMARY KEY (url, computer_reference)
);
INSERT INTO "software12" VALUES('/srv/slapgrid//srv//runner/project//slapos/software.cfg','computer','available');
CREATE TABLE computer12 (
reference VARCHAR(255) DEFAULT 'computer',
address VARCHAR(255),
netmask VARCHAR(255),
CONSTRAINT uniq PRIMARY KEY (reference)
);
INSERT INTO "computer12" VALUES('computer','127.0.0.1','255.255.255.255');
CREATE TABLE partition12 (
reference VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
slap_state VARCHAR(255) DEFAULT 'free',
software_release VARCHAR(255),
xml TEXT,
connection_xml TEXT,
slave_instance_list TEXT,
software_type VARCHAR(255),
partition_reference VARCHAR(255), -- name of the instance
requested_by VARCHAR(255), -- only used for debugging,
-- slapproxy does not support proper scope
requested_state VARCHAR(255) NOT NULL DEFAULT 'started',
CONSTRAINT uniq PRIMARY KEY (reference, computer_reference)
);
INSERT INTO "partition12" VALUES('slappart0','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="json">{
"site-id": "erp5"
}
}</parameter>
</instance>
',NULL,NULL,'production','slapos',NULL,'started');
INSERT INTO "partition12" VALUES('slappart1','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">mysql://127.0.0.1:45678/erp5</parameter>
</instance>
',NULL,'mariadb','MariaDB DataBase','slappart0','started');
INSERT INTO "partition12" VALUES('slappart2','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="cloudooo-json"></parameter>
</instance>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">cloudooo://127.0.0.1:23000/</parameter>
</instance>
',NULL,'cloudooo','Cloudooo','slappart0','started');
INSERT INTO "partition12" VALUES('slappart3','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">memcached://127.0.0.1:11000/</parameter>
</instance>
',NULL,'memcached','Memcached','slappart0','started');
INSERT INTO "partition12" VALUES('slappart4','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance/>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">memcached://127.0.0.1:13301/</parameter>
</instance>
',NULL,'kumofs','KumoFS','slappart0','started');
INSERT INTO "partition12" VALUES('slappart5','computer','busy','/srv/slapgrid//srv//runner/project//slapos/software.cfg','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="kumofs-url">memcached://127.0.0.1:13301/</parameter>
<parameter id="memcached-url">memcached://127.0.0.1:11000/</parameter>
<parameter id="cloudooo-url">cloudooo://127.0.0.1:23000/</parameter>
</instance>
','<?xml version=''1.0'' encoding=''utf-8''?>
<instance>
<parameter id="url">https://[fc00::1]:10001</parameter>
</instance>
',NULL,'tidstorage','TidStorage','slappart0','started');
INSERT INTO "partition12" VALUES('slappart6','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
INSERT INTO "partition12" VALUES('slappart7','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
INSERT INTO "partition12" VALUES('slappart8','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
INSERT INTO "partition12" VALUES('slappart9','computer','free',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'started');
CREATE TABLE slave12 (
reference VARCHAR(255), -- unique slave reference
computer_reference VARCHAR(255) DEFAULT 'computer',
connection_xml TEXT,
hosted_by VARCHAR(255),
asked_by VARCHAR(255) -- only used for debugging,
-- slapproxy does not support proper scope
);
CREATE TABLE partition_network12 (
partition_reference VARCHAR(255),
computer_reference VARCHAR(255) DEFAULT 'computer',
reference VARCHAR(255),
address VARCHAR(255),
netmask VARCHAR(255)
);
INSERT INTO "partition_network12" VALUES('slappart0','computer','slappart0','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network12" VALUES('slappart0','computer','slappart0','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network12" VALUES('slappart1','computer','slappart1','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network12" VALUES('slappart1','computer','slappart1','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network12" VALUES('slappart2','computer','slappart2','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network12" VALUES('slappart2','computer','slappart2','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network12" VALUES('slappart3','computer','slappart3','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network12" VALUES('slappart3','computer','slappart3','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network12" VALUES('slappart4','computer','slappart4','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network12" VALUES('slappart4','computer','slappart4','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network12" VALUES('slappart5','computer','slappart5','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network12" VALUES('slappart5','computer','slappart5','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network12" VALUES('slappart6','computer','slappart6','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network12" VALUES('slappart6','computer','slappart6','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network12" VALUES('slappart7','computer','slappart7','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network12" VALUES('slappart7','computer','slappart7','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network12" VALUES('slappart8','computer','slappart8','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network12" VALUES('slappart8','computer','slappart8','fc00::1','ffff:ffff:ffff::');
INSERT INTO "partition_network12" VALUES('slappart9','computer','slappart9','127.0.0.1','255.255.255.255');
INSERT INTO "partition_network12" VALUES('slappart9','computer','slappart9','fc00::1','ffff:ffff:ffff::');
CREATE TABLE forwarded_partition_request12 (
partition_reference VARCHAR(255), -- a.k.a source_instance_id
master_url VARCHAR(255)
);
COMMIT;
slapos/tests/test_slapproxy/generate_dump.sh 0 → 100755
View file @ c0912336
#!/bin/bash
# Update an old dump to current format. Useful to update tests date
# when increasing database version number.
# Note that because it's what test expects, $COMPUTER_ID must be set to
# `computer` for slapos/tests/test_slapproxy/database_dump_version_??.sql
# and `slaprunner` for slapos/tests/test_slapproxy/database_dump_version_current.sql
if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3"]; then
echo "Usage: $0 computer_id dump_before.sql dump_after.sql"
exit 1;
fi
COMPUTER_ID=$1
DUMP_BEFORE=$2
DUMP_AFTER=$3
PORT=6123
echo "Using slapos $(which slapos) : $(slapos --version)"
TMPD=$(mktemp -d)
if [ ! -e $TMPD ]; then
>&2 echo "Failed to create temp directory"
exit 1
fi
trap "exit 1" HUP INT PIPE QUIT TERM
trap 'rm -rf "$TMPD"' exit
cat <<EOF > ${TMPD}/slapos.cfg
[slapos]
computer_id = $COMPUTER_ID
[slapproxy]
host = 127.0.0.1
port = $PORT
database_uri = ${TMPD}/proxy.db
EOF
sqlite3 ${TMPD}/proxy.db < $DUMP_BEFORE
slapos proxy start --cfg ${TMPD}/slapos.cfg &
SLAPOS_PROXY_PID=$!
curl --silent --retry-connrefuse --retry 3 http://127.0.0.1:${PORT}/getComputerInformation?computer_id=$COMPUTER_ID
echo ".dump" | sqlite3 ${TMPD}/proxy.db > $DUMP_AFTER
kill $SLAPOS_PROXY_PID
wait
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7