Commit 86b03edf authored by Rafael Monnerat's avatar Rafael Monnerat

collect: small optimization on garbage collect

This changes a query like:

    SELECT date FROM user WHERE reported = 1 AND date != '2020-04-13' AND date != '2020-04-12'  AND date != '2020-04-11'  AND date != '2020-04-10'  AND date != '2020-04-09'  AND date != '2020-04-08'  AND date != '2020-04-07'  AND date != '2020-04-06'  AND date != '2020-04-05' AND date != '2020-04-04'  AND date != '2020-04-03'  AND date != '2020-04-02'  AND date != '2020-04-01'  AND date != '2020-03-31'  AND date != '2020-03-30'  LIMIT 1

    EXPLAIN QUERY PLAN ...
    0|0|0|SCAN TABLE user USING COVERING INDEX user_date_reported_index

which took ~3 seconds on a 1.5Go collector.db into:

    SELECT date FROM user WHERE reported = 1 AND (date < '2020-03-30'  OR  date > '2020-04-13')  LIMIT 1

    EXPLAIN QUERY PLAN ...
    0|0|0|SEARCH TABLE user USING COVERING INDEX user_date_reported_index (date<?)
    0|0|0|SEARCH TABLE user USING COVERING INDEX user_date_reported_index (date>?)

which is instant.

See merge request !201
parents 49ec3ece 686289b8
...@@ -286,10 +286,11 @@ class Database: ...@@ -286,10 +286,11 @@ class Database:
""" """
date_list = self._getGarbageCollectionDateList(days_to_preserve) date_list = self._getGarbageCollectionDateList(days_to_preserve)
where_clause = "reported = 1" where_clause = "reported = 1"
for _date in date_list: if date_list:
where_clause += " AND date != '%s' " % _date where_clause += " AND (date < '%s' " % min(date_list)
where_clause += " OR date > '%s') " % max(date_list)
vacuum = 0 vacuum = False
delete_sql = "DELETE FROM %s WHERE %s" delete_sql = "DELETE FROM %s WHERE %s"
select_sql = "SELECT date FROM %s WHERE %s LIMIT 1" select_sql = "SELECT date FROM %s WHERE %s LIMIT 1"
...@@ -298,7 +299,7 @@ class Database: ...@@ -298,7 +299,7 @@ class Database:
if table not in self.preserve_table_list: if table not in self.preserve_table_list:
if self._execute(select_sql % (table, where_clause)).fetchone() is not None: if self._execute(select_sql % (table, where_clause)).fetchone() is not None:
self._execute(delete_sql % (table, where_clause)) self._execute(delete_sql % (table, where_clause))
vacuum = 1 vacuum = True
self.commit() self.commit()
if vacuum: if vacuum:
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment