X ANALYZE from under NEO + STATUS delta

neo/storage/database/mysqldb.py

@@ -27,6 +27,7 @@ import re
 import string
 import struct
 import time
+import sys
 
 from . import LOG_QUERIES
 from .manager import CreationUndone, DatabaseManager, splitOIDField
@@ -119,7 +120,7 @@ class MySQLDatabaseManager(DatabaseManager):
             conn = self.conn
             try:
                 conn.query(query)
-                if query.startswith("SELECT "):
+                if query.startswith("SELECT ") or query.startswith("ANALYZE ") or query.startswith("SHOW "):
                     r = conn.store_result()
                     return tuple([
                         tuple([d.tostring() if isinstance(d, array) else d
@@ -136,7 +137,7 @@ class MySQLDatabaseManager(DatabaseManager):
         if r in ("INSERT", "REPLACE", "DELETE", "UPDATE"):
             self._active = 1
         else:
-            assert r in ("ALTER", "CREATE", "DROP"), query
+            assert r in ("ALTER", "CREATE", "DROP", "FLUSH"), query
 
     @property
     def escape(self):
@@ -330,7 +331,10 @@ class MySQLDatabaseManager(DatabaseManager):
         return util.p64(r[0][0]) if r else None
 
     def _getNextTID(self, *args): # partition, oid, tid
+                       #" USE INDEX(PRIMARY)"
+                       # " FORCE INDEX(`partition`)"
         r = self.query("SELECT tid FROM obj"
+                       " USE INDEX(`partition`)"
                        " WHERE `partition`=%d AND oid=%d AND tid>%d"
                        " ORDER BY tid LIMIT 1" % args)
         return r[0][0] if r else None
@@ -667,15 +671,85 @@ class MySQLDatabaseManager(DatabaseManager):
         p64 = util.p64
         min_tid = u64(min_tid)
         max_tid = u64(max_tid)
+        # AND tid >= %(min_tid)d AND tid <= %(max_tid)d
+        # ORDER BY tid ASC LIMIT %(length)d""" % {
+        # ORDER BY tid ASC LIMIT %(length)d""" % {
+        print >>sys.stderr, 'AAA', min_tid, max_tid, length, partition
+
+        self.query("""FLUSH STATUS""")
+        def status():
+            st = {}
+            r = self.query("SHOW STATUS")
+            for k,v in r:
+                try:
+                    v = int(v)
+                except ValueError:
+                    try:
+                        v = float(v)
+                    except ValueError:
+                        pass
+
+                st[k] = v
+            return st
+
+        # (s2 - s1) -> {} k -> (v, delta(v))
+        def stdiff(s1, s2):
+            keys1 = s1.keys(); keys1.sort()
+            keys2 = s2.keys(); keys2.sort()
+            assert keys1 == keys2
+            d = {}
+            for k in keys1:
+                v1, v2 = s1[k], s2[k]
+                if v1 != v2:
+                    vdelta = v1
+                    try:
+                        vdelta = v2 - v1
+                    except TypeError:
+                        pass
+                    d[k] = (v2, vdelta)
+            return d
+
+        S1 = status()
+        Tstart = time.time()
+        #r = self.query("""ANALYZE FORMAT=JSON SELECT tid FROM trans
+        r = self.query("""ANALYZE SELECT tid FROM trans
+                    FORCE INDEX FOR ORDER BY (PRIMARY)
+                    WHERE `partition` = %(partition)d
+                    AND tid BETWEEN %(min_tid)d AND %(max_tid)d
+                    ORDER BY tid ASC LIMIT %(length)d""" % {
+            'partition': partition,
+            'min_tid': min_tid,
+            'max_tid': max_tid,
+            'length': length,
+        })
+        Tend = time.time()
+        print >>sys.stderr, '%.3f' % (Tend-Tstart), r
+        #for i, _ in enumerate(r):
+        #    print >>sys.stderr, '[%i]: %s' % (i, _[0]), _[1:]
+        # r = self.query("""SHOW STATUS LIKE 'Handler%'""")
+        # #print >>sys.stderr, r
+        # for _ in r:
+        #     print >>sys.stderr, '%s:\t%s' % _
+        S2 = status()
+        D = stdiff(S1, S2)
+        keys = D.keys(); keys.sort()
+        for k in keys:
+            v, vdelta = D[k]
+            print >>sys.stderr, '%-64s:\t%s\t+%s' % (k, v, vdelta)
+
+        Tstart = time.time()
         r = self.query("""SELECT tid FROM trans
+                    FORCE INDEX FOR ORDER BY (PRIMARY)
                     WHERE `partition` = %(partition)d
-                    AND tid >= %(min_tid)d AND tid <= %(max_tid)d
+                    AND tid BETWEEN %(min_tid)d AND %(max_tid)d
                     ORDER BY tid ASC LIMIT %(length)d""" % {
             'partition': partition,
             'min_tid': min_tid,
             'max_tid': max_tid,
             'length': length,
         })
+        Tend = time.time()
+        print >>sys.stderr, '(%.3f)' % (Tend - Tstart)
         return [p64(t[0]) for t in r]
 
     def _updatePackFuture(self, oid, orig_serial, max_serial):