trying to stabilize floating-point tests

mysql-test/main/vector.result

@@ -108,14 +108,14 @@ id	hex(v)	vec_totext(v)
 flush tables;
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-9	0.4719976290006591
-10	0.5069011044450041
-3	0.5865673124650332
+9	0.47199
+10	0.50690
+3	0.58656
 select id,vec_distance_euclidean(x'b047263C9f87233fcfd27e3eae493e3f0329f43e', v) d from t1 order by d limit 3;
 id	d
-9	0.4719976290006591
-10	0.5069011044450041
-3	0.5865673124650332
+9	0.47199
+10	0.50690
+3	0.58656
 select id>0,vec_distance_euclidean(v, NULL) d from t1 order by d limit 3;
 id>0	d
 1	NULL
@@ -138,156 +138,156 @@ id1	id2	vec_distance_euclidean(t1.v, t2.v)
 8	8	0
 9	9	0
 10	10	0
-7	10	0.35209010323904116
-10	7	0.35209010323904116
-1	7	0.557267332724855
-7	1	0.557267332724855
-2	3	0.6065128837978769
-3	2	0.6065128837978769
-1	3	0.6128238020507096
-3	1	0.6128238020507096
-5	8	0.6219995745138945
-8	5	0.6219995745138945
-3	10	0.6523185662547816
-10	3	0.6523185662547816
-9	10	0.6732681362788765
-10	9	0.6732681362788765
-3	7	0.6799892416547949
-7	3	0.6799892416547949
-3	9	0.6820752294088018
-9	3	0.6820752294088018
-2	10	0.6916305331777215
-10	2	0.6916305331777215
-2	9	0.6966650510789955
-9	2	0.6966650510789955
-3	6	0.7102823580937639
-6	3	0.7102823580937639
-2	7	0.7120217580666971
-7	2	0.7120217580666971
-2	6	0.7351618106552689
-6	2	0.7351618106552689
-1	10	0.7386864491588024
-10	1	0.7386864491588024
-4	6	0.7784357824370262
-6	4	0.7784357824370262
-4	8	0.7795837407361241
-8	4	0.7795837407361241
-4	5	0.8132007346697969
-5	4	0.8132007346697969
-2	4	0.8260925223296488
-4	2	0.8260925223296488
-5	10	0.8286488932765299
-10	5	0.8286488932765299
-5	9	0.8769351333060768
-9	5	0.8769351333060768
-1	6	0.8861410875047832
-6	1	0.8861410875047832
-3	5	0.9224201772876247
-5	3	0.9224201772876247
-4	7	0.9347916246876117
-7	4	0.9347916246876117
-7	9	0.9364253407685257
-9	7	0.9364253407685257
-3	4	0.9757105842688992
-4	3	0.9757105842688992
-1	2	0.9810272439433514
-2	1	0.9810272439433514
-1	4	0.9965475544626712
-4	1	0.9965475544626712
-5	7	0.9976863778073342
-7	5	0.9976863778073342
-4	10	1.0109345944029724
-10	4	1.0109345944029724
-1	5	1.0208359400987237
-5	1	1.0208359400987237
-6	7	1.0221332668982412
-7	6	1.0221332668982412
-2	5	1.050769316594881
-5	2	1.050769316594881
-6	8	1.103420381318026
-8	6	1.103420381318026
-3	8	1.1170300826294572
-8	3	1.1170300826294572
-6	10	1.1523451990991307
-10	6	1.1523451990991307
-1	9	1.1637750565139302
-9	1	1.1637750565139302
-2	8	1.1736571017573874
-8	2	1.1736571017573874
-4	9	1.1746893942711878
-9	4	1.1746893942711878
-1	8	1.1909959973982214
-8	1	1.1909959973982214
-8	10	1.209359617652948
-10	8	1.209359617652948
-6	9	1.214529873940304
-9	6	1.214529873940304
-5	6	1.227278506501395
-6	5	1.227278506501395
-8	9	1.2575258643523053
-9	8	1.2575258643523053
-7	8	1.288239696195716
-8	7	1.288239696195716
+7	10	0.35209
+10	7	0.35209
+1	7	0.55726
+7	1	0.55726
+2	3	0.60651
+3	2	0.60651
+1	3	0.61282
+3	1	0.61282
+5	8	0.62199
+8	5	0.62199
+3	10	0.65231
+10	3	0.65231
+9	10	0.67326
+10	9	0.67326
+3	7	0.67998
+7	3	0.67998
+3	9	0.68207
+9	3	0.68207
+2	10	0.69163
+10	2	0.69163
+2	9	0.69666
+9	2	0.69666
+3	6	0.71028
+6	3	0.71028
+2	7	0.71202
+7	2	0.71202
+2	6	0.73516
+6	2	0.73516
+1	10	0.73868
+10	1	0.73868
+4	6	0.77843
+6	4	0.77843
+4	8	0.77958
+8	4	0.77958
+4	5	0.81320
+5	4	0.81320
+2	4	0.82609
+4	2	0.82609
+5	10	0.82864
+10	5	0.82864
+5	9	0.87693
+9	5	0.87693
+1	6	0.88614
+6	1	0.88614
+3	5	0.92242
+5	3	0.92242
+4	7	0.93479
+7	4	0.93479
+7	9	0.93642
+9	7	0.93642
+3	4	0.97571
+4	3	0.97571
+1	2	0.98102
+2	1	0.98102
+1	4	0.99654
+4	1	0.99654
+5	7	0.99768
+7	5	0.99768
+4	10	1.01093
+10	4	1.01093
+1	5	1.02083
+5	1	1.02083
+6	7	1.02213
+7	6	1.02213
+2	5	1.05076
+5	2	1.05076
+6	8	1.10342
+8	6	1.10342
+3	8	1.11703
+8	3	1.11703
+6	10	1.15234
+10	6	1.15234
+1	9	1.16377
+9	1	1.16377
+2	8	1.17365
+8	2	1.17365
+4	9	1.17468
+9	4	1.17468
+1	8	1.19099
+8	1	1.19099
+8	10	1.20935
+10	8	1.20935
+6	9	1.21452
+9	6	1.21452
+5	6	1.22727
+6	5	1.22727
+8	9	1.25752
+9	8	1.25752
+7	8	1.28823
+8	7	1.28823
 select * from (
 select id,vec_distance_euclidean(v, x'b047263C9F87233fcfd27e3eae493e3f0329f43e') d
 from t1 where id < 10
 ) u order by d limit 3;
 id	d
-9	0.4719976290006591
-3	0.5865673124650332
+9	0.47199
+3	0.58656
 flush session status;
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-9	0.4719976290006591
-10	0.5069011044450041
-3	0.5865673124650332
+9	0.47199
+10	0.50690
+3	0.58656
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	0
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 use index () order by d limit 3;
 id	d
-9	0.4719976290006591
-10	0.5069011044450041
-3	0.5865673124650332
+9	0.47199
+10	0.50690
+3	0.58656
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	11
 flush session status;
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-10	0.05905546376032378
-9	0.06546887818344715
-3	0.10750282439505232
+10	0.05905
+9	0.06546
+3	0.10750
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	11
 delete from t1 where v = x'7b713f3e5258323f80d1113d673b2b3f66e3583f';
 select id,vec_distance_euclidean(v, x'B047263C9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-10	0.5069011044450041
-3	0.5865673124650332
-7	0.7344464697214867
+10	0.50690
+3	0.58656
+7	0.73444
 insert t1 (v) values (x'7b713f3e5258323f80d1113d673b2b3f66e3583f');
 select id,vec_distance_euclidean(v, x'b047263c9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-11	0.4719976290006591
-10	0.5069011044450041
-3	0.5865673124650332
+11	0.47199
+10	0.50690
+3	0.58656
 select id,vec_distance_euclidean(v, x'B047263c9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 id	d
-11	0.4719976290006591
-10	0.5069011044450041
-3	0.5865673124650332
-7	0.7344464697214867
-5	0.7671033529042712
+11	0.47199
+10	0.50690
+3	0.58656
+7	0.73444
+5	0.76710
 update t1 set v=x'76EDFC3E4B57243F10F8423FB158713F020BAA3E' where v=x'6CA1D43E9DF91B3FE580DA3E1C247D3F147CF33E';
 select id,vec_distance_euclidean(v, x'B047263C9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 id	d
-11	0.4719976290006591
-3	0.5865673124650332
-7	0.7344464697214867
-10	0.746836719209219
-5	0.7671033529042712
+11	0.47199
+3	0.58656
+7	0.73444
+10	0.74683
+5	0.76710
 delete from t1;
 insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
 (x'f511303f72224a3fdd05fe3eb22a133ffae86a3f'),
@@ -301,11 +301,11 @@ insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
 (x'6ca1d43e9df91b3fe580da3e1c247d3f147cf33e');
 select id,vec_distance_euclidean(v, x'b047263c9f87233Fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 id	d
-20	0.4719976290006591
-21	0.5069011044450041
-14	0.5865673124650332
-18	0.7344464697214867
-16	0.7671033529042712
+20	0.47199
+21	0.50690
+14	0.58656
+18	0.73444
+16	0.76710
 insert t1 (v) values ('');
 ERROR 22007: Incorrect vector value: '...' for column `test`.`t1`.`v` at row 1
 insert t1 (v) values (x'1234');
@@ -436,32 +436,32 @@ insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
 (x'6ca1d43e9df91b3fe580da3e1c247d3f147cf33e');
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-10	0.05905546376032378
-9	0.06546887818344715
-3	0.10750282439505232
+10	0.05905
+9	0.06546
+3	0.10750
 flush session status;
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-10	0.05905546376032378
-9	0.06546887818344715
-3	0.10750282439505232
+10	0.05905
+9	0.06546
+3	0.10750
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	0
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 use index () order by d limit 3;
 id	d
-10	0.05905546376032378
-9	0.06546887818344715
-3	0.10750282439505232
+10	0.05905
+9	0.06546
+3	0.10750
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	11
 flush session status;
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 id	d
-9	0.4719976290006591
-10	0.5069011044450041
-3	0.5865673124650332
+9	0.47199
+10	0.50690
+3	0.58656
 show status like 'handler_read_rnd_next';
 Variable_name	Value
 Handler_read_rnd_next	11

mysql-test/main/vector.test

@@ -41,16 +41,22 @@ insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
 select id, hex(v), vec_totext(v) from t1;
 flush tables;
 # test with a valid query vector
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 # swapped arguments
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(x'b047263C9f87233fcfd27e3eae493e3f0329f43e', v) d from t1 order by d limit 3;
 # test with NULL (id is unpredictable)
+--replace_regex /(\.\d{5})\d+/\1/
 select id>0,vec_distance_euclidean(v, NULL) d from t1 order by d limit 3;
 # test with invalid query vector (id is unpredictable)
+--replace_regex /(\.\d{5})\d+/\1/
 select id>0,vec_distance_euclidean(v, x'123456') d from t1 order by d limit 3;
+--replace_regex /(\.\d{5})\d+/\1/
 select t1.id as id1, t2.id as id2, vec_distance_euclidean(t1.v, t2.v) from t1, t1 as t2 order by 3,1,2;
 
 # subquery
+--replace_regex /(\.\d{5})\d+/\1/
 select * from (
   select id,vec_distance_euclidean(v, x'b047263C9F87233fcfd27e3eae493e3f0329f43e') d
   from t1 where id < 10
@@ -60,11 +66,14 @@ select * from (
 --disable_view_protocol
 --disable_ps2_protocol
 flush session status;
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 show status like 'handler_read_rnd_next'; # used
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 use index () order by d limit 3;
 show status like 'handler_read_rnd_next'; # not used
 flush session status;
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 show status like 'handler_read_rnd_next'; # not used, wrong distance metric
 --enable_ps2_protocol
@@ -72,15 +81,19 @@ show status like 'handler_read_rnd_next'; # not used, wrong distance metric
 
 # test delete
 delete from t1 where v = x'7b713f3e5258323f80d1113d673b2b3f66e3583f';
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'B047263C9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 
 # test insert deleted vec
 insert t1 (v) values (x'7b713f3e5258323f80d1113d673b2b3f66e3583f');
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'b047263c9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 
 # test update
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'B047263c9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 update t1 set v=x'76EDFC3E4B57243F10F8423FB158713F020BAA3E' where v=x'6CA1D43E9DF91B3FE580DA3E1C247D3F147CF33E';
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'B047263C9F87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 
 # test delete all and reinsert
@@ -95,6 +108,7 @@ insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
                      (x'56926c3fdf098d3e2c8c5e3d1ad4953daa9d0b3e'),
                      (x'7b713f3e5258323f80d1113d673b2b3f66e3583f'),
                      (x'6ca1d43e9df91b3fe580da3e1c247d3f147cf33e');
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'b047263c9f87233Fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 5;
 
 
@@ -198,16 +212,20 @@ insert t1 (v) values (x'e360d63ebe554f3fcdbc523f4522193f5236083d'),
                      (x'6ca1d43e9df91b3fe580da3e1c247d3f147cf33e');
 
 # make sure the graph is loaded
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 
 --disable_view_protocol
 --disable_ps2_protocol
 flush session status;
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 show status like 'handler_read_rnd_next';
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_cosine(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 use index () order by d limit 3;
 show status like 'handler_read_rnd_next';
 flush session status;
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'B047263c9f87233fcfd27e3eae493e3f0329f43e') d from t1 order by d limit 3;
 show status like 'handler_read_rnd_next';
 --enable_ps2_protocol

mysql-test/main/vector_innodb.result

@@ -40,18 +40,18 @@ delete from t1 where id=7;
 rollback;
 select id,vec_distance_euclidean(v, x'c923e33dc0da313fe7c7983e526b3d3fde63963e6eaf3a3f27fa133fe27a583f') d from t1 order by d limit 5;
 id	d
-2	0.8781474260354732
-10	0.8856208347761952
-30	1.0162643974895857
-7	1.026397313888122
-5	1.0308161006949719
+2	0.87814
+10	0.88562
+30	1.01626
+7	1.02639
+5	1.03081
 select id,vec_distance_euclidean(v, x'754b5f3ea2312b3fc169f43e4604883e1d20173e8dd7443f421b703fb11e0d3e') d from t1 order by d limit 5;
 id	d
-2	0.9426904171992334
-33	0.9477554826856
-30	1.111405427702547
-10	1.118630286292343
-8	1.1405733350751739
+2	0.94269
+33	0.94775
+30	1.11140
+10	1.11863
+8	1.14057
 create table t2 (id int auto_increment primary key, v blob not null, vector index (v)) engine=innodb;
 insert t2 (v) values
 (x'45cf153f830a313f7a0a113fb1ff533f47a1533fcf9e6e3f'),
@@ -73,18 +73,18 @@ insert t2 values
 commit;
 select id,vec_distance_euclidean(v, x'1f4d053f7056493f937da03dd8c97a3f220cbb3c926c1c3facca213ec0618a3e') d from t1 order by d limit 5;
 id	d
-6	0.9309383181777582
-5	0.9706304662574956
-30	0.98144492002831
-50	1.079862635421575
-2	1.0907138991979892
+6	0.93093
+5	0.97063
+30	0.98144
+50	1.07986
+2	1.09071
 select id,vec_distance_euclidean(v, x'f618663f256be73e62cd453f8bcdbf3e16ae503c3858313f') d from t2 order by d limit 5;
 id	d
-21	0.43559180321379337
-20	0.6435053022072372
-6	0.6942000623336242
-2	0.7971622099055623
-9	0.8298589136476077
+21	0.43559
+20	0.64350
+6	0.69420
+2	0.79716
+9	0.82985
 drop table t1, t2;
 #
 # MDEV-34989 After selecting from empty table with vector key the next insert hangs

mysql-test/main/vector_innodb.test

@@ -37,7 +37,9 @@ insert t1 values
 delete from t1 where id=7;
 rollback;
 
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'c923e33dc0da313fe7c7983e526b3d3fde63963e6eaf3a3f27fa133fe27a583f') d from t1 order by d limit 5;
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'754b5f3ea2312b3fc169f43e4604883e1d20173e8dd7443f421b703fb11e0d3e') d from t1 order by d limit 5;
 
 ### two indexes in one transaction:
@@ -63,7 +65,9 @@ insert t2 values
      (21, x'35e05d3f18e8513fb81a3d3f8acf7d3e794a1d3c72f9613f');
 commit;
 
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'1f4d053f7056493f937da03dd8c97a3f220cbb3c926c1c3facca213ec0618a3e') d from t1 order by d limit 5;
+--replace_regex /(\.\d{5})\d+/\1/
 select id,vec_distance_euclidean(v, x'f618663f256be73e62cd453f8bcdbf3e16ae503c3858313f') d from t2 order by d limit 5;
 
 drop table t1, t2;