MDEV-35793: Server crashes in Item_func_vec_distance_common::get_const_arg

The problem was caused by this scenario: The query had both SELECT DISTINCT
and ORDER BY. DISTINCT was converted into GROUP BY. Then, vector index was
used to resolve the GROUP BY.
When join_read_first() initialized vector index scan, it used the ORDER BY
clause instead of GROUP BY, which caused a crash.

Fixed by making test_if_skip_sort_order() remember which ordering the scan
produces in JOIN_TAB::full_index_scan_order, and join_read_first() using that.

mysql-test/main/vector_innodb.result

@@ -241,3 +241,15 @@ hex(v)
 3737373700000000
 3131313100000000
 drop table t;
+#
+# MDEV-35793: Server crashes in
+# Item_func_vec_distance_common::get_const_arg
+#
+create table t (pk int primary key, v vector(1) not null, vector(v)) engine=innodb;
+insert into t values (1,0x31313131),(2,0x32323232);
+select distinct vec_distance_euclidean(v, 0x30303030) as d from t order by pk;
+d
+0.0000000019375161827791346
+0.000000009731407668281116
+drop table t;
+# End of 11.7 tests

mysql-test/main/vector_innodb.test

@@ -242,3 +242,15 @@ insert into t values (1,0x38383838),(2,0x37373737),(3,0x31313131);
 alter table t modify v vector(2) not null;
 select hex(v) from t order by a;
 drop table t;
+
+--echo #
+--echo # MDEV-35793: Server crashes in
+--echo # Item_func_vec_distance_common::get_const_arg
+--echo #
+
+create table t (pk int primary key, v vector(1) not null, vector(v)) engine=innodb;
+insert into t values (1,0x31313131),(2,0x32323232); # optional, fails either way
+select distinct vec_distance_euclidean(v, 0x30303030) as d from t order by pk;
+drop table t;
+
+--echo # End of 11.7 tests

sql/sql_select.cc

@@ -3683,7 +3683,8 @@ bool JOIN::make_aggr_tables_info()
   bool is_having_added_as_table_cond= false;
   DBUG_ENTER("JOIN::make_aggr_tables_info");
 
-  
+  DBUG_ASSERT(current_ref_ptrs == items0);
+
   sort_and_group_aggr_tab= NULL;
 
   if (group_optimized_away)
@@ -3790,7 +3791,6 @@ bool JOIN::make_aggr_tables_info()
         */
         init_items_ref_array();
         items1= ref_ptr_array_slice(2);
-        //items1= items0 + all_fields.elements;
         if (change_to_use_tmp_fields(thd, items1,
                                      tmp_fields_list1, tmp_all_fields1,
                                      fields_list.elements, all_fields))
@@ -25199,12 +25199,13 @@ join_read_first(JOIN_TAB *tab)
   tab->read_record.table=table;
   if (tab->index >= table->s->keys)
   {
-    ORDER *order= tab->join->order ? tab->join->order : tab->join->group_list;
+    ORDER *order= tab->full_index_scan_order;
     DBUG_ASSERT(tab->index < table->s->total_keys);
     DBUG_ASSERT(tab->index == table->s->keys);
     DBUG_ASSERT(tab->sorted);
     DBUG_ASSERT(order);
     DBUG_ASSERT(order->next == NULL);
+    DBUG_ASSERT(order->item[0]->real_item()->type() == Item::FUNC_ITEM);
     tab->read_record.read_record_func= join_hlindex_read_next;
     error= tab->table->hlindex_read_first(tab->index, *order->item,
                                           tab->join->select_limit);
@@ -27110,6 +27111,7 @@ test_if_skip_sort_order(JOIN_TAB *tab,ORDER *order,ha_rows select_limit,
   int best_key= -1;
   bool changed_key= false;
   THD *thd= tab->join->thd;
+  ORDER *best_key_order= 0;
   Json_writer_object trace_wrapper(thd);
   Json_writer_array  trace_arr(thd, "test_if_skip_sort_order");
   DBUG_ENTER("test_if_skip_sort_order");
@@ -27342,6 +27344,7 @@ test_if_skip_sort_order(JOIN_TAB *tab,ORDER *order,ha_rows select_limit,
          !table->is_clustering_key(best_key)))
       goto use_filesort;
 
+    best_key_order= order;
     if (select && table->opt_range_keys.is_set(best_key) && best_key != ref_key)
     {
       key_map tmp_map;
@@ -27441,6 +27444,7 @@ check_reverse_order:
                                  join_read_first:
                                  join_read_last);
         tab->type=JT_NEXT;           // Read with index_first(), index_next()
+        tab->full_index_scan_order= best_key_order;
 
         /*
           Currently usage of rowid filters is not supported in InnoDB

sql/sql_select.h

@@ -556,6 +556,13 @@ typedef struct st_join_table {
   /** HAVING condition for checking prior saving a record into tmp table*/
   Item *having;
 
+  /**
+    Ordering to be produced when doing full index scan.
+    Important for vector indexes, set by test_if_skip_sort_order() when it
+    decides to use full index to produce rows in order.
+  */
+  ORDER *full_index_scan_order;
+
   /** TRUE <=> remove duplicates on this table. */
   bool distinct;