Commit 4ecb78a3 authored by Rucha Deodhar's avatar Rucha Deodhar

MDEV-32854: Make JSON_DEPTH_LIMIT configurable

Create server variable to store limit for json depth with default value of
32, minimum 0 and maximum 100. Replace use of the previous direct or
indirect use JSON_DEPTH_LIMIT examples in array.
parent 9811d23b
......@@ -7,7 +7,10 @@
extern "C" {
#endif
#define JSON_DEPTH_LIMIT 32
#define JSON_DEPTH_DEFAULT 32
#define JSON_DEPTH_LIMIT JSON_DEPTH_DEFAULT /* Still used in column store. */
extern int (*get_json_depth)(void);
/*
When error happens, the c_next of the JSON engine contains the
......@@ -27,8 +30,6 @@ enum json_errors {
JE_STRING_CONST= -5, /* Character disallowed in string constant. */
JE_ESCAPING= -6, /* Error in the escaping. */
JE_DEPTH= -7, /* The limit on the JSON depth was overrun. */
};
......@@ -104,8 +105,8 @@ typedef struct st_json_path_step_t
typedef struct st_json_path_t
{
json_string_t s; /* The string to be parsed. */
json_path_step_t steps[JSON_DEPTH_LIMIT]; /* Steps of the path. */
json_path_step_t *last_step; /* Points to the last step. */
MEM_ROOT_DYNAMIC_ARRAY steps; /* Steps of the path. */
json_path_step_t* last_step; /* Points to the last step. */
int mode_strict; /* TRUE if the path specified as 'strict' */
enum json_path_step_types types_used; /* The '|' of all step's 'type'-s */
......@@ -225,8 +226,7 @@ typedef struct st_json_engine_t
const uchar *value_end; /* Points to the next character after the value. */
int value_len; /* The length of the value. Does not count quotations for */
/* string constants. */
int stack[JSON_DEPTH_LIMIT]; /* Keeps the stack of nested JSON structures. */
MEM_ROOT_DYNAMIC_ARRAY stack; /* Keeps the stack of nested JSON structures. */
int stack_p; /* The 'stack' pointer. */
volatile uchar *killed_ptr;
} json_engine_t;
......@@ -235,6 +235,7 @@ typedef struct st_json_engine_t
int json_scan_start(json_engine_t *je,
CHARSET_INFO *i_cs, const uchar *str, const uchar *end);
int json_scan_next(json_engine_t *j);
void reset_json_engine(json_engine_t *je);
/*
......@@ -341,6 +342,13 @@ int json_skip_level_and_count(json_engine_t *j, int *n_items_skipped);
*/
#define json_value_scalar(je) ((je)->value_type > JSON_VALUE_ARRAY)
#define report_json_error(js, je, n_param) \
report_json_error_ex(js->ptr(), je, func_name(), n_param, \
Sql_condition::WARN_LEVEL_WARN)
#define report_path_error(js, je, n_param) \
report_path_error_ex(js->ptr(), je, func_name(), n_param,\
Sql_condition::WARN_LEVEL_WARN)
/*
Look for the JSON PATH in the json string.
......@@ -350,7 +358,7 @@ int json_skip_level_and_count(json_engine_t *j, int *n_items_skipped);
initialized with the JSON string, and the json_path_t with the JSON path
appropriately. The 'p_cur_step' should point at the first
step of the path.
The 'array_counters' is the array of JSON_DEPTH_LIMIT size.
The 'array_counters' is the array of 'curr_json_depth_limit' size.
It stores the array counters of the parsed JSON.
If function returns 0, it means it found the match. The position of
the match is je->s.c_str. Then we can call the json_find_path()
......@@ -360,7 +368,7 @@ int json_skip_level_and_count(json_engine_t *j, int *n_items_skipped);
*/
int json_find_path(json_engine_t *je,
json_path_t *p, json_path_step_t **p_cur_step,
int *array_counters);
MEM_ROOT_DYNAMIC_ARRAY *array_counters);
typedef struct st_json_find_paths_t
......@@ -369,7 +377,7 @@ typedef struct st_json_find_paths_t
json_path_t *paths;
uint cur_depth;
uint *path_depths;
int array_counters[JSON_DEPTH_LIMIT];
MEM_ROOT_DYNAMIC_ARRAY array_counters;
} json_find_paths_t;
......@@ -430,9 +438,11 @@ int json_get_path_start(json_engine_t *je, CHARSET_INFO *i_cs,
int json_get_path_next(json_engine_t *je, json_path_t *p);
int json_path_compare(const json_path_t *a, const json_path_t *b,
enum json_value_types vt, const int* array_size_counter);
enum json_value_types vt,
MEM_ROOT_DYNAMIC_ARRAY* array_size_counter);
int json_valid(const char *js, size_t js_len, CHARSET_INFO *cs);
int json_valid(const char *js, size_t js_len,
CHARSET_INFO *cs, json_engine_t *je);
int json_locate_key(const char *js, const char *js_end,
const char *kname,
......@@ -440,7 +450,8 @@ int json_locate_key(const char *js, const char *js_end,
int *comma_pos);
int json_normalize(DYNAMIC_STRING *result,
const char *s, size_t size, CHARSET_INFO *cs);
const char *s, size_t size, CHARSET_INFO *cs,
MEM_ROOT *current_mem_root);
int json_skip_array_and_count(json_engine_t *j, int* n_item);
......
......@@ -354,6 +354,17 @@ typedef struct st_dynamic_array
myf malloc_flags;
} DYNAMIC_ARRAY;
typedef struct st_mem_root_dynamic_array
{
MEM_ROOT *mem_root;
uchar *buffer;
size_t elements, max_element;
size_t alloc_increment;
size_t size_of_element;
PSI_memory_key m_psi_key;
myf malloc_flags;
} MEM_ROOT_DYNAMIC_ARRAY;
typedef struct st_dynamic_array_append
{
......@@ -1163,6 +1174,20 @@ extern void thd_increment_bytes_sent(void *thd, size_t length);
extern void thd_increment_bytes_received(void *thd, size_t length);
extern void thd_increment_net_big_packet_count(void *thd, size_t length);
extern int mem_root_dynamic_array_init(MEM_ROOT *mem_root,
PSI_memory_key psi_key,
MEM_ROOT_DYNAMIC_ARRAY *array,
size_t element_size, void *init_buffer,
size_t init_alloc,
size_t alloc_increment,
myf my_flags);
extern int mem_root_dynamic_array_set_val(MEM_ROOT_DYNAMIC_ARRAY *array,
const void *element, size_t idx);
extern void* mem_root_dynamic_array_get_val(MEM_ROOT_DYNAMIC_ARRAY *array, size_t idx);
extern void mem_root_dynamic_array_reset(MEM_ROOT_DYNAMIC_ARRAY *array);
extern void mem_root_dynamic_array_copy(MEM_ROOT_DYNAMIC_ARRAY *dest, MEM_ROOT_DYNAMIC_ARRAY *src);
extern void* mem_root_dynamic_array_get_val_ptr(MEM_ROOT_DYNAMIC_ARRAY *array, size_t idx);
#include <mysql/psi/psi.h>
#ifdef HAVE_PSI_INTERFACE
......
......@@ -43,7 +43,7 @@ NULL
select json_query('{"key1":123, "key1": [1,2,3]}', '$.key1');
json_query('{"key1":123, "key1": [1,2,3]}', '$.key1')
[1,2,3]
select json_query('{"key1":123, "key1": [1,2,3]}', concat('$', repeat('.k', 1000))) as exp;
select json_query('{"key1":123, "key1": [1,2,3]}', concat('$', repeat('.k', 100))) as exp;
exp
NULL
select json_array();
......@@ -651,8 +651,6 @@ max_allowed_packet 2048
select json_array(repeat('a',1024),repeat('a',1024)) as ex;
ex
NULL
Warnings:
Warning 1301 Result of json_array() was larger than max_allowed_packet (2048) - truncated
select json_object("a", repeat('a',1024),"b", repeat('a',1024)) as ex;
ex
NULL
......@@ -4797,16 +4795,16 @@ JSON_SCHEMA_VALID(@schema_array, '[')
0
Warnings:
Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_schema_valid'
SELECT JSON_SCHEMA_VALID(repeat('[', 100000), json_object());
JSON_SCHEMA_VALID(repeat('[', 100000), json_object())
SELECT JSON_SCHEMA_VALID(repeat('[', 100), json_object());
JSON_SCHEMA_VALID(repeat('[', 100), json_object())
NULL
Warnings:
Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 1 to function 'json_schema_valid' at position 32
SELECT JSON_SCHEMA_VALID(json_object(), repeat('[', 10000000));
JSON_SCHEMA_VALID(json_object(), repeat('[', 10000000))
Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_schema_valid'
SELECT JSON_SCHEMA_VALID(json_object(), repeat('[', 100));
JSON_SCHEMA_VALID(json_object(), repeat('[', 100))
0
Warnings:
Warning 4040 Limit of 32 on JSON nested structures depth is reached in argument 2 to function 'json_schema_valid' at position 32
Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_schema_valid'
#
# MDEV-30677: Incorrect result for "SELECT JSON_SCHEMA_VALID('{}', NULL)"
#
......@@ -4851,6 +4849,8 @@ NULL
SELECT JSON_KEY_VALUE('', '$.a');
JSON_KEY_VALUE('', '$.a')
NULL
Warnings:
Warning 4037 Unexpected end of JSON text in argument 1 to function ''
SELECT JSON_KEY_VALUE('[1,2,3]', '');
JSON_KEY_VALUE('[1,2,3]', '')
NULL
......
......@@ -18,7 +18,7 @@ select json_query('{"key1":{"a":1, "b":[1,2]}}', '$.key2');
select json_query('{"key1":{"a":1, "b":[1,2]}}', '$.key1');
select json_query('{"key1": 1}', '$.key1');
select json_query('{"key1":123, "key1": [1,2,3]}', '$.key1');
select json_query('{"key1":123, "key1": [1,2,3]}', concat('$', repeat('.k', 1000))) as exp;
select json_query('{"key1":123, "key1": [1,2,3]}', concat('$', repeat('.k', 100))) as exp;
select json_array();
select json_array(1);
......@@ -3742,10 +3742,10 @@ SET @schema_array= '{"type":"array"}';
SELECT JSON_SCHEMA_VALID(@schema_array, '[');
--disable_view_protocol
SELECT JSON_SCHEMA_VALID(repeat('[', 100000), json_object());
SELECT JSON_SCHEMA_VALID(repeat('[', 100), json_object());
--enable_view_protocol
SELECT JSON_SCHEMA_VALID(json_object(), repeat('[', 10000000));
SELECT JSON_SCHEMA_VALID(json_object(), repeat('[', 100));
--echo #
--echo # MDEV-30677: Incorrect result for "SELECT JSON_SCHEMA_VALID('{}', NULL)"
......
......@@ -10,12 +10,20 @@ NULL
select json_equals("", "");
json_equals("", "")
NULL
Warnings:
Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_equals'
Warning 4037 Unexpected end of JSON text in argument 2 to function 'json_equals'
select json_equals("", 1);
json_equals("", 1)
NULL
Warnings:
Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_equals'
select json_equals(now(), now());
json_equals(now(), now())
NULL
Warnings:
Warning 4038 Syntax error in JSON text in argument 1 to function 'json_equals' at position 5
Warning 4038 Syntax error in JSON text in argument 2 to function 'json_equals' at position 5
select json_equals('{"a":[1, 2, 3]}', '{"a":[1, 2, 3, 4]}');
json_equals('{"a":[1, 2, 3]}', '{"a":[1, 2, 3, 4]}')
0
......@@ -65,7 +73,7 @@ select json_equals('{"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"ob
select json_equals('{"obj":{"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"key": "value"}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}',
'{"obj":{"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"obj": {"key": "value"}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}') as 32_levels;
32_levels
NULL
1
#
# test values from different charset
# (UTF-8 two-bytes vs. latin1 single high-byte)
......
......@@ -43,6 +43,8 @@ NULL
select json_normalize('{ "invalid": "no_close"');
json_normalize('{ "invalid": "no_close"')
NULL
Warnings:
Warning 4037 Unexpected end of JSON text in argument 1 to function 'json_normalize'
drop table t1;
drop view v1;
create table t1 (text varchar(200) character set 'latin1');
......
select json_equals('{"a":[1, 2, 3]}', '{"a":[1, 2, 3, 4]}');
select json_query('{"key1":123, "key1": [1,2,3]}', concat('$', repeat('.k', 100))) as exp;
......@@ -1452,6 +1452,16 @@ NUMERIC_BLOCK_SIZE 1
ENUM_VALUE_LIST NULL
READ_ONLY NO
COMMAND_LINE_ARGUMENT REQUIRED
VARIABLE_NAME JSON_DEPTH_LIMIT
VARIABLE_SCOPE SESSION
VARIABLE_TYPE INT UNSIGNED
VARIABLE_COMMENT Max for JSON
NUMERIC_MIN_VALUE 0
NUMERIC_MAX_VALUE 100
NUMERIC_BLOCK_SIZE 1
ENUM_VALUE_LIST NULL
READ_ONLY YES
COMMAND_LINE_ARGUMENT REQUIRED
VARIABLE_NAME KEEP_FILES_ON_CREATE
VARIABLE_SCOPE SESSION
VARIABLE_TYPE BOOLEAN
......
select json_valid('[1, 2]');
select json_valid('"string"}');
select json_valid('{"key1":1, "key2":[2,3]}');
select json_valid('[false, true, null]');
select json_valid(repeat('[', 1000));
select json_valid(repeat('{"a":', 1000));
......@@ -411,3 +411,141 @@ void freeze_size(DYNAMIC_ARRAY *array)
array->max_element= elements;
}
}
int mem_root_dynamic_array_init(MEM_ROOT *current_mem_root, PSI_memory_key psi_key,
MEM_ROOT_DYNAMIC_ARRAY *array,
size_t element_size, void *init_buffer,
size_t init_alloc, size_t alloc_increment,
myf my_flags)
{
DBUG_ENTER("init_mem_root_dynamic_array");
if (!alloc_increment)
{
alloc_increment=MY_MAX((8192-MALLOC_OVERHEAD)/element_size,16);
if (init_alloc > 8 && alloc_increment > init_alloc * 2)
alloc_increment=init_alloc*2;
}
array->elements=0;
array->max_element=init_alloc;
array->alloc_increment=alloc_increment;
array->size_of_element=element_size;
array->m_psi_key= psi_key;
array->malloc_flags= my_flags;
DBUG_ASSERT((my_flags & MY_INIT_BUFFER_USED) == 0);
if ((array->buffer= (uchar*)init_buffer))
{
array->malloc_flags|= MY_INIT_BUFFER_USED;
DBUG_RETURN(FALSE);
}
array->mem_root= current_mem_root;
/*
Since the dynamic array is usable even if allocation fails here malloc
should not throw an error
*/
if (init_alloc &&
!(array->buffer= (uchar*) alloc_root(array->mem_root,
array->size_of_element*array->max_element)))
array->max_element=0;
//memset(array->buffer, 0, array->size_of_element*array->max_element);
DBUG_RETURN(FALSE);
}
void mem_root_dynamic_array_reset(MEM_ROOT_DYNAMIC_ARRAY *array)
{
memset(array->buffer, 0, (array->size_of_element)*(array->max_element));
}
int mem_root_allocate_dynamic(MEM_ROOT *mem_root,
MEM_ROOT_DYNAMIC_ARRAY *array,
size_t max_elements)
{
DBUG_ENTER("allocate_dynamic");
if (max_elements >= array->max_element)
{
size_t size;
uchar *new_ptr;
size= (max_elements + array->alloc_increment);
if (array->malloc_flags & MY_INIT_BUFFER_USED)
{
/*
In this senerio, the buffer is statically preallocated,
so we have to create an all-new malloc since we overflowed
*/
if (!(new_ptr= (uchar *) alloc_root(mem_root,
size * array->size_of_element)))
DBUG_RETURN(0);
array->malloc_flags&= ~MY_INIT_BUFFER_USED;
}
else if (!(new_ptr= (uchar*) alloc_root(mem_root, size*array->size_of_element)))
DBUG_RETURN(TRUE);
memcpy(new_ptr, array->buffer,
array->max_element * array->size_of_element);
memset(new_ptr+((array->max_element) * array->size_of_element), 0, array->alloc_increment*array->size_of_element);
array->buffer= new_ptr;
array->max_element= size;
}
DBUG_RETURN(FALSE);
}
int mem_root_dynamic_array_set_val(MEM_ROOT_DYNAMIC_ARRAY *array,
const void *element, size_t idx)
{
if (idx >= array->max_element)
{
if (mem_root_allocate_dynamic(array->mem_root, array, idx))
return TRUE;
array->elements++;
}
memcpy(array->buffer+(idx * array->size_of_element), element,
array->size_of_element);
return FALSE;
}
void* mem_root_dynamic_array_get_val(MEM_ROOT_DYNAMIC_ARRAY *array, size_t idx)
{
void* element_ptr;
if (idx >= array->max_element)
{
if (mem_root_allocate_dynamic(array->mem_root, array, idx))
return 0;
}
// Calculate the pointer to the desired element in the array
element_ptr = array->buffer + (idx * array->size_of_element);
return element_ptr;
}
void* mem_root_dynamic_array_get_val_ptr(MEM_ROOT_DYNAMIC_ARRAY *array, size_t idx)
{
if (idx >= array->max_element)
{
if (mem_root_allocate_dynamic(array->mem_root, array, idx))
return 0;
}
// Calculate the pointer to the desired element in the array
return (array->buffer + (idx * array->size_of_element));
}
void mem_root_dynamic_array_copy(MEM_ROOT_DYNAMIC_ARRAY *dest, MEM_ROOT_DYNAMIC_ARRAY *src)
{
for (int i=0; i<src->max_element; i++)
{
void *element= src->buffer+(i*src->size_of_element);
mem_root_dynamic_array_set_val(dest, element, i);
}
dest->elements= src->elements;
dest->max_element=src->max_element;
dest->alloc_increment=src->alloc_increment;
dest->size_of_element=src->size_of_element;
dest->m_psi_key= src->m_psi_key;
dest->malloc_flags= src->malloc_flags;
}
......@@ -114,6 +114,20 @@ String *Item_func_geometry_from_wkb::val_str(String *str)
return str;
}
bool Item_func_geometry_from_json::fix_length_and_dec(THD *thd)
{
if (!mem_root_inited)
init_alloc_root(PSI_NOT_INSTRUMENTED, &current_mem_root, 1024, 0, MYF(0));
mem_root_inited= true;
mem_root_dynamic_array_init(&current_mem_root, PSI_NOT_INSTRUMENTED,
&je.stack,
sizeof(int), NULL,
32, 32, MYF(0));
return Item_geometry_func::fix_length_and_dec(thd);
}
String *Item_func_geometry_from_json::val_str(String *str)
{
......@@ -122,7 +136,6 @@ String *Item_func_geometry_from_json::val_str(String *str)
String *js= args[0]->val_str_ascii(&tmp_js);
uint32 srid= 0;
longlong options= 0;
json_engine_t je;
if ((null_value= args[0]->null_value))
return 0;
......
......@@ -249,6 +249,10 @@ class Item_func_geometry_from_wkb: public Item_geometry_func
class Item_func_geometry_from_json: public Item_geometry_func
{
String tmp_js;
json_engine_t je;
MEM_ROOT current_mem_root;
int mem_root_inited;
bool check_arguments() const override
{
// TODO: check with Alexey, for better args[1] and args[2] type control
......@@ -256,19 +260,27 @@ class Item_func_geometry_from_json: public Item_geometry_func
check_argument_types_traditional_scalar(1, MY_MIN(3, arg_count));
}
public:
Item_func_geometry_from_json(THD *thd, Item *js): Item_geometry_func(thd, js) {}
Item_func_geometry_from_json(THD *thd, Item *js): Item_geometry_func(thd, js)
{ mem_root_inited= false; }
Item_func_geometry_from_json(THD *thd, Item *js, Item *opt):
Item_geometry_func(thd, js, opt) {}
Item_geometry_func(thd, js, opt) { mem_root_inited= false; }
Item_func_geometry_from_json(THD *thd, Item *js, Item *opt, Item *srid):
Item_geometry_func(thd, js, opt, srid) {}
Item_geometry_func(thd, js, opt, srid) { mem_root_inited= false; }
LEX_CSTRING func_name_cstring() const override
{
static LEX_CSTRING name= {STRING_WITH_LEN("st_geomfromgeojson") };
return name;
}
bool fix_length_and_dec(THD *thd) override;
String *val_str(String *) override;
Item *do_get_copy(THD *thd) const override
{ return get_item_copy<Item_func_geometry_from_json>(thd, this); }
void cleanup() override
{
if (mem_root_inited)
free_root(&current_mem_root, MYF(0));
Item_geometry_func ::cleanup();
}
};
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -59,7 +59,7 @@ uchar* get_key_name(const char *key_name, size_t *length,
}
void json_get_normalized_string(json_engine_t *je, String *res,
int *error)
int *error, MEM_ROOT *current_mem_root)
{
char *val_begin= (char*)je->value, *val_end= NULL;
String val("",0,je->s.cs);
......@@ -83,7 +83,7 @@ void json_get_normalized_string(json_engine_t *je, String *res,
je->value_type == JSON_VALUE_OBJECT)
{
if (json_normalize(&a_res, (const char*)val.ptr(),
val_end-val_begin, je->s.cs))
val_end-val_begin, je->s.cs, current_mem_root))
goto error;
}
else if(je->value_type == JSON_VALUE_STRING)
......
......@@ -26,5 +26,5 @@ bool json_assign_type(uint *curr_type, json_engine_t *je);
uchar* get_key_name(const char *key_name, size_t *length,
my_bool /* unused */);
void json_get_normalized_string(json_engine_t *je, String *res,
int *error);
int *error, MEM_ROOT *current_mem_root);
#endif
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment