1 """
2 This module includes helper variables and methods for the unittests.
3 """
4
5 import random
6 from sys_appends import *
7
8 import sys
9
10 sys.path.append(b + 'src/vsql_core/utils/factored_parser_udf')
11
12 from tuple_processor import *
13
14
15 limit_test = 200
16
17
18 tuple_size = 1000
19
20 test_num = tuple_size
21
22 vec_size = 100
23
24
25 row_num = 30
26
27 col_num = 50
28
29
30 type_non_array = 0
31
32 type_1d_array = 1
33
34 type_2d_array = 2
35
36
37 with_sql = 1
38
39 without_sql = 0
40
41 size_int = 4
42 struct_int4 = struct.Struct('>i')
43
44 temp_table_name = 'temp_tbl_for_test'
45 temp_file_name = "temp_file.dat"
46
47
48 exact_equal = 0
49
50 almost_equal = 1
51 almost_eq_places = 6
52
53
54 enumerate_fnc = lambda x: [[y for j, y in enumerate(set(x)) if (i >> j) & 1] for i in range(2**len(set(x)))]
55
57 """
58 Compares non-array data type
59 @type cls: class
60 @param cls: unittest class for assertEqual functions
61 @type data1: data
62 @param data1: data1 to compare
63 @type data2: data
64 @param data2: data2 to compare
65 @type method_name: string
66 @param method_name: name of the method
67 @type is_almost: bool
68 @param is_almost: whether we want exact comparison or almost comparison
69 """
70 if is_almost:
71 cls.assertAlmostEqual(data1, data2, almost_eq_places, method_name + ': tuple content is different')
72 else:
73 cls.assertEqual(data1, data2, method_name + ': tuple content is different')
74
76 """
77 Compares 1d array data type
78 @type cls: class
79 @param cls: unittest class for assertEqual functions
80 @type array1: array
81 @param array1: array1 to compare
82 @type array2: array
83 @param array2: array2 to compare
84 @type method_name: string
85 @param method_name: name of the method
86 @type is_almost: bool
87 @param is_almost: whether we want exact comparison or almost comparison
88 """
89 cls.assertEqual(len(array1), len(array2), method_name + ': array sizes are different')
90 for i in range(len(array1)):
91 compare_non_array_data(cls, array1[i], array2[i], method_name, is_almost)
92
93
95 """
96 Compares 2d array data type
97 @type cls: class
98 @param cls: unittest class for assertEqual functions
99 @type array1: array
100 @param array1: array1 to compare
101 @type array2: array
102 @param array2: array2 to compare
103 @type method_name: string
104 @param method_name: name of the method
105 @type is_almost: bool
106 @param is_almost: whether we want exact comparison or almost comparison
107 """
108 cls.assertEqual(len(array1), len(array2), method_name + ': row sizes are different')
109 for i in range(len(array1)):
110 compare_array_data(cls, array1[i], array2[i], method_name, is_almost)
111
112 -def compare_data(is_array_type, cls, rand_data, ret_data, method_name, is_almost):
113 """
114 Compares data type(non-array, 1d or 2d array)
115 @type is_array_type: number
116 @param is_array_type: whether data is non-array, 1d or 2d array
117 @type cls: class
118 @param cls: unittest class for assertEqual functions
119 @type rand_data: data
120 @param rand_data: data to compare
121 @type ret_data: data
122 @param ret_data: data to compare
123 @type method_name: string
124 @param method_name: name of the method
125 @type is_almost: bool
126 @param is_almost: whether we want exact comparison or almost comparison
127 """
128
129 if is_array_type == type_2d_array:
130 compare_2d_array_data(cls, rand_data, ret_data, method_name, is_almost)
131 elif is_array_type == type_1d_array:
132 compare_array_data(cls, rand_data, ret_data, method_name, is_almost)
133 elif is_array_type == type_non_array:
134 compare_non_array_data(cls, rand_data, ret_data, method_name, is_almost)
135
136 -def compare_tuples(cls, type_params, rand_tuple, ret_tuple, method_name, is_almost):
137 """
138 Compares tuples that can contain one or more data types
139 @type cls: class
140 @param cls: unittest class for assertEqual functions
141 @type type_params: tuple
142 @param type_params: contains information about the data types in the tuple
143 @type rand_tuple: array
144 @param rand_tuple: array to compare
145 @type ret_tuple: array
146 @param ret_tuple: array to compare
147 @type method_name: string
148 @param method_name: name of the method
149 @type is_almost: bool
150 @param is_almost: whether we want exact comparison or almost comparison
151 """
152 cls.assertEqual(len(rand_tuple), len(ret_tuple), 'tuple sizes are different')
153 for i in range(len(rand_tuple)):
154 compare_data(type_params[i][0], cls, rand_tuple[i], ret_tuple[i], method_name,
155 is_almost and (type_params[i][3] == float8oid))
156
158 """
159 Compares set of tuples that can contain one or more tuples that can contain one or more data types
160 @type cls: class
161 @param cls: unittest class for assertEqual functions
162 @type type_params: tuple
163 @param type_params: contains information about the data types in the tuple
164 @type tuple_set1: array
165 @param tuple_set1: array to compare
166 @type tuple_set2: array
167 @param tuple_set2: array to compare
168 @type method_name: string
169 @param method_name: name of the method
170 @type is_almost: bool
171 @param is_almost: whether we want exact comparison or almost comparison
172 """
173 cls.assertEqual(len(tuple_set1), len(tuple_set2), method_name + '# of tuples differs')
174 for i in range(len(tuple_set1)):
175 compare_tuples(cls, type_params, tuple_set1[i], tuple_set2[i], method_name, is_almost)
176
178 """
179 Converts char to sql representation
180 @type v: char
181 @param v: character
182 @rtype: string
183 @return sql representation of the given type
184 """
185 return '\'' + v + '\''
186
188 """
189 Converts bool and numeric types to sql representation
190 @type v: bool or a numeric type
191 @param v: bool or a numeric type
192 @rtype: string
193 @return sql representation of the given type
194 """
195 return repr(v)
196
198 """
199 Converts char array to sql representation
200 @type v: char array
201 @param v: character array
202 @rtype: string
203 @return sql representation of the given type
204 """
205 str = '\'{'
206 str += ','.join(v)
207 str += '}\''
208 return str
209
211 """
212 Converts bool and numeric array types to sql representation
213 @type v: bool or a numeric array type
214 @param v: bool or a numeric array type
215 @rtype: string
216 @return sql representation of the given type
217 """
218 return '\'' + convert_bool_and_numeric_array_without_quotes(v) + '\''
219
221 """
222 Converts bool and numeric array types to sql representation without quotes
223 @type v: bool or a numeric array type
224 @param v: bool or a numeric array type
225 @rtype: string
226 @return sql representation of the given type
227 """
228 return '{' + ','.join([repr(el) for el in v]) + '}'
229
231 """
232 Converts bool and numeric 2d array types to sql representation
233 @type v: bool or a numeric 2d array type
234 @param v: bool or a numeric 2d array type
235 @rtype: string
236 @return sql representation of the given type
237 """
238 str = '\'{'
239 str += ','.join([convert_bool_and_numeric_array_without_quotes(el) for el in v])
240 str += '}\''
241 return str
242
244 """
245 Writes non-array data to the buffer
246 @type b: buffer
247 @param b: buffer that the data will be written
248 @type data: data
249 @param data: non-array type data
250 @type data_size: number
251 @param data_size: size of the data
252 @type pack_fnc: function
253 @param pack_fnc: pack function for the data
254 """
255
256 b.write(struct_int4.pack(data_size))
257
258 b.write(pack_fnc(data))
259
261 """
262 Writes 1d array data to the buffer
263 @type b: buffer
264 @param b: buffer that the data will be written
265 @type data_array: data
266 @param data_array: 1d array type data
267 @type elm_size: number
268 @param elm_size: size of the data
269 @type elm_type: number
270 @param elm_type: oid of the element
271 @type pack_fnc: function
272 @param pack_fnc: pack function for the data
273 """
274
275 b.write(struct_int4.pack(size_int * 5 + len(data_array) * (size_int + elm_size)))
276 b.write(struct_int4.pack(1))
277 b.write(struct_int4.pack(0))
278 b.write(struct_int4.pack(elm_type))
279
280 b.write(struct_int4.pack(len(data_array)))
281 b.write(struct_int4.pack(1))
282
283 for val in data_array:
284 b.write(struct_int4.pack(elm_size))
285 b.write(pack_fnc(val))
286
288 """
289 Writes 2d array data to the buffer
290 @type b: buffer
291 @param b: buffer that the data will be written
292 @type data_array: data
293 @param data_array: 2d array type data
294 @type elm_size: number
295 @param elm_size: size of the data
296 @type elm_type: number
297 @param elm_type: oid of the element
298 @type pack_fnc: function
299 @param pack_fnc: pack function for the data
300 """
301
302 b.write(struct_int4.pack(size_int * (3 + 4) + len(data_array) * len(data_array[0]) * (size_int + elm_size)))
303 b.write(struct_int4.pack(2))
304 b.write(struct_int4.pack(0))
305 b.write(struct_int4.pack(elm_type))
306
307 b.write(struct_int4.pack(len(data_array)))
308 b.write(struct_int4.pack(1))
309 b.write(struct_int4.pack(len(data_array[0])))
310 b.write(struct_int4.pack(1))
311
312 for x in data_array:
313 for val in x:
314 b.write(struct_int4.pack(elm_size))
315 b.write(pack_fnc(val))
316
317 -def write_rand_data(is_array_type, b, rand_data, elm_size, elm_type, pack_fnc):
318 """
319 Writes data(non-array, 1d or 2d array) to the buffer
320 @type is_array_type: number
321 @param is_array_type: whether data is non-array, 1d or 2d array
322 @type b: buffer
323 @param b: buffer that the data will be written
324 @type rand_data: data
325 @param rand_data: non-array, 1d or 2d array type data
326 @type elm_size: number
327 @param elm_size: size of the data
328 @type elm_type: number
329 @param elm_type: oid of the element
330 @type pack_fnc: function
331 @param pack_fnc: pack function for the data
332 """
333 if is_array_type == type_2d_array:
334 write_2d_array_data(b, rand_data, elm_size, elm_type, pack_fnc)
335 elif is_array_type == type_1d_array:
336 write_array_data(b, rand_data, elm_size, elm_type, pack_fnc)
337 elif is_array_type == type_non_array:
338 write_non_array_data(b, rand_data, elm_size, pack_fnc)
339
341 """
342 Creates random data(non-array, 1d or 2d array)
343 @type is_array_type: number
344 @param is_array_type: whether data is non-array, 1d or 2d array
345 @type random_fnc: function
346 @param random_fnc: random function to create a random element
347 @rtype: data
348 @return: returns random data(non-array, 1d or 2d array)
349 """
350
351 if is_array_type == type_2d_array:
352 rand_data = create_2d_generic_array(random_fnc)
353 elif is_array_type == type_1d_array:
354 rand_data = create_1d_generic_array(random_fnc)
355 elif is_array_type == type_non_array:
356 rand_data = random_fnc()
357
358 return rand_data
359
361 """
362 Creates random char
363 @rtype: char
364 @return: random char
365 """
366 rand = random.randint(0,35)
367 ch = '0'
368 if(rand < 10):
369 ch = str(rand)
370 else:
371 ch = chr(rand + 55)
372 return ch
373
375 """
376 Creates random bool
377 @rtype: bool
378 @return: random bool
379 """
380 rand = random.randint(0,1)
381 if rand == 1:
382 return True
383 else:
384 return False
385
387 """
388 Creates random int2
389 @rtype: int2
390 @return: random int2
391 """
392 return random.randint(-2 ** 15, 2 ** 15 - 1)
393
395 """
396 Creates random int4
397 @rtype: int4
398 @return: random int4
399 """
400 return random.randint(-2 ** 31, 2 ** 31 - 1)
401
403 """
404 Creates random int8
405 @rtype: int8
406 @return: random int8
407 """
408 return random.randint(-2 ** 63, 2 ** 63 - 1)
409
410
412 """
413 Creates random float4
414 @rtype: float4
415 @return: random float4
416 """
417 return 3.5
418
419
421 """
422 Creates random float8
423 @rtype: float8
424 @return: random float8
425 """
426 return random.uniform(0, 2000000)
427
429 """
430 Creates 1d array
431 @type create_random_elm_fnc: function
432 @param create_random_elm_fnc: function to create random elements of the array
433 @rtype: array
434 @return: random 1d array with the given type
435 """
436 ret = []
437 for i in range(vec_size):
438 ret.append(create_random_elm_fnc())
439
440 return ret
441
443 """
444 Creates 2d array
445 @type create_random_elm_fnc: function
446 @param create_random_elm_fnc: function to create random elements of the array
447 @rtype: array
448 @return: random 2d array with the given type
449 """
450 ret = []
451 for i in range(row_num):
452 row_array = []
453 for j in range(col_num):
454 row_array.append(create_random_elm_fnc())
455 ret.append(row_array)
456
457 return ret
458
460 """
461 Includes a function(to convert data to sql type) and sql type of data
462 to the given type param
463 @type t: tuple
464 @param t: type param
465 @type convert_fnc: function
466 @param convert_fnc: function to convert data to the sql representation
467 @type sql_type: string
468 @param sql_type: data type in sql (e.g. 'char' for char, 'float8[]' for float8 array)
469 @rtype: tuple
470 @return: new type param that includes convert function and sql type
471 """
472 new_t = []
473 for elm in t:
474 new_t.append(elm)
475 new_t.append(convert_fnc)
476 new_t.append(sql_type)
477 return new_t
478
479
480 char_type_param = (type_non_array, oid_dict[charoid][data_size_field], create_random_char, charoid, struct_char.pack)
481 bool_type_param = (type_non_array, oid_dict[booloid][data_size_field], create_random_bool, booloid, struct_bool.pack)
482 int2_type_param = (type_non_array, oid_dict[int2oid][data_size_field], create_random_int2, int2oid, struct_int2.pack)
483 int4_type_param = (type_non_array, oid_dict[int4oid][data_size_field], create_random_int4, int4oid, struct_int4.pack)
484 int8_type_param = (type_non_array, oid_dict[int8oid][data_size_field], create_random_int8, int8oid, struct_int8.pack)
485 float4_type_param = (type_non_array, oid_dict[float4oid][data_size_field], create_random_float4, float4oid, struct_float4.pack)
486 float8_type_param = (type_non_array, oid_dict[float8oid][data_size_field], create_random_float8, float8oid, struct_float8.pack)
487 char_array_type_param = (type_1d_array, oid_dict[charoid][data_size_field], create_random_char, charoid, struct_char.pack)
488 bool_array_type_param = (type_1d_array, oid_dict[booloid][data_size_field], create_random_bool, booloid, struct_bool.pack)
489 int2_array_type_param = (type_1d_array, oid_dict[int2oid][data_size_field], create_random_int2, int2oid, struct_int2.pack)
490 int4_array_type_param = (type_1d_array, oid_dict[int4oid][data_size_field], create_random_int4, int4oid, struct_int4.pack)
491 int8_array_type_param = (type_1d_array, oid_dict[int8oid][data_size_field], create_random_int8, int8oid, struct_int8.pack)
492 float4_array_type_param = (type_1d_array, oid_dict[float4oid][data_size_field], create_random_float4, float4oid, struct_float4.pack)
493 float8_array_type_param = (type_1d_array, oid_dict[float8oid][data_size_field], create_random_float8, float8oid, struct_float8.pack)
494 float8_2d_array_type_param = (type_2d_array, oid_dict[float8oid][data_size_field], create_random_float8, float8oid, struct_float8.pack)
495
496
497 char_vdb_type_param = get_vdb_tuple(char_type_param, convert_char, 'char')
498 bool_vdb_type_param = get_vdb_tuple(bool_type_param, convert_bool_and_numeric, 'bool')
499 int2_vdb_type_param = get_vdb_tuple(int2_type_param, convert_bool_and_numeric, 'int2')
500 int4_vdb_type_param = get_vdb_tuple(int4_type_param, convert_bool_and_numeric, 'int4')
501 int8_vdb_type_param = get_vdb_tuple(int8_type_param, convert_bool_and_numeric, 'int8')
502 float4_vdb_type_param = get_vdb_tuple(float4_type_param, convert_bool_and_numeric, 'float4')
503 float8_vdb_type_param = get_vdb_tuple(float8_type_param, convert_bool_and_numeric, 'float8')
504 bool_array_vdb_type_param = get_vdb_tuple(bool_array_type_param, convert_bool_and_numeric_array, 'bool[]')
505 int2_array_vdb_type_param = get_vdb_tuple(int2_array_type_param, convert_bool_and_numeric_array, 'int2[]')
506 int4_array_vdb_type_param = get_vdb_tuple(int4_array_type_param, convert_bool_and_numeric_array, 'int4[]')
507 int8_array_vdb_type_param = get_vdb_tuple(int8_array_type_param, convert_bool_and_numeric_array, 'int8[]')
508 float4_array_vdb_type_param = get_vdb_tuple(float4_array_type_param, convert_bool_and_numeric_array, 'float4[]')
509 float8_array_vdb_type_param = get_vdb_tuple(float8_array_type_param, convert_bool_and_numeric_array, 'float8[]')
510 float8_2d_array_vdb_type_param = get_vdb_tuple(float8_2d_array_type_param, convert_bool_and_numeric_2d_array, 'float8[][]')
511
513 """
514 This method enumerates all subsets of the given types and creates all combination
515 of different schemas (e.g. [char, int2] or [char, int4, float8array]).
516 Then, for each combination, test method(test_fnc) is called.
517 # of tests is: 2 ** len(type_params) - 1
518 @type test_fnc: function
519 @param test_fnc: name of the generic test function
520 @type type_params: tuple
521 @param type_params: contains information(is_array_type, elm_size, random_fnc, elm_type, pack_fnc)
522 about each data type in the tuple.
523 @type oids: tuple
524 @param oids: oids of the data types in the tuple
525 @type limit_test: number
526 @param limit_test: when enumerations are too many to execute, limit_test is used to limit the number of enumerations
527 """
528
529 assert(len(type_params) == len(oids))
530 enumerated_list = enumerate_fnc(xrange(len(type_params)))
531
532 for i in range(len(enumerated_list)):
533 if(len(enumerated_list[i]) != 0 and i < limit_test):
534 param_array_for_fnc = []
535 oid_array_for_fnc = []
536 method_name = ''
537 for j in range(len(enumerated_list[i])):
538 param_array_for_fnc.append(type_params[enumerated_list[i][j]])
539 oid_of_elm = oids[enumerated_list[i][j]]
540 oid_array_for_fnc.append(oid_of_elm)
541 method_name += '_' + oid_dict[oid_of_elm][0]
542
543 test_fnc(oid_array_for_fnc, param_array_for_fnc, method_name)
544
546 """
547 This method tests tuples with possible non-array data type combinations by calling test_fnc.
548 Since there are 7 non-array data types, all combinations produce 127 different schema.
549 Therefore, this method includes min(127, limit_test) tests
550 @type test_fnc: function
551 @param test_fnc: name of the generic test function
552 @type limit_test: number
553 @param limit_test: when enumerations are too many to execute, limit_test is used to limit the number of enumerations
554 @type sql_interaction: bool
555 @param sql_interaction: whether test involves postgres or not
556 """
557 if sql_interaction:
558 type_params = [char_vdb_type_param, bool_vdb_type_param, int2_vdb_type_param,
559 int4_vdb_type_param, int8_vdb_type_param, float4_vdb_type_param, float8_vdb_type_param]
560 else:
561 type_params = [char_type_param, bool_type_param, int2_type_param, int4_type_param, int8_type_param,
562 float4_type_param, float8_type_param]
563
564 oids = [charoid, booloid, int2oid, int4oid, int8oid, float4oid, float8oid]
565 enumerate_and_test_tuples(test_fnc, type_params, oids, limit_test)
566
568 """
569 This method tests tuples with possible array data type combinations by calling test_fnc.
570 Since there are 8 array data types(for the ones with sql interaction, we dont use char arrays, since they are
571 interpreted as strings, so it is 7), all combinations produce 255(127) different schema.
572 Therefore, this method includes min(255(127), limit_test) tests
573 @type test_fnc: function
574 @param test_fnc: name of the generic test function
575 @type limit_test: number
576 @param limit_test: when enumerations are too many to execute, limit_test is used to limit the number of enumerations
577 @type sql_interaction: bool
578 @param sql_interaction: whether test involves postgres or not
579 """
580 if sql_interaction:
581 type_params = [bool_array_vdb_type_param, int2_array_vdb_type_param,
582 int4_array_vdb_type_param, int8_array_vdb_type_param,
583 float4_array_vdb_type_param, float8_array_vdb_type_param,
584 float8_2d_array_vdb_type_param]
585 oids = [boolarrayoid, int2arrayoid, int4arrayoid,
586 int8arrayoid, float4arrayoid, float8arrayoid, float82darrayoid]
587 else:
588 type_params = [char_array_type_param, bool_array_type_param, int2_array_type_param,
589 int4_array_type_param, int8_array_type_param, float4_array_type_param,
590 float8_array_type_param, float8_2d_array_type_param]
591 oids = [chararrayoid, boolarrayoid, int2arrayoid, int4arrayoid,
592 int8arrayoid, float4arrayoid, float8arrayoid, float82darrayoid]
593
594 enumerate_and_test_tuples(test_fnc, type_params, oids, limit_test)
595
597 """
598 This method tests tuples with possible non-array and array data type combinations by calling test_fnc.
599 Since there are 7 non-array and 8 array data types(for the ones with sql interaction, we dont use char arrays,
600 since they are interpreted as strings, so it is 7), all combinations produce 2 ** 15 - 1(2 ** 14 - 1) different schema.
601 Therefore, this method includes min(2 ** 15 - 1(2 ** 14 - 1), limit_test) tests
602 @type test_fnc: function
603 @param test_fnc: name of the generic test function
604 @type limit_test: number
605 @param limit_test: when enumerations are too many to execute, limit_test is used to limit the number of enumerations
606 @type sql_interaction: bool
607 @param sql_interaction: whether test involves postgres or not
608 """
609 if sql_interaction:
610 type_params = [char_vdb_type_param, bool_vdb_type_param, int2_vdb_type_param,
611 int4_vdb_type_param, int8_vdb_type_param, float4_vdb_type_param,
612 float8_vdb_type_param, bool_array_vdb_type_param,
613 int2_array_vdb_type_param, int4_array_vdb_type_param,
614 int8_array_vdb_type_param, float4_array_vdb_type_param,
615 float8_array_vdb_type_param, float8_2d_array_vdb_type_param]
616 oids = [charoid, booloid, int2oid, int4oid, int8oid, float4oid, float8oid,
617 boolarrayoid, int2arrayoid, int4arrayoid, int8arrayoid, float4arrayoid,
618 float8arrayoid, float82darrayoid]
619 else:
620 type_params = [char_type_param, bool_type_param, int2_type_param, int4_type_param,
621 int8_type_param, float4_type_param, float8_type_param,
622 char_array_type_param, bool_array_type_param, int2_array_type_param,
623 int4_array_type_param, int8_array_type_param, float4_array_type_param,
624 float8_array_type_param, float8_2d_array_type_param]
625 oids = [charoid, booloid, int2oid, int4oid, int8oid, float4oid, float8oid,
626 chararrayoid, boolarrayoid, int2arrayoid, int4arrayoid,
627 int8arrayoid, float4arrayoid, float8arrayoid, float82darrayoid]
628
629 enumerate_and_test_tuples(test_fnc, type_params, oids, limit_test)
630
632 """
633 From given type params, this method creates "create table query statement"
634 @type type_params: array
635 @param type_params: type param for each data type in the query
636 @rtype: string
637 @return: create table statement
638 """
639 create_table_query = 'DROP TABLE IF EXISTS ' + temp_table_name + ';'
640 create_table_query += 'CREATE TABLE ' + temp_table_name + '('
641 create_table_query += ','.join('f' + str(i) + ' ' + type_params[i][6] for i in range(len(type_params)))
642
643 create_table_query += ');'
644
645 return create_table_query
646