|
|
|
|
@ -18,14 +18,15 @@ |
|
|
|
|
import io |
|
|
|
|
from datetime import datetime |
|
|
|
|
|
|
|
|
|
from minio import Minio |
|
|
|
|
from minio.select.options import (SelectObjectOptions, CSVInput, JSONInput, |
|
|
|
|
RequestProgress, InputSerialization, |
|
|
|
|
OutputSerialization, CSVOutput, JsonOutput) |
|
|
|
|
from minio.select.options import (CSVInput, CSVOutput, InputSerialization, |
|
|
|
|
JSONInput, OutputSerialization, |
|
|
|
|
RequestProgress, SelectObjectOptions) |
|
|
|
|
|
|
|
|
|
from utils import * |
|
|
|
|
from utils import generate_bucket_name, generate_object_name |
|
|
|
|
|
|
|
|
|
def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql_output, tests, log_output): |
|
|
|
|
|
|
|
|
|
def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, |
|
|
|
|
sql_output, tests, log_output): |
|
|
|
|
bucket_name = generate_bucket_name() |
|
|
|
|
object_name = generate_object_name() |
|
|
|
|
|
|
|
|
|
@ -51,7 +52,8 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql |
|
|
|
|
) |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
data = client.select_object_content(bucket_name, object_name, options) |
|
|
|
|
data = client.select_object_content( |
|
|
|
|
bucket_name, object_name, options) |
|
|
|
|
|
|
|
|
|
# Get the records |
|
|
|
|
records = io.BytesIO() |
|
|
|
|
@ -62,13 +64,15 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql |
|
|
|
|
if got_output != expected_output: |
|
|
|
|
if type(expected_output) == datetime: |
|
|
|
|
# Attempt to parse the date which will throw an exception for any issue |
|
|
|
|
datetime.strptime(got_output.decode("utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z') |
|
|
|
|
datetime.strptime(got_output.decode( |
|
|
|
|
"utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z') |
|
|
|
|
else: |
|
|
|
|
raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(idx+1, expected_output, got_output)) |
|
|
|
|
raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format( |
|
|
|
|
idx+1, expected_output, got_output)) |
|
|
|
|
|
|
|
|
|
log_output.args['total_success'] += 1 |
|
|
|
|
except Exception as err: |
|
|
|
|
continue ## TODO, raise instead |
|
|
|
|
continue # TODO, raise instead |
|
|
|
|
# raise Exception(err) |
|
|
|
|
finally: |
|
|
|
|
client.remove_object(bucket_name, object_name) |
|
|
|
|
@ -100,24 +104,31 @@ def test_sql_operators(client, log_output): |
|
|
|
|
tests = [ |
|
|
|
|
# Logical operators |
|
|
|
|
("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'), |
|
|
|
|
("NOT", "select * from S3Object s where NOT s.id = 1", b'2,Elliot,4\n3,Yves,5\n4,,0\n'), |
|
|
|
|
("OR", "select * from S3Object s where s.id = 1 OR s.id = 3", b'1,John,3\n3,Yves,5\n'), |
|
|
|
|
("NOT", "select * from S3Object s where NOT s.id = 1", |
|
|
|
|
b'2,Elliot,4\n3,Yves,5\n4,,0\n'), |
|
|
|
|
("OR", "select * from S3Object s where s.id = 1 OR s.id = 3", |
|
|
|
|
b'1,John,3\n3,Yves,5\n'), |
|
|
|
|
# Comparison Operators |
|
|
|
|
("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'), |
|
|
|
|
(">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'), |
|
|
|
|
("<=", "select * from S3Object s where s.age <= 4", b'1,John,3\n2,Elliot,4\n4,,0\n'), |
|
|
|
|
("<=", "select * from S3Object s where s.age <= 4", |
|
|
|
|
b'1,John,3\n2,Elliot,4\n4,,0\n'), |
|
|
|
|
(">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'), |
|
|
|
|
("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'), |
|
|
|
|
("<>", "select * from S3Object s where s.age <> 4", b'1,John,3\n3,Yves,5\n4,,0\n'), |
|
|
|
|
("!=", "select * from S3Object s where s.age != 4", b'1,John,3\n3,Yves,5\n4,,0\n'), |
|
|
|
|
("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5", b'2,Elliot,4\n3,Yves,5\n'), |
|
|
|
|
("<>", "select * from S3Object s where s.age <> 4", |
|
|
|
|
b'1,John,3\n3,Yves,5\n4,,0\n'), |
|
|
|
|
("!=", "select * from S3Object s where s.age != 4", |
|
|
|
|
b'1,John,3\n3,Yves,5\n4,,0\n'), |
|
|
|
|
("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5", |
|
|
|
|
b'2,Elliot,4\n3,Yves,5\n'), |
|
|
|
|
("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'), |
|
|
|
|
# Pattern Matching Operators |
|
|
|
|
("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'), |
|
|
|
|
("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'), |
|
|
|
|
# Unitary Operators |
|
|
|
|
("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'), |
|
|
|
|
("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL", b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'), |
|
|
|
|
("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL", |
|
|
|
|
b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'), |
|
|
|
|
# Math Operators |
|
|
|
|
("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'), |
|
|
|
|
("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'), |
|
|
|
|
@ -168,7 +179,6 @@ def test_sql_operators_precedence(client, log_output): |
|
|
|
|
print(log_output.json_report()) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_sql_functions_agg_cond_conv(client, log_output): |
|
|
|
|
|
|
|
|
|
json_testfile = """{"id": 1, "name": "John", "age": 3} |
|
|
|
|
@ -187,8 +197,9 @@ def test_sql_functions_agg_cond_conv(client, log_output): |
|
|
|
|
# Conditional functions |
|
|
|
|
("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'), |
|
|
|
|
("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'), |
|
|
|
|
## Conversion functions |
|
|
|
|
("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s", b'3.0\n4.0\n5.0\n\n0.0\n'), |
|
|
|
|
# Conversion functions |
|
|
|
|
("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s", |
|
|
|
|
b'3.0\n4.0\n5.0\n\n0.0\n'), |
|
|
|
|
|
|
|
|
|
] |
|
|
|
|
|
|
|
|
|
@ -211,12 +222,18 @@ def test_sql_functions_date(client, log_output): |
|
|
|
|
|
|
|
|
|
tests = [ |
|
|
|
|
# DATE_ADD |
|
|
|
|
("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2022-01-02T03:04:05.006+07:30\n'), |
|
|
|
|
("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-02-02T03:04:05.006+07:30\n'), |
|
|
|
|
("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-01T03:04:05.006+07:30\n'), |
|
|
|
|
("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T04:04:05.006+07:30\n'), |
|
|
|
|
("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:09:05.006+07:30\n'), |
|
|
|
|
("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:04:10.006+07:30\n'), |
|
|
|
|
("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", |
|
|
|
|
b'2022-01-02T03:04:05.006+07:30\n'), |
|
|
|
|
("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", |
|
|
|
|
b'2017-02-02T03:04:05.006+07:30\n'), |
|
|
|
|
("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s", |
|
|
|
|
b'2017-01-01T03:04:05.006+07:30\n'), |
|
|
|
|
("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", |
|
|
|
|
b'2017-01-02T04:04:05.006+07:30\n'), |
|
|
|
|
("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", |
|
|
|
|
b'2017-01-02T03:09:05.006+07:30\n'), |
|
|
|
|
("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", |
|
|
|
|
b'2017-01-02T03:04:10.006+07:30\n'), |
|
|
|
|
# DATE_DIFF |
|
|
|
|
("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'), |
|
|
|
|
("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'), |
|
|
|
|
@ -229,16 +246,23 @@ def test_sql_functions_date(client, log_output): |
|
|
|
|
("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'), |
|
|
|
|
("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'), |
|
|
|
|
# TO_STRING |
|
|
|
|
("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s", b'"January 2, 2017"\n'), |
|
|
|
|
("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s", |
|
|
|
|
b'"January 2, 2017"\n'), |
|
|
|
|
("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'), |
|
|
|
|
("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'), |
|
|
|
|
("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'), |
|
|
|
|
("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s", b'"January 2, 2017 3:4 AM"\n'), |
|
|
|
|
("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), |
|
|
|
|
("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), |
|
|
|
|
("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), |
|
|
|
|
("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s", b'2017-01-02T3:4:05+07:30\n'), |
|
|
|
|
("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s", b'2017-01-02T03:04:05.006+07:30\n'), |
|
|
|
|
("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s", |
|
|
|
|
b'"January 2, 2017 3:4 AM"\n'), |
|
|
|
|
("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", |
|
|
|
|
b'2017-01-02T3:4:05+0730\n'), |
|
|
|
|
("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", |
|
|
|
|
b'2017-01-02T3:4:05+0730\n'), |
|
|
|
|
("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s", |
|
|
|
|
b'2017-01-02T3:4:05+0730\n'), |
|
|
|
|
("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s", |
|
|
|
|
b'2017-01-02T3:4:05+07:30\n'), |
|
|
|
|
("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s", |
|
|
|
|
b'2017-01-02T03:04:05.006+07:30\n'), |
|
|
|
|
("UTCNOW", "select UTCNOW() from S3Object", datetime(1, 1, 1)), |
|
|
|
|
|
|
|
|
|
] |
|
|
|
|
@ -253,6 +277,7 @@ def test_sql_functions_date(client, log_output): |
|
|
|
|
# Test passes |
|
|
|
|
print(log_output.json_report()) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_sql_functions_string(client, log_output): |
|
|
|
|
|
|
|
|
|
json_testfile = """ |
|
|
|
|
@ -264,7 +289,8 @@ def test_sql_functions_string(client, log_output): |
|
|
|
|
tests = [ |
|
|
|
|
# CHAR_LENGTH |
|
|
|
|
("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), |
|
|
|
|
("CHARACTER_LENGTH", "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), |
|
|
|
|
("CHARACTER_LENGTH", |
|
|
|
|
"select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), |
|
|
|
|
# LOWER |
|
|
|
|
("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'), |
|
|
|
|
# SUBSTRING |
|
|
|
|
@ -273,8 +299,10 @@ def test_sql_functions_string(client, log_output): |
|
|
|
|
("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'), |
|
|
|
|
# TRIM |
|
|
|
|
("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'), |
|
|
|
|
("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t \n'), |
|
|
|
|
("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2", b' \tfoobar\t\n'), |
|
|
|
|
("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2", |
|
|
|
|
b'\tfoobar\t \n'), |
|
|
|
|
("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2", |
|
|
|
|
b' \tfoobar\t\n'), |
|
|
|
|
("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'), |
|
|
|
|
("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'), |
|
|
|
|
# UPPER |
|
|
|
|
@ -304,7 +332,8 @@ def test_sql_datatypes(client, log_output): |
|
|
|
|
("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'), |
|
|
|
|
("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'), |
|
|
|
|
("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'), |
|
|
|
|
("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object", b'2007-04-05T14:30Z\n'), |
|
|
|
|
("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object", |
|
|
|
|
b'2007-04-05T14:30Z\n'), |
|
|
|
|
] |
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
@ -323,9 +352,12 @@ def test_sql_select(client, log_output): |
|
|
|
|
json_testfile = """{"id": 1, "created": "June 27", "modified": "July 6" } |
|
|
|
|
{"id": 2, "Created": "June 28", "Modified": "July 7", "Cast": "Random Date" }""" |
|
|
|
|
tests = [ |
|
|
|
|
("select_1", "select * from S3Object", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), |
|
|
|
|
("select_2", "select * from S3Object s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), |
|
|
|
|
("select_3", "select * from S3Object as s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), |
|
|
|
|
("select_1", "select * from S3Object", |
|
|
|
|
b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), |
|
|
|
|
("select_2", "select * from S3Object s", |
|
|
|
|
b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), |
|
|
|
|
("select_3", "select * from S3Object as s", |
|
|
|
|
b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), |
|
|
|
|
("select_4", "select s.line from S3Object as s", b'\n\n'), |
|
|
|
|
("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'), |
|
|
|
|
("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'), |
|
|
|
|
@ -343,16 +375,22 @@ def test_sql_select(client, log_output): |
|
|
|
|
# Test passes |
|
|
|
|
print(log_output.json_report()) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_sql_select_json(client, log_output): |
|
|
|
|
json_testcontent = """{ "Rules": [ {"id": "1"}, {"expr": "y > x"}, {"id": "2", "expr": "z = DEBUG"} ]} |
|
|
|
|
{ "created": "June 27", "modified": "July 6" } |
|
|
|
|
""" |
|
|
|
|
tests = [ |
|
|
|
|
("select_1", "SELECT id FROM S3Object[*].Rules[*].id", b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'), |
|
|
|
|
("select_2", "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'), |
|
|
|
|
("select_3", "SELECT d.created, d.modified FROM S3Object[*] d", b'{}\n{"created":"June 27","modified":"July 6"}\n'), |
|
|
|
|
("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]", b'{}\n{"created":"June 27","modified":"July 6"}\n'), |
|
|
|
|
("select_5", "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'), |
|
|
|
|
("select_1", "SELECT id FROM S3Object[*].Rules[*].id", |
|
|
|
|
b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'), |
|
|
|
|
("select_2", |
|
|
|
|
"SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'), |
|
|
|
|
("select_3", "SELECT d.created, d.modified FROM S3Object[*] d", |
|
|
|
|
b'{}\n{"created":"June 27","modified":"July 6"}\n'), |
|
|
|
|
("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]", |
|
|
|
|
b'{}\n{"created":"June 27","modified":"July 6"}\n'), |
|
|
|
|
("select_5", |
|
|
|
|
"Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'), |
|
|
|
|
] |
|
|
|
|
|
|
|
|
|
input_serialization = InputSerialization(json=JSONInput(Type="DOCUMENT")) |
|
|
|
|
@ -395,5 +433,3 @@ val4,val5,val6 |
|
|
|
|
|
|
|
|
|
# Test passes |
|
|
|
|
print(log_output.json_report()) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Harshavardhana
4 years ago
committed by