oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fix: s3select tests with new minio-py SDK (#10995)

Browse Source
master
Harshavardhana 4 years ago committed by GitHub
parent bdd094bc39
commit ae4ded7fd1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 64 additions and 68 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      mint/run/core/minio-go/go.mod
  2. 82
      mint/run/core/s3select/csv.py
  3. 47
      mint/run/core/s3select/sql_ops.py
  4. 1
      mint/run/core/s3select/tests.py

2
mint/run/core/minio-go/go.mod
Unescape View File

@ -1,3 +1,5 @@
module mint.minio.io/minio-go module mint.minio.io/minio-go
go 1.14 go 1.14
require github.com/minio/minio-go/v7 v7.0.6 // indirect

82
mint/run/core/s3select/csv.py
Unescape View File

@ -19,9 +19,13 @@ import io
import os import os
from minio import Minio from minio import Minio
from minio.select.options import (CSVInput, CSVOutput, InputSerialization, from minio.selectrequest import (COMPRESSION_TYPE_NONE, FILE_HEADER_INFO_NONE,
JSONOutput, OutputSerialization, JSON_TYPE_DOCUMENT, QUOTE_FIELDS_ALWAYS,
RequestProgress, SelectObjectOptions) QUOTE_FIELDS_ASNEEDED, CSVInputSerialization,
CSVOutputSerialization,
JSONInputSerialization,
JSONOutputSerialization, SelectRequest)
from utils import * from utils import *
@ -93,26 +97,22 @@ def test_csv_input_custom_quote_char(client, log_output):
try: try:
for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests): for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests):
sql_opts = SelectObjectOptions( sql_opts = SelectRequest(
expression="select * from s3object", "select * from s3object",
input_serialization=InputSerialization( CSVInputSerialization(
compression_type="NONE", compression_type=COMPRESSION_TYPE_NONE,
csv=CSVInput(file_header_info="NONE", file_header_info=FILE_HEADER_INFO_NONE,
record_delimiter="\n", record_delimiter="\n",
field_delimiter=",", field_delimiter=",",
quote_character=quote_char, quote_character=quote_char,
quote_escape_character=escape_char, quote_escape_character=escape_char,
comments="#", comments="#",
allow_quoted_record_delimiter="FALSE",), allow_quoted_record_delimiter="FALSE",
), ),
output_serialization=OutputSerialization( JSONOutputSerialization(
json=JSONOutput( record_delimiter="\n",
record_delimiter="\n",
)
), ),
request_progress=RequestProgress( request_progress=False,
enabled="False"
)
) )
test_sql_api(f'test_{idx}', client, bucket_name, test_sql_api(f'test_{idx}', client, bucket_name,
@ -150,30 +150,26 @@ def test_csv_output_custom_quote_char(client, log_output):
try: try:
for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests): for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests):
sql_opts = SelectObjectOptions( sql_opts = SelectRequest(
expression="select * from s3object", "select * from s3object",
input_serialization=InputSerialization( CSVInputSerialization(
compression_type="NONE", compression_type=COMPRESSION_TYPE_NONE,
csv=CSVInput(file_header_info="NONE", file_header_info=FILE_HEADER_INFO_NONE,
record_delimiter="\n", record_delimiter="\n",
field_delimiter=",", field_delimiter=",",
quote_character='"', quote_character='"',
quote_escape_character='"', quote_escape_character='"',
comments="#", comments="#",
allow_quoted_record_delimiter="FALSE", allow_quoted_record_delimiter="FALSE",
),
), ),
output_serialization=OutputSerialization( CSVOutputSerialization(
csv=CSVOutput(quote_fields="ALWAYS", quote_fields=QUOTE_FIELDS_ALWAYS,
record_delimiter="\n", record_delimiter="\n",
field_delimiter=",", field_delimiter=",",
quote_character=quote_char, quote_character=quote_char,
quote_escape_character=escape_char, quote_escape_character=escape_char,
)
), ),
request_progress=RequestProgress( request_progress=False,
enabled="False"
)
) )
test_sql_api(f'test_{idx}', client, bucket_name, test_sql_api(f'test_{idx}', client, bucket_name,

47
mint/run/core/s3select/sql_ops.py
Unescape View File

@ -18,9 +18,12 @@
import io import io
from datetime import datetime from datetime import datetime
from minio.select.options import (CSVInput, CSVOutput, InputSerialization, from minio.selectrequest import (FILE_HEADER_INFO_NONE, JSON_TYPE_DOCUMENT,
JSONInput, JSONOutput, OutputSerialization, QUOTE_FIELDS_ASNEEDED, CSVInputSerialization,
RequestProgress, SelectObjectOptions) CSVOutputSerialization,
JSONInputSerialization,
JSONOutputSerialization, SelectRequest)
from utils import generate_bucket_name, generate_object_name from utils import generate_bucket_name, generate_object_name
@ -42,17 +45,15 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input,
continue continue
try: try:
log_output.args['total_tests'] += 1 log_output.args['total_tests'] += 1
options = SelectObjectOptions( sreq = SelectRequest(
expression=select_expression, select_expression,
input_serialization=sql_input, sql_input,
output_serialization=sql_output, sql_output,
request_progress=RequestProgress( request_progress=False
enabled="False"
)
) )
data = client.select_object_content( data = client.select_object_content(
bucket_name, object_name, options) bucket_name, object_name, sreq)
# Get the records # Get the records
records = io.BytesIO() records = io.BytesIO()
@ -79,14 +80,13 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input,
def test_sql_expressions(client, input_json_bytes, tests, log_output): def test_sql_expressions(client, input_json_bytes, tests, log_output):
input_serialization = InputSerialization( input_serialization = JSONInputSerialization(
compression_type="NONE", compression_type="NONE",
json=JSONInput(json_type="DOCUMENT"), json_type=JSON_TYPE_DOCUMENT,
) )
output_serialization = OutputSerialization( output_serialization = CSVOutputSerialization(
csv=CSVOutput(quote_fields="ASNEEDED") quote_fields=QUOTE_FIELDS_ASNEEDED)
)
test_sql_expressions_custom_input_output(client, input_json_bytes, test_sql_expressions_custom_input_output(client, input_json_bytes,
input_serialization, output_serialization, tests, log_output) input_serialization, output_serialization, tests, log_output)
@ -392,8 +392,8 @@ def test_sql_select_json(client, log_output):
"Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'), "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'),
] ]
input_serialization = InputSerialization(json=JSONInput(json_type="DOCUMENT")) input_serialization = JSONInputSerialization(json_type=JSON_TYPE_DOCUMENT)
output_serialization = OutputSerialization(json=JSONOutput()) output_serialization = JSONOutputSerialization()
try: try:
test_sql_expressions_custom_input_output(client, json_testcontent, test_sql_expressions_custom_input_output(client, json_testcontent,
input_serialization, output_serialization, tests, log_output) input_serialization, output_serialization, tests, log_output)
@ -414,14 +414,11 @@ val4,val5,val6
("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'), ("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'),
] ]
input_serialization = InputSerialization( input_serialization = CSVInputSerialization(
csv=CSVInput( file_header_info=FILE_HEADER_INFO_NONE,
file_header_info="NONE", allow_quoted_record_delimiter="FALSE",
allow_quoted_record_delimiter="FALSE",
),
) )
output_serialization = CSVOutputSerialization()
output_serialization = OutputSerialization(csv=CSVOutput())
try: try:
test_sql_expressions_custom_input_output(client, json_testcontent, test_sql_expressions_custom_input_output(client, json_testcontent,
input_serialization, output_serialization, tests, log_output) input_serialization, output_serialization, tests, log_output)

1
mint/run/core/s3select/tests.py
Unescape View File

@ -21,6 +21,7 @@ from csv import (test_csv_input_custom_quote_char,
test_csv_output_custom_quote_char) test_csv_output_custom_quote_char)
from minio import Minio from minio import Minio
from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv, from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv,
test_sql_functions_date, test_sql_functions_string, test_sql_functions_date, test_sql_functions_string,
test_sql_operators, test_sql_operators_precedence, test_sql_operators, test_sql_operators_precedence,

Write Preview
Loading…
Cancel
Save