fix: s3select tests with new minio-py SDK (#10995)

master
Harshavardhana 4 years ago committed by GitHub
parent bdd094bc39
commit ae4ded7fd1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      mint/run/core/minio-go/go.mod
  2. 50
      mint/run/core/s3select/csv.py
  3. 45
      mint/run/core/s3select/sql_ops.py
  4. 1
      mint/run/core/s3select/tests.py

@ -1,3 +1,5 @@
module mint.minio.io/minio-go
go 1.14
require github.com/minio/minio-go/v7 v7.0.6 // indirect

@ -19,9 +19,13 @@ import io
import os
from minio import Minio
from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
JSONOutput, OutputSerialization,
RequestProgress, SelectObjectOptions)
from minio.selectrequest import (COMPRESSION_TYPE_NONE, FILE_HEADER_INFO_NONE,
JSON_TYPE_DOCUMENT, QUOTE_FIELDS_ALWAYS,
QUOTE_FIELDS_ASNEEDED, CSVInputSerialization,
CSVOutputSerialization,
JSONInputSerialization,
JSONOutputSerialization, SelectRequest)
from utils import *
@ -93,26 +97,22 @@ def test_csv_input_custom_quote_char(client, log_output):
try:
for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests):
sql_opts = SelectObjectOptions(
expression="select * from s3object",
input_serialization=InputSerialization(
compression_type="NONE",
csv=CSVInput(file_header_info="NONE",
sql_opts = SelectRequest(
"select * from s3object",
CSVInputSerialization(
compression_type=COMPRESSION_TYPE_NONE,
file_header_info=FILE_HEADER_INFO_NONE,
record_delimiter="\n",
field_delimiter=",",
quote_character=quote_char,
quote_escape_character=escape_char,
comments="#",
allow_quoted_record_delimiter="FALSE",),
allow_quoted_record_delimiter="FALSE",
),
output_serialization=OutputSerialization(
json=JSONOutput(
JSONOutputSerialization(
record_delimiter="\n",
)
),
request_progress=RequestProgress(
enabled="False"
)
request_progress=False,
)
test_sql_api(f'test_{idx}', client, bucket_name,
@ -150,11 +150,11 @@ def test_csv_output_custom_quote_char(client, log_output):
try:
for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests):
sql_opts = SelectObjectOptions(
expression="select * from s3object",
input_serialization=InputSerialization(
compression_type="NONE",
csv=CSVInput(file_header_info="NONE",
sql_opts = SelectRequest(
"select * from s3object",
CSVInputSerialization(
compression_type=COMPRESSION_TYPE_NONE,
file_header_info=FILE_HEADER_INFO_NONE,
record_delimiter="\n",
field_delimiter=",",
quote_character='"',
@ -162,18 +162,14 @@ def test_csv_output_custom_quote_char(client, log_output):
comments="#",
allow_quoted_record_delimiter="FALSE",
),
),
output_serialization=OutputSerialization(
csv=CSVOutput(quote_fields="ALWAYS",
CSVOutputSerialization(
quote_fields=QUOTE_FIELDS_ALWAYS,
record_delimiter="\n",
field_delimiter=",",
quote_character=quote_char,
quote_escape_character=escape_char,
)
),
request_progress=RequestProgress(
enabled="False"
)
request_progress=False,
)
test_sql_api(f'test_{idx}', client, bucket_name,

@ -18,9 +18,12 @@
import io
from datetime import datetime
from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
JSONInput, JSONOutput, OutputSerialization,
RequestProgress, SelectObjectOptions)
from minio.selectrequest import (FILE_HEADER_INFO_NONE, JSON_TYPE_DOCUMENT,
QUOTE_FIELDS_ASNEEDED, CSVInputSerialization,
CSVOutputSerialization,
JSONInputSerialization,
JSONOutputSerialization, SelectRequest)
from utils import generate_bucket_name, generate_object_name
@ -42,17 +45,15 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input,
continue
try:
log_output.args['total_tests'] += 1
options = SelectObjectOptions(
expression=select_expression,
input_serialization=sql_input,
output_serialization=sql_output,
request_progress=RequestProgress(
enabled="False"
)
sreq = SelectRequest(
select_expression,
sql_input,
sql_output,
request_progress=False
)
data = client.select_object_content(
bucket_name, object_name, options)
bucket_name, object_name, sreq)
# Get the records
records = io.BytesIO()
@ -79,14 +80,13 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input,
def test_sql_expressions(client, input_json_bytes, tests, log_output):
input_serialization = InputSerialization(
input_serialization = JSONInputSerialization(
compression_type="NONE",
json=JSONInput(json_type="DOCUMENT"),
json_type=JSON_TYPE_DOCUMENT,
)
output_serialization = OutputSerialization(
csv=CSVOutput(quote_fields="ASNEEDED")
)
output_serialization = CSVOutputSerialization(
quote_fields=QUOTE_FIELDS_ASNEEDED)
test_sql_expressions_custom_input_output(client, input_json_bytes,
input_serialization, output_serialization, tests, log_output)
@ -392,8 +392,8 @@ def test_sql_select_json(client, log_output):
"Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'),
]
input_serialization = InputSerialization(json=JSONInput(json_type="DOCUMENT"))
output_serialization = OutputSerialization(json=JSONOutput())
input_serialization = JSONInputSerialization(json_type=JSON_TYPE_DOCUMENT)
output_serialization = JSONOutputSerialization()
try:
test_sql_expressions_custom_input_output(client, json_testcontent,
input_serialization, output_serialization, tests, log_output)
@ -414,14 +414,11 @@ val4,val5,val6
("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'),
]
input_serialization = InputSerialization(
csv=CSVInput(
file_header_info="NONE",
input_serialization = CSVInputSerialization(
file_header_info=FILE_HEADER_INFO_NONE,
allow_quoted_record_delimiter="FALSE",
),
)
output_serialization = OutputSerialization(csv=CSVOutput())
output_serialization = CSVOutputSerialization()
try:
test_sql_expressions_custom_input_output(client, json_testcontent,
input_serialization, output_serialization, tests, log_output)

@ -21,6 +21,7 @@ from csv import (test_csv_input_custom_quote_char,
test_csv_output_custom_quote_char)
from minio import Minio
from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv,
test_sql_functions_date, test_sql_functions_string,
test_sql_operators, test_sql_operators_precedence,

Loading…
Cancel
Save