From 9179cdfc9d49af461a7f2b9ebbe3a34da689977f Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 10 Aug 2020 14:32:17 -0700 Subject: [PATCH] update mint tests with new minio-py APIs (#10238) --- mint/run/core/s3select/csv.py | 43 ++++++++++++++++--------------- mint/run/core/s3select/sql_ops.py | 11 ++++---- mint/run/core/s3select/tests.py | 1 - 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/mint/run/core/s3select/csv.py b/mint/run/core/s3select/csv.py index 298415484..9f5759af0 100644 --- a/mint/run/core/s3select/csv.py +++ b/mint/run/core/s3select/csv.py @@ -22,7 +22,6 @@ from minio import Minio from minio.select.options import (CSVInput, CSVOutput, InputSerialization, JSONOutput, OutputSerialization, RequestProgress, SelectObjectOptions) - from utils import * @@ -98,17 +97,17 @@ def test_csv_input_custom_quote_char(client, log_output): expression="select * from s3object", input_serialization=InputSerialization( compression_type="NONE", - csv=CSVInput(FileHeaderInfo="NONE", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter=quote_char, - QuoteEscapeCharacter=escape_char, - Comments="#", - AllowQuotedRecordDelimiter="FALSE",), + csv=CSVInput(file_header_info="NONE", + record_delimiter="\n", + field_delimiter=",", + quote_character=quote_char, + quote_escape_character=escape_char, + comments="#", + allow_quoted_record_delimiter="FALSE",), ), output_serialization=OutputSerialization( json=JSONOutput( - RecordDelimiter="\n", + record_delimiter="\n", ) ), request_progress=RequestProgress( @@ -155,20 +154,22 @@ def test_csv_output_custom_quote_char(client, log_output): expression="select * from s3object", input_serialization=InputSerialization( compression_type="NONE", - csv=CSVInput(FileHeaderInfo="NONE", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter='"', - QuoteEscapeCharacter='"', - Comments="#", - AllowQuotedRecordDelimiter="FALSE",), + csv=CSVInput(file_header_info="NONE", + record_delimiter="\n", + field_delimiter=",", + quote_character='"', + quote_escape_character='"', + comments="#", + allow_quoted_record_delimiter="FALSE", + ), ), output_serialization=OutputSerialization( - csv=CSVOutput(QuoteFields="ALWAYS", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter=quote_char, - QuoteEscapeCharacter=escape_char,) + csv=CSVOutput(quote_fields="ALWAYS", + record_delimiter="\n", + field_delimiter=",", + quote_character=quote_char, + quote_escape_character=escape_char, + ) ), request_progress=RequestProgress( enabled="False" diff --git a/mint/run/core/s3select/sql_ops.py b/mint/run/core/s3select/sql_ops.py index 8b2099a21..7b67c535d 100644 --- a/mint/run/core/s3select/sql_ops.py +++ b/mint/run/core/s3select/sql_ops.py @@ -21,7 +21,6 @@ from datetime import datetime from minio.select.options import (CSVInput, CSVOutput, InputSerialization, JSONInput, JSONOutput, OutputSerialization, RequestProgress, SelectObjectOptions) - from utils import generate_bucket_name, generate_object_name @@ -82,11 +81,11 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, def test_sql_expressions(client, input_json_bytes, tests, log_output): input_serialization = InputSerialization( compression_type="NONE", - json=JSONInput(Type="DOCUMENT"), + json=JSONInput(json_type="DOCUMENT"), ) output_serialization = OutputSerialization( - csv=CSVOutput(QuoteFields="ASNEEDED") + csv=CSVOutput(quote_fields="ASNEEDED") ) test_sql_expressions_custom_input_output(client, input_json_bytes, @@ -393,7 +392,7 @@ def test_sql_select_json(client, log_output): "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'), ] - input_serialization = InputSerialization(json=JSONInput(Type="DOCUMENT")) + input_serialization = InputSerialization(json=JSONInput(json_type="DOCUMENT")) output_serialization = OutputSerialization(json=JSONOutput()) try: test_sql_expressions_custom_input_output(client, json_testcontent, @@ -417,8 +416,8 @@ val4,val5,val6 input_serialization = InputSerialization( csv=CSVInput( - FileHeaderInfo="NONE", - AllowQuotedRecordDelimiter="FALSE", + file_header_info="NONE", + allow_quoted_record_delimiter="FALSE", ), ) diff --git a/mint/run/core/s3select/tests.py b/mint/run/core/s3select/tests.py index b13091137..3ba0eb934 100644 --- a/mint/run/core/s3select/tests.py +++ b/mint/run/core/s3select/tests.py @@ -21,7 +21,6 @@ from csv import (test_csv_input_custom_quote_char, test_csv_output_custom_quote_char) from minio import Minio - from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv, test_sql_functions_date, test_sql_functions_string, test_sql_operators, test_sql_operators_precedence,