From 1e2ebc9945259ba2a15d48d791daa2b68f8c82c9 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Mon, 10 Aug 2020 09:02:29 -0700 Subject: [PATCH] feat: time to bring back http2.0 support (#10230) Bonus move our CI/CD to go1.14 --- .github/workflows/go.yml | 2 +- cmd/http-tracer.go | 1 + cmd/http/server.go | 9 +- cmd/lock-rest-client.go | 1 - cmd/prepare-storage.go | 1 - cmd/signals.go | 4 +- cmd/storage-rest-client.go | 1 - cmd/utils.go | 16 +- go.mod | 2 +- go.sum | 4 +- mint/run/core/s3select/csv.py | 198 +++++++++--------- mint/run/core/s3select/sql_ops.py | 334 +++++++++++++++++------------- mint/run/core/s3select/tests.py | 51 +++-- mint/run/core/s3select/utils.py | 6 +- pkg/trace/trace.go | 1 + 15 files changed, 349 insertions(+), 282 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 9bc3970d5..ac9569fe9 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go-version: [1.13.x] + go-version: [1.14.x] os: [ubuntu-latest, windows-latest] steps: - uses: actions/checkout@v2 diff --git a/cmd/http-tracer.go b/cmd/http-tracer.go index fe97dcaa1..ee225674c 100644 --- a/cmd/http-tracer.go +++ b/cmd/http-tracer.go @@ -129,6 +129,7 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ rq := trace.RequestInfo{ Time: time.Now().UTC(), + Proto: r.Proto, Method: r.Method, Path: r.URL.Path, RawQuery: r.URL.RawQuery, diff --git a/cmd/http/server.go b/cmd/http/server.go index 0ad0d8cd3..f8f594b40 100644 --- a/cmd/http/server.go +++ b/cmd/http/server.go @@ -191,14 +191,7 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat // TLS hardening PreferServerCipherSuites: true, MinVersion: tls.VersionTLS12, - // Do not edit the next line, protos priority is kept - // on purpose in this manner for HTTP 2.0, we would - // still like HTTP 2.0 clients to negotiate connection - // to server if needed but by default HTTP 1.1 is - // expected. We need to change this in future - // when we wish to go back to HTTP 2.0 as default - // priority for HTTP protocol negotiation. - NextProtos: []string{"http/1.1", "h2"}, + NextProtos: []string{"h2", "http/1.1"}, } tlsConfig.GetCertificate = getCert } diff --git a/cmd/lock-rest-client.go b/cmd/lock-rest-client.go index 91e63e6e6..1c3d2d174 100644 --- a/cmd/lock-rest-client.go +++ b/cmd/lock-rest-client.go @@ -149,7 +149,6 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient { tlsConfig = &tls.Config{ ServerName: endpoint.Hostname(), RootCAs: globalRootCAs, - NextProtos: []string{"http/1.1"}, // Force http1.1 } } diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go index c73b750e9..13a56d911 100644 --- a/cmd/prepare-storage.go +++ b/cmd/prepare-storage.go @@ -185,7 +185,6 @@ func IsServerResolvable(endpoint Endpoint) error { tlsConfig = &tls.Config{ ServerName: endpoint.Hostname(), RootCAs: globalRootCAs, - NextProtos: []string{"http/1.1"}, // Force http1.1 } } diff --git a/cmd/signals.go b/cmd/signals.go index 965a85cc6..82b9639b9 100644 --- a/cmd/signals.go +++ b/cmd/signals.go @@ -18,6 +18,8 @@ package cmd import ( "context" + "errors" + "net/http" "os" "strings" @@ -73,7 +75,7 @@ func handleSignals() { if objAPI := newObjectLayerWithoutSafeModeFn(); objAPI != nil { objAPI.Shutdown(context.Background()) } - if err != nil { + if err != nil && !errors.Is(err, http.ErrServerClosed) { logger.Fatal(err, "Unable to start MinIO server") } exit(true) diff --git a/cmd/storage-rest-client.go b/cmd/storage-rest-client.go index 156a0e761..3402895c9 100644 --- a/cmd/storage-rest-client.go +++ b/cmd/storage-rest-client.go @@ -663,7 +663,6 @@ func newStorageRESTClient(endpoint Endpoint) *storageRESTClient { tlsConfig = &tls.Config{ ServerName: endpoint.Hostname(), RootCAs: globalRootCAs, - NextProtos: []string{"http/1.1"}, // Force http1.1 } } diff --git a/cmd/utils.go b/cmd/utils.go index cd2c5addb..6a129facf 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -39,13 +39,13 @@ import ( "sync" "time" + humanize "github.com/dustin/go-humanize" + "github.com/gorilla/mux" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" "github.com/minio/minio/pkg/handlers" "github.com/minio/minio/pkg/madmin" - - humanize "github.com/dustin/go-humanize" - "github.com/gorilla/mux" + "golang.org/x/net/http2" ) const ( @@ -467,6 +467,11 @@ func newInternodeHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) // in raw stream. DisableCompression: true, } + + if tlsConfig != nil { + http2.ConfigureTransport(tr) + } + return func() *http.Transport { return tr } @@ -490,6 +495,11 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) fu // in raw stream. DisableCompression: true, } + + if tlsConfig != nil { + http2.ConfigureTransport(tr) + } + return func() *http.Transport { return tr } diff --git a/go.mod b/go.mod index e81b4e409..e7596e232 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/minio/minio-go/v7 v7.0.3 github.com/minio/selfupdate v0.3.1 github.com/minio/sha256-simd v0.1.1 - github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37 + github.com/minio/simdjson-go v0.1.5 github.com/minio/sio v0.2.0 github.com/mitchellh/go-homedir v1.1.0 github.com/mmcloughlin/avo v0.0.0-20200523190732-4439b6b2c061 // indirect diff --git a/go.sum b/go.sum index ec267072e..97e6b9900 100644 --- a/go.sum +++ b/go.sum @@ -297,8 +297,8 @@ github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlE github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37 h1:pDeao6M5AEd8hwTtGmE0pVKomlL56JFRa5SiXDZAuJE= -github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI= +github.com/minio/simdjson-go v0.1.5 h1:6T5mHh7r3kUvgwhmFWQAjoPV5Yt5oD/VPjAI9ViH1kM= +github.com/minio/simdjson-go v0.1.5/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI= github.com/minio/sio v0.2.0 h1:NCRCFLx0r5pRbXf65LVNjxbCGZgNQvNFQkgX3XF4BoA= github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= diff --git a/mint/run/core/s3select/csv.py b/mint/run/core/s3select/csv.py index faee96518..b34ba2afc 100644 --- a/mint/run/core/s3select/csv.py +++ b/mint/run/core/s3select/csv.py @@ -15,37 +15,42 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import io +import os from minio import Minio -from minio.select.options import (SelectObjectOptions, CSVInput, - RequestProgress, InputSerialization, - OutputSerialization, CSVOutput, JsonOutput) +from minio.select.options import (CSVInput, CSVOutput, InputSerialization, + JsonOutput, OutputSerialization, + RequestProgress, SelectObjectOptions) from utils import * + def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_output): """ Test if the passed SQL request has the output equal to the passed execpted one""" object_name = generate_object_name() got_output = b'' try: bytes_content = io.BytesIO(input_data) - client.put_object(bucket_name, object_name, io.BytesIO(input_data), len(input_data)) + client.put_object(bucket_name, object_name, + io.BytesIO(input_data), len(input_data)) data = client.select_object_content(bucket_name, object_name, sql_opts) # Get the records records = io.BytesIO() for d in data.stream(10*1024): records.write(d.encode('utf-8')) - got_output = records.getvalue() + got_output = records.getvalue() except Exception as select_err: if not isinstance(expected_output, Exception): - raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) + raise ValueError( + 'Test {} unexpectedly failed with: {}'.format(test_name, select_err)) else: if isinstance(expected_output, Exception): - raise ValueError('Test {}: expected an exception, got {}'.format(test_name, got_output)) + raise ValueError( + 'Test {}: expected an exception, got {}'.format(test_name, got_output)) if got_output != expected_output: - raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format(test_name, expected_output, got_output)) + raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format( + test_name, expected_output, got_output)) finally: client.remove_object(bucket_name, object_name) @@ -55,28 +60,34 @@ def test_csv_input_custom_quote_char(client, log_output): log_output.args['bucket_name'] = bucket_name = generate_bucket_name() tests = [ - # Invalid quote character, should fail - ('""', '"', b'col1,col2,col3\n', Exception()), - # UTF-8 quote character - ('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(), b'{"_1":"col1","_2":"col2","_3":"col3"}\n'), - # Only one field is quoted - ('"', '"', b'"col1",col2,col3\n', b'{"_1":"col1","_2":"col2","_3":"col3"}\n'), - ('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'), - ('\'', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), - ('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), - ('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), - ('', '"', b'"col1","col2","col3"\n', b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'), - ('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'), - ('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'), - ('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'), - ('"', '\\', b'A\\B,C\n', b'{"_1":"A\\\\B","_2":"C"}\n'), - ('"', '"', b'"A""B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'), - ('"', '\\', b'"A\\B","CD"\n', b'{"_1":"AB","_2":"CD"}\n'), - ('"', '\\', b'"A\\,","CD"\n', b'{"_1":"A,","_2":"CD"}\n'), - ('"', '\\', b'"A\\"B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'), - ('"', '\\', b'"A\\""\n', b'{"_1":"A\\""}\n'), - ('"', '\\', b'"A\\"\\"B"\n', b'{"_1":"A\\"\\"B"}\n'), - ('"', '\\', b'"A\\"","\\"B"\n', b'{"_1":"A\\"","_2":"\\"B"}\n'), + # Invalid quote character, should fail + ('""', '"', b'col1,col2,col3\n', Exception()), + # UTF-8 quote character + ('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(), + b'{"_1":"col1","_2":"col2","_3":"col3"}\n'), + # Only one field is quoted + ('"', '"', b'"col1",col2,col3\n', + b'{"_1":"col1","_2":"col2","_3":"col3"}\n'), + ('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'), + ('\'', '"', b'"col1",col2,col3\n', + b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), + ('', '"', b'"col1",col2,col3\n', + b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), + ('', '"', b'"col1",col2,col3\n', + b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), + ('', '"', b'"col1","col2","col3"\n', + b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'), + ('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'), + ('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'), + ('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'), + ('"', '\\', b'A\\B,C\n', b'{"_1":"A\\\\B","_2":"C"}\n'), + ('"', '"', b'"A""B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'), + ('"', '\\', b'"A\\B","CD"\n', b'{"_1":"AB","_2":"CD"}\n'), + ('"', '\\', b'"A\\,","CD"\n', b'{"_1":"A,","_2":"CD"}\n'), + ('"', '\\', b'"A\\"B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'), + ('"', '\\', b'"A\\""\n', b'{"_1":"A\\""}\n'), + ('"', '\\', b'"A\\"\\"B"\n', b'{"_1":"A\\"\\"B"}\n'), + ('"', '\\', b'"A\\"","\\"B"\n', b'{"_1":"A\\"","_2":"\\"B"}\n'), ] client.make_bucket(bucket_name) @@ -84,54 +95,56 @@ def test_csv_input_custom_quote_char(client, log_output): try: for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests): sql_opts = SelectObjectOptions( - expression="select * from s3object", - input_serialization=InputSerialization( - compression_type="NONE", - csv=CSVInput(FileHeaderInfo="NONE", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter=quote_char, - QuoteEscapeCharacter=escape_char, - Comments="#", - AllowQuotedRecordDelimiter="FALSE",), - ), - output_serialization=OutputSerialization( - json = JsonOutput( - RecordDelimiter="\n", - ) - ), - request_progress=RequestProgress( - enabled="False" - ) + expression="select * from s3object", + input_serialization=InputSerialization( + compression_type="NONE", + csv=CSVInput(FileHeaderInfo="NONE", + RecordDelimiter="\n", + FieldDelimiter=",", + QuoteCharacter=quote_char, + QuoteEscapeCharacter=escape_char, + Comments="#", + AllowQuotedRecordDelimiter="FALSE",), + ), + output_serialization=OutputSerialization( + json=JsonOutput( + RecordDelimiter="\n", ) - - test_sql_api(f'test_{idx}', client, bucket_name, data, sql_opts, expected_output) + ), + request_progress=RequestProgress( + enabled="False" + ) + ) + + test_sql_api(f'test_{idx}', client, bucket_name, + data, sql_opts, expected_output) finally: client.remove_bucket(bucket_name) # Test passes print(log_output.json_report()) + def test_csv_output_custom_quote_char(client, log_output): # Get a unique bucket_name and object_name log_output.args['bucket_name'] = bucket_name = generate_bucket_name() tests = [ - # UTF-8 quote character - ("''", "''", b'col1,col2,col3\n', Exception()), - ("'", "'", b'col1,col2,col3\n', b"'col1','col2','col3'\n"), - ("", '"', b'col1,col2,col3\n', b'\x00col1\x00,\x00col2\x00,\x00col3\x00\n'), - ('"', '"', b'col1,col2,col3\n', b'"col1","col2","col3"\n'), - ('"', '"', b'col"1,col2,col3\n', b'"col""1","col2","col3"\n'), - ('"', '"', b'""""\n', b'""""\n'), - ('"', '"', b'\n', b''), - ("'", "\\", b'col1,col2,col3\n', b"'col1','col2','col3'\n"), - ("'", "\\", b'col""1,col2,col3\n', b"'col\"\"1','col2','col3'\n"), - ("'", "\\", b'col\'1,col2,col3\n', b"'col\\'1','col2','col3'\n"), - ("'", "\\", b'"col\'1","col2","col3"\n', b"'col\\'1','col2','col3'\n"), - ("'", "\\", b'col\'\n', b"'col\\''\n"), - # Two consecutive escaped quotes - ("'", "\\", b'"a"""""\n', b"'a\"\"'\n"), + # UTF-8 quote character + ("''", "''", b'col1,col2,col3\n', Exception()), + ("'", "'", b'col1,col2,col3\n', b"'col1','col2','col3'\n"), + ("", '"', b'col1,col2,col3\n', b'\x00col1\x00,\x00col2\x00,\x00col3\x00\n'), + ('"', '"', b'col1,col2,col3\n', b'"col1","col2","col3"\n'), + ('"', '"', b'col"1,col2,col3\n', b'"col""1","col2","col3"\n'), + ('"', '"', b'""""\n', b'""""\n'), + ('"', '"', b'\n', b''), + ("'", "\\", b'col1,col2,col3\n', b"'col1','col2','col3'\n"), + ("'", "\\", b'col""1,col2,col3\n', b"'col\"\"1','col2','col3'\n"), + ("'", "\\", b'col\'1,col2,col3\n', b"'col\\'1','col2','col3'\n"), + ("'", "\\", b'"col\'1","col2","col3"\n', b"'col\\'1','col2','col3'\n"), + ("'", "\\", b'col\'\n', b"'col\\''\n"), + # Two consecutive escaped quotes + ("'", "\\", b'"a"""""\n', b"'a\"\"'\n"), ] client.make_bucket(bucket_name) @@ -139,34 +152,33 @@ def test_csv_output_custom_quote_char(client, log_output): try: for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests): sql_opts = SelectObjectOptions( - expression="select * from s3object", - input_serialization=InputSerialization( - compression_type="NONE", - csv=CSVInput(FileHeaderInfo="NONE", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter='"', - QuoteEscapeCharacter='"', - Comments="#", - AllowQuotedRecordDelimiter="FALSE",), - ), - output_serialization=OutputSerialization( - csv=CSVOutput(QuoteFields="ALWAYS", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter=quote_char, - QuoteEscapeCharacter=escape_char,) - ), - request_progress=RequestProgress( - enabled="False" - ) - ) - - test_sql_api(f'test_{idx}', client, bucket_name, input_data, sql_opts, expected_output) + expression="select * from s3object", + input_serialization=InputSerialization( + compression_type="NONE", + csv=CSVInput(FileHeaderInfo="NONE", + RecordDelimiter="\n", + FieldDelimiter=",", + QuoteCharacter='"', + QuoteEscapeCharacter='"', + Comments="#", + AllowQuotedRecordDelimiter="FALSE",), + ), + output_serialization=OutputSerialization( + csv=CSVOutput(QuoteFields="ALWAYS", + RecordDelimiter="\n", + FieldDelimiter=",", + QuoteCharacter=quote_char, + QuoteEscapeCharacter=escape_char,) + ), + request_progress=RequestProgress( + enabled="False" + ) + ) + + test_sql_api(f'test_{idx}', client, bucket_name, + input_data, sql_opts, expected_output) finally: client.remove_bucket(bucket_name) # Test passes print(log_output.json_report()) - - diff --git a/mint/run/core/s3select/sql_ops.py b/mint/run/core/s3select/sql_ops.py index 83931ad21..bf3c0bd84 100644 --- a/mint/run/core/s3select/sql_ops.py +++ b/mint/run/core/s3select/sql_ops.py @@ -18,14 +18,15 @@ import io from datetime import datetime -from minio import Minio -from minio.select.options import (SelectObjectOptions, CSVInput, JSONInput, - RequestProgress, InputSerialization, - OutputSerialization, CSVOutput, JsonOutput) +from minio.select.options import (CSVInput, CSVOutput, InputSerialization, + JSONInput, OutputSerialization, + RequestProgress, SelectObjectOptions) -from utils import * +from utils import generate_bucket_name, generate_object_name -def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql_output, tests, log_output): + +def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, + sql_output, tests, log_output): bucket_name = generate_bucket_name() object_name = generate_object_name() @@ -48,10 +49,11 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql output_serialization=sql_output, request_progress=RequestProgress( enabled="False" - ) ) + ) - data = client.select_object_content(bucket_name, object_name, options) + data = client.select_object_content( + bucket_name, object_name, options) # Get the records records = io.BytesIO() @@ -62,13 +64,15 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql if got_output != expected_output: if type(expected_output) == datetime: # Attempt to parse the date which will throw an exception for any issue - datetime.strptime(got_output.decode("utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z') + datetime.strptime(got_output.decode( + "utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z') else: - raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(idx+1, expected_output, got_output)) + raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format( + idx+1, expected_output, got_output)) log_output.args['total_success'] += 1 except Exception as err: - continue ## TODO, raise instead + continue # TODO, raise instead # raise Exception(err) finally: client.remove_object(bucket_name, object_name) @@ -77,16 +81,16 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql def test_sql_expressions(client, input_json_bytes, tests, log_output): input_serialization = InputSerialization( - compression_type="NONE", - json=JSONInput(Type="DOCUMENT"), - ) + compression_type="NONE", + json=JSONInput(Type="DOCUMENT"), + ) - output_serialization=OutputSerialization( - csv=CSVOutput(QuoteFields="ASNEEDED") - ) + output_serialization = OutputSerialization( + csv=CSVOutput(QuoteFields="ASNEEDED") + ) test_sql_expressions_custom_input_output(client, input_json_bytes, - input_serialization, output_serialization, tests, log_output) + input_serialization, output_serialization, tests, log_output) def test_sql_operators(client, log_output): @@ -98,31 +102,38 @@ def test_sql_operators(client, log_output): """ tests = [ - # Logical operators - ("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'), - ("NOT", "select * from S3Object s where NOT s.id = 1", b'2,Elliot,4\n3,Yves,5\n4,,0\n'), - ("OR", "select * from S3Object s where s.id = 1 OR s.id = 3", b'1,John,3\n3,Yves,5\n'), - # Comparison Operators - ("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'), - (">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'), - ("<=", "select * from S3Object s where s.age <= 4", b'1,John,3\n2,Elliot,4\n4,,0\n'), - (">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'), - ("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'), - ("<>", "select * from S3Object s where s.age <> 4", b'1,John,3\n3,Yves,5\n4,,0\n'), - ("!=", "select * from S3Object s where s.age != 4", b'1,John,3\n3,Yves,5\n4,,0\n'), - ("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5", b'2,Elliot,4\n3,Yves,5\n'), - ("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'), - # Pattern Matching Operators - ("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'), - ("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'), - # Unitary Operators - ("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'), - ("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL", b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'), - # Math Operators - ("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'), - ("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'), - ("*", "select * from S3Object s where s.age = 2*2 ", b'2,Elliot,4\n'), - ("%", "select * from S3Object s where s.age = 10%6 ", b'2,Elliot,4\n'), + # Logical operators + ("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'), + ("NOT", "select * from S3Object s where NOT s.id = 1", + b'2,Elliot,4\n3,Yves,5\n4,,0\n'), + ("OR", "select * from S3Object s where s.id = 1 OR s.id = 3", + b'1,John,3\n3,Yves,5\n'), + # Comparison Operators + ("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'), + (">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'), + ("<=", "select * from S3Object s where s.age <= 4", + b'1,John,3\n2,Elliot,4\n4,,0\n'), + (">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'), + ("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'), + ("<>", "select * from S3Object s where s.age <> 4", + b'1,John,3\n3,Yves,5\n4,,0\n'), + ("!=", "select * from S3Object s where s.age != 4", + b'1,John,3\n3,Yves,5\n4,,0\n'), + ("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5", + b'2,Elliot,4\n3,Yves,5\n'), + ("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'), + # Pattern Matching Operators + ("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'), + ("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'), + # Unitary Operators + ("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'), + ("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL", + b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'), + # Math Operators + ("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'), + ("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'), + ("*", "select * from S3Object s where s.age = 2*2 ", b'2,Elliot,4\n'), + ("%", "select * from S3Object s where s.age = 10%6 ", b'2,Elliot,4\n'), ] try: @@ -141,20 +152,20 @@ def test_sql_operators_precedence(client, log_output): json_testfile = """{"id": 1, "name": "Eric"}""" tests = [ - ("-_1", "select -3*3 from S3Object", b'-9\n'), - ("*", "select 10-3*2 from S3Object", b'4\n'), - ("/", "select 13-10/5 from S3Object", b'11\n'), - ("%", "select 13-10%5 from S3Object", b'13\n'), - ("+", "select 1+1*3 from S3Object", b'4\n'), - ("-_2", "select 1-1*3 from S3Object", b'-2\n'), - ("=", "select * from S3Object as s where s.id = 13-12", b'1,Eric\n'), - ("<>", "select * from S3Object as s where s.id <> 1-1", b'1,Eric\n'), - ("NOT", "select * from S3Object where false OR NOT false", b'1,Eric\n'), - ("AND", "select * from S3Object where true AND true OR false ", b'1,Eric\n'), - ("OR", "select * from S3Object where false OR NOT false", b'1,Eric\n'), - ("IN", "select * from S3Object as s where s.id <> -1 AND s.id IN (1,2,3)", b'1,Eric\n'), - ("BETWEEN", "select * from S3Object as s where s.id <> -1 AND s.id BETWEEN -1 AND 3", b'1,Eric\n'), - ("LIKE", "select * from S3Object as s where s.id <> -1 AND s.name LIKE 'E%'", b'1,Eric\n'), + ("-_1", "select -3*3 from S3Object", b'-9\n'), + ("*", "select 10-3*2 from S3Object", b'4\n'), + ("/", "select 13-10/5 from S3Object", b'11\n'), + ("%", "select 13-10%5 from S3Object", b'13\n'), + ("+", "select 1+1*3 from S3Object", b'4\n'), + ("-_2", "select 1-1*3 from S3Object", b'-2\n'), + ("=", "select * from S3Object as s where s.id = 13-12", b'1,Eric\n'), + ("<>", "select * from S3Object as s where s.id <> 1-1", b'1,Eric\n'), + ("NOT", "select * from S3Object where false OR NOT false", b'1,Eric\n'), + ("AND", "select * from S3Object where true AND true OR false ", b'1,Eric\n'), + ("OR", "select * from S3Object where false OR NOT false", b'1,Eric\n'), + ("IN", "select * from S3Object as s where s.id <> -1 AND s.id IN (1,2,3)", b'1,Eric\n'), + ("BETWEEN", "select * from S3Object as s where s.id <> -1 AND s.id BETWEEN -1 AND 3", b'1,Eric\n'), + ("LIKE", "select * from S3Object as s where s.id <> -1 AND s.name LIKE 'E%'", b'1,Eric\n'), ] try: @@ -168,7 +179,6 @@ def test_sql_operators_precedence(client, log_output): print(log_output.json_report()) - def test_sql_functions_agg_cond_conv(client, log_output): json_testfile = """{"id": 1, "name": "John", "age": 3} @@ -178,17 +188,18 @@ def test_sql_functions_agg_cond_conv(client, log_output): {"id": 5, "name": "Eric", "age": 0} """ tests = [ - # Aggregate functions - ("COUNT", "select count(*) from S3Object s", b'5\n'), - ("AVG", "select avg(s.age) from S3Object s", b'3\n'), - ("MAX", "select max(s.age) from S3Object s", b'5\n'), - ("MIN", "select min(s.age) from S3Object s", b'0\n'), - ("SUM", "select sum(s.age) from S3Object s", b'12\n'), - # Conditional functions - ("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'), - ("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'), - ## Conversion functions - ("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s", b'3.0\n4.0\n5.0\n\n0.0\n'), + # Aggregate functions + ("COUNT", "select count(*) from S3Object s", b'5\n'), + ("AVG", "select avg(s.age) from S3Object s", b'3\n'), + ("MAX", "select max(s.age) from S3Object s", b'5\n'), + ("MIN", "select min(s.age) from S3Object s", b'0\n'), + ("SUM", "select sum(s.age) from S3Object s", b'12\n'), + # Conditional functions + ("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'), + ("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'), + # Conversion functions + ("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s", + b'3.0\n4.0\n5.0\n\n0.0\n'), ] @@ -210,36 +221,49 @@ def test_sql_functions_date(client, log_output): """ tests = [ - # DATE_ADD - ("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2022-01-02T03:04:05.006+07:30\n'), - ("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-02-02T03:04:05.006+07:30\n'), - ("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-01T03:04:05.006+07:30\n'), - ("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T04:04:05.006+07:30\n'), - ("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:09:05.006+07:30\n'), - ("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:04:10.006+07:30\n'), - # DATE_DIFF - ("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'), - ("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'), - ("DATE_DIFF_3", "select DATE_DIFF(day, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2010-01-02T')) from S3Object as s", b'-2556\n'), - # EXTRACT - ("EXTRACT_1", "select EXTRACT(year FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017\n'), - ("EXTRACT_2", "select EXTRACT(month FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'1\n'), - ("EXTRACT_3", "select EXTRACT(hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'3\n'), - ("EXTRACT_4", "select EXTRACT(minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'4\n'), - ("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'), - ("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'), - # TO_STRING - ("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s", b'"January 2, 2017"\n'), - ("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'), - ("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'), - ("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'), - ("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s", b'"January 2, 2017 3:4 AM"\n'), - ("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), - ("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), - ("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), - ("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s", b'2017-01-02T3:4:05+07:30\n'), - ("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s", b'2017-01-02T03:04:05.006+07:30\n'), - ("UTCNOW", "select UTCNOW() from S3Object", datetime(1,1,1)), + # DATE_ADD + ("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", + b'2022-01-02T03:04:05.006+07:30\n'), + ("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", + b'2017-02-02T03:04:05.006+07:30\n'), + ("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s", + b'2017-01-01T03:04:05.006+07:30\n'), + ("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", + b'2017-01-02T04:04:05.006+07:30\n'), + ("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", + b'2017-01-02T03:09:05.006+07:30\n'), + ("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", + b'2017-01-02T03:04:10.006+07:30\n'), + # DATE_DIFF + ("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'), + ("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'), + ("DATE_DIFF_3", "select DATE_DIFF(day, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2010-01-02T')) from S3Object as s", b'-2556\n'), + # EXTRACT + ("EXTRACT_1", "select EXTRACT(year FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017\n'), + ("EXTRACT_2", "select EXTRACT(month FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'1\n'), + ("EXTRACT_3", "select EXTRACT(hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'3\n'), + ("EXTRACT_4", "select EXTRACT(minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'4\n'), + ("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'), + ("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'), + # TO_STRING + ("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s", + b'"January 2, 2017"\n'), + ("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'), + ("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'), + ("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'), + ("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s", + b'"January 2, 2017 3:4 AM"\n'), + ("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", + b'2017-01-02T3:4:05+0730\n'), + ("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", + b'2017-01-02T3:4:05+0730\n'), + ("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s", + b'2017-01-02T3:4:05+0730\n'), + ("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s", + b'2017-01-02T3:4:05+07:30\n'), + ("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s", + b'2017-01-02T03:04:05.006+07:30\n'), + ("UTCNOW", "select UTCNOW() from S3Object", datetime(1, 1, 1)), ] @@ -253,6 +277,7 @@ def test_sql_functions_date(client, log_output): # Test passes print(log_output.json_report()) + def test_sql_functions_string(client, log_output): json_testfile = """ @@ -262,23 +287,26 @@ def test_sql_functions_string(client, log_output): """ tests = [ - # CHAR_LENGTH - ("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), - ("CHARACTER_LENGTH", "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), - # LOWER - ("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'), - # SUBSTRING - ("SUBSTRING_1", "select SUBSTRING(s.name FROM 2) from S3Object as s where s.id = 1", b'ohn\n'), - ("SUBSTRING_2", "select SUBSTRING(s.name FROM 2 FOR 2) from S3Object as s where s.id = 1", b'oh\n'), - ("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'), - # TRIM - ("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'), - ("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t \n'), - ("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2", b' \tfoobar\t\n'), - ("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'), - ("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'), - # UPPER - ("UPPER", "select UPPER(s.name) from S3Object as s where s.id= 1", b'JOHN\n'), + # CHAR_LENGTH + ("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), + ("CHARACTER_LENGTH", + "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), + # LOWER + ("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'), + # SUBSTRING + ("SUBSTRING_1", "select SUBSTRING(s.name FROM 2) from S3Object as s where s.id = 1", b'ohn\n'), + ("SUBSTRING_2", "select SUBSTRING(s.name FROM 2 FOR 2) from S3Object as s where s.id = 1", b'oh\n'), + ("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'), + # TRIM + ("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'), + ("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2", + b'\tfoobar\t \n'), + ("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2", + b' \tfoobar\t\n'), + ("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'), + ("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'), + # UPPER + ("UPPER", "select UPPER(s.name) from S3Object as s where s.id= 1", b'JOHN\n'), ] try: @@ -297,14 +325,15 @@ def test_sql_datatypes(client, log_output): {"name": "John"} """ tests = [ - ("bool", "select CAST('true' AS BOOL) from S3Object", b'true\n'), - ("int", "select CAST('13' AS INT) from S3Object", b'13\n'), - ("integer", "select CAST('13' AS INTEGER) from S3Object", b'13\n'), - ("string", "select CAST(true AS STRING) from S3Object", b'true\n'), - ("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'), - ("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'), - ("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'), - ("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object", b'2007-04-05T14:30Z\n'), + ("bool", "select CAST('true' AS BOOL) from S3Object", b'true\n'), + ("int", "select CAST('13' AS INT) from S3Object", b'13\n'), + ("integer", "select CAST('13' AS INTEGER) from S3Object", b'13\n'), + ("string", "select CAST(true AS STRING) from S3Object", b'true\n'), + ("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'), + ("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'), + ("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'), + ("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object", + b'2007-04-05T14:30Z\n'), ] try: @@ -323,14 +352,17 @@ def test_sql_select(client, log_output): json_testfile = """{"id": 1, "created": "June 27", "modified": "July 6" } {"id": 2, "Created": "June 28", "Modified": "July 7", "Cast": "Random Date" }""" tests = [ - ("select_1", "select * from S3Object", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), - ("select_2", "select * from S3Object s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), - ("select_3", "select * from S3Object as s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), - ("select_4", "select s.line from S3Object as s", b'\n\n'), - ("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'), - ("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'), - ("where", 'select s.created from S3Object as s', b'June 27\nJune 28\n'), - ("limit", 'select * from S3Object as s LIMIT 1', b'1,June 27,July 6\n'), + ("select_1", "select * from S3Object", + b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), + ("select_2", "select * from S3Object s", + b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), + ("select_3", "select * from S3Object as s", + b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), + ("select_4", "select s.line from S3Object as s", b'\n\n'), + ("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'), + ("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'), + ("where", 'select s.created from S3Object as s', b'June 27\nJune 28\n'), + ("limit", 'select * from S3Object as s LIMIT 1', b'1,June 27,July 6\n'), ] try: @@ -343,23 +375,29 @@ def test_sql_select(client, log_output): # Test passes print(log_output.json_report()) + def test_sql_select_json(client, log_output): json_testcontent = """{ "Rules": [ {"id": "1"}, {"expr": "y > x"}, {"id": "2", "expr": "z = DEBUG"} ]} { "created": "June 27", "modified": "July 6" } """ tests = [ - ("select_1", "SELECT id FROM S3Object[*].Rules[*].id", b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'), - ("select_2", "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'), - ("select_3", "SELECT d.created, d.modified FROM S3Object[*] d", b'{}\n{"created":"June 27","modified":"July 6"}\n'), - ("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]", b'{}\n{"created":"June 27","modified":"July 6"}\n'), - ("select_5", "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'), + ("select_1", "SELECT id FROM S3Object[*].Rules[*].id", + b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'), + ("select_2", + "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'), + ("select_3", "SELECT d.created, d.modified FROM S3Object[*] d", + b'{}\n{"created":"June 27","modified":"July 6"}\n'), + ("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]", + b'{}\n{"created":"June 27","modified":"July 6"}\n'), + ("select_5", + "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'), ] input_serialization = InputSerialization(json=JSONInput(Type="DOCUMENT")) output_serialization = OutputSerialization(json=JsonOutput()) try: test_sql_expressions_custom_input_output(client, json_testcontent, - input_serialization, output_serialization, tests, log_output) + input_serialization, output_serialization, tests, log_output) except Exception as select_err: raise select_err # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) @@ -374,20 +412,20 @@ def test_sql_select_csv_no_header(client, log_output): val4,val5,val6 """ tests = [ - ("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'), + ("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'), ] - input_serialization=InputSerialization( - csv=CSVInput( - FileHeaderInfo="NONE", - AllowQuotedRecordDelimiter="FALSE", - ), - ) + input_serialization = InputSerialization( + csv=CSVInput( + FileHeaderInfo="NONE", + AllowQuotedRecordDelimiter="FALSE", + ), + ) - output_serialization=OutputSerialization(csv=CSVOutput()) + output_serialization = OutputSerialization(csv=CSVOutput()) try: test_sql_expressions_custom_input_output(client, json_testcontent, - input_serialization, output_serialization, tests, log_output) + input_serialization, output_serialization, tests, log_output) except Exception as select_err: raise select_err # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) @@ -395,5 +433,3 @@ val4,val5,val6 # Test passes print(log_output.json_report()) - - diff --git a/mint/run/core/s3select/tests.py b/mint/run/core/s3select/tests.py index 3e8c26dde..b13091137 100644 --- a/mint/run/core/s3select/tests.py +++ b/mint/run/core/s3select/tests.py @@ -16,12 +16,19 @@ # limitations under the License. import os -from sys import exit +import sys +from csv import (test_csv_input_custom_quote_char, + test_csv_output_custom_quote_char) + from minio import Minio +from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv, + test_sql_functions_date, test_sql_functions_string, + test_sql_operators, test_sql_operators_precedence, + test_sql_select, test_sql_select_csv_no_header, + test_sql_select_json) from utils import LogOutput -from sql_ops import * -from csv import * + def main(): """ @@ -39,48 +46,56 @@ def main(): secret_key = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' secure = True - client = Minio(server_endpoint, access_key, secret_key, secure=False) + client = Minio(server_endpoint, access_key, secret_key, secure=secure) - log_output = LogOutput(client.select_object_content, 'test_csv_input_quote_char') + log_output = LogOutput(client.select_object_content, + 'test_csv_input_quote_char') test_csv_input_custom_quote_char(client, log_output) - log_output = LogOutput(client.select_object_content, 'test_csv_output_quote_char') + log_output = LogOutput(client.select_object_content, + 'test_csv_output_quote_char') test_csv_output_custom_quote_char(client, log_output) - log_output = LogOutput(client.select_object_content, 'test_sql_operators') + log_output = LogOutput( + client.select_object_content, 'test_sql_operators') test_sql_operators(client, log_output) - log_output = LogOutput(client.select_object_content, 'test_sql_operators_precedence') + log_output = LogOutput(client.select_object_content, + 'test_sql_operators_precedence') test_sql_operators_precedence(client, log_output) - log_output = LogOutput(client.select_object_content, 'test_sql_functions_agg_cond_conv') + log_output = LogOutput(client.select_object_content, + 'test_sql_functions_agg_cond_conv') test_sql_functions_agg_cond_conv(client, log_output) - log_output = LogOutput(client.select_object_content, 'test_sql_functions_date') + log_output = LogOutput( + client.select_object_content, 'test_sql_functions_date') test_sql_functions_date(client, log_output) - log_output = LogOutput(client.select_object_content, 'test_sql_functions_string') + log_output = LogOutput(client.select_object_content, + 'test_sql_functions_string') test_sql_functions_string(client, log_output) - log_output = LogOutput(client.select_object_content, 'test_sql_datatypes') + log_output = LogOutput( + client.select_object_content, 'test_sql_datatypes') test_sql_datatypes(client, log_output) log_output = LogOutput(client.select_object_content, 'test_sql_select') test_sql_select(client, log_output) - log_output = LogOutput(client.select_object_content, 'test_sql_select_json') + log_output = LogOutput( + client.select_object_content, 'test_sql_select_json') test_sql_select_json(client, log_output) - log_output = LogOutput(client.select_object_content, 'test_sql_select_csv') + log_output = LogOutput( + client.select_object_content, 'test_sql_select_csv') test_sql_select_csv_no_header(client, log_output) except Exception as err: print(log_output.json_report(err)) - exit(1) + sys.exit(1) + if __name__ == "__main__": # Execute only if run as a script main() - - - diff --git a/mint/run/core/s3select/utils.py b/mint/run/core/s3select/utils.py index 1baabf983..db31d3521 100644 --- a/mint/run/core/s3select/utils.py +++ b/mint/run/core/s3select/utils.py @@ -15,11 +15,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import uuid import inspect import json import time import traceback +import uuid + class LogOutput(object): """ @@ -100,7 +101,6 @@ class LogOutput(object): def generate_bucket_name(): return "s3select-test-" + str(uuid.uuid4()) + def generate_object_name(): return str(uuid.uuid4()) - - diff --git a/pkg/trace/trace.go b/pkg/trace/trace.go index 5cc875787..c010bbc03 100644 --- a/pkg/trace/trace.go +++ b/pkg/trace/trace.go @@ -42,6 +42,7 @@ type CallStats struct { // RequestInfo represents trace of http request type RequestInfo struct { Time time.Time `json:"time"` + Proto string `json:"proto"` Method string `json:"method"` Path string `json:"path,omitempty"` RawQuery string `json:"rawquery,omitempty"`