oyd
/
minio
6
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

feat: time to bring back http2.0 support (#10230)

Browse Source 
Bonus move our CI/CD to go1.14
master
Harshavardhana 4 years ago committed by GitHub
parent a49e3647b6
commit 1e2ebc9945
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 349 additions and 282 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      .github/workflows/go.yml
  2. 1
      cmd/http-tracer.go
  3. 9
      cmd/http/server.go
  4. 1
      cmd/lock-rest-client.go
  5. 1
      cmd/prepare-storage.go
  6. 4
      cmd/signals.go
  7. 1
      cmd/storage-rest-client.go
  8. 16
      cmd/utils.go
  9. 2
      go.mod
  10. 4
      go.sum
  11. 198
      mint/run/core/s3select/csv.py
  12. 334
      mint/run/core/s3select/sql_ops.py
  13. 51
      mint/run/core/s3select/tests.py
  14. 6
      mint/run/core/s3select/utils.py
  15. 1
      pkg/trace/trace.go

2
.github/workflows/go.yml
Unescape View File

@ -12,7 +12,7 @@ jobs:
runs-on: ${{ matrix.os }} runs-on: ${{ matrix.os }}
strategy: strategy:
matrix: matrix:
go-version: [1.13.x] go-version: [1.14.x]
os: [ubuntu-latest, windows-latest] os: [ubuntu-latest, windows-latest]
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

1
cmd/http-tracer.go
Unescape View File

@ -129,6 +129,7 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
rq := trace.RequestInfo{ rq := trace.RequestInfo{
Time: time.Now().UTC(), Time: time.Now().UTC(),
Proto: r.Proto,
Method: r.Method, Method: r.Method,
Path: r.URL.Path, Path: r.URL.Path,
RawQuery: r.URL.RawQuery, RawQuery: r.URL.RawQuery,

9
cmd/http/server.go
Unescape View File

@ -191,14 +191,7 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
// TLS hardening // TLS hardening
PreferServerCipherSuites: true, PreferServerCipherSuites: true,
MinVersion: tls.VersionTLS12, MinVersion: tls.VersionTLS12,
// Do not edit the next line, protos priority is kept NextProtos: []string{"h2", "http/1.1"},
// on purpose in this manner for HTTP 2.0, we would
// still like HTTP 2.0 clients to negotiate connection
// to server if needed but by default HTTP 1.1 is
// expected. We need to change this in future
// when we wish to go back to HTTP 2.0 as default
// priority for HTTP protocol negotiation.
NextProtos: []string{"http/1.1", "h2"},
} }
tlsConfig.GetCertificate = getCert tlsConfig.GetCertificate = getCert
} }

1
cmd/lock-rest-client.go
Unescape View File

@ -149,7 +149,6 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
tlsConfig = &tls.Config{ tlsConfig = &tls.Config{
ServerName: endpoint.Hostname(), ServerName: endpoint.Hostname(),
RootCAs: globalRootCAs, RootCAs: globalRootCAs,
NextProtos: []string{"http/1.1"}, // Force http1.1
} }
} }

1
cmd/prepare-storage.go
Unescape View File

@ -185,7 +185,6 @@ func IsServerResolvable(endpoint Endpoint) error {
tlsConfig = &tls.Config{ tlsConfig = &tls.Config{
ServerName: endpoint.Hostname(), ServerName: endpoint.Hostname(),
RootCAs: globalRootCAs, RootCAs: globalRootCAs,
NextProtos: []string{"http/1.1"}, // Force http1.1
} }
} }

4
cmd/signals.go
Unescape View File

@ -18,6 +18,8 @@ package cmd
import ( import (
"context" "context"
"errors"
"net/http"
"os" "os"
"strings" "strings"
@ -73,7 +75,7 @@ func handleSignals() {
if objAPI := newObjectLayerWithoutSafeModeFn(); objAPI != nil { if objAPI := newObjectLayerWithoutSafeModeFn(); objAPI != nil {
objAPI.Shutdown(context.Background()) objAPI.Shutdown(context.Background())
} }
if err != nil { if err != nil && !errors.Is(err, http.ErrServerClosed) {
logger.Fatal(err, "Unable to start MinIO server") logger.Fatal(err, "Unable to start MinIO server")
} }
exit(true) exit(true)

1
cmd/storage-rest-client.go
Unescape View File

@ -663,7 +663,6 @@ func newStorageRESTClient(endpoint Endpoint) *storageRESTClient {
tlsConfig = &tls.Config{ tlsConfig = &tls.Config{
ServerName: endpoint.Hostname(), ServerName: endpoint.Hostname(),
RootCAs: globalRootCAs, RootCAs: globalRootCAs,
NextProtos: []string{"http/1.1"}, // Force http1.1
} }
} }

16
cmd/utils.go
Unescape View File

@ -39,13 +39,13 @@ import (
"sync" "sync"
"time" "time"
humanize "github.com/dustin/go-humanize"
"github.com/gorilla/mux"
xhttp "github.com/minio/minio/cmd/http" xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger" "github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/handlers" "github.com/minio/minio/pkg/handlers"
"github.com/minio/minio/pkg/madmin" "github.com/minio/minio/pkg/madmin"
"golang.org/x/net/http2"
humanize "github.com/dustin/go-humanize"
"github.com/gorilla/mux"
) )
const ( const (
@ -467,6 +467,11 @@ func newInternodeHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration)
// in raw stream. // in raw stream.
DisableCompression: true, DisableCompression: true,
} }
if tlsConfig != nil {
http2.ConfigureTransport(tr)
}
return func() *http.Transport { return func() *http.Transport {
return tr return tr
} }
@ -490,6 +495,11 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) fu
// in raw stream. // in raw stream.
DisableCompression: true, DisableCompression: true,
} }
if tlsConfig != nil {
http2.ConfigureTransport(tr)
}
return func() *http.Transport { return func() *http.Transport {
return tr return tr
} }

2
go.mod
Unescape View File

@ -50,7 +50,7 @@ require (
github.com/minio/minio-go/v7 v7.0.3 github.com/minio/minio-go/v7 v7.0.3
github.com/minio/selfupdate v0.3.1 github.com/minio/selfupdate v0.3.1
github.com/minio/sha256-simd v0.1.1 github.com/minio/sha256-simd v0.1.1
github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37 github.com/minio/simdjson-go v0.1.5
github.com/minio/sio v0.2.0 github.com/minio/sio v0.2.0
github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/go-homedir v1.1.0
github.com/mmcloughlin/avo v0.0.0-20200523190732-4439b6b2c061 // indirect github.com/mmcloughlin/avo v0.0.0-20200523190732-4439b6b2c061 // indirect

4
go.sum
Unescape View File

@ -297,8 +297,8 @@ github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlE
github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM= github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37 h1:pDeao6M5AEd8hwTtGmE0pVKomlL56JFRa5SiXDZAuJE= github.com/minio/simdjson-go v0.1.5 h1:6T5mHh7r3kUvgwhmFWQAjoPV5Yt5oD/VPjAI9ViH1kM=
github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI= github.com/minio/simdjson-go v0.1.5/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI=
github.com/minio/sio v0.2.0 h1:NCRCFLx0r5pRbXf65LVNjxbCGZgNQvNFQkgX3XF4BoA= github.com/minio/sio v0.2.0 h1:NCRCFLx0r5pRbXf65LVNjxbCGZgNQvNFQkgX3XF4BoA=
github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0= github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=

198
mint/run/core/s3select/csv.py
Unescape View File

@ -15,37 +15,42 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import os
import io import io
import os
from minio import Minio from minio import Minio
from minio.select.options import (SelectObjectOptions, CSVInput, from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
RequestProgress, InputSerialization, JsonOutput, OutputSerialization,
OutputSerialization, CSVOutput, JsonOutput) RequestProgress, SelectObjectOptions)
from utils import * from utils import *
def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_output): def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_output):
""" Test if the passed SQL request has the output equal to the passed execpted one""" """ Test if the passed SQL request has the output equal to the passed execpted one"""
object_name = generate_object_name() object_name = generate_object_name()
got_output = b'' got_output = b''
try: try:
bytes_content = io.BytesIO(input_data) bytes_content = io.BytesIO(input_data)
client.put_object(bucket_name, object_name, io.BytesIO(input_data), len(input_data)) client.put_object(bucket_name, object_name,
io.BytesIO(input_data), len(input_data))
data = client.select_object_content(bucket_name, object_name, sql_opts) data = client.select_object_content(bucket_name, object_name, sql_opts)
# Get the records # Get the records
records = io.BytesIO() records = io.BytesIO()
for d in data.stream(10*1024): for d in data.stream(10*1024):
records.write(d.encode('utf-8')) records.write(d.encode('utf-8'))
got_output = records.getvalue() got_output = records.getvalue()
except Exception as select_err: except Exception as select_err:
if not isinstance(expected_output, Exception): if not isinstance(expected_output, Exception):
raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) raise ValueError(
'Test {} unexpectedly failed with: {}'.format(test_name, select_err))
else: else:
if isinstance(expected_output, Exception): if isinstance(expected_output, Exception):
raise ValueError('Test {}: expected an exception, got {}'.format(test_name, got_output)) raise ValueError(
'Test {}: expected an exception, got {}'.format(test_name, got_output))
if got_output != expected_output: if got_output != expected_output:
raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format(test_name, expected_output, got_output)) raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format(
test_name, expected_output, got_output))
finally: finally:
client.remove_object(bucket_name, object_name) client.remove_object(bucket_name, object_name)
@ -55,28 +60,34 @@ def test_csv_input_custom_quote_char(client, log_output):
log_output.args['bucket_name'] = bucket_name = generate_bucket_name() log_output.args['bucket_name'] = bucket_name = generate_bucket_name()
tests = [ tests = [
# Invalid quote character, should fail # Invalid quote character, should fail
('""', '"', b'col1,col2,col3\n', Exception()), ('""', '"', b'col1,col2,col3\n', Exception()),
# UTF-8 quote character # UTF-8 quote character
('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(), b'{"_1":"col1","_2":"col2","_3":"col3"}\n'), ('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(),
# Only one field is quoted b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
('"', '"', b'"col1",col2,col3\n', b'{"_1":"col1","_2":"col2","_3":"col3"}\n'), # Only one field is quoted
('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'), ('"', '"', b'"col1",col2,col3\n',
('\'', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), ('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'),
('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'), ('\'', '"', b'"col1",col2,col3\n',
('', '"', b'"col1","col2","col3"\n', b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'), b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'), ('', '"', b'"col1",col2,col3\n',
('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'), b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'), ('', '"', b'"col1",col2,col3\n',
('"', '\\', b'A\\B,C\n', b'{"_1":"A\\\\B","_2":"C"}\n'), b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
('"', '"', b'"A""B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'), ('', '"', b'"col1","col2","col3"\n',
('"', '\\', b'"A\\B","CD"\n', b'{"_1":"AB","_2":"CD"}\n'), b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'),
('"', '\\', b'"A\\,","CD"\n', b'{"_1":"A,","_2":"CD"}\n'), ('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'),
('"', '\\', b'"A\\"B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'), ('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'),
('"', '\\', b'"A\\""\n', b'{"_1":"A\\""}\n'), ('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'),
('"', '\\', b'"A\\"\\"B"\n', b'{"_1":"A\\"\\"B"}\n'), ('"', '\\', b'A\\B,C\n', b'{"_1":"A\\\\B","_2":"C"}\n'),
('"', '\\', b'"A\\"","\\"B"\n', b'{"_1":"A\\"","_2":"\\"B"}\n'), ('"', '"', b'"A""B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'),
('"', '\\', b'"A\\B","CD"\n', b'{"_1":"AB","_2":"CD"}\n'),
('"', '\\', b'"A\\,","CD"\n', b'{"_1":"A,","_2":"CD"}\n'),
('"', '\\', b'"A\\"B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'),
('"', '\\', b'"A\\""\n', b'{"_1":"A\\""}\n'),
('"', '\\', b'"A\\"\\"B"\n', b'{"_1":"A\\"\\"B"}\n'),
('"', '\\', b'"A\\"","\\"B"\n', b'{"_1":"A\\"","_2":"\\"B"}\n'),
] ]
client.make_bucket(bucket_name) client.make_bucket(bucket_name)
@ -84,54 +95,56 @@ def test_csv_input_custom_quote_char(client, log_output):
try: try:
for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests): for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests):
sql_opts = SelectObjectOptions( sql_opts = SelectObjectOptions(
expression="select * from s3object", expression="select * from s3object",
input_serialization=InputSerialization( input_serialization=InputSerialization(
compression_type="NONE", compression_type="NONE",
csv=CSVInput(FileHeaderInfo="NONE", csv=CSVInput(FileHeaderInfo="NONE",
RecordDelimiter="\n", RecordDelimiter="\n",
FieldDelimiter=",", FieldDelimiter=",",
QuoteCharacter=quote_char, QuoteCharacter=quote_char,
QuoteEscapeCharacter=escape_char, QuoteEscapeCharacter=escape_char,
Comments="#", Comments="#",
AllowQuotedRecordDelimiter="FALSE",), AllowQuotedRecordDelimiter="FALSE",),
), ),
output_serialization=OutputSerialization( output_serialization=OutputSerialization(
json = JsonOutput( json=JsonOutput(
RecordDelimiter="\n", RecordDelimiter="\n",
)
),
request_progress=RequestProgress(
enabled="False"
)
) )
),
test_sql_api(f'test_{idx}', client, bucket_name, data, sql_opts, expected_output) request_progress=RequestProgress(
enabled="False"
)
)
test_sql_api(f'test_{idx}', client, bucket_name,
data, sql_opts, expected_output)
finally: finally:
client.remove_bucket(bucket_name) client.remove_bucket(bucket_name)
# Test passes # Test passes
print(log_output.json_report()) print(log_output.json_report())
def test_csv_output_custom_quote_char(client, log_output): def test_csv_output_custom_quote_char(client, log_output):
# Get a unique bucket_name and object_name # Get a unique bucket_name and object_name
log_output.args['bucket_name'] = bucket_name = generate_bucket_name() log_output.args['bucket_name'] = bucket_name = generate_bucket_name()
tests = [ tests = [
# UTF-8 quote character # UTF-8 quote character
("''", "''", b'col1,col2,col3\n', Exception()), ("''", "''", b'col1,col2,col3\n', Exception()),
("'", "'", b'col1,col2,col3\n', b"'col1','col2','col3'\n"), ("'", "'", b'col1,col2,col3\n', b"'col1','col2','col3'\n"),
("", '"', b'col1,col2,col3\n', b'\x00col1\x00,\x00col2\x00,\x00col3\x00\n'), ("", '"', b'col1,col2,col3\n', b'\x00col1\x00,\x00col2\x00,\x00col3\x00\n'),
('"', '"', b'col1,col2,col3\n', b'"col1","col2","col3"\n'), ('"', '"', b'col1,col2,col3\n', b'"col1","col2","col3"\n'),
('"', '"', b'col"1,col2,col3\n', b'"col""1","col2","col3"\n'), ('"', '"', b'col"1,col2,col3\n', b'"col""1","col2","col3"\n'),
('"', '"', b'""""\n', b'""""\n'), ('"', '"', b'""""\n', b'""""\n'),
('"', '"', b'\n', b''), ('"', '"', b'\n', b''),
("'", "\\", b'col1,col2,col3\n', b"'col1','col2','col3'\n"), ("'", "\\", b'col1,col2,col3\n', b"'col1','col2','col3'\n"),
("'", "\\", b'col""1,col2,col3\n', b"'col\"\"1','col2','col3'\n"), ("'", "\\", b'col""1,col2,col3\n', b"'col\"\"1','col2','col3'\n"),
("'", "\\", b'col\'1,col2,col3\n', b"'col\\'1','col2','col3'\n"), ("'", "\\", b'col\'1,col2,col3\n', b"'col\\'1','col2','col3'\n"),
("'", "\\", b'"col\'1","col2","col3"\n', b"'col\\'1','col2','col3'\n"), ("'", "\\", b'"col\'1","col2","col3"\n', b"'col\\'1','col2','col3'\n"),
("'", "\\", b'col\'\n', b"'col\\''\n"), ("'", "\\", b'col\'\n', b"'col\\''\n"),
# Two consecutive escaped quotes # Two consecutive escaped quotes
("'", "\\", b'"a"""""\n', b"'a\"\"'\n"), ("'", "\\", b'"a"""""\n', b"'a\"\"'\n"),
] ]
client.make_bucket(bucket_name) client.make_bucket(bucket_name)
@ -139,34 +152,33 @@ def test_csv_output_custom_quote_char(client, log_output):
try: try:
for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests): for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests):
sql_opts = SelectObjectOptions( sql_opts = SelectObjectOptions(
expression="select * from s3object", expression="select * from s3object",
input_serialization=InputSerialization( input_serialization=InputSerialization(
compression_type="NONE", compression_type="NONE",
csv=CSVInput(FileHeaderInfo="NONE", csv=CSVInput(FileHeaderInfo="NONE",
RecordDelimiter="\n", RecordDelimiter="\n",
FieldDelimiter=",", FieldDelimiter=",",
QuoteCharacter='"', QuoteCharacter='"',
QuoteEscapeCharacter='"', QuoteEscapeCharacter='"',
Comments="#", Comments="#",
AllowQuotedRecordDelimiter="FALSE",), AllowQuotedRecordDelimiter="FALSE",),
), ),
output_serialization=OutputSerialization( output_serialization=OutputSerialization(
csv=CSVOutput(QuoteFields="ALWAYS", csv=CSVOutput(QuoteFields="ALWAYS",
RecordDelimiter="\n", RecordDelimiter="\n",
FieldDelimiter=",", FieldDelimiter=",",
QuoteCharacter=quote_char, QuoteCharacter=quote_char,
QuoteEscapeCharacter=escape_char,) QuoteEscapeCharacter=escape_char,)
), ),
request_progress=RequestProgress( request_progress=RequestProgress(
enabled="False" enabled="False"
) )
) )
test_sql_api(f'test_{idx}', client, bucket_name, input_data, sql_opts, expected_output) test_sql_api(f'test_{idx}', client, bucket_name,
input_data, sql_opts, expected_output)
finally: finally:
client.remove_bucket(bucket_name) client.remove_bucket(bucket_name)
# Test passes # Test passes
print(log_output.json_report()) print(log_output.json_report())

334
mint/run/core/s3select/sql_ops.py
Unescape View File

@ -18,14 +18,15 @@
import io import io
from datetime import datetime from datetime import datetime
from minio import Minio from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
from minio.select.options import (SelectObjectOptions, CSVInput, JSONInput, JSONInput, OutputSerialization,
RequestProgress, InputSerialization, RequestProgress, SelectObjectOptions)
OutputSerialization, CSVOutput, JsonOutput)
from utils import * from utils import generate_bucket_name, generate_object_name
def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql_output, tests, log_output):
def test_sql_expressions_custom_input_output(client, input_bytes, sql_input,
sql_output, tests, log_output):
bucket_name = generate_bucket_name() bucket_name = generate_bucket_name()
object_name = generate_object_name() object_name = generate_object_name()
@ -48,10 +49,11 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql
output_serialization=sql_output, output_serialization=sql_output,
request_progress=RequestProgress( request_progress=RequestProgress(
enabled="False" enabled="False"
)
) )
)
data = client.select_object_content(bucket_name, object_name, options) data = client.select_object_content(
bucket_name, object_name, options)
# Get the records # Get the records
records = io.BytesIO() records = io.BytesIO()
@ -62,13 +64,15 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql
if got_output != expected_output: if got_output != expected_output:
if type(expected_output) == datetime: if type(expected_output) == datetime:
# Attempt to parse the date which will throw an exception for any issue # Attempt to parse the date which will throw an exception for any issue
datetime.strptime(got_output.decode("utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z') datetime.strptime(got_output.decode(
"utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z')
else: else:
raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(idx+1, expected_output, got_output)) raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(
idx+1, expected_output, got_output))
log_output.args['total_success'] += 1 log_output.args['total_success'] += 1
except Exception as err: except Exception as err:
continue ## TODO, raise instead continue # TODO, raise instead
# raise Exception(err) # raise Exception(err)
finally: finally:
client.remove_object(bucket_name, object_name) client.remove_object(bucket_name, object_name)
@ -77,16 +81,16 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql
def test_sql_expressions(client, input_json_bytes, tests, log_output): def test_sql_expressions(client, input_json_bytes, tests, log_output):
input_serialization = InputSerialization( input_serialization = InputSerialization(
compression_type="NONE", compression_type="NONE",
json=JSONInput(Type="DOCUMENT"), json=JSONInput(Type="DOCUMENT"),
) )
output_serialization=OutputSerialization( output_serialization = OutputSerialization(
csv=CSVOutput(QuoteFields="ASNEEDED") csv=CSVOutput(QuoteFields="ASNEEDED")
) )
test_sql_expressions_custom_input_output(client, input_json_bytes, test_sql_expressions_custom_input_output(client, input_json_bytes,
input_serialization, output_serialization, tests, log_output) input_serialization, output_serialization, tests, log_output)
def test_sql_operators(client, log_output): def test_sql_operators(client, log_output):
@ -98,31 +102,38 @@ def test_sql_operators(client, log_output):
""" """
tests = [ tests = [
# Logical operators # Logical operators
("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'), ("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'),
("NOT", "select * from S3Object s where NOT s.id = 1", b'2,Elliot,4\n3,Yves,5\n4,,0\n'), ("NOT", "select * from S3Object s where NOT s.id = 1",
("OR", "select * from S3Object s where s.id = 1 OR s.id = 3", b'1,John,3\n3,Yves,5\n'), b'2,Elliot,4\n3,Yves,5\n4,,0\n'),
# Comparison Operators ("OR", "select * from S3Object s where s.id = 1 OR s.id = 3",
("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'), b'1,John,3\n3,Yves,5\n'),
(">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'), # Comparison Operators
("<=", "select * from S3Object s where s.age <= 4", b'1,John,3\n2,Elliot,4\n4,,0\n'), ("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'),
(">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'), (">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'),
("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'), ("<=", "select * from S3Object s where s.age <= 4",
("<>", "select * from S3Object s where s.age <> 4", b'1,John,3\n3,Yves,5\n4,,0\n'), b'1,John,3\n2,Elliot,4\n4,,0\n'),
("!=", "select * from S3Object s where s.age != 4", b'1,John,3\n3,Yves,5\n4,,0\n'), (">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'),
("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5", b'2,Elliot,4\n3,Yves,5\n'), ("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'),
("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'), ("<>", "select * from S3Object s where s.age <> 4",
# Pattern Matching Operators b'1,John,3\n3,Yves,5\n4,,0\n'),
("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'), ("!=", "select * from S3Object s where s.age != 4",
("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'), b'1,John,3\n3,Yves,5\n4,,0\n'),
# Unitary Operators ("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5",
("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'), b'2,Elliot,4\n3,Yves,5\n'),
("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL", b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'), ("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'),
# Math Operators # Pattern Matching Operators
("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'), ("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'),
("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'), ("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'),
("*", "select * from S3Object s where s.age = 2*2 ", b'2,Elliot,4\n'), # Unitary Operators
("%", "select * from S3Object s where s.age = 10%6 ", b'2,Elliot,4\n'), ("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'),
("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL",
b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'),
# Math Operators
("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'),
("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'),
("*", "select * from S3Object s where s.age = 2*2 ", b'2,Elliot,4\n'),
("%", "select * from S3Object s where s.age = 10%6 ", b'2,Elliot,4\n'),
] ]
try: try:
@ -141,20 +152,20 @@ def test_sql_operators_precedence(client, log_output):
json_testfile = """{"id": 1, "name": "Eric"}""" json_testfile = """{"id": 1, "name": "Eric"}"""
tests = [ tests = [
("-_1", "select -3*3 from S3Object", b'-9\n'), ("-_1", "select -3*3 from S3Object", b'-9\n'),
("*", "select 10-3*2 from S3Object", b'4\n'), ("*", "select 10-3*2 from S3Object", b'4\n'),
("/", "select 13-10/5 from S3Object", b'11\n'), ("/", "select 13-10/5 from S3Object", b'11\n'),
("%", "select 13-10%5 from S3Object", b'13\n'), ("%", "select 13-10%5 from S3Object", b'13\n'),
("+", "select 1+1*3 from S3Object", b'4\n'), ("+", "select 1+1*3 from S3Object", b'4\n'),
("-_2", "select 1-1*3 from S3Object", b'-2\n'), ("-_2", "select 1-1*3 from S3Object", b'-2\n'),
("=", "select * from S3Object as s where s.id = 13-12", b'1,Eric\n'), ("=", "select * from S3Object as s where s.id = 13-12", b'1,Eric\n'),
("<>", "select * from S3Object as s where s.id <> 1-1", b'1,Eric\n'), ("<>", "select * from S3Object as s where s.id <> 1-1", b'1,Eric\n'),
("NOT", "select * from S3Object where false OR NOT false", b'1,Eric\n'), ("NOT", "select * from S3Object where false OR NOT false", b'1,Eric\n'),
("AND", "select * from S3Object where true AND true OR false ", b'1,Eric\n'), ("AND", "select * from S3Object where true AND true OR false ", b'1,Eric\n'),
("OR", "select * from S3Object where false OR NOT false", b'1,Eric\n'), ("OR", "select * from S3Object where false OR NOT false", b'1,Eric\n'),
("IN", "select * from S3Object as s where s.id <> -1 AND s.id IN (1,2,3)", b'1,Eric\n'), ("IN", "select * from S3Object as s where s.id <> -1 AND s.id IN (1,2,3)", b'1,Eric\n'),
("BETWEEN", "select * from S3Object as s where s.id <> -1 AND s.id BETWEEN -1 AND 3", b'1,Eric\n'), ("BETWEEN", "select * from S3Object as s where s.id <> -1 AND s.id BETWEEN -1 AND 3", b'1,Eric\n'),
("LIKE", "select * from S3Object as s where s.id <> -1 AND s.name LIKE 'E%'", b'1,Eric\n'), ("LIKE", "select * from S3Object as s where s.id <> -1 AND s.name LIKE 'E%'", b'1,Eric\n'),
] ]
try: try:
@ -168,7 +179,6 @@ def test_sql_operators_precedence(client, log_output):
print(log_output.json_report()) print(log_output.json_report())
def test_sql_functions_agg_cond_conv(client, log_output): def test_sql_functions_agg_cond_conv(client, log_output):
json_testfile = """{"id": 1, "name": "John", "age": 3} json_testfile = """{"id": 1, "name": "John", "age": 3}
@ -178,17 +188,18 @@ def test_sql_functions_agg_cond_conv(client, log_output):
{"id": 5, "name": "Eric", "age": 0} {"id": 5, "name": "Eric", "age": 0}
""" """
tests = [ tests = [
# Aggregate functions # Aggregate functions
("COUNT", "select count(*) from S3Object s", b'5\n'), ("COUNT", "select count(*) from S3Object s", b'5\n'),
("AVG", "select avg(s.age) from S3Object s", b'3\n'), ("AVG", "select avg(s.age) from S3Object s", b'3\n'),
("MAX", "select max(s.age) from S3Object s", b'5\n'), ("MAX", "select max(s.age) from S3Object s", b'5\n'),
("MIN", "select min(s.age) from S3Object s", b'0\n'), ("MIN", "select min(s.age) from S3Object s", b'0\n'),
("SUM", "select sum(s.age) from S3Object s", b'12\n'), ("SUM", "select sum(s.age) from S3Object s", b'12\n'),
# Conditional functions # Conditional functions
("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'), ("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'),
("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'), ("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'),
## Conversion functions # Conversion functions
("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s", b'3.0\n4.0\n5.0\n\n0.0\n'), ("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s",
b'3.0\n4.0\n5.0\n\n0.0\n'),
] ]
@ -210,36 +221,49 @@ def test_sql_functions_date(client, log_output):
""" """
tests = [ tests = [
# DATE_ADD # DATE_ADD
("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2022-01-02T03:04:05.006+07:30\n'), ("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-02-02T03:04:05.006+07:30\n'), b'2022-01-02T03:04:05.006+07:30\n'),
("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-01T03:04:05.006+07:30\n'), ("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s",
("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T04:04:05.006+07:30\n'), b'2017-02-02T03:04:05.006+07:30\n'),
("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:09:05.006+07:30\n'), ("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s",
("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:04:10.006+07:30\n'), b'2017-01-01T03:04:05.006+07:30\n'),
# DATE_DIFF ("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s",
("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'), b'2017-01-02T04:04:05.006+07:30\n'),
("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'), ("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
("DATE_DIFF_3", "select DATE_DIFF(day, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2010-01-02T')) from S3Object as s", b'-2556\n'), b'2017-01-02T03:09:05.006+07:30\n'),
# EXTRACT ("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
("EXTRACT_1", "select EXTRACT(year FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017\n'), b'2017-01-02T03:04:10.006+07:30\n'),
("EXTRACT_2", "select EXTRACT(month FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'1\n'), # DATE_DIFF
("EXTRACT_3", "select EXTRACT(hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'3\n'), ("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'),
("EXTRACT_4", "select EXTRACT(minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'4\n'), ("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'),
("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'), ("DATE_DIFF_3", "select DATE_DIFF(day, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2010-01-02T')) from S3Object as s", b'-2556\n'),
("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'), # EXTRACT
# TO_STRING ("EXTRACT_1", "select EXTRACT(year FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017\n'),
("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s", b'"January 2, 2017"\n'), ("EXTRACT_2", "select EXTRACT(month FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'1\n'),
("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'), ("EXTRACT_3", "select EXTRACT(hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'3\n'),
("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'), ("EXTRACT_4", "select EXTRACT(minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'4\n'),
("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'), ("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'),
("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s", b'"January 2, 2017 3:4 AM"\n'), ("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'),
("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), # TO_STRING
("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), ("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s",
("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'), b'"January 2, 2017"\n'),
("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s", b'2017-01-02T3:4:05+07:30\n'), ("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'),
("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s", b'2017-01-02T03:04:05.006+07:30\n'), ("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'),
("UTCNOW", "select UTCNOW() from S3Object", datetime(1,1,1)), ("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'),
("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s",
b'"January 2, 2017 3:4 AM"\n'),
("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s",
b'2017-01-02T3:4:05+0730\n'),
("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s",
b'2017-01-02T3:4:05+0730\n'),
("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s",
b'2017-01-02T3:4:05+0730\n'),
("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s",
b'2017-01-02T3:4:05+07:30\n'),
("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s",
b'2017-01-02T03:04:05.006+07:30\n'),
("UTCNOW", "select UTCNOW() from S3Object", datetime(1, 1, 1)),
] ]
@ -253,6 +277,7 @@ def test_sql_functions_date(client, log_output):
# Test passes # Test passes
print(log_output.json_report()) print(log_output.json_report())
def test_sql_functions_string(client, log_output): def test_sql_functions_string(client, log_output):
json_testfile = """ json_testfile = """
@ -262,23 +287,26 @@ def test_sql_functions_string(client, log_output):
""" """
tests = [ tests = [
# CHAR_LENGTH # CHAR_LENGTH
("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), ("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
("CHARACTER_LENGTH", "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'), ("CHARACTER_LENGTH",
# LOWER "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'), # LOWER
# SUBSTRING ("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'),
("SUBSTRING_1", "select SUBSTRING(s.name FROM 2) from S3Object as s where s.id = 1", b'ohn\n'), # SUBSTRING
("SUBSTRING_2", "select SUBSTRING(s.name FROM 2 FOR 2) from S3Object as s where s.id = 1", b'oh\n'), ("SUBSTRING_1", "select SUBSTRING(s.name FROM 2) from S3Object as s where s.id = 1", b'ohn\n'),
("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'), ("SUBSTRING_2", "select SUBSTRING(s.name FROM 2 FOR 2) from S3Object as s where s.id = 1", b'oh\n'),
# TRIM ("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'),
("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'), # TRIM
("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t \n'), ("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2", b' \tfoobar\t\n'), ("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2",
("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'), b'\tfoobar\t \n'),
("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'), ("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2",
# UPPER b' \tfoobar\t\n'),
("UPPER", "select UPPER(s.name) from S3Object as s where s.id= 1", b'JOHN\n'), ("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'),
# UPPER
("UPPER", "select UPPER(s.name) from S3Object as s where s.id= 1", b'JOHN\n'),
] ]
try: try:
@ -297,14 +325,15 @@ def test_sql_datatypes(client, log_output):
{"name": "John"} {"name": "John"}
""" """
tests = [ tests = [
("bool", "select CAST('true' AS BOOL) from S3Object", b'true\n'), ("bool", "select CAST('true' AS BOOL) from S3Object", b'true\n'),
("int", "select CAST('13' AS INT) from S3Object", b'13\n'), ("int", "select CAST('13' AS INT) from S3Object", b'13\n'),
("integer", "select CAST('13' AS INTEGER) from S3Object", b'13\n'), ("integer", "select CAST('13' AS INTEGER) from S3Object", b'13\n'),
("string", "select CAST(true AS STRING) from S3Object", b'true\n'), ("string", "select CAST(true AS STRING) from S3Object", b'true\n'),
("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'), ("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'),
("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'), ("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'), ("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object", b'2007-04-05T14:30Z\n'), ("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object",
b'2007-04-05T14:30Z\n'),
] ]
try: try:
@ -323,14 +352,17 @@ def test_sql_select(client, log_output):
json_testfile = """{"id": 1, "created": "June 27", "modified": "July 6" } json_testfile = """{"id": 1, "created": "June 27", "modified": "July 6" }
{"id": 2, "Created": "June 28", "Modified": "July 7", "Cast": "Random Date" }""" {"id": 2, "Created": "June 28", "Modified": "July 7", "Cast": "Random Date" }"""
tests = [ tests = [
("select_1", "select * from S3Object", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), ("select_1", "select * from S3Object",
("select_2", "select * from S3Object s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
("select_3", "select * from S3Object as s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'), ("select_2", "select * from S3Object s",
("select_4", "select s.line from S3Object as s", b'\n\n'), b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'), ("select_3", "select * from S3Object as s",
("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'), b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
("where", 'select s.created from S3Object as s', b'June 27\nJune 28\n'), ("select_4", "select s.line from S3Object as s", b'\n\n'),
("limit", 'select * from S3Object as s LIMIT 1', b'1,June 27,July 6\n'), ("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'),
("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'),
("where", 'select s.created from S3Object as s', b'June 27\nJune 28\n'),
("limit", 'select * from S3Object as s LIMIT 1', b'1,June 27,July 6\n'),
] ]
try: try:
@ -343,23 +375,29 @@ def test_sql_select(client, log_output):
# Test passes # Test passes
print(log_output.json_report()) print(log_output.json_report())
def test_sql_select_json(client, log_output): def test_sql_select_json(client, log_output):
json_testcontent = """{ "Rules": [ {"id": "1"}, {"expr": "y > x"}, {"id": "2", "expr": "z = DEBUG"} ]} json_testcontent = """{ "Rules": [ {"id": "1"}, {"expr": "y > x"}, {"id": "2", "expr": "z = DEBUG"} ]}
{ "created": "June 27", "modified": "July 6" } { "created": "June 27", "modified": "July 6" }
""" """
tests = [ tests = [
("select_1", "SELECT id FROM S3Object[*].Rules[*].id", b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'), ("select_1", "SELECT id FROM S3Object[*].Rules[*].id",
("select_2", "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'), b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'),
("select_3", "SELECT d.created, d.modified FROM S3Object[*] d", b'{}\n{"created":"June 27","modified":"July 6"}\n'), ("select_2",
("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]", b'{}\n{"created":"June 27","modified":"July 6"}\n'), "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'),
("select_5", "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'), ("select_3", "SELECT d.created, d.modified FROM S3Object[*] d",
b'{}\n{"created":"June 27","modified":"July 6"}\n'),
("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]",
b'{}\n{"created":"June 27","modified":"July 6"}\n'),
("select_5",
"Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'),
] ]
input_serialization = InputSerialization(json=JSONInput(Type="DOCUMENT")) input_serialization = InputSerialization(json=JSONInput(Type="DOCUMENT"))
output_serialization = OutputSerialization(json=JsonOutput()) output_serialization = OutputSerialization(json=JsonOutput())
try: try:
test_sql_expressions_custom_input_output(client, json_testcontent, test_sql_expressions_custom_input_output(client, json_testcontent,
input_serialization, output_serialization, tests, log_output) input_serialization, output_serialization, tests, log_output)
except Exception as select_err: except Exception as select_err:
raise select_err raise select_err
# raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
@ -374,20 +412,20 @@ def test_sql_select_csv_no_header(client, log_output):
val4,val5,val6 val4,val5,val6
""" """
tests = [ tests = [
("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'), ("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'),
] ]
input_serialization=InputSerialization( input_serialization = InputSerialization(
csv=CSVInput( csv=CSVInput(
FileHeaderInfo="NONE", FileHeaderInfo="NONE",
AllowQuotedRecordDelimiter="FALSE", AllowQuotedRecordDelimiter="FALSE",
), ),
) )
output_serialization=OutputSerialization(csv=CSVOutput()) output_serialization = OutputSerialization(csv=CSVOutput())
try: try:
test_sql_expressions_custom_input_output(client, json_testcontent, test_sql_expressions_custom_input_output(client, json_testcontent,
input_serialization, output_serialization, tests, log_output) input_serialization, output_serialization, tests, log_output)
except Exception as select_err: except Exception as select_err:
raise select_err raise select_err
# raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err)) # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
@ -395,5 +433,3 @@ val4,val5,val6
# Test passes # Test passes
print(log_output.json_report()) print(log_output.json_report())

51
mint/run/core/s3select/tests.py
Unescape View File

@ -16,12 +16,19 @@
# limitations under the License. # limitations under the License.
import os import os
from sys import exit import sys
from csv import (test_csv_input_custom_quote_char,
test_csv_output_custom_quote_char)
from minio import Minio from minio import Minio
from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv,
test_sql_functions_date, test_sql_functions_string,
test_sql_operators, test_sql_operators_precedence,
test_sql_select, test_sql_select_csv_no_header,
test_sql_select_json)
from utils import LogOutput from utils import LogOutput
from sql_ops import *
from csv import *
def main(): def main():
""" """
@ -39,48 +46,56 @@ def main():
secret_key = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' secret_key = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG'
secure = True secure = True
client = Minio(server_endpoint, access_key, secret_key, secure=False) client = Minio(server_endpoint, access_key, secret_key, secure=secure)
log_output = LogOutput(client.select_object_content, 'test_csv_input_quote_char') log_output = LogOutput(client.select_object_content,
'test_csv_input_quote_char')
test_csv_input_custom_quote_char(client, log_output) test_csv_input_custom_quote_char(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_csv_output_quote_char') log_output = LogOutput(client.select_object_content,
'test_csv_output_quote_char')
test_csv_output_custom_quote_char(client, log_output) test_csv_output_custom_quote_char(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_operators') log_output = LogOutput(
client.select_object_content, 'test_sql_operators')
test_sql_operators(client, log_output) test_sql_operators(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_operators_precedence') log_output = LogOutput(client.select_object_content,
'test_sql_operators_precedence')
test_sql_operators_precedence(client, log_output) test_sql_operators_precedence(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_functions_agg_cond_conv') log_output = LogOutput(client.select_object_content,
'test_sql_functions_agg_cond_conv')
test_sql_functions_agg_cond_conv(client, log_output) test_sql_functions_agg_cond_conv(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_functions_date') log_output = LogOutput(
client.select_object_content, 'test_sql_functions_date')
test_sql_functions_date(client, log_output) test_sql_functions_date(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_functions_string') log_output = LogOutput(client.select_object_content,
'test_sql_functions_string')
test_sql_functions_string(client, log_output) test_sql_functions_string(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_datatypes') log_output = LogOutput(
client.select_object_content, 'test_sql_datatypes')
test_sql_datatypes(client, log_output) test_sql_datatypes(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_select') log_output = LogOutput(client.select_object_content, 'test_sql_select')
test_sql_select(client, log_output) test_sql_select(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_select_json') log_output = LogOutput(
client.select_object_content, 'test_sql_select_json')
test_sql_select_json(client, log_output) test_sql_select_json(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_select_csv') log_output = LogOutput(
client.select_object_content, 'test_sql_select_csv')
test_sql_select_csv_no_header(client, log_output) test_sql_select_csv_no_header(client, log_output)
except Exception as err: except Exception as err:
print(log_output.json_report(err)) print(log_output.json_report(err))
exit(1) sys.exit(1)
if __name__ == "__main__": if __name__ == "__main__":
# Execute only if run as a script # Execute only if run as a script
main() main()

6
mint/run/core/s3select/utils.py
Unescape View File

@ -15,11 +15,12 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import uuid
import inspect import inspect
import json import json
import time import time
import traceback import traceback
import uuid
class LogOutput(object): class LogOutput(object):
""" """
@ -100,7 +101,6 @@ class LogOutput(object):
def generate_bucket_name(): def generate_bucket_name():
return "s3select-test-" + str(uuid.uuid4()) return "s3select-test-" + str(uuid.uuid4())
def generate_object_name(): def generate_object_name():
return str(uuid.uuid4()) return str(uuid.uuid4())

1
pkg/trace/trace.go
Unescape View File

@ -42,6 +42,7 @@ type CallStats struct {
// RequestInfo represents trace of http request // RequestInfo represents trace of http request
type RequestInfo struct { type RequestInfo struct {
Time time.Time `json:"time"` Time time.Time `json:"time"`
Proto string `json:"proto"`
Method string `json:"method"` Method string `json:"method"`
Path string `json:"path,omitempty"` Path string `json:"path,omitempty"`
RawQuery string `json:"rawquery,omitempty"` RawQuery string `json:"rawquery,omitempty"`

Write Preview
Loading…
Cancel
Save