feat: time to bring back http2.0 support (#10230)

Bonus move our CI/CD to go1.14
master
Harshavardhana 4 years ago committed by GitHub
parent a49e3647b6
commit 1e2ebc9945
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      .github/workflows/go.yml
  2. 1
      cmd/http-tracer.go
  3. 9
      cmd/http/server.go
  4. 1
      cmd/lock-rest-client.go
  5. 1
      cmd/prepare-storage.go
  6. 4
      cmd/signals.go
  7. 1
      cmd/storage-rest-client.go
  8. 16
      cmd/utils.go
  9. 2
      go.mod
  10. 4
      go.sum
  11. 50
      mint/run/core/s3select/csv.py
  12. 138
      mint/run/core/s3select/sql_ops.py
  13. 51
      mint/run/core/s3select/tests.py
  14. 6
      mint/run/core/s3select/utils.py
  15. 1
      pkg/trace/trace.go

@ -12,7 +12,7 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
go-version: [1.13.x]
go-version: [1.14.x]
os: [ubuntu-latest, windows-latest]
steps:
- uses: actions/checkout@v2

@ -129,6 +129,7 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
rq := trace.RequestInfo{
Time: time.Now().UTC(),
Proto: r.Proto,
Method: r.Method,
Path: r.URL.Path,
RawQuery: r.URL.RawQuery,

@ -191,14 +191,7 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
// TLS hardening
PreferServerCipherSuites: true,
MinVersion: tls.VersionTLS12,
// Do not edit the next line, protos priority is kept
// on purpose in this manner for HTTP 2.0, we would
// still like HTTP 2.0 clients to negotiate connection
// to server if needed but by default HTTP 1.1 is
// expected. We need to change this in future
// when we wish to go back to HTTP 2.0 as default
// priority for HTTP protocol negotiation.
NextProtos: []string{"http/1.1", "h2"},
NextProtos: []string{"h2", "http/1.1"},
}
tlsConfig.GetCertificate = getCert
}

@ -149,7 +149,6 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
tlsConfig = &tls.Config{
ServerName: endpoint.Hostname(),
RootCAs: globalRootCAs,
NextProtos: []string{"http/1.1"}, // Force http1.1
}
}

@ -185,7 +185,6 @@ func IsServerResolvable(endpoint Endpoint) error {
tlsConfig = &tls.Config{
ServerName: endpoint.Hostname(),
RootCAs: globalRootCAs,
NextProtos: []string{"http/1.1"}, // Force http1.1
}
}

@ -18,6 +18,8 @@ package cmd
import (
"context"
"errors"
"net/http"
"os"
"strings"
@ -73,7 +75,7 @@ func handleSignals() {
if objAPI := newObjectLayerWithoutSafeModeFn(); objAPI != nil {
objAPI.Shutdown(context.Background())
}
if err != nil {
if err != nil && !errors.Is(err, http.ErrServerClosed) {
logger.Fatal(err, "Unable to start MinIO server")
}
exit(true)

@ -663,7 +663,6 @@ func newStorageRESTClient(endpoint Endpoint) *storageRESTClient {
tlsConfig = &tls.Config{
ServerName: endpoint.Hostname(),
RootCAs: globalRootCAs,
NextProtos: []string{"http/1.1"}, // Force http1.1
}
}

@ -39,13 +39,13 @@ import (
"sync"
"time"
humanize "github.com/dustin/go-humanize"
"github.com/gorilla/mux"
xhttp "github.com/minio/minio/cmd/http"
"github.com/minio/minio/cmd/logger"
"github.com/minio/minio/pkg/handlers"
"github.com/minio/minio/pkg/madmin"
humanize "github.com/dustin/go-humanize"
"github.com/gorilla/mux"
"golang.org/x/net/http2"
)
const (
@ -467,6 +467,11 @@ func newInternodeHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration)
// in raw stream.
DisableCompression: true,
}
if tlsConfig != nil {
http2.ConfigureTransport(tr)
}
return func() *http.Transport {
return tr
}
@ -490,6 +495,11 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) fu
// in raw stream.
DisableCompression: true,
}
if tlsConfig != nil {
http2.ConfigureTransport(tr)
}
return func() *http.Transport {
return tr
}

@ -50,7 +50,7 @@ require (
github.com/minio/minio-go/v7 v7.0.3
github.com/minio/selfupdate v0.3.1
github.com/minio/sha256-simd v0.1.1
github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37
github.com/minio/simdjson-go v0.1.5
github.com/minio/sio v0.2.0
github.com/mitchellh/go-homedir v1.1.0
github.com/mmcloughlin/avo v0.0.0-20200523190732-4439b6b2c061 // indirect

@ -297,8 +297,8 @@ github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlE
github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37 h1:pDeao6M5AEd8hwTtGmE0pVKomlL56JFRa5SiXDZAuJE=
github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI=
github.com/minio/simdjson-go v0.1.5 h1:6T5mHh7r3kUvgwhmFWQAjoPV5Yt5oD/VPjAI9ViH1kM=
github.com/minio/simdjson-go v0.1.5/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI=
github.com/minio/sio v0.2.0 h1:NCRCFLx0r5pRbXf65LVNjxbCGZgNQvNFQkgX3XF4BoA=
github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=

@ -15,23 +15,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import io
import os
from minio import Minio
from minio.select.options import (SelectObjectOptions, CSVInput,
RequestProgress, InputSerialization,
OutputSerialization, CSVOutput, JsonOutput)
from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
JsonOutput, OutputSerialization,
RequestProgress, SelectObjectOptions)
from utils import *
def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_output):
""" Test if the passed SQL request has the output equal to the passed execpted one"""
object_name = generate_object_name()
got_output = b''
try:
bytes_content = io.BytesIO(input_data)
client.put_object(bucket_name, object_name, io.BytesIO(input_data), len(input_data))
client.put_object(bucket_name, object_name,
io.BytesIO(input_data), len(input_data))
data = client.select_object_content(bucket_name, object_name, sql_opts)
# Get the records
records = io.BytesIO()
@ -40,12 +42,15 @@ def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_
got_output = records.getvalue()
except Exception as select_err:
if not isinstance(expected_output, Exception):
raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
raise ValueError(
'Test {} unexpectedly failed with: {}'.format(test_name, select_err))
else:
if isinstance(expected_output, Exception):
raise ValueError('Test {}: expected an exception, got {}'.format(test_name, got_output))
raise ValueError(
'Test {}: expected an exception, got {}'.format(test_name, got_output))
if got_output != expected_output:
raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format(test_name, expected_output, got_output))
raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format(
test_name, expected_output, got_output))
finally:
client.remove_object(bucket_name, object_name)
@ -58,14 +63,20 @@ def test_csv_input_custom_quote_char(client, log_output):
# Invalid quote character, should fail
('""', '"', b'col1,col2,col3\n', Exception()),
# UTF-8 quote character
('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(), b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(),
b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
# Only one field is quoted
('"', '"', b'"col1",col2,col3\n', b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
('"', '"', b'"col1",col2,col3\n',
b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'),
('\'', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
('', '"', b'"col1","col2","col3"\n', b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'),
('\'', '"', b'"col1",col2,col3\n',
b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
('', '"', b'"col1",col2,col3\n',
b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
('', '"', b'"col1",col2,col3\n',
b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
('', '"', b'"col1","col2","col3"\n',
b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'),
('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'),
('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'),
('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'),
@ -96,7 +107,7 @@ def test_csv_input_custom_quote_char(client, log_output):
AllowQuotedRecordDelimiter="FALSE",),
),
output_serialization=OutputSerialization(
json = JsonOutput(
json=JsonOutput(
RecordDelimiter="\n",
)
),
@ -105,13 +116,15 @@ def test_csv_input_custom_quote_char(client, log_output):
)
)
test_sql_api(f'test_{idx}', client, bucket_name, data, sql_opts, expected_output)
test_sql_api(f'test_{idx}', client, bucket_name,
data, sql_opts, expected_output)
finally:
client.remove_bucket(bucket_name)
# Test passes
print(log_output.json_report())
def test_csv_output_custom_quote_char(client, log_output):
# Get a unique bucket_name and object_name
log_output.args['bucket_name'] = bucket_name = generate_bucket_name()
@ -162,11 +175,10 @@ def test_csv_output_custom_quote_char(client, log_output):
)
)
test_sql_api(f'test_{idx}', client, bucket_name, input_data, sql_opts, expected_output)
test_sql_api(f'test_{idx}', client, bucket_name,
input_data, sql_opts, expected_output)
finally:
client.remove_bucket(bucket_name)
# Test passes
print(log_output.json_report())

@ -18,14 +18,15 @@
import io
from datetime import datetime
from minio import Minio
from minio.select.options import (SelectObjectOptions, CSVInput, JSONInput,
RequestProgress, InputSerialization,
OutputSerialization, CSVOutput, JsonOutput)
from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
JSONInput, OutputSerialization,
RequestProgress, SelectObjectOptions)
from utils import *
from utils import generate_bucket_name, generate_object_name
def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql_output, tests, log_output):
def test_sql_expressions_custom_input_output(client, input_bytes, sql_input,
sql_output, tests, log_output):
bucket_name = generate_bucket_name()
object_name = generate_object_name()
@ -51,7 +52,8 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql
)
)
data = client.select_object_content(bucket_name, object_name, options)
data = client.select_object_content(
bucket_name, object_name, options)
# Get the records
records = io.BytesIO()
@ -62,13 +64,15 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql
if got_output != expected_output:
if type(expected_output) == datetime:
# Attempt to parse the date which will throw an exception for any issue
datetime.strptime(got_output.decode("utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z')
datetime.strptime(got_output.decode(
"utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z')
else:
raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(idx+1, expected_output, got_output))
raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(
idx+1, expected_output, got_output))
log_output.args['total_success'] += 1
except Exception as err:
continue ## TODO, raise instead
continue # TODO, raise instead
# raise Exception(err)
finally:
client.remove_object(bucket_name, object_name)
@ -81,7 +85,7 @@ def test_sql_expressions(client, input_json_bytes, tests, log_output):
json=JSONInput(Type="DOCUMENT"),
)
output_serialization=OutputSerialization(
output_serialization = OutputSerialization(
csv=CSVOutput(QuoteFields="ASNEEDED")
)
@ -100,24 +104,31 @@ def test_sql_operators(client, log_output):
tests = [
# Logical operators
("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'),
("NOT", "select * from S3Object s where NOT s.id = 1", b'2,Elliot,4\n3,Yves,5\n4,,0\n'),
("OR", "select * from S3Object s where s.id = 1 OR s.id = 3", b'1,John,3\n3,Yves,5\n'),
("NOT", "select * from S3Object s where NOT s.id = 1",
b'2,Elliot,4\n3,Yves,5\n4,,0\n'),
("OR", "select * from S3Object s where s.id = 1 OR s.id = 3",
b'1,John,3\n3,Yves,5\n'),
# Comparison Operators
("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'),
(">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'),
("<=", "select * from S3Object s where s.age <= 4", b'1,John,3\n2,Elliot,4\n4,,0\n'),
("<=", "select * from S3Object s where s.age <= 4",
b'1,John,3\n2,Elliot,4\n4,,0\n'),
(">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'),
("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'),
("<>", "select * from S3Object s where s.age <> 4", b'1,John,3\n3,Yves,5\n4,,0\n'),
("!=", "select * from S3Object s where s.age != 4", b'1,John,3\n3,Yves,5\n4,,0\n'),
("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5", b'2,Elliot,4\n3,Yves,5\n'),
("<>", "select * from S3Object s where s.age <> 4",
b'1,John,3\n3,Yves,5\n4,,0\n'),
("!=", "select * from S3Object s where s.age != 4",
b'1,John,3\n3,Yves,5\n4,,0\n'),
("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5",
b'2,Elliot,4\n3,Yves,5\n'),
("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'),
# Pattern Matching Operators
("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'),
("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'),
# Unitary Operators
("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'),
("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL", b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'),
("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL",
b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'),
# Math Operators
("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'),
("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'),
@ -168,7 +179,6 @@ def test_sql_operators_precedence(client, log_output):
print(log_output.json_report())
def test_sql_functions_agg_cond_conv(client, log_output):
json_testfile = """{"id": 1, "name": "John", "age": 3}
@ -187,8 +197,9 @@ def test_sql_functions_agg_cond_conv(client, log_output):
# Conditional functions
("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'),
("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'),
## Conversion functions
("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s", b'3.0\n4.0\n5.0\n\n0.0\n'),
# Conversion functions
("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s",
b'3.0\n4.0\n5.0\n\n0.0\n'),
]
@ -211,12 +222,18 @@ def test_sql_functions_date(client, log_output):
tests = [
# DATE_ADD
("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2022-01-02T03:04:05.006+07:30\n'),
("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-02-02T03:04:05.006+07:30\n'),
("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-01T03:04:05.006+07:30\n'),
("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T04:04:05.006+07:30\n'),
("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:09:05.006+07:30\n'),
("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:04:10.006+07:30\n'),
("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
b'2022-01-02T03:04:05.006+07:30\n'),
("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s",
b'2017-02-02T03:04:05.006+07:30\n'),
("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s",
b'2017-01-01T03:04:05.006+07:30\n'),
("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s",
b'2017-01-02T04:04:05.006+07:30\n'),
("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
b'2017-01-02T03:09:05.006+07:30\n'),
("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
b'2017-01-02T03:04:10.006+07:30\n'),
# DATE_DIFF
("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'),
("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'),
@ -229,17 +246,24 @@ def test_sql_functions_date(client, log_output):
("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'),
("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'),
# TO_STRING
("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s", b'"January 2, 2017"\n'),
("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s",
b'"January 2, 2017"\n'),
("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'),
("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'),
("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'),
("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s", b'"January 2, 2017 3:4 AM"\n'),
("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s", b'2017-01-02T3:4:05+07:30\n'),
("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s", b'2017-01-02T03:04:05.006+07:30\n'),
("UTCNOW", "select UTCNOW() from S3Object", datetime(1,1,1)),
("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s",
b'"January 2, 2017 3:4 AM"\n'),
("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s",
b'2017-01-02T3:4:05+0730\n'),
("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s",
b'2017-01-02T3:4:05+0730\n'),
("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s",
b'2017-01-02T3:4:05+0730\n'),
("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s",
b'2017-01-02T3:4:05+07:30\n'),
("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s",
b'2017-01-02T03:04:05.006+07:30\n'),
("UTCNOW", "select UTCNOW() from S3Object", datetime(1, 1, 1)),
]
@ -253,6 +277,7 @@ def test_sql_functions_date(client, log_output):
# Test passes
print(log_output.json_report())
def test_sql_functions_string(client, log_output):
json_testfile = """
@ -264,7 +289,8 @@ def test_sql_functions_string(client, log_output):
tests = [
# CHAR_LENGTH
("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
("CHARACTER_LENGTH", "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
("CHARACTER_LENGTH",
"select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
# LOWER
("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'),
# SUBSTRING
@ -273,8 +299,10 @@ def test_sql_functions_string(client, log_output):
("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'),
# TRIM
("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t \n'),
("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2", b' \tfoobar\t\n'),
("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2",
b'\tfoobar\t \n'),
("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2",
b' \tfoobar\t\n'),
("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'),
# UPPER
@ -304,7 +332,8 @@ def test_sql_datatypes(client, log_output):
("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'),
("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object", b'2007-04-05T14:30Z\n'),
("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object",
b'2007-04-05T14:30Z\n'),
]
try:
@ -323,9 +352,12 @@ def test_sql_select(client, log_output):
json_testfile = """{"id": 1, "created": "June 27", "modified": "July 6" }
{"id": 2, "Created": "June 28", "Modified": "July 7", "Cast": "Random Date" }"""
tests = [
("select_1", "select * from S3Object", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
("select_2", "select * from S3Object s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
("select_3", "select * from S3Object as s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
("select_1", "select * from S3Object",
b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
("select_2", "select * from S3Object s",
b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
("select_3", "select * from S3Object as s",
b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
("select_4", "select s.line from S3Object as s", b'\n\n'),
("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'),
("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'),
@ -343,16 +375,22 @@ def test_sql_select(client, log_output):
# Test passes
print(log_output.json_report())
def test_sql_select_json(client, log_output):
json_testcontent = """{ "Rules": [ {"id": "1"}, {"expr": "y > x"}, {"id": "2", "expr": "z = DEBUG"} ]}
{ "created": "June 27", "modified": "July 6" }
"""
tests = [
("select_1", "SELECT id FROM S3Object[*].Rules[*].id", b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'),
("select_2", "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'),
("select_3", "SELECT d.created, d.modified FROM S3Object[*] d", b'{}\n{"created":"June 27","modified":"July 6"}\n'),
("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]", b'{}\n{"created":"June 27","modified":"July 6"}\n'),
("select_5", "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'),
("select_1", "SELECT id FROM S3Object[*].Rules[*].id",
b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'),
("select_2",
"SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'),
("select_3", "SELECT d.created, d.modified FROM S3Object[*] d",
b'{}\n{"created":"June 27","modified":"July 6"}\n'),
("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]",
b'{}\n{"created":"June 27","modified":"July 6"}\n'),
("select_5",
"Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'),
]
input_serialization = InputSerialization(json=JSONInput(Type="DOCUMENT"))
@ -377,14 +415,14 @@ val4,val5,val6
("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'),
]
input_serialization=InputSerialization(
input_serialization = InputSerialization(
csv=CSVInput(
FileHeaderInfo="NONE",
AllowQuotedRecordDelimiter="FALSE",
),
)
output_serialization=OutputSerialization(csv=CSVOutput())
output_serialization = OutputSerialization(csv=CSVOutput())
try:
test_sql_expressions_custom_input_output(client, json_testcontent,
input_serialization, output_serialization, tests, log_output)
@ -395,5 +433,3 @@ val4,val5,val6
# Test passes
print(log_output.json_report())

@ -16,12 +16,19 @@
# limitations under the License.
import os
from sys import exit
import sys
from csv import (test_csv_input_custom_quote_char,
test_csv_output_custom_quote_char)
from minio import Minio
from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv,
test_sql_functions_date, test_sql_functions_string,
test_sql_operators, test_sql_operators_precedence,
test_sql_select, test_sql_select_csv_no_header,
test_sql_select_json)
from utils import LogOutput
from sql_ops import *
from csv import *
def main():
"""
@ -39,48 +46,56 @@ def main():
secret_key = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG'
secure = True
client = Minio(server_endpoint, access_key, secret_key, secure=False)
client = Minio(server_endpoint, access_key, secret_key, secure=secure)
log_output = LogOutput(client.select_object_content, 'test_csv_input_quote_char')
log_output = LogOutput(client.select_object_content,
'test_csv_input_quote_char')
test_csv_input_custom_quote_char(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_csv_output_quote_char')
log_output = LogOutput(client.select_object_content,
'test_csv_output_quote_char')
test_csv_output_custom_quote_char(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_operators')
log_output = LogOutput(
client.select_object_content, 'test_sql_operators')
test_sql_operators(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_operators_precedence')
log_output = LogOutput(client.select_object_content,
'test_sql_operators_precedence')
test_sql_operators_precedence(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_functions_agg_cond_conv')
log_output = LogOutput(client.select_object_content,
'test_sql_functions_agg_cond_conv')
test_sql_functions_agg_cond_conv(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_functions_date')
log_output = LogOutput(
client.select_object_content, 'test_sql_functions_date')
test_sql_functions_date(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_functions_string')
log_output = LogOutput(client.select_object_content,
'test_sql_functions_string')
test_sql_functions_string(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_datatypes')
log_output = LogOutput(
client.select_object_content, 'test_sql_datatypes')
test_sql_datatypes(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_select')
test_sql_select(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_select_json')
log_output = LogOutput(
client.select_object_content, 'test_sql_select_json')
test_sql_select_json(client, log_output)
log_output = LogOutput(client.select_object_content, 'test_sql_select_csv')
log_output = LogOutput(
client.select_object_content, 'test_sql_select_csv')
test_sql_select_csv_no_header(client, log_output)
except Exception as err:
print(log_output.json_report(err))
exit(1)
sys.exit(1)
if __name__ == "__main__":
# Execute only if run as a script
main()

@ -15,11 +15,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import uuid
import inspect
import json
import time
import traceback
import uuid
class LogOutput(object):
"""
@ -100,7 +101,6 @@ class LogOutput(object):
def generate_bucket_name():
return "s3select-test-" + str(uuid.uuid4())
def generate_object_name():
return str(uuid.uuid4())

@ -42,6 +42,7 @@ type CallStats struct {
// RequestInfo represents trace of http request
type RequestInfo struct {
Time time.Time `json:"time"`
Proto string `json:"proto"`
Method string `json:"method"`
Path string `json:"path,omitempty"`
RawQuery string `json:"rawquery,omitempty"`

Loading…
Cancel
Save