From 6128304f6ea8f29a6540521355c5a78e91dba650 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 18 Dec 2020 19:45:59 -0800 Subject: [PATCH] fix: regression introduced in aws-sdk-go tests regression was introduced by cce5d7152a6666b5401505a9ebc5ddc78562d5f5 which incorrectly implemented the tests and fixed a wrong requirement, CompleteMultipart should never succeed in the tests. --- mint/build/minio-py/install.sh | 4 ++-- mint/run/core/aws-sdk-go/main.go | 34 +++++++++++++++++-------------- mint/run/core/s3select/csv.py | 13 ++++++------ mint/run/core/s3select/sql_ops.py | 11 +++++----- 4 files changed, 32 insertions(+), 30 deletions(-) diff --git a/mint/build/minio-py/install.sh b/mint/build/minio-py/install.sh index ca877cc01..0319454c8 100755 --- a/mint/build/minio-py/install.sh +++ b/mint/build/minio-py/install.sh @@ -23,5 +23,5 @@ fi test_run_dir="$MINT_RUN_CORE_DIR/minio-py" pip3 install --user faker -pip3 install git+http://github.com/minio/minio-py -$WGET --output-document="$test_run_dir/tests.py" "https://raw.githubusercontent.com/minio/minio-py/master/tests/functional/tests.py" +pip3 install minio==${MINIO_PY_VERSION} +$WGET --output-document="$test_run_dir/tests.py" "https://raw.githubusercontent.com/minio/minio-py/${MINIO_PY_VERSION}/tests/functional/tests.py" diff --git a/mint/run/core/aws-sdk-go/main.go b/mint/run/core/aws-sdk-go/main.go index fe7c3f9f6..b7010314d 100644 --- a/mint/run/core/aws-sdk-go/main.go +++ b/mint/run/core/aws-sdk-go/main.go @@ -885,7 +885,7 @@ func testListMultipartUploads(s3Client *s3.S3) { failureLog(function, args, startTime, "", "AWS SDK Go createMultipartupload API failed", err).Fatal() return } - parts := make(map[*int64]*string) + parts := make([]*string, 5) for i := 0; i < 5; i++ { result, errUpload := s3Client.UploadPart(&s3.UploadPartInput{ Bucket: aws.String(bucket), @@ -903,7 +903,7 @@ func testListMultipartUploads(s3Client *s3.S3) { failureLog(function, args, startTime, "", "AWS SDK Go uploadPart API failed for", errUpload).Fatal() return } - parts[aws.Int64(int64(i+1))] = result.ETag + parts[i] = result.ETag } listParts, errParts := s3Client.ListParts(&s3.ListPartsInput{ @@ -917,21 +917,20 @@ func testListMultipartUploads(s3Client *s3.S3) { } if len(parts) != len(listParts.Parts) { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go ListParts.Parts len mismatch want: %v got: %v", len(parts), len(listParts.Parts)), err).Fatal() + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go ListParts.Parts len mismatch want: %d got: %d", len(parts), len(listParts.Parts)), err).Fatal() return } completedParts := make([]*s3.CompletedPart, len(parts)) - for _, part := range listParts.Parts { - if tag, ok := parts[part.PartNumber]; ok { - if tag != part.ETag { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go ListParts.Parts output mismatch want: %v got: %v", tag, part.ETag), err).Fatal() - return - } - completedParts = append(completedParts, &s3.CompletedPart{ - ETag: part.ETag, - PartNumber: part.PartNumber, - }) + for i, part := range listParts.Parts { + tag := parts[i] + if *tag != *part.ETag { + failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go ListParts.Parts output mismatch want: %#v got: %#v", tag, part.ETag), err).Fatal() + return + } + completedParts[i] = &s3.CompletedPart{ + ETag: part.ETag, + PartNumber: part.PartNumber, } } @@ -942,8 +941,13 @@ func testListMultipartUploads(s3Client *s3.S3) { Parts: completedParts}, UploadId: multipartUpload.UploadId, }) - if err != nil { - failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go CompleteMultipartUpload failed"), err).Fatal() + if err == nil { + failureLog(function, args, startTime, "", "AWS SDK Go CompleteMultipartUpload is expected to fail but succeeded", errors.New("expected nil")).Fatal() + return + } + + if err.(s3.RequestFailure).Code() != "EntityTooSmall" { + failureLog(function, args, startTime, "", "AWS SDK Go CompleteMultipartUpload is expected to fail with EntityTooSmall", err).Fatal() return } diff --git a/mint/run/core/s3select/csv.py b/mint/run/core/s3select/csv.py index cf1eaaaa3..7d4bf2321 100644 --- a/mint/run/core/s3select/csv.py +++ b/mint/run/core/s3select/csv.py @@ -19,12 +19,11 @@ import io import os from minio import Minio -from minio.selectrequest import (COMPRESSION_TYPE_NONE, FILE_HEADER_INFO_NONE, - JSON_TYPE_DOCUMENT, QUOTE_FIELDS_ALWAYS, - QUOTE_FIELDS_ASNEEDED, CSVInputSerialization, - CSVOutputSerialization, - JSONInputSerialization, - JSONOutputSerialization, SelectRequest) +from minio.select import (COMPRESSION_TYPE_NONE, FILE_HEADER_INFO_NONE, + JSON_TYPE_DOCUMENT, QUOTE_FIELDS_ALWAYS, + QUOTE_FIELDS_ASNEEDED, CSVInputSerialization, + CSVOutputSerialization, JSONInputSerialization, + JSONOutputSerialization, SelectRequest) from utils import * @@ -41,7 +40,7 @@ def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_ # Get the records records = io.BytesIO() for d in data.stream(10*1024): - records.write(d.encode('utf-8')) + records.write(d) got_output = records.getvalue() except Exception as select_err: if not isinstance(expected_output, Exception): diff --git a/mint/run/core/s3select/sql_ops.py b/mint/run/core/s3select/sql_ops.py index e43b06321..0db598e67 100644 --- a/mint/run/core/s3select/sql_ops.py +++ b/mint/run/core/s3select/sql_ops.py @@ -18,11 +18,10 @@ import io from datetime import datetime -from minio.selectrequest import (FILE_HEADER_INFO_NONE, JSON_TYPE_DOCUMENT, - QUOTE_FIELDS_ASNEEDED, CSVInputSerialization, - CSVOutputSerialization, - JSONInputSerialization, - JSONOutputSerialization, SelectRequest) +from minio.select import (FILE_HEADER_INFO_NONE, JSON_TYPE_DOCUMENT, + QUOTE_FIELDS_ASNEEDED, CSVInputSerialization, + CSVOutputSerialization, JSONInputSerialization, + JSONOutputSerialization, SelectRequest) from utils import generate_bucket_name, generate_object_name @@ -58,7 +57,7 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, # Get the records records = io.BytesIO() for d in data.stream(10*1024): - records.write(d.encode('utf-8')) + records.write(d) got_output = records.getvalue() if got_output != expected_output: