minio/pkg/s3select/internal/parquet-go/data/column_test.go

/*
 * Minio Cloud Storage, (C) 2019 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package data

import (
	"reflect"
	"testing"

	"github.com/minio/minio/pkg/s3select/internal/parquet-go/gen-go/parquet"
	"github.com/minio/minio/pkg/s3select/internal/parquet-go/schema"
)

var (
	v10    = int32(10)
	v20    = int32(20)
	v30    = int32(30)
	ten    = []byte("ten")
	foo    = []byte("foo")
	bar    = []byte("bar")
	phone1 = []byte("1-234-567-8901")
	phone2 = []byte("1-234-567-1098")
	phone3 = []byte("1-111-222-3333")
)

func TestAddressBookExample(t *testing.T) {
	// message AddressBook {
	//   required string owner;
	//   repeated string ownerPhoneNumbers;
	//   repeated group contacts {
	//     required string name;
	//     optional string phoneNumber;
	//   }
	// }
	t.Skip("Broken")

	addressBook := schema.NewTree()
	{
		owner, err := schema.NewElement("owner", parquet.FieldRepetitionType_REQUIRED,
			parquet.TypePtr(parquet.Type_BYTE_ARRAY), parquet.ConvertedTypePtr(parquet.ConvertedType_UTF8),
			nil, nil, nil)
		if err != nil {
			t.Fatal(err)
		}

		ownerPhoneNumbers, err := schema.NewElement("ownerPhoneNumbers", parquet.FieldRepetitionType_OPTIONAL,
			nil, parquet.ConvertedTypePtr(parquet.ConvertedType_LIST),
			nil, nil, nil)
		if err != nil {
			t.Fatal(err)
		}

		ownerPhoneNumbersList, err := schema.NewElement("list", parquet.FieldRepetitionType_REPEATED,
			nil, nil,
			nil, nil, nil)
		if err != nil {
			t.Fatal(err)
		}

		ownerPhoneNumbersElement, err := schema.NewElement("element", parquet.FieldRepetitionType_REQUIRED,
			parquet.TypePtr(parquet.Type_BYTE_ARRAY), parquet.ConvertedTypePtr(parquet.ConvertedType_UTF8),
			nil, nil, nil)
		if err != nil {
			t.Fatal(err)
		}

		contacts, err := schema.NewElement("contacts", parquet.FieldRepetitionType_OPTIONAL,
			nil, parquet.ConvertedTypePtr(parquet.ConvertedType_LIST),
			nil, nil, nil)
		if err != nil {
			t.Fatal(err)
		}

		contactsList, err := schema.NewElement("list", parquet.FieldRepetitionType_REPEATED,
			nil, nil,
			nil, nil, nil)
		if err != nil {
			t.Fatal(err)
		}

		contactsElement, err := schema.NewElement("element", parquet.FieldRepetitionType_REQUIRED,
			nil, nil,
			nil, nil, nil)
		if err != nil {
			t.Fatal(err)
		}

		contactName, err := schema.NewElement("name", parquet.FieldRepetitionType_REQUIRED,
			parquet.TypePtr(parquet.Type_BYTE_ARRAY), parquet.ConvertedTypePtr(parquet.ConvertedType_UTF8),
			nil, nil, nil)
		if err != nil {
			t.Fatal(err)
		}

		contactPhoneNumber, err := schema.NewElement("phoneNumber", parquet.FieldRepetitionType_OPTIONAL,
			parquet.TypePtr(parquet.Type_BYTE_ARRAY), parquet.ConvertedTypePtr(parquet.ConvertedType_UTF8),
			nil, nil, nil)
		if err != nil {
			t.Fatal(err)
		}
		if err = addressBook.Set("owner", owner); err != nil {
			t.Fatal(err)
		}

		if err = addressBook.Set("ownerPhoneNumbers", ownerPhoneNumbers); err != nil {
			t.Fatal(err)
		}
		if err = addressBook.Set("ownerPhoneNumbers.list", ownerPhoneNumbersList); err != nil {
			t.Fatal(err)
		}
		if err = addressBook.Set("ownerPhoneNumbers.list.element", ownerPhoneNumbersElement); err != nil {
			t.Fatal(err)
		}

		if err = addressBook.Set("contacts", contacts); err != nil {
			t.Fatal(err)
		}
		if err = addressBook.Set("contacts.list", contactsList); err != nil {
			t.Fatal(err)
		}
		if err = addressBook.Set("contacts.list.element", contactsElement); err != nil {
			t.Fatal(err)
		}
		if err = addressBook.Set("contacts.list.element.name", contactName); err != nil {
			t.Fatal(err)
		}
		if err = addressBook.Set("contacts.list.element.phoneNumber", contactPhoneNumber); err != nil {
			t.Fatal(err)
		}
	}

	if _, _, err := addressBook.ToParquetSchema(); err != nil {
		t.Fatal(err)
	}

	case2Data := `{
    "owner": "foo"
}`
	result2 := map[string]*Column{
		"owner": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{foo},
			definitionLevels: []int64{0},
			repetitionLevels: []int64{0},
		},
		"ownerPhoneNumbers.list.element": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{nil},
			definitionLevels: []int64{0},
			repetitionLevels: []int64{0},
		},
		"contacts.list.element.name": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{nil},
			definitionLevels: []int64{0},
			repetitionLevels: []int64{0},
		},
	}

	case3Data := `{
    "owner": "foo",
    "ownerPhoneNumbers": [
        "1-234-567-8901"
    ]
}
`
	result3 := map[string]*Column{
		"owner": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{foo},
			definitionLevels: []int64{0},
			repetitionLevels: []int64{0},
		},
		"ownerPhoneNumbers.list.element": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{phone1},
			definitionLevels: []int64{2},
			repetitionLevels: []int64{0},
		},
		"contacts.list.element.name": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{nil},
			definitionLevels: []int64{0},
			repetitionLevels: []int64{0},
		},
	}

	case4Data := `{
    "owner": "foo",
    "ownerPhoneNumbers": [
        "1-234-567-8901",
        "1-234-567-1098"
    ]
}
`
	result4 := map[string]*Column{
		"owner": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{foo},
			definitionLevels: []int64{0},
			repetitionLevels: []int64{0},
		},
		"ownerPhoneNumbers.list.element": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{phone1, phone2},
			definitionLevels: []int64{2, 2},
			repetitionLevels: []int64{0, 1},
		},
		"contacts.list.element.name": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{nil},
			definitionLevels: []int64{0},
			repetitionLevels: []int64{0},
		},
	}

	case5Data := `{
    "contacts": [
        {
            "name": "bar"
        }
    ],
    "owner": "foo"
}`
	result5 := map[string]*Column{
		"owner": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{foo},
			definitionLevels: []int64{0},
			repetitionLevels: []int64{0},
		},
		"ownerPhoneNumbers.list.element": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{nil},
			definitionLevels: []int64{0},
			repetitionLevels: []int64{0},
		},
		"contacts.list.element.name": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{bar},
			definitionLevels: []int64{2},
			repetitionLevels: []int64{0},
		},
		"contacts.list.element.phoneNumber": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{nil},
			definitionLevels: []int64{2},
			repetitionLevels: []int64{1},
		},
	}

	case6Data := `{
    "contacts": [
        {
            "name": "bar",
            "phoneNumber": "1-111-222-3333"
        }
    ],
    "owner": "foo"
}`
	result6 := map[string]*Column{
		"owner": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{foo},
			definitionLevels: []int64{0},
			repetitionLevels: []int64{0},
		},
		"ownerPhoneNumbers.list.element": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{nil},
			definitionLevels: []int64{0},
			repetitionLevels: []int64{0},
		},
		"contacts.list.element.name": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{bar},
			definitionLevels: []int64{2},
			repetitionLevels: []int64{0},
		},
		"contacts.list.element.phoneNumber": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{phone3},
			definitionLevels: []int64{3},
			repetitionLevels: []int64{1},
		},
	}

	case7Data := `{
    "contacts": [
        {
            "name": "bar",
            "phoneNumber": "1-111-222-3333"
        }
    ],
    "owner": "foo",
    "ownerPhoneNumbers": [
        "1-234-567-8901",
        "1-234-567-1098"
    ]
}`
	result7 := map[string]*Column{
		"owner": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{foo},
			definitionLevels: []int64{0},
			repetitionLevels: []int64{0},
		},
		"ownerPhoneNumbers.list.element": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{phone1, phone2},
			definitionLevels: []int64{2, 2},
			repetitionLevels: []int64{0, 1},
		},
		"contacts.list.element.name": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{bar},
			definitionLevels: []int64{2},
			repetitionLevels: []int64{0},
		},
		"contacts.list.element.phoneNumber": {
			parquetType:      parquet.Type_BYTE_ARRAY,
			values:           []interface{}{phone3},
			definitionLevels: []int64{3},
			repetitionLevels: []int64{1},
		},
	}

	testCases := []struct {
		data           string
		expectedResult map[string]*Column
		expectErr      bool
	}{
		{`{}`, nil, true}, // err: owner: nil value for required field
		{case2Data, result2, false},
		{case3Data, result3, false},
		{case4Data, result4, false},
		{case5Data, result5, false},
		{case6Data, result6, false},
		{case7Data, result7, false},
	}

	for i, testCase := range testCases {
		result, err := UnmarshalJSON([]byte(testCase.data), addressBook)
		expectErr := (err != nil)

		if testCase.expectErr != expectErr {
			t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
		}

		if !testCase.expectErr {
			if !reflect.DeepEqual(result, testCase.expectedResult) {
				t.Errorf("case %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result)
			}
		}
	}
}