diff --git a/docs/select/README.md b/docs/select/README.md index 17745302c..5e46eecc1 100644 --- a/docs/select/README.md +++ b/docs/select/README.md @@ -103,6 +103,6 @@ For a more detailed SELECT SQL reference, please see [here](https://docs.aws.ama - All [operators](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-operators.html) are supported. - All aggregation, conditional, type-conversion and string functions are supported. - JSON path expressions such as `FROM S3Object[*].path` are not yet evaluated. -- Large numbers (more than 64-bit) are not yet supported. -- Date [functions](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-date.html) are not yet supported (EXTRACT, DATE_DIFF, etc). +- Large numbers (outside of the signed 64-bit range) are not yet supported. +- The Date [functions](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-date.html) `DATE_ADD`, `DATE_DIFF`, `EXTRACT` and `UTCNOW` along with type conversion using `CAST` to the `TIMESTAMP` data type are currently supported. - AWS S3's [reserved keywords](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-keyword-list.html) list is not yet respected. diff --git a/pkg/s3select/json/record.go b/pkg/s3select/json/record.go index d3003ee56..087551535 100644 --- a/pkg/s3select/json/record.go +++ b/pkg/s3select/json/record.go @@ -60,6 +60,8 @@ func (r *Record) Set(name string, value *sql.Value) (err error) { v = f } else if i, ok := value.ToInt(); ok { v = i + } else if t, ok := value.ToTimestamp(); ok { + v = sql.FormatSQLTimestamp(t) } else if s, ok := value.ToString(); ok { v = s } else if value.IsNull() { diff --git a/pkg/s3select/sql/analysis.go b/pkg/s3select/sql/analysis.go index 56c227b27..e897be874 100644 --- a/pkg/s3select/sql/analysis.go +++ b/pkg/s3select/sql/analysis.go @@ -201,6 +201,16 @@ func (e *FuncExpr) analyze(s *Select) (result qProp) { case sqlFnExtract: return e.Extract.From.analyze(s) + case sqlFnDateAdd: + result.combine(e.DateAdd.Quantity.analyze(s)) + result.combine(e.DateAdd.Timestamp.analyze(s)) + return result + + case sqlFnDateDiff: + result.combine(e.DateDiff.Timestamp1.analyze(s)) + result.combine(e.DateDiff.Timestamp2.analyze(s)) + return result + // Handle aggregation function calls case aggFnAvg, aggFnMax, aggFnMin, aggFnSum, aggFnCount: // Initialize accumulator @@ -283,6 +293,11 @@ func (e *FuncExpr) analyze(s *Select) (result qProp) { } return result + case sqlFnUTCNow: + if len(e.SFunc.ArgsList) != 0 { + result.err = fmt.Errorf("%s() takes no arguments", string(funcName)) + } + return result } // TODO: implement other functions diff --git a/pkg/s3select/sql/evaluate.go b/pkg/s3select/sql/evaluate.go index 3ee28597d..f1c1480ab 100644 --- a/pkg/s3select/sql/evaluate.go +++ b/pkg/s3select/sql/evaluate.go @@ -305,10 +305,9 @@ func (e *UnaryTerm) evalNode(r Record) (*Value, error) { } inferTypeForArithOp(v) - if ival, ok := v.ToInt(); ok { - return FromInt(-ival), nil - } else if fval, ok := v.ToFloat(); ok { - return FromFloat(-fval), nil + v.negate() + if v.isNumeric() { + return v, nil } return nil, errArithMismatchedTypes } diff --git a/pkg/s3select/sql/funceval.go b/pkg/s3select/sql/funceval.go index 132383281..6d5321fa7 100644 --- a/pkg/s3select/sql/funceval.go +++ b/pkg/s3select/sql/funceval.go @@ -21,6 +21,7 @@ import ( "fmt" "strconv" "strings" + "time" ) // FuncName - SQL function name. @@ -52,21 +53,10 @@ const ( sqlFnUpper FuncName = "UPPER" ) -// Allowed cast types -const ( - castBool = "BOOL" - castInt = "INT" - castInteger = "INTEGER" - castString = "STRING" - castFloat = "FLOAT" - castDecimal = "DECIMAL" - castNumeric = "NUMERIC" - castTimestamp = "TIMESTAMP" -) - var ( errUnimplementedCast = errors.New("This cast not yet implemented") errNonStringTrimArg = errors.New("TRIM() received a non-string argument") + errNonTimestampArg = errors.New("Expected a timestamp argument") ) func (e *FuncExpr) getFunctionName() FuncName { @@ -83,6 +73,10 @@ func (e *FuncExpr) getFunctionName() FuncName { return sqlFnExtract case e.Trim != nil: return sqlFnTrim + case e.DateAdd != nil: + return sqlFnDateAdd + case e.DateDiff != nil: + return sqlFnDateDiff default: return "" } @@ -102,10 +96,17 @@ func (e *FuncExpr) evalSQLFnNode(r Record) (res *Value, err error) { return handleSQLSubstring(r, e.Substring) case sqlFnExtract: - return nil, errNotImplemented + return handleSQLExtract(r, e.Extract) case sqlFnTrim: return handleSQLTrim(r, e.Trim) + + case sqlFnDateAdd: + return handleDateAdd(r, e.DateAdd) + + case sqlFnDateDiff: + return handleDateDiff(r, e.DateDiff) + } // For all simple argument functions, we evaluate the arguments here @@ -119,30 +120,33 @@ func (e *FuncExpr) evalSQLFnNode(r Record) (res *Value, err error) { switch e.getFunctionName() { case sqlFnCoalesce: - return coalesce(r, argVals) + return coalesce(argVals) case sqlFnNullIf: - return nullif(r, argVals[0], argVals[1]) + return nullif(argVals[0], argVals[1]) case sqlFnCharLength, sqlFnCharacterLength: - return charlen(r, argVals[0]) + return charlen(argVals[0]) case sqlFnLower: - return lowerCase(r, argVals[0]) + return lowerCase(argVals[0]) case sqlFnUpper: - return upperCase(r, argVals[0]) + return upperCase(argVals[0]) + + case sqlFnUTCNow: + return handleUTCNow() - case sqlFnDateAdd, sqlFnDateDiff, sqlFnToString, sqlFnToTimestamp, sqlFnUTCNow: + case sqlFnToString, sqlFnToTimestamp: // TODO: implement fallthrough default: - return nil, errInvalidASTNode + return nil, errNotImplemented } } -func coalesce(r Record, args []*Value) (res *Value, err error) { +func coalesce(args []*Value) (res *Value, err error) { for _, arg := range args { if arg.IsNull() { continue @@ -152,7 +156,7 @@ func coalesce(r Record, args []*Value) (res *Value, err error) { return FromNull(), nil } -func nullif(r Record, v1, v2 *Value) (res *Value, err error) { +func nullif(v1, v2 *Value) (res *Value, err error) { // Handle Null cases if v1.IsNull() || v2.IsNull() { return v1, nil @@ -185,7 +189,7 @@ func nullif(r Record, v1, v2 *Value) (res *Value, err error) { return v1, nil } -func charlen(r Record, v *Value) (*Value, error) { +func charlen(v *Value) (*Value, error) { inferTypeAsString(v) s, ok := v.ToString() if !ok { @@ -195,7 +199,7 @@ func charlen(r Record, v *Value) (*Value, error) { return FromInt(int64(len(s))), nil } -func lowerCase(r Record, v *Value) (*Value, error) { +func lowerCase(v *Value) (*Value, error) { inferTypeAsString(v) s, ok := v.ToString() if !ok { @@ -205,7 +209,7 @@ func lowerCase(r Record, v *Value) (*Value, error) { return FromString(strings.ToLower(s)), nil } -func upperCase(r Record, v *Value) (*Value, error) { +func upperCase(v *Value) (*Value, error) { inferTypeAsString(v) s, ok := v.ToString() if !ok { @@ -215,6 +219,58 @@ func upperCase(r Record, v *Value) (*Value, error) { return FromString(strings.ToUpper(s)), nil } +func handleDateAdd(r Record, d *DateAddFunc) (*Value, error) { + q, err := d.Quantity.evalNode(r) + if err != nil { + return nil, err + } + inferTypeForArithOp(q) + qty, ok := q.ToFloat() + if !ok { + return nil, fmt.Errorf("QUANTITY must be a numeric argument to %s()", sqlFnDateAdd) + } + + ts, err := d.Timestamp.evalNode(r) + if err != nil { + return nil, err + } + inferTypeAsTimestamp(ts) + t, ok := ts.ToTimestamp() + if !ok { + return nil, fmt.Errorf("%s() expects a timestamp argument", sqlFnDateAdd) + } + + return dateAdd(strings.ToUpper(d.DatePart), qty, t) +} + +func handleDateDiff(r Record, d *DateDiffFunc) (*Value, error) { + tval1, err := d.Timestamp1.evalNode(r) + if err != nil { + return nil, err + } + inferTypeAsTimestamp(tval1) + ts1, ok := tval1.ToTimestamp() + if !ok { + return nil, fmt.Errorf("%s() expects two timestamp arguments", sqlFnDateDiff) + } + + tval2, err := d.Timestamp2.evalNode(r) + if err != nil { + return nil, err + } + inferTypeAsTimestamp(tval2) + ts2, ok := tval2.ToTimestamp() + if !ok { + return nil, fmt.Errorf("%s() expects two timestamp arguments", sqlFnDateDiff) + } + + return dateDiff(strings.ToUpper(d.DatePart), ts1, ts2) +} + +func handleUTCNow() (*Value, error) { + return FromTimestamp(time.Now().UTC()), nil +} + func handleSQLSubstring(r Record, e *SubstringFunc) (val *Value, err error) { // Both forms `SUBSTRING('abc' FROM 2 FOR 1)` and // SUBSTRING('abc', 2, 1) are supported. @@ -301,6 +357,22 @@ func handleSQLTrim(r Record, e *TrimFunc) (res *Value, err error) { return FromString(result), nil } +func handleSQLExtract(r Record, e *ExtractFunc) (res *Value, err error) { + timeVal, verr := e.From.evalNode(r) + if verr != nil { + return nil, verr + } + + inferTypeAsTimestamp(timeVal) + + t, ok := timeVal.ToTimestamp() + if !ok { + return nil, errNonTimestampArg + } + + return extract(strings.ToUpper(e.Timeword), t) +} + func errUnsupportedCast(fromType, toType string) error { return fmt.Errorf("Cannot cast from %v to %v", fromType, toType) } @@ -309,12 +381,23 @@ func errCastFailure(msg string) error { return fmt.Errorf("Error casting: %s", msg) } +// Allowed cast types +const ( + castBool = "BOOL" + castInt = "INT" + castInteger = "INTEGER" + castString = "STRING" + castFloat = "FLOAT" + castDecimal = "DECIMAL" + castNumeric = "NUMERIC" + castTimestamp = "TIMESTAMP" +) + func (e *Expression) castTo(r Record, castType string) (res *Value, err error) { v, err := e.evalNode(r) if err != nil { return nil, err } - fmt.Println("Cast to ", castType) switch castType { case castInt, castInteger: @@ -329,7 +412,15 @@ func (e *Expression) castTo(r Record, castType string) (res *Value, err error) { s, err := stringCast(v) return FromString(s), err - case castBool, castDecimal, castNumeric, castTimestamp: + case castTimestamp: + t, err := timestampCast(v) + return FromTimestamp(t), err + + case castBool: + b, err := boolCast(v) + return FromBool(b), err + + case castDecimal, castNumeric: fallthrough default: @@ -431,3 +522,44 @@ func stringCast(v *Value) (string, error) { // This does not happen return "", nil } + +func timestampCast(v *Value) (t time.Time, _ error) { + switch v.vType { + case typeString: + s, _ := v.ToString() + return parseSQLTimestamp(s) + case typeBytes: + b, _ := v.ToBytes() + return parseSQLTimestamp(string(b)) + case typeTimestamp: + t, _ = v.ToTimestamp() + return t, nil + default: + return t, errCastFailure(fmt.Sprintf("cannot cast %v to Timestamp type", v.GetTypeString())) + } +} + +func boolCast(v *Value) (b bool, _ error) { + sToB := func(s string) (bool, error) { + if s == "true" { + return true, nil + } else if s == "false" { + return false, nil + } else { + return false, errCastFailure("cannot cast to Bool") + } + } + switch v.vType { + case typeBool: + b, _ := v.ToBool() + return b, nil + case typeString: + s, _ := v.ToString() + return sToB(strings.ToLower(s)) + case typeBytes: + b, _ := v.ToBytes() + return sToB(strings.ToLower(string(b))) + default: + return false, errCastFailure("cannot cast %v to Bool") + } +} diff --git a/pkg/s3select/sql/parser.go b/pkg/s3select/sql/parser.go index bfdb7d0e4..f5bce5e41 100644 --- a/pkg/s3select/sql/parser.go +++ b/pkg/s3select/sql/parser.go @@ -247,6 +247,8 @@ type FuncExpr struct { Substring *SubstringFunc `parser:"| @@"` Extract *ExtractFunc `parser:"| @@"` Trim *TrimFunc `parser:"| @@"` + DateAdd *DateAddFunc `parser:"| @@"` + DateDiff *DateDiffFunc `parser:"| @@"` // Used during evaluation for aggregation funcs aggregate *aggVal @@ -255,7 +257,7 @@ type FuncExpr struct { // SimpleArgFunc represents functions with simple expression // arguments. type SimpleArgFunc struct { - FunctionName string `parser:" @(\"AVG\" | \"MAX\" | \"MIN\" | \"SUM\" | \"COALESCE\" | \"NULLIF\" | \"DATE_ADD\" | \"DATE_DIFF\" | \"TO_STRING\" | \"TO_TIMESTAMP\" | \"UTCNOW\" | \"CHAR_LENGTH\" | \"CHARACTER_LENGTH\" | \"LOWER\" | \"UPPER\") "` + FunctionName string `parser:" @(\"AVG\" | \"MAX\" | \"MIN\" | \"SUM\" | \"COALESCE\" | \"NULLIF\" | \"TO_STRING\" | \"TO_TIMESTAMP\" | \"UTCNOW\" | \"CHAR_LENGTH\" | \"CHARACTER_LENGTH\" | \"LOWER\" | \"UPPER\") "` ArgsList []*Expression `parser:"\"(\" (@@ (\",\" @@)*)?\")\""` } @@ -294,6 +296,20 @@ type TrimFunc struct { TrimFrom *PrimaryTerm `parser:" \"FROM\" )? @@ \")\" "` } +// DateAddFunc represents the DATE_ADD function +type DateAddFunc struct { + DatePart string `parser:" \"DATE_ADD\" \"(\" @( \"YEAR\":Timeword | \"MONTH\":Timeword | \"DAY\":Timeword | \"HOUR\":Timeword | \"MINUTE\":Timeword | \"SECOND\":Timeword ) \",\""` + Quantity *Operand `parser:" @@ \",\""` + Timestamp *PrimaryTerm `parser:" @@ \")\""` +} + +// DateDiffFunc represents the DATE_DIFF function +type DateDiffFunc struct { + DatePart string `parser:" \"DATE_DIFF\" \"(\" @( \"YEAR\":Timeword | \"MONTH\":Timeword | \"DAY\":Timeword | \"HOUR\":Timeword | \"MINUTE\":Timeword | \"SECOND\":Timeword ) \",\" "` + Timestamp1 *PrimaryTerm `parser:" @@ \",\" "` + Timestamp2 *PrimaryTerm `parser:" @@ \")\" "` +} + // LitValue represents a literal value parsed from the sql type LitValue struct { Number *float64 `parser:"( @Number"` diff --git a/pkg/s3select/sql/statement.go b/pkg/s3select/sql/statement.go index 35b93dbd7..7a0347dad 100644 --- a/pkg/s3select/sql/statement.go +++ b/pkg/s3select/sql/statement.go @@ -84,7 +84,6 @@ func ParseSelectStatement(s string) (stmt SelectStatement, err error) { stmt.selectQProp = selectAST.Expression.analyze(&selectAST) err = stmt.selectQProp.err if err != nil { - fmt.Println("Got Analysis err:", err) err = errQueryAnalysisFailure(err) } return diff --git a/pkg/s3select/sql/timestampfuncs.go b/pkg/s3select/sql/timestampfuncs.go new file mode 100644 index 000000000..75ec8add1 --- /dev/null +++ b/pkg/s3select/sql/timestampfuncs.go @@ -0,0 +1,199 @@ +/* + * Minio Cloud Storage, (C) 2019 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sql + +import ( + "time" +) + +const ( + layoutYear = "2006T" + layoutMonth = "2006-01T" + layoutDay = "2006-01-02T" + layoutMinute = "2006-01-02T15:04Z07:00" + layoutSecond = "2006-01-02T15:04:05Z07:00" + layoutNanosecond = "2006-01-02T15:04:05.999999999Z07:00" +) + +var ( + tformats = []string{ + layoutYear, + layoutMonth, + layoutDay, + layoutMinute, + layoutSecond, + layoutNanosecond, + } + oneNanoSecond = 1 +) + +func parseSQLTimestamp(s string) (t time.Time, err error) { + for _, f := range tformats { + t, err = time.Parse(f, s) + if err == nil { + break + } + } + return +} + +// FormatSQLTimestamp - returns the a string representation of the +// timestamp as used in S3 Select +func FormatSQLTimestamp(t time.Time) string { + _, zoneOffset := t.Zone() + hasZone := zoneOffset != 0 + hasFracSecond := t.Nanosecond() != 0 + hasSecond := t.Second() != 0 + hasTime := t.Hour() != 0 || t.Minute() != 0 + hasDay := t.Day() != 1 + hasMonth := t.Month() != 1 + + switch { + case hasFracSecond: + return t.Format(layoutNanosecond) + case hasSecond: + return t.Format(layoutSecond) + case hasTime || hasZone: + return t.Format(layoutMinute) + case hasDay: + return t.Format(layoutDay) + case hasMonth: + return t.Format(layoutMonth) + default: + return t.Format(layoutYear) + } +} + +const ( + timePartYear = "YEAR" + timePartMonth = "MONTH" + timePartDay = "DAY" + timePartHour = "HOUR" + timePartMinute = "MINUTE" + timePartSecond = "SECOND" + timePartTimezoneHour = "TIMEZONE_HOUR" + timePartTimezoneMinute = "TIMEZONE_MINUTE" +) + +func extract(what string, t time.Time) (v *Value, err error) { + switch what { + case timePartYear: + return FromInt(int64(t.Year())), nil + case timePartMonth: + return FromInt(int64(t.Month())), nil + case timePartDay: + return FromInt(int64(t.Day())), nil + case timePartHour: + return FromInt(int64(t.Hour())), nil + case timePartMinute: + return FromInt(int64(t.Minute())), nil + case timePartSecond: + return FromInt(int64(t.Second())), nil + case timePartTimezoneHour: + _, zoneOffset := t.Zone() + return FromInt(int64(zoneOffset / 3600)), nil + case timePartTimezoneMinute: + _, zoneOffset := t.Zone() + return FromInt(int64((zoneOffset % 3600) / 60)), nil + default: + // This does not happen + return nil, errNotImplemented + } +} + +func dateAdd(timePart string, qty float64, t time.Time) (*Value, error) { + var duration time.Duration + switch timePart { + case timePartYear: + return FromTimestamp(t.AddDate(int(qty), 0, 0)), nil + case timePartMonth: + return FromTimestamp(t.AddDate(0, int(qty), 0)), nil + case timePartDay: + return FromTimestamp(t.AddDate(0, 0, int(qty))), nil + case timePartHour: + duration = time.Duration(qty) * time.Hour + case timePartMinute: + duration = time.Duration(qty) * time.Minute + case timePartSecond: + duration = time.Duration(qty) * time.Second + default: + return nil, errNotImplemented + } + return FromTimestamp(t.Add(duration)), nil +} + +const ( + dayInNanoseconds = time.Hour * 24 +) + +// dateDiff computes the difference between two times in terms of the +// `timePart` which can be years, months, days, hours, minutes or +// seconds. For difference in years, months or days, the time part, +// including timezone is ignored. +func dateDiff(timePart string, ts1, ts2 time.Time) (*Value, error) { + if ts2.Before(ts1) { + v, err := dateDiff(timePart, ts2, ts1) + v.negate() + return v, err + } + + duration := ts2.Sub(ts1) + y1, m1, d1 := ts1.Date() + y2, m2, d2 := ts2.Date() + dy, dm := int64(y2-y1), int64(m2-m1) + + switch timePart { + case timePartYear: + if m2 > m1 || (m2 == m1 && d2 >= d1) { + return FromInt(dy), nil + } + return FromInt(dy - 1), nil + case timePartMonth: + months := 12 * dy + if m2 >= m1 { + months += dm + } else { + months += 12 + dm + } + if d2 < d1 { + months-- + } + return FromInt(months), nil + case timePartDay: + // To compute the number of days between two times + // using the time package, zero out the time portions + // of the timestamps, compute the difference duration + // and then divide by the length of a day. + d1 := time.Date(y1, m1, d1, 0, 0, 0, 0, time.UTC) + d2 := time.Date(y2, m2, d2, 0, 0, 0, 0, time.UTC) + diff := d2.Sub(d1) + days := diff / dayInNanoseconds + return FromInt(int64(days)), nil + case timePartHour: + hours := duration / time.Hour + return FromInt(int64(hours)), nil + case timePartMinute: + minutes := duration / time.Minute + return FromInt(int64(minutes)), nil + case timePartSecond: + seconds := duration / time.Second + return FromInt(int64(seconds)), nil + default: + + } + return nil, errNotImplemented +} diff --git a/pkg/s3select/sql/timestampfuncs_test.go b/pkg/s3select/sql/timestampfuncs_test.go new file mode 100644 index 000000000..9a9c73da1 --- /dev/null +++ b/pkg/s3select/sql/timestampfuncs_test.go @@ -0,0 +1,60 @@ +/* + * Minio Cloud Storage, (C) 2019 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sql + +import ( + "testing" + "time" +) + +func TestParseAndDisplaySQLTimestamp(t *testing.T) { + beijing := time.FixedZone("", int((8 * time.Hour).Seconds())) + fakeLosAngeles := time.FixedZone("", -int((8 * time.Hour).Seconds())) + cases := []struct { + s string + t time.Time + }{ + {"2010T", time.Date(2010, 1, 1, 0, 0, 0, 0, time.UTC)}, + {"2010-02T", time.Date(2010, 2, 1, 0, 0, 0, 0, time.UTC)}, + {"2010-02-03T", time.Date(2010, 2, 3, 0, 0, 0, 0, time.UTC)}, + {"2010-02-03T04:11Z", time.Date(2010, 2, 3, 4, 11, 0, 0, time.UTC)}, + {"2010-02-03T04:11:30Z", time.Date(2010, 2, 3, 4, 11, 30, 0, time.UTC)}, + {"2010-02-03T04:11:30.23Z", time.Date(2010, 2, 3, 4, 11, 30, 230000000, time.UTC)}, + {"2010-02-03T04:11+08:00", time.Date(2010, 2, 3, 4, 11, 0, 0, beijing)}, + {"2010-02-03T04:11:30+08:00", time.Date(2010, 2, 3, 4, 11, 30, 0, beijing)}, + {"2010-02-03T04:11:30.23+08:00", time.Date(2010, 2, 3, 4, 11, 30, 230000000, beijing)}, + {"2010-02-03T04:11:30-08:00", time.Date(2010, 2, 3, 4, 11, 30, 0, fakeLosAngeles)}, + {"2010-02-03T04:11:30.23-08:00", time.Date(2010, 2, 3, 4, 11, 30, 230000000, fakeLosAngeles)}, + } + for i, tc := range cases { + tval, err := parseSQLTimestamp(tc.s) + if err != nil { + t.Errorf("Case %d: Unexpected error: %v", i+1, err) + continue + } + if !tval.Equal(tc.t) { + t.Errorf("Case %d: Expected %v got %v", i+1, tc.t, tval) + continue + } + + tstr := FormatSQLTimestamp(tc.t) + if tstr != tc.s { + t.Errorf("Case %d: Expected %s got %s", i+1, tc.s, tstr) + continue + } + } +} diff --git a/pkg/s3select/sql/value.go b/pkg/s3select/sql/value.go index 49420c00d..f5cf172d6 100644 --- a/pkg/s3select/sql/value.go +++ b/pkg/s3select/sql/value.go @@ -22,6 +22,7 @@ import ( "math" "strconv" "strings" + "time" ) var ( @@ -48,6 +49,9 @@ const ( // 64-bit floating point typeFloat + // timestamp type + typeTimestamp + // This type refers to untyped values, e.g. as read from CSV typeBytes ) @@ -77,6 +81,8 @@ func (v *Value) GetTypeString() string { return "INT" case typeFloat: return "FLOAT" + case typeTimestamp: + return "TIMESTAMP" case typeBytes: return "BYTES" } @@ -90,6 +96,8 @@ func (v *Value) Repr() string { return ":NULL" case typeBool, typeInt, typeFloat: return fmt.Sprintf("%v:%s", v.value, v.GetTypeString()) + case typeTimestamp: + return fmt.Sprintf("%s:TIMESTAMP", v.value.(*time.Time)) case typeString: return fmt.Sprintf("\"%s\":%s", v.value.(string), v.GetTypeString()) case typeBytes: @@ -119,6 +127,11 @@ func FromBool(b bool) *Value { return &Value{value: b, vType: typeBool} } +// FromTimestamp creates a Value from a timestamp +func FromTimestamp(t time.Time) *Value { + return &Value{value: t, vType: typeTimestamp} +} + // FromNull creates a Value with Null value func FromNull() *Value { return &Value{vType: typeNull} @@ -173,6 +186,15 @@ func (v *Value) ToBool() (val bool, ok bool) { return false, false } +// ToTimestamp returns the timestamp value if present. +func (v *Value) ToTimestamp() (t time.Time, ok bool) { + switch v.vType { + case typeTimestamp: + return v.value.(time.Time), true + } + return t, false +} + // ToBytes converts Value to byte-slice. func (v *Value) ToBytes() ([]byte, bool) { switch v.vType { @@ -213,6 +235,11 @@ func (v *Value) setBool(b bool) { v.value = b } +func (v *Value) setTimestamp(t time.Time) { + v.vType = typeTimestamp + v.value = t +} + // CSVString - convert to string for CSV serialization func (v *Value) CSVString() string { switch v.vType { @@ -226,6 +253,8 @@ func (v *Value) CSVString() string { return fmt.Sprintf("%v", v.value.(int64)) case typeFloat: return fmt.Sprintf("%v", v.value.(float64)) + case typeTimestamp: + return FormatSQLTimestamp(v.value.(time.Time)) case typeBytes: return fmt.Sprintf("%v", string(v.value.([]byte))) default: @@ -242,6 +271,16 @@ func floatToValue(f float64) *Value { return FromFloat(f) } +// negate negates a numeric value +func (v *Value) negate() { + switch v.vType { + case typeFloat: + v.value = -(v.value.(float64)) + case typeInt: + v.value = -(v.value.(int64)) + } +} + // Value comparison functions: we do not expose them outside the // module. Logical operators "<", ">", ">=", "<=" work on strings and // numbers. Equality operators "=", "!=" work on strings, @@ -571,6 +610,24 @@ func (v *Value) minmax(a *Value, isMax, isFirstRow bool) error { return nil } +func inferTypeAsTimestamp(v *Value) { + if s, ok := v.ToString(); ok { + t, err := parseSQLTimestamp(s) + if err != nil { + return + } + v.setTimestamp(t) + } else if b, ok := v.ToBytes(); ok { + s := string(b) + t, err := parseSQLTimestamp(s) + if err != nil { + return + } + v.setTimestamp(t) + } + return +} + // inferTypeAsString is used to convert untyped values to string - it // is called when the caller requires a string context to proceed. func inferTypeAsString(v *Value) {