Update vendorized bcicen/jstream (#7257)

- Includes an error handling fix that is waiting to be merged upstream
- Uses order-preserving (un)marshalling for JSON objects.
master
Aditya Manthramurthy 6 years ago committed by kannappanr
parent bedcb7442a
commit 80a351633f
  1. 2
      pkg/s3select/json/reader.go
  2. 110
      vendor/github.com/bcicen/jstream/decoder.go
  3. 6
      vendor/vendor.json

@ -71,7 +71,7 @@ func (r *Reader) Close() error {
// NewReader - creates new JSON reader using readCloser. // NewReader - creates new JSON reader using readCloser.
func NewReader(readCloser io.ReadCloser, args *ReaderArgs) *Reader { func NewReader(readCloser io.ReadCloser, args *ReaderArgs) *Reader {
d := jstream.NewDecoder(readCloser, 0) d := jstream.NewDecoder(readCloser, 0).ObjectAsKVS()
return &Reader{ return &Reader{
args: args, args: args,
decoder: d, decoder: d,

@ -42,8 +42,8 @@ type KV struct {
// KVS - represents key values in an JSON object // KVS - represents key values in an JSON object
type KVS []KV type KVS []KV
// MarshalJSON - implements converting slice of KVS into a JSON object // MarshalJSON - implements converting a KVS datastructure into a JSON
// with multiple keys and values. // object with multiple keys and values.
func (kvs KVS) MarshalJSON() ([]byte, error) { func (kvs KVS) MarshalJSON() ([]byte, error) {
b := new(bytes.Buffer) b := new(bytes.Buffer)
b.Write([]byte("{")) b.Write([]byte("{"))
@ -69,6 +69,7 @@ type Decoder struct {
emitDepth int emitDepth int
emitKV bool emitKV bool
emitRecursive bool emitRecursive bool
objectAsKVS bool
depth int depth int
scratch *scratch scratch *scratch
@ -97,6 +98,15 @@ func NewDecoder(r io.Reader, emitDepth int) *Decoder {
return d return d
} }
// ObjectAsKVS - by default JSON returns map[string]interface{} this
// is usually fine in most cases, but when you need to preserve the
// input order its not a right data structure. To preserve input
// order please use this option.
func (d *Decoder) ObjectAsKVS() *Decoder {
d.objectAsKVS = true
return d
}
// EmitKV enables emitting a jstream.KV struct when the items(s) parsed // EmitKV enables emitting a jstream.KV struct when the items(s) parsed
// at configured emit depth are within a JSON object. By default, only // at configured emit depth are within a JSON object. By default, only
// the object values are emitted. // the object values are emitted.
@ -217,7 +227,13 @@ func (d *Decoder) any() (interface{}, ValueType, error) {
i, err := d.array() i, err := d.array()
return i, Array, err return i, Array, err
case '{': case '{':
i, err := d.object() var i interface{}
var err error
if d.objectAsKVS {
i, err = d.objectOrdered()
} else {
i, err = d.object()
}
return i, Object, err return i, Object, err
default: default:
return nil, Unknown, d.mkError(ErrSyntax, "looking for beginning of value") return nil, Unknown, d.mkError(ErrSyntax, "looking for beginning of value")
@ -442,7 +458,92 @@ out:
} }
// object accept valid JSON array value // object accept valid JSON array value
func (d *Decoder) object() (KVS, error) { func (d *Decoder) object() (map[string]interface{}, error) {
d.depth++
var (
c byte
k string
v interface{}
t ValueType
err error
obj map[string]interface{}
)
// skip allocating map if it will not be emitted
if d.depth > d.emitDepth {
obj = make(map[string]interface{})
}
// if the object has no keys
if c = d.skipSpaces(); c == '}' {
goto out
}
scan:
for {
offset := d.pos - 1
// read string key
if c != '"' {
err = d.mkError(ErrSyntax, "looking for beginning of object key string")
break
}
if k, err = d.string(); err != nil {
break
}
// read colon before value
if c = d.skipSpaces(); c != ':' {
err = d.mkError(ErrSyntax, "after object key")
break
}
// read value
d.skipSpaces()
if d.emitKV {
if v, t, err = d.any(); err != nil {
break
}
if d.willEmit() {
d.metaCh <- &MetaValue{
Offset: int(offset),
Length: int(d.pos - offset),
Depth: d.depth,
Value: KV{k, v},
ValueType: t,
}
}
} else {
if v, err = d.emitAny(); err != nil {
break
}
}
if obj != nil {
obj[k] = v
}
// next token must be ',' or '}'
switch c = d.skipSpaces(); c {
case '}':
goto out
case ',':
c = d.skipSpaces()
goto scan
default:
err = d.mkError(ErrSyntax, "after object key:value pair")
goto out
}
}
out:
d.depth--
return obj, err
}
// object (ordered) accept valid JSON array value
func (d *Decoder) objectOrdered() (KVS, error) {
d.depth++ d.depth++
var ( var (
@ -517,6 +618,7 @@ scan:
goto scan goto scan
default: default:
err = d.mkError(ErrSyntax, "after object key:value pair") err = d.mkError(ErrSyntax, "after object key:value pair")
goto out
} }
} }

@ -116,10 +116,10 @@
"revisionTime": "2017-09-25T03:23:15Z" "revisionTime": "2017-09-25T03:23:15Z"
}, },
{ {
"checksumSHA1": "mSo9Sti7F6+laUs4NLx/p23rHD0=", "checksumSHA1": "YPGDETCyuMkceQvAXafFnj6ndkY=",
"path": "github.com/bcicen/jstream", "path": "github.com/bcicen/jstream",
"revision": "f306cd3e1fa602a1b513114521a0d6a5a9d5c919", "revision": "16c1f8af81c2a9967b30d365a29472126274f998",
"revisionTime": "2019-02-06T02:23:53Z" "revisionTime": "2019-02-20T04:59:26Z"
}, },
{ {
"checksumSHA1": "0rido7hYHQtfq3UJzVT5LClLAWc=", "checksumSHA1": "0rido7hYHQtfq3UJzVT5LClLAWc=",

Loading…
Cancel
Save