pax_global_header00006660000000000000000000000064131716713400014515gustar00rootroot0000000000000052 comment=81f5327eb8367be6f106cd62f136b6f25b4c1678 canonicaljson-go-1.0.3/000077500000000000000000000000001317167134000147425ustar00rootroot00000000000000canonicaljson-go-1.0.3/.gitignore000066400000000000000000000000161317167134000167270ustar00rootroot00000000000000canonicaljson canonicaljson-go-1.0.3/.gitmodules000066400000000000000000000001631317167134000171170ustar00rootroot00000000000000[submodule "canonicaljson-spec"] path = canonicaljson-spec url = git@github.com:gibson042/canonicaljson-spec.git canonicaljson-go-1.0.3/LICENSE000066400000000000000000000031761317167134000157560ustar00rootroot00000000000000Copyright (c) 2016 Richard Gibson. All rights reserved. Based on the Go standard package "encoding/json" from golang/go@9d77ad8d34ce56e182adc30cd21af50a4b00932c:src/encoding/json Copyright (c) 2012 The Go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. canonicaljson-go-1.0.3/README.md000066400000000000000000000402351317167134000162250ustar00rootroot00000000000000# canonicaljson-go Go library for producing JSON in canonical format as specified by [https://gibson042.github.io/canonicaljson-spec/](https://gibson042.github.io/canonicaljson-spec/). The provided interface matches that of standard package "[encoding/json](https://golang.org/pkg/encoding/json/)" wherever they overlap: * [func `Marshal`](https://golang.org/pkg/encoding/json/#Marshal) * [func `MarshalIndent`](https://golang.org/pkg/encoding/json/#MarshalIndent) * [type `Encoder`](https://golang.org/pkg/encoding/json/#Encoder) Types from the standard package are also accepted wherever relevant. Test this package by invoking `test.sh`. ``` godoc github.com/gibson042/canonicaljson-go PACKAGE DOCUMENTATION package canonicaljson import "github.com/gibson042/canonicaljson-go" Package canonicaljson implements canonical serialization of Go objects to canonical-form JSON as specified at https://gibson042.github.io/canonicaljson-spec/ . The provided interface should match that of standard package "encoding/json" (from which it is derived) wherever they overlap (and in fact, this package is essentially a 2016-03-09 fork from golang/go@9d77ad8d34ce56e182adc30cd21af50a4b00932c:src/encoding/json ). Notable differences: - Object keys are sorted lexicographically by code point - Non-integer JSON numbers are represented in capital-E exponential notation with significand in (-10, 10) and no insignificant signs or zeroes beyond those required to force a decimal point. - JSON strings are represented in UTF-8 with minimal byte length, using escapes only when necessary for validity and Unicode escapes (uppercase hex) only when there is no shorter option. FUNCTIONS func Marshal(v interface{}) ([]byte, error) Marshal returns the canonical UTF-8 JSON encoding of v. Marshal traverses the value v recursively. If an encountered value implements the json.Marshaler interface and is not a nil pointer, Marshal calls its MarshalJSON method to produce JSON. If no MarshalJSON method is present but the value implements encoding.TextMarshaler instead, Marshal calls its MarshalText method. The nil pointer exception is not strictly necessary but mimics a similar, necessary exception in the behavior of json.UnmarshalJSON. Otherwise, Marshal uses the following type-dependent default encodings: Boolean values encode as JSON booleans. Floating point, integer, and Number values encode as JSON numbers. Non-fractional values become sequences of digits without leading spaces; fractional values are represented in capital-E exponential notation with the shortest possible significand of magnitude less than 10 that includes at least one digit both before and after the decimal point, and the shortest possible non-empty exponent. String values encode as JSON strings, treating ill-formed UTF-8 input as an error (with the exception of "WTF-8" encodings of lone surrogates—code points U+D800 through U+DFFF [inclusive] that could not be interpreted as part of a surrogate pair). This is in contrast to encoding/json, which replaces ill-formed sequences with U+FFFD REPLACEMENT CHARACTER. Also in contrast to encoding/json, characters appear unescaped whenever possible. Control characters U+0000 through U+001F and lone surrogates U+D800 through U+DFFF are replaced with their shortest escape sequence, 4 uppercase hex characters except for the following: - \b U+0008 BACKSPACE - \t U+0009 CHARACTER TABULATION ("tab") - \n U+000A LINE FEED ("newline") - \f U+000C FORM FEED - \r U+000D CARRIAGE RETURN Array and slice values encode as JSON arrays, except that []byte encodes as a base64-encoded string, and a nil slice encodes as the null JSON object. Struct values encode as JSON objects. Each exported struct field becomes a member of the object unless - the field's tag is "-", or - the field is empty and its tag specifies the "omitempty" option. The empty values are false, 0, any nil pointer or interface value, and any array, slice, map, or string of length zero. The object's default key string is the struct field name but can be specified in the struct field's tag value. The "json" key in the struct field's tag value is the key name, followed by an optional comma and options. Examples: // Field is ignored by this package. Field int `json:"-"` // Field appears in JSON as key "myName". Field int `json:"myName"` // Field appears in JSON as key "myName" and // the field is omitted from the object if its value is empty, // as defined above. Field int `json:"myName,omitempty"` // Field appears in JSON as key "Field" (the default), but // the field is skipped if empty. // Note the leading comma. Field int `json:",omitempty"` The "string" option signals that a field is stored as JSON inside a JSON-encoded string. It applies only to fields of string, floating point, integer, or boolean types. This extra level of encoding is sometimes used when communicating with JavaScript programs: Int64String int64 `json:",string"` The key name will be used if it's a non-empty string consisting of only Unicode letters, digits, dollar signs, percent signs, hyphens, underscores and slashes. Anonymous struct fields are usually marshaled as if their inner exported fields were fields in the outer struct, subject to the usual Go visibility rules amended as described in the next paragraph. An anonymous struct field with a name given in its JSON tag is treated as having that name, rather than being anonymous. An anonymous struct field of interface type is treated the same as having that type as its name, rather than being anonymous. The Go visibility rules for struct fields are amended for JSON when deciding which field to marshal. If there are multiple fields at the same level, and that level is the least nested (and would therefore be the nesting level selected by the usual Go rules), the following extra rules apply: 1) Of those fields, if any are JSON-tagged, only tagged fields are considered, even if there are multiple untagged fields that would otherwise conflict. 2) If there is exactly one field (tagged or not according to the first rule), that is selected. 3) Otherwise there are multiple fields, and all are ignored; no error occurs. Handling of anonymous struct fields is new in Go 1.1. Prior to Go 1.1, anonymous struct fields were ignored. To force ignoring of an anonymous struct field in both current and earlier versions, give the field a JSON tag of "-". Map values encode as JSON objects. The map's key type must be string; the map keys are used as JSON object keys, subject to the UTF-8 coercion described for string values above. Pointer values encode as the value pointed to. A nil pointer encodes as the null JSON object. Interface values encode as the value contained in the interface. A nil interface value encodes as the null JSON object. Channel, complex, and function values cannot be encoded in JSON. Attempting to encode such a value causes Marshal to return an UnsupportedTypeError. JSON cannot represent cyclic data structures and Marshal does not handle them. Passing cyclic structures to Marshal will result in an infinite recursion. func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) MarshalIndent is like Marshal, but adds whitespace for more readable output. func Unmarshal(data []byte, v interface{}) error Unmarshal parses the JSON-encoded UTF-8 data and stores the result in the value pointed to by v. Unmarshal uses the inverse of the encodings that Marshal uses, allocating maps, slices, and pointers as necessary, with the following additional rules: To unmarshal JSON into a pointer, Unmarshal first handles the case of the JSON being the JSON literal null. In that case, Unmarshal sets the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into the value pointed at by the pointer. If the pointer is nil, Unmarshal allocates a new value for it to point to. To unmarshal JSON into a struct, Unmarshal matches incoming object keys to the keys used by Marshal (either the struct field name or its tag), preferring an exact match but also accepting a case-insensitive match. Unmarshal will only set exported fields of the struct. To unmarshal JSON into an interface value, Unmarshal stores one of these in the interface value: bool, for JSON booleans float64, for JSON numbers string, for JSON strings []interface{}, for JSON arrays map[string]interface{}, for JSON objects nil for JSON null To unmarshal a JSON array into a slice, Unmarshal resets the slice length to zero and then appends each element to the slice. As a special case, to unmarshal an empty JSON array into a slice, Unmarshal replaces the slice with a new empty slice. To unmarshal a JSON array into a Go array, Unmarshal decodes JSON array elements into corresponding Go array elements. If the Go array is smaller than the JSON array, the additional JSON array elements are discarded. If the JSON array is smaller than the Go array, the additional Go array elements are set to zero values. To unmarshal a JSON object into a string-keyed map, Unmarshal first establishes a map to use, If the map is nil, Unmarshal allocates a new map. Otherwise Unmarshal reuses the existing map, keeping existing entries. Unmarshal then stores key-value pairs from the JSON object into the map. If a JSON value is not appropriate for a given target type, or if a JSON number overflows the target type, Unmarshal skips that field and completes the unmarshaling as best it can. If no more serious errors are encountered, Unmarshal returns an UnmarshalTypeError describing the earliest such error. The JSON null value unmarshals into an interface, map, pointer, or slice by setting that Go value to nil. Because null is often used in JSON to mean ``not present,'' unmarshaling a JSON null into any other Go type has no effect on the value and produces no error. Invalid UTF-8 input is always treated as an error, even if it is encountered when unmarshaling a quoted string (in contrast to encoding/json, which replaces bad octets with U+FFFD REPLACEMENT CHARACTER). However, `\uXXXX` JSON string escape sequences specifying lone surrogates (code points U+D800 through U+DFFF [inclusive] that are not part of a surrogate pair) are interpreted to produce "WTF-8" runes that cannot be losslessly serialized to UTF-8 and might produce unexpected behavior if passed to functions expecting all strings to contain valid UTF-8. This package's Marshal function checks for such runes and emits them as valid JSON escape sequences. TYPES type Decoder struct { // contains filtered or unexported fields } A Decoder reads and decodes JSON objects from an input stream. func NewDecoder(r io.Reader) *Decoder NewDecoder returns a new decoder that reads from r. The decoder introduces its own buffering and may read data from r beyond the JSON values requested. func (dec *Decoder) Buffered() io.Reader Buffered returns a reader of the data remaining in the Decoder's buffer. The reader is valid until the next call to Decode. func (dec *Decoder) Decode(v interface{}) error Decode reads the next JSON-encoded value from its input and stores it in the value pointed to by v. See the documentation for Unmarshal for details about the conversion of JSON into a Go value. func (dec *Decoder) More() bool More reports whether there is another element in the current array or object being parsed. func (dec *Decoder) Token() (Token, error) Token returns the next JSON token in the input stream. At the end of the input stream, Token returns nil, io.EOF. Token guarantees that the delimiters [ ] { } it returns are properly nested and matched: if Token encounters an unexpected delimiter in the input, it will return an error. The input stream consists of basic JSON values—bool, string, number, and null—along with delimiters [ ] { } of type Delim to mark the start and end of arrays and objects. Commas and colons are elided. func (dec *Decoder) UseNumber() UseNumber causes the Decoder to unmarshal a number into an interface{} as a Number instead of as a float64. type Delim rune A Delim is a JSON array or object delimiter, one of [ ] { or }. func (d Delim) String() string type Encoder struct { // contains filtered or unexported fields } An Encoder writes JSON objects to an output stream. func NewEncoder(w io.Writer) *Encoder NewEncoder returns a new encoder that writes to w. func (enc *Encoder) Encode(v interface{}) error Encode writes the JSON encoding of v to the stream, followed by a newline character. See the documentation for Marshal for details about the conversion of Go values to JSON. type InvalidUnmarshalError struct { Type reflect.Type } An InvalidUnmarshalError describes an invalid argument passed to Unmarshal. (The argument to Unmarshal must be a non-nil pointer.) func (e *InvalidUnmarshalError) Error() string type Marshaler interface { MarshalJSON() ([]byte, error) } Marshaler is the interface implemented by objects that can marshal themselves into valid JSON. type MarshalerError struct { Type reflect.Type Err error } func (e *MarshalerError) Error() string type Number string A Number represents a JSON number literal. TODO(go>=1.9): type Number = json.Number func (n Number) Float64() (float64, error) Float64 returns the number as a float64. func (n Number) Int64() (int64, error) Int64 returns the number as an int64. func (n Number) String() string String returns the literal text of the number. type RawMessage []byte RawMessage is a raw encoded JSON object. It implements Marshaler and Unmarshaler and can be used to delay JSON decoding or precompute a JSON encoding. func (m *RawMessage) MarshalJSON() ([]byte, error) MarshalJSON returns *m as the JSON encoding of m. func (m *RawMessage) UnmarshalJSON(data []byte) error UnmarshalJSON sets *m to a copy of data. type SyntaxError struct { Offset int64 // error occurred after reading Offset bytes // contains filtered or unexported fields } A SyntaxError is a description of a JSON syntax error. func (e *SyntaxError) Error() string type Token interface{} A Token holds a value of one of these types: Delim, for the four JSON delimiters [ ] { } bool, for JSON booleans float64, for JSON numbers Number, for JSON numbers string, for JSON string literals nil, for JSON null type UnmarshalFieldError struct { Key string Type reflect.Type Field reflect.StructField } An UnmarshalFieldError describes a JSON object key that led to an unexported (and therefore unwritable) struct field. (No longer used; kept for compatibility.) func (e *UnmarshalFieldError) Error() string type UnmarshalTypeError struct { Value string // description of JSON value - "bool", "array", "number -5" Type reflect.Type // type of Go value it could not be assigned to Offset int64 // error occurred after reading Offset bytes } An UnmarshalTypeError describes a JSON value that was not appropriate for a value of a specific Go type. func (e *UnmarshalTypeError) Error() string type Unmarshaler interface { UnmarshalJSON([]byte) error } Unmarshaler is the interface implemented by objects that can unmarshal a JSON description of themselves. The input can be assumed to be a valid encoding of a JSON value. UnmarshalJSON must copy the JSON data if it wishes to retain the data after returning. type UnsupportedTypeError struct { Type reflect.Type } An UnsupportedTypeError is returned by Marshal when attempting to encode an unsupported value type. func (e *UnsupportedTypeError) Error() string type UnsupportedValueError struct { Value reflect.Value Str string } func (e *UnsupportedValueError) Error() string SUBDIRECTORIES canonicaljson-spec cli ``` canonicaljson-go-1.0.3/canonicaljson-spec/000077500000000000000000000000001317167134000205135ustar00rootroot00000000000000canonicaljson-go-1.0.3/cli/000077500000000000000000000000001317167134000155115ustar00rootroot00000000000000canonicaljson-go-1.0.3/cli/canonicaljson.go000066400000000000000000000020551317167134000206630ustar00rootroot00000000000000package main import ( "flag" "fmt" "github.com/gibson042/canonicaljson-go" "log" "os" ) func main() { flag.Parse() srcFiles := flag.Args() if len(srcFiles) == 0 { srcFiles = []string{"-"} } for _, srcFile := range srcFiles { var data interface{} var decoder *canonicaljson.Decoder if srcFile == "-" { decoder = canonicaljson.NewDecoder(os.Stdin) } else { file, err := os.Open(srcFile) if err != nil { log.Fatal(err) } decoder = canonicaljson.NewDecoder(file) } // Handle numbers with infinite precision. decoder.UseNumber() // Read as many JSON values as possible from standard input. for srcFile != "-" || decoder.More() { if err := decoder.Decode(&data); err != nil { log.Fatal(err) } if result, err := canonicaljson.Marshal(&data); err != nil { log.Fatal(err) } else { fmt.Printf("%s", string(result)) } // Read only a single value from each file. if srcFile != "-" { if decoder.More() { log.Fatal("Trailing data in file: %s\n", srcFile) } break } } } } canonicaljson-go-1.0.3/decode.go000066400000000000000000000725421317167134000165260ustar00rootroot00000000000000// Copyright 2016 Richard Gibson. All rights reserved. // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Represents JSON data structure using native Go types: booleans, floats, // strings, arrays, and maps. package canonicaljson import ( "bytes" "encoding" "encoding/base64" "errors" "fmt" "reflect" "runtime" "strconv" "unicode" "unicode/utf16" "unicode/utf8" ) // Unmarshal parses the JSON-encoded UTF-8 data and stores the result // in the value pointed to by v. // // Unmarshal uses the inverse of the encodings that // Marshal uses, allocating maps, slices, and pointers as necessary, // with the following additional rules: // // To unmarshal JSON into a pointer, Unmarshal first handles the case of // the JSON being the JSON literal null. In that case, Unmarshal sets // the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into // the value pointed at by the pointer. If the pointer is nil, Unmarshal // allocates a new value for it to point to. // // To unmarshal JSON into a struct, Unmarshal matches incoming object // keys to the keys used by Marshal (either the struct field name or its tag), // preferring an exact match but also accepting a case-insensitive match. // Unmarshal will only set exported fields of the struct. // // To unmarshal JSON into an interface value, // Unmarshal stores one of these in the interface value: // // bool, for JSON booleans // float64, for JSON numbers // string, for JSON strings // []interface{}, for JSON arrays // map[string]interface{}, for JSON objects // nil for JSON null // // To unmarshal a JSON array into a slice, Unmarshal resets the slice length // to zero and then appends each element to the slice. // As a special case, to unmarshal an empty JSON array into a slice, // Unmarshal replaces the slice with a new empty slice. // // To unmarshal a JSON array into a Go array, Unmarshal decodes // JSON array elements into corresponding Go array elements. // If the Go array is smaller than the JSON array, // the additional JSON array elements are discarded. // If the JSON array is smaller than the Go array, // the additional Go array elements are set to zero values. // // To unmarshal a JSON object into a string-keyed map, Unmarshal first // establishes a map to use, If the map is nil, Unmarshal allocates a new map. // Otherwise Unmarshal reuses the existing map, keeping existing entries. // Unmarshal then stores key-value pairs from the JSON object into the map. // // If a JSON value is not appropriate for a given target type, // or if a JSON number overflows the target type, Unmarshal // skips that field and completes the unmarshaling as best it can. // If no more serious errors are encountered, Unmarshal returns // an UnmarshalTypeError describing the earliest such error. // // The JSON null value unmarshals into an interface, map, pointer, or slice // by setting that Go value to nil. Because null is often used in JSON to mean // ``not present,'' unmarshaling a JSON null into any other Go type has no effect // on the value and produces no error. // // Invalid UTF-8 input is always treated as an error, even if it is // encountered when unmarshaling a quoted string (in contrast to // encoding/json, which replaces bad octets with U+FFFD REPLACEMENT // CHARACTER). // However, `\uXXXX` JSON string escape sequences specifying lone // surrogates (code points U+D800 through U+DFFF [inclusive] that are // not part of a surrogate pair) are interpreted to produce "WTF-8" // runes that cannot be losslessly serialized to UTF-8 and might produce // unexpected behavior if passed to functions expecting all strings to // contain valid UTF-8. // This package's Marshal function checks for such runes and emits them // as valid JSON escape sequences. // func Unmarshal(data []byte, v interface{}) error { // Check for well-formedness. // Avoids filling out half a data structure // before discovering a JSON syntax error. var d decodeState err := checkValid(data, &d.scan) if err != nil { return err } d.init(data) return d.unmarshal(v) } // Unmarshaler is the interface implemented by objects // that can unmarshal a JSON description of themselves. // The input can be assumed to be a valid encoding of // a JSON value. UnmarshalJSON must copy the JSON data // if it wishes to retain the data after returning. type Unmarshaler interface { UnmarshalJSON([]byte) error } // An UnmarshalTypeError describes a JSON value that was // not appropriate for a value of a specific Go type. type UnmarshalTypeError struct { Value string // description of JSON value - "bool", "array", "number -5" Type reflect.Type // type of Go value it could not be assigned to Offset int64 // error occurred after reading Offset bytes } func (e *UnmarshalTypeError) Error() string { return "canonicaljson: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() } // An UnmarshalFieldError describes a JSON object key that // led to an unexported (and therefore unwritable) struct field. // (No longer used; kept for compatibility.) type UnmarshalFieldError struct { Key string Type reflect.Type Field reflect.StructField } func (e *UnmarshalFieldError) Error() string { return "canonicaljson: cannot unmarshal object key " + strconv.Quote(e.Key) + " into unexported field " + e.Field.Name + " of type " + e.Type.String() } // An InvalidUnmarshalError describes an invalid argument passed to Unmarshal. // (The argument to Unmarshal must be a non-nil pointer.) type InvalidUnmarshalError struct { Type reflect.Type } func (e *InvalidUnmarshalError) Error() string { if e.Type == nil { return "canonicaljson: Unmarshal(nil)" } if e.Type.Kind() != reflect.Ptr { return "canonicaljson: Unmarshal(non-pointer " + e.Type.String() + ")" } return "canonicaljson: Unmarshal(nil " + e.Type.String() + ")" } func (d *decodeState) unmarshal(v interface{}) (err error) { defer func() { if r := recover(); r != nil { if _, ok := r.(runtime.Error); ok { panic(r) } err = r.(error) } }() rv := reflect.ValueOf(v) if rv.Kind() != reflect.Ptr || rv.IsNil() { return &InvalidUnmarshalError{reflect.TypeOf(v)} } d.scan.reset() // We decode rv not rv.Elem because the Unmarshaler interface // test must be applied at the top level of the value. d.value(rv) return d.savedError } // A Number represents a JSON number literal. // TODO(go>=1.9): type Number = json.Number type Number string // String returns the literal text of the number. func (n Number) String() string { return string(n) } // Float64 returns the number as a float64. func (n Number) Float64() (float64, error) { return strconv.ParseFloat(string(n), 64) } // Int64 returns the number as an int64. func (n Number) Int64() (int64, error) { return strconv.ParseInt(string(n), 10, 64) } // isValidNumber reports whether s is a valid JSON number literal. func isValidNumber(s string) bool { // This function implements the JSON numbers grammar. // See https://tools.ietf.org/html/rfc7159#section-6 // and http://json.org/number.gif if s == "" { return false } // Optional - if s[0] == '-' { s = s[1:] if s == "" { return false } } // Digits switch { default: return false case s[0] == '0': s = s[1:] case '1' <= s[0] && s[0] <= '9': s = s[1:] for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { s = s[1:] } } // . followed by 1 or more digits. if len(s) >= 2 && s[0] == '.' && '0' <= s[1] && s[1] <= '9' { s = s[2:] for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { s = s[1:] } } // e or E followed by an optional - or + and // 1 or more digits. if len(s) >= 2 && (s[0] == 'e' || s[0] == 'E') { s = s[1:] if s[0] == '+' || s[0] == '-' { s = s[1:] if s == "" { return false } } for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { s = s[1:] } } // Make sure we are at the end. return s == "" } // decodeState represents the state while decoding a JSON value. type decodeState struct { data []byte off int // read offset in data scan scanner nextscan scanner // for calls to nextValue savedError error useNumber bool } // errPhase is used for errors that should not happen unless // there is a bug in the JSON decoder or something is editing // the data slice while the decoder executes. var errPhase = errors.New("JSON decoder out of sync - data changing underfoot?") func (d *decodeState) init(data []byte) *decodeState { d.data = data d.off = 0 d.savedError = nil return d } // error aborts the decoding by panicking with err. func (d *decodeState) error(err error) { panic(err) } // saveError saves the first err it is called with, // for reporting at the end of the unmarshal. func (d *decodeState) saveError(err error) { if d.savedError == nil { d.savedError = err } } // next cuts off and returns the next full JSON value in d.data[d.off:]. // The next value is known to be an object or array, not a literal. func (d *decodeState) next() []byte { c := d.data[d.off] item, rest, err := nextValue(d.data[d.off:], &d.nextscan) if err != nil { d.error(err) } d.off = len(d.data) - len(rest) // Our scanner has seen the opening brace/bracket // and thinks we're still in the middle of the object. // invent a closing brace/bracket to get it out. if c == '{' { d.scan.step(&d.scan, '}') } else { d.scan.step(&d.scan, ']') } return item } // scanWhile processes bytes in d.data[d.off:] until it // receives a scan code not equal to op. // It updates d.off and returns the new scan code. func (d *decodeState) scanWhile(op int) int { var newOp int for { if d.off >= len(d.data) { newOp = d.scan.eof() d.off = len(d.data) + 1 // mark processed EOF with len+1 } else { c := d.data[d.off] d.off++ newOp = d.scan.step(&d.scan, c) } if newOp != op { break } } return newOp } // value decodes a JSON value from d.data[d.off:] into the value. // it updates d.off to point past the decoded value. func (d *decodeState) value(v reflect.Value) { if !v.IsValid() { _, rest, err := nextValue(d.data[d.off:], &d.nextscan) if err != nil { d.error(err) } d.off = len(d.data) - len(rest) // d.scan thinks we're still at the beginning of the item. // Feed in an empty string - the shortest, simplest value - // so that it knows we got to the end of the value. if d.scan.redo { // rewind. d.scan.redo = false d.scan.step = stateBeginValue } d.scan.step(&d.scan, '"') d.scan.step(&d.scan, '"') n := len(d.scan.parseState) if n > 0 && d.scan.parseState[n-1] == parseObjectKey { // d.scan thinks we just read an object key; finish the object d.scan.step(&d.scan, ':') d.scan.step(&d.scan, '"') d.scan.step(&d.scan, '"') d.scan.step(&d.scan, '}') } return } switch op := d.scanWhile(scanSkipSpace); op { default: d.error(errPhase) case scanBeginArray: d.array(v) case scanBeginObject: d.object(v) case scanBeginLiteral: d.literal(v) } } type unquotedValue struct{} // valueQuoted is like value but decodes a // quoted string literal or literal null into an interface value. // If it finds anything other than a quoted string literal or null, // valueQuoted returns unquotedValue{}. func (d *decodeState) valueQuoted() interface{} { switch op := d.scanWhile(scanSkipSpace); op { default: d.error(errPhase) case scanBeginArray: d.array(reflect.Value{}) case scanBeginObject: d.object(reflect.Value{}) case scanBeginLiteral: switch v := d.literalInterface().(type) { case nil, string: return v } } return unquotedValue{} } // indirect walks down v allocating pointers as needed, // until it gets to a non-pointer. // if it encounters an Unmarshaler, indirect stops and returns that. // if decodingNull is true, indirect stops at the last pointer so it can be set to nil. func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { // If v is a named type and is addressable, // start with its address, so that if the type has pointer methods, // we find them. if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() { v = v.Addr() } for { // Load value from interface, but only if the result will be // usefully addressable. if v.Kind() == reflect.Interface && !v.IsNil() { e := v.Elem() if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) { v = e continue } } if v.Kind() != reflect.Ptr { break } if v.Elem().Kind() != reflect.Ptr && decodingNull && v.CanSet() { break } if v.IsNil() { v.Set(reflect.New(v.Type().Elem())) } if v.Type().NumMethod() > 0 { if u, ok := v.Interface().(Unmarshaler); ok { return u, nil, reflect.Value{} } if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { return nil, u, reflect.Value{} } } v = v.Elem() } return nil, nil, v } // array consumes an array from d.data[d.off-1:], decoding into the value v. // the first byte of the array ('[') has been read already. func (d *decodeState) array(v reflect.Value) { // Check for unmarshaler. u, ut, pv := d.indirect(v, false) if u != nil { d.off-- err := u.UnmarshalJSON(d.next()) if err != nil { d.error(err) } return } if ut != nil { d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)}) d.off-- d.next() return } v = pv // Check type of target. switch v.Kind() { case reflect.Interface: if v.NumMethod() == 0 { // Decoding into nil interface? Switch to non-reflect code. v.Set(reflect.ValueOf(d.arrayInterface())) return } // Otherwise it's invalid. fallthrough default: d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)}) d.off-- d.next() return case reflect.Array: case reflect.Slice: break } i := 0 for { // Look ahead for ] - can only happen on first iteration. op := d.scanWhile(scanSkipSpace) if op == scanEndArray { break } // Back up so d.value can have the byte we just read. d.off-- d.scan.undo(op) // Get element of array, growing if necessary. if v.Kind() == reflect.Slice { // Grow slice if necessary if i >= v.Cap() { newcap := v.Cap() + v.Cap()/2 if newcap < 4 { newcap = 4 } newv := reflect.MakeSlice(v.Type(), v.Len(), newcap) reflect.Copy(newv, v) v.Set(newv) } if i >= v.Len() { v.SetLen(i + 1) } } if i < v.Len() { // Decode into element. d.value(v.Index(i)) } else { // Ran out of fixed array: skip. d.value(reflect.Value{}) } i++ // Next token must be , or ]. op = d.scanWhile(scanSkipSpace) if op == scanEndArray { break } if op != scanArrayValue { d.error(errPhase) } } if i < v.Len() { if v.Kind() == reflect.Array { // Array. Zero the rest. z := reflect.Zero(v.Type().Elem()) for ; i < v.Len(); i++ { v.Index(i).Set(z) } } else { v.SetLen(i) } } if i == 0 && v.Kind() == reflect.Slice { v.Set(reflect.MakeSlice(v.Type(), 0, 0)) } } var nullLiteral = []byte("null") // object consumes an object from d.data[d.off-1:], decoding into the value v. // the first byte ('{') of the object has been read already. func (d *decodeState) object(v reflect.Value) { // Check for unmarshaler. u, ut, pv := d.indirect(v, false) if u != nil { d.off-- err := u.UnmarshalJSON(d.next()) if err != nil { d.error(err) } return } if ut != nil { d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) d.off-- d.next() // skip over { } in input return } v = pv // Decoding into nil interface? Switch to non-reflect code. if v.Kind() == reflect.Interface && v.NumMethod() == 0 { v.Set(reflect.ValueOf(d.objectInterface())) return } // Check type of target: struct or map[string]T switch v.Kind() { case reflect.Map: // map must have string kind t := v.Type() if t.Key().Kind() != reflect.String { d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) d.off-- d.next() // skip over { } in input return } if v.IsNil() { v.Set(reflect.MakeMap(t)) } case reflect.Struct: default: d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) d.off-- d.next() // skip over { } in input return } var mapElem reflect.Value for { // Read opening " of string key or closing }. op := d.scanWhile(scanSkipSpace) if op == scanEndObject { // closing } - can only happen on first iteration. break } if op != scanBeginLiteral { d.error(errPhase) } // Read key. start := d.off - 1 op = d.scanWhile(scanContinue) item := d.data[start : d.off-1] key, ok := unquoteBytes(item) if !ok { d.error(errPhase) } // Figure out field corresponding to key. var subv reflect.Value destring := false // whether the value is wrapped in a string to be decoded first if v.Kind() == reflect.Map { elemType := v.Type().Elem() if !mapElem.IsValid() { mapElem = reflect.New(elemType).Elem() } else { mapElem.Set(reflect.Zero(elemType)) } subv = mapElem } else { var f *field fields := cachedTypeFields(v.Type()) for i := range fields { ff := &fields[i] if bytes.Equal(ff.nameBytes, key) { f = ff break } if f == nil && ff.equalFold(ff.nameBytes, key) { f = ff } } if f != nil { subv = v destring = f.quoted for _, i := range f.index { if subv.Kind() == reflect.Ptr { if subv.IsNil() { subv.Set(reflect.New(subv.Type().Elem())) } subv = subv.Elem() } subv = subv.Field(i) } } } // Read : before value. if op == scanSkipSpace { op = d.scanWhile(scanSkipSpace) } if op != scanObjectKey { d.error(errPhase) } // Read value. if destring { switch qv := d.valueQuoted().(type) { case nil: d.literalStore(nullLiteral, subv, false) case string: d.literalStore([]byte(qv), subv, true) default: d.saveError(fmt.Errorf("canonicaljson: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", subv.Type())) } } else { d.value(subv) } // Write value back to map; // if using struct, subv points into struct already. if v.Kind() == reflect.Map { kv := reflect.ValueOf(key).Convert(v.Type().Key()) v.SetMapIndex(kv, subv) } // Next token must be , or }. op = d.scanWhile(scanSkipSpace) if op == scanEndObject { break } if op != scanObjectValue { d.error(errPhase) } } } // literal consumes a literal from d.data[d.off-1:], decoding into the value v. // The first byte of the literal has been read already // (that's how the caller knows it's a literal). func (d *decodeState) literal(v reflect.Value) { // All bytes inside literal return scanContinue op code. start := d.off - 1 op := d.scanWhile(scanContinue) // Scan read one byte too far; back up. d.off-- d.scan.undo(op) d.literalStore(d.data[start:d.off], v, false) } // convertNumber converts the number literal s to a float64 or a Number // depending on the setting of d.useNumber. func (d *decodeState) convertNumber(s string) (interface{}, error) { if d.useNumber { return Number(s), nil } f, err := strconv.ParseFloat(s, 64) if err != nil { return nil, &UnmarshalTypeError{"number " + s, reflect.TypeOf(0.0), int64(d.off)} } return f, nil } var numberType = reflect.TypeOf(Number("")) // literalStore decodes a literal stored in item into v. // // fromQuoted indicates whether this literal came from unwrapping a // string from the ",string" struct tag option. this is used only to // produce more helpful error messages. func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool) { // Check for unmarshaler. if len(item) == 0 { //Empty string given d.saveError(fmt.Errorf("canonicaljson: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) return } wantptr := item[0] == 'n' // null u, ut, pv := d.indirect(v, wantptr) if u != nil { err := u.UnmarshalJSON(item) if err != nil { d.error(err) } return } if ut != nil { if item[0] != '"' { if fromQuoted { d.saveError(fmt.Errorf("canonicaljson: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) } return } s, ok := unquoteBytes(item) if !ok { if fromQuoted { d.error(fmt.Errorf("canonicaljson: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { d.error(errPhase) } } err := ut.UnmarshalText(s) if err != nil { d.error(err) } return } v = pv switch c := item[0]; c { case 'n': // null switch v.Kind() { case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: v.Set(reflect.Zero(v.Type())) // otherwise, ignore null for primitives/string } case 't', 'f': // true, false value := c == 't' switch v.Kind() { default: if fromQuoted { d.saveError(fmt.Errorf("canonicaljson: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)}) } case reflect.Bool: v.SetBool(value) case reflect.Interface: if v.NumMethod() == 0 { v.Set(reflect.ValueOf(value)) } else { d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)}) } } case '"': // string s, ok := unquoteBytes(item) if !ok { if fromQuoted { d.error(fmt.Errorf("canonicaljson: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { d.error(errPhase) } } switch v.Kind() { default: d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) case reflect.Slice: if v.Type().Elem().Kind() != reflect.Uint8 { d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) break } b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) n, err := base64.StdEncoding.Decode(b, s) if err != nil { d.saveError(err) break } v.SetBytes(b[:n]) case reflect.String: v.SetString(string(s)) case reflect.Interface: if v.NumMethod() == 0 { v.Set(reflect.ValueOf(string(s))) } else { d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) } } default: // number if c != '-' && (c < '0' || c > '9') { if fromQuoted { d.error(fmt.Errorf("canonicaljson: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { d.error(errPhase) } } s := string(item) switch v.Kind() { default: if v.Kind() == reflect.String && v.Type() == numberType { v.SetString(s) if !isValidNumber(s) { d.error(fmt.Errorf("canonicaljson: invalid number literal, trying to unmarshal %q into Number", item)) } break } if fromQuoted { d.error(fmt.Errorf("canonicaljson: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { d.error(&UnmarshalTypeError{"number", v.Type(), int64(d.off)}) } case reflect.Interface: n, err := d.convertNumber(s) if err != nil { d.saveError(err) break } if v.NumMethod() != 0 { d.saveError(&UnmarshalTypeError{"number", v.Type(), int64(d.off)}) break } v.Set(reflect.ValueOf(n)) case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: n, err := strconv.ParseInt(s, 10, 64) if err != nil || v.OverflowInt(n) { d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) break } v.SetInt(n) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: n, err := strconv.ParseUint(s, 10, 64) if err != nil || v.OverflowUint(n) { d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) break } v.SetUint(n) case reflect.Float32, reflect.Float64: n, err := strconv.ParseFloat(s, v.Type().Bits()) if err != nil || v.OverflowFloat(n) { d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) break } v.SetFloat(n) } } } // The xxxInterface routines build up a value to be stored // in an empty interface. They are not strictly necessary, // but they avoid the weight of reflection in this common case. // valueInterface is like value but returns interface{} func (d *decodeState) valueInterface() interface{} { switch d.scanWhile(scanSkipSpace) { default: d.error(errPhase) panic("unreachable") case scanBeginArray: return d.arrayInterface() case scanBeginObject: return d.objectInterface() case scanBeginLiteral: return d.literalInterface() } } // arrayInterface is like array but returns []interface{}. func (d *decodeState) arrayInterface() []interface{} { var v = make([]interface{}, 0) for { // Look ahead for ] - can only happen on first iteration. op := d.scanWhile(scanSkipSpace) if op == scanEndArray { break } // Back up so d.value can have the byte we just read. d.off-- d.scan.undo(op) v = append(v, d.valueInterface()) // Next token must be , or ]. op = d.scanWhile(scanSkipSpace) if op == scanEndArray { break } if op != scanArrayValue { d.error(errPhase) } } return v } // objectInterface is like object but returns map[string]interface{}. func (d *decodeState) objectInterface() map[string]interface{} { m := make(map[string]interface{}) for { // Read opening " of string key or closing }. op := d.scanWhile(scanSkipSpace) if op == scanEndObject { // closing } - can only happen on first iteration. break } if op != scanBeginLiteral { d.error(errPhase) } // Read string key. start := d.off - 1 op = d.scanWhile(scanContinue) item := d.data[start : d.off-1] key, ok := unquote(item) if !ok { d.error(errPhase) } // Read : before value. if op == scanSkipSpace { op = d.scanWhile(scanSkipSpace) } if op != scanObjectKey { d.error(errPhase) } // Read value. m[key] = d.valueInterface() // Next token must be , or }. op = d.scanWhile(scanSkipSpace) if op == scanEndObject { break } if op != scanObjectValue { d.error(errPhase) } } return m } // literalInterface is like literal but returns an interface value. func (d *decodeState) literalInterface() interface{} { // All bytes inside literal return scanContinue op code. start := d.off - 1 op := d.scanWhile(scanContinue) // Scan read one byte too far; back up. d.off-- d.scan.undo(op) item := d.data[start:d.off] switch c := item[0]; c { case 'n': // null return nil case 't', 'f': // true, false return c == 't' case '"': // string s, ok := unquote(item) if !ok { d.error(errPhase) } return s default: // number if c != '-' && (c < '0' || c > '9') { d.error(errPhase) } n, err := d.convertNumber(string(item)) if err != nil { d.saveError(err) } return n } } // getu4 decodes \uXXXX from the beginning of s, returning the hex value, // or it returns -1. func getu4(s []byte) rune { if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { return -1 } r, err := strconv.ParseUint(string(s[2:6]), 16, 64) if err != nil { return -1 } return rune(r) } // unquote converts a quoted JSON string literal s into an actual string t. // The rules are different than for Go, so cannot use strconv.Unquote. func unquote(s []byte) (t string, ok bool) { s, ok = unquoteBytes(s) t = string(s) return } func unquoteBytes(s []byte) (t []byte, ok bool) { if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { return } s = s[1 : len(s)-1] // Check for unusual characters. If there are none, // then no unquoting is needed, so return a slice of the // original bytes. r := 0 for r < len(s) { c := s[r] if c == '\\' || c == '"' || c < ' ' { break } if c < utf8.RuneSelf { r++ continue } rr, size := utf8.DecodeRune(s[r:]) if rr == utf8.RuneError && size == 1 { break } r += size } if r == len(s) { return s, true } b := make([]byte, len(s)+2*utf8.UTFMax) w := copy(b, s[0:r]) for r < len(s) { // Out of room? Can only happen if s is full of // malformed UTF-8 and we're replacing each // byte with RuneError. if w >= len(b)-2*utf8.UTFMax { nb := make([]byte, (len(b)+utf8.UTFMax)*2) copy(nb, b[0:w]) b = nb } switch c := s[r]; { case c == '\\': r++ if r >= len(s) { return } switch s[r] { default: return case '"', '\\', '/', '\'': b[w] = s[r] r++ w++ case 'b': b[w] = '\b' r++ w++ case 'f': b[w] = '\f' r++ w++ case 'n': b[w] = '\n' r++ w++ case 'r': b[w] = '\r' r++ w++ case 't': b[w] = '\t' r++ w++ case 'u': r-- rr := getu4(s[r:]) if rr < 0 { return } r += 6 if utf16.IsSurrogate(rr) { rr1 := getu4(s[r:]) if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { // A valid pair; consume. r += 6 w += utf8.EncodeRune(b[w:], dec) break } // Lone surrogate; represent with a "WTF-8" encoding. w += copy(b[w:], []byte{ // 3-octet leading byte (4 bits of data) 0xE0 | byte(rr>>12), // first continuation byte (6 bits of data) 0x80 | byte((rr>>6)&0x3F), // second continuation byte (6 bits of data) 0x80 | byte(rr&0x3F), }) break } w += utf8.EncodeRune(b[w:], rr) } // Quote, control characters are invalid. case c == '"', c < ' ': return // ASCII case c < utf8.RuneSelf: b[w] = c r++ w++ // Reject ill-formed UTF-8. default: rr, size := utf8.DecodeRune(s[r:]) if rr == utf8.RuneError && size == 1 { return } r += size w += utf8.EncodeRune(b[w:], rr) } } return b[0:w], true } canonicaljson-go-1.0.3/decode_test.go000066400000000000000000001023411317167134000175540ustar00rootroot00000000000000// Copyright 2016 Richard Gibson. All rights reserved. // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package canonicaljson import ( "bytes" "encoding" "fmt" "image" "net" "reflect" "strings" "testing" "time" ) type T struct { X string Y int Z int `json:"-"` } type U struct { Alphabet string `json:"alpha"` } type V struct { F1 interface{} F2 int32 F3 Number } // ifaceNumAsFloat64/ifaceNumAsNumber are used to test unmarshaling with and // without UseNumber var ifaceNumAsFloat64 = map[string]interface{}{ "k1": float64(1), "k2": "s", "k3": []interface{}{float64(1), float64(2.0), float64(3e-3)}, "k4": map[string]interface{}{"kk1": "s", "kk2": float64(2)}, } var ifaceNumAsNumber = map[string]interface{}{ "k1": Number("1"), "k2": "s", "k3": []interface{}{Number("1"), Number("2"), Number("3.0E-3")}, "k4": map[string]interface{}{"kk1": "s", "kk2": Number("2")}, } type tx struct { x int } // A type that can unmarshal itself. type unmarshaler struct { T bool } func (u *unmarshaler) UnmarshalJSON(b []byte) error { *u = unmarshaler{true} // All we need to see that UnmarshalJSON is called. return nil } type ustruct struct { M unmarshaler } type unmarshalerText struct { T bool } // needed for re-marshaling tests func (u *unmarshalerText) MarshalText() ([]byte, error) { return []byte(""), nil } func (u *unmarshalerText) UnmarshalText(b []byte) error { *u = unmarshalerText{true} // All we need to see that UnmarshalText is called. return nil } var _ encoding.TextUnmarshaler = (*unmarshalerText)(nil) type ustructText struct { M unmarshalerText } var ( um0, um1 unmarshaler // target2 of unmarshaling ump = &um1 umtrue = unmarshaler{true} umslice = []unmarshaler{{true}} umslicep = new([]unmarshaler) umstruct = ustruct{unmarshaler{true}} um0T, um1T unmarshalerText // target2 of unmarshaling umpT = &um1T umtrueT = unmarshalerText{true} umsliceT = []unmarshalerText{{true}} umslicepT = new([]unmarshalerText) umstructT = ustructText{unmarshalerText{true}} ) // Test data structures for anonymous fields. type Point struct { Z int } type Top struct { Level0 int Embed0 *Embed0a *Embed0b `json:"e,omitempty"` // treated as named Embed0c `json:"-"` // ignored Loop Embed0p // has Point with X, Y, used Embed0q // has Point with Z, used embed // contains exported field } type Embed0 struct { Level1a int // overridden by Embed0a's Level1a with json tag Level1b int // used because Embed0a's Level1b is renamed Level1c int // used because Embed0a's Level1c is ignored Level1d int // annihilated by Embed0a's Level1d Level1e int `json:"x"` // annihilated by Embed0a.Level1e } type Embed0a struct { Level1a int `json:"Level1a,omitempty"` Level1b int `json:"LEVEL1B,omitempty"` Level1c int `json:"-"` Level1d int // annihilated by Embed0's Level1d Level1f int `json:"x"` // annihilated by Embed0's Level1e } type Embed0b Embed0 type Embed0c Embed0 type Embed0p struct { image.Point } type Embed0q struct { Point } type embed struct { Q int } type Loop struct { Loop1 int `json:",omitempty"` Loop2 int `json:",omitempty"` *Loop } // From reflect test: // The X in S6 and S7 annihilate, but they also block the X in S8.S9. type S5 struct { S6 S7 S8 } type S6 struct { X int } type S7 S6 type S8 struct { S9 } type S9 struct { X int Y int } // From reflect test: // The X in S11.S6 and S12.S6 annihilate, but they also block the X in S13.S8.S9. type S10 struct { S11 S12 S13 } type S11 struct { S6 } type S12 struct { S6 } type S13 struct { S8 } type unmarshalTest struct { label string in string ptr interface{} out interface{} err error useNumber bool } type Ambig struct { // Given "hello", the first match should win. First int `json:"HELLO"` Second int `json:"Hello"` } type XYZ struct { X interface{} Y interface{} Z interface{} } func sliceAddr(x []int) *[]int { return &x } func mapAddr(x map[string]int) *map[string]int { return &x } // wtf8 returns a UTF-8 like encoding of a code point, even if it is a surrogate. // See http://www.unicode.org/versions/Unicode10.0.0/ch03.pdf#G7404 func wtf8(cp rune) string { var result []byte if cp < 0x80 { result = []byte{byte(cp)} } else if cp < 0x800 { result = []byte{ 0xC0 | byte(cp>>6), 0x80 | byte(cp&0x3F), } } else if cp < 0x10000 { result = []byte{ 0xE0 | byte(cp>>12), 0x80 | byte((cp>>6)&0x3F), 0x80 | byte(cp&0x3F), } } else { result = []byte{ 0xF0 | byte(cp>>18), 0x80 | byte((cp>>12)&0x3F), 0x80 | byte((cp>>6)&0x3F), 0x80 | byte(cp&0x3F), } } return string(result) } var unmarshalTests = []unmarshalTest{ // basic types {in: `true`, ptr: new(bool), out: true}, {in: `1`, ptr: new(int), out: 1}, {in: `1.2`, ptr: new(float64), out: 1.2}, {in: `-5`, ptr: new(int16), out: int16(-5)}, {in: `2`, ptr: new(Number), out: Number("2"), useNumber: true}, {in: `2`, ptr: new(Number), out: Number("2")}, {in: `2`, ptr: new(interface{}), out: float64(2.0)}, {in: `2`, ptr: new(interface{}), out: Number("2"), useNumber: true}, {in: `"a\u1234"`, ptr: new(string), out: "a\u1234"}, {in: `"http:\/\/"`, ptr: new(string), out: "http://"}, {in: `"g-clef: \uD834\uDD1E"`, ptr: new(string), out: "g-clef: \U0001D11E"}, {in: `"lone surrogates: \uD834x\uDD1E"`, ptr: new(string), out: "lone surrogates: " + wtf8(0xD834) + "x" + wtf8(0xDD1E)}, {in: "null", ptr: new(interface{}), out: nil}, {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &UnmarshalTypeError{"array", reflect.TypeOf(""), 7}}, {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: Number("3")}}, {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: Number("1"), F2: int32(2), F3: Number("3")}, useNumber: true}, {label: "interface as float64", in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsFloat64}, {label: "interface as Number", in: `{"k1":1,"k2":"s","k3":[1,2,3.0E-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsNumber, useNumber: true}, // raw values with whitespace {in: "\n true ", ptr: new(bool), out: true}, {in: "\t 1 ", ptr: new(int), out: 1}, {in: "\r 1.2 ", ptr: new(float64), out: 1.2}, {in: "\t -5 \n", ptr: new(int16), out: int16(-5)}, {in: "\t \"a\\u1234\" \n", ptr: new(string), out: "a\u1234"}, // Z has a "-" tag. {in: `{"Y": 1, "Z": 2}`, ptr: new(T), out: T{Y: 1}}, {in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), out: U{Alphabet: "abc"}}, {in: `{"alpha": "abc"}`, ptr: new(U), out: U{Alphabet: "abc"}}, {in: `{"alphabet": "xyz"}`, ptr: new(U), out: U{}}, // syntax errors {in: `{"X": "foo", "Y"}`, err: &SyntaxError{"invalid character '}' after object key", 17}}, {in: `[1, 2, 3+]`, err: &SyntaxError{"invalid character '+' after array element", 9}}, {in: `{"X":12x}`, err: &SyntaxError{"invalid character 'x' after object key:value pair", 8}, useNumber: true}, // raw value errors {in: "\x01 42", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, {in: " 42 \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 5}}, {in: "\x01 true", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, {in: " false \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 8}}, {in: "\x01 1.2", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, {in: " 3.4 \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 6}}, {in: "\x01 \"string\"", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, {in: " \"string\" \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 11}}, // array tests {in: `[1, 2, 3]`, ptr: new([3]int), out: [3]int{1, 2, 3}}, {in: `[1, 2, 3]`, ptr: new([1]int), out: [1]int{1}}, {in: `[1, 2, 3]`, ptr: new([5]int), out: [5]int{1, 2, 3, 0, 0}}, // empty array to interface test {in: `[]`, ptr: new([]interface{}), out: []interface{}{}}, {in: `null`, ptr: new([]interface{}), out: []interface{}(nil)}, {in: `{"T":[]}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": []interface{}{}}}, {in: `{"T":null}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": interface{}(nil)}}, // composite tests {in: allValueIndent, ptr: new(All), out: allValue}, {in: allValueCompact, ptr: new(All), out: allValue}, {in: allValueIndent, ptr: new(*All), out: &allValue}, {in: allValueCompact, ptr: new(*All), out: &allValue}, {in: pallValueIndent, ptr: new(All), out: pallValue}, {in: pallValueCompact, ptr: new(All), out: pallValue}, {in: pallValueIndent, ptr: new(*All), out: &pallValue}, {in: pallValueCompact, ptr: new(*All), out: &pallValue}, // unmarshal interface test {in: `{"T":false}`, ptr: &um0, out: umtrue}, // use "false" so test will fail if custom unmarshaler is not called {in: `{"T":false}`, ptr: &ump, out: &umtrue}, {in: `[{"T":false}]`, ptr: &umslice, out: umslice}, {in: `[{"T":false}]`, ptr: &umslicep, out: &umslice}, {in: `{"M":{"T":false}}`, ptr: &umstruct, out: umstruct}, // UnmarshalText interface test {in: `"X"`, ptr: &um0T, out: umtrueT}, // use "false" so test will fail if custom unmarshaler is not called {in: `"X"`, ptr: &umpT, out: &umtrueT}, {in: `["X"]`, ptr: &umsliceT, out: umsliceT}, {in: `["X"]`, ptr: &umslicepT, out: &umsliceT}, {in: `{"M":"X"}`, ptr: &umstructT, out: umstructT}, // Overwriting of data. // This is different from package xml, but it's what we've always done. // Now documented and tested. {in: `[2]`, ptr: sliceAddr([]int{1}), out: []int{2}}, {in: `{"key": 2}`, ptr: mapAddr(map[string]int{"old": 0, "key": 1}), out: map[string]int{"key": 2}}, { in: `{ "Level0": 1, "Level1b": 2, "Level1c": 3, "x": 4, "Level1a": 5, "LEVEL1B": 6, "e": { "Level1a": 8, "Level1b": 9, "Level1c": 10, "Level1d": 11, "x": 12 }, "Loop1": 13, "Loop2": 14, "X": 15, "Y": 16, "Z": 17, "Q": 18 }`, ptr: new(Top), out: Top{ Level0: 1, Embed0: Embed0{ Level1b: 2, Level1c: 3, }, Embed0a: &Embed0a{ Level1a: 5, Level1b: 6, }, Embed0b: &Embed0b{ Level1a: 8, Level1b: 9, Level1c: 10, Level1d: 11, Level1e: 12, }, Loop: Loop{ Loop1: 13, Loop2: 14, }, Embed0p: Embed0p{ Point: image.Point{X: 15, Y: 16}, }, Embed0q: Embed0q{ Point: Point{Z: 17}, }, embed: embed{ Q: 18, }, }, }, { in: `{"hello": 1}`, ptr: new(Ambig), out: Ambig{First: 1}, }, { in: `{"X": 1,"Y":2}`, ptr: new(S5), out: S5{S8: S8{S9: S9{Y: 2}}}, }, { in: `{"X": 1,"Y":2}`, ptr: new(S10), out: S10{S13: S13{S8: S8{S9: S9{Y: 2}}}}, }, // Ill-formed UTF-8 is rejected. { in: "\"hello\xffworld\"", err: &SyntaxError{"invalid character '\\xff' in string literal (expecting UTF-8 leading byte)", 7}, }, { in: "\"hello\xc2\xc2world\"", err: &SyntaxError{"invalid character '\\xc2' in UTF-8 multi-byte sequence (expecting UTF-8 continuation byte)", 8}, }, { in: "\"hello\xc2\xffworld\"", err: &SyntaxError{"invalid character '\\xff' in UTF-8 multi-byte sequence (expecting UTF-8 continuation byte)", 8}, }, { in: "\"hello\\ud800world\"", ptr: new(string), out: "hello" + wtf8(0xd800) + "world", }, { in: "\"hello\\ud800\\ud800world\"", ptr: new(string), out: "hello" + wtf8(0xd800) + wtf8(0xd800) + "world", }, { in: "\"hello\xed\xa0\x80\xed\xb0\x80world\"", err: &SyntaxError{"invalid character '\\xa0' in UTF-8 three-byte sequence (expecting well-formed UTF-8 continuation byte)", 8}, }, { label: "issue 8305", in: `{"2009-11-10T23:00:00Z": "hello world"}`, ptr: &map[time.Time]string{}, err: &UnmarshalTypeError{"object", reflect.TypeOf(map[time.Time]string{}), 1}, }, } // TestMarshalNumberZeroVal ensures that, unlike encoding/json, we don't // marshal empty strings as "0". func TestMarshalNumberZeroVal(t *testing.T) { n := Number("") _, err := Marshal(n) got := fmt.Sprintf("%v", err) want := `canonicaljson: invalid number literal ""` if got != want { t.Fatalf("got err = %q, want %q", got, err) } } func TestMarshalEmbeds(t *testing.T) { top := &Top{ Level0: 1, Embed0: Embed0{ Level1b: 2, Level1c: 3, }, Embed0a: &Embed0a{ Level1a: 5, Level1b: 6, }, Embed0b: &Embed0b{ Level1a: 8, Level1b: 9, Level1c: 10, Level1d: 11, Level1e: 12, }, Loop: Loop{ Loop1: 13, Loop2: 14, }, Embed0p: Embed0p{ Point: image.Point{X: 15, Y: 16}, }, Embed0q: Embed0q{ Point: Point{Z: 17}, }, embed: embed{ Q: 18, }, } b, err := Marshal(top) if err != nil { t.Fatal(err) } want := `{"LEVEL1B":6,"Level0":1,"Level1a":5,"Level1b":2,"Level1c":3,"Loop1":13,"Loop2":14,"Q":18,"X":15,"Y":16,"Z":17,"e":{"Level1a":8,"Level1b":9,"Level1c":10,"Level1d":11,"x":12}}` if string(b) != want { t.Errorf("Wrong marshal result.\n got: %q\nwant: %q", b, want) } } func TestUnmarshal(t *testing.T) { for i, tt := range unmarshalTests { label := fmt.Sprintf("#%d %s [%q]", i, tt.label, tt.in) var scan scanner in := []byte(tt.in) if err := checkValid(in, &scan); err != nil { if !reflect.DeepEqual(err, tt.err) { t.Errorf("%s: checkValid: %#v", label, err) continue } } if tt.ptr == nil { continue } // v = new(right-type) v := reflect.New(reflect.TypeOf(tt.ptr).Elem()) dec := NewDecoder(bytes.NewReader(in)) if tt.useNumber { dec.UseNumber() } if err := dec.Decode(v.Interface()); !reflect.DeepEqual(err, tt.err) { t.Errorf("%s: Decode error: have %v, want %v", label, err, tt.err) continue } else if err != nil { continue } if !reflect.DeepEqual(v.Elem().Interface(), tt.out) { t.Errorf("%s: Decode mismatch\nhave: %#+v\nwant: %#+v", label, v.Elem().Interface(), tt.out) data, _ := Marshal(v.Elem().Interface()) println(string(data)) data, _ = Marshal(tt.out) println(string(data)) continue } // Check round trip. if tt.err == nil { enc, err := Marshal(v.Interface()) if err != nil { t.Errorf("%s: error re-marshaling: %v", label, err) continue } vv := reflect.New(reflect.TypeOf(tt.ptr).Elem()) dec = NewDecoder(bytes.NewReader(enc)) if tt.useNumber { dec.UseNumber() } if err := dec.Decode(vv.Interface()); err != nil { t.Errorf("%s: error re-unmarshaling %#q: %v", label, enc, err) continue } if !reflect.DeepEqual(v.Elem().Interface(), vv.Elem().Interface()) { t.Errorf("%s: round-trip mismatch\nhave: %#+v\nwant: %#+v", label, v.Elem().Interface(), vv.Elem().Interface()) t.Errorf(" In: %q", strings.Map(noSpace, string(in))) t.Errorf("Marshal: %q", strings.Map(noSpace, string(enc))) continue } } } } var numberTests = []struct { in string i int64 intErr string f float64 floatErr string }{ {in: "-1.23e1", intErr: "strconv.ParseInt: parsing \"-1.23e1\": invalid syntax", f: -1.23e1}, {in: "-12", i: -12, f: -12.0}, {in: "1e1000", intErr: "strconv.ParseInt: parsing \"1e1000\": invalid syntax", floatErr: "strconv.ParseFloat: parsing \"1e1000\": value out of range"}, } // Independent of Decode, basic coverage of the accessors in Number func TestNumberAccessors(t *testing.T) { for _, tt := range numberTests { n := Number(tt.in) if s := n.String(); s != tt.in { t.Errorf("Number(%q).String() is %q", tt.in, s) } if i, err := n.Int64(); err == nil && tt.intErr == "" && i != tt.i { t.Errorf("Number(%q).Int64() is %d", tt.in, i) } else if (err == nil && tt.intErr != "") || (err != nil && err.Error() != tt.intErr) { t.Errorf("Number(%q).Int64() wanted error %q but got: %v", tt.in, tt.intErr, err) } if f, err := n.Float64(); err == nil && tt.floatErr == "" && f != tt.f { t.Errorf("Number(%q).Float64() is %g", tt.in, f) } else if (err == nil && tt.floatErr != "") || (err != nil && err.Error() != tt.floatErr) { t.Errorf("Number(%q).Float64() wanted error %q but got: %v", tt.in, tt.floatErr, err) } } } func TestLargeByteSlice(t *testing.T) { s0 := make([]byte, 2000) for i := range s0 { s0[i] = byte(i) } b, err := Marshal(s0) if err != nil { t.Fatalf("Marshal: %v", err) } var s1 []byte if err := Unmarshal(b, &s1); err != nil { t.Fatalf("Unmarshal: %v", err) } if !bytes.Equal(s0, s1) { t.Errorf("Marshal large byte slice") diff(t, s0, s1) } } type Xint struct { X int } func TestUnmarshalInterface(t *testing.T) { var xint Xint var i interface{} = &xint if err := Unmarshal([]byte(`{"X":1}`), &i); err != nil { t.Fatalf("Unmarshal: %v", err) } if xint.X != 1 { t.Fatalf("Did not write to xint") } } func TestUnmarshalPtrPtr(t *testing.T) { var xint Xint pxint := &xint if err := Unmarshal([]byte(`{"X":1}`), &pxint); err != nil { t.Fatalf("Unmarshal: %v", err) } if xint.X != 1 { t.Fatalf("Did not write to xint") } } func TestEscape(t *testing.T) { const input = `"foobar"` + " [\u2028 \u2029]" const expected = `"` + `\"foobar\"` + " [\u2028 \u2029]" + `"` b, err := Marshal(input) if err != nil { t.Fatalf("Marshal error: %v", err) } if s := string(b); s != expected { t.Errorf("Encoding of [%s]:\n got [%s]\nwant [%s]", input, s, expected) } } // WrongString is a struct that's misusing the ,string modifier. type WrongString struct { Message string `json:"result,string"` } type wrongStringTest struct { in, err string } var wrongStringTests = []wrongStringTest{ {`{"result":"x"}`, `canonicaljson: invalid use of ,string struct tag, trying to unmarshal "x" into string`}, {`{"result":"foo"}`, `canonicaljson: invalid use of ,string struct tag, trying to unmarshal "foo" into string`}, {`{"result":"123"}`, `canonicaljson: invalid use of ,string struct tag, trying to unmarshal "123" into string`}, {`{"result":123}`, `canonicaljson: invalid use of ,string struct tag, trying to unmarshal unquoted value into string`}, } // If people misuse the ,string modifier, the error message should be // helpful, telling the user that they're doing it wrong. func TestErrorMessageFromMisusedString(t *testing.T) { for n, tt := range wrongStringTests { r := strings.NewReader(tt.in) var s WrongString err := NewDecoder(r).Decode(&s) got := fmt.Sprintf("%v", err) if got != tt.err { t.Errorf("%d. got err = %q, want %q", n, got, tt.err) } } } func noSpace(c rune) rune { if isSpace(byte(c)) { //only used for ascii return -1 } return c } type All struct { Bool bool Int int Int8 int8 Int16 int16 Int32 int32 Int64 int64 Uint uint Uint8 uint8 Uint16 uint16 Uint32 uint32 Uint64 uint64 Uintptr uintptr Float32 float32 Float64 float64 Foo string `json:"bar"` Foo2 string `json:"bar2,dummyopt"` IntStr int64 `json:",string"` PBool *bool PInt *int PInt8 *int8 PInt16 *int16 PInt32 *int32 PInt64 *int64 PUint *uint PUint8 *uint8 PUint16 *uint16 PUint32 *uint32 PUint64 *uint64 PUintptr *uintptr PFloat32 *float32 PFloat64 *float64 String string PString *string Map map[string]Small MapP map[string]*Small PMap *map[string]Small PMapP *map[string]*Small EmptyMap map[string]Small NilMap map[string]Small Slice []Small SliceP []*Small PSlice *[]Small PSliceP *[]*Small EmptySlice []Small NilSlice []Small StringSlice []string ByteSlice []byte Small Small PSmall *Small PPSmall **Small Interface interface{} PInterface *interface{} unexported int } type Small struct { Tag string } var allValue = All{ Bool: true, Int: 2, Int8: 3, Int16: 4, Int32: 5, Int64: 6, Uint: 7, Uint8: 8, Uint16: 9, Uint32: 10, Uint64: 11, Uintptr: 12, Float32: 14.1, Float64: 15.1, Foo: "foo", Foo2: "foo2", IntStr: 42, String: "16", Map: map[string]Small{ "17": {Tag: "tag17"}, "18": {Tag: "tag18"}, }, MapP: map[string]*Small{ "19": {Tag: "tag19"}, "20": nil, }, EmptyMap: map[string]Small{}, Slice: []Small{{Tag: "tag20"}, {Tag: "tag21"}}, SliceP: []*Small{{Tag: "tag22"}, nil, {Tag: "tag23"}}, EmptySlice: []Small{}, StringSlice: []string{"str24", "str25", "str26"}, ByteSlice: []byte{27, 28, 29}, Small: Small{Tag: "tag30"}, PSmall: &Small{Tag: "tag31"}, Interface: 5.2, } var pallValue = All{ PBool: &allValue.Bool, PInt: &allValue.Int, PInt8: &allValue.Int8, PInt16: &allValue.Int16, PInt32: &allValue.Int32, PInt64: &allValue.Int64, PUint: &allValue.Uint, PUint8: &allValue.Uint8, PUint16: &allValue.Uint16, PUint32: &allValue.Uint32, PUint64: &allValue.Uint64, PUintptr: &allValue.Uintptr, PFloat32: &allValue.Float32, PFloat64: &allValue.Float64, PString: &allValue.String, PMap: &allValue.Map, PMapP: &allValue.MapP, PSlice: &allValue.Slice, PSliceP: &allValue.SliceP, PPSmall: &allValue.PSmall, PInterface: &allValue.Interface, } var allValueIndent = `{ "Bool": true, "Int": 2, "Int8": 3, "Int16": 4, "Int32": 5, "Int64": 6, "Uint": 7, "Uint8": 8, "Uint16": 9, "Uint32": 10, "Uint64": 11, "Uintptr": 12, "Float32": 14.1, "Float64": 15.1, "bar": "foo", "bar2": "foo2", "IntStr": "42", "PBool": null, "PInt": null, "PInt8": null, "PInt16": null, "PInt32": null, "PInt64": null, "PUint": null, "PUint8": null, "PUint16": null, "PUint32": null, "PUint64": null, "PUintptr": null, "PFloat32": null, "PFloat64": null, "String": "16", "PString": null, "Map": { "17": { "Tag": "tag17" }, "18": { "Tag": "tag18" } }, "MapP": { "19": { "Tag": "tag19" }, "20": null }, "PMap": null, "PMapP": null, "EmptyMap": {}, "NilMap": null, "Slice": [ { "Tag": "tag20" }, { "Tag": "tag21" } ], "SliceP": [ { "Tag": "tag22" }, null, { "Tag": "tag23" } ], "PSlice": null, "PSliceP": null, "EmptySlice": [], "NilSlice": null, "StringSlice": [ "str24", "str25", "str26" ], "ByteSlice": "Gxwd", "Small": { "Tag": "tag30" }, "PSmall": { "Tag": "tag31" }, "PPSmall": null, "Interface": 5.2, "PInterface": null }` var allValueCompact = strings.Map(noSpace, allValueIndent) var pallValueIndent = `{ "Bool": false, "Int": 0, "Int8": 0, "Int16": 0, "Int32": 0, "Int64": 0, "Uint": 0, "Uint8": 0, "Uint16": 0, "Uint32": 0, "Uint64": 0, "Uintptr": 0, "Float32": 0, "Float64": 0, "bar": "", "bar2": "", "IntStr": "0", "PBool": true, "PInt": 2, "PInt8": 3, "PInt16": 4, "PInt32": 5, "PInt64": 6, "PUint": 7, "PUint8": 8, "PUint16": 9, "PUint32": 10, "PUint64": 11, "PUintptr": 12, "PFloat32": 14.1, "PFloat64": 15.1, "String": "", "PString": "16", "Map": null, "MapP": null, "PMap": { "17": { "Tag": "tag17" }, "18": { "Tag": "tag18" } }, "PMapP": { "19": { "Tag": "tag19" }, "20": null }, "EmptyMap": null, "NilMap": null, "Slice": null, "SliceP": null, "PSlice": [ { "Tag": "tag20" }, { "Tag": "tag21" } ], "PSliceP": [ { "Tag": "tag22" }, null, { "Tag": "tag23" } ], "EmptySlice": null, "NilSlice": null, "StringSlice": null, "ByteSlice": null, "Small": { "Tag": "" }, "PSmall": null, "PPSmall": { "Tag": "tag31" }, "Interface": null, "PInterface": 5.2 }` var pallValueCompact = strings.Map(noSpace, pallValueIndent) func TestRefUnmarshal(t *testing.T) { type S struct { // Ref is defined in encode_test.go. R0 Ref R1 *Ref R2 RefText R3 *RefText } want := S{ R0: 12, R1: new(Ref), R2: 13, R3: new(RefText), } *want.R1 = 12 *want.R3 = 13 var got S if err := Unmarshal([]byte(`{"R0":"ref","R1":"ref","R2":"ref","R3":"ref"}`), &got); err != nil { t.Fatalf("Unmarshal: %v", err) } if !reflect.DeepEqual(got, want) { t.Errorf("got %+v, want %+v", got, want) } } // Test that the empty string doesn't panic decoding when ,string is specified // Issue 3450 func TestEmptyString(t *testing.T) { type T2 struct { Number1 int `json:",string"` Number2 int `json:",string"` } data := `{"Number1":"1", "Number2":""}` dec := NewDecoder(strings.NewReader(data)) var t2 T2 err := dec.Decode(&t2) if err == nil { t.Fatal("Decode: did not return error") } if t2.Number1 != 1 { t.Fatal("Decode: did not set Number1") } } // Test that a null for ,string is not replaced with the previous quoted string (issue 7046). // It should also not be an error (issue 2540, issue 8587). func TestNullString(t *testing.T) { type T struct { A int `json:",string"` B int `json:",string"` C *int `json:",string"` } data := []byte(`{"A": "1", "B": null, "C": null}`) var s T s.B = 1 s.C = new(int) *s.C = 2 err := Unmarshal(data, &s) if err != nil { t.Fatalf("Unmarshal: %v", err) } if s.B != 1 || s.C != nil { t.Fatalf("after Unmarshal, s.B=%d, s.C=%p, want 1, nil", s.B, s.C) } } func intp(x int) *int { p := new(int) *p = x return p } func intpp(x *int) **int { pp := new(*int) *pp = x return pp } var interfaceSetTests = []struct { pre interface{} json string post interface{} }{ {"foo", `"bar"`, "bar"}, {"foo", `2`, 2.0}, {"foo", `true`, true}, {"foo", `null`, nil}, {nil, `null`, nil}, {new(int), `null`, nil}, {(*int)(nil), `null`, nil}, {new(*int), `null`, new(*int)}, {(**int)(nil), `null`, nil}, {intp(1), `null`, nil}, {intpp(nil), `null`, intpp(nil)}, {intpp(intp(1)), `null`, intpp(nil)}, } func TestInterfaceSet(t *testing.T) { for _, tt := range interfaceSetTests { b := struct{ X interface{} }{tt.pre} blob := `{"X":` + tt.json + `}` if err := Unmarshal([]byte(blob), &b); err != nil { t.Errorf("Unmarshal %#q: %v", blob, err) continue } if !reflect.DeepEqual(b.X, tt.post) { t.Errorf("Unmarshal %#q into %#v: X=%#v, want %#v", blob, tt.pre, b.X, tt.post) } } } // JSON null values should be ignored for primitives and string values instead of resulting in an error. // Issue 2540 func TestUnmarshalNulls(t *testing.T) { jsonData := []byte(`{ "Bool" : null, "Int" : null, "Int8" : null, "Int16" : null, "Int32" : null, "Int64" : null, "Uint" : null, "Uint8" : null, "Uint16" : null, "Uint32" : null, "Uint64" : null, "Float32" : null, "Float64" : null, "String" : null}`) nulls := All{ Bool: true, Int: 2, Int8: 3, Int16: 4, Int32: 5, Int64: 6, Uint: 7, Uint8: 8, Uint16: 9, Uint32: 10, Uint64: 11, Float32: 12.1, Float64: 13.1, String: "14"} err := Unmarshal(jsonData, &nulls) if err != nil { t.Errorf("Unmarshal of null values failed: %v", err) } if !nulls.Bool || nulls.Int != 2 || nulls.Int8 != 3 || nulls.Int16 != 4 || nulls.Int32 != 5 || nulls.Int64 != 6 || nulls.Uint != 7 || nulls.Uint8 != 8 || nulls.Uint16 != 9 || nulls.Uint32 != 10 || nulls.Uint64 != 11 || nulls.Float32 != 12.1 || nulls.Float64 != 13.1 || nulls.String != "14" { t.Errorf("Unmarshal of null values affected primitives") } } func TestStringKind(t *testing.T) { type stringKind string var m1, m2 map[stringKind]int m1 = map[stringKind]int{ "foo": 42, } data, err := Marshal(m1) if err != nil { t.Errorf("Unexpected error marshaling: %v", err) } err = Unmarshal(data, &m2) if err != nil { t.Errorf("Unexpected error unmarshaling: %v", err) } if !reflect.DeepEqual(m1, m2) { t.Error("Items should be equal after encoding and then decoding") } } // Custom types with []byte as underlying type could not be marshalled // and then unmarshalled. // Issue 8962. func TestByteKind(t *testing.T) { type byteKind []byte a := byteKind("hello") data, err := Marshal(a) if err != nil { t.Error(err) } var b byteKind err = Unmarshal(data, &b) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(a, b) { t.Errorf("expected %v == %v", a, b) } } // The fix for issue 8962 introduced a regression. // Issue 12921. func TestSliceOfCustomByte(t *testing.T) { type Uint8 uint8 a := []Uint8("hello") data, err := Marshal(a) if err != nil { t.Fatal(err) } var b []Uint8 err = Unmarshal(data, &b) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(a, b) { t.Fatal("expected %v == %v", a, b) } } var decodeTypeErrorTests = []struct { dest interface{} src string }{ {new(string), `{"user": "name"}`}, // issue 4628. {new(error), `{}`}, // issue 4222 {new(error), `[]`}, {new(error), `""`}, {new(error), `123`}, {new(error), `true`}, } func TestUnmarshalTypeError(t *testing.T) { for _, item := range decodeTypeErrorTests { err := Unmarshal([]byte(item.src), item.dest) if _, ok := err.(*UnmarshalTypeError); !ok { t.Errorf("expected type error for Unmarshal(%q, type %T): got %T", item.src, item.dest, err) } } } var unmarshalSyntaxTests = []string{ "tru", "fals", "nul", "123e", `"hello`, `[1,2,3`, `{"key":1`, `{"key":1,`, } func TestUnmarshalSyntax(t *testing.T) { var x interface{} for _, src := range unmarshalSyntaxTests { err := Unmarshal([]byte(src), &x) if _, ok := err.(*SyntaxError); !ok { t.Errorf("expected syntax error for Unmarshal(%q): got %T", src, err) } } } // Test handling of unexported fields that should be ignored. // Issue 4660 type unexportedFields struct { Name string m map[string]interface{} `json:"-"` m2 map[string]interface{} `json:"abcd"` } func TestUnmarshalUnexported(t *testing.T) { input := `{"Name": "Bob", "m": {"x": 123}, "m2": {"y": 456}, "abcd": {"z": 789}}` want := &unexportedFields{Name: "Bob"} out := &unexportedFields{} err := Unmarshal([]byte(input), out) if err != nil { t.Errorf("got error %v, expected nil", err) } if !reflect.DeepEqual(out, want) { t.Errorf("got %q, want %q", out, want) } } // Time3339 is a time.Time which encodes to and from JSON // as an RFC 3339 time in UTC. type Time3339 time.Time func (t *Time3339) UnmarshalJSON(b []byte) error { if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' { return fmt.Errorf("types: failed to unmarshal non-string value %q as an RFC 3339 time", b) } tm, err := time.Parse(time.RFC3339, string(b[1:len(b)-1])) if err != nil { return err } *t = Time3339(tm) return nil } func TestUnmarshalJSONLiteralError(t *testing.T) { var t3 Time3339 err := Unmarshal([]byte(`"0000-00-00T00:00:00Z"`), &t3) if err == nil { t.Fatalf("expected error; got time %v", time.Time(t3)) } if !strings.Contains(err.Error(), "range") { t.Errorf("got err = %v; want out of range error", err) } } // Test that extra object elements in an array do not result in a // "data changing underfoot" error. // Issue 3717 func TestSkipArrayObjects(t *testing.T) { json := `[{}]` var dest [0]interface{} err := Unmarshal([]byte(json), &dest) if err != nil { t.Errorf("got error %q, want nil", err) } } // Test semantics of pre-filled struct fields and pre-filled map fields. // Issue 4900. func TestPrefilled(t *testing.T) { ptrToMap := func(m map[string]interface{}) *map[string]interface{} { return &m } // Values here change, cannot reuse table across runs. var prefillTests = []struct { in string ptr interface{} out interface{} }{ { in: `{"X": 1, "Y": 2}`, ptr: &XYZ{X: float32(3), Y: int16(4), Z: 1.5}, out: &XYZ{X: float64(1), Y: float64(2), Z: 1.5}, }, { in: `{"X": 1, "Y": 2}`, ptr: ptrToMap(map[string]interface{}{"X": float32(3), "Y": int16(4), "Z": 1.5}), out: ptrToMap(map[string]interface{}{"X": float64(1), "Y": float64(2), "Z": 1.5}), }, } for _, tt := range prefillTests { ptrstr := fmt.Sprintf("%v", tt.ptr) err := Unmarshal([]byte(tt.in), tt.ptr) // tt.ptr edited here if err != nil { t.Errorf("Unmarshal: %v", err) } if !reflect.DeepEqual(tt.ptr, tt.out) { t.Errorf("Unmarshal(%#q, %s): have %v, want %v", tt.in, ptrstr, tt.ptr, tt.out) } } } var invalidUnmarshalTests = []struct { v interface{} want string }{ {nil, "canonicaljson: Unmarshal(nil)"}, {struct{}{}, "canonicaljson: Unmarshal(non-pointer struct {})"}, {(*int)(nil), "canonicaljson: Unmarshal(nil *int)"}, } func TestInvalidUnmarshal(t *testing.T) { buf := []byte(`{"a":"1"}`) for _, tt := range invalidUnmarshalTests { err := Unmarshal(buf, tt.v) if err == nil { t.Errorf("Unmarshal expecting error, got nil") continue } if got := err.Error(); got != tt.want { t.Errorf("Unmarshal = %q; want %q", got, tt.want) } } } var invalidUnmarshalTextTests = []struct { v interface{} want string }{ {nil, "canonicaljson: Unmarshal(nil)"}, {struct{}{}, "canonicaljson: Unmarshal(non-pointer struct {})"}, {(*int)(nil), "canonicaljson: Unmarshal(nil *int)"}, {new(net.IP), "canonicaljson: cannot unmarshal string into Go value of type *net.IP"}, } func TestInvalidUnmarshalText(t *testing.T) { buf := []byte(`123`) for _, tt := range invalidUnmarshalTextTests { err := Unmarshal(buf, tt.v) if err == nil { t.Errorf("Unmarshal expecting error, got nil") continue } if got := err.Error(); got != tt.want { t.Errorf("Unmarshal = %q; want %q", got, tt.want) } } } // Test that string option is ignored for invalid types. // Issue 9812. func TestInvalidStringOption(t *testing.T) { num := 0 item := struct { T time.Time `json:",string"` M map[string]string `json:",string"` S []string `json:",string"` A [1]string `json:",string"` I interface{} `json:",string"` P *int `json:",string"` }{M: make(map[string]string), S: make([]string, 0), I: num, P: &num} data, err := Marshal(item) if err != nil { t.Fatalf("Marshal: %v", err) } err = Unmarshal(data, &item) if err != nil { t.Fatalf("Unmarshal: %v", err) } } canonicaljson-go-1.0.3/encode.go000066400000000000000000000773561317167134000165500ustar00rootroot00000000000000// Copyright 2016 Richard Gibson. All rights reserved. // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package canonicaljson implements canonical serialization of Go // objects to canonical-form JSON as specified at // https://gibson042.github.io/canonicaljson-spec/ . // The provided interface should match that of standard package // "encoding/json" (from which it is derived) wherever they overlap (and // in fact, this package is essentially a 2016-03-09 fork from // golang/go@9d77ad8d34ce56e182adc30cd21af50a4b00932c:src/encoding/json // ). Notable differences: // - Object keys are sorted lexicographically by code point // - Non-integer JSON numbers are represented in capital-E exponential // notation with significand in (-10, 10) and no insignificant signs // or zeroes beyond those required to force a decimal point. // - JSON strings are represented in UTF-8 with minimal byte length, // using escapes only when necessary for validity and Unicode // escapes (uppercase hex) only when there is no shorter option. package canonicaljson import ( "bytes" "encoding" "encoding/base64" "encoding/json" "fmt" "math" "reflect" "regexp" "runtime" "sort" "strconv" "strings" "sync" "unicode" "unicode/utf8" ) // Marshal returns the canonical UTF-8 JSON encoding of v. // // Marshal traverses the value v recursively. // If an encountered value implements the json.Marshaler interface // and is not a nil pointer, Marshal calls its MarshalJSON method // to produce JSON. If no MarshalJSON method is present but the // value implements encoding.TextMarshaler instead, Marshal calls // its MarshalText method. // The nil pointer exception is not strictly necessary // but mimics a similar, necessary exception in the behavior of // json.UnmarshalJSON. // // Otherwise, Marshal uses the following type-dependent default encodings: // // Boolean values encode as JSON booleans. // // Floating point, integer, and Number values encode as JSON numbers. // Non-fractional values become sequences of digits without leading // spaces; fractional values are represented in capital-E exponential // notation with the shortest possible significand of magnitude less // than 10 that includes at least one digit both before and after the // decimal point, and the shortest possible non-empty exponent. // // String values encode as JSON strings, treating ill-formed UTF-8 input // as an error (with the exception of "WTF-8" encodings of lone // surrogates—code points U+D800 through U+DFFF [inclusive] that could // not be interpreted as part of a surrogate pair). // This is in contrast to encoding/json, which replaces ill-formed // sequences with U+FFFD REPLACEMENT CHARACTER. // Also in contrast to encoding/json, characters appear unescaped // whenever possible. // Control characters U+0000 through U+001F and lone surrogates U+D800 // through U+DFFF are replaced with their shortest escape sequence, // 4 uppercase hex characters except for the following: // - \b U+0008 BACKSPACE // - \t U+0009 CHARACTER TABULATION ("tab") // - \n U+000A LINE FEED ("newline") // - \f U+000C FORM FEED // - \r U+000D CARRIAGE RETURN // // Array and slice values encode as JSON arrays, except that // []byte encodes as a base64-encoded string, and a nil slice // encodes as the null JSON object. // // Struct values encode as JSON objects. Each exported struct field // becomes a member of the object unless // - the field's tag is "-", or // - the field is empty and its tag specifies the "omitempty" option. // The empty values are false, 0, any // nil pointer or interface value, and any array, slice, map, or string of // length zero. The object's default key string is the struct field name // but can be specified in the struct field's tag value. The "json" key in // the struct field's tag value is the key name, followed by an optional comma // and options. Examples: // // // Field is ignored by this package. // Field int `json:"-"` // // // Field appears in JSON as key "myName". // Field int `json:"myName"` // // // Field appears in JSON as key "myName" and // // the field is omitted from the object if its value is empty, // // as defined above. // Field int `json:"myName,omitempty"` // // // Field appears in JSON as key "Field" (the default), but // // the field is skipped if empty. // // Note the leading comma. // Field int `json:",omitempty"` // // The "string" option signals that a field is stored as JSON inside a // JSON-encoded string. It applies only to fields of string, floating point, // integer, or boolean types. This extra level of encoding is sometimes used // when communicating with JavaScript programs: // // Int64String int64 `json:",string"` // // The key name will be used if it's a non-empty string consisting of // only Unicode letters, digits, dollar signs, percent signs, hyphens, // underscores and slashes. // // Anonymous struct fields are usually marshaled as if their inner exported fields // were fields in the outer struct, subject to the usual Go visibility rules amended // as described in the next paragraph. // An anonymous struct field with a name given in its JSON tag is treated as // having that name, rather than being anonymous. // An anonymous struct field of interface type is treated the same as having // that type as its name, rather than being anonymous. // // The Go visibility rules for struct fields are amended for JSON when // deciding which field to marshal. If there are // multiple fields at the same level, and that level is the least // nested (and would therefore be the nesting level selected by the // usual Go rules), the following extra rules apply: // // 1) Of those fields, if any are JSON-tagged, only tagged fields are considered, // even if there are multiple untagged fields that would otherwise conflict. // 2) If there is exactly one field (tagged or not according to the first rule), that is selected. // 3) Otherwise there are multiple fields, and all are ignored; no error occurs. // // Handling of anonymous struct fields is new in Go 1.1. // Prior to Go 1.1, anonymous struct fields were ignored. To force ignoring of // an anonymous struct field in both current and earlier versions, give the field // a JSON tag of "-". // // Map values encode as JSON objects. // The map's key type must be string; the map keys are used as JSON object // keys, subject to the UTF-8 coercion described for string values above. // // Pointer values encode as the value pointed to. // A nil pointer encodes as the null JSON object. // // Interface values encode as the value contained in the interface. // A nil interface value encodes as the null JSON object. // // Channel, complex, and function values cannot be encoded in JSON. // Attempting to encode such a value causes Marshal to return // an UnsupportedTypeError. // // JSON cannot represent cyclic data structures and Marshal does not // handle them. Passing cyclic structures to Marshal will result in // an infinite recursion. // func Marshal(v interface{}) ([]byte, error) { e := &encodeState{} err := e.marshal(v) if err != nil { return nil, err } return e.Bytes(), nil } // MarshalIndent is like Marshal, but adds whitespace for more readable output. func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { b, err := Marshal(v) if err != nil { return nil, err } var buf bytes.Buffer err = addIndentation(&buf, b, prefix, indent) if err != nil { return nil, err } return buf.Bytes(), nil } // Marshaler is the interface implemented by objects that // can marshal themselves into valid JSON. type Marshaler interface { MarshalJSON() ([]byte, error) } // An UnsupportedTypeError is returned by Marshal when attempting // to encode an unsupported value type. type UnsupportedTypeError struct { Type reflect.Type } func (e *UnsupportedTypeError) Error() string { return "canonicaljson: unsupported type: " + e.Type.String() } type UnsupportedValueError struct { Value reflect.Value Str string } func (e *UnsupportedValueError) Error() string { return "canonicaljson: unsupported value: " + e.Str } type MarshalerError struct { Type reflect.Type Err error } func (e *MarshalerError) Error() string { return "canonicaljson: error calling MarshalJSON for type " + e.Type.String() + ": " + e.Err.Error() } var hex = "0123456789ABCDEF" var jsonNumberType = reflect.TypeOf(json.Number("")) // An encodeState encodes JSON into a bytes.Buffer. type encodeState struct { bytes.Buffer // accumulated output scratch [64]byte } var encodeStatePool sync.Pool func newEncodeState() *encodeState { if v := encodeStatePool.Get(); v != nil { e := v.(*encodeState) e.Reset() return e } return new(encodeState) } func (e *encodeState) marshal(v interface{}) (err error) { defer func() { if r := recover(); r != nil { if _, ok := r.(runtime.Error); ok { panic(r) } if s, ok := r.(string); ok { panic(s) } err = r.(error) } }() e.reflectValue(reflect.ValueOf(v)) return nil } func (e *encodeState) error(err error) { panic(err) } func isEmptyValue(v reflect.Value) bool { switch v.Kind() { case reflect.Array, reflect.Map, reflect.Slice, reflect.String: return v.Len() == 0 case reflect.Bool: return !v.Bool() case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return v.Int() == 0 case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: return v.Uint() == 0 case reflect.Float32, reflect.Float64: return v.Float() == 0 case reflect.Interface, reflect.Ptr: return v.IsNil() } return false } func (e *encodeState) reflectValue(v reflect.Value) { valueEncoder(v)(e, v, false) } type encoderFunc func(e *encodeState, v reflect.Value, quoted bool) var encoderCache struct { sync.RWMutex m map[reflect.Type]encoderFunc } func valueEncoder(v reflect.Value) encoderFunc { if !v.IsValid() { return invalidValueEncoder } return typeEncoder(v.Type()) } func typeEncoder(t reflect.Type) encoderFunc { encoderCache.RLock() f := encoderCache.m[t] encoderCache.RUnlock() if f != nil { return f } // To deal with recursive types, populate the map with an // indirect func before we build it. This type waits on the // real func (f) to be ready and then calls it. This indirect // func is only used for recursive types. encoderCache.Lock() if encoderCache.m == nil { encoderCache.m = make(map[reflect.Type]encoderFunc) } var wg sync.WaitGroup wg.Add(1) encoderCache.m[t] = func(e *encodeState, v reflect.Value, quoted bool) { wg.Wait() f(e, v, quoted) } encoderCache.Unlock() // Compute fields without lock. // Might duplicate effort but won't hold other computations back. f = newTypeEncoder(t, true) wg.Done() encoderCache.Lock() encoderCache.m[t] = f encoderCache.Unlock() return f } var ( marshalerType = reflect.TypeOf(new(json.Marshaler)).Elem() textMarshalerType = reflect.TypeOf(new(encoding.TextMarshaler)).Elem() ) // newTypeEncoder constructs an encoderFunc for a type. // The returned encoder only checks CanAddr when allowAddr is true. func newTypeEncoder(t reflect.Type, allowAddr bool) encoderFunc { if t.Implements(marshalerType) { return marshalerEncoder } if t.Kind() != reflect.Ptr && allowAddr { if reflect.PtrTo(t).Implements(marshalerType) { return newCondAddrEncoder(addrMarshalerEncoder, newTypeEncoder(t, false)) } } if t.Implements(textMarshalerType) { return textMarshalerEncoder } if t.Kind() != reflect.Ptr && allowAddr { if reflect.PtrTo(t).Implements(textMarshalerType) { return newCondAddrEncoder(addrTextMarshalerEncoder, newTypeEncoder(t, false)) } } switch t.Kind() { case reflect.Bool: return boolEncoder case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return intEncoder case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: return uintEncoder case reflect.Float32: return float32Encoder case reflect.Float64: return float64Encoder case reflect.String: return stringEncoder case reflect.Interface: return interfaceEncoder case reflect.Struct: return newStructEncoder(t) case reflect.Map: return newMapEncoder(t) case reflect.Slice: return newSliceEncoder(t) case reflect.Array: return newArrayEncoder(t) case reflect.Ptr: return newPtrEncoder(t) default: return unsupportedTypeEncoder } } func invalidValueEncoder(e *encodeState, v reflect.Value, quoted bool) { e.WriteString("null") } func marshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { if v.Kind() == reflect.Ptr && v.IsNil() { e.WriteString("null") return } m := v.Interface().(json.Marshaler) b, err := m.MarshalJSON() // Remarshal JSON, checking validity. var data interface{} var jsonBlob []byte if err == nil { err = Unmarshal(b, &data) } if err == nil { jsonBlob, err = Marshal(data) } if err == nil { e.Buffer.Write(jsonBlob) } if err != nil { e.error(&MarshalerError{v.Type(), err}) } } func addrMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { va := v.Addr() if va.IsNil() { e.WriteString("null") return } m := va.Interface().(json.Marshaler) b, err := m.MarshalJSON() // Remarshal JSON, checking validity. var data interface{} var jsonBlob []byte if err == nil { err = Unmarshal(b, &data) } if err == nil { jsonBlob, err = Marshal(data) } if err == nil { e.Buffer.Write(jsonBlob) } if err != nil { e.error(&MarshalerError{v.Type(), err}) } } func textMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { if v.Kind() == reflect.Ptr && v.IsNil() { e.WriteString("null") return } m := v.Interface().(encoding.TextMarshaler) b, err := m.MarshalText() if err != nil { e.error(&MarshalerError{v.Type(), err}) } e.stringBytes(b) } func addrTextMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { va := v.Addr() if va.IsNil() { e.WriteString("null") return } m := va.Interface().(encoding.TextMarshaler) b, err := m.MarshalText() if err != nil { e.error(&MarshalerError{v.Type(), err}) } e.stringBytes(b) } func boolEncoder(e *encodeState, v reflect.Value, quoted bool) { if quoted { e.WriteByte('"') } if v.Bool() { e.WriteString("true") } else { e.WriteString("false") } if quoted { e.WriteByte('"') } } func intEncoder(e *encodeState, v reflect.Value, quoted bool) { b := strconv.AppendInt(e.scratch[:0], v.Int(), 10) if quoted { e.WriteByte('"') e.Write(b) e.WriteByte('"') } else { e.Write(b) } } func uintEncoder(e *encodeState, v reflect.Value, quoted bool) { b := strconv.AppendUint(e.scratch[:0], v.Uint(), 10) if quoted { e.WriteByte('"') e.Write(b) e.WriteByte('"') } else { e.Write(b) } } // Force capital-E exponent, remove + signs and leading zeroes var expNormalizer = regexp.MustCompile("(?:E(?:[+]0*|(-|)0+)|e(?:[+]|(-|))0*)([0-9])") var expNormalizerReplacement = "E$1$2$3" // Find the significant digits in a numeric string var significantDigits = regexp.MustCompile(`^(?:0(?:\.0*|)|)([0-9.]+?)(?:0*\.?0*|)(?:E|$)`) // normalizeNumber normalizes a valid JSON numeric string and writes the result on its encoder. func normalizeNumber(e *encodeState, stringBytes []byte) { s := expNormalizer.ReplaceAllString(string(stringBytes), expNormalizerReplacement) // detect negative sign negative := s[0] == '-' if negative { s = s[1:] } // detect exponent exp := 0 expPos := strings.LastIndexByte(s, 'E') if expPos == -1 { expPos = len(s) } else { var err error exp, err = strconv.Atoi(s[expPos+1:]) if err != nil { e.error(err) } } // characterize the significant digits, finishing early on zero significantRange := significantDigits.FindStringSubmatchIndex(s) if significantRange == nil { e.error(fmt.Errorf("canonicaljson: no significand in %q", s)) } significantRange = significantRange[2:4] if s[significantRange[0]] == '0' { e.WriteByte('0') return } pointPos := strings.IndexByte(s, '.') effectivePointPos := pointPos if effectivePointPos == -1 { effectivePointPos = expPos } var significand string if pointPos > significantRange[0] && pointPos < significantRange[1] { significand = s[significantRange[0]:pointPos] + s[pointPos+1:significantRange[1]] } else { significand = s[significantRange[0]:significantRange[1]] } // detect integers (i.e., significand fits within exponent) isInt := (exp <= 0 && (effectivePointPos+exp) >= significantRange[1]) || (exp > 0 && (effectivePointPos+exp+1) >= significantRange[1]) // effective exponent increases/decreases by excess/deficient significand magnitude if effectivePointPos > 1 { exp += effectivePointPos - 1 } else if significantRange[0] > 0 { exp -= significantRange[0] - 1 } // write result if negative { e.WriteByte('-') } if isInt { // integer: render without exponent e.WriteString(significand) if len(significand) < exp+1 { e.WriteString(strings.Repeat("0", exp+1-len(significand))) } } else { // non-integer: render minimal significand with decimal point and non-empty exponent if len(significand) == 1 { significand += "0" } e.WriteString(significand[:1] + "." + significand[1:] + "E" + strconv.Itoa(exp)) } } type floatEncoder int // number of bits func (bits floatEncoder) encode(e *encodeState, v reflect.Value, quoted bool) { // Get a float value *not* equal to negative zero. f := v.Float() + 0 if math.IsInf(f, 0) || math.IsNaN(f) { e.error(&UnsupportedValueError{v, strconv.FormatFloat(f, 'g', -1, int(bits))}) } b := strconv.AppendFloat(e.scratch[:0], f, 'E', -1, int(bits)) if quoted { e.WriteByte('"') e.Write(b) e.WriteByte('"') } else { normalizeNumber(e, b) } } var ( float32Encoder = (floatEncoder(32)).encode float64Encoder = (floatEncoder(64)).encode ) func stringEncoder(e *encodeState, v reflect.Value, quoted bool) { if t := v.Type(); t == numberType || t == jsonNumberType { numStr := v.String() if !isValidNumber(numStr) { e.error(fmt.Errorf("canonicaljson: invalid number literal %q", numStr)) } normalizeNumber(e, []byte(numStr)) return } if quoted { sb, err := Marshal(v.String()) if err != nil { e.error(err) } e.string(string(sb)) } else { e.string(v.String()) } } func interfaceEncoder(e *encodeState, v reflect.Value, quoted bool) { if v.IsNil() { e.WriteString("null") return } e.reflectValue(v.Elem()) } func unsupportedTypeEncoder(e *encodeState, v reflect.Value, quoted bool) { e.error(&UnsupportedTypeError{v.Type()}) } type structEncoder struct { fields []field fieldEncs []encoderFunc } func (se *structEncoder) encode(e *encodeState, v reflect.Value, quoted bool) { e.WriteByte('{') first := true for i, f := range se.fields { fv := fieldByIndex(v, f.index) if !fv.IsValid() || f.omitEmpty && isEmptyValue(fv) { continue } if first { first = false } else { e.WriteByte(',') } e.string(f.name) e.WriteByte(':') se.fieldEncs[i](e, fv, f.quoted) } e.WriteByte('}') } func newStructEncoder(t reflect.Type) encoderFunc { fields := cachedTypeFields(t) se := &structEncoder{ fields: fields, fieldEncs: make([]encoderFunc, len(fields)), } for i, f := range fields { se.fieldEncs[i] = typeEncoder(typeByIndex(t, f.index)) } return se.encode } type mapEncoder struct { elemEnc encoderFunc } func (me *mapEncoder) encode(e *encodeState, v reflect.Value, _ bool) { if v.IsNil() { e.WriteString("null") return } e.WriteByte('{') var sv stringValues = v.MapKeys() sort.Sort(sv) for i, k := range sv { if i > 0 { e.WriteByte(',') } e.string(k.String()) e.WriteByte(':') me.elemEnc(e, v.MapIndex(k), false) } e.WriteByte('}') } func newMapEncoder(t reflect.Type) encoderFunc { if t.Key().Kind() != reflect.String { return unsupportedTypeEncoder } me := &mapEncoder{typeEncoder(t.Elem())} return me.encode } func encodeByteSlice(e *encodeState, v reflect.Value, _ bool) { if v.IsNil() { e.WriteString("null") return } s := v.Bytes() e.WriteByte('"') if len(s) < 1024 { // for small buffers, using Encode directly is much faster. dst := make([]byte, base64.StdEncoding.EncodedLen(len(s))) base64.StdEncoding.Encode(dst, s) e.Write(dst) } else { // for large buffers, avoid unnecessary extra temporary // buffer space. enc := base64.NewEncoder(base64.StdEncoding, e) enc.Write(s) enc.Close() } e.WriteByte('"') } // sliceEncoder just wraps an arrayEncoder, checking to make sure the value isn't nil. type sliceEncoder struct { arrayEnc encoderFunc } func (se *sliceEncoder) encode(e *encodeState, v reflect.Value, _ bool) { if v.IsNil() { e.WriteString("null") return } se.arrayEnc(e, v, false) } func newSliceEncoder(t reflect.Type) encoderFunc { // Byte slices get special treatment; arrays don't. if t.Elem().Kind() == reflect.Uint8 { return encodeByteSlice } enc := &sliceEncoder{newArrayEncoder(t)} return enc.encode } type arrayEncoder struct { elemEnc encoderFunc } func (ae *arrayEncoder) encode(e *encodeState, v reflect.Value, _ bool) { e.WriteByte('[') n := v.Len() for i := 0; i < n; i++ { if i > 0 { e.WriteByte(',') } ae.elemEnc(e, v.Index(i), false) } e.WriteByte(']') } func newArrayEncoder(t reflect.Type) encoderFunc { enc := &arrayEncoder{typeEncoder(t.Elem())} return enc.encode } type ptrEncoder struct { elemEnc encoderFunc } func (pe *ptrEncoder) encode(e *encodeState, v reflect.Value, quoted bool) { if v.IsNil() { e.WriteString("null") return } pe.elemEnc(e, v.Elem(), quoted) } func newPtrEncoder(t reflect.Type) encoderFunc { enc := &ptrEncoder{typeEncoder(t.Elem())} return enc.encode } type condAddrEncoder struct { canAddrEnc, elseEnc encoderFunc } func (ce *condAddrEncoder) encode(e *encodeState, v reflect.Value, quoted bool) { if v.CanAddr() { ce.canAddrEnc(e, v, quoted) } else { ce.elseEnc(e, v, quoted) } } // newCondAddrEncoder returns an encoder that checks whether its value // CanAddr and delegates to canAddrEnc if so, else to elseEnc. func newCondAddrEncoder(canAddrEnc, elseEnc encoderFunc) encoderFunc { enc := &condAddrEncoder{canAddrEnc: canAddrEnc, elseEnc: elseEnc} return enc.encode } func isValidTag(s string) bool { if s == "" { return false } for _, c := range s { switch { case strings.ContainsRune("!#$%&()*+-./:<=>?@[]^_{|}~ ", c): // Backslash and quote chars are reserved, but // otherwise any punctuation chars are allowed // in a tag name. default: if !unicode.IsLetter(c) && !unicode.IsDigit(c) { return false } } } return true } func fieldByIndex(v reflect.Value, index []int) reflect.Value { for _, i := range index { if v.Kind() == reflect.Ptr { if v.IsNil() { return reflect.Value{} } v = v.Elem() } v = v.Field(i) } return v } func typeByIndex(t reflect.Type, index []int) reflect.Type { for _, i := range index { if t.Kind() == reflect.Ptr { t = t.Elem() } t = t.Field(i).Type } return t } // stringValues is a slice of reflect.Value holding *reflect.StringValue. // It implements the methods to sort by string. type stringValues []reflect.Value func (sv stringValues) Len() int { return len(sv) } func (sv stringValues) Swap(i, j int) { sv[i], sv[j] = sv[j], sv[i] } func (sv stringValues) Less(i, j int) bool { return sv.get(i) < sv.get(j) } func (sv stringValues) get(i int) string { return sv[i].String() } func (e *encodeState) string(s string) int { return e.stringBytes([]byte(s)) } func (e *encodeState) stringBytes(s []byte) int { len0 := e.Len() e.WriteByte('"') start := 0 rejectLowSurrogateAt := -1 for i := 0; i < len(s); { if b := s[i]; b < utf8.RuneSelf { if 0x20 <= b && b != '\\' && b != '"' { i++ continue } if start < i { e.Write(s[start:i]) } e.WriteByte('\\') switch b { case '\\', '"': e.WriteByte(b) case '\n': e.WriteByte('n') case '\r': e.WriteByte('r') case '\t': e.WriteByte('t') case '\x08': // \b e.WriteByte('b') case '\x0c': // \f e.WriteByte('f') default: // This encodes other bytes < 0x20 as \u00xx. e.WriteString("u00") e.WriteByte(hex[b>>4]) e.WriteByte(hex[b&0xF]) } i++ start = i continue } c, size := utf8.DecodeRune(s[i:]) if c == utf8.RuneError && size == 1 { if start < i { e.Write(s[start:i]) } // Check for a "WTF-8" lone surrogate (and escape it). // High surrogate (U+D800 through U+DBFF): 1110 1101 10 10bbbb 10 bbbbbb // Low surrogate (U+DC00 through U+DFFF): 1110 1101 10 11bbbb 10 bbbbbb if s[i] == 0xED && i+2 < len(s) { c2 := s[i+1] c3 := s[i+2] if c2 >= 0xA0 && c2 <= 0xBF && (c3&0xC0) == 0x80 { // Don't let this special-case logic sneak through a valid surrogate pair. if isHigh := (c2 & 0x10) == 0; isHigh || i != rejectLowSurrogateAt { if isHigh { rejectLowSurrogateAt = i + 3 } e.WriteString(`\u`) e.WriteByte(hex[0xD]) e.WriteByte(hex[(c2>>2)&0x0F]) e.WriteByte(hex[((c2<<2)&0x0C)|((c3>>4)&0x03)]) e.WriteByte(hex[c3&0x0F]) i += 3 start = i continue } } } e.error(&UnsupportedValueError{reflect.ValueOf(s), fmt.Sprintf("%q", string(s))}) } i += size } if start < len(s) { e.Write(s[start:]) } e.WriteByte('"') return e.Len() - len0 } // A field represents a single field found in a struct. type field struct { name string nameBytes []byte // []byte(name) equalFold func(s, t []byte) bool // bytes.EqualFold or equivalent tag bool index []int typ reflect.Type omitEmpty bool quoted bool } func fillField(f field) field { f.nameBytes = []byte(f.name) f.equalFold = foldFunc(f.nameBytes) return f } // byName sorts field by name, breaking ties with depth, // then breaking ties with "name came from json tag", then // breaking ties with index sequence. type byName []field func (x byName) Len() int { return len(x) } func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } func (x byName) Less(i, j int) bool { if x[i].name != x[j].name { return x[i].name < x[j].name } if len(x[i].index) != len(x[j].index) { return len(x[i].index) < len(x[j].index) } if x[i].tag != x[j].tag { return x[i].tag } return byIndex(x).Less(i, j) } // byIndex sorts field by index sequence. type byIndex []field func (x byIndex) Len() int { return len(x) } func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } func (x byIndex) Less(i, j int) bool { for k, xik := range x[i].index { if k >= len(x[j].index) { return false } if xik != x[j].index[k] { return xik < x[j].index[k] } } return len(x[i].index) < len(x[j].index) } // typeFields returns a list of fields that JSON should recognize for the given type. // The algorithm is breadth-first search over the set of structs to include - the top struct // and then any reachable anonymous structs. func typeFields(t reflect.Type) []field { // Anonymous fields to explore at the current level and the next. current := []field{} next := []field{{typ: t}} // Count of queued names for current level and the next. count := map[reflect.Type]int{} nextCount := map[reflect.Type]int{} // Types already visited at an earlier level. visited := map[reflect.Type]bool{} // Fields found. var fields []field for len(next) > 0 { current, next = next, current[:0] count, nextCount = nextCount, map[reflect.Type]int{} for _, f := range current { if visited[f.typ] { continue } visited[f.typ] = true // Scan f.typ for fields to include. for i := 0; i < f.typ.NumField(); i++ { sf := f.typ.Field(i) if sf.PkgPath != "" && !sf.Anonymous { // unexported continue } tag := sf.Tag.Get("json") if tag == "-" { continue } name, opts := parseTag(tag) if !isValidTag(name) { name = "" } index := make([]int, len(f.index)+1) copy(index, f.index) index[len(f.index)] = i ft := sf.Type if ft.Name() == "" && ft.Kind() == reflect.Ptr { // Follow pointer. ft = ft.Elem() } // Only strings, floats, integers, and booleans can be quoted. quoted := false if opts.Contains("string") { switch ft.Kind() { case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64, reflect.String: quoted = true } } // Record found field and index sequence. if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { tagged := name != "" if name == "" { name = sf.Name } fields = append(fields, fillField(field{ name: name, tag: tagged, index: index, typ: ft, omitEmpty: opts.Contains("omitempty"), quoted: quoted, })) if count[f.typ] > 1 { // If there were multiple instances, add a second, // so that the annihilation code will see a duplicate. // It only cares about the distinction between 1 or 2, // so don't bother generating any more copies. fields = append(fields, fields[len(fields)-1]) } continue } // Record new anonymous struct to explore in next round. nextCount[ft]++ if nextCount[ft] == 1 { next = append(next, fillField(field{name: ft.Name(), index: index, typ: ft})) } } } } sort.Sort(byName(fields)) // Delete all fields that are hidden by the Go rules for embedded fields, // except that fields with JSON tags are promoted. // The fields are sorted in primary order of name, secondary order // of field index length. Loop over names; for each name, delete // hidden fields by choosing the one dominant field that survives. out := fields[:0] for advance, i := 0, 0; i < len(fields); i += advance { // One iteration per name. // Find the sequence of fields with the name of this first field. fi := fields[i] name := fi.name for advance = 1; i+advance < len(fields); advance++ { fj := fields[i+advance] if fj.name != name { break } } if advance == 1 { // Only one field with this name out = append(out, fi) continue } dominant, ok := dominantField(fields[i : i+advance]) if ok { out = append(out, dominant) } } return out } // dominantField looks through the fields, all of which are known to // have the same name, to find the single field that dominates the // others using Go's embedding rules, modified by the presence of // JSON tags. If there are multiple top-level fields, the boolean // will be false: This condition is an error in Go and we skip all // the fields. func dominantField(fields []field) (field, bool) { // The fields are sorted in increasing index-length order. The winner // must therefore be one with the shortest index length. Drop all // longer entries, which is easy: just truncate the slice. length := len(fields[0].index) tagged := -1 // Index of first tagged field. for i, f := range fields { if len(f.index) > length { fields = fields[:i] break } if f.tag { if tagged >= 0 { // Multiple tagged fields at the same level: conflict. // Return no field. return field{}, false } tagged = i } } if tagged >= 0 { return fields[tagged], true } // All remaining fields have the same length. If there's more than one, // we have a conflict (two fields named "X" at the same level) and we // return no field. if len(fields) > 1 { return field{}, false } return fields[0], true } var fieldCache struct { sync.RWMutex m map[reflect.Type][]field } // cachedTypeFields is like typeFields but uses a cache to avoid repeated work. func cachedTypeFields(t reflect.Type) []field { fieldCache.RLock() f := fieldCache.m[t] fieldCache.RUnlock() if f != nil { return f } // Compute fields without lock. // Might duplicate effort but won't hold other computations back. f = typeFields(t) if f == nil { f = []field{} } fieldCache.Lock() if fieldCache.m == nil { fieldCache.m = map[reflect.Type][]field{} } fieldCache.m[t] = f fieldCache.Unlock() return f } canonicaljson-go-1.0.3/encode_test.go000066400000000000000000000341321317167134000175700ustar00rootroot00000000000000// Copyright 2016 Richard Gibson. All rights reserved. // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package canonicaljson import ( "bytes" "encoding/json" "fmt" "math" "reflect" "strconv" "testing" "unicode" ) type Optionals struct { Sr string `json:"sr"` So string `json:"so,omitempty"` Sw string `json:"-"` Ir int `json:"omitempty"` // actually named omitempty, not an option Io int `json:"io,omitempty"` Slr []string `json:"slr,random"` Slo []string `json:"slo,omitempty"` Mr map[string]interface{} `json:"mr"` Mo map[string]interface{} `json:",omitempty"` Fr float64 `json:"fr"` Fo float64 `json:"fo,omitempty"` Br bool `json:"br"` Bo bool `json:"bo,omitempty"` Ur uint `json:"ur"` Uo uint `json:"uo,omitempty"` Str struct{} `json:"str"` Sto struct{} `json:"sto,omitempty"` } var optionalsExpected = `{ "br": false, "fr": 0, "mr": {}, "omitempty": 0, "slr": null, "sr": "", "sto": {}, "str": {}, "ur": 0 }` func TestOmitEmpty(t *testing.T) { var o Optionals o.Sw = "something" o.Mr = map[string]interface{}{} o.Mo = map[string]interface{}{} got, err := MarshalIndent(&o, "", " ") if err != nil { t.Fatal(err) } if got := string(got); got != optionalsExpected { t.Errorf(" got: %s\nwant: %s\n", got, optionalsExpected) } } type StringTag struct { BoolStr bool `json:",string"` IntStr int64 `json:",string"` StrStr string `json:",string"` } var stringTagExpected = `{ "BoolStr": "true", "IntStr": "42", "StrStr": "\"xzbit\"" }` func TestStringTag(t *testing.T) { var s StringTag s.BoolStr = true s.IntStr = 42 s.StrStr = "xzbit" got, err := MarshalIndent(&s, "", " ") if err != nil { t.Fatal(err) } if got := string(got); got != stringTagExpected { t.Fatalf(" got: %s\nwant: %s\n", got, stringTagExpected) } // Verify that it round-trips with the standard package. var s2 StringTag err = json.NewDecoder(bytes.NewReader(got)).Decode(&s2) if err != nil { t.Fatalf("Decode: %v", err) } if !reflect.DeepEqual(s, s2) { t.Fatalf("decode didn't match.\nsource: %#v\nEncoded as:\n%s\ndecode: %#v", s, string(got), s2) } } // byte slices are special even if they're renamed types. type renamedByte byte type renamedByteSlice []byte type renamedRenamedByteSlice []renamedByte func TestEncodeRenamedByteSlice(t *testing.T) { s := renamedByteSlice("abc") result, err := Marshal(s) if err != nil { t.Fatal(err) } expect := `"YWJj"` if string(result) != expect { t.Errorf(" got %s want %s", result, expect) } r := renamedRenamedByteSlice("abc") result, err = Marshal(r) if err != nil { t.Fatal(err) } if string(result) != expect { t.Errorf(" got %s want %s", result, expect) } } var unsupportedValues = []interface{}{ math.NaN(), math.Inf(-1), math.Inf(1), // Ill-formed UTF-8 "\x80", "\xcf", "\xed", "\xed\xa0", "\xed\xa0\x7e", "\xed\xa0\xc0", "\xf8\x82\x83\x84", "\xed\xa0\x80\xed\xbf\xbf", "\xed\xa0\x80\xed", "hello\xffworld", "\xff", "\xff\xff", "a\xffb", "\xe6\x97\xa5\xe6\x9c\xac\xff\xaa\x9e", } func TestUnsupportedValues(t *testing.T) { for i, v := range unsupportedValues { label := fmt.Sprintf("#%d [%#v]", i, v) if _, err := Marshal(v); err != nil { if _, ok := err.(*UnsupportedValueError); !ok { t.Errorf("%s: got %T, want UnsupportedValueError", label, err) } } else { t.Errorf("%s: expected error", label) } } } // Ref has Marshaler and Unmarshaler methods with pointer receiver. type Ref int func (*Ref) MarshalJSON() ([]byte, error) { return []byte(`"ref"`), nil } func (r *Ref) UnmarshalJSON([]byte) error { *r = 12 return nil } // Val has Marshaler methods with value receiver. type Val int func (Val) MarshalJSON() ([]byte, error) { return []byte(`"val"`), nil } // RefText has Marshaler and Unmarshaler methods with pointer receiver. type RefText int func (*RefText) MarshalText() ([]byte, error) { return []byte(`"ref"`), nil } func (r *RefText) UnmarshalText([]byte) error { *r = 13 return nil } // ValText has Marshaler methods with value receiver. type ValText int func (ValText) MarshalText() ([]byte, error) { return []byte(`"val"`), nil } func TestRefValMarshal(t *testing.T) { var s = struct { R0 Ref R1 *Ref R2 RefText R3 *RefText V0 Val V1 *Val V2 ValText V3 *ValText }{ R0: 12, R1: new(Ref), R2: 14, R3: new(RefText), V0: 13, V1: new(Val), V2: 15, V3: new(ValText), } const want = `{"R0":"ref","R1":"ref","R2":"\"ref\"","R3":"\"ref\"","V0":"val","V1":"val","V2":"\"val\"","V3":"\"val\""}` b, err := Marshal(&s) if err != nil { t.Fatalf("Marshal: %v", err) } if got := string(b); got != want { t.Errorf("got %q, want %q", got, want) } } // C implements Marshaler and returns unescaped JSON. type C int func (C) MarshalJSON() ([]byte, error) { return []byte(`"<&>"`), nil } // CText implements Marshaler and returns unescaped text. type CText int func (CText) MarshalText() ([]byte, error) { return []byte(`"<&>"`), nil } func TestMarshalerEscaping(t *testing.T) { var c C want := `"<&>"` b, err := Marshal(c) if err != nil { t.Fatalf("Marshal(c): %v", err) } if got := string(b); got != want { t.Errorf("Marshal(c) = %#q, want %#q", got, want) } var ct CText want = `"\"<&>\""` b, err = Marshal(ct) if err != nil { t.Fatalf("Marshal(ct): %v", err) } if got := string(b); got != want { t.Errorf("Marshal(ct) = %#q, want %#q", got, want) } } type IntType int type MyStruct struct { IntType } func TestAnonymousNonstruct(t *testing.T) { var i IntType = 11 a := MyStruct{i} const want = `{"IntType":11}` b, err := Marshal(a) if err != nil { t.Fatalf("Marshal: %v", err) } if got := string(b); got != want { t.Errorf("got %q, want %q", got, want) } } type BugA struct { S string } type BugB struct { BugA S string } type BugC struct { S string } // Legal Go: We never use the repeated embedded field (S). type BugX struct { A int BugA BugB } // Issue 5245. func TestEmbeddedBug(t *testing.T) { v := BugB{ BugA{"A"}, "B", } b, err := Marshal(v) if err != nil { t.Fatal("Marshal:", err) } want := `{"S":"B"}` got := string(b) if got != want { t.Fatalf("Marshal: got %s want %s", got, want) } // Now check that the duplicate field, S, does not appear. x := BugX{ A: 23, } b, err = Marshal(x) if err != nil { t.Fatal("Marshal:", err) } want = `{"A":23}` got = string(b) if got != want { t.Fatalf("Marshal: got %s want %s", got, want) } } type BugD struct { // Same as BugA after tagging. XXX string `json:"S"` } // BugD's tagged S field should dominate BugA's. type BugY struct { BugA BugD } // Test that a field with a tag dominates untagged fields. func TestTaggedFieldDominates(t *testing.T) { v := BugY{ BugA{"BugA"}, BugD{"BugD"}, } b, err := Marshal(v) if err != nil { t.Fatal("Marshal:", err) } want := `{"S":"BugD"}` got := string(b) if got != want { t.Fatalf("Marshal: got %s want %s", got, want) } } // There are no tags here, so S should not appear. type BugZ struct { BugA BugC BugY // Contains a tagged S field through BugD; should not dominate. } func TestDuplicatedFieldDisappears(t *testing.T) { v := BugZ{ BugA{"BugA"}, BugC{"BugC"}, BugY{ BugA{"nested BugA"}, BugD{"nested BugD"}, }, } b, err := Marshal(v) if err != nil { t.Fatal("Marshal:", err) } want := `{}` got := string(b) if got != want { t.Fatalf("Marshal: got %s want %s", got, want) } } func TestStringBytes(t *testing.T) { // Test that encodeState.stringBytes and encodeState.string use the same encoding. es := &encodeState{} var r []rune for i := '\u0000'; i <= unicode.MaxRune; i++ { r = append(r, i) } // Include some valid non-UTF-8 sequences. s := string(r) + "\xed\xbf\xbf\xed\xa0\x80" es.string(s) esBytes := &encodeState{} esBytes.stringBytes([]byte(s)) enc := es.Buffer.String() encBytes := esBytes.Buffer.String() if enc != encBytes { i := 0 for i < len(enc) && i < len(encBytes) && enc[i] == encBytes[i] { i++ } enc = enc[i:] encBytes = encBytes[i:] i = 0 for i < len(enc) && i < len(encBytes) && enc[len(enc)-i-1] == encBytes[len(encBytes)-i-1] { i++ } enc = enc[:len(enc)-i] encBytes = encBytes[:len(encBytes)-i] if len(enc) > 20 { enc = enc[:20] + "..." } if len(encBytes) > 20 { encBytes = encBytes[:20] + "..." } t.Errorf("encodings differ at %#q vs %#q", enc, encBytes) } } func TestIssue6458(t *testing.T) { type Foo struct { M RawMessage } x := Foo{RawMessage(`"foo"`)} b, err := Marshal(&x) if err != nil { t.Fatal(err) } if want := `{"M":"foo"}`; string(b) != want { t.Errorf("Marshal(&x) = %#q; want %#q", b, want) } b, err = Marshal(x) if err != nil { t.Fatal(err) } if want := `{"M":"ImZvbyI="}`; string(b) != want { t.Errorf("Marshal(x) = %#q; want %#q", b, want) } } func TestIssue10281(t *testing.T) { type Foo struct { N json.Number } x := Foo{json.Number(`invalid`)} b, err := Marshal(&x) if err == nil { t.Errorf("Marshal(&x) = %#q; want error", b) } } // golang.org/issue/8582 func TestEncodePointerString(t *testing.T) { type stringPointer struct { N *int64 `json:"n,string"` } var n int64 = 42 b, err := Marshal(stringPointer{N: &n}) if err != nil { t.Fatalf("Marshal: %v", err) } if got, want := string(b), `{"n":"42"}`; got != want { t.Errorf("Marshal = %s, want %s", got, want) } var back stringPointer err = json.Unmarshal(b, &back) if err != nil { t.Fatalf("Unmarshal: %v", err) } if back.N == nil { t.Fatalf("Unmarshalled nil N field") } if *back.N != 42 { t.Fatalf("*N = %d; want 42", *back.N) } } var encodeStringTests = []struct { in string out string }{ {"", `""`}, {"\x00", `"\u0000"`}, {"\x01", `"\u0001"`}, {"\x02", `"\u0002"`}, {"\x03", `"\u0003"`}, {"\x04", `"\u0004"`}, {"\x05", `"\u0005"`}, {"\x06", `"\u0006"`}, {"\x07", `"\u0007"`}, {"\x08", `"\b"`}, {"\x09", `"\t"`}, {"\x0a", `"\n"`}, {"\x0b", `"\u000B"`}, {"\x0c", `"\f"`}, {"\x0d", `"\r"`}, {"\x0e", `"\u000E"`}, {"\x0f", `"\u000F"`}, {"\x10", `"\u0010"`}, {"\x11", `"\u0011"`}, {"\x12", `"\u0012"`}, {"\x13", `"\u0013"`}, {"\x14", `"\u0014"`}, {"\x15", `"\u0015"`}, {"\x16", `"\u0016"`}, {"\x17", `"\u0017"`}, {"\x18", `"\u0018"`}, {"\x19", `"\u0019"`}, {"\x1a", `"\u001A"`}, {"\x1b", `"\u001B"`}, {"\x1c", `"\u001C"`}, {"\x1d", `"\u001D"`}, {"\x1e", `"\u001E"`}, {"\x1f", `"\u001F"`}, {"\x7f", "\"\x7f\""}, {"\xe6\x97\xa5\xe6\x9c\xac", `"日本"`}, {"\xed\xa0\x80", `"\uD800"`}, {"\xed\xa0\x80a", `"\uD800a"`}, {"\xed\xbf\xbf", `"\uDFFF"`}, {"\xed\xbf\xbfa", `"\uDFFFa"`}, {"\xed\xbf\xbf\xed\xa0\x80", `"\uDFFF\uD800"`}, {"\xed\xbf\xbf\xed\xa0\x80a", `"\uDFFF\uD800a"`}, } func TestEncodeString(t *testing.T) { for _, tt := range encodeStringTests { b, err := Marshal(tt.in) if err != nil { t.Errorf("Marshal(%q): %v", tt.in, err) continue } out := string(b) if out != tt.out { t.Errorf("Marshal(%q) = %#q, want %#q", tt.in, out, tt.out) } } } var named = map[string]interface{}{ "\x08": nil, "\x09": nil, "\x0a": nil, "\x0c": nil, "\x0d": nil, "\x00": nil, "\x01": nil, "\x02": nil, "\x03": nil, "\x04": nil, "\x05": nil, "\x06": nil, "\x07": nil, "\x0b": nil, "\x0e": nil, "\x0f": nil, "\x10": nil, "\x11": nil, "\x12": nil, "\x13": nil, "\x14": nil, "\x15": nil, "\x16": nil, "\x17": nil, "\x18": nil, "\x19": nil, "\x1a": nil, "\x1b": nil, "\x1c": nil, "\x1d": nil, "\x1e": nil, "\x1f": nil, "\x7f": nil, } var namedExpected = `{ "\u0000": null, "\u0001": null, "\u0002": null, "\u0003": null, "\u0004": null, "\u0005": null, "\u0006": null, "\u0007": null, "\b": null, "\t": null, "\n": null, "\u000B": null, "\f": null, "\r": null, "\u000E": null, "\u000F": null, "\u0010": null, "\u0011": null, "\u0012": null, "\u0013": null, "\u0014": null, "\u0015": null, "\u0016": null, "\u0017": null, "\u0018": null, "\u0019": null, "\u001A": null, "\u001B": null, "\u001C": null, "\u001D": null, "\u001E": null, "\u001F": null, "` + "\x7f" + `": null }` func TestEncodeKey(t *testing.T) { result, err := MarshalIndent(named, "", " ") if err != nil { t.Fatal(err) } if result := string(result); result != namedExpected { t.Errorf(" got: %s\nwant: %s\n", result, namedExpected) } } var floats = map[string][]string{ "2.5E-3": []string{ "0.025e-1", "0.0250e-1", "0.02500e-1", "0.25e-2", "0.250e-2", "0.2500e-2", }, "2.5E-2": []string{ "0.025e0", "0.0250e0", "0.02500e0", "0.025", "0.0250", "0.02500", "0.25e-1", "0.250e-1", "0.2500e-1", "2.5e-2", "2.50e-2", "2.500e-2", }, "2.5E-1": []string{ "0.025e1", "0.0250e1", "0.02500e1", "0.25e0", "0.250e0", "0.2500e0", "0.25", "0.250", "0.2500", "2.5e-1", "2.50e-1", "2.500e-1", "25e-2", "25.0e-2", "25.00e-2", }, "2.5E0": []string{ "0.025e2", "0.0250e2", "0.02500e2", "0.25e1", "0.250e1", "0.2500e1", "2.5e0", "2.50e0", "2.500e0", "2.5", "2.50", "2.500", "25e-1", "25.0e-1", "25.00e-1", "250e-2", "250.0e-2", "250.00e-2", }, "25": []string{ "0.25e2", "0.250e2", "0.2500e2", "2.5e1", "2.50e1", "2.500e1", "25e0", "25.0e0", "25.00e0", "25", "25.0", "25.00", "250e-1", "250.0e-1", "250.00e-1", }, "250": []string{ "2.5e2", "2.50e2", "2.500e2", "25e1", "25.0e1", "25.00e1", "250e0", "250.0e0", "250.00e0", "250", "250.0", "250.00", }, "2500": []string{ "25e2", "25.0e2", "25.00e2", "250e1", "250.0e1", "250.00e1", }, } func TestFloat(t *testing.T) { for expected, inputs := range floats { for _, input := range inputs { inputFloat, _ := strconv.ParseFloat(input, 64) result, err := Marshal(inputFloat) if err != nil { t.Fatal(err) } if string(result) != expected { t.Errorf(" float(%s)\n got: %s\n want: %s\n", input, result, expected) } inputNumber := Number(input) result, err = Marshal(inputNumber) if err != nil { t.Fatal(err) } if string(result) != expected { t.Errorf(" Number(%s)\n got: %s\n want: %s\n", input, result, expected) } inputJsonNumber := json.Number(input) result, err = Marshal(inputJsonNumber) if err != nil { t.Fatal(err) } if string(result) != expected { t.Errorf(" json.Number(%s)\n got: %s\n want: %s\n", input, result, expected) } } } } canonicaljson-go-1.0.3/fold.go000066400000000000000000000067171317167134000162300ustar00rootroot00000000000000// Copyright 2016 Richard Gibson. All rights reserved. // Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package canonicaljson import ( "bytes" "unicode/utf8" ) const ( caseMask = ^byte(0x20) // Mask to ignore case in ASCII. kelvin = '\u212A' smallLongEss = '\u017F' ) // foldFunc returns one of four different case folding equivalence // functions, from most general (and slow) to fastest: // // 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8 // 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S') // 3) asciiEqualFold, no special, but includes non-letters (including _) // 4) simpleLetterEqualFold, no specials, no non-letters. // // The letters S and K are special because they map to 3 runes, not just 2: // * S maps to s and to U+017F 'ſ' Latin small letter long s // * k maps to K and to U+212A 'K' Kelvin sign // See https://play.golang.org/p/tTxjOc0OGo // // The returned function is specialized for matching against s and // should only be given s. It's not curried for performance reasons. func foldFunc(s []byte) func(s, t []byte) bool { nonLetter := false special := false // special letter for _, b := range s { if b >= utf8.RuneSelf { return bytes.EqualFold } upper := b & caseMask if upper < 'A' || upper > 'Z' { nonLetter = true } else if upper == 'K' || upper == 'S' { // See above for why these letters are special. special = true } } if special { return equalFoldRight } if nonLetter { return asciiEqualFold } return simpleLetterEqualFold } // equalFoldRight is a specialization of bytes.EqualFold when s is // known to be all ASCII (including punctuation), but contains an 's', // 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t. // See comments on foldFunc. func equalFoldRight(s, t []byte) bool { for _, sb := range s { if len(t) == 0 { return false } tb := t[0] if tb < utf8.RuneSelf { if sb != tb { sbUpper := sb & caseMask if 'A' <= sbUpper && sbUpper <= 'Z' { if sbUpper != tb&caseMask { return false } } else { return false } } t = t[1:] continue } // sb is ASCII and t is not. t must be either kelvin // sign or long s; sb must be s, S, k, or K. tr, size := utf8.DecodeRune(t) switch sb { case 's', 'S': if tr != smallLongEss { return false } case 'k', 'K': if tr != kelvin { return false } default: return false } t = t[size:] } if len(t) > 0 { return false } return true } // asciiEqualFold is a specialization of bytes.EqualFold for use when // s is all ASCII (but may contain non-letters) and contains no // special-folding letters. // See comments on foldFunc. func asciiEqualFold(s, t []byte) bool { if len(s) != len(t) { return false } for i, sb := range s { tb := t[i] if sb == tb { continue } if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') { if sb&caseMask != tb&caseMask { return false } } else { return false } } return true } // simpleLetterEqualFold is a specialization of bytes.EqualFold for // use when s is all ASCII letters (no underscores, etc) and also // doesn't contain 'k', 'K', 's', or 'S'. // See comments on foldFunc. func simpleLetterEqualFold(s, t []byte) bool { if len(s) != len(t) { return false } for i, b := range s { if b&caseMask != t[i]&caseMask { return false } } return true } canonicaljson-go-1.0.3/fold_test.go000066400000000000000000000057201317167134000172600ustar00rootroot00000000000000// Copyright 2016 Richard Gibson. All rights reserved. // Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package canonicaljson import ( "bytes" "strings" "testing" "unicode/utf8" ) var foldTests = []struct { fn func(s, t []byte) bool s, t string want bool }{ {equalFoldRight, "", "", true}, {equalFoldRight, "a", "a", true}, {equalFoldRight, "", "a", false}, {equalFoldRight, "a", "", false}, {equalFoldRight, "a", "A", true}, {equalFoldRight, "AB", "ab", true}, {equalFoldRight, "AB", "ac", false}, {equalFoldRight, "sbkKc", "ſbKKc", true}, {equalFoldRight, "SbKkc", "ſbKKc", true}, {equalFoldRight, "SbKkc", "ſbKK", false}, {equalFoldRight, "e", "é", false}, {equalFoldRight, "s", "S", true}, {simpleLetterEqualFold, "", "", true}, {simpleLetterEqualFold, "abc", "abc", true}, {simpleLetterEqualFold, "abc", "ABC", true}, {simpleLetterEqualFold, "abc", "ABCD", false}, {simpleLetterEqualFold, "abc", "xxx", false}, {asciiEqualFold, "a_B", "A_b", true}, {asciiEqualFold, "aa@", "aa`", false}, // verify 0x40 and 0x60 aren't case-equivalent } func TestFold(t *testing.T) { for i, tt := range foldTests { if got := tt.fn([]byte(tt.s), []byte(tt.t)); got != tt.want { t.Errorf("%d. %q, %q = %v; want %v", i, tt.s, tt.t, got, tt.want) } truth := strings.EqualFold(tt.s, tt.t) if truth != tt.want { t.Errorf("strings.EqualFold doesn't agree with case %d", i) } } } func TestFoldAgainstUnicode(t *testing.T) { const bufSize = 5 buf1 := make([]byte, 0, bufSize) buf2 := make([]byte, 0, bufSize) var runes []rune for i := 0x20; i <= 0x7f; i++ { runes = append(runes, rune(i)) } runes = append(runes, kelvin, smallLongEss) funcs := []struct { name string fold func(s, t []byte) bool letter bool // must be ASCII letter simple bool // must be simple ASCII letter (not 'S' or 'K') }{ { name: "equalFoldRight", fold: equalFoldRight, }, { name: "asciiEqualFold", fold: asciiEqualFold, simple: true, }, { name: "simpleLetterEqualFold", fold: simpleLetterEqualFold, simple: true, letter: true, }, } for _, ff := range funcs { for _, r := range runes { if r >= utf8.RuneSelf { continue } if ff.letter && !isASCIILetter(byte(r)) { continue } if ff.simple && (r == 's' || r == 'S' || r == 'k' || r == 'K') { continue } for _, r2 := range runes { buf1 := append(buf1[:0], 'x') buf2 := append(buf2[:0], 'x') buf1 = buf1[:1+utf8.EncodeRune(buf1[1:bufSize], r)] buf2 = buf2[:1+utf8.EncodeRune(buf2[1:bufSize], r2)] buf1 = append(buf1, 'x') buf2 = append(buf2, 'x') want := bytes.EqualFold(buf1, buf2) if got := ff.fold(buf1, buf2); got != want { t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want) } } } } } func isASCIILetter(b byte) bool { return ('A' <= b && b <= 'Z') || ('a' <= b && b <= 'z') } canonicaljson-go-1.0.3/indent.go000066400000000000000000000045321317167134000165560ustar00rootroot00000000000000// Copyright 2016 Richard Gibson. All rights reserved. // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package canonicaljson import "bytes" func newline(dst *bytes.Buffer, prefix, indent string, depth int) { dst.WriteByte('\n') dst.WriteString(prefix) for i := 0; i < depth; i++ { dst.WriteString(indent) } } // addIndentation appends to dst an indented form of the JSON-encoded src. // Each element in a JSON object or array begins on a new, // indented line beginning with prefix followed by one or more // copies of indent according to the indentation nesting. // The data appended to dst does not begin with the prefix nor // any indentation, to make it easier to embed inside other formatted JSON data. // Although leading space characters (space, tab, carriage return, newline) // at the beginning of src are dropped, trailing space characters // at the end of src are preserved and copied to dst. // For example, if src has no trailing spaces, neither will dst; // if src ends in a trailing newline, so will dst. func addIndentation(dst *bytes.Buffer, src []byte, prefix, indent string) error { origLen := dst.Len() var scan scanner scan.reset() needIndent := false depth := 0 for _, c := range src { scan.bytes++ v := scan.step(&scan, c) if v == scanSkipSpace { continue } if v == scanError { break } if needIndent && v != scanEndObject && v != scanEndArray { needIndent = false depth++ newline(dst, prefix, indent, depth) } // Emit semantically uninteresting bytes // (in particular, punctuation in strings) unmodified. if v == scanContinue { dst.WriteByte(c) continue } // Add spacing around real punctuation. switch c { case '{', '[': // delay indent so that empty object and array are formatted as {} and []. needIndent = true dst.WriteByte(c) case ',': dst.WriteByte(c) newline(dst, prefix, indent, depth) case ':': dst.WriteByte(c) dst.WriteByte(' ') case '}', ']': if needIndent { // suppress indent in empty object/array needIndent = false } else { depth-- newline(dst, prefix, indent, depth) } dst.WriteByte(c) default: dst.WriteByte(c) } } if scan.eof() == scanError { dst.Truncate(origLen) return scan.err } return nil } canonicaljson-go-1.0.3/scanner.go000066400000000000000000000500451317167134000167260ustar00rootroot00000000000000// Copyright 2016 Richard Gibson. All rights reserved. // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package canonicaljson // JSON value parser state machine. // Just about at the limit of what is reasonable to write by hand. // Some parts are a bit tedious, but overall it nicely factors out the // otherwise common code from the multiple scanning functions // in this package (checkValid, nextValue, etc). // // This file starts with two simple examples using the scanner // before diving into the scanner itself. import ( "fmt" "strconv" ) // checkValid verifies that data is valid JSON-encoded data. // scan is passed in for use by checkValid to avoid an allocation. func checkValid(data []byte, scan *scanner) error { scan.reset() for _, c := range data { scan.bytes++ if scan.step(scan, c) == scanError { return scan.err } } if scan.eof() == scanError { return scan.err } return nil } // nextValue splits data after the next whole JSON value, // returning that value and the bytes that follow it as separate slices. // scan is passed in for use by nextValue to avoid an allocation. func nextValue(data []byte, scan *scanner) (value, rest []byte, err error) { scan.reset() for i, c := range data { v := scan.step(scan, c) if v >= scanEndObject { switch v { // probe the scanner with a space to determine whether we will // get scanEnd on the next character. Otherwise, if the next character // is not a space, scanEndTop allocates a needless error. case scanEndObject, scanEndArray: if scan.step(scan, ' ') == scanEnd { return data[:i+1], data[i+1:], nil } case scanError: return nil, nil, scan.err case scanEnd: return data[:i], data[i:], nil } } } if scan.eof() == scanError { return nil, nil, scan.err } return data, nil, nil } // A SyntaxError is a description of a JSON syntax error. type SyntaxError struct { msg string // description of error Offset int64 // error occurred after reading Offset bytes } func (e *SyntaxError) Error() string { return e.msg } // A scanner is a JSON scanning state machine. // Callers call scan.reset() and then pass bytes in one at a time // by calling scan.step(&scan, c) for each byte. // The return value, referred to as an opcode, tells the // caller about significant parsing events like beginning // and ending literals, objects, and arrays, so that the // caller can follow along if it wishes. // The return value scanEnd indicates that a single top-level // JSON value has been completed, *before* the byte that // just got passed in. (The indication must be delayed in order // to recognize the end of numbers: is 123 a whole value or // the beginning of 12345e+6?). type scanner struct { // The step is a func to be called to execute the next transition. // Also tried using an integer constant and a single func // with a switch, but using the func directly was 10% faster // on a 64-bit Mac Mini, and it's nicer to read. step func(*scanner, byte) int // Reached end of top-level value. endTop bool // Stack of what we're in the middle of - array values, object keys, object values. parseState []int // Error that happened, if any. err error // 1-byte redo (see undo method) redo bool redoCode int redoState func(*scanner, byte) int // total bytes consumed, updated by decoder.Decode bytes int64 } // These values are returned by the state transition functions // assigned to scanner.state and the method scanner.eof. // They give details about the current state of the scan that // callers might be interested to know about. // It is okay to ignore the return value of any particular // call to scanner.state: if one call returns scanError, // every subsequent call will return scanError too. const ( // Continue. scanContinue = iota // uninteresting byte scanBeginLiteral // end implied by next result != scanContinue scanBeginObject // begin object scanObjectKey // just finished object key (string) scanObjectValue // just finished non-last object value scanEndObject // end object (implies scanObjectValue if possible) scanBeginArray // begin array scanArrayValue // just finished array value scanEndArray // end array (implies scanArrayValue if possible) scanSkipSpace // space byte; can skip; known to be last "continue" result // Stop. scanEnd // top-level value ended *before* this byte; known to be first "stop" result scanError // hit an error, scanner.err. ) // These values are stored in the parseState stack. // They give the current state of a composite value // being scanned. If the parser is inside a nested value // the parseState describes the nested state, outermost at entry 0. const ( parseObjectKey = iota // parsing object key (before colon) parseObjectValue // parsing object value (after colon) parseArrayValue // parsing array value ) // reset prepares the scanner for use. // It must be called before calling s.step. func (s *scanner) reset() { s.step = stateBeginValue s.parseState = s.parseState[0:0] s.err = nil s.redo = false s.endTop = false } // eof tells the scanner that the end of input has been reached. // It returns a scan status just as s.step does. func (s *scanner) eof() int { if s.err != nil { return scanError } if s.endTop { return scanEnd } s.step(s, ' ') if s.endTop { return scanEnd } if s.err == nil { s.err = &SyntaxError{"unexpected end of JSON input", s.bytes} } return scanError } // pushParseState pushes a new parse state p onto the parse stack. func (s *scanner) pushParseState(p int) { s.parseState = append(s.parseState, p) } // popParseState pops a parse state (already obtained) off the stack // and updates s.step accordingly. func (s *scanner) popParseState() { n := len(s.parseState) - 1 s.parseState = s.parseState[0:n] s.redo = false if n == 0 { s.step = stateEndTop s.endTop = true } else { s.step = stateEndValue } } func isSpace(c byte) bool { return c == ' ' || c == '\t' || c == '\r' || c == '\n' } // stateBeginValueOrEmpty is the state after reading `[`. func stateBeginValueOrEmpty(s *scanner, c byte) int { if c <= ' ' && isSpace(c) { return scanSkipSpace } if c == ']' { return stateEndValue(s, c) } return stateBeginValue(s, c) } // stateBeginValue is the state at the beginning of the input. func stateBeginValue(s *scanner, c byte) int { if c <= ' ' && isSpace(c) { return scanSkipSpace } switch c { case '{': s.step = stateBeginStringOrEmpty s.pushParseState(parseObjectKey) return scanBeginObject case '[': s.step = stateBeginValueOrEmpty s.pushParseState(parseArrayValue) return scanBeginArray case '"': s.step = stateInString return scanBeginLiteral case '-': s.step = stateNeg return scanBeginLiteral case '0': // beginning of 0.123 s.step = state0 return scanBeginLiteral case 't': // beginning of true s.step = stateT return scanBeginLiteral case 'f': // beginning of false s.step = stateF return scanBeginLiteral case 'n': // beginning of null s.step = stateN return scanBeginLiteral } if '1' <= c && c <= '9' { // beginning of 1234.5 s.step = state1 return scanBeginLiteral } return s.error(c, "looking for beginning of value") } // stateBeginStringOrEmpty is the state after reading `{`. func stateBeginStringOrEmpty(s *scanner, c byte) int { if c <= ' ' && isSpace(c) { return scanSkipSpace } if c == '}' { n := len(s.parseState) s.parseState[n-1] = parseObjectValue return stateEndValue(s, c) } return stateBeginString(s, c) } // stateBeginString is the state after reading `{"key": value,`. func stateBeginString(s *scanner, c byte) int { if c <= ' ' && isSpace(c) { return scanSkipSpace } if c == '"' { s.step = stateInString return scanBeginLiteral } return s.error(c, "looking for beginning of object key string") } // stateEndValue is the state after completing a value, // such as after reading `{}` or `true` or `["x"`. func stateEndValue(s *scanner, c byte) int { n := len(s.parseState) if n == 0 { // Completed top-level before the current byte. s.step = stateEndTop s.endTop = true return stateEndTop(s, c) } if c <= ' ' && isSpace(c) { s.step = stateEndValue return scanSkipSpace } ps := s.parseState[n-1] switch ps { case parseObjectKey: if c == ':' { s.parseState[n-1] = parseObjectValue s.step = stateBeginValue return scanObjectKey } return s.error(c, "after object key") case parseObjectValue: if c == ',' { s.parseState[n-1] = parseObjectKey s.step = stateBeginString return scanObjectValue } if c == '}' { s.popParseState() return scanEndObject } return s.error(c, "after object key:value pair") case parseArrayValue: if c == ',' { s.step = stateBeginValue return scanArrayValue } if c == ']' { s.popParseState() return scanEndArray } return s.error(c, "after array element") } return s.error(c, "") } // stateEndTop is the state after finishing the top-level value, // such as after reading `{}` or `[1,2,3]`. // Only space characters should be seen now. func stateEndTop(s *scanner, c byte) int { if c != ' ' && c != '\t' && c != '\r' && c != '\n' { // Complain about non-space byte on next call. s.error(c, "after top-level value") } return scanEnd } // stateInString is the state after reading `"`. func stateInString(s *scanner, c byte) int { if c == '"' { s.step = stateEndValue return scanContinue } if c == '\\' { s.step = stateInStringEsc return scanContinue } if c < 0x20 { return s.error(c, "in string literal") } else if c >= 0x80 { // Require a well-formed UTF-8 sequence. // See Table 3-7 in http://www.unicode.org/versions/Unicode10.0.0/ch03.pdf#G7404 if c >= 0xC2 && c <= 0xDF { s.step = stateInStringContinuation1 } else if c == 0xE0 { s.step = stateInStringContinuation2_E0 } else if c == 0xED { s.step = stateInStringContinuation2_ED } else if c >= 0xE1 && c <= 0xEF { s.step = stateInStringContinuation2 } else if c == 0xF0 { s.step = stateInStringContinuation3_F0 } else if c == 0xF4 { s.step = stateInStringContinuation3_F4 } else if c >= 0xF1 && c <= 0xF3 { s.step = stateInStringContinuation3 } else { return s.error(c, "in string literal (expecting UTF-8 leading byte)") } } return scanContinue } // stateInStringContinuation1 is the state after reading all but the // final byte of a UTF-8 multi-byte sequence in a quoted string. func stateInStringContinuation1(s *scanner, c byte) int { if c&0xC0 != 0x80 { return s.error(c, "in UTF-8 multi-byte sequence (expecting UTF-8 continuation byte)") } s.step = stateInString return scanContinue } // stateInStringContinuation2 is the state after reading all but the // final two bytes of a UTF-8 multi-byte sequence in a quoted string. func stateInStringContinuation2(s *scanner, c byte) int { if c&0xC0 != 0x80 { return s.error(c, "in UTF-8 multi-byte sequence (expecting UTF-8 continuation byte)") } s.step = stateInStringContinuation1 return scanContinue } // stateInStringContinuation2_E0 is the state after reading an 0xE0 byte // in a quoted string. func stateInStringContinuation2_E0(s *scanner, c byte) int { if c&0xC0 != 0x80 { return s.error(c, "in UTF-8 three-byte sequence (expecting UTF-8 continuation byte)") } else if c < 0xA0 { return s.error(c, "in UTF-8 three-byte sequence (expecting well-formed UTF-8 continuation byte)") } s.step = stateInStringContinuation1 return scanContinue } // stateInStringContinuation2_ED is the state after reading an 0xED byte // in a quoted string. func stateInStringContinuation2_ED(s *scanner, c byte) int { if c&0xC0 != 0x80 { return s.error(c, "in UTF-8 three-byte sequence (expecting UTF-8 continuation byte)") } else if c > 0x9F { return s.error(c, "in UTF-8 three-byte sequence (expecting well-formed UTF-8 continuation byte)") } s.step = stateInStringContinuation1 return scanContinue } // stateInStringContinuation3 is the state after reading all but the // final three bytes of a UTF-8 multi-byte sequence in a quoted string. func stateInStringContinuation3(s *scanner, c byte) int { if c&0xC0 != 0x80 { return s.error(c, "in UTF-8 multi-byte sequence (expecting UTF-8 continuation byte)") } s.step = stateInStringContinuation2 return scanContinue } // stateInStringContinuation3_F0 is the state after reading an 0xF0 byte // in a quoted string. func stateInStringContinuation3_F0(s *scanner, c byte) int { if c&0xC0 != 0x80 { return s.error(c, "in UTF-8 four-byte sequence (expecting UTF-8 continuation byte)") } else if c < 0x90 { return s.error(c, "in UTF-8 four-byte sequence (expecting well-formed UTF-8 continuation byte)") } s.step = stateInStringContinuation2 return scanContinue } // stateInStringContinuation3_F4 is the state after reading an 0xF4 byte // in a quoted string. func stateInStringContinuation3_F4(s *scanner, c byte) int { if c&0xC0 != 0x80 { return s.error(c, "in UTF-8 four-byte sequence (expecting UTF-8 continuation byte)") } else if c > 0x8F { return s.error(c, "in UTF-8 four-byte sequence (expecting well-formed UTF-8 continuation byte)") } s.step = stateInStringContinuation2 return scanContinue } // stateInStringEsc is the state after reading `"\` during a quoted string. func stateInStringEsc(s *scanner, c byte) int { switch c { case 'b', 'f', 'n', 'r', 't', '\\', '/', '"': s.step = stateInString return scanContinue case 'u': s.step = stateInStringEscU return scanContinue } return s.error(c, "in string escape code") } // stateInStringEscU is the state after reading `"\u` during a quoted string. func stateInStringEscU(s *scanner, c byte) int { if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { s.step = stateInStringEscU1 return scanContinue } // numbers return s.error(c, "in \\u hexadecimal character escape") } // stateInStringEscU1 is the state after reading `"\u1` during a quoted string. func stateInStringEscU1(s *scanner, c byte) int { if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { s.step = stateInStringEscU12 return scanContinue } // numbers return s.error(c, "in \\u hexadecimal character escape") } // stateInStringEscU12 is the state after reading `"\u12` during a quoted string. func stateInStringEscU12(s *scanner, c byte) int { if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { s.step = stateInStringEscU123 return scanContinue } // numbers return s.error(c, "in \\u hexadecimal character escape") } // stateInStringEscU123 is the state after reading `"\u123` during a quoted string. func stateInStringEscU123(s *scanner, c byte) int { if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { s.step = stateInString return scanContinue } // numbers return s.error(c, "in \\u hexadecimal character escape") } // stateNeg is the state after reading `-` during a number. func stateNeg(s *scanner, c byte) int { if c == '0' { s.step = state0 return scanContinue } if '1' <= c && c <= '9' { s.step = state1 return scanContinue } return s.error(c, "in numeric literal") } // state1 is the state after reading a non-zero integer during a number, // such as after reading `1` or `100` but not `0`. func state1(s *scanner, c byte) int { if '0' <= c && c <= '9' { s.step = state1 return scanContinue } return state0(s, c) } // state0 is the state after reading `0` during a number. func state0(s *scanner, c byte) int { if c == '.' { s.step = stateDot return scanContinue } if c == 'e' || c == 'E' { s.step = stateE return scanContinue } return stateEndValue(s, c) } // stateDot is the state after reading the integer and decimal point in a number, // such as after reading `1.`. func stateDot(s *scanner, c byte) int { if '0' <= c && c <= '9' { s.step = stateDot0 return scanContinue } return s.error(c, "after decimal point in numeric literal") } // stateDot0 is the state after reading the integer, decimal point, and subsequent // digits of a number, such as after reading `3.14`. func stateDot0(s *scanner, c byte) int { if '0' <= c && c <= '9' { return scanContinue } if c == 'e' || c == 'E' { s.step = stateE return scanContinue } return stateEndValue(s, c) } // stateE is the state after reading the significand and e in a number, // such as after reading `314e` or `0.314e`. func stateE(s *scanner, c byte) int { if c == '+' || c == '-' { s.step = stateESign return scanContinue } return stateESign(s, c) } // stateESign is the state after reading the significand, e, and sign in a number, // such as after reading `314e-` or `0.314e+`. func stateESign(s *scanner, c byte) int { if '0' <= c && c <= '9' { s.step = stateE0 return scanContinue } return s.error(c, "in exponent of numeric literal") } // stateE0 is the state after reading the significand, e, optional sign, // and at least one digit of the exponent in a number, // such as after reading `314e-2` or `0.314e+1` or `3.14e0`. func stateE0(s *scanner, c byte) int { if '0' <= c && c <= '9' { return scanContinue } return stateEndValue(s, c) } // stateT is the state after reading `t`. func stateT(s *scanner, c byte) int { if c == 'r' { s.step = stateTr return scanContinue } return s.error(c, "in literal true (expecting 'r')") } // stateTr is the state after reading `tr`. func stateTr(s *scanner, c byte) int { if c == 'u' { s.step = stateTru return scanContinue } return s.error(c, "in literal true (expecting 'u')") } // stateTru is the state after reading `tru`. func stateTru(s *scanner, c byte) int { if c == 'e' { s.step = stateEndValue return scanContinue } return s.error(c, "in literal true (expecting 'e')") } // stateF is the state after reading `f`. func stateF(s *scanner, c byte) int { if c == 'a' { s.step = stateFa return scanContinue } return s.error(c, "in literal false (expecting 'a')") } // stateFa is the state after reading `fa`. func stateFa(s *scanner, c byte) int { if c == 'l' { s.step = stateFal return scanContinue } return s.error(c, "in literal false (expecting 'l')") } // stateFal is the state after reading `fal`. func stateFal(s *scanner, c byte) int { if c == 's' { s.step = stateFals return scanContinue } return s.error(c, "in literal false (expecting 's')") } // stateFals is the state after reading `fals`. func stateFals(s *scanner, c byte) int { if c == 'e' { s.step = stateEndValue return scanContinue } return s.error(c, "in literal false (expecting 'e')") } // stateN is the state after reading `n`. func stateN(s *scanner, c byte) int { if c == 'u' { s.step = stateNu return scanContinue } return s.error(c, "in literal null (expecting 'u')") } // stateNu is the state after reading `nu`. func stateNu(s *scanner, c byte) int { if c == 'l' { s.step = stateNul return scanContinue } return s.error(c, "in literal null (expecting 'l')") } // stateNul is the state after reading `nul`. func stateNul(s *scanner, c byte) int { if c == 'l' { s.step = stateEndValue return scanContinue } return s.error(c, "in literal null (expecting 'l')") } // stateError is the state after reaching a syntax error, // such as after reading `[1}` or `5.1.2`. func stateError(s *scanner, c byte) int { return scanError } // error records an error and switches to the error state. func (s *scanner) error(c byte, context string) int { s.step = stateError s.err = &SyntaxError{"invalid character " + quoteChar(c) + " " + context, s.bytes} return scanError } // quoteChar formats c as a quoted character literal func quoteChar(c byte) string { // Force escapes (where necessary) to use \x. if c < 0x7F { return strconv.QuoteRune(rune(c)) } return fmt.Sprintf("'\\x%02x'", c) } // undo causes the scanner to return scanCode from the next state transition. // This gives callers a simple 1-byte undo mechanism. func (s *scanner) undo(scanCode int) { if s.redo { panic("canonicaljson: invalid use of scanner") } s.redoCode = scanCode s.redoState = s.step s.step = stateRedo s.redo = true } // stateRedo helps implement the scanner's 1-byte undo. func stateRedo(s *scanner, c byte) int { s.redo = false s.step = s.redoState return s.redoCode } canonicaljson-go-1.0.3/scanner_test.go000066400000000000000000000056521317167134000177710ustar00rootroot00000000000000// Copyright 2016 Richard Gibson. All rights reserved. // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package canonicaljson import ( "encoding/json" "testing" ) // Tests of simple examples. type example struct { compact string indent string } var examples = []example{ {`1`, `1`}, {`{}`, `{}`}, {`[]`, `[]`}, {`{"":2}`, "{\n\t\"\": 2\n}"}, {`[3]`, "[\n\t3\n]"}, {`[1,2,3]`, "[\n\t1,\n\t2,\n\t3\n]"}, {`{"x":1}`, "{\n\t\"x\": 1\n}"}, {ex1, ex1i}, } var ex1 = `[true,false,null,"x",1,1.5E0,0,-500]` var ex1i = `[ true, false, null, "x", 1, 1.5E0, 0, -500 ]` func TestMarshal(t *testing.T) { var obj interface{} for _, tt := range examples { if err := json.Unmarshal([]byte(tt.compact), &obj); err != nil { t.Errorf("json.Unmarshal(%#q): %v", tt.compact, err) } else { s, err := Marshal(obj) if err != nil { t.Errorf("Marshal(%#q): %v", tt.compact, err) } else if string(s) != tt.compact { t.Errorf("Marshal(%#q) = %#q, want original", tt.compact, string(s)) } } if err := json.Unmarshal([]byte(tt.indent), &obj); err != nil { t.Errorf("json.Unmarshal(%#q): %v", tt.indent, err) } else { s, err := Marshal(obj) if err != nil { t.Errorf("Marshal(%#q): %v", tt.indent, err) } else if string(s) != tt.compact { t.Errorf("Marshal(%#q) = %#q, want %#q", tt.indent, string(s), tt.compact) } } } } func TestMarshalSeparators(t *testing.T) { // U+2028 and U+2029 should be unescaped inside strings. // They should not appear outside strings. tests := []struct { in, compact string }{ {"{\"\u2028\": 1}", "{\"\u2028\":1}"}, {"{\"\u2029\" :2}", "{\"\u2029\":2}"}, } for _, tt := range tests { var obj interface{} if err := json.Unmarshal([]byte(tt.in), &obj); err != nil { t.Errorf("json.Unmarshal(%q): %v", tt.in, err) } else { s, err := Marshal(obj) if err != nil { t.Errorf("Marshal(%#q): %v", tt.in, err) } else if string(s) != tt.compact { t.Errorf("Marshal(%q) = %q, want %q", tt.in, s, tt.compact) } } } } func TestMarshalIndent(t *testing.T) { var obj interface{} for _, tt := range examples { if err := json.Unmarshal([]byte(tt.indent), &obj); err != nil { t.Errorf("json.Unmarshal(%#q): %v", tt.indent, err) } else { s, err := MarshalIndent(obj, "", "\t") if err != nil { t.Errorf("MarshalIndent(%#q): %v", tt.indent, err) } else if string(s) != tt.indent { t.Errorf("MarshalIndent(%#q) = %#q, want original", tt.indent, s) } } if err := json.Unmarshal([]byte(tt.compact), &obj); err != nil { t.Errorf("json.Unmarshal(%#q): %v", tt.compact, err) } else { s, err := MarshalIndent(obj, "", "\t") if err != nil { t.Errorf("MarshalIndent(%#q): %v", tt.compact, err) } else if string(s) != tt.indent { t.Errorf("MarshalIndent(%#q) = %#q, want %#q", tt.compact, s, tt.indent) } } } } canonicaljson-go-1.0.3/stream.go000066400000000000000000000266351317167134000166000ustar00rootroot00000000000000// Copyright 2016 Richard Gibson. All rights reserved. // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package canonicaljson import ( "bytes" "errors" "io" ) // A Decoder reads and decodes JSON objects from an input stream. type Decoder struct { r io.Reader buf []byte d decodeState scanp int // start of unread data in buf scan scanner err error tokenState int tokenStack []int } // NewDecoder returns a new decoder that reads from r. // // The decoder introduces its own buffering and may // read data from r beyond the JSON values requested. func NewDecoder(r io.Reader) *Decoder { return &Decoder{r: r} } // UseNumber causes the Decoder to unmarshal a number into an interface{} as a // Number instead of as a float64. func (dec *Decoder) UseNumber() { dec.d.useNumber = true } // Decode reads the next JSON-encoded value from its // input and stores it in the value pointed to by v. // // See the documentation for Unmarshal for details about // the conversion of JSON into a Go value. func (dec *Decoder) Decode(v interface{}) error { if dec.err != nil { return dec.err } if err := dec.tokenPrepareForDecode(); err != nil { return err } if !dec.tokenValueAllowed() { return &SyntaxError{msg: "not at beginning of value"} } // Read whole value into buffer. n, err := dec.readValue() if err != nil { return err } dec.d.init(dec.buf[dec.scanp : dec.scanp+n]) dec.scanp += n // Don't save err from unmarshal into dec.err: // the connection is still usable since we read a complete JSON // object from it before the error happened. err = dec.d.unmarshal(v) // fixup token streaming state dec.tokenValueEnd() return err } // Buffered returns a reader of the data remaining in the Decoder's // buffer. The reader is valid until the next call to Decode. func (dec *Decoder) Buffered() io.Reader { return bytes.NewReader(dec.buf[dec.scanp:]) } // readValue reads a JSON value into dec.buf. // It returns the length of the encoding. func (dec *Decoder) readValue() (int, error) { dec.scan.reset() scanp := dec.scanp var err error Input: for { // Look in the buffer for a new value. for i, c := range dec.buf[scanp:] { dec.scan.bytes++ v := dec.scan.step(&dec.scan, c) if v == scanEnd { scanp += i break Input } // scanEnd is delayed one byte. // We might block trying to get that byte from src, // so instead invent a space byte. if (v == scanEndObject || v == scanEndArray) && dec.scan.step(&dec.scan, ' ') == scanEnd { scanp += i + 1 break Input } if v == scanError { dec.err = dec.scan.err return 0, dec.scan.err } } scanp = len(dec.buf) // Did the last read have an error? // Delayed until now to allow buffer scan. if err != nil { if err == io.EOF { if dec.scan.step(&dec.scan, ' ') == scanEnd { break Input } if nonSpace(dec.buf) { err = io.ErrUnexpectedEOF } } dec.err = err return 0, err } n := scanp - dec.scanp err = dec.refill() scanp = dec.scanp + n } return scanp - dec.scanp, nil } func (dec *Decoder) refill() error { // Make room to read more into the buffer. // First slide down data already consumed. if dec.scanp > 0 { n := copy(dec.buf, dec.buf[dec.scanp:]) dec.buf = dec.buf[:n] dec.scanp = 0 } // Grow buffer if not large enough. const minRead = 512 if cap(dec.buf)-len(dec.buf) < minRead { newBuf := make([]byte, len(dec.buf), 2*cap(dec.buf)+minRead) copy(newBuf, dec.buf) dec.buf = newBuf } // Read. Delay error for next iteration (after scan). n, err := dec.r.Read(dec.buf[len(dec.buf):cap(dec.buf)]) dec.buf = dec.buf[0 : len(dec.buf)+n] return err } func nonSpace(b []byte) bool { for _, c := range b { if !isSpace(c) { return true } } return false } // An Encoder writes JSON objects to an output stream. type Encoder struct { w io.Writer err error } // NewEncoder returns a new encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { return &Encoder{w: w} } // Encode writes the JSON encoding of v to the stream, // followed by a newline character. // // See the documentation for Marshal for details about the // conversion of Go values to JSON. func (enc *Encoder) Encode(v interface{}) error { if enc.err != nil { return enc.err } e := newEncodeState() err := e.marshal(v) if err != nil { return err } // Terminate each value with a newline. // This makes the output look a little nicer // when debugging, and some kind of space // is required if the encoded value was a number, // so that the reader knows there aren't more // digits coming. e.WriteByte('\n') if _, err = enc.w.Write(e.Bytes()); err != nil { enc.err = err } encodeStatePool.Put(e) return err } // RawMessage is a raw encoded JSON object. // It implements Marshaler and Unmarshaler and can // be used to delay JSON decoding or precompute a JSON encoding. type RawMessage []byte // MarshalJSON returns *m as the JSON encoding of m. func (m *RawMessage) MarshalJSON() ([]byte, error) { return *m, nil } // UnmarshalJSON sets *m to a copy of data. func (m *RawMessage) UnmarshalJSON(data []byte) error { if m == nil { return errors.New("canonicaljson.RawMessage: UnmarshalJSON on nil pointer") } *m = append((*m)[0:0], data...) return nil } var _ Marshaler = (*RawMessage)(nil) var _ Unmarshaler = (*RawMessage)(nil) // A Token holds a value of one of these types: // // Delim, for the four JSON delimiters [ ] { } // bool, for JSON booleans // float64, for JSON numbers // Number, for JSON numbers // string, for JSON string literals // nil, for JSON null // type Token interface{} const ( tokenTopValue = iota tokenArrayStart tokenArrayValue tokenArrayComma tokenObjectStart tokenObjectKey tokenObjectColon tokenObjectValue tokenObjectComma ) // advance tokenstate from a separator state to a value state func (dec *Decoder) tokenPrepareForDecode() error { // Note: Not calling peek before switch, to avoid // putting peek into the standard Decode path. // peek is only called when using the Token API. switch dec.tokenState { case tokenArrayComma: c, err := dec.peek() if err != nil { return err } if c != ',' { return &SyntaxError{"expected comma after array element", 0} } dec.scanp++ dec.tokenState = tokenArrayValue case tokenObjectColon: c, err := dec.peek() if err != nil { return err } if c != ':' { return &SyntaxError{"expected colon after object key", 0} } dec.scanp++ dec.tokenState = tokenObjectValue } return nil } func (dec *Decoder) tokenValueAllowed() bool { switch dec.tokenState { case tokenTopValue, tokenArrayStart, tokenArrayValue, tokenObjectValue: return true } return false } func (dec *Decoder) tokenValueEnd() { switch dec.tokenState { case tokenArrayStart, tokenArrayValue: dec.tokenState = tokenArrayComma case tokenObjectValue: dec.tokenState = tokenObjectComma } } // A Delim is a JSON array or object delimiter, one of [ ] { or }. type Delim rune func (d Delim) String() string { return string(d) } // Token returns the next JSON token in the input stream. // At the end of the input stream, Token returns nil, io.EOF. // // Token guarantees that the delimiters [ ] { } it returns are // properly nested and matched: if Token encounters an unexpected // delimiter in the input, it will return an error. // // The input stream consists of basic JSON values—bool, string, // number, and null—along with delimiters [ ] { } of type Delim // to mark the start and end of arrays and objects. // Commas and colons are elided. func (dec *Decoder) Token() (Token, error) { for { c, err := dec.peek() if err != nil { return nil, err } switch c { case '[': if !dec.tokenValueAllowed() { return dec.tokenError(c) } dec.scanp++ dec.tokenStack = append(dec.tokenStack, dec.tokenState) dec.tokenState = tokenArrayStart return Delim('['), nil case ']': if dec.tokenState != tokenArrayStart && dec.tokenState != tokenArrayComma { return dec.tokenError(c) } dec.scanp++ dec.tokenState = dec.tokenStack[len(dec.tokenStack)-1] dec.tokenStack = dec.tokenStack[:len(dec.tokenStack)-1] dec.tokenValueEnd() return Delim(']'), nil case '{': if !dec.tokenValueAllowed() { return dec.tokenError(c) } dec.scanp++ dec.tokenStack = append(dec.tokenStack, dec.tokenState) dec.tokenState = tokenObjectStart return Delim('{'), nil case '}': if dec.tokenState != tokenObjectStart && dec.tokenState != tokenObjectComma { return dec.tokenError(c) } dec.scanp++ dec.tokenState = dec.tokenStack[len(dec.tokenStack)-1] dec.tokenStack = dec.tokenStack[:len(dec.tokenStack)-1] dec.tokenValueEnd() return Delim('}'), nil case ':': if dec.tokenState != tokenObjectColon { return dec.tokenError(c) } dec.scanp++ dec.tokenState = tokenObjectValue continue case ',': if dec.tokenState == tokenArrayComma { dec.scanp++ dec.tokenState = tokenArrayValue continue } if dec.tokenState == tokenObjectComma { dec.scanp++ dec.tokenState = tokenObjectKey continue } return dec.tokenError(c) case '"': if dec.tokenState == tokenObjectStart || dec.tokenState == tokenObjectKey { var x string old := dec.tokenState dec.tokenState = tokenTopValue err := dec.Decode(&x) dec.tokenState = old if err != nil { clearOffset(err) return nil, err } dec.tokenState = tokenObjectColon return x, nil } fallthrough default: if !dec.tokenValueAllowed() { return dec.tokenError(c) } var x interface{} if err := dec.Decode(&x); err != nil { clearOffset(err) return nil, err } return x, nil } } } func clearOffset(err error) { if s, ok := err.(*SyntaxError); ok { s.Offset = 0 } } func (dec *Decoder) tokenError(c byte) (Token, error) { var context string switch dec.tokenState { case tokenTopValue: context = " looking for beginning of value" case tokenArrayStart, tokenArrayValue, tokenObjectValue: context = " looking for beginning of value" case tokenArrayComma: context = " after array element" case tokenObjectKey: context = " looking for beginning of object key string" case tokenObjectColon: context = " after object key" case tokenObjectComma: context = " after object key:value pair" } return nil, &SyntaxError{"invalid character " + quoteChar(c) + " " + context, 0} } // More reports whether there is another element in the // current array or object being parsed. func (dec *Decoder) More() bool { c, err := dec.peek() return err == nil && c != ']' && c != '}' } func (dec *Decoder) peek() (byte, error) { var err error for { for i := dec.scanp; i < len(dec.buf); i++ { c := dec.buf[i] if isSpace(c) { continue } dec.scanp = i return c, nil } // buffer has been scanned, now report any error if err != nil { return 0, err } err = dec.refill() } } /* TODO // EncodeToken writes the given JSON token to the stream. // It returns an error if the delimiters [ ] { } are not properly used. // // EncodeToken does not call Flush, because usually it is part of // a larger operation such as Encode, and those will call Flush when finished. // Callers that create an Encoder and then invoke EncodeToken directly, // without using Encode, need to call Flush when finished to ensure that // the JSON is written to the underlying writer. func (e *Encoder) EncodeToken(t Token) error { ... } */ canonicaljson-go-1.0.3/stream_test.go000066400000000000000000000124301317167134000176230ustar00rootroot00000000000000// Copyright 2016 Richard Gibson. All rights reserved. // Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package canonicaljson import ( "bytes" "encoding/json" "io/ioutil" "testing" ) // Test values for the stream test. // One of each JSON kind. var streamTest = []interface{}{ 0.1, "hello", nil, true, false, []interface{}{"a", "b", "c"}, map[string]interface{}{"K": "Kelvin", "ß": "long s"}, 3.14, // another value to make sure something can follow map } var streamEncoded = `1.0E-1 "hello" null true false ["a","b","c"] {"ß":"long s","K":"Kelvin"} 3.14E0 ` func TestEncoder(t *testing.T) { for i := 0; i <= len(streamTest); i++ { var buf bytes.Buffer enc := NewEncoder(&buf) for j, v := range streamTest[0:i] { if err := enc.Encode(v); err != nil { t.Fatalf("encode #%d: %v", j, err) } } if have, want := buf.String(), nlines(streamEncoded, i); have != want { t.Errorf("encoding %d items: mismatch", i) diff(t, []byte(have), []byte(want)) break } } } func nlines(s string, n int) string { if n <= 0 { return "" } for i, c := range s { if c == '\n' { if n--; n == 0 { return s[0 : i+1] } } } return s } func TestRawMessage(t *testing.T) { // TODO(rsc): Should not need the * in *RawMessage var data struct { X float64 Id *RawMessage Y float32 } const raw = `["\u0056",null]` const msg = `{"Id":["V",null],"X":1.0E-1,"Y":2.0E-1}` data.X = 0.1 data.Y = 0.2 id := RawMessage(raw) data.Id = &id b, err := Marshal(&data) if err != nil { t.Fatalf("Marshal: %v", err) } if string(b) != msg { t.Fatalf("Marshal: have %#q want %#q", b, msg) } } func TestNullRawMessage(t *testing.T) { // TODO(rsc): Should not need the * in *RawMessage var data struct { X float64 Id *RawMessage Y float32 } data.Id = new(RawMessage) const msg = `{"Id":null,"X":1.0E-1,"Y":2.0E-1}` err := json.Unmarshal([]byte(msg), &data) if err != nil { t.Fatalf("Unmarshal: %v", err) } if data.Id != nil { t.Fatalf("Raw mismatch: have non-nil, want nil") } b, err := Marshal(&data) if err != nil { t.Fatalf("Marshal: %v", err) } if string(b) != msg { t.Fatalf("Marshal: have %#q want %#q", b, msg) } } var blockingTests = []string{ `{"x": 1}`, `[1, 2, 3]`, } func BenchmarkEncoderEncode(b *testing.B) { b.ReportAllocs() type T struct { X, Y string } v := &T{"foo", "bar"} for i := 0; i < b.N; i++ { if err := NewEncoder(ioutil.Discard).Encode(v); err != nil { b.Fatal(err) } } } type tokenStreamCase struct { json string expTokens []interface{} } type decodeThis struct { v interface{} } var tokenStreamCases []tokenStreamCase = []tokenStreamCase{ // streaming token cases {json: `10`, expTokens: []interface{}{float64(10)}}, {json: ` [10] `, expTokens: []interface{}{ Delim('['), float64(10), Delim(']')}}, {json: ` [false,10,"b"] `, expTokens: []interface{}{ Delim('['), false, float64(10), "b", Delim(']')}}, {json: `{ "a": 1 }`, expTokens: []interface{}{ Delim('{'), "a", float64(1), Delim('}')}}, {json: `{"a": 1, "b":"3"}`, expTokens: []interface{}{ Delim('{'), "a", float64(1), "b", "3", Delim('}')}}, {json: ` [{"a": 1},{"a": 2}] `, expTokens: []interface{}{ Delim('['), Delim('{'), "a", float64(1), Delim('}'), Delim('{'), "a", float64(2), Delim('}'), Delim(']')}}, {json: `{"obj": {"a": 1}}`, expTokens: []interface{}{ Delim('{'), "obj", Delim('{'), "a", float64(1), Delim('}'), Delim('}')}}, {json: `{"obj": [{"a": 1}]}`, expTokens: []interface{}{ Delim('{'), "obj", Delim('['), Delim('{'), "a", float64(1), Delim('}'), Delim(']'), Delim('}')}}, // streaming tokens with intermittent Decode() {json: `{ "a": 1 }`, expTokens: []interface{}{ Delim('{'), "a", decodeThis{float64(1)}, Delim('}')}}, {json: ` [ { "a" : 1 } ] `, expTokens: []interface{}{ Delim('['), decodeThis{map[string]interface{}{"a": float64(1)}}, Delim(']')}}, {json: ` [{"a": 1},{"a": 2}] `, expTokens: []interface{}{ Delim('['), decodeThis{map[string]interface{}{"a": float64(1)}}, decodeThis{map[string]interface{}{"a": float64(2)}}, Delim(']')}}, {json: `{ "obj" : [ { "a" : 1 } ] }`, expTokens: []interface{}{ Delim('{'), "obj", Delim('['), decodeThis{map[string]interface{}{"a": float64(1)}}, Delim(']'), Delim('}')}}, {json: `{"obj": {"a": 1}}`, expTokens: []interface{}{ Delim('{'), "obj", decodeThis{map[string]interface{}{"a": float64(1)}}, Delim('}')}}, {json: `{"obj": [{"a": 1}]}`, expTokens: []interface{}{ Delim('{'), "obj", decodeThis{[]interface{}{ map[string]interface{}{"a": float64(1)}, }}, Delim('}')}}, {json: ` [{"a": 1} {"a": 2}] `, expTokens: []interface{}{ Delim('['), decodeThis{map[string]interface{}{"a": float64(1)}}, decodeThis{&SyntaxError{"expected comma after array element", 0}}, }}, {json: `{ "a" 1 }`, expTokens: []interface{}{ Delim('{'), "a", decodeThis{&SyntaxError{"expected colon after object key", 0}}, }}, } func diff(t *testing.T, a, b []byte) { for i := 0; ; i++ { if i >= len(a) || i >= len(b) || a[i] != b[i] { j := i - 10 if j < 0 { j = 0 } t.Errorf("diverge at %d: «%s» vs «%s»", i, trim(a[j:]), trim(b[j:])) return } } } func trim(b []byte) []byte { if len(b) > 20 { return b[0:20] } return b } canonicaljson-go-1.0.3/tagkey_test.go000066400000000000000000000052371317167134000176230ustar00rootroot00000000000000// Copyright 2016 Richard Gibson. All rights reserved. // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package canonicaljson import ( "testing" ) type basicLatin2xTag struct { V string `json:"$%-/"` } type basicLatin3xTag struct { V string `json:"0123456789"` } type basicLatin4xTag struct { V string `json:"ABCDEFGHIJKLMO"` } type basicLatin5xTag struct { V string `json:"PQRSTUVWXYZ_"` } type basicLatin6xTag struct { V string `json:"abcdefghijklmno"` } type basicLatin7xTag struct { V string `json:"pqrstuvwxyz"` } type miscPlaneTag struct { V string `json:"色は匂へど"` } type percentSlashTag struct { V string `json:"text/html%"` // https://golang.org/issue/2718 } type punctuationTag struct { V string `json:"!#$%&()*+-./:<=>?@[]^_{|}~"` // https://golang.org/issue/3546 } type emptyTag struct { W string } type misnamedTag struct { X string `jsom:"Misnamed"` } type badFormatTag struct { Y string `:"BadFormat"` } type badCodeTag struct { Z string `json:" !\"#&'()*+,."` } type spaceTag struct { Q string `json:"With space"` } type unicodeTag struct { W string `json:"Ελλάδα"` } var structTagObjectKeyTests = []struct { raw interface{} value string key string }{ {basicLatin2xTag{"2x"}, "2x", "$%-/"}, {basicLatin3xTag{"3x"}, "3x", "0123456789"}, {basicLatin4xTag{"4x"}, "4x", "ABCDEFGHIJKLMO"}, {basicLatin5xTag{"5x"}, "5x", "PQRSTUVWXYZ_"}, {basicLatin6xTag{"6x"}, "6x", "abcdefghijklmno"}, {basicLatin7xTag{"7x"}, "7x", "pqrstuvwxyz"}, {miscPlaneTag{"いろはにほへと"}, "いろはにほへと", "色は匂へど"}, {emptyTag{"Pour Moi"}, "Pour Moi", "W"}, {misnamedTag{"Animal Kingdom"}, "Animal Kingdom", "X"}, {badFormatTag{"Orfevre"}, "Orfevre", "Y"}, {badCodeTag{"Reliable Man"}, "Reliable Man", "Z"}, {percentSlashTag{"brut"}, "brut", "text/html%"}, {punctuationTag{"Union Rags"}, "Union Rags", "!#$%&()*+-./:<=>?@[]^_{|}~"}, {spaceTag{"Perreddu"}, "Perreddu", "With space"}, {unicodeTag{"Loukanikos"}, "Loukanikos", "Ελλάδα"}, } func TestStructTagObjectKey(t *testing.T) { for _, tt := range structTagObjectKeyTests { b, err := Marshal(tt.raw) if err != nil { t.Fatalf("Marshal(%#q) failed: %v", tt.raw, err) } var f interface{} err = Unmarshal(b, &f) if err != nil { t.Fatalf("Unmarshal(%#q) failed: %v", b, err) } for i, v := range f.(map[string]interface{}) { switch i { case tt.key: if s, ok := v.(string); !ok || s != tt.value { t.Fatalf("Unexpected value: %#q, want %v", s, tt.value) } default: t.Fatalf("Unexpected key: %#q, from %#q", i, b) } } } } canonicaljson-go-1.0.3/tags.go000066400000000000000000000021641317167134000162320ustar00rootroot00000000000000// Copyright 2016 Richard Gibson. All rights reserved. // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package canonicaljson import ( "strings" ) // tagOptions is the string following a comma in a struct field's "json" // tag, or the empty string. It does not include the leading comma. type tagOptions string // parseTag splits a struct field's json tag into its name and // comma-separated options. func parseTag(tag string) (string, tagOptions) { if idx := strings.Index(tag, ","); idx != -1 { return tag[:idx], tagOptions(tag[idx+1:]) } return tag, tagOptions("") } // Contains reports whether a comma-separated list of options // contains a particular substr flag. substr must be surrounded by a // string boundary or commas. func (o tagOptions) Contains(optionName string) bool { if len(o) == 0 { return false } s := string(o) for s != "" { var next string i := strings.Index(s, ",") if i >= 0 { s, next = s[:i], s[i+1:] } if s == optionName { return true } s = next } return false } canonicaljson-go-1.0.3/tags_test.go000066400000000000000000000011651317167134000172710ustar00rootroot00000000000000// Copyright 2016 Richard Gibson. All rights reserved. // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package canonicaljson import ( "testing" ) func TestTagParsing(t *testing.T) { name, opts := parseTag("field,foobar,foo") if name != "field" { t.Fatalf("name = %q, want field", name) } for _, tt := range []struct { opt string want bool }{ {"foobar", true}, {"foo", true}, {"bar", false}, } { if opts.Contains(tt.opt) != tt.want { t.Errorf("Contains(%q) = %v", tt.opt, !tt.want) } } } canonicaljson-go-1.0.3/test.sh000077500000000000000000000003741317167134000162640ustar00rootroot00000000000000#!/bin/sh # Switch to source directory. cd "`dirname "$0"`" #git submodule update --init --recursive if go build && go test && go build cli/canonicaljson.go && canonicaljson-spec/test.sh ./canonicaljson; then echo PASS else echo FAIL exit 1 fi