You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

1272 line
36 KiB

  1. // Go support for Protocol Buffers - Google's data interchange format
  2. //
  3. // Copyright 2015 The Go Authors. All rights reserved.
  4. // https://github.com/golang/protobuf
  5. //
  6. // Redistribution and use in source and binary forms, with or without
  7. // modification, are permitted provided that the following conditions are
  8. // met:
  9. //
  10. // * Redistributions of source code must retain the above copyright
  11. // notice, this list of conditions and the following disclaimer.
  12. // * Redistributions in binary form must reproduce the above
  13. // copyright notice, this list of conditions and the following disclaimer
  14. // in the documentation and/or other materials provided with the
  15. // distribution.
  16. // * Neither the name of Google Inc. nor the names of its
  17. // contributors may be used to endorse or promote products derived from
  18. // this software without specific prior written permission.
  19. //
  20. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21. // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22. // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23. // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24. // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25. // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26. // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27. // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28. // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29. // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31. /*
  32. Package jsonpb provides marshaling and unmarshaling between protocol buffers and JSON.
  33. It follows the specification at https://developers.google.com/protocol-buffers/docs/proto3#json.
  34. This package produces a different output than the standard "encoding/json" package,
  35. which does not operate correctly on protocol buffers.
  36. */
  37. package jsonpb
  38. import (
  39. "bytes"
  40. "encoding/json"
  41. "errors"
  42. "fmt"
  43. "io"
  44. "math"
  45. "reflect"
  46. "sort"
  47. "strconv"
  48. "strings"
  49. "time"
  50. "github.com/golang/protobuf/proto"
  51. stpb "github.com/golang/protobuf/ptypes/struct"
  52. )
  53. const secondInNanos = int64(time.Second / time.Nanosecond)
  54. // Marshaler is a configurable object for converting between
  55. // protocol buffer objects and a JSON representation for them.
  56. type Marshaler struct {
  57. // Whether to render enum values as integers, as opposed to string values.
  58. EnumsAsInts bool
  59. // Whether to render fields with zero values.
  60. EmitDefaults bool
  61. // A string to indent each level by. The presence of this field will
  62. // also cause a space to appear between the field separator and
  63. // value, and for newlines to be appear between fields and array
  64. // elements.
  65. Indent string
  66. // Whether to use the original (.proto) name for fields.
  67. OrigName bool
  68. // A custom URL resolver to use when marshaling Any messages to JSON.
  69. // If unset, the default resolution strategy is to extract the
  70. // fully-qualified type name from the type URL and pass that to
  71. // proto.MessageType(string).
  72. AnyResolver AnyResolver
  73. }
  74. // AnyResolver takes a type URL, present in an Any message, and resolves it into
  75. // an instance of the associated message.
  76. type AnyResolver interface {
  77. Resolve(typeUrl string) (proto.Message, error)
  78. }
  79. func defaultResolveAny(typeUrl string) (proto.Message, error) {
  80. // Only the part of typeUrl after the last slash is relevant.
  81. mname := typeUrl
  82. if slash := strings.LastIndex(mname, "/"); slash >= 0 {
  83. mname = mname[slash+1:]
  84. }
  85. mt := proto.MessageType(mname)
  86. if mt == nil {
  87. return nil, fmt.Errorf("unknown message type %q", mname)
  88. }
  89. return reflect.New(mt.Elem()).Interface().(proto.Message), nil
  90. }
  91. // JSONPBMarshaler is implemented by protobuf messages that customize the
  92. // way they are marshaled to JSON. Messages that implement this should
  93. // also implement JSONPBUnmarshaler so that the custom format can be
  94. // parsed.
  95. //
  96. // The JSON marshaling must follow the proto to JSON specification:
  97. // https://developers.google.com/protocol-buffers/docs/proto3#json
  98. type JSONPBMarshaler interface {
  99. MarshalJSONPB(*Marshaler) ([]byte, error)
  100. }
  101. // JSONPBUnmarshaler is implemented by protobuf messages that customize
  102. // the way they are unmarshaled from JSON. Messages that implement this
  103. // should also implement JSONPBMarshaler so that the custom format can be
  104. // produced.
  105. //
  106. // The JSON unmarshaling must follow the JSON to proto specification:
  107. // https://developers.google.com/protocol-buffers/docs/proto3#json
  108. type JSONPBUnmarshaler interface {
  109. UnmarshalJSONPB(*Unmarshaler, []byte) error
  110. }
  111. // Marshal marshals a protocol buffer into JSON.
  112. func (m *Marshaler) Marshal(out io.Writer, pb proto.Message) error {
  113. v := reflect.ValueOf(pb)
  114. if pb == nil || (v.Kind() == reflect.Ptr && v.IsNil()) {
  115. return errors.New("Marshal called with nil")
  116. }
  117. // Check for unset required fields first.
  118. if err := checkRequiredFields(pb); err != nil {
  119. return err
  120. }
  121. writer := &errWriter{writer: out}
  122. return m.marshalObject(writer, pb, "", "")
  123. }
  124. // MarshalToString converts a protocol buffer object to JSON string.
  125. func (m *Marshaler) MarshalToString(pb proto.Message) (string, error) {
  126. var buf bytes.Buffer
  127. if err := m.Marshal(&buf, pb); err != nil {
  128. return "", err
  129. }
  130. return buf.String(), nil
  131. }
  132. type int32Slice []int32
  133. var nonFinite = map[string]float64{
  134. `"NaN"`: math.NaN(),
  135. `"Infinity"`: math.Inf(1),
  136. `"-Infinity"`: math.Inf(-1),
  137. }
  138. // For sorting extensions ids to ensure stable output.
  139. func (s int32Slice) Len() int { return len(s) }
  140. func (s int32Slice) Less(i, j int) bool { return s[i] < s[j] }
  141. func (s int32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
  142. type wkt interface {
  143. XXX_WellKnownType() string
  144. }
  145. // marshalObject writes a struct to the Writer.
  146. func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent, typeURL string) error {
  147. if jsm, ok := v.(JSONPBMarshaler); ok {
  148. b, err := jsm.MarshalJSONPB(m)
  149. if err != nil {
  150. return err
  151. }
  152. if typeURL != "" {
  153. // we are marshaling this object to an Any type
  154. var js map[string]*json.RawMessage
  155. if err = json.Unmarshal(b, &js); err != nil {
  156. return fmt.Errorf("type %T produced invalid JSON: %v", v, err)
  157. }
  158. turl, err := json.Marshal(typeURL)
  159. if err != nil {
  160. return fmt.Errorf("failed to marshal type URL %q to JSON: %v", typeURL, err)
  161. }
  162. js["@type"] = (*json.RawMessage)(&turl)
  163. if b, err = json.Marshal(js); err != nil {
  164. return err
  165. }
  166. }
  167. out.write(string(b))
  168. return out.err
  169. }
  170. s := reflect.ValueOf(v).Elem()
  171. // Handle well-known types.
  172. if wkt, ok := v.(wkt); ok {
  173. switch wkt.XXX_WellKnownType() {
  174. case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value",
  175. "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue":
  176. // "Wrappers use the same representation in JSON
  177. // as the wrapped primitive type, ..."
  178. sprop := proto.GetProperties(s.Type())
  179. return m.marshalValue(out, sprop.Prop[0], s.Field(0), indent)
  180. case "Any":
  181. // Any is a bit more involved.
  182. return m.marshalAny(out, v, indent)
  183. case "Duration":
  184. // "Generated output always contains 0, 3, 6, or 9 fractional digits,
  185. // depending on required precision."
  186. s, ns := s.Field(0).Int(), s.Field(1).Int()
  187. if ns <= -secondInNanos || ns >= secondInNanos {
  188. return fmt.Errorf("ns out of range (%v, %v)", -secondInNanos, secondInNanos)
  189. }
  190. if (s > 0 && ns < 0) || (s < 0 && ns > 0) {
  191. return errors.New("signs of seconds and nanos do not match")
  192. }
  193. if s < 0 {
  194. ns = -ns
  195. }
  196. x := fmt.Sprintf("%d.%09d", s, ns)
  197. x = strings.TrimSuffix(x, "000")
  198. x = strings.TrimSuffix(x, "000")
  199. x = strings.TrimSuffix(x, ".000")
  200. out.write(`"`)
  201. out.write(x)
  202. out.write(`s"`)
  203. return out.err
  204. case "Struct", "ListValue":
  205. // Let marshalValue handle the `Struct.fields` map or the `ListValue.values` slice.
  206. // TODO: pass the correct Properties if needed.
  207. return m.marshalValue(out, &proto.Properties{}, s.Field(0), indent)
  208. case "Timestamp":
  209. // "RFC 3339, where generated output will always be Z-normalized
  210. // and uses 0, 3, 6 or 9 fractional digits."
  211. s, ns := s.Field(0).Int(), s.Field(1).Int()
  212. if ns < 0 || ns >= secondInNanos {
  213. return fmt.Errorf("ns out of range [0, %v)", secondInNanos)
  214. }
  215. t := time.Unix(s, ns).UTC()
  216. // time.RFC3339Nano isn't exactly right (we need to get 3/6/9 fractional digits).
  217. x := t.Format("2006-01-02T15:04:05.000000000")
  218. x = strings.TrimSuffix(x, "000")
  219. x = strings.TrimSuffix(x, "000")
  220. x = strings.TrimSuffix(x, ".000")
  221. out.write(`"`)
  222. out.write(x)
  223. out.write(`Z"`)
  224. return out.err
  225. case "Value":
  226. // Value has a single oneof.
  227. kind := s.Field(0)
  228. if kind.IsNil() {
  229. // "absence of any variant indicates an error"
  230. return errors.New("nil Value")
  231. }
  232. // oneof -> *T -> T -> T.F
  233. x := kind.Elem().Elem().Field(0)
  234. // TODO: pass the correct Properties if needed.
  235. return m.marshalValue(out, &proto.Properties{}, x, indent)
  236. }
  237. }
  238. out.write("{")
  239. if m.Indent != "" {
  240. out.write("\n")
  241. }
  242. firstField := true
  243. if typeURL != "" {
  244. if err := m.marshalTypeURL(out, indent, typeURL); err != nil {
  245. return err
  246. }
  247. firstField = false
  248. }
  249. for i := 0; i < s.NumField(); i++ {
  250. value := s.Field(i)
  251. valueField := s.Type().Field(i)
  252. if strings.HasPrefix(valueField.Name, "XXX_") {
  253. continue
  254. }
  255. // IsNil will panic on most value kinds.
  256. switch value.Kind() {
  257. case reflect.Chan, reflect.Func, reflect.Interface:
  258. if value.IsNil() {
  259. continue
  260. }
  261. }
  262. if !m.EmitDefaults {
  263. switch value.Kind() {
  264. case reflect.Bool:
  265. if !value.Bool() {
  266. continue
  267. }
  268. case reflect.Int32, reflect.Int64:
  269. if value.Int() == 0 {
  270. continue
  271. }
  272. case reflect.Uint32, reflect.Uint64:
  273. if value.Uint() == 0 {
  274. continue
  275. }
  276. case reflect.Float32, reflect.Float64:
  277. if value.Float() == 0 {
  278. continue
  279. }
  280. case reflect.String:
  281. if value.Len() == 0 {
  282. continue
  283. }
  284. case reflect.Map, reflect.Ptr, reflect.Slice:
  285. if value.IsNil() {
  286. continue
  287. }
  288. }
  289. }
  290. // Oneof fields need special handling.
  291. if valueField.Tag.Get("protobuf_oneof") != "" {
  292. // value is an interface containing &T{real_value}.
  293. sv := value.Elem().Elem() // interface -> *T -> T
  294. value = sv.Field(0)
  295. valueField = sv.Type().Field(0)
  296. }
  297. prop := jsonProperties(valueField, m.OrigName)
  298. if !firstField {
  299. m.writeSep(out)
  300. }
  301. if err := m.marshalField(out, prop, value, indent); err != nil {
  302. return err
  303. }
  304. firstField = false
  305. }
  306. // Handle proto2 extensions.
  307. if ep, ok := v.(proto.Message); ok {
  308. extensions := proto.RegisteredExtensions(v)
  309. // Sort extensions for stable output.
  310. ids := make([]int32, 0, len(extensions))
  311. for id, desc := range extensions {
  312. if !proto.HasExtension(ep, desc) {
  313. continue
  314. }
  315. ids = append(ids, id)
  316. }
  317. sort.Sort(int32Slice(ids))
  318. for _, id := range ids {
  319. desc := extensions[id]
  320. if desc == nil {
  321. // unknown extension
  322. continue
  323. }
  324. ext, extErr := proto.GetExtension(ep, desc)
  325. if extErr != nil {
  326. return extErr
  327. }
  328. value := reflect.ValueOf(ext)
  329. var prop proto.Properties
  330. prop.Parse(desc.Tag)
  331. prop.JSONName = fmt.Sprintf("[%s]", desc.Name)
  332. if !firstField {
  333. m.writeSep(out)
  334. }
  335. if err := m.marshalField(out, &prop, value, indent); err != nil {
  336. return err
  337. }
  338. firstField = false
  339. }
  340. }
  341. if m.Indent != "" {
  342. out.write("\n")
  343. out.write(indent)
  344. }
  345. out.write("}")
  346. return out.err
  347. }
  348. func (m *Marshaler) writeSep(out *errWriter) {
  349. if m.Indent != "" {
  350. out.write(",\n")
  351. } else {
  352. out.write(",")
  353. }
  354. }
  355. func (m *Marshaler) marshalAny(out *errWriter, any proto.Message, indent string) error {
  356. // "If the Any contains a value that has a special JSON mapping,
  357. // it will be converted as follows: {"@type": xxx, "value": yyy}.
  358. // Otherwise, the value will be converted into a JSON object,
  359. // and the "@type" field will be inserted to indicate the actual data type."
  360. v := reflect.ValueOf(any).Elem()
  361. turl := v.Field(0).String()
  362. val := v.Field(1).Bytes()
  363. var msg proto.Message
  364. var err error
  365. if m.AnyResolver != nil {
  366. msg, err = m.AnyResolver.Resolve(turl)
  367. } else {
  368. msg, err = defaultResolveAny(turl)
  369. }
  370. if err != nil {
  371. return err
  372. }
  373. if err := proto.Unmarshal(val, msg); err != nil {
  374. return err
  375. }
  376. if _, ok := msg.(wkt); ok {
  377. out.write("{")
  378. if m.Indent != "" {
  379. out.write("\n")
  380. }
  381. if err := m.marshalTypeURL(out, indent, turl); err != nil {
  382. return err
  383. }
  384. m.writeSep(out)
  385. if m.Indent != "" {
  386. out.write(indent)
  387. out.write(m.Indent)
  388. out.write(`"value": `)
  389. } else {
  390. out.write(`"value":`)
  391. }
  392. if err := m.marshalObject(out, msg, indent+m.Indent, ""); err != nil {
  393. return err
  394. }
  395. if m.Indent != "" {
  396. out.write("\n")
  397. out.write(indent)
  398. }
  399. out.write("}")
  400. return out.err
  401. }
  402. return m.marshalObject(out, msg, indent, turl)
  403. }
  404. func (m *Marshaler) marshalTypeURL(out *errWriter, indent, typeURL string) error {
  405. if m.Indent != "" {
  406. out.write(indent)
  407. out.write(m.Indent)
  408. }
  409. out.write(`"@type":`)
  410. if m.Indent != "" {
  411. out.write(" ")
  412. }
  413. b, err := json.Marshal(typeURL)
  414. if err != nil {
  415. return err
  416. }
  417. out.write(string(b))
  418. return out.err
  419. }
  420. // marshalField writes field description and value to the Writer.
  421. func (m *Marshaler) marshalField(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error {
  422. if m.Indent != "" {
  423. out.write(indent)
  424. out.write(m.Indent)
  425. }
  426. out.write(`"`)
  427. out.write(prop.JSONName)
  428. out.write(`":`)
  429. if m.Indent != "" {
  430. out.write(" ")
  431. }
  432. if err := m.marshalValue(out, prop, v, indent); err != nil {
  433. return err
  434. }
  435. return nil
  436. }
  437. // marshalValue writes the value to the Writer.
  438. func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error {
  439. var err error
  440. v = reflect.Indirect(v)
  441. // Handle nil pointer
  442. if v.Kind() == reflect.Invalid {
  443. out.write("null")
  444. return out.err
  445. }
  446. // Handle repeated elements.
  447. if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 {
  448. out.write("[")
  449. comma := ""
  450. for i := 0; i < v.Len(); i++ {
  451. sliceVal := v.Index(i)
  452. out.write(comma)
  453. if m.Indent != "" {
  454. out.write("\n")
  455. out.write(indent)
  456. out.write(m.Indent)
  457. out.write(m.Indent)
  458. }
  459. if err := m.marshalValue(out, prop, sliceVal, indent+m.Indent); err != nil {
  460. return err
  461. }
  462. comma = ","
  463. }
  464. if m.Indent != "" {
  465. out.write("\n")
  466. out.write(indent)
  467. out.write(m.Indent)
  468. }
  469. out.write("]")
  470. return out.err
  471. }
  472. // Handle well-known types.
  473. // Most are handled up in marshalObject (because 99% are messages).
  474. if wkt, ok := v.Interface().(wkt); ok {
  475. switch wkt.XXX_WellKnownType() {
  476. case "NullValue":
  477. out.write("null")
  478. return out.err
  479. }
  480. }
  481. // Handle enumerations.
  482. if !m.EnumsAsInts && prop.Enum != "" {
  483. // Unknown enum values will are stringified by the proto library as their
  484. // value. Such values should _not_ be quoted or they will be interpreted
  485. // as an enum string instead of their value.
  486. enumStr := v.Interface().(fmt.Stringer).String()
  487. var valStr string
  488. if v.Kind() == reflect.Ptr {
  489. valStr = strconv.Itoa(int(v.Elem().Int()))
  490. } else {
  491. valStr = strconv.Itoa(int(v.Int()))
  492. }
  493. isKnownEnum := enumStr != valStr
  494. if isKnownEnum {
  495. out.write(`"`)
  496. }
  497. out.write(enumStr)
  498. if isKnownEnum {
  499. out.write(`"`)
  500. }
  501. return out.err
  502. }
  503. // Handle nested messages.
  504. if v.Kind() == reflect.Struct {
  505. return m.marshalObject(out, v.Addr().Interface().(proto.Message), indent+m.Indent, "")
  506. }
  507. // Handle maps.
  508. // Since Go randomizes map iteration, we sort keys for stable output.
  509. if v.Kind() == reflect.Map {
  510. out.write(`{`)
  511. keys := v.MapKeys()
  512. sort.Sort(mapKeys(keys))
  513. for i, k := range keys {
  514. if i > 0 {
  515. out.write(`,`)
  516. }
  517. if m.Indent != "" {
  518. out.write("\n")
  519. out.write(indent)
  520. out.write(m.Indent)
  521. out.write(m.Indent)
  522. }
  523. // TODO handle map key prop properly
  524. b, err := json.Marshal(k.Interface())
  525. if err != nil {
  526. return err
  527. }
  528. s := string(b)
  529. // If the JSON is not a string value, encode it again to make it one.
  530. if !strings.HasPrefix(s, `"`) {
  531. b, err := json.Marshal(s)
  532. if err != nil {
  533. return err
  534. }
  535. s = string(b)
  536. }
  537. out.write(s)
  538. out.write(`:`)
  539. if m.Indent != "" {
  540. out.write(` `)
  541. }
  542. vprop := prop
  543. if prop != nil && prop.MapValProp != nil {
  544. vprop = prop.MapValProp
  545. }
  546. if err := m.marshalValue(out, vprop, v.MapIndex(k), indent+m.Indent); err != nil {
  547. return err
  548. }
  549. }
  550. if m.Indent != "" {
  551. out.write("\n")
  552. out.write(indent)
  553. out.write(m.Indent)
  554. }
  555. out.write(`}`)
  556. return out.err
  557. }
  558. // Handle non-finite floats, e.g. NaN, Infinity and -Infinity.
  559. if v.Kind() == reflect.Float32 || v.Kind() == reflect.Float64 {
  560. f := v.Float()
  561. var sval string
  562. switch {
  563. case math.IsInf(f, 1):
  564. sval = `"Infinity"`
  565. case math.IsInf(f, -1):
  566. sval = `"-Infinity"`
  567. case math.IsNaN(f):
  568. sval = `"NaN"`
  569. }
  570. if sval != "" {
  571. out.write(sval)
  572. return out.err
  573. }
  574. }
  575. // Default handling defers to the encoding/json library.
  576. b, err := json.Marshal(v.Interface())
  577. if err != nil {
  578. return err
  579. }
  580. needToQuote := string(b[0]) != `"` && (v.Kind() == reflect.Int64 || v.Kind() == reflect.Uint64)
  581. if needToQuote {
  582. out.write(`"`)
  583. }
  584. out.write(string(b))
  585. if needToQuote {
  586. out.write(`"`)
  587. }
  588. return out.err
  589. }
  590. // Unmarshaler is a configurable object for converting from a JSON
  591. // representation to a protocol buffer object.
  592. type Unmarshaler struct {
  593. // Whether to allow messages to contain unknown fields, as opposed to
  594. // failing to unmarshal.
  595. AllowUnknownFields bool
  596. // A custom URL resolver to use when unmarshaling Any messages from JSON.
  597. // If unset, the default resolution strategy is to extract the
  598. // fully-qualified type name from the type URL and pass that to
  599. // proto.MessageType(string).
  600. AnyResolver AnyResolver
  601. }
  602. // UnmarshalNext unmarshals the next protocol buffer from a JSON object stream.
  603. // This function is lenient and will decode any options permutations of the
  604. // related Marshaler.
  605. func (u *Unmarshaler) UnmarshalNext(dec *json.Decoder, pb proto.Message) error {
  606. inputValue := json.RawMessage{}
  607. if err := dec.Decode(&inputValue); err != nil {
  608. return err
  609. }
  610. if err := u.unmarshalValue(reflect.ValueOf(pb).Elem(), inputValue, nil); err != nil {
  611. return err
  612. }
  613. return checkRequiredFields(pb)
  614. }
  615. // Unmarshal unmarshals a JSON object stream into a protocol
  616. // buffer. This function is lenient and will decode any options
  617. // permutations of the related Marshaler.
  618. func (u *Unmarshaler) Unmarshal(r io.Reader, pb proto.Message) error {
  619. dec := json.NewDecoder(r)
  620. return u.UnmarshalNext(dec, pb)
  621. }
  622. // UnmarshalNext unmarshals the next protocol buffer from a JSON object stream.
  623. // This function is lenient and will decode any options permutations of the
  624. // related Marshaler.
  625. func UnmarshalNext(dec *json.Decoder, pb proto.Message) error {
  626. return new(Unmarshaler).UnmarshalNext(dec, pb)
  627. }
  628. // Unmarshal unmarshals a JSON object stream into a protocol
  629. // buffer. This function is lenient and will decode any options
  630. // permutations of the related Marshaler.
  631. func Unmarshal(r io.Reader, pb proto.Message) error {
  632. return new(Unmarshaler).Unmarshal(r, pb)
  633. }
  634. // UnmarshalString will populate the fields of a protocol buffer based
  635. // on a JSON string. This function is lenient and will decode any options
  636. // permutations of the related Marshaler.
  637. func UnmarshalString(str string, pb proto.Message) error {
  638. return new(Unmarshaler).Unmarshal(strings.NewReader(str), pb)
  639. }
  640. // unmarshalValue converts/copies a value into the target.
  641. // prop may be nil.
  642. func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMessage, prop *proto.Properties) error {
  643. targetType := target.Type()
  644. // Allocate memory for pointer fields.
  645. if targetType.Kind() == reflect.Ptr {
  646. // If input value is "null" and target is a pointer type, then the field should be treated as not set
  647. // UNLESS the target is structpb.Value, in which case it should be set to structpb.NullValue.
  648. _, isJSONPBUnmarshaler := target.Interface().(JSONPBUnmarshaler)
  649. if string(inputValue) == "null" && targetType != reflect.TypeOf(&stpb.Value{}) && !isJSONPBUnmarshaler {
  650. return nil
  651. }
  652. target.Set(reflect.New(targetType.Elem()))
  653. return u.unmarshalValue(target.Elem(), inputValue, prop)
  654. }
  655. if jsu, ok := target.Addr().Interface().(JSONPBUnmarshaler); ok {
  656. return jsu.UnmarshalJSONPB(u, []byte(inputValue))
  657. }
  658. // Handle well-known types that are not pointers.
  659. if w, ok := target.Addr().Interface().(wkt); ok {
  660. switch w.XXX_WellKnownType() {
  661. case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value",
  662. "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue":
  663. return u.unmarshalValue(target.Field(0), inputValue, prop)
  664. case "Any":
  665. // Use json.RawMessage pointer type instead of value to support pre-1.8 version.
  666. // 1.8 changed RawMessage.MarshalJSON from pointer type to value type, see
  667. // https://github.com/golang/go/issues/14493
  668. var jsonFields map[string]*json.RawMessage
  669. if err := json.Unmarshal(inputValue, &jsonFields); err != nil {
  670. return err
  671. }
  672. val, ok := jsonFields["@type"]
  673. if !ok || val == nil {
  674. return errors.New("Any JSON doesn't have '@type'")
  675. }
  676. var turl string
  677. if err := json.Unmarshal([]byte(*val), &turl); err != nil {
  678. return fmt.Errorf("can't unmarshal Any's '@type': %q", *val)
  679. }
  680. target.Field(0).SetString(turl)
  681. var m proto.Message
  682. var err error
  683. if u.AnyResolver != nil {
  684. m, err = u.AnyResolver.Resolve(turl)
  685. } else {
  686. m, err = defaultResolveAny(turl)
  687. }
  688. if err != nil {
  689. return err
  690. }
  691. if _, ok := m.(wkt); ok {
  692. val, ok := jsonFields["value"]
  693. if !ok {
  694. return errors.New("Any JSON doesn't have 'value'")
  695. }
  696. if err := u.unmarshalValue(reflect.ValueOf(m).Elem(), *val, nil); err != nil {
  697. return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err)
  698. }
  699. } else {
  700. delete(jsonFields, "@type")
  701. nestedProto, err := json.Marshal(jsonFields)
  702. if err != nil {
  703. return fmt.Errorf("can't generate JSON for Any's nested proto to be unmarshaled: %v", err)
  704. }
  705. if err = u.unmarshalValue(reflect.ValueOf(m).Elem(), nestedProto, nil); err != nil {
  706. return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err)
  707. }
  708. }
  709. b, err := proto.Marshal(m)
  710. if err != nil {
  711. return fmt.Errorf("can't marshal proto %T into Any.Value: %v", m, err)
  712. }
  713. target.Field(1).SetBytes(b)
  714. return nil
  715. case "Duration":
  716. unq, err := unquote(string(inputValue))
  717. if err != nil {
  718. return err
  719. }
  720. d, err := time.ParseDuration(unq)
  721. if err != nil {
  722. return fmt.Errorf("bad Duration: %v", err)
  723. }
  724. ns := d.Nanoseconds()
  725. s := ns / 1e9
  726. ns %= 1e9
  727. target.Field(0).SetInt(s)
  728. target.Field(1).SetInt(ns)
  729. return nil
  730. case "Timestamp":
  731. unq, err := unquote(string(inputValue))
  732. if err != nil {
  733. return err
  734. }
  735. t, err := time.Parse(time.RFC3339Nano, unq)
  736. if err != nil {
  737. return fmt.Errorf("bad Timestamp: %v", err)
  738. }
  739. target.Field(0).SetInt(t.Unix())
  740. target.Field(1).SetInt(int64(t.Nanosecond()))
  741. return nil
  742. case "Struct":
  743. var m map[string]json.RawMessage
  744. if err := json.Unmarshal(inputValue, &m); err != nil {
  745. return fmt.Errorf("bad StructValue: %v", err)
  746. }
  747. target.Field(0).Set(reflect.ValueOf(map[string]*stpb.Value{}))
  748. for k, jv := range m {
  749. pv := &stpb.Value{}
  750. if err := u.unmarshalValue(reflect.ValueOf(pv).Elem(), jv, prop); err != nil {
  751. return fmt.Errorf("bad value in StructValue for key %q: %v", k, err)
  752. }
  753. target.Field(0).SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(pv))
  754. }
  755. return nil
  756. case "ListValue":
  757. var s []json.RawMessage
  758. if err := json.Unmarshal(inputValue, &s); err != nil {
  759. return fmt.Errorf("bad ListValue: %v", err)
  760. }
  761. target.Field(0).Set(reflect.ValueOf(make([]*stpb.Value, len(s))))
  762. for i, sv := range s {
  763. if err := u.unmarshalValue(target.Field(0).Index(i), sv, prop); err != nil {
  764. return err
  765. }
  766. }
  767. return nil
  768. case "Value":
  769. ivStr := string(inputValue)
  770. if ivStr == "null" {
  771. target.Field(0).Set(reflect.ValueOf(&stpb.Value_NullValue{}))
  772. } else if v, err := strconv.ParseFloat(ivStr, 0); err == nil {
  773. target.Field(0).Set(reflect.ValueOf(&stpb.Value_NumberValue{v}))
  774. } else if v, err := unquote(ivStr); err == nil {
  775. target.Field(0).Set(reflect.ValueOf(&stpb.Value_StringValue{v}))
  776. } else if v, err := strconv.ParseBool(ivStr); err == nil {
  777. target.Field(0).Set(reflect.ValueOf(&stpb.Value_BoolValue{v}))
  778. } else if err := json.Unmarshal(inputValue, &[]json.RawMessage{}); err == nil {
  779. lv := &stpb.ListValue{}
  780. target.Field(0).Set(reflect.ValueOf(&stpb.Value_ListValue{lv}))
  781. return u.unmarshalValue(reflect.ValueOf(lv).Elem(), inputValue, prop)
  782. } else if err := json.Unmarshal(inputValue, &map[string]json.RawMessage{}); err == nil {
  783. sv := &stpb.Struct{}
  784. target.Field(0).Set(reflect.ValueOf(&stpb.Value_StructValue{sv}))
  785. return u.unmarshalValue(reflect.ValueOf(sv).Elem(), inputValue, prop)
  786. } else {
  787. return fmt.Errorf("unrecognized type for Value %q", ivStr)
  788. }
  789. return nil
  790. }
  791. }
  792. // Handle enums, which have an underlying type of int32,
  793. // and may appear as strings.
  794. // The case of an enum appearing as a number is handled
  795. // at the bottom of this function.
  796. if inputValue[0] == '"' && prop != nil && prop.Enum != "" {
  797. vmap := proto.EnumValueMap(prop.Enum)
  798. // Don't need to do unquoting; valid enum names
  799. // are from a limited character set.
  800. s := inputValue[1 : len(inputValue)-1]
  801. n, ok := vmap[string(s)]
  802. if !ok {
  803. return fmt.Errorf("unknown value %q for enum %s", s, prop.Enum)
  804. }
  805. if target.Kind() == reflect.Ptr { // proto2
  806. target.Set(reflect.New(targetType.Elem()))
  807. target = target.Elem()
  808. }
  809. if targetType.Kind() != reflect.Int32 {
  810. return fmt.Errorf("invalid target %q for enum %s", targetType.Kind(), prop.Enum)
  811. }
  812. target.SetInt(int64(n))
  813. return nil
  814. }
  815. // Handle nested messages.
  816. if targetType.Kind() == reflect.Struct {
  817. var jsonFields map[string]json.RawMessage
  818. if err := json.Unmarshal(inputValue, &jsonFields); err != nil {
  819. return err
  820. }
  821. consumeField := func(prop *proto.Properties) (json.RawMessage, bool) {
  822. // Be liberal in what names we accept; both orig_name and camelName are okay.
  823. fieldNames := acceptedJSONFieldNames(prop)
  824. vOrig, okOrig := jsonFields[fieldNames.orig]
  825. vCamel, okCamel := jsonFields[fieldNames.camel]
  826. if !okOrig && !okCamel {
  827. return nil, false
  828. }
  829. // If, for some reason, both are present in the data, favour the camelName.
  830. var raw json.RawMessage
  831. if okOrig {
  832. raw = vOrig
  833. delete(jsonFields, fieldNames.orig)
  834. }
  835. if okCamel {
  836. raw = vCamel
  837. delete(jsonFields, fieldNames.camel)
  838. }
  839. return raw, true
  840. }
  841. sprops := proto.GetProperties(targetType)
  842. for i := 0; i < target.NumField(); i++ {
  843. ft := target.Type().Field(i)
  844. if strings.HasPrefix(ft.Name, "XXX_") {
  845. continue
  846. }
  847. valueForField, ok := consumeField(sprops.Prop[i])
  848. if !ok {
  849. continue
  850. }
  851. if err := u.unmarshalValue(target.Field(i), valueForField, sprops.Prop[i]); err != nil {
  852. return err
  853. }
  854. }
  855. // Check for any oneof fields.
  856. if len(jsonFields) > 0 {
  857. for _, oop := range sprops.OneofTypes {
  858. raw, ok := consumeField(oop.Prop)
  859. if !ok {
  860. continue
  861. }
  862. nv := reflect.New(oop.Type.Elem())
  863. target.Field(oop.Field).Set(nv)
  864. if err := u.unmarshalValue(nv.Elem().Field(0), raw, oop.Prop); err != nil {
  865. return err
  866. }
  867. }
  868. }
  869. // Handle proto2 extensions.
  870. if len(jsonFields) > 0 {
  871. if ep, ok := target.Addr().Interface().(proto.Message); ok {
  872. for _, ext := range proto.RegisteredExtensions(ep) {
  873. name := fmt.Sprintf("[%s]", ext.Name)
  874. raw, ok := jsonFields[name]
  875. if !ok {
  876. continue
  877. }
  878. delete(jsonFields, name)
  879. nv := reflect.New(reflect.TypeOf(ext.ExtensionType).Elem())
  880. if err := u.unmarshalValue(nv.Elem(), raw, nil); err != nil {
  881. return err
  882. }
  883. if err := proto.SetExtension(ep, ext, nv.Interface()); err != nil {
  884. return err
  885. }
  886. }
  887. }
  888. }
  889. if !u.AllowUnknownFields && len(jsonFields) > 0 {
  890. // Pick any field to be the scapegoat.
  891. var f string
  892. for fname := range jsonFields {
  893. f = fname
  894. break
  895. }
  896. return fmt.Errorf("unknown field %q in %v", f, targetType)
  897. }
  898. return nil
  899. }
  900. // Handle arrays (which aren't encoded bytes)
  901. if targetType.Kind() == reflect.Slice && targetType.Elem().Kind() != reflect.Uint8 {
  902. var slc []json.RawMessage
  903. if err := json.Unmarshal(inputValue, &slc); err != nil {
  904. return err
  905. }
  906. if slc != nil {
  907. l := len(slc)
  908. target.Set(reflect.MakeSlice(targetType, l, l))
  909. for i := 0; i < l; i++ {
  910. if err := u.unmarshalValue(target.Index(i), slc[i], prop); err != nil {
  911. return err
  912. }
  913. }
  914. }
  915. return nil
  916. }
  917. // Handle maps (whose keys are always strings)
  918. if targetType.Kind() == reflect.Map {
  919. var mp map[string]json.RawMessage
  920. if err := json.Unmarshal(inputValue, &mp); err != nil {
  921. return err
  922. }
  923. if mp != nil {
  924. target.Set(reflect.MakeMap(targetType))
  925. for ks, raw := range mp {
  926. // Unmarshal map key. The core json library already decoded the key into a
  927. // string, so we handle that specially. Other types were quoted post-serialization.
  928. var k reflect.Value
  929. if targetType.Key().Kind() == reflect.String {
  930. k = reflect.ValueOf(ks)
  931. } else {
  932. k = reflect.New(targetType.Key()).Elem()
  933. var kprop *proto.Properties
  934. if prop != nil && prop.MapKeyProp != nil {
  935. kprop = prop.MapKeyProp
  936. }
  937. if err := u.unmarshalValue(k, json.RawMessage(ks), kprop); err != nil {
  938. return err
  939. }
  940. }
  941. // Unmarshal map value.
  942. v := reflect.New(targetType.Elem()).Elem()
  943. var vprop *proto.Properties
  944. if prop != nil && prop.MapValProp != nil {
  945. vprop = prop.MapValProp
  946. }
  947. if err := u.unmarshalValue(v, raw, vprop); err != nil {
  948. return err
  949. }
  950. target.SetMapIndex(k, v)
  951. }
  952. }
  953. return nil
  954. }
  955. // Non-finite numbers can be encoded as strings.
  956. isFloat := targetType.Kind() == reflect.Float32 || targetType.Kind() == reflect.Float64
  957. if isFloat {
  958. if num, ok := nonFinite[string(inputValue)]; ok {
  959. target.SetFloat(num)
  960. return nil
  961. }
  962. }
  963. // integers & floats can be encoded as strings. In this case we drop
  964. // the quotes and proceed as normal.
  965. isNum := targetType.Kind() == reflect.Int64 || targetType.Kind() == reflect.Uint64 ||
  966. targetType.Kind() == reflect.Int32 || targetType.Kind() == reflect.Uint32 ||
  967. targetType.Kind() == reflect.Float32 || targetType.Kind() == reflect.Float64
  968. if isNum && strings.HasPrefix(string(inputValue), `"`) {
  969. inputValue = inputValue[1 : len(inputValue)-1]
  970. }
  971. // Use the encoding/json for parsing other value types.
  972. return json.Unmarshal(inputValue, target.Addr().Interface())
  973. }
  974. func unquote(s string) (string, error) {
  975. var ret string
  976. err := json.Unmarshal([]byte(s), &ret)
  977. return ret, err
  978. }
  979. // jsonProperties returns parsed proto.Properties for the field and corrects JSONName attribute.
  980. func jsonProperties(f reflect.StructField, origName bool) *proto.Properties {
  981. var prop proto.Properties
  982. prop.Init(f.Type, f.Name, f.Tag.Get("protobuf"), &f)
  983. if origName || prop.JSONName == "" {
  984. prop.JSONName = prop.OrigName
  985. }
  986. return &prop
  987. }
  988. type fieldNames struct {
  989. orig, camel string
  990. }
  991. func acceptedJSONFieldNames(prop *proto.Properties) fieldNames {
  992. opts := fieldNames{orig: prop.OrigName, camel: prop.OrigName}
  993. if prop.JSONName != "" {
  994. opts.camel = prop.JSONName
  995. }
  996. return opts
  997. }
  998. // Writer wrapper inspired by https://blog.golang.org/errors-are-values
  999. type errWriter struct {
  1000. writer io.Writer
  1001. err error
  1002. }
  1003. func (w *errWriter) write(str string) {
  1004. if w.err != nil {
  1005. return
  1006. }
  1007. _, w.err = w.writer.Write([]byte(str))
  1008. }
  1009. // Map fields may have key types of non-float scalars, strings and enums.
  1010. // The easiest way to sort them in some deterministic order is to use fmt.
  1011. // If this turns out to be inefficient we can always consider other options,
  1012. // such as doing a Schwartzian transform.
  1013. //
  1014. // Numeric keys are sorted in numeric order per
  1015. // https://developers.google.com/protocol-buffers/docs/proto#maps.
  1016. type mapKeys []reflect.Value
  1017. func (s mapKeys) Len() int { return len(s) }
  1018. func (s mapKeys) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
  1019. func (s mapKeys) Less(i, j int) bool {
  1020. if k := s[i].Kind(); k == s[j].Kind() {
  1021. switch k {
  1022. case reflect.String:
  1023. return s[i].String() < s[j].String()
  1024. case reflect.Int32, reflect.Int64:
  1025. return s[i].Int() < s[j].Int()
  1026. case reflect.Uint32, reflect.Uint64:
  1027. return s[i].Uint() < s[j].Uint()
  1028. }
  1029. }
  1030. return fmt.Sprint(s[i].Interface()) < fmt.Sprint(s[j].Interface())
  1031. }
  1032. // checkRequiredFields returns an error if any required field in the given proto message is not set.
  1033. // This function is used by both Marshal and Unmarshal. While required fields only exist in a
  1034. // proto2 message, a proto3 message can contain proto2 message(s).
  1035. func checkRequiredFields(pb proto.Message) error {
  1036. // Most well-known type messages do not contain required fields. The "Any" type may contain
  1037. // a message that has required fields.
  1038. //
  1039. // When an Any message is being marshaled, the code will invoked proto.Unmarshal on Any.Value
  1040. // field in order to transform that into JSON, and that should have returned an error if a
  1041. // required field is not set in the embedded message.
  1042. //
  1043. // When an Any message is being unmarshaled, the code will have invoked proto.Marshal on the
  1044. // embedded message to store the serialized message in Any.Value field, and that should have
  1045. // returned an error if a required field is not set.
  1046. if _, ok := pb.(wkt); ok {
  1047. return nil
  1048. }
  1049. v := reflect.ValueOf(pb)
  1050. // Skip message if it is not a struct pointer.
  1051. if v.Kind() != reflect.Ptr {
  1052. return nil
  1053. }
  1054. v = v.Elem()
  1055. if v.Kind() != reflect.Struct {
  1056. return nil
  1057. }
  1058. for i := 0; i < v.NumField(); i++ {
  1059. field := v.Field(i)
  1060. sfield := v.Type().Field(i)
  1061. if sfield.PkgPath != "" {
  1062. // blank PkgPath means the field is exported; skip if not exported
  1063. continue
  1064. }
  1065. if strings.HasPrefix(sfield.Name, "XXX_") {
  1066. continue
  1067. }
  1068. // Oneof field is an interface implemented by wrapper structs containing the actual oneof
  1069. // field, i.e. an interface containing &T{real_value}.
  1070. if sfield.Tag.Get("protobuf_oneof") != "" {
  1071. if field.Kind() != reflect.Interface {
  1072. continue
  1073. }
  1074. v := field.Elem()
  1075. if v.Kind() != reflect.Ptr || v.IsNil() {
  1076. continue
  1077. }
  1078. v = v.Elem()
  1079. if v.Kind() != reflect.Struct || v.NumField() < 1 {
  1080. continue
  1081. }
  1082. field = v.Field(0)
  1083. sfield = v.Type().Field(0)
  1084. }
  1085. protoTag := sfield.Tag.Get("protobuf")
  1086. if protoTag == "" {
  1087. continue
  1088. }
  1089. var prop proto.Properties
  1090. prop.Init(sfield.Type, sfield.Name, protoTag, &sfield)
  1091. switch field.Kind() {
  1092. case reflect.Map:
  1093. if field.IsNil() {
  1094. continue
  1095. }
  1096. // Check each map value.
  1097. keys := field.MapKeys()
  1098. for _, k := range keys {
  1099. v := field.MapIndex(k)
  1100. if err := checkRequiredFieldsInValue(v); err != nil {
  1101. return err
  1102. }
  1103. }
  1104. case reflect.Slice:
  1105. // Handle non-repeated type, e.g. bytes.
  1106. if !prop.Repeated {
  1107. if prop.Required && field.IsNil() {
  1108. return fmt.Errorf("required field %q is not set", prop.Name)
  1109. }
  1110. continue
  1111. }
  1112. // Handle repeated type.
  1113. if field.IsNil() {
  1114. continue
  1115. }
  1116. // Check each slice item.
  1117. for i := 0; i < field.Len(); i++ {
  1118. v := field.Index(i)
  1119. if err := checkRequiredFieldsInValue(v); err != nil {
  1120. return err
  1121. }
  1122. }
  1123. case reflect.Ptr:
  1124. if field.IsNil() {
  1125. if prop.Required {
  1126. return fmt.Errorf("required field %q is not set", prop.Name)
  1127. }
  1128. continue
  1129. }
  1130. if err := checkRequiredFieldsInValue(field); err != nil {
  1131. return err
  1132. }
  1133. }
  1134. }
  1135. // Handle proto2 extensions.
  1136. for _, ext := range proto.RegisteredExtensions(pb) {
  1137. if !proto.HasExtension(pb, ext) {
  1138. continue
  1139. }
  1140. ep, err := proto.GetExtension(pb, ext)
  1141. if err != nil {
  1142. return err
  1143. }
  1144. err = checkRequiredFieldsInValue(reflect.ValueOf(ep))
  1145. if err != nil {
  1146. return err
  1147. }
  1148. }
  1149. return nil
  1150. }
  1151. func checkRequiredFieldsInValue(v reflect.Value) error {
  1152. if pm, ok := v.Interface().(proto.Message); ok {
  1153. return checkRequiredFields(pm)
  1154. }
  1155. return nil
  1156. }