You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

872 lines
23 KiB

  1. // Copyright 2015 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package bigquery
  15. import (
  16. "encoding/base64"
  17. "errors"
  18. "fmt"
  19. "math"
  20. "math/big"
  21. "reflect"
  22. "strconv"
  23. "strings"
  24. "time"
  25. "cloud.google.com/go/civil"
  26. bq "google.golang.org/api/bigquery/v2"
  27. )
  28. // Value stores the contents of a single cell from a BigQuery result.
  29. type Value interface{}
  30. // ValueLoader stores a slice of Values representing a result row from a Read operation.
  31. // See RowIterator.Next for more information.
  32. type ValueLoader interface {
  33. Load(v []Value, s Schema) error
  34. }
  35. // valueList converts a []Value to implement ValueLoader.
  36. type valueList []Value
  37. // Load stores a sequence of values in a valueList.
  38. // It resets the slice length to zero, then appends each value to it.
  39. func (vs *valueList) Load(v []Value, _ Schema) error {
  40. *vs = append((*vs)[:0], v...)
  41. return nil
  42. }
  43. // valueMap converts a map[string]Value to implement ValueLoader.
  44. type valueMap map[string]Value
  45. // Load stores a sequence of values in a valueMap.
  46. func (vm *valueMap) Load(v []Value, s Schema) error {
  47. if *vm == nil {
  48. *vm = map[string]Value{}
  49. }
  50. loadMap(*vm, v, s)
  51. return nil
  52. }
  53. func loadMap(m map[string]Value, vals []Value, s Schema) {
  54. for i, f := range s {
  55. val := vals[i]
  56. var v interface{}
  57. switch {
  58. case val == nil:
  59. v = val
  60. case f.Schema == nil:
  61. v = val
  62. case !f.Repeated:
  63. m2 := map[string]Value{}
  64. loadMap(m2, val.([]Value), f.Schema)
  65. v = m2
  66. default: // repeated and nested
  67. sval := val.([]Value)
  68. vs := make([]Value, len(sval))
  69. for j, e := range sval {
  70. m2 := map[string]Value{}
  71. loadMap(m2, e.([]Value), f.Schema)
  72. vs[j] = m2
  73. }
  74. v = vs
  75. }
  76. m[f.Name] = v
  77. }
  78. }
  79. type structLoader struct {
  80. typ reflect.Type // type of struct
  81. err error
  82. ops []structLoaderOp
  83. vstructp reflect.Value // pointer to current struct value; changed by set
  84. }
  85. // A setFunc is a function that sets a struct field or slice/array
  86. // element to a value.
  87. type setFunc func(v reflect.Value, val interface{}) error
  88. // A structLoaderOp instructs the loader to set a struct field to a row value.
  89. type structLoaderOp struct {
  90. fieldIndex []int
  91. valueIndex int
  92. setFunc setFunc
  93. repeated bool
  94. }
  95. var errNoNulls = errors.New("bigquery: NULL values cannot be read into structs")
  96. func setAny(v reflect.Value, x interface{}) error {
  97. if x == nil {
  98. return errNoNulls
  99. }
  100. v.Set(reflect.ValueOf(x))
  101. return nil
  102. }
  103. func setInt(v reflect.Value, x interface{}) error {
  104. if x == nil {
  105. return errNoNulls
  106. }
  107. xx := x.(int64)
  108. if v.OverflowInt(xx) {
  109. return fmt.Errorf("bigquery: value %v overflows struct field of type %v", xx, v.Type())
  110. }
  111. v.SetInt(xx)
  112. return nil
  113. }
  114. func setUint(v reflect.Value, x interface{}) error {
  115. if x == nil {
  116. return errNoNulls
  117. }
  118. xx := x.(int64)
  119. if xx < 0 || v.OverflowUint(uint64(xx)) {
  120. return fmt.Errorf("bigquery: value %v overflows struct field of type %v", xx, v.Type())
  121. }
  122. v.SetUint(uint64(xx))
  123. return nil
  124. }
  125. func setFloat(v reflect.Value, x interface{}) error {
  126. if x == nil {
  127. return errNoNulls
  128. }
  129. xx := x.(float64)
  130. if v.OverflowFloat(xx) {
  131. return fmt.Errorf("bigquery: value %v overflows struct field of type %v", xx, v.Type())
  132. }
  133. v.SetFloat(xx)
  134. return nil
  135. }
  136. func setBool(v reflect.Value, x interface{}) error {
  137. if x == nil {
  138. return errNoNulls
  139. }
  140. v.SetBool(x.(bool))
  141. return nil
  142. }
  143. func setString(v reflect.Value, x interface{}) error {
  144. if x == nil {
  145. return errNoNulls
  146. }
  147. v.SetString(x.(string))
  148. return nil
  149. }
  150. func setBytes(v reflect.Value, x interface{}) error {
  151. if x == nil {
  152. v.SetBytes(nil)
  153. } else {
  154. v.SetBytes(x.([]byte))
  155. }
  156. return nil
  157. }
  158. func setNull(v reflect.Value, x interface{}, build func() interface{}) error {
  159. if x == nil {
  160. v.Set(reflect.Zero(v.Type()))
  161. } else {
  162. n := build()
  163. v.Set(reflect.ValueOf(n))
  164. }
  165. return nil
  166. }
  167. // set remembers a value for the next call to Load. The value must be
  168. // a pointer to a struct. (This is checked in RowIterator.Next.)
  169. func (sl *structLoader) set(structp interface{}, schema Schema) error {
  170. if sl.err != nil {
  171. return sl.err
  172. }
  173. sl.vstructp = reflect.ValueOf(structp)
  174. typ := sl.vstructp.Type().Elem()
  175. if sl.typ == nil {
  176. // First call: remember the type and compile the schema.
  177. sl.typ = typ
  178. ops, err := compileToOps(typ, schema)
  179. if err != nil {
  180. sl.err = err
  181. return err
  182. }
  183. sl.ops = ops
  184. } else if sl.typ != typ {
  185. return fmt.Errorf("bigquery: struct type changed from %s to %s", sl.typ, typ)
  186. }
  187. return nil
  188. }
  189. // compileToOps produces a sequence of operations that will set the fields of a
  190. // value of structType to the contents of a row with schema.
  191. func compileToOps(structType reflect.Type, schema Schema) ([]structLoaderOp, error) {
  192. var ops []structLoaderOp
  193. fields, err := fieldCache.Fields(structType)
  194. if err != nil {
  195. return nil, err
  196. }
  197. for i, schemaField := range schema {
  198. // Look for an exported struct field with the same name as the schema
  199. // field, ignoring case (BigQuery column names are case-insensitive,
  200. // and we want to act like encoding/json anyway).
  201. structField := fields.Match(schemaField.Name)
  202. if structField == nil {
  203. // Ignore schema fields with no corresponding struct field.
  204. continue
  205. }
  206. op := structLoaderOp{
  207. fieldIndex: structField.Index,
  208. valueIndex: i,
  209. }
  210. t := structField.Type
  211. if schemaField.Repeated {
  212. if t.Kind() != reflect.Slice && t.Kind() != reflect.Array {
  213. return nil, fmt.Errorf("bigquery: repeated schema field %s requires slice or array, but struct field %s has type %s",
  214. schemaField.Name, structField.Name, t)
  215. }
  216. t = t.Elem()
  217. op.repeated = true
  218. }
  219. if schemaField.Type == RecordFieldType {
  220. // Field can be a struct or a pointer to a struct.
  221. if t.Kind() == reflect.Ptr {
  222. t = t.Elem()
  223. }
  224. if t.Kind() != reflect.Struct {
  225. return nil, fmt.Errorf("bigquery: field %s has type %s, expected struct or *struct",
  226. structField.Name, structField.Type)
  227. }
  228. nested, err := compileToOps(t, schemaField.Schema)
  229. if err != nil {
  230. return nil, err
  231. }
  232. op.setFunc = func(v reflect.Value, val interface{}) error {
  233. return setNested(nested, v, val)
  234. }
  235. } else {
  236. op.setFunc = determineSetFunc(t, schemaField.Type)
  237. if op.setFunc == nil {
  238. return nil, fmt.Errorf("bigquery: schema field %s of type %s is not assignable to struct field %s of type %s",
  239. schemaField.Name, schemaField.Type, structField.Name, t)
  240. }
  241. }
  242. ops = append(ops, op)
  243. }
  244. return ops, nil
  245. }
  246. // determineSetFunc chooses the best function for setting a field of type ftype
  247. // to a value whose schema field type is stype. It returns nil if stype
  248. // is not assignable to ftype.
  249. // determineSetFunc considers only basic types. See compileToOps for
  250. // handling of repetition and nesting.
  251. func determineSetFunc(ftype reflect.Type, stype FieldType) setFunc {
  252. switch stype {
  253. case StringFieldType:
  254. if ftype.Kind() == reflect.String {
  255. return setString
  256. }
  257. if ftype == typeOfNullString {
  258. return func(v reflect.Value, x interface{}) error {
  259. return setNull(v, x, func() interface{} {
  260. return NullString{StringVal: x.(string), Valid: true}
  261. })
  262. }
  263. }
  264. case BytesFieldType:
  265. if ftype == typeOfByteSlice {
  266. return setBytes
  267. }
  268. case IntegerFieldType:
  269. if isSupportedUintType(ftype) {
  270. return setUint
  271. } else if isSupportedIntType(ftype) {
  272. return setInt
  273. }
  274. if ftype == typeOfNullInt64 {
  275. return func(v reflect.Value, x interface{}) error {
  276. return setNull(v, x, func() interface{} {
  277. return NullInt64{Int64: x.(int64), Valid: true}
  278. })
  279. }
  280. }
  281. case FloatFieldType:
  282. switch ftype.Kind() {
  283. case reflect.Float32, reflect.Float64:
  284. return setFloat
  285. }
  286. if ftype == typeOfNullFloat64 {
  287. return func(v reflect.Value, x interface{}) error {
  288. return setNull(v, x, func() interface{} {
  289. return NullFloat64{Float64: x.(float64), Valid: true}
  290. })
  291. }
  292. }
  293. case BooleanFieldType:
  294. if ftype.Kind() == reflect.Bool {
  295. return setBool
  296. }
  297. if ftype == typeOfNullBool {
  298. return func(v reflect.Value, x interface{}) error {
  299. return setNull(v, x, func() interface{} {
  300. return NullBool{Bool: x.(bool), Valid: true}
  301. })
  302. }
  303. }
  304. case TimestampFieldType:
  305. if ftype == typeOfGoTime {
  306. return setAny
  307. }
  308. if ftype == typeOfNullTimestamp {
  309. return func(v reflect.Value, x interface{}) error {
  310. return setNull(v, x, func() interface{} {
  311. return NullTimestamp{Timestamp: x.(time.Time), Valid: true}
  312. })
  313. }
  314. }
  315. case DateFieldType:
  316. if ftype == typeOfDate {
  317. return setAny
  318. }
  319. if ftype == typeOfNullDate {
  320. return func(v reflect.Value, x interface{}) error {
  321. return setNull(v, x, func() interface{} {
  322. return NullDate{Date: x.(civil.Date), Valid: true}
  323. })
  324. }
  325. }
  326. case TimeFieldType:
  327. if ftype == typeOfTime {
  328. return setAny
  329. }
  330. if ftype == typeOfNullTime {
  331. return func(v reflect.Value, x interface{}) error {
  332. return setNull(v, x, func() interface{} {
  333. return NullTime{Time: x.(civil.Time), Valid: true}
  334. })
  335. }
  336. }
  337. case DateTimeFieldType:
  338. if ftype == typeOfDateTime {
  339. return setAny
  340. }
  341. if ftype == typeOfNullDateTime {
  342. return func(v reflect.Value, x interface{}) error {
  343. return setNull(v, x, func() interface{} {
  344. return NullDateTime{DateTime: x.(civil.DateTime), Valid: true}
  345. })
  346. }
  347. }
  348. case NumericFieldType:
  349. if ftype == typeOfRat {
  350. return func(v reflect.Value, x interface{}) error {
  351. return setNull(v, x, func() interface{} { return x.(*big.Rat) })
  352. }
  353. }
  354. }
  355. return nil
  356. }
  357. func (sl *structLoader) Load(values []Value, _ Schema) error {
  358. if sl.err != nil {
  359. return sl.err
  360. }
  361. return runOps(sl.ops, sl.vstructp.Elem(), values)
  362. }
  363. // runOps executes a sequence of ops, setting the fields of vstruct to the
  364. // supplied values.
  365. func runOps(ops []structLoaderOp, vstruct reflect.Value, values []Value) error {
  366. for _, op := range ops {
  367. field := vstruct.FieldByIndex(op.fieldIndex)
  368. var err error
  369. if op.repeated {
  370. err = setRepeated(field, values[op.valueIndex].([]Value), op.setFunc)
  371. } else {
  372. err = op.setFunc(field, values[op.valueIndex])
  373. }
  374. if err != nil {
  375. return err
  376. }
  377. }
  378. return nil
  379. }
  380. func setNested(ops []structLoaderOp, v reflect.Value, val interface{}) error {
  381. // v is either a struct or a pointer to a struct.
  382. if v.Kind() == reflect.Ptr {
  383. // If the value is nil, set the pointer to nil.
  384. if val == nil {
  385. v.Set(reflect.Zero(v.Type()))
  386. return nil
  387. }
  388. // If the pointer is nil, set it to a zero struct value.
  389. if v.IsNil() {
  390. v.Set(reflect.New(v.Type().Elem()))
  391. }
  392. v = v.Elem()
  393. }
  394. return runOps(ops, v, val.([]Value))
  395. }
  396. func setRepeated(field reflect.Value, vslice []Value, setElem setFunc) error {
  397. vlen := len(vslice)
  398. var flen int
  399. switch field.Type().Kind() {
  400. case reflect.Slice:
  401. // Make a slice of the right size, avoiding allocation if possible.
  402. switch {
  403. case field.Len() < vlen:
  404. field.Set(reflect.MakeSlice(field.Type(), vlen, vlen))
  405. case field.Len() > vlen:
  406. field.SetLen(vlen)
  407. }
  408. flen = vlen
  409. case reflect.Array:
  410. flen = field.Len()
  411. if flen > vlen {
  412. // Set extra elements to their zero value.
  413. z := reflect.Zero(field.Type().Elem())
  414. for i := vlen; i < flen; i++ {
  415. field.Index(i).Set(z)
  416. }
  417. }
  418. default:
  419. return fmt.Errorf("bigquery: impossible field type %s", field.Type())
  420. }
  421. for i, val := range vslice {
  422. if i < flen { // avoid writing past the end of a short array
  423. if err := setElem(field.Index(i), val); err != nil {
  424. return err
  425. }
  426. }
  427. }
  428. return nil
  429. }
  430. // A ValueSaver returns a row of data to be inserted into a table.
  431. type ValueSaver interface {
  432. // Save returns a row to be inserted into a BigQuery table, represented
  433. // as a map from field name to Value.
  434. // If insertID is non-empty, BigQuery will use it to de-duplicate
  435. // insertions of this row on a best-effort basis.
  436. Save() (row map[string]Value, insertID string, err error)
  437. }
  438. // ValuesSaver implements ValueSaver for a slice of Values.
  439. type ValuesSaver struct {
  440. Schema Schema
  441. // If non-empty, BigQuery will use InsertID to de-duplicate insertions
  442. // of this row on a best-effort basis.
  443. InsertID string
  444. Row []Value
  445. }
  446. // Save implements ValueSaver.
  447. func (vls *ValuesSaver) Save() (map[string]Value, string, error) {
  448. m, err := valuesToMap(vls.Row, vls.Schema)
  449. return m, vls.InsertID, err
  450. }
  451. func valuesToMap(vs []Value, schema Schema) (map[string]Value, error) {
  452. if len(vs) != len(schema) {
  453. return nil, errors.New("Schema does not match length of row to be inserted")
  454. }
  455. m := make(map[string]Value)
  456. for i, fieldSchema := range schema {
  457. if vs[i] == nil {
  458. m[fieldSchema.Name] = nil
  459. continue
  460. }
  461. if fieldSchema.Type != RecordFieldType {
  462. m[fieldSchema.Name] = toUploadValue(vs[i], fieldSchema)
  463. continue
  464. }
  465. // Nested record, possibly repeated.
  466. vals, ok := vs[i].([]Value)
  467. if !ok {
  468. return nil, errors.New("nested record is not a []Value")
  469. }
  470. if !fieldSchema.Repeated {
  471. value, err := valuesToMap(vals, fieldSchema.Schema)
  472. if err != nil {
  473. return nil, err
  474. }
  475. m[fieldSchema.Name] = value
  476. continue
  477. }
  478. // A repeated nested field is converted into a slice of maps.
  479. var maps []Value
  480. for _, v := range vals {
  481. sv, ok := v.([]Value)
  482. if !ok {
  483. return nil, errors.New("nested record in slice is not a []Value")
  484. }
  485. value, err := valuesToMap(sv, fieldSchema.Schema)
  486. if err != nil {
  487. return nil, err
  488. }
  489. maps = append(maps, value)
  490. }
  491. m[fieldSchema.Name] = maps
  492. }
  493. return m, nil
  494. }
  495. // StructSaver implements ValueSaver for a struct.
  496. // The struct is converted to a map of values by using the values of struct
  497. // fields corresponding to schema fields. Additional and missing
  498. // fields are ignored, as are nested struct pointers that are nil.
  499. type StructSaver struct {
  500. // Schema determines what fields of the struct are uploaded. It should
  501. // match the table's schema.
  502. // Schema is optional for StructSavers that are passed to Uploader.Put.
  503. Schema Schema
  504. // If non-empty, BigQuery will use InsertID to de-duplicate insertions
  505. // of this row on a best-effort basis.
  506. InsertID string
  507. // Struct should be a struct or a pointer to a struct.
  508. Struct interface{}
  509. }
  510. // Save implements ValueSaver.
  511. func (ss *StructSaver) Save() (row map[string]Value, insertID string, err error) {
  512. vstruct := reflect.ValueOf(ss.Struct)
  513. row, err = structToMap(vstruct, ss.Schema)
  514. if err != nil {
  515. return nil, "", err
  516. }
  517. return row, ss.InsertID, nil
  518. }
  519. func structToMap(vstruct reflect.Value, schema Schema) (map[string]Value, error) {
  520. if vstruct.Kind() == reflect.Ptr {
  521. vstruct = vstruct.Elem()
  522. }
  523. if !vstruct.IsValid() {
  524. return nil, nil
  525. }
  526. m := map[string]Value{}
  527. if vstruct.Kind() != reflect.Struct {
  528. return nil, fmt.Errorf("bigquery: type is %s, need struct or struct pointer", vstruct.Type())
  529. }
  530. fields, err := fieldCache.Fields(vstruct.Type())
  531. if err != nil {
  532. return nil, err
  533. }
  534. for _, schemaField := range schema {
  535. // Look for an exported struct field with the same name as the schema
  536. // field, ignoring case.
  537. structField := fields.Match(schemaField.Name)
  538. if structField == nil {
  539. continue
  540. }
  541. val, err := structFieldToUploadValue(vstruct.FieldByIndex(structField.Index), schemaField)
  542. if err != nil {
  543. return nil, err
  544. }
  545. // Add the value to the map, unless it is nil.
  546. if val != nil {
  547. m[schemaField.Name] = val
  548. }
  549. }
  550. return m, nil
  551. }
  552. // structFieldToUploadValue converts a struct field to a value suitable for ValueSaver.Save, using
  553. // the schemaField as a guide.
  554. // structFieldToUploadValue is careful to return a true nil interface{} when needed, so its
  555. // caller can easily identify a nil value.
  556. func structFieldToUploadValue(vfield reflect.Value, schemaField *FieldSchema) (interface{}, error) {
  557. if schemaField.Repeated && (vfield.Kind() != reflect.Slice && vfield.Kind() != reflect.Array) {
  558. return nil, fmt.Errorf("bigquery: repeated schema field %s requires slice or array, but value has type %s",
  559. schemaField.Name, vfield.Type())
  560. }
  561. // A non-nested field can be represented by its Go value, except for some types.
  562. if schemaField.Type != RecordFieldType {
  563. return toUploadValueReflect(vfield, schemaField), nil
  564. }
  565. // A non-repeated nested field is converted into a map[string]Value.
  566. if !schemaField.Repeated {
  567. m, err := structToMap(vfield, schemaField.Schema)
  568. if err != nil {
  569. return nil, err
  570. }
  571. if m == nil {
  572. return nil, nil
  573. }
  574. return m, nil
  575. }
  576. // A repeated nested field is converted into a slice of maps.
  577. if vfield.Len() == 0 {
  578. return nil, nil
  579. }
  580. var vals []Value
  581. for i := 0; i < vfield.Len(); i++ {
  582. m, err := structToMap(vfield.Index(i), schemaField.Schema)
  583. if err != nil {
  584. return nil, err
  585. }
  586. vals = append(vals, m)
  587. }
  588. return vals, nil
  589. }
  590. func toUploadValue(val interface{}, fs *FieldSchema) interface{} {
  591. if fs.Type == TimeFieldType || fs.Type == DateTimeFieldType || fs.Type == NumericFieldType {
  592. return toUploadValueReflect(reflect.ValueOf(val), fs)
  593. }
  594. return val
  595. }
  596. func toUploadValueReflect(v reflect.Value, fs *FieldSchema) interface{} {
  597. switch fs.Type {
  598. case TimeFieldType:
  599. if v.Type() == typeOfNullTime {
  600. return v.Interface()
  601. }
  602. return formatUploadValue(v, fs, func(v reflect.Value) string {
  603. return CivilTimeString(v.Interface().(civil.Time))
  604. })
  605. case DateTimeFieldType:
  606. if v.Type() == typeOfNullDateTime {
  607. return v.Interface()
  608. }
  609. return formatUploadValue(v, fs, func(v reflect.Value) string {
  610. return CivilDateTimeString(v.Interface().(civil.DateTime))
  611. })
  612. case NumericFieldType:
  613. if r, ok := v.Interface().(*big.Rat); ok && r == nil {
  614. return nil
  615. }
  616. return formatUploadValue(v, fs, func(v reflect.Value) string {
  617. return NumericString(v.Interface().(*big.Rat))
  618. })
  619. default:
  620. if !fs.Repeated || v.Len() > 0 {
  621. return v.Interface()
  622. }
  623. // The service treats a null repeated field as an error. Return
  624. // nil to omit the field entirely.
  625. return nil
  626. }
  627. }
  628. func formatUploadValue(v reflect.Value, fs *FieldSchema, cvt func(reflect.Value) string) interface{} {
  629. if !fs.Repeated {
  630. return cvt(v)
  631. }
  632. if v.Len() == 0 {
  633. return nil
  634. }
  635. s := make([]string, v.Len())
  636. for i := 0; i < v.Len(); i++ {
  637. s[i] = cvt(v.Index(i))
  638. }
  639. return s
  640. }
  641. // CivilTimeString returns a string representing a civil.Time in a format compatible
  642. // with BigQuery SQL. It rounds the time to the nearest microsecond and returns a
  643. // string with six digits of sub-second precision.
  644. //
  645. // Use CivilTimeString when using civil.Time in DML, for example in INSERT
  646. // statements.
  647. func CivilTimeString(t civil.Time) string {
  648. if t.Nanosecond == 0 {
  649. return t.String()
  650. } else {
  651. micro := (t.Nanosecond + 500) / 1000 // round to nearest microsecond
  652. t.Nanosecond = 0
  653. return t.String() + fmt.Sprintf(".%06d", micro)
  654. }
  655. }
  656. // CivilDateTimeString returns a string representing a civil.DateTime in a format compatible
  657. // with BigQuery SQL. It separate the date and time with a space, and formats the time
  658. // with CivilTimeString.
  659. //
  660. // Use CivilDateTimeString when using civil.DateTime in DML, for example in INSERT
  661. // statements.
  662. func CivilDateTimeString(dt civil.DateTime) string {
  663. return dt.Date.String() + " " + CivilTimeString(dt.Time)
  664. }
  665. // parseCivilDateTime parses a date-time represented in a BigQuery SQL
  666. // compatible format and returns a civil.DateTime.
  667. func parseCivilDateTime(s string) (civil.DateTime, error) {
  668. parts := strings.Fields(s)
  669. if len(parts) != 2 {
  670. return civil.DateTime{}, fmt.Errorf("bigquery: bad DATETIME value %q", s)
  671. }
  672. return civil.ParseDateTime(parts[0] + "T" + parts[1])
  673. }
  674. const (
  675. // The maximum number of digits in a NUMERIC value.
  676. NumericPrecisionDigits = 38
  677. // The maximum number of digits after the decimal point in a NUMERIC value.
  678. NumericScaleDigits = 9
  679. )
  680. // NumericString returns a string representing a *big.Rat in a format compatible
  681. // with BigQuery SQL. It returns a floating-point literal with 9 digits
  682. // after the decimal point.
  683. func NumericString(r *big.Rat) string {
  684. return r.FloatString(NumericScaleDigits)
  685. }
  686. // convertRows converts a series of TableRows into a series of Value slices.
  687. // schema is used to interpret the data from rows; its length must match the
  688. // length of each row.
  689. func convertRows(rows []*bq.TableRow, schema Schema) ([][]Value, error) {
  690. var rs [][]Value
  691. for _, r := range rows {
  692. row, err := convertRow(r, schema)
  693. if err != nil {
  694. return nil, err
  695. }
  696. rs = append(rs, row)
  697. }
  698. return rs, nil
  699. }
  700. func convertRow(r *bq.TableRow, schema Schema) ([]Value, error) {
  701. if len(schema) != len(r.F) {
  702. return nil, errors.New("schema length does not match row length")
  703. }
  704. var values []Value
  705. for i, cell := range r.F {
  706. fs := schema[i]
  707. v, err := convertValue(cell.V, fs.Type, fs.Schema)
  708. if err != nil {
  709. return nil, err
  710. }
  711. values = append(values, v)
  712. }
  713. return values, nil
  714. }
  715. func convertValue(val interface{}, typ FieldType, schema Schema) (Value, error) {
  716. switch val := val.(type) {
  717. case nil:
  718. return nil, nil
  719. case []interface{}:
  720. return convertRepeatedRecord(val, typ, schema)
  721. case map[string]interface{}:
  722. return convertNestedRecord(val, schema)
  723. case string:
  724. return convertBasicType(val, typ)
  725. default:
  726. return nil, fmt.Errorf("got value %v; expected a value of type %s", val, typ)
  727. }
  728. }
  729. func convertRepeatedRecord(vals []interface{}, typ FieldType, schema Schema) (Value, error) {
  730. var values []Value
  731. for _, cell := range vals {
  732. // each cell contains a single entry, keyed by "v"
  733. val := cell.(map[string]interface{})["v"]
  734. v, err := convertValue(val, typ, schema)
  735. if err != nil {
  736. return nil, err
  737. }
  738. values = append(values, v)
  739. }
  740. return values, nil
  741. }
  742. func convertNestedRecord(val map[string]interface{}, schema Schema) (Value, error) {
  743. // convertNestedRecord is similar to convertRow, as a record has the same structure as a row.
  744. // Nested records are wrapped in a map with a single key, "f".
  745. record := val["f"].([]interface{})
  746. if len(record) != len(schema) {
  747. return nil, errors.New("schema length does not match record length")
  748. }
  749. var values []Value
  750. for i, cell := range record {
  751. // each cell contains a single entry, keyed by "v"
  752. val := cell.(map[string]interface{})["v"]
  753. fs := schema[i]
  754. v, err := convertValue(val, fs.Type, fs.Schema)
  755. if err != nil {
  756. return nil, err
  757. }
  758. values = append(values, v)
  759. }
  760. return values, nil
  761. }
  762. // convertBasicType returns val as an interface with a concrete type specified by typ.
  763. func convertBasicType(val string, typ FieldType) (Value, error) {
  764. switch typ {
  765. case StringFieldType:
  766. return val, nil
  767. case BytesFieldType:
  768. return base64.StdEncoding.DecodeString(val)
  769. case IntegerFieldType:
  770. return strconv.ParseInt(val, 10, 64)
  771. case FloatFieldType:
  772. return strconv.ParseFloat(val, 64)
  773. case BooleanFieldType:
  774. return strconv.ParseBool(val)
  775. case TimestampFieldType:
  776. f, err := strconv.ParseFloat(val, 64)
  777. if err != nil {
  778. return nil, err
  779. }
  780. secs := math.Trunc(f)
  781. nanos := (f - secs) * 1e9
  782. return Value(time.Unix(int64(secs), int64(nanos)).UTC()), nil
  783. case DateFieldType:
  784. return civil.ParseDate(val)
  785. case TimeFieldType:
  786. return civil.ParseTime(val)
  787. case DateTimeFieldType:
  788. return civil.ParseDateTime(val)
  789. case NumericFieldType:
  790. r, ok := (&big.Rat{}).SetString(val)
  791. if !ok {
  792. return nil, fmt.Errorf("bigquery: invalid NUMERIC value %q", val)
  793. }
  794. return Value(r), nil
  795. default:
  796. return nil, fmt.Errorf("unrecognized type: %s", typ)
  797. }
  798. }