You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

893 lines
24 KiB

  1. // Copyright 2015 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package bigquery
  15. import (
  16. "encoding/base64"
  17. "errors"
  18. "fmt"
  19. "math"
  20. "math/big"
  21. "reflect"
  22. "strconv"
  23. "strings"
  24. "time"
  25. "cloud.google.com/go/civil"
  26. bq "google.golang.org/api/bigquery/v2"
  27. )
  28. // Value stores the contents of a single cell from a BigQuery result.
  29. type Value interface{}
  30. // ValueLoader stores a slice of Values representing a result row from a Read operation.
  31. // See RowIterator.Next for more information.
  32. type ValueLoader interface {
  33. Load(v []Value, s Schema) error
  34. }
  35. // valueList converts a []Value to implement ValueLoader.
  36. type valueList []Value
  37. // Load stores a sequence of values in a valueList.
  38. // It resets the slice length to zero, then appends each value to it.
  39. func (vs *valueList) Load(v []Value, _ Schema) error {
  40. *vs = append((*vs)[:0], v...)
  41. return nil
  42. }
  43. // valueMap converts a map[string]Value to implement ValueLoader.
  44. type valueMap map[string]Value
  45. // Load stores a sequence of values in a valueMap.
  46. func (vm *valueMap) Load(v []Value, s Schema) error {
  47. if *vm == nil {
  48. *vm = map[string]Value{}
  49. }
  50. loadMap(*vm, v, s)
  51. return nil
  52. }
  53. func loadMap(m map[string]Value, vals []Value, s Schema) {
  54. for i, f := range s {
  55. val := vals[i]
  56. var v interface{}
  57. switch {
  58. case val == nil:
  59. v = val
  60. case f.Schema == nil:
  61. v = val
  62. case !f.Repeated:
  63. m2 := map[string]Value{}
  64. loadMap(m2, val.([]Value), f.Schema)
  65. v = m2
  66. default: // repeated and nested
  67. sval := val.([]Value)
  68. vs := make([]Value, len(sval))
  69. for j, e := range sval {
  70. m2 := map[string]Value{}
  71. loadMap(m2, e.([]Value), f.Schema)
  72. vs[j] = m2
  73. }
  74. v = vs
  75. }
  76. m[f.Name] = v
  77. }
  78. }
  79. type structLoader struct {
  80. typ reflect.Type // type of struct
  81. err error
  82. ops []structLoaderOp
  83. vstructp reflect.Value // pointer to current struct value; changed by set
  84. }
  85. // A setFunc is a function that sets a struct field or slice/array
  86. // element to a value.
  87. type setFunc func(v reflect.Value, val interface{}) error
  88. // A structLoaderOp instructs the loader to set a struct field to a row value.
  89. type structLoaderOp struct {
  90. fieldIndex []int
  91. valueIndex int
  92. setFunc setFunc
  93. repeated bool
  94. }
  95. var errNoNulls = errors.New("bigquery: NULL values cannot be read into structs")
  96. func setAny(v reflect.Value, x interface{}) error {
  97. if x == nil {
  98. return errNoNulls
  99. }
  100. v.Set(reflect.ValueOf(x))
  101. return nil
  102. }
  103. func setInt(v reflect.Value, x interface{}) error {
  104. if x == nil {
  105. return errNoNulls
  106. }
  107. xx := x.(int64)
  108. if v.OverflowInt(xx) {
  109. return fmt.Errorf("bigquery: value %v overflows struct field of type %v", xx, v.Type())
  110. }
  111. v.SetInt(xx)
  112. return nil
  113. }
  114. func setUint(v reflect.Value, x interface{}) error {
  115. if x == nil {
  116. return errNoNulls
  117. }
  118. xx := x.(int64)
  119. if xx < 0 || v.OverflowUint(uint64(xx)) {
  120. return fmt.Errorf("bigquery: value %v overflows struct field of type %v", xx, v.Type())
  121. }
  122. v.SetUint(uint64(xx))
  123. return nil
  124. }
  125. func setFloat(v reflect.Value, x interface{}) error {
  126. if x == nil {
  127. return errNoNulls
  128. }
  129. xx := x.(float64)
  130. if v.OverflowFloat(xx) {
  131. return fmt.Errorf("bigquery: value %v overflows struct field of type %v", xx, v.Type())
  132. }
  133. v.SetFloat(xx)
  134. return nil
  135. }
  136. func setBool(v reflect.Value, x interface{}) error {
  137. if x == nil {
  138. return errNoNulls
  139. }
  140. v.SetBool(x.(bool))
  141. return nil
  142. }
  143. func setString(v reflect.Value, x interface{}) error {
  144. if x == nil {
  145. return errNoNulls
  146. }
  147. v.SetString(x.(string))
  148. return nil
  149. }
  150. func setGeography(v reflect.Value, x interface{}) error {
  151. if x == nil {
  152. return errNoNulls
  153. }
  154. v.SetString(x.(string))
  155. return nil
  156. }
  157. func setBytes(v reflect.Value, x interface{}) error {
  158. if x == nil {
  159. v.SetBytes(nil)
  160. } else {
  161. v.SetBytes(x.([]byte))
  162. }
  163. return nil
  164. }
  165. func setNull(v reflect.Value, x interface{}, build func() interface{}) error {
  166. if x == nil {
  167. v.Set(reflect.Zero(v.Type()))
  168. } else {
  169. n := build()
  170. v.Set(reflect.ValueOf(n))
  171. }
  172. return nil
  173. }
  174. // set remembers a value for the next call to Load. The value must be
  175. // a pointer to a struct. (This is checked in RowIterator.Next.)
  176. func (sl *structLoader) set(structp interface{}, schema Schema) error {
  177. if sl.err != nil {
  178. return sl.err
  179. }
  180. sl.vstructp = reflect.ValueOf(structp)
  181. typ := sl.vstructp.Type().Elem()
  182. if sl.typ == nil {
  183. // First call: remember the type and compile the schema.
  184. sl.typ = typ
  185. ops, err := compileToOps(typ, schema)
  186. if err != nil {
  187. sl.err = err
  188. return err
  189. }
  190. sl.ops = ops
  191. } else if sl.typ != typ {
  192. return fmt.Errorf("bigquery: struct type changed from %s to %s", sl.typ, typ)
  193. }
  194. return nil
  195. }
  196. // compileToOps produces a sequence of operations that will set the fields of a
  197. // value of structType to the contents of a row with schema.
  198. func compileToOps(structType reflect.Type, schema Schema) ([]structLoaderOp, error) {
  199. var ops []structLoaderOp
  200. fields, err := fieldCache.Fields(structType)
  201. if err != nil {
  202. return nil, err
  203. }
  204. for i, schemaField := range schema {
  205. // Look for an exported struct field with the same name as the schema
  206. // field, ignoring case (BigQuery column names are case-insensitive,
  207. // and we want to act like encoding/json anyway).
  208. structField := fields.Match(schemaField.Name)
  209. if structField == nil {
  210. // Ignore schema fields with no corresponding struct field.
  211. continue
  212. }
  213. op := structLoaderOp{
  214. fieldIndex: structField.Index,
  215. valueIndex: i,
  216. }
  217. t := structField.Type
  218. if schemaField.Repeated {
  219. if t.Kind() != reflect.Slice && t.Kind() != reflect.Array {
  220. return nil, fmt.Errorf("bigquery: repeated schema field %s requires slice or array, but struct field %s has type %s",
  221. schemaField.Name, structField.Name, t)
  222. }
  223. t = t.Elem()
  224. op.repeated = true
  225. }
  226. if schemaField.Type == RecordFieldType {
  227. // Field can be a struct or a pointer to a struct.
  228. if t.Kind() == reflect.Ptr {
  229. t = t.Elem()
  230. }
  231. if t.Kind() != reflect.Struct {
  232. return nil, fmt.Errorf("bigquery: field %s has type %s, expected struct or *struct",
  233. structField.Name, structField.Type)
  234. }
  235. nested, err := compileToOps(t, schemaField.Schema)
  236. if err != nil {
  237. return nil, err
  238. }
  239. op.setFunc = func(v reflect.Value, val interface{}) error {
  240. return setNested(nested, v, val)
  241. }
  242. } else {
  243. op.setFunc = determineSetFunc(t, schemaField.Type)
  244. if op.setFunc == nil {
  245. return nil, fmt.Errorf("bigquery: schema field %s of type %s is not assignable to struct field %s of type %s",
  246. schemaField.Name, schemaField.Type, structField.Name, t)
  247. }
  248. }
  249. ops = append(ops, op)
  250. }
  251. return ops, nil
  252. }
  253. // determineSetFunc chooses the best function for setting a field of type ftype
  254. // to a value whose schema field type is stype. It returns nil if stype
  255. // is not assignable to ftype.
  256. // determineSetFunc considers only basic types. See compileToOps for
  257. // handling of repetition and nesting.
  258. func determineSetFunc(ftype reflect.Type, stype FieldType) setFunc {
  259. switch stype {
  260. case StringFieldType:
  261. if ftype.Kind() == reflect.String {
  262. return setString
  263. }
  264. if ftype == typeOfNullString {
  265. return func(v reflect.Value, x interface{}) error {
  266. return setNull(v, x, func() interface{} {
  267. return NullString{StringVal: x.(string), Valid: true}
  268. })
  269. }
  270. }
  271. case GeographyFieldType:
  272. if ftype.Kind() == reflect.String {
  273. return setGeography
  274. }
  275. if ftype == typeOfNullGeography {
  276. return func(v reflect.Value, x interface{}) error {
  277. return setNull(v, x, func() interface{} {
  278. return NullGeography{GeographyVal: x.(string), Valid: true}
  279. })
  280. }
  281. }
  282. case BytesFieldType:
  283. if ftype == typeOfByteSlice {
  284. return setBytes
  285. }
  286. case IntegerFieldType:
  287. if isSupportedUintType(ftype) {
  288. return setUint
  289. } else if isSupportedIntType(ftype) {
  290. return setInt
  291. }
  292. if ftype == typeOfNullInt64 {
  293. return func(v reflect.Value, x interface{}) error {
  294. return setNull(v, x, func() interface{} {
  295. return NullInt64{Int64: x.(int64), Valid: true}
  296. })
  297. }
  298. }
  299. case FloatFieldType:
  300. switch ftype.Kind() {
  301. case reflect.Float32, reflect.Float64:
  302. return setFloat
  303. }
  304. if ftype == typeOfNullFloat64 {
  305. return func(v reflect.Value, x interface{}) error {
  306. return setNull(v, x, func() interface{} {
  307. return NullFloat64{Float64: x.(float64), Valid: true}
  308. })
  309. }
  310. }
  311. case BooleanFieldType:
  312. if ftype.Kind() == reflect.Bool {
  313. return setBool
  314. }
  315. if ftype == typeOfNullBool {
  316. return func(v reflect.Value, x interface{}) error {
  317. return setNull(v, x, func() interface{} {
  318. return NullBool{Bool: x.(bool), Valid: true}
  319. })
  320. }
  321. }
  322. case TimestampFieldType:
  323. if ftype == typeOfGoTime {
  324. return setAny
  325. }
  326. if ftype == typeOfNullTimestamp {
  327. return func(v reflect.Value, x interface{}) error {
  328. return setNull(v, x, func() interface{} {
  329. return NullTimestamp{Timestamp: x.(time.Time), Valid: true}
  330. })
  331. }
  332. }
  333. case DateFieldType:
  334. if ftype == typeOfDate {
  335. return setAny
  336. }
  337. if ftype == typeOfNullDate {
  338. return func(v reflect.Value, x interface{}) error {
  339. return setNull(v, x, func() interface{} {
  340. return NullDate{Date: x.(civil.Date), Valid: true}
  341. })
  342. }
  343. }
  344. case TimeFieldType:
  345. if ftype == typeOfTime {
  346. return setAny
  347. }
  348. if ftype == typeOfNullTime {
  349. return func(v reflect.Value, x interface{}) error {
  350. return setNull(v, x, func() interface{} {
  351. return NullTime{Time: x.(civil.Time), Valid: true}
  352. })
  353. }
  354. }
  355. case DateTimeFieldType:
  356. if ftype == typeOfDateTime {
  357. return setAny
  358. }
  359. if ftype == typeOfNullDateTime {
  360. return func(v reflect.Value, x interface{}) error {
  361. return setNull(v, x, func() interface{} {
  362. return NullDateTime{DateTime: x.(civil.DateTime), Valid: true}
  363. })
  364. }
  365. }
  366. case NumericFieldType:
  367. if ftype == typeOfRat {
  368. return func(v reflect.Value, x interface{}) error {
  369. return setNull(v, x, func() interface{} { return x.(*big.Rat) })
  370. }
  371. }
  372. }
  373. return nil
  374. }
  375. func (sl *structLoader) Load(values []Value, _ Schema) error {
  376. if sl.err != nil {
  377. return sl.err
  378. }
  379. return runOps(sl.ops, sl.vstructp.Elem(), values)
  380. }
  381. // runOps executes a sequence of ops, setting the fields of vstruct to the
  382. // supplied values.
  383. func runOps(ops []structLoaderOp, vstruct reflect.Value, values []Value) error {
  384. for _, op := range ops {
  385. field := vstruct.FieldByIndex(op.fieldIndex)
  386. var err error
  387. if op.repeated {
  388. err = setRepeated(field, values[op.valueIndex].([]Value), op.setFunc)
  389. } else {
  390. err = op.setFunc(field, values[op.valueIndex])
  391. }
  392. if err != nil {
  393. return err
  394. }
  395. }
  396. return nil
  397. }
  398. func setNested(ops []structLoaderOp, v reflect.Value, val interface{}) error {
  399. // v is either a struct or a pointer to a struct.
  400. if v.Kind() == reflect.Ptr {
  401. // If the value is nil, set the pointer to nil.
  402. if val == nil {
  403. v.Set(reflect.Zero(v.Type()))
  404. return nil
  405. }
  406. // If the pointer is nil, set it to a zero struct value.
  407. if v.IsNil() {
  408. v.Set(reflect.New(v.Type().Elem()))
  409. }
  410. v = v.Elem()
  411. }
  412. return runOps(ops, v, val.([]Value))
  413. }
  414. func setRepeated(field reflect.Value, vslice []Value, setElem setFunc) error {
  415. vlen := len(vslice)
  416. var flen int
  417. switch field.Type().Kind() {
  418. case reflect.Slice:
  419. // Make a slice of the right size, avoiding allocation if possible.
  420. switch {
  421. case field.Len() < vlen:
  422. field.Set(reflect.MakeSlice(field.Type(), vlen, vlen))
  423. case field.Len() > vlen:
  424. field.SetLen(vlen)
  425. }
  426. flen = vlen
  427. case reflect.Array:
  428. flen = field.Len()
  429. if flen > vlen {
  430. // Set extra elements to their zero value.
  431. z := reflect.Zero(field.Type().Elem())
  432. for i := vlen; i < flen; i++ {
  433. field.Index(i).Set(z)
  434. }
  435. }
  436. default:
  437. return fmt.Errorf("bigquery: impossible field type %s", field.Type())
  438. }
  439. for i, val := range vslice {
  440. if i < flen { // avoid writing past the end of a short array
  441. if err := setElem(field.Index(i), val); err != nil {
  442. return err
  443. }
  444. }
  445. }
  446. return nil
  447. }
  448. // A ValueSaver returns a row of data to be inserted into a table.
  449. type ValueSaver interface {
  450. // Save returns a row to be inserted into a BigQuery table, represented
  451. // as a map from field name to Value.
  452. // If insertID is non-empty, BigQuery will use it to de-duplicate
  453. // insertions of this row on a best-effort basis.
  454. Save() (row map[string]Value, insertID string, err error)
  455. }
  456. // ValuesSaver implements ValueSaver for a slice of Values.
  457. type ValuesSaver struct {
  458. Schema Schema
  459. // If non-empty, BigQuery will use InsertID to de-duplicate insertions
  460. // of this row on a best-effort basis.
  461. InsertID string
  462. Row []Value
  463. }
  464. // Save implements ValueSaver.
  465. func (vls *ValuesSaver) Save() (map[string]Value, string, error) {
  466. m, err := valuesToMap(vls.Row, vls.Schema)
  467. return m, vls.InsertID, err
  468. }
  469. func valuesToMap(vs []Value, schema Schema) (map[string]Value, error) {
  470. if len(vs) != len(schema) {
  471. return nil, errors.New("schema does not match length of row to be inserted")
  472. }
  473. m := make(map[string]Value)
  474. for i, fieldSchema := range schema {
  475. if vs[i] == nil {
  476. m[fieldSchema.Name] = nil
  477. continue
  478. }
  479. if fieldSchema.Type != RecordFieldType {
  480. m[fieldSchema.Name] = toUploadValue(vs[i], fieldSchema)
  481. continue
  482. }
  483. // Nested record, possibly repeated.
  484. vals, ok := vs[i].([]Value)
  485. if !ok {
  486. return nil, errors.New("nested record is not a []Value")
  487. }
  488. if !fieldSchema.Repeated {
  489. value, err := valuesToMap(vals, fieldSchema.Schema)
  490. if err != nil {
  491. return nil, err
  492. }
  493. m[fieldSchema.Name] = value
  494. continue
  495. }
  496. // A repeated nested field is converted into a slice of maps.
  497. var maps []Value
  498. for _, v := range vals {
  499. sv, ok := v.([]Value)
  500. if !ok {
  501. return nil, errors.New("nested record in slice is not a []Value")
  502. }
  503. value, err := valuesToMap(sv, fieldSchema.Schema)
  504. if err != nil {
  505. return nil, err
  506. }
  507. maps = append(maps, value)
  508. }
  509. m[fieldSchema.Name] = maps
  510. }
  511. return m, nil
  512. }
  513. // StructSaver implements ValueSaver for a struct.
  514. // The struct is converted to a map of values by using the values of struct
  515. // fields corresponding to schema fields. Additional and missing
  516. // fields are ignored, as are nested struct pointers that are nil.
  517. type StructSaver struct {
  518. // Schema determines what fields of the struct are uploaded. It should
  519. // match the table's schema.
  520. // Schema is optional for StructSavers that are passed to Uploader.Put.
  521. Schema Schema
  522. // If non-empty, BigQuery will use InsertID to de-duplicate insertions
  523. // of this row on a best-effort basis.
  524. InsertID string
  525. // Struct should be a struct or a pointer to a struct.
  526. Struct interface{}
  527. }
  528. // Save implements ValueSaver.
  529. func (ss *StructSaver) Save() (row map[string]Value, insertID string, err error) {
  530. vstruct := reflect.ValueOf(ss.Struct)
  531. row, err = structToMap(vstruct, ss.Schema)
  532. if err != nil {
  533. return nil, "", err
  534. }
  535. return row, ss.InsertID, nil
  536. }
  537. func structToMap(vstruct reflect.Value, schema Schema) (map[string]Value, error) {
  538. if vstruct.Kind() == reflect.Ptr {
  539. vstruct = vstruct.Elem()
  540. }
  541. if !vstruct.IsValid() {
  542. return nil, nil
  543. }
  544. m := map[string]Value{}
  545. if vstruct.Kind() != reflect.Struct {
  546. return nil, fmt.Errorf("bigquery: type is %s, need struct or struct pointer", vstruct.Type())
  547. }
  548. fields, err := fieldCache.Fields(vstruct.Type())
  549. if err != nil {
  550. return nil, err
  551. }
  552. for _, schemaField := range schema {
  553. // Look for an exported struct field with the same name as the schema
  554. // field, ignoring case.
  555. structField := fields.Match(schemaField.Name)
  556. if structField == nil {
  557. continue
  558. }
  559. val, err := structFieldToUploadValue(vstruct.FieldByIndex(structField.Index), schemaField)
  560. if err != nil {
  561. return nil, err
  562. }
  563. // Add the value to the map, unless it is nil.
  564. if val != nil {
  565. m[schemaField.Name] = val
  566. }
  567. }
  568. return m, nil
  569. }
  570. // structFieldToUploadValue converts a struct field to a value suitable for ValueSaver.Save, using
  571. // the schemaField as a guide.
  572. // structFieldToUploadValue is careful to return a true nil interface{} when needed, so its
  573. // caller can easily identify a nil value.
  574. func structFieldToUploadValue(vfield reflect.Value, schemaField *FieldSchema) (interface{}, error) {
  575. if schemaField.Repeated && (vfield.Kind() != reflect.Slice && vfield.Kind() != reflect.Array) {
  576. return nil, fmt.Errorf("bigquery: repeated schema field %s requires slice or array, but value has type %s",
  577. schemaField.Name, vfield.Type())
  578. }
  579. // A non-nested field can be represented by its Go value, except for some types.
  580. if schemaField.Type != RecordFieldType {
  581. return toUploadValueReflect(vfield, schemaField), nil
  582. }
  583. // A non-repeated nested field is converted into a map[string]Value.
  584. if !schemaField.Repeated {
  585. m, err := structToMap(vfield, schemaField.Schema)
  586. if err != nil {
  587. return nil, err
  588. }
  589. if m == nil {
  590. return nil, nil
  591. }
  592. return m, nil
  593. }
  594. // A repeated nested field is converted into a slice of maps.
  595. // If the field is zero-length (but not nil), we return a zero-length []Value.
  596. if vfield.IsNil() {
  597. return nil, nil
  598. }
  599. vals := []Value{}
  600. for i := 0; i < vfield.Len(); i++ {
  601. m, err := structToMap(vfield.Index(i), schemaField.Schema)
  602. if err != nil {
  603. return nil, err
  604. }
  605. vals = append(vals, m)
  606. }
  607. return vals, nil
  608. }
  609. func toUploadValue(val interface{}, fs *FieldSchema) interface{} {
  610. if fs.Type == TimeFieldType || fs.Type == DateTimeFieldType || fs.Type == NumericFieldType {
  611. return toUploadValueReflect(reflect.ValueOf(val), fs)
  612. }
  613. return val
  614. }
  615. func toUploadValueReflect(v reflect.Value, fs *FieldSchema) interface{} {
  616. switch fs.Type {
  617. case TimeFieldType:
  618. if v.Type() == typeOfNullTime {
  619. return v.Interface()
  620. }
  621. return formatUploadValue(v, fs, func(v reflect.Value) string {
  622. return CivilTimeString(v.Interface().(civil.Time))
  623. })
  624. case DateTimeFieldType:
  625. if v.Type() == typeOfNullDateTime {
  626. return v.Interface()
  627. }
  628. return formatUploadValue(v, fs, func(v reflect.Value) string {
  629. return CivilDateTimeString(v.Interface().(civil.DateTime))
  630. })
  631. case NumericFieldType:
  632. if r, ok := v.Interface().(*big.Rat); ok && r == nil {
  633. return nil
  634. }
  635. return formatUploadValue(v, fs, func(v reflect.Value) string {
  636. return NumericString(v.Interface().(*big.Rat))
  637. })
  638. default:
  639. if !fs.Repeated || v.Len() > 0 {
  640. return v.Interface()
  641. }
  642. // The service treats a null repeated field as an error. Return
  643. // nil to omit the field entirely.
  644. return nil
  645. }
  646. }
  647. func formatUploadValue(v reflect.Value, fs *FieldSchema, cvt func(reflect.Value) string) interface{} {
  648. if !fs.Repeated {
  649. return cvt(v)
  650. }
  651. if v.Len() == 0 {
  652. return nil
  653. }
  654. s := make([]string, v.Len())
  655. for i := 0; i < v.Len(); i++ {
  656. s[i] = cvt(v.Index(i))
  657. }
  658. return s
  659. }
  660. // CivilTimeString returns a string representing a civil.Time in a format compatible
  661. // with BigQuery SQL. It rounds the time to the nearest microsecond and returns a
  662. // string with six digits of sub-second precision.
  663. //
  664. // Use CivilTimeString when using civil.Time in DML, for example in INSERT
  665. // statements.
  666. func CivilTimeString(t civil.Time) string {
  667. if t.Nanosecond == 0 {
  668. return t.String()
  669. }
  670. micro := (t.Nanosecond + 500) / 1000 // round to nearest microsecond
  671. t.Nanosecond = 0
  672. return t.String() + fmt.Sprintf(".%06d", micro)
  673. }
  674. // CivilDateTimeString returns a string representing a civil.DateTime in a format compatible
  675. // with BigQuery SQL. It separate the date and time with a space, and formats the time
  676. // with CivilTimeString.
  677. //
  678. // Use CivilDateTimeString when using civil.DateTime in DML, for example in INSERT
  679. // statements.
  680. func CivilDateTimeString(dt civil.DateTime) string {
  681. return dt.Date.String() + " " + CivilTimeString(dt.Time)
  682. }
  683. // parseCivilDateTime parses a date-time represented in a BigQuery SQL
  684. // compatible format and returns a civil.DateTime.
  685. func parseCivilDateTime(s string) (civil.DateTime, error) {
  686. parts := strings.Fields(s)
  687. if len(parts) != 2 {
  688. return civil.DateTime{}, fmt.Errorf("bigquery: bad DATETIME value %q", s)
  689. }
  690. return civil.ParseDateTime(parts[0] + "T" + parts[1])
  691. }
  692. const (
  693. // NumericPrecisionDigits is the maximum number of digits in a NUMERIC value.
  694. NumericPrecisionDigits = 38
  695. // NumericScaleDigits is the maximum number of digits after the decimal point in a NUMERIC value.
  696. NumericScaleDigits = 9
  697. )
  698. // NumericString returns a string representing a *big.Rat in a format compatible
  699. // with BigQuery SQL. It returns a floating-point literal with 9 digits
  700. // after the decimal point.
  701. func NumericString(r *big.Rat) string {
  702. return r.FloatString(NumericScaleDigits)
  703. }
  704. // convertRows converts a series of TableRows into a series of Value slices.
  705. // schema is used to interpret the data from rows; its length must match the
  706. // length of each row.
  707. func convertRows(rows []*bq.TableRow, schema Schema) ([][]Value, error) {
  708. var rs [][]Value
  709. for _, r := range rows {
  710. row, err := convertRow(r, schema)
  711. if err != nil {
  712. return nil, err
  713. }
  714. rs = append(rs, row)
  715. }
  716. return rs, nil
  717. }
  718. func convertRow(r *bq.TableRow, schema Schema) ([]Value, error) {
  719. if len(schema) != len(r.F) {
  720. return nil, errors.New("schema length does not match row length")
  721. }
  722. var values []Value
  723. for i, cell := range r.F {
  724. fs := schema[i]
  725. v, err := convertValue(cell.V, fs.Type, fs.Schema)
  726. if err != nil {
  727. return nil, err
  728. }
  729. values = append(values, v)
  730. }
  731. return values, nil
  732. }
  733. func convertValue(val interface{}, typ FieldType, schema Schema) (Value, error) {
  734. switch val := val.(type) {
  735. case nil:
  736. return nil, nil
  737. case []interface{}:
  738. return convertRepeatedRecord(val, typ, schema)
  739. case map[string]interface{}:
  740. return convertNestedRecord(val, schema)
  741. case string:
  742. return convertBasicType(val, typ)
  743. default:
  744. return nil, fmt.Errorf("got value %v; expected a value of type %s", val, typ)
  745. }
  746. }
  747. func convertRepeatedRecord(vals []interface{}, typ FieldType, schema Schema) (Value, error) {
  748. var values []Value
  749. for _, cell := range vals {
  750. // each cell contains a single entry, keyed by "v"
  751. val := cell.(map[string]interface{})["v"]
  752. v, err := convertValue(val, typ, schema)
  753. if err != nil {
  754. return nil, err
  755. }
  756. values = append(values, v)
  757. }
  758. return values, nil
  759. }
  760. func convertNestedRecord(val map[string]interface{}, schema Schema) (Value, error) {
  761. // convertNestedRecord is similar to convertRow, as a record has the same structure as a row.
  762. // Nested records are wrapped in a map with a single key, "f".
  763. record := val["f"].([]interface{})
  764. if len(record) != len(schema) {
  765. return nil, errors.New("schema length does not match record length")
  766. }
  767. var values []Value
  768. for i, cell := range record {
  769. // each cell contains a single entry, keyed by "v"
  770. val := cell.(map[string]interface{})["v"]
  771. fs := schema[i]
  772. v, err := convertValue(val, fs.Type, fs.Schema)
  773. if err != nil {
  774. return nil, err
  775. }
  776. values = append(values, v)
  777. }
  778. return values, nil
  779. }
  780. // convertBasicType returns val as an interface with a concrete type specified by typ.
  781. func convertBasicType(val string, typ FieldType) (Value, error) {
  782. switch typ {
  783. case StringFieldType:
  784. return val, nil
  785. case BytesFieldType:
  786. return base64.StdEncoding.DecodeString(val)
  787. case IntegerFieldType:
  788. return strconv.ParseInt(val, 10, 64)
  789. case FloatFieldType:
  790. return strconv.ParseFloat(val, 64)
  791. case BooleanFieldType:
  792. return strconv.ParseBool(val)
  793. case TimestampFieldType:
  794. f, err := strconv.ParseFloat(val, 64)
  795. if err != nil {
  796. return nil, err
  797. }
  798. secs := math.Trunc(f)
  799. nanos := (f - secs) * 1e9
  800. return Value(time.Unix(int64(secs), int64(nanos)).UTC()), nil
  801. case DateFieldType:
  802. return civil.ParseDate(val)
  803. case TimeFieldType:
  804. return civil.ParseTime(val)
  805. case DateTimeFieldType:
  806. return civil.ParseDateTime(val)
  807. case NumericFieldType:
  808. r, ok := (&big.Rat{}).SetString(val)
  809. if !ok {
  810. return nil, fmt.Errorf("bigquery: invalid NUMERIC value %q", val)
  811. }
  812. return Value(r), nil
  813. case GeographyFieldType:
  814. return val, nil
  815. default:
  816. return nil, fmt.Errorf("unrecognized type: %s", typ)
  817. }
  818. }