You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

847 lines
29 KiB

  1. // Copyright 2016 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // API/gRPC features intentionally missing from this client:
  15. // - You cannot have the server pick the time of the entry. This client
  16. // always sends a time.
  17. // - There is no way to provide a protocol buffer payload.
  18. // - No support for the "partial success" feature when writing log entries.
  19. // TODO(jba): test whether forward-slash characters in the log ID must be URL-encoded.
  20. // These features are missing now, but will likely be added:
  21. // - There is no way to specify CallOptions.
  22. package logging
  23. import (
  24. "context"
  25. "encoding/json"
  26. "errors"
  27. "fmt"
  28. "log"
  29. "net/http"
  30. "strconv"
  31. "strings"
  32. "sync"
  33. "time"
  34. "cloud.google.com/go/compute/metadata"
  35. "cloud.google.com/go/internal/version"
  36. vkit "cloud.google.com/go/logging/apiv2"
  37. "cloud.google.com/go/logging/internal"
  38. "github.com/golang/protobuf/proto"
  39. "github.com/golang/protobuf/ptypes"
  40. structpb "github.com/golang/protobuf/ptypes/struct"
  41. tspb "github.com/golang/protobuf/ptypes/timestamp"
  42. "google.golang.org/api/option"
  43. "google.golang.org/api/support/bundler"
  44. mrpb "google.golang.org/genproto/googleapis/api/monitoredres"
  45. logtypepb "google.golang.org/genproto/googleapis/logging/type"
  46. logpb "google.golang.org/genproto/googleapis/logging/v2"
  47. )
  48. const (
  49. // ReadScope is the scope for reading from the logging service.
  50. ReadScope = "https://www.googleapis.com/auth/logging.read"
  51. // WriteScope is the scope for writing to the logging service.
  52. WriteScope = "https://www.googleapis.com/auth/logging.write"
  53. // AdminScope is the scope for administrative actions on the logging service.
  54. AdminScope = "https://www.googleapis.com/auth/logging.admin"
  55. )
  56. const (
  57. // defaultErrorCapacity is the capacity of the channel used to deliver
  58. // errors to the OnError function.
  59. defaultErrorCapacity = 10
  60. // DefaultDelayThreshold is the default value for the DelayThreshold LoggerOption.
  61. DefaultDelayThreshold = time.Second
  62. // DefaultEntryCountThreshold is the default value for the EntryCountThreshold LoggerOption.
  63. DefaultEntryCountThreshold = 1000
  64. // DefaultEntryByteThreshold is the default value for the EntryByteThreshold LoggerOption.
  65. DefaultEntryByteThreshold = 1 << 20 // 1MiB
  66. // DefaultBufferedByteLimit is the default value for the BufferedByteLimit LoggerOption.
  67. DefaultBufferedByteLimit = 1 << 30 // 1GiB
  68. // defaultWriteTimeout is the timeout for the underlying write API calls. As
  69. // write API calls are not idempotent, they are not retried on timeout. This
  70. // timeout is to allow clients to degrade gracefully if underlying logging
  71. // service is temporarily impaired for some reason.
  72. defaultWriteTimeout = 10 * time.Minute
  73. )
  74. // For testing:
  75. var now = time.Now
  76. // ErrOverflow signals that the number of buffered entries for a Logger
  77. // exceeds its BufferLimit.
  78. var ErrOverflow = bundler.ErrOverflow
  79. // ErrOversizedEntry signals that an entry's size exceeds the maximum number of
  80. // bytes that will be sent in a single call to the logging service.
  81. var ErrOversizedEntry = bundler.ErrOversizedItem
  82. // Client is a Logging client. A Client is associated with a single Cloud project.
  83. type Client struct {
  84. client *vkit.Client // client for the logging service
  85. parent string // e.g. "projects/proj-id"
  86. errc chan error // should be buffered to minimize dropped errors
  87. donec chan struct{} // closed on Client.Close to close Logger bundlers
  88. loggers sync.WaitGroup // so we can wait for loggers to close
  89. closed bool
  90. mu sync.Mutex
  91. nErrs int // number of errors we saw
  92. lastErr error // last error we saw
  93. // OnError is called when an error occurs in a call to Log or Flush. The
  94. // error may be due to an invalid Entry, an overflow because BufferLimit
  95. // was reached (in which case the error will be ErrOverflow) or an error
  96. // communicating with the logging service. OnError is called with errors
  97. // from all Loggers. It is never called concurrently. OnError is expected
  98. // to return quickly; if errors occur while OnError is running, some may
  99. // not be reported. The default behavior is to call log.Printf.
  100. //
  101. // This field should be set only once, before any method of Client is called.
  102. OnError func(err error)
  103. }
  104. // NewClient returns a new logging client associated with the provided parent.
  105. // A parent can take any of the following forms:
  106. // projects/PROJECT_ID
  107. // folders/FOLDER_ID
  108. // billingAccounts/ACCOUNT_ID
  109. // organizations/ORG_ID
  110. // for backwards compatibility, a string with no '/' is also allowed and is interpreted
  111. // as a project ID.
  112. //
  113. // By default NewClient uses WriteScope. To use a different scope, call
  114. // NewClient using a WithScopes option (see https://godoc.org/google.golang.org/api/option#WithScopes).
  115. func NewClient(ctx context.Context, parent string, opts ...option.ClientOption) (*Client, error) {
  116. if !strings.ContainsRune(parent, '/') {
  117. parent = "projects/" + parent
  118. }
  119. opts = append([]option.ClientOption{
  120. option.WithEndpoint(internal.ProdAddr),
  121. option.WithScopes(WriteScope),
  122. }, opts...)
  123. c, err := vkit.NewClient(ctx, opts...)
  124. if err != nil {
  125. return nil, err
  126. }
  127. c.SetGoogleClientInfo("gccl", version.Repo)
  128. client := &Client{
  129. client: c,
  130. parent: parent,
  131. errc: make(chan error, defaultErrorCapacity), // create a small buffer for errors
  132. donec: make(chan struct{}),
  133. OnError: func(e error) { log.Printf("logging client: %v", e) },
  134. }
  135. // Call the user's function synchronously, to make life easier for them.
  136. go func() {
  137. for err := range client.errc {
  138. // This reference to OnError is memory-safe if the user sets OnError before
  139. // calling any client methods. The reference happens before the first read from
  140. // client.errc, which happens before the first write to client.errc, which
  141. // happens before any call, which happens before the user sets OnError.
  142. if fn := client.OnError; fn != nil {
  143. fn(err)
  144. } else {
  145. log.Printf("logging (parent %q): %v", parent, err)
  146. }
  147. }
  148. }()
  149. return client, nil
  150. }
  151. var unixZeroTimestamp *tspb.Timestamp
  152. func init() {
  153. var err error
  154. unixZeroTimestamp, err = ptypes.TimestampProto(time.Unix(0, 0))
  155. if err != nil {
  156. panic(err)
  157. }
  158. }
  159. // Ping reports whether the client's connection to the logging service and the
  160. // authentication configuration are valid. To accomplish this, Ping writes a
  161. // log entry "ping" to a log named "ping".
  162. func (c *Client) Ping(ctx context.Context) error {
  163. ent := &logpb.LogEntry{
  164. Payload: &logpb.LogEntry_TextPayload{TextPayload: "ping"},
  165. Timestamp: unixZeroTimestamp, // Identical timestamps and insert IDs are both
  166. InsertId: "ping", // necessary for the service to dedup these entries.
  167. }
  168. _, err := c.client.WriteLogEntries(ctx, &logpb.WriteLogEntriesRequest{
  169. LogName: internal.LogPath(c.parent, "ping"),
  170. Resource: monitoredResource(c.parent),
  171. Entries: []*logpb.LogEntry{ent},
  172. })
  173. return err
  174. }
  175. // error puts the error on the client's error channel
  176. // without blocking, and records summary error info.
  177. func (c *Client) error(err error) {
  178. select {
  179. case c.errc <- err:
  180. default:
  181. }
  182. c.mu.Lock()
  183. c.lastErr = err
  184. c.nErrs++
  185. c.mu.Unlock()
  186. }
  187. func (c *Client) extractErrorInfo() error {
  188. var err error
  189. c.mu.Lock()
  190. if c.lastErr != nil {
  191. err = fmt.Errorf("saw %d errors; last: %v", c.nErrs, c.lastErr)
  192. c.nErrs = 0
  193. c.lastErr = nil
  194. }
  195. c.mu.Unlock()
  196. return err
  197. }
  198. // A Logger is used to write log messages to a single log. It can be configured
  199. // with a log ID, common monitored resource, and a set of common labels.
  200. type Logger struct {
  201. client *Client
  202. logName string // "projects/{projectID}/logs/{logID}"
  203. stdLoggers map[Severity]*log.Logger
  204. bundler *bundler.Bundler
  205. // Options
  206. commonResource *mrpb.MonitoredResource
  207. commonLabels map[string]string
  208. ctxFunc func() (context.Context, func())
  209. }
  210. // A LoggerOption is a configuration option for a Logger.
  211. type LoggerOption interface {
  212. set(*Logger)
  213. }
  214. // CommonResource sets the monitored resource associated with all log entries
  215. // written from a Logger. If not provided, the resource is automatically
  216. // detected based on the running environment. This value can be overridden
  217. // per-entry by setting an Entry's Resource field.
  218. func CommonResource(r *mrpb.MonitoredResource) LoggerOption { return commonResource{r} }
  219. type commonResource struct{ *mrpb.MonitoredResource }
  220. func (r commonResource) set(l *Logger) { l.commonResource = r.MonitoredResource }
  221. var detectedResource struct {
  222. pb *mrpb.MonitoredResource
  223. once sync.Once
  224. }
  225. func detectResource() *mrpb.MonitoredResource {
  226. detectedResource.once.Do(func() {
  227. if !metadata.OnGCE() {
  228. return
  229. }
  230. projectID, err := metadata.ProjectID()
  231. if err != nil {
  232. return
  233. }
  234. id, err := metadata.InstanceID()
  235. if err != nil {
  236. return
  237. }
  238. zone, err := metadata.Zone()
  239. if err != nil {
  240. return
  241. }
  242. detectedResource.pb = &mrpb.MonitoredResource{
  243. Type: "gce_instance",
  244. Labels: map[string]string{
  245. "project_id": projectID,
  246. "instance_id": id,
  247. "zone": zone,
  248. },
  249. }
  250. })
  251. return detectedResource.pb
  252. }
  253. var resourceInfo = map[string]struct{ rtype, label string }{
  254. "organizations": {"organization", "organization_id"},
  255. "folders": {"folder", "folder_id"},
  256. "projects": {"project", "project_id"},
  257. "billingAccounts": {"billing_account", "account_id"},
  258. }
  259. func monitoredResource(parent string) *mrpb.MonitoredResource {
  260. parts := strings.SplitN(parent, "/", 2)
  261. if len(parts) != 2 {
  262. return globalResource(parent)
  263. }
  264. info, ok := resourceInfo[parts[0]]
  265. if !ok {
  266. return globalResource(parts[1])
  267. }
  268. return &mrpb.MonitoredResource{
  269. Type: info.rtype,
  270. Labels: map[string]string{info.label: parts[1]},
  271. }
  272. }
  273. func globalResource(projectID string) *mrpb.MonitoredResource {
  274. return &mrpb.MonitoredResource{
  275. Type: "global",
  276. Labels: map[string]string{
  277. "project_id": projectID,
  278. },
  279. }
  280. }
  281. // CommonLabels are labels that apply to all log entries written from a Logger,
  282. // so that you don't have to repeat them in each log entry's Labels field. If
  283. // any of the log entries contains a (key, value) with the same key that is in
  284. // CommonLabels, then the entry's (key, value) overrides the one in
  285. // CommonLabels.
  286. func CommonLabels(m map[string]string) LoggerOption { return commonLabels(m) }
  287. type commonLabels map[string]string
  288. func (c commonLabels) set(l *Logger) { l.commonLabels = c }
  289. // ConcurrentWriteLimit determines how many goroutines will send log entries to the
  290. // underlying service. The default is 1. Set ConcurrentWriteLimit to a higher value to
  291. // increase throughput.
  292. func ConcurrentWriteLimit(n int) LoggerOption { return concurrentWriteLimit(n) }
  293. type concurrentWriteLimit int
  294. func (c concurrentWriteLimit) set(l *Logger) { l.bundler.HandlerLimit = int(c) }
  295. // DelayThreshold is the maximum amount of time that an entry should remain
  296. // buffered in memory before a call to the logging service is triggered. Larger
  297. // values of DelayThreshold will generally result in fewer calls to the logging
  298. // service, while increasing the risk that log entries will be lost if the
  299. // process crashes.
  300. // The default is DefaultDelayThreshold.
  301. func DelayThreshold(d time.Duration) LoggerOption { return delayThreshold(d) }
  302. type delayThreshold time.Duration
  303. func (d delayThreshold) set(l *Logger) { l.bundler.DelayThreshold = time.Duration(d) }
  304. // EntryCountThreshold is the maximum number of entries that will be buffered
  305. // in memory before a call to the logging service is triggered. Larger values
  306. // will generally result in fewer calls to the logging service, while
  307. // increasing both memory consumption and the risk that log entries will be
  308. // lost if the process crashes.
  309. // The default is DefaultEntryCountThreshold.
  310. func EntryCountThreshold(n int) LoggerOption { return entryCountThreshold(n) }
  311. type entryCountThreshold int
  312. func (e entryCountThreshold) set(l *Logger) { l.bundler.BundleCountThreshold = int(e) }
  313. // EntryByteThreshold is the maximum number of bytes of entries that will be
  314. // buffered in memory before a call to the logging service is triggered. See
  315. // EntryCountThreshold for a discussion of the tradeoffs involved in setting
  316. // this option.
  317. // The default is DefaultEntryByteThreshold.
  318. func EntryByteThreshold(n int) LoggerOption { return entryByteThreshold(n) }
  319. type entryByteThreshold int
  320. func (e entryByteThreshold) set(l *Logger) { l.bundler.BundleByteThreshold = int(e) }
  321. // EntryByteLimit is the maximum number of bytes of entries that will be sent
  322. // in a single call to the logging service. ErrOversizedEntry is returned if an
  323. // entry exceeds EntryByteLimit. This option limits the size of a single RPC
  324. // payload, to account for network or service issues with large RPCs. If
  325. // EntryByteLimit is smaller than EntryByteThreshold, the latter has no effect.
  326. // The default is zero, meaning there is no limit.
  327. func EntryByteLimit(n int) LoggerOption { return entryByteLimit(n) }
  328. type entryByteLimit int
  329. func (e entryByteLimit) set(l *Logger) { l.bundler.BundleByteLimit = int(e) }
  330. // BufferedByteLimit is the maximum number of bytes that the Logger will keep
  331. // in memory before returning ErrOverflow. This option limits the total memory
  332. // consumption of the Logger (but note that each Logger has its own, separate
  333. // limit). It is possible to reach BufferedByteLimit even if it is larger than
  334. // EntryByteThreshold or EntryByteLimit, because calls triggered by the latter
  335. // two options may be enqueued (and hence occupying memory) while new log
  336. // entries are being added.
  337. // The default is DefaultBufferedByteLimit.
  338. func BufferedByteLimit(n int) LoggerOption { return bufferedByteLimit(n) }
  339. type bufferedByteLimit int
  340. func (b bufferedByteLimit) set(l *Logger) { l.bundler.BufferedByteLimit = int(b) }
  341. // ContextFunc is a function that will be called to obtain a context.Context for the
  342. // WriteLogEntries RPC executed in the background for calls to Logger.Log. The
  343. // default is a function that always returns context.Background. The second return
  344. // value of the function is a function to call after the RPC completes.
  345. //
  346. // The function is not used for calls to Logger.LogSync, since the caller can pass
  347. // in the context directly.
  348. //
  349. // This option is EXPERIMENTAL. It may be changed or removed.
  350. func ContextFunc(f func() (ctx context.Context, afterCall func())) LoggerOption {
  351. return contextFunc(f)
  352. }
  353. type contextFunc func() (ctx context.Context, afterCall func())
  354. func (c contextFunc) set(l *Logger) { l.ctxFunc = c }
  355. // Logger returns a Logger that will write entries with the given log ID, such as
  356. // "syslog". A log ID must be less than 512 characters long and can only
  357. // include the following characters: upper and lower case alphanumeric
  358. // characters: [A-Za-z0-9]; and punctuation characters: forward-slash,
  359. // underscore, hyphen, and period.
  360. func (c *Client) Logger(logID string, opts ...LoggerOption) *Logger {
  361. r := detectResource()
  362. if r == nil {
  363. r = monitoredResource(c.parent)
  364. }
  365. l := &Logger{
  366. client: c,
  367. logName: internal.LogPath(c.parent, logID),
  368. commonResource: r,
  369. ctxFunc: func() (context.Context, func()) { return context.Background(), nil },
  370. }
  371. l.bundler = bundler.NewBundler(&logpb.LogEntry{}, func(entries interface{}) {
  372. l.writeLogEntries(entries.([]*logpb.LogEntry))
  373. })
  374. l.bundler.DelayThreshold = DefaultDelayThreshold
  375. l.bundler.BundleCountThreshold = DefaultEntryCountThreshold
  376. l.bundler.BundleByteThreshold = DefaultEntryByteThreshold
  377. l.bundler.BufferedByteLimit = DefaultBufferedByteLimit
  378. for _, opt := range opts {
  379. opt.set(l)
  380. }
  381. l.stdLoggers = map[Severity]*log.Logger{}
  382. for s := range severityName {
  383. l.stdLoggers[s] = log.New(severityWriter{l, s}, "", 0)
  384. }
  385. c.loggers.Add(1)
  386. // Start a goroutine that cleans up the bundler, its channel
  387. // and the writer goroutines when the client is closed.
  388. go func() {
  389. defer c.loggers.Done()
  390. <-c.donec
  391. l.bundler.Flush()
  392. }()
  393. return l
  394. }
  395. type severityWriter struct {
  396. l *Logger
  397. s Severity
  398. }
  399. func (w severityWriter) Write(p []byte) (n int, err error) {
  400. w.l.Log(Entry{
  401. Severity: w.s,
  402. Payload: string(p),
  403. })
  404. return len(p), nil
  405. }
  406. // Close waits for all opened loggers to be flushed and closes the client.
  407. func (c *Client) Close() error {
  408. if c.closed {
  409. return nil
  410. }
  411. close(c.donec) // close Logger bundlers
  412. c.loggers.Wait() // wait for all bundlers to flush and close
  413. // Now there can be no more errors.
  414. close(c.errc) // terminate error goroutine
  415. // Prefer errors arising from logging to the error returned from Close.
  416. err := c.extractErrorInfo()
  417. err2 := c.client.Close()
  418. if err == nil {
  419. err = err2
  420. }
  421. c.closed = true
  422. return err
  423. }
  424. // Severity is the severity of the event described in a log entry. These
  425. // guideline severity levels are ordered, with numerically smaller levels
  426. // treated as less severe than numerically larger levels.
  427. type Severity int
  428. const (
  429. // Default means the log entry has no assigned severity level.
  430. Default = Severity(logtypepb.LogSeverity_DEFAULT)
  431. // Debug means debug or trace information.
  432. Debug = Severity(logtypepb.LogSeverity_DEBUG)
  433. // Info means routine information, such as ongoing status or performance.
  434. Info = Severity(logtypepb.LogSeverity_INFO)
  435. // Notice means normal but significant events, such as start up, shut down, or configuration.
  436. Notice = Severity(logtypepb.LogSeverity_NOTICE)
  437. // Warning means events that might cause problems.
  438. Warning = Severity(logtypepb.LogSeverity_WARNING)
  439. // Error means events that are likely to cause problems.
  440. Error = Severity(logtypepb.LogSeverity_ERROR)
  441. // Critical means events that cause more severe problems or brief outages.
  442. Critical = Severity(logtypepb.LogSeverity_CRITICAL)
  443. // Alert means a person must take an action immediately.
  444. Alert = Severity(logtypepb.LogSeverity_ALERT)
  445. // Emergency means one or more systems are unusable.
  446. Emergency = Severity(logtypepb.LogSeverity_EMERGENCY)
  447. )
  448. var severityName = map[Severity]string{
  449. Default: "Default",
  450. Debug: "Debug",
  451. Info: "Info",
  452. Notice: "Notice",
  453. Warning: "Warning",
  454. Error: "Error",
  455. Critical: "Critical",
  456. Alert: "Alert",
  457. Emergency: "Emergency",
  458. }
  459. // String converts a severity level to a string.
  460. func (v Severity) String() string {
  461. // same as proto.EnumName
  462. s, ok := severityName[v]
  463. if ok {
  464. return s
  465. }
  466. return strconv.Itoa(int(v))
  467. }
  468. // ParseSeverity returns the Severity whose name equals s, ignoring case. It
  469. // returns Default if no Severity matches.
  470. func ParseSeverity(s string) Severity {
  471. sl := strings.ToLower(s)
  472. for sev, name := range severityName {
  473. if strings.ToLower(name) == sl {
  474. return sev
  475. }
  476. }
  477. return Default
  478. }
  479. // Entry is a log entry.
  480. // See https://cloud.google.com/logging/docs/view/logs_index for more about entries.
  481. type Entry struct {
  482. // Timestamp is the time of the entry. If zero, the current time is used.
  483. Timestamp time.Time
  484. // Severity is the entry's severity level.
  485. // The zero value is Default.
  486. Severity Severity
  487. // Payload must be either a string, or something that marshals via the
  488. // encoding/json package to a JSON object (and not any other type of JSON value).
  489. Payload interface{}
  490. // Labels optionally specifies key/value labels for the log entry.
  491. // The Logger.Log method takes ownership of this map. See Logger.CommonLabels
  492. // for more about labels.
  493. Labels map[string]string
  494. // InsertID is a unique ID for the log entry. If you provide this field,
  495. // the logging service considers other log entries in the same log with the
  496. // same ID as duplicates which can be removed. If omitted, the logging
  497. // service will generate a unique ID for this log entry. Note that because
  498. // this client retries RPCs automatically, it is possible (though unlikely)
  499. // that an Entry without an InsertID will be written more than once.
  500. InsertID string
  501. // HTTPRequest optionally specifies metadata about the HTTP request
  502. // associated with this log entry, if applicable. It is optional.
  503. HTTPRequest *HTTPRequest
  504. // Operation optionally provides information about an operation associated
  505. // with the log entry, if applicable.
  506. Operation *logpb.LogEntryOperation
  507. // LogName is the full log name, in the form
  508. // "projects/{ProjectID}/logs/{LogID}". It is set by the client when
  509. // reading entries. It is an error to set it when writing entries.
  510. LogName string
  511. // Resource is the monitored resource associated with the entry.
  512. Resource *mrpb.MonitoredResource
  513. // Trace is the resource name of the trace associated with the log entry,
  514. // if any. If it contains a relative resource name, the name is assumed to
  515. // be relative to //tracing.googleapis.com.
  516. Trace string
  517. // ID of the span within the trace associated with the log entry.
  518. // The ID is a 16-character hexadecimal encoding of an 8-byte array.
  519. SpanID string
  520. // Optional. Source code location information associated with the log entry,
  521. // if any.
  522. SourceLocation *logpb.LogEntrySourceLocation
  523. }
  524. // HTTPRequest contains an http.Request as well as additional
  525. // information about the request and its response.
  526. type HTTPRequest struct {
  527. // Request is the http.Request passed to the handler.
  528. Request *http.Request
  529. // RequestSize is the size of the HTTP request message in bytes, including
  530. // the request headers and the request body.
  531. RequestSize int64
  532. // Status is the response code indicating the status of the response.
  533. // Examples: 200, 404.
  534. Status int
  535. // ResponseSize is the size of the HTTP response message sent back to the client, in bytes,
  536. // including the response headers and the response body.
  537. ResponseSize int64
  538. // Latency is the request processing latency on the server, from the time the request was
  539. // received until the response was sent.
  540. Latency time.Duration
  541. // LocalIP is the IP address (IPv4 or IPv6) of the origin server that the request
  542. // was sent to.
  543. LocalIP string
  544. // RemoteIP is the IP address (IPv4 or IPv6) of the client that issued the
  545. // HTTP request. Examples: "192.168.1.1", "FE80::0202:B3FF:FE1E:8329".
  546. RemoteIP string
  547. // CacheHit reports whether an entity was served from cache (with or without
  548. // validation).
  549. CacheHit bool
  550. // CacheValidatedWithOriginServer reports whether the response was
  551. // validated with the origin server before being served from cache. This
  552. // field is only meaningful if CacheHit is true.
  553. CacheValidatedWithOriginServer bool
  554. }
  555. func fromHTTPRequest(r *HTTPRequest) *logtypepb.HttpRequest {
  556. if r == nil {
  557. return nil
  558. }
  559. if r.Request == nil {
  560. panic("HTTPRequest must have a non-nil Request")
  561. }
  562. u := *r.Request.URL
  563. u.Fragment = ""
  564. pb := &logtypepb.HttpRequest{
  565. RequestMethod: r.Request.Method,
  566. RequestUrl: u.String(),
  567. RequestSize: r.RequestSize,
  568. Status: int32(r.Status),
  569. ResponseSize: r.ResponseSize,
  570. UserAgent: r.Request.UserAgent(),
  571. ServerIp: r.LocalIP,
  572. RemoteIp: r.RemoteIP, // TODO(jba): attempt to parse http.Request.RemoteAddr?
  573. Referer: r.Request.Referer(),
  574. CacheHit: r.CacheHit,
  575. CacheValidatedWithOriginServer: r.CacheValidatedWithOriginServer,
  576. }
  577. if r.Latency != 0 {
  578. pb.Latency = ptypes.DurationProto(r.Latency)
  579. }
  580. return pb
  581. }
  582. // toProtoStruct converts v, which must marshal into a JSON object,
  583. // into a Google Struct proto.
  584. func toProtoStruct(v interface{}) (*structpb.Struct, error) {
  585. // Fast path: if v is already a *structpb.Struct, nothing to do.
  586. if s, ok := v.(*structpb.Struct); ok {
  587. return s, nil
  588. }
  589. // v is a Go value that supports JSON marshalling. We want a Struct
  590. // protobuf. Some day we may have a more direct way to get there, but right
  591. // now the only way is to marshal the Go value to JSON, unmarshal into a
  592. // map, and then build the Struct proto from the map.
  593. var jb []byte
  594. var err error
  595. if raw, ok := v.(json.RawMessage); ok { // needed for Go 1.7 and below
  596. jb = []byte(raw)
  597. } else {
  598. jb, err = json.Marshal(v)
  599. if err != nil {
  600. return nil, fmt.Errorf("logging: json.Marshal: %v", err)
  601. }
  602. }
  603. var m map[string]interface{}
  604. err = json.Unmarshal(jb, &m)
  605. if err != nil {
  606. return nil, fmt.Errorf("logging: json.Unmarshal: %v", err)
  607. }
  608. return jsonMapToProtoStruct(m), nil
  609. }
  610. func jsonMapToProtoStruct(m map[string]interface{}) *structpb.Struct {
  611. fields := map[string]*structpb.Value{}
  612. for k, v := range m {
  613. fields[k] = jsonValueToStructValue(v)
  614. }
  615. return &structpb.Struct{Fields: fields}
  616. }
  617. func jsonValueToStructValue(v interface{}) *structpb.Value {
  618. switch x := v.(type) {
  619. case bool:
  620. return &structpb.Value{Kind: &structpb.Value_BoolValue{BoolValue: x}}
  621. case float64:
  622. return &structpb.Value{Kind: &structpb.Value_NumberValue{NumberValue: x}}
  623. case string:
  624. return &structpb.Value{Kind: &structpb.Value_StringValue{StringValue: x}}
  625. case nil:
  626. return &structpb.Value{Kind: &structpb.Value_NullValue{}}
  627. case map[string]interface{}:
  628. return &structpb.Value{Kind: &structpb.Value_StructValue{StructValue: jsonMapToProtoStruct(x)}}
  629. case []interface{}:
  630. var vals []*structpb.Value
  631. for _, e := range x {
  632. vals = append(vals, jsonValueToStructValue(e))
  633. }
  634. return &structpb.Value{Kind: &structpb.Value_ListValue{ListValue: &structpb.ListValue{Values: vals}}}
  635. default:
  636. panic(fmt.Sprintf("bad type %T for JSON value", v))
  637. }
  638. }
  639. // LogSync logs the Entry synchronously without any buffering. Because LogSync is slow
  640. // and will block, it is intended primarily for debugging or critical errors.
  641. // Prefer Log for most uses.
  642. // TODO(jba): come up with a better name (LogNow?) or eliminate.
  643. func (l *Logger) LogSync(ctx context.Context, e Entry) error {
  644. ent, err := l.toLogEntry(e)
  645. if err != nil {
  646. return err
  647. }
  648. _, err = l.client.client.WriteLogEntries(ctx, &logpb.WriteLogEntriesRequest{
  649. LogName: l.logName,
  650. Resource: l.commonResource,
  651. Labels: l.commonLabels,
  652. Entries: []*logpb.LogEntry{ent},
  653. })
  654. return err
  655. }
  656. // Log buffers the Entry for output to the logging service. It never blocks.
  657. func (l *Logger) Log(e Entry) {
  658. ent, err := l.toLogEntry(e)
  659. if err != nil {
  660. l.client.error(err)
  661. return
  662. }
  663. if err := l.bundler.Add(ent, proto.Size(ent)); err != nil {
  664. l.client.error(err)
  665. }
  666. }
  667. // Flush blocks until all currently buffered log entries are sent.
  668. //
  669. // If any errors occurred since the last call to Flush from any Logger, or the
  670. // creation of the client if this is the first call, then Flush returns a non-nil
  671. // error with summary information about the errors. This information is unlikely to
  672. // be actionable. For more accurate error reporting, set Client.OnError.
  673. func (l *Logger) Flush() error {
  674. l.bundler.Flush()
  675. return l.client.extractErrorInfo()
  676. }
  677. func (l *Logger) writeLogEntries(entries []*logpb.LogEntry) {
  678. req := &logpb.WriteLogEntriesRequest{
  679. LogName: l.logName,
  680. Resource: l.commonResource,
  681. Labels: l.commonLabels,
  682. Entries: entries,
  683. }
  684. ctx, afterCall := l.ctxFunc()
  685. ctx, cancel := context.WithTimeout(ctx, defaultWriteTimeout)
  686. defer cancel()
  687. _, err := l.client.client.WriteLogEntries(ctx, req)
  688. if err != nil {
  689. l.client.error(err)
  690. }
  691. if afterCall != nil {
  692. afterCall()
  693. }
  694. }
  695. // StandardLogger returns a *log.Logger for the provided severity.
  696. //
  697. // This method is cheap. A single log.Logger is pre-allocated for each
  698. // severity level in each Logger. Callers may mutate the returned log.Logger
  699. // (for example by calling SetFlags or SetPrefix).
  700. func (l *Logger) StandardLogger(s Severity) *log.Logger { return l.stdLoggers[s] }
  701. func (l *Logger) toLogEntry(e Entry) (*logpb.LogEntry, error) {
  702. if e.LogName != "" {
  703. return nil, errors.New("logging: Entry.LogName should be not be set when writing")
  704. }
  705. t := e.Timestamp
  706. if t.IsZero() {
  707. t = now()
  708. }
  709. ts, err := ptypes.TimestampProto(t)
  710. if err != nil {
  711. return nil, err
  712. }
  713. if e.Trace == "" && e.HTTPRequest != nil && e.HTTPRequest.Request != nil {
  714. traceHeader := e.HTTPRequest.Request.Header.Get("X-Cloud-Trace-Context")
  715. if traceHeader != "" {
  716. // Set to a relative resource name, as described at
  717. // https://cloud.google.com/appengine/docs/flexible/go/writing-application-logs.
  718. e.Trace = fmt.Sprintf("%s/traces/%s", l.client.parent, traceHeader)
  719. }
  720. }
  721. ent := &logpb.LogEntry{
  722. Timestamp: ts,
  723. Severity: logtypepb.LogSeverity(e.Severity),
  724. InsertId: e.InsertID,
  725. HttpRequest: fromHTTPRequest(e.HTTPRequest),
  726. Operation: e.Operation,
  727. Labels: e.Labels,
  728. Trace: e.Trace,
  729. SpanId: e.SpanID,
  730. Resource: e.Resource,
  731. SourceLocation: e.SourceLocation,
  732. }
  733. switch p := e.Payload.(type) {
  734. case string:
  735. ent.Payload = &logpb.LogEntry_TextPayload{TextPayload: p}
  736. default:
  737. s, err := toProtoStruct(p)
  738. if err != nil {
  739. return nil, err
  740. }
  741. ent.Payload = &logpb.LogEntry_JsonPayload{JsonPayload: s}
  742. }
  743. return ent, nil
  744. }