|
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406 |
- // Copyright 2016 Google LLC
- //
- // Licensed under the Apache License, Version 2.0 (the "License");
- // you may not use this file except in compliance with the License.
- // You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing, software
- // distributed under the License is distributed on an "AS IS" BASIS,
- // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- // See the License for the specific language governing permissions and
- // limitations under the License.
-
- // These features are missing now, but will likely be added:
- // - There is no way to specify CallOptions.
-
- // Package logadmin contains a Stackdriver Logging client that can be used
- // for reading logs and working with sinks, metrics and monitored resources.
- // For a client that can write logs, see package cloud.google.com/go/logging.
- //
- // The client uses Logging API v2.
- // See https://cloud.google.com/logging/docs/api/v2/ for an introduction to the API.
- //
- // Note: This package is in beta. Some backwards-incompatible changes may occur.
- package logadmin // import "cloud.google.com/go/logging/logadmin"
-
- import (
- "fmt"
- "math"
- "net/http"
- "net/url"
- "strings"
- "time"
-
- "cloud.google.com/go/internal/version"
- "cloud.google.com/go/logging"
- vkit "cloud.google.com/go/logging/apiv2"
- "cloud.google.com/go/logging/internal"
- "github.com/golang/protobuf/ptypes"
- gax "github.com/googleapis/gax-go"
- "golang.org/x/net/context"
- "google.golang.org/api/iterator"
- "google.golang.org/api/option"
- logtypepb "google.golang.org/genproto/googleapis/logging/type"
- logpb "google.golang.org/genproto/googleapis/logging/v2"
- "google.golang.org/grpc/codes"
-
- // Import the following so EntryIterator can unmarshal log protos.
- _ "google.golang.org/genproto/googleapis/appengine/logging/v1"
- _ "google.golang.org/genproto/googleapis/cloud/audit"
- )
-
- // Client is a Logging client. A Client is associated with a single Cloud project.
- type Client struct {
- lClient *vkit.Client // logging client
- sClient *vkit.ConfigClient // sink client
- mClient *vkit.MetricsClient // metric client
- parent string
- closed bool
- }
-
- // NewClient returns a new logging client associated with the provided project ID.
- //
- // By default NewClient uses AdminScope. To use a different scope, call
- // NewClient using a WithScopes option (see https://godoc.org/google.golang.org/api/option#WithScopes).
- func NewClient(ctx context.Context, parent string, opts ...option.ClientOption) (*Client, error) {
- if !strings.ContainsRune(parent, '/') {
- parent = "projects/" + parent
- }
- opts = append([]option.ClientOption{
- option.WithEndpoint(internal.ProdAddr),
- option.WithScopes(logging.AdminScope),
- }, opts...)
- lc, err := vkit.NewClient(ctx, opts...)
- if err != nil {
- return nil, err
- }
- // TODO(jba): pass along any client options that should be provided to all clients.
- sc, err := vkit.NewConfigClient(ctx, option.WithGRPCConn(lc.Connection()))
- if err != nil {
- return nil, err
- }
- mc, err := vkit.NewMetricsClient(ctx, option.WithGRPCConn(lc.Connection()))
- if err != nil {
- return nil, err
- }
- // Retry some non-idempotent methods on INTERNAL, because it happens sometimes
- // and in all observed cases the operation did not complete.
- retryerOnInternal := func() gax.Retryer {
- return gax.OnCodes([]codes.Code{
- codes.Internal,
- }, gax.Backoff{
- Initial: 100 * time.Millisecond,
- Max: 1000 * time.Millisecond,
- Multiplier: 1.2,
- })
- }
- mc.CallOptions.CreateLogMetric = []gax.CallOption{gax.WithRetry(retryerOnInternal)}
- mc.CallOptions.UpdateLogMetric = []gax.CallOption{gax.WithRetry(retryerOnInternal)}
-
- lc.SetGoogleClientInfo("gccl", version.Repo)
- sc.SetGoogleClientInfo("gccl", version.Repo)
- mc.SetGoogleClientInfo("gccl", version.Repo)
- client := &Client{
- lClient: lc,
- sClient: sc,
- mClient: mc,
- parent: parent,
- }
- return client, nil
- }
-
- // Close closes the client.
- func (c *Client) Close() error {
- if c.closed {
- return nil
- }
- // Return only the first error. Since all clients share an underlying connection,
- // Closes after the first always report a "connection is closing" error.
- err := c.lClient.Close()
- _ = c.sClient.Close()
- _ = c.mClient.Close()
- c.closed = true
- return err
- }
-
- // DeleteLog deletes a log and all its log entries. The log will reappear if it receives new entries.
- // logID identifies the log within the project. An example log ID is "syslog". Requires AdminScope.
- func (c *Client) DeleteLog(ctx context.Context, logID string) error {
- return c.lClient.DeleteLog(ctx, &logpb.DeleteLogRequest{
- LogName: internal.LogPath(c.parent, logID),
- })
- }
-
- func toHTTPRequest(p *logtypepb.HttpRequest) (*logging.HTTPRequest, error) {
- if p == nil {
- return nil, nil
- }
- u, err := url.Parse(p.RequestUrl)
- if err != nil {
- return nil, err
- }
- var dur time.Duration
- if p.Latency != nil {
- dur, err = ptypes.Duration(p.Latency)
- if err != nil {
- return nil, err
- }
- }
- hr := &http.Request{
- Method: p.RequestMethod,
- URL: u,
- Header: map[string][]string{},
- }
- if p.UserAgent != "" {
- hr.Header.Set("User-Agent", p.UserAgent)
- }
- if p.Referer != "" {
- hr.Header.Set("Referer", p.Referer)
- }
- return &logging.HTTPRequest{
- Request: hr,
- RequestSize: p.RequestSize,
- Status: int(p.Status),
- ResponseSize: p.ResponseSize,
- Latency: dur,
- RemoteIP: p.RemoteIp,
- CacheHit: p.CacheHit,
- CacheValidatedWithOriginServer: p.CacheValidatedWithOriginServer,
- }, nil
- }
-
- // An EntriesOption is an option for listing log entries.
- type EntriesOption interface {
- set(*logpb.ListLogEntriesRequest)
- }
-
- // ProjectIDs sets the project IDs or project numbers from which to retrieve
- // log entries. Examples of a project ID: "my-project-1A", "1234567890".
- func ProjectIDs(pids []string) EntriesOption { return projectIDs(pids) }
-
- type projectIDs []string
-
- func (p projectIDs) set(r *logpb.ListLogEntriesRequest) {
- r.ResourceNames = make([]string, len(p))
- for i, v := range p {
- r.ResourceNames[i] = fmt.Sprintf("projects/%s", v)
- }
- }
-
- // ResourceNames sets the resource names from which to retrieve
- // log entries. Examples: "projects/my-project-1A", "organizations/my-org".
- func ResourceNames(rns []string) EntriesOption { return resourceNames(rns) }
-
- type resourceNames []string
-
- func (rn resourceNames) set(r *logpb.ListLogEntriesRequest) {
- r.ResourceNames = append([]string(nil), rn...)
- }
-
- // Filter sets an advanced logs filter for listing log entries (see
- // https://cloud.google.com/logging/docs/view/advanced_filters). The filter is
- // compared against all log entries in the projects specified by ProjectIDs.
- // Only entries that match the filter are retrieved. An empty filter (the
- // default) matches all log entries.
- //
- // In the filter string, log names must be written in their full form, as
- // "projects/PROJECT-ID/logs/LOG-ID". Forward slashes in LOG-ID must be
- // replaced by %2F before calling Filter.
- //
- // Timestamps in the filter string must be written in RFC 3339 format. See the
- // timestamp example.
- func Filter(f string) EntriesOption { return filter(f) }
-
- type filter string
-
- func (f filter) set(r *logpb.ListLogEntriesRequest) { r.Filter = string(f) }
-
- // NewestFirst causes log entries to be listed from most recent (newest) to
- // least recent (oldest). By default, they are listed from oldest to newest.
- func NewestFirst() EntriesOption { return newestFirst{} }
-
- type newestFirst struct{}
-
- func (newestFirst) set(r *logpb.ListLogEntriesRequest) { r.OrderBy = "timestamp desc" }
-
- // Entries returns an EntryIterator for iterating over log entries. By default,
- // the log entries will be restricted to those from the project passed to
- // NewClient. This may be overridden by passing a ProjectIDs option. Requires ReadScope or AdminScope.
- func (c *Client) Entries(ctx context.Context, opts ...EntriesOption) *EntryIterator {
- it := &EntryIterator{
- it: c.lClient.ListLogEntries(ctx, listLogEntriesRequest(c.parent, opts)),
- }
- it.pageInfo, it.nextFunc = iterator.NewPageInfo(
- it.fetch,
- func() int { return len(it.items) },
- func() interface{} { b := it.items; it.items = nil; return b })
- return it
- }
-
- func listLogEntriesRequest(parent string, opts []EntriesOption) *logpb.ListLogEntriesRequest {
- req := &logpb.ListLogEntriesRequest{
- ResourceNames: []string{parent},
- }
- for _, opt := range opts {
- opt.set(req)
- }
- return req
- }
-
- // An EntryIterator iterates over log entries.
- type EntryIterator struct {
- it *vkit.LogEntryIterator
- pageInfo *iterator.PageInfo
- nextFunc func() error
- items []*logging.Entry
- }
-
- // PageInfo supports pagination. See https://godoc.org/google.golang.org/api/iterator package for details.
- func (it *EntryIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
-
- // Next returns the next result. Its second return value is iterator.Done
- // (https://godoc.org/google.golang.org/api/iterator) if there are no more
- // results. Once Next returns Done, all subsequent calls will return Done.
- func (it *EntryIterator) Next() (*logging.Entry, error) {
- if err := it.nextFunc(); err != nil {
- return nil, err
- }
- item := it.items[0]
- it.items = it.items[1:]
- return item, nil
- }
-
- func (it *EntryIterator) fetch(pageSize int, pageToken string) (string, error) {
- return iterFetch(pageSize, pageToken, it.it.PageInfo(), func() error {
- item, err := it.it.Next()
- if err != nil {
- return err
- }
- e, err := fromLogEntry(item)
- if err != nil {
- return err
- }
- it.items = append(it.items, e)
- return nil
- })
- }
-
- func trunc32(i int) int32 {
- if i > math.MaxInt32 {
- i = math.MaxInt32
- }
- return int32(i)
- }
-
- var slashUnescaper = strings.NewReplacer("%2F", "/", "%2f", "/")
-
- func fromLogEntry(le *logpb.LogEntry) (*logging.Entry, error) {
- time, err := ptypes.Timestamp(le.Timestamp)
- if err != nil {
- return nil, err
- }
- var payload interface{}
- switch x := le.Payload.(type) {
- case *logpb.LogEntry_TextPayload:
- payload = x.TextPayload
-
- case *logpb.LogEntry_ProtoPayload:
- var d ptypes.DynamicAny
- if err := ptypes.UnmarshalAny(x.ProtoPayload, &d); err != nil {
- return nil, fmt.Errorf("logging: unmarshalling proto payload: %v", err)
- }
- payload = d.Message
-
- case *logpb.LogEntry_JsonPayload:
- // Leave this as a Struct.
- // TODO(jba): convert to map[string]interface{}?
- payload = x.JsonPayload
-
- default:
- return nil, fmt.Errorf("logging: unknown payload type: %T", le.Payload)
- }
- hr, err := toHTTPRequest(le.HttpRequest)
- if err != nil {
- return nil, err
- }
- return &logging.Entry{
- Timestamp: time,
- Severity: logging.Severity(le.Severity),
- Payload: payload,
- Labels: le.Labels,
- InsertID: le.InsertId,
- HTTPRequest: hr,
- Operation: le.Operation,
- LogName: slashUnescaper.Replace(le.LogName),
- Resource: le.Resource,
- Trace: le.Trace,
- }, nil
- }
-
- // Logs lists the logs owned by the parent resource of the client.
- func (c *Client) Logs(ctx context.Context) *LogIterator {
- it := &LogIterator{
- parentResource: c.parent,
- it: c.lClient.ListLogs(ctx, &logpb.ListLogsRequest{Parent: c.parent}),
- }
- it.pageInfo, it.nextFunc = iterator.NewPageInfo(
- it.fetch,
- func() int { return len(it.items) },
- func() interface{} { b := it.items; it.items = nil; return b })
- return it
- }
-
- // A LogIterator iterates over logs.
- type LogIterator struct {
- parentResource string
- it *vkit.StringIterator
- pageInfo *iterator.PageInfo
- nextFunc func() error
- items []string
- }
-
- // PageInfo supports pagination. See https://godoc.org/google.golang.org/api/iterator package for details.
- func (it *LogIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }
-
- // Next returns the next result. Its second return value is iterator.Done
- // (https://godoc.org/google.golang.org/api/iterator) if there are no more
- // results. Once Next returns Done, all subsequent calls will return Done.
- func (it *LogIterator) Next() (string, error) {
- if err := it.nextFunc(); err != nil {
- return "", err
- }
- item := it.items[0]
- it.items = it.items[1:]
- return item, nil
- }
-
- func (it *LogIterator) fetch(pageSize int, pageToken string) (string, error) {
- return iterFetch(pageSize, pageToken, it.it.PageInfo(), func() error {
- logPath, err := it.it.Next()
- if err != nil {
- return err
- }
- logID := internal.LogIDFromPath(it.parentResource, logPath)
- it.items = append(it.items, logID)
- return nil
- })
- }
-
- // Common fetch code for iterators that are backed by vkit iterators.
- func iterFetch(pageSize int, pageToken string, pi *iterator.PageInfo, next func() error) (string, error) {
- pi.MaxSize = pageSize
- pi.Token = pageToken
- // Get one item, which will fill the buffer.
- if err := next(); err != nil {
- return "", err
- }
- // Collect the rest of the buffer.
- for pi.Remaining() > 0 {
- if err := next(); err != nil {
- return "", err
- }
- }
- return pi.Token, nil
- }
|