/* * Copyright 2019 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ // Package edsbalancer implements a balancer to handle EDS responses. package edsbalancer import ( "context" "encoding/json" "fmt" "net" "reflect" "strconv" "sync" xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" xdstypepb "github.com/envoyproxy/go-control-plane/envoy/type" "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/roundrobin" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/resolver" "google.golang.org/grpc/status" ) type localityConfig struct { weight uint32 addrs []resolver.Address } // EDSBalancer does load balancing based on the EDS responses. Note that it // doesn't implement the balancer interface. It's intended to be used by a high // level balancer implementation. // // The localities are picked as weighted round robin. A configurable child // policy is used to manage endpoints in each locality. type EDSBalancer struct { balancer.ClientConn bg *balancerGroup subBalancerBuilder balancer.Builder lidToConfig map[string]*localityConfig pickerMu sync.Mutex drops []*dropper innerPicker balancer.Picker // The picker without drop support. innerState connectivity.State // The state of the picker. } // NewXDSBalancer create a new EDSBalancer. func NewXDSBalancer(cc balancer.ClientConn) *EDSBalancer { xdsB := &EDSBalancer{ ClientConn: cc, subBalancerBuilder: balancer.Get(roundrobin.Name), lidToConfig: make(map[string]*localityConfig), } // Don't start balancer group here. Start it when handling the first EDS // response. Otherwise the balancer group will be started with round-robin, // and if users specify a different sub-balancer, all balancers in balancer // group will be closed and recreated when sub-balancer update happens. return xdsB } // HandleChildPolicy updates the child balancers handling endpoints. Child // policy is roundrobin by default. If the specified balancer is not installed, // the old child balancer will be used. // // HandleChildPolicy and HandleEDSResponse must be called by the same goroutine. func (xdsB *EDSBalancer) HandleChildPolicy(name string, config json.RawMessage) { // name could come from cdsResp.GetLbPolicy().String(). LbPolicy.String() // are all UPPER_CASE with underscore. // // No conversion is needed here because balancer package converts all names // into lower_case before registering/looking up. xdsB.updateSubBalancerName(name) // TODO: (eds) send balancer config to the new child balancers. } func (xdsB *EDSBalancer) updateSubBalancerName(subBalancerName string) { if xdsB.subBalancerBuilder.Name() == subBalancerName { return } newSubBalancerBuilder := balancer.Get(subBalancerName) if newSubBalancerBuilder == nil { grpclog.Infof("EDSBalancer: failed to find balancer with name %q, keep using %q", subBalancerName, xdsB.subBalancerBuilder.Name()) return } xdsB.subBalancerBuilder = newSubBalancerBuilder if xdsB.bg != nil { // xdsB.bg == nil until the first EDS response is handled. There's no // need to update balancer group before that. for id, config := range xdsB.lidToConfig { // TODO: (eds) add support to balancer group to support smoothly // switching sub-balancers (keep old balancer around until new // balancer becomes ready). xdsB.bg.remove(id) xdsB.bg.add(id, config.weight, xdsB.subBalancerBuilder) xdsB.bg.handleResolvedAddrs(id, config.addrs) } } } // updateDrops compares new drop policies with the old. If they are different, // it updates the drop policies and send ClientConn an updated picker. func (xdsB *EDSBalancer) updateDrops(dropPolicies []*xdspb.ClusterLoadAssignment_Policy_DropOverload) { var ( newDrops []*dropper dropsChanged bool ) for i, dropPolicy := range dropPolicies { percentage := dropPolicy.GetDropPercentage() var ( numerator = percentage.GetNumerator() denominator uint32 ) switch percentage.GetDenominator() { case xdstypepb.FractionalPercent_HUNDRED: denominator = 100 case xdstypepb.FractionalPercent_TEN_THOUSAND: denominator = 10000 case xdstypepb.FractionalPercent_MILLION: denominator = 1000000 } newDrops = append(newDrops, newDropper(numerator, denominator)) // The following reading xdsB.drops doesn't need mutex because it can only // be updated by the code following. if dropsChanged { continue } if i >= len(xdsB.drops) { dropsChanged = true continue } if oldDrop := xdsB.drops[i]; numerator != oldDrop.numerator || denominator != oldDrop.denominator { dropsChanged = true } } if dropsChanged { xdsB.pickerMu.Lock() xdsB.drops = newDrops if xdsB.innerPicker != nil { // Update picker with old inner picker, new drops. xdsB.ClientConn.UpdateBalancerState(xdsB.innerState, newDropPicker(xdsB.innerPicker, newDrops)) } xdsB.pickerMu.Unlock() } } // HandleEDSResponse handles the EDS response and creates/deletes localities and // SubConns. It also handles drops. // // HandleCDSResponse and HandleEDSResponse must be called by the same goroutine. func (xdsB *EDSBalancer) HandleEDSResponse(edsResp *xdspb.ClusterLoadAssignment) { // Create balancer group if it's never created (this is the first EDS // response). if xdsB.bg == nil { xdsB.bg = newBalancerGroup(xdsB) } // TODO: Unhandled fields from EDS response: // - edsResp.GetPolicy().GetOverprovisioningFactor() // - locality.GetPriority() // - lbEndpoint.GetMetadata(): contains BNS name, send to sub-balancers // - as service config or as resolved address // - if socketAddress is not ip:port // - socketAddress.GetNamedPort(), socketAddress.GetResolverName() // - resolve endpoint's name with another resolver xdsB.updateDrops(edsResp.GetPolicy().GetDropOverloads()) // newLocalitiesSet contains all names of localitis in the new EDS response. // It's used to delete localities that are removed in the new EDS response. newLocalitiesSet := make(map[string]struct{}) for _, locality := range edsResp.Endpoints { // One balancer for each locality. l := locality.GetLocality() if l == nil { grpclog.Warningf("xds: received LocalityLbEndpoints with Locality") continue } lid := fmt.Sprintf("%s-%s-%s", l.Region, l.Zone, l.SubZone) newLocalitiesSet[lid] = struct{}{} newWeight := locality.GetLoadBalancingWeight().GetValue() if newWeight == 0 { // Weight can never be 0. newWeight = 1 } var newAddrs []resolver.Address for _, lbEndpoint := range locality.GetLbEndpoints() { socketAddress := lbEndpoint.GetEndpoint().GetAddress().GetSocketAddress() newAddrs = append(newAddrs, resolver.Address{ Addr: net.JoinHostPort(socketAddress.GetAddress(), strconv.Itoa(int(socketAddress.GetPortValue()))), }) } var weightChanged, addrsChanged bool config, ok := xdsB.lidToConfig[lid] if !ok { // A new balancer, add it to balancer group and balancer map. xdsB.bg.add(lid, newWeight, xdsB.subBalancerBuilder) config = &localityConfig{ weight: newWeight, } xdsB.lidToConfig[lid] = config // weightChanged is false for new locality, because there's no need to // update weight in bg. addrsChanged = true } else { // Compare weight and addrs. if config.weight != newWeight { weightChanged = true } if !reflect.DeepEqual(config.addrs, newAddrs) { addrsChanged = true } } if weightChanged { config.weight = newWeight xdsB.bg.changeWeight(lid, newWeight) } if addrsChanged { config.addrs = newAddrs xdsB.bg.handleResolvedAddrs(lid, newAddrs) } } // Delete localities that are removed in the latest response. for lid := range xdsB.lidToConfig { if _, ok := newLocalitiesSet[lid]; !ok { xdsB.bg.remove(lid) delete(xdsB.lidToConfig, lid) } } } // HandleSubConnStateChange handles the state change and update pickers accordingly. func (xdsB *EDSBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { xdsB.bg.handleSubConnStateChange(sc, s) } // UpdateBalancerState overrides balancer.ClientConn to wrap the picker in a // dropPicker. func (xdsB *EDSBalancer) UpdateBalancerState(s connectivity.State, p balancer.Picker) { xdsB.pickerMu.Lock() defer xdsB.pickerMu.Unlock() xdsB.innerPicker = p xdsB.innerState = s // Don't reset drops when it's a state change. xdsB.ClientConn.UpdateBalancerState(s, newDropPicker(p, xdsB.drops)) } // Close closes the balancer. func (xdsB *EDSBalancer) Close() { xdsB.bg.close() } type dropPicker struct { drops []*dropper p balancer.Picker } func newDropPicker(p balancer.Picker, drops []*dropper) *dropPicker { return &dropPicker{ drops: drops, p: p, } } func (d *dropPicker) Pick(ctx context.Context, opts balancer.PickOptions) (conn balancer.SubConn, done func(balancer.DoneInfo), err error) { var drop bool for _, dp := range d.drops { // It's necessary to call drop on all droppers if the droppers are // stateful. For example, if the second drop only drops 1/2, and only // drops even number picks, we need to call it's drop() even if the // first dropper already returned true. // // It won't be necessary if droppers are stateless, like toss a coin. drop = drop || dp.drop() } if drop { return nil, nil, status.Errorf(codes.Unavailable, "RPC is dropped") } // TODO: (eds) don't drop unless the inner picker is READY. Similar to // https://github.com/grpc/grpc-go/issues/2622. return d.p.Pick(ctx, opts) }