|
- /*
- * Copyright 2019 gRPC authors.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- package edsbalancer
-
- import (
- "context"
- "sync"
-
- "google.golang.org/grpc/balancer"
- "google.golang.org/grpc/balancer/base"
- "google.golang.org/grpc/connectivity"
- "google.golang.org/grpc/grpclog"
- "google.golang.org/grpc/resolver"
- )
-
- type pickerState struct {
- weight uint32
- picker balancer.Picker
- state connectivity.State
- }
-
- // balancerGroup takes a list of balancers, and make then into one balancer.
- //
- // Note that this struct doesn't implement balancer.Balancer, because it's not
- // intended to be used directly as a balancer. It's expected to be used as a
- // sub-balancer manager by a high level balancer.
- //
- // Updates from ClientConn are forwarded to sub-balancers
- // - service config update
- // - Not implemented
- // - address update
- // - subConn state change
- // - find the corresponding balancer and forward
- //
- // Actions from sub-balances are forwarded to parent ClientConn
- // - new/remove SubConn
- // - picker update and health states change
- // - sub-pickers are grouped into a group-picker
- // - aggregated connectivity state is the overall state of all pickers.
- // - resolveNow
- type balancerGroup struct {
- cc balancer.ClientConn
-
- mu sync.Mutex
- idToBalancer map[string]balancer.Balancer
- scToID map[balancer.SubConn]string
-
- pickerMu sync.Mutex
- // All balancer IDs exist as keys in this map. If an ID is not in map, it's
- // either removed or never added.
- idToPickerState map[string]*pickerState
- }
-
- func newBalancerGroup(cc balancer.ClientConn) *balancerGroup {
- return &balancerGroup{
- cc: cc,
-
- scToID: make(map[balancer.SubConn]string),
- idToBalancer: make(map[string]balancer.Balancer),
- idToPickerState: make(map[string]*pickerState),
- }
- }
-
- // add adds a balancer built by builder to the group, with given id and weight.
- func (bg *balancerGroup) add(id string, weight uint32, builder balancer.Builder) {
- bg.mu.Lock()
- if _, ok := bg.idToBalancer[id]; ok {
- bg.mu.Unlock()
- grpclog.Warningf("balancer group: adding a balancer with existing ID: %s", id)
- return
- }
- bg.mu.Unlock()
- bgcc := &balancerGroupCC{
- id: id,
- group: bg,
- }
- b := builder.Build(bgcc, balancer.BuildOptions{})
- bg.mu.Lock()
- bg.idToBalancer[id] = b
- bg.mu.Unlock()
-
- bg.pickerMu.Lock()
- bg.idToPickerState[id] = &pickerState{
- weight: weight,
- // Start everything in IDLE. It's doesn't affect the overall state
- // because we don't count IDLE when aggregating (as opposite to e.g.
- // READY, 1 READY results in overall READY).
- state: connectivity.Idle,
- }
- bg.pickerMu.Unlock()
- }
-
- // remove removes the balancer with id from the group, and closes the balancer.
- //
- // It also removes the picker generated from this balancer from the picker
- // group. It always results in a picker update.
- func (bg *balancerGroup) remove(id string) {
- bg.mu.Lock()
- // Close balancer.
- if b, ok := bg.idToBalancer[id]; ok {
- b.Close()
- delete(bg.idToBalancer, id)
- }
- // Remove SubConns.
- for sc, bid := range bg.scToID {
- if bid == id {
- bg.cc.RemoveSubConn(sc)
- delete(bg.scToID, sc)
- }
- }
- bg.mu.Unlock()
-
- bg.pickerMu.Lock()
- // Remove id and picker from picker map. This also results in future updates
- // for this ID to be ignored.
- delete(bg.idToPickerState, id)
- // Update state and picker to reflect the changes.
- bg.cc.UpdateBalancerState(buildPickerAndState(bg.idToPickerState))
- bg.pickerMu.Unlock()
- }
-
- // changeWeight changes the weight of the balancer.
- //
- // NOTE: It always results in a picker update now. This probably isn't
- // necessary. But it seems better to do the update because it's a change in the
- // picker (which is balancer's snapshot).
- func (bg *balancerGroup) changeWeight(id string, newWeight uint32) {
- bg.pickerMu.Lock()
- defer bg.pickerMu.Unlock()
- pState, ok := bg.idToPickerState[id]
- if !ok {
- return
- }
- if pState.weight == newWeight {
- return
- }
- pState.weight = newWeight
- // Update state and picker to reflect the changes.
- bg.cc.UpdateBalancerState(buildPickerAndState(bg.idToPickerState))
- }
-
- // Following are actions from the parent grpc.ClientConn, forward to sub-balancers.
-
- // SubConn state change: find the corresponding balancer and then forward.
- func (bg *balancerGroup) handleSubConnStateChange(sc balancer.SubConn, state connectivity.State) {
- grpclog.Infof("balancer group: handle subconn state change: %p, %v", sc, state)
- bg.mu.Lock()
- var b balancer.Balancer
- if id, ok := bg.scToID[sc]; ok {
- if state == connectivity.Shutdown {
- // Only delete sc from the map when state changed to Shutdown.
- delete(bg.scToID, sc)
- }
- b = bg.idToBalancer[id]
- }
- bg.mu.Unlock()
- if b == nil {
- grpclog.Infof("balancer group: balancer not found for sc state change")
- return
- }
- b.HandleSubConnStateChange(sc, state)
- }
-
- // Address change: forward to balancer.
- func (bg *balancerGroup) handleResolvedAddrs(id string, addrs []resolver.Address) {
- bg.mu.Lock()
- b, ok := bg.idToBalancer[id]
- bg.mu.Unlock()
- if !ok {
- grpclog.Infof("balancer group: balancer with id %q not found", id)
- return
- }
- b.HandleResolvedAddrs(addrs, nil)
- }
-
- // TODO: handleServiceConfig()
- //
- // For BNS address for slicer, comes from endpoint.Metadata. It will be sent
- // from parent to sub-balancers as service config.
-
- // Following are actions from sub-balancers, forward to ClientConn.
-
- // newSubConn: forward to ClientConn, and also create a map from sc to balancer,
- // so state update will find the right balancer.
- //
- // One note about removing SubConn: only forward to ClientConn, but not delete
- // from map. Delete sc from the map only when state changes to Shutdown. Since
- // it's just forwarding the action, there's no need for a removeSubConn()
- // wrapper function.
- func (bg *balancerGroup) newSubConn(id string, addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
- sc, err := bg.cc.NewSubConn(addrs, opts)
- if err != nil {
- return nil, err
- }
- bg.mu.Lock()
- bg.scToID[sc] = id
- bg.mu.Unlock()
- return sc, nil
- }
-
- // updateBalancerState: create an aggregated picker and an aggregated
- // connectivity state, then forward to ClientConn.
- func (bg *balancerGroup) updateBalancerState(id string, state connectivity.State, picker balancer.Picker) {
- grpclog.Infof("balancer group: update balancer state: %v, %v, %p", id, state, picker)
- bg.pickerMu.Lock()
- defer bg.pickerMu.Unlock()
- pickerSt, ok := bg.idToPickerState[id]
- if !ok {
- // All state starts in IDLE. If ID is not in map, it's either removed,
- // or never existed.
- grpclog.Infof("balancer group: pickerState not found when update picker/state")
- return
- }
- pickerSt.picker = picker
- pickerSt.state = state
- bg.cc.UpdateBalancerState(buildPickerAndState(bg.idToPickerState))
- }
-
- func (bg *balancerGroup) close() {
- bg.mu.Lock()
- for _, b := range bg.idToBalancer {
- b.Close()
- }
- // Also remove all SubConns.
- for sc := range bg.scToID {
- bg.cc.RemoveSubConn(sc)
- }
- bg.mu.Unlock()
- }
-
- func buildPickerAndState(m map[string]*pickerState) (connectivity.State, balancer.Picker) {
- var readyN, connectingN int
- readyPickerWithWeights := make([]pickerState, 0, len(m))
- for _, ps := range m {
- switch ps.state {
- case connectivity.Ready:
- readyN++
- readyPickerWithWeights = append(readyPickerWithWeights, *ps)
- case connectivity.Connecting:
- connectingN++
- }
- }
- var aggregatedState connectivity.State
- switch {
- case readyN > 0:
- aggregatedState = connectivity.Ready
- case connectingN > 0:
- aggregatedState = connectivity.Connecting
- default:
- aggregatedState = connectivity.TransientFailure
- }
- if aggregatedState == connectivity.TransientFailure {
- return aggregatedState, base.NewErrPicker(balancer.ErrTransientFailure)
- }
- return aggregatedState, newPickerGroup(readyPickerWithWeights)
- }
-
- type pickerGroup struct {
- readyPickerWithWeights []pickerState
- length int
-
- mu sync.Mutex
- idx int // The index of the picker that will be picked
- count uint32 // The number of times the current picker has been picked.
- }
-
- // newPickerGroup takes pickers with weights, and group them into one picker.
- //
- // Note it only takes ready pickers. The map shouldn't contain non-ready
- // pickers.
- //
- // TODO: (bg) confirm this is the expected behavior: non-ready balancers should
- // be ignored when picking. Only ready balancers are picked.
- func newPickerGroup(readyPickerWithWeights []pickerState) *pickerGroup {
- return &pickerGroup{
- readyPickerWithWeights: readyPickerWithWeights,
- length: len(readyPickerWithWeights),
- }
- }
-
- func (pg *pickerGroup) Pick(ctx context.Context, opts balancer.PickOptions) (conn balancer.SubConn, done func(balancer.DoneInfo), err error) {
- if pg.length <= 0 {
- return nil, nil, balancer.ErrNoSubConnAvailable
- }
- // TODO: the WRR algorithm needs a design.
- // MAYBE: move WRR implmentation to util.go as a separate struct.
- pg.mu.Lock()
- pickerSt := pg.readyPickerWithWeights[pg.idx]
- p := pickerSt.picker
- pg.count++
- if pg.count >= pickerSt.weight {
- pg.idx = (pg.idx + 1) % pg.length
- pg.count = 0
- }
- pg.mu.Unlock()
- return p.Pick(ctx, opts)
- }
-
- // balancerGroupCC implements the balancer.ClientConn API and get passed to each
- // sub-balancer. It contains the sub-balancer ID, so the parent balancer can
- // keep track of SubConn/pickers and the sub-balancers they belong to.
- //
- // Some of the actions are forwarded to the parent ClientConn with no change.
- // Some are forward to balancer group with the sub-balancer ID.
- type balancerGroupCC struct {
- id string
- group *balancerGroup
- }
-
- func (bgcc *balancerGroupCC) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
- return bgcc.group.newSubConn(bgcc.id, addrs, opts)
- }
- func (bgcc *balancerGroupCC) RemoveSubConn(sc balancer.SubConn) {
- bgcc.group.cc.RemoveSubConn(sc)
- }
- func (bgcc *balancerGroupCC) UpdateBalancerState(state connectivity.State, picker balancer.Picker) {
- bgcc.group.updateBalancerState(bgcc.id, state, picker)
- }
- func (bgcc *balancerGroupCC) ResolveNow(opt resolver.ResolveNowOption) {
- bgcc.group.cc.ResolveNow(opt)
- }
- func (bgcc *balancerGroupCC) Target() string {
- return bgcc.group.cc.Target()
- }
|