transfer.sh/vendor/google.golang.org/grpc/balancer/xds/edsbalancer/balancergroup_test.go

/*
 * Copyright 2019 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package edsbalancer

import (
	"context"
	"reflect"
	"testing"

	"google.golang.org/grpc/balancer"
	"google.golang.org/grpc/balancer/roundrobin"
	"google.golang.org/grpc/connectivity"
	"google.golang.org/grpc/resolver"
)

var (
	rrBuilder        = balancer.Get(roundrobin.Name)
	testBalancerIDs  = []string{"b1", "b2", "b3"}
	testBackendAddrs = []resolver.Address{{Addr: "1.1.1.1:1"}, {Addr: "2.2.2.2:2"}, {Addr: "3.3.3.3:3"}, {Addr: "4.4.4.4:4"}}
)

// 1 balancer, 1 backend -> 2 backends -> 1 backend.
func TestBalancerGroup_OneRR_AddRemoveBackend(t *testing.T) {
	cc := newTestClientConn(t)
	bg := newBalancerGroup(cc)

	// Add one balancer to group.
	bg.add(testBalancerIDs[0], 1, rrBuilder)
	// Send one resolved address.
	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:1])

	// Send subconn state change.
	sc1 := <-cc.newSubConnCh
	bg.handleSubConnStateChange(sc1, connectivity.Connecting)
	bg.handleSubConnStateChange(sc1, connectivity.Ready)

	// Test pick with one backend.
	p1 := <-cc.newPickerCh
	for i := 0; i < 5; i++ {
		gotSC, _, _ := p1.Pick(context.Background(), balancer.PickOptions{})
		if !reflect.DeepEqual(gotSC, sc1) {
			t.Fatalf("picker.Pick, got %v, want %v", gotSC, sc1)
		}
	}

	// Send two addresses.
	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
	// Expect one new subconn, send state update.
	sc2 := <-cc.newSubConnCh
	bg.handleSubConnStateChange(sc2, connectivity.Connecting)
	bg.handleSubConnStateChange(sc2, connectivity.Ready)

	// Test roundrobin pick.
	p2 := <-cc.newPickerCh
	want := []balancer.SubConn{sc1, sc2}
	if err := isRoundRobin(want, func() balancer.SubConn {
		sc, _, _ := p2.Pick(context.Background(), balancer.PickOptions{})
		return sc
	}); err != nil {
		t.Fatalf("want %v, got %v", want, err)
	}

	// Remove the first address.
	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[1:2])
	scToRemove := <-cc.removeSubConnCh
	if !reflect.DeepEqual(scToRemove, sc1) {
		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
	}
	bg.handleSubConnStateChange(scToRemove, connectivity.Shutdown)

	// Test pick with only the second subconn.
	p3 := <-cc.newPickerCh
	for i := 0; i < 5; i++ {
		gotSC, _, _ := p3.Pick(context.Background(), balancer.PickOptions{})
		if !reflect.DeepEqual(gotSC, sc2) {
			t.Fatalf("picker.Pick, got %v, want %v", gotSC, sc2)
		}
	}
}

// 2 balancers, each with 1 backend.
func TestBalancerGroup_TwoRR_OneBackend(t *testing.T) {
	cc := newTestClientConn(t)
	bg := newBalancerGroup(cc)

	// Add two balancers to group and send one resolved address to both
	// balancers.
	bg.add(testBalancerIDs[0], 1, rrBuilder)
	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:1])
	sc1 := <-cc.newSubConnCh

	bg.add(testBalancerIDs[1], 1, rrBuilder)
	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[0:1])
	sc2 := <-cc.newSubConnCh

	// Send state changes for both subconns.
	bg.handleSubConnStateChange(sc1, connectivity.Connecting)
	bg.handleSubConnStateChange(sc1, connectivity.Ready)
	bg.handleSubConnStateChange(sc2, connectivity.Connecting)
	bg.handleSubConnStateChange(sc2, connectivity.Ready)

	// Test roundrobin on the last picker.
	p1 := <-cc.newPickerCh
	want := []balancer.SubConn{sc1, sc2}
	if err := isRoundRobin(want, func() balancer.SubConn {
		sc, _, _ := p1.Pick(context.Background(), balancer.PickOptions{})
		return sc
	}); err != nil {
		t.Fatalf("want %v, got %v", want, err)
	}
}

// 2 balancers, each with more than 1 backends.
func TestBalancerGroup_TwoRR_MoreBackends(t *testing.T) {
	cc := newTestClientConn(t)
	bg := newBalancerGroup(cc)

	// Add two balancers to group and send one resolved address to both
	// balancers.
	bg.add(testBalancerIDs[0], 1, rrBuilder)
	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
	sc1 := <-cc.newSubConnCh
	sc2 := <-cc.newSubConnCh

	bg.add(testBalancerIDs[1], 1, rrBuilder)
	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
	sc3 := <-cc.newSubConnCh
	sc4 := <-cc.newSubConnCh

	// Send state changes for both subconns.
	bg.handleSubConnStateChange(sc1, connectivity.Connecting)
	bg.handleSubConnStateChange(sc1, connectivity.Ready)
	bg.handleSubConnStateChange(sc2, connectivity.Connecting)
	bg.handleSubConnStateChange(sc2, connectivity.Ready)
	bg.handleSubConnStateChange(sc3, connectivity.Connecting)
	bg.handleSubConnStateChange(sc3, connectivity.Ready)
	bg.handleSubConnStateChange(sc4, connectivity.Connecting)
	bg.handleSubConnStateChange(sc4, connectivity.Ready)

	// Test roundrobin on the last picker.
	p1 := <-cc.newPickerCh
	want := []balancer.SubConn{sc1, sc2, sc3, sc4}
	if err := isRoundRobin(want, func() balancer.SubConn {
		sc, _, _ := p1.Pick(context.Background(), balancer.PickOptions{})
		return sc
	}); err != nil {
		t.Fatalf("want %v, got %v", want, err)
	}

	// Turn sc2's connection down, should be RR between balancers.
	bg.handleSubConnStateChange(sc2, connectivity.TransientFailure)
	p2 := <-cc.newPickerCh
	// Expect two sc1's in the result, because balancer1 will be picked twice,
	// but there's only one sc in it.
	want = []balancer.SubConn{sc1, sc1, sc3, sc4}
	if err := isRoundRobin(want, func() balancer.SubConn {
		sc, _, _ := p2.Pick(context.Background(), balancer.PickOptions{})
		return sc
	}); err != nil {
		t.Fatalf("want %v, got %v", want, err)
	}

	// Remove sc3's addresses.
	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[3:4])
	scToRemove := <-cc.removeSubConnCh
	if !reflect.DeepEqual(scToRemove, sc3) {
		t.Fatalf("RemoveSubConn, want %v, got %v", sc3, scToRemove)
	}
	bg.handleSubConnStateChange(scToRemove, connectivity.Shutdown)
	p3 := <-cc.newPickerCh
	want = []balancer.SubConn{sc1, sc4}
	if err := isRoundRobin(want, func() balancer.SubConn {
		sc, _, _ := p3.Pick(context.Background(), balancer.PickOptions{})
		return sc
	}); err != nil {
		t.Fatalf("want %v, got %v", want, err)
	}

	// Turn sc1's connection down.
	bg.handleSubConnStateChange(sc1, connectivity.TransientFailure)
	p4 := <-cc.newPickerCh
	want = []balancer.SubConn{sc4}
	if err := isRoundRobin(want, func() balancer.SubConn {
		sc, _, _ := p4.Pick(context.Background(), balancer.PickOptions{})
		return sc
	}); err != nil {
		t.Fatalf("want %v, got %v", want, err)
	}

	// Turn last connection to connecting.
	bg.handleSubConnStateChange(sc4, connectivity.Connecting)
	p5 := <-cc.newPickerCh
	for i := 0; i < 5; i++ {
		if _, _, err := p5.Pick(context.Background(), balancer.PickOptions{}); err != balancer.ErrNoSubConnAvailable {
			t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err)
		}
	}

	// Turn all connections down.
	bg.handleSubConnStateChange(sc4, connectivity.TransientFailure)
	p6 := <-cc.newPickerCh
	for i := 0; i < 5; i++ {
		if _, _, err := p6.Pick(context.Background(), balancer.PickOptions{}); err != balancer.ErrTransientFailure {
			t.Fatalf("want pick error %v, got %v", balancer.ErrTransientFailure, err)
		}
	}
}

// 2 balancers with different weights.
func TestBalancerGroup_TwoRR_DifferentWeight_MoreBackends(t *testing.T) {
	cc := newTestClientConn(t)
	bg := newBalancerGroup(cc)

	// Add two balancers to group and send two resolved addresses to both
	// balancers.
	bg.add(testBalancerIDs[0], 2, rrBuilder)
	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
	sc1 := <-cc.newSubConnCh
	sc2 := <-cc.newSubConnCh

	bg.add(testBalancerIDs[1], 1, rrBuilder)
	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
	sc3 := <-cc.newSubConnCh
	sc4 := <-cc.newSubConnCh

	// Send state changes for both subconns.
	bg.handleSubConnStateChange(sc1, connectivity.Connecting)
	bg.handleSubConnStateChange(sc1, connectivity.Ready)
	bg.handleSubConnStateChange(sc2, connectivity.Connecting)
	bg.handleSubConnStateChange(sc2, connectivity.Ready)
	bg.handleSubConnStateChange(sc3, connectivity.Connecting)
	bg.handleSubConnStateChange(sc3, connectivity.Ready)
	bg.handleSubConnStateChange(sc4, connectivity.Connecting)
	bg.handleSubConnStateChange(sc4, connectivity.Ready)

	// Test roundrobin on the last picker.
	p1 := <-cc.newPickerCh
	want := []balancer.SubConn{sc1, sc1, sc2, sc2, sc3, sc4}
	if err := isRoundRobin(want, func() balancer.SubConn {
		sc, _, _ := p1.Pick(context.Background(), balancer.PickOptions{})
		return sc
	}); err != nil {
		t.Fatalf("want %v, got %v", want, err)
	}
}

// totally 3 balancers, add/remove balancer.
func TestBalancerGroup_ThreeRR_RemoveBalancer(t *testing.T) {
	cc := newTestClientConn(t)
	bg := newBalancerGroup(cc)

	// Add three balancers to group and send one resolved address to both
	// balancers.
	bg.add(testBalancerIDs[0], 1, rrBuilder)
	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:1])
	sc1 := <-cc.newSubConnCh

	bg.add(testBalancerIDs[1], 1, rrBuilder)
	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[1:2])
	sc2 := <-cc.newSubConnCh

	bg.add(testBalancerIDs[2], 1, rrBuilder)
	bg.handleResolvedAddrs(testBalancerIDs[2], testBackendAddrs[1:2])
	sc3 := <-cc.newSubConnCh

	// Send state changes for both subconns.
	bg.handleSubConnStateChange(sc1, connectivity.Connecting)
	bg.handleSubConnStateChange(sc1, connectivity.Ready)
	bg.handleSubConnStateChange(sc2, connectivity.Connecting)
	bg.handleSubConnStateChange(sc2, connectivity.Ready)
	bg.handleSubConnStateChange(sc3, connectivity.Connecting)
	bg.handleSubConnStateChange(sc3, connectivity.Ready)

	p1 := <-cc.newPickerCh
	want := []balancer.SubConn{sc1, sc2, sc3}
	if err := isRoundRobin(want, func() balancer.SubConn {
		sc, _, _ := p1.Pick(context.Background(), balancer.PickOptions{})
		return sc
	}); err != nil {
		t.Fatalf("want %v, got %v", want, err)
	}

	// Remove the second balancer, while the others two are ready.
	bg.remove(testBalancerIDs[1])
	scToRemove := <-cc.removeSubConnCh
	if !reflect.DeepEqual(scToRemove, sc2) {
		t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove)
	}
	p2 := <-cc.newPickerCh
	want = []balancer.SubConn{sc1, sc3}
	if err := isRoundRobin(want, func() balancer.SubConn {
		sc, _, _ := p2.Pick(context.Background(), balancer.PickOptions{})
		return sc
	}); err != nil {
		t.Fatalf("want %v, got %v", want, err)
	}

	// move balancer 3 into transient failure.
	bg.handleSubConnStateChange(sc3, connectivity.TransientFailure)
	// Remove the first balancer, while the third is transient failure.
	bg.remove(testBalancerIDs[0])
	scToRemove = <-cc.removeSubConnCh
	if !reflect.DeepEqual(scToRemove, sc1) {
		t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove)
	}
	p3 := <-cc.newPickerCh
	for i := 0; i < 5; i++ {
		if _, _, err := p3.Pick(context.Background(), balancer.PickOptions{}); err != balancer.ErrTransientFailure {
			t.Fatalf("want pick error %v, got %v", balancer.ErrTransientFailure, err)
		}
	}
}

// 2 balancers, change balancer weight.
func TestBalancerGroup_TwoRR_ChangeWeight_MoreBackends(t *testing.T) {
	cc := newTestClientConn(t)
	bg := newBalancerGroup(cc)

	// Add two balancers to group and send two resolved addresses to both
	// balancers.
	bg.add(testBalancerIDs[0], 2, rrBuilder)
	bg.handleResolvedAddrs(testBalancerIDs[0], testBackendAddrs[0:2])
	sc1 := <-cc.newSubConnCh
	sc2 := <-cc.newSubConnCh

	bg.add(testBalancerIDs[1], 1, rrBuilder)
	bg.handleResolvedAddrs(testBalancerIDs[1], testBackendAddrs[2:4])
	sc3 := <-cc.newSubConnCh
	sc4 := <-cc.newSubConnCh

	// Send state changes for both subconns.
	bg.handleSubConnStateChange(sc1, connectivity.Connecting)
	bg.handleSubConnStateChange(sc1, connectivity.Ready)
	bg.handleSubConnStateChange(sc2, connectivity.Connecting)
	bg.handleSubConnStateChange(sc2, connectivity.Ready)
	bg.handleSubConnStateChange(sc3, connectivity.Connecting)
	bg.handleSubConnStateChange(sc3, connectivity.Ready)
	bg.handleSubConnStateChange(sc4, connectivity.Connecting)
	bg.handleSubConnStateChange(sc4, connectivity.Ready)

	// Test roundrobin on the last picker.
	p1 := <-cc.newPickerCh
	want := []balancer.SubConn{sc1, sc1, sc2, sc2, sc3, sc4}
	if err := isRoundRobin(want, func() balancer.SubConn {
		sc, _, _ := p1.Pick(context.Background(), balancer.PickOptions{})
		return sc
	}); err != nil {
		t.Fatalf("want %v, got %v", want, err)
	}

	bg.changeWeight(testBalancerIDs[0], 3)

	// Test roundrobin with new weight.
	p2 := <-cc.newPickerCh
	want = []balancer.SubConn{sc1, sc1, sc1, sc2, sc2, sc2, sc3, sc4}
	if err := isRoundRobin(want, func() balancer.SubConn {
		sc, _, _ := p2.Pick(context.Background(), balancer.PickOptions{})
		return sc
	}); err != nil {
		t.Fatalf("want %v, got %v", want, err)
	}
}