backfeed/main.go

package main

import (
	"context"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	redisprom "github.com/globocom/go-redis-prometheus"
	"github.com/go-redis/redis/v8"
	"github.com/gorilla/mux"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/tevino/abool/v2"
)

const (
	ItemChannelBuffer = 100000
	ItemWrapSize      = 100000
)

func main() {
	log.SetFlags(log.Flags() | log.Lshortfile)

	trackerRedisOptions, err := redis.ParseURL(os.Getenv("REDIS_TRACKER"))
	if err != nil {
		log.Panicf("invalid REDIS_TRACKER url: %s", err)
	}
	trackerRedisOptions.ReadTimeout = 15 * time.Minute
	trackerRedisClient := redis.NewClient(trackerRedisOptions)

	backfeedRedisClient := redis.NewClusterClient(&redis.ClusterOptions{
		Addrs:       strings.Split(os.Getenv("REDIS_BACKFEED_ADDRS"), ","),
		Username:    os.Getenv("REDIS_BACKFEED_USERNAME"),
		Password:    os.Getenv("REDIS_BACKFEED_PASSWORD"),
		ReadTimeout: 15 * time.Minute,
		PoolSize:    256,
	})

	backfeedRedisMetricsHook := redisprom.NewHook(
		redisprom.WithInstanceName("backfeed"),
	)
	backfeedRedisClient.AddHook(backfeedRedisMetricsHook)
	trackerRedisMetricsHook := redisprom.NewHook(
		redisprom.WithInstanceName("tracker"),
	)
	trackerRedisClient.AddHook(trackerRedisMetricsHook)

	if err := trackerRedisClient.Ping(context.Background()).Err(); err != nil {
		log.Panicf("unable to ping tracker redis: %s", err)
	}
	if err := backfeedRedisClient.Ping(context.Background()).Err(); err != nil {
		log.Panicf("unable to ping backfeed redis: %s", err)
	}
	err = backfeedRedisClient.ForEachShard(context.Background(), func(ctx context.Context, client *redis.Client) error {
		client.ClientGetName(ctx)
		return client.Ping(ctx).Err()
	})

	globalBackfeedManager := &GlobalBackfeedManager{
		ActiveFeeds:   map[string]*ProjectBackfeedManager{},
		ActiveSlugs:   map[string]string{},
		TrackerRedis:  trackerRedisClient,
		BackfeedRedis: backfeedRedisClient,
		Populated:     abool.New(),
	}

	globalBackfeedManager.Context, globalBackfeedManager.Cancel = context.WithCancel(context.Background())
	defer globalBackfeedManager.CancelAllFeeds()

	if err := globalBackfeedManager.RefreshFeeds(); err != nil {
		log.Panicf("unable to set up backfeed projects: %s", err)
	}
	r := mux.NewRouter()
	r.Methods(http.MethodPost).Path("/legacy/{slug}").HandlerFunc(globalBackfeedManager.HandleLegacy)
	r.Methods(http.MethodGet).Path("/ping").HandlerFunc(globalBackfeedManager.HandlePing)
	r.Methods(http.MethodGet).Path("/health").HandlerFunc(globalBackfeedManager.HandleHealth)
	r.Methods(http.MethodGet).Path("/manage/lastaccessstats").HandlerFunc(globalBackfeedManager.HandleLastAccessStats)
	r.Methods(http.MethodGet).Path("/manage/dump/{key}").HandlerFunc(globalBackfeedManager.HandleDump)
	r.Methods(http.MethodPut).Path("/manage/load").HandlerFunc(globalBackfeedManager.HandleLoad)
	r.Methods(http.MethodDelete).Path("/manage/unlink/{key}").HandlerFunc(globalBackfeedManager.HandleUnlink)
	r.Methods(http.MethodGet).Path("/manage/redisinfo").HandlerFunc(globalBackfeedManager.HandleRedisInfo)
	rMetrics := mux.NewRouter()
	rMetrics.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux)
	rMetrics.Path("/metrics").Handler(promhttp.Handler())
	doneChan := make(chan bool)
	serveErrChan := make(chan error)
	go func() {
		s := &http.Server{
			Addr:           os.Getenv("HTTP_ADDR"),
			IdleTimeout:    1 * time.Hour,
			MaxHeaderBytes: 1 * 1024 * 1024,
			Handler:        r,
		}
		serveErrChan <- s.ListenAndServe()
	}()
	metricsErrChan := make(chan error)
	go func() {
		if os.Getenv("METRICS_ADDR") != "" {
			s := &http.Server{
				Addr:           os.Getenv("METRICS_ADDR"),
				IdleTimeout:    1 * time.Hour,
				MaxHeaderBytes: 1 * 1024 * 1024,
				Handler:        rMetrics,
			}
			metricsErrChan <- s.ListenAndServe()
		} else {
			<-doneChan
			metricsErrChan <- nil
		}
	}()
	log.Printf("backfeed listening on %s", os.Getenv("HTTP_ADDR"))
	if os.Getenv("METRICS_ADDR") != "" {
		log.Printf("metrics/debug listening on %s", os.Getenv("METRICS_ADDR"))
	}
	sc := make(chan os.Signal, 1)
	signal.Notify(sc, syscall.SIGINT, syscall.SIGTERM, os.Interrupt, os.Kill)
	ticker := time.NewTicker(1 * time.Second)
	for {
		select {
		case <-sc:
			return
		case <-ticker.C:
		}
		if err := globalBackfeedManager.RefreshFeeds(); err != nil {
			log.Printf("unable to refresh backfeed projects: %s", err)
		}
	}
}