package main import ( "context" "log" "net/http" _ "net/http/pprof" "os" "os/signal" "strings" "syscall" "time" redisprom "github.com/globocom/go-redis-prometheus" "github.com/go-redis/redis/v8" "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/tevino/abool/v2" ) const ( ItemChannelBuffer = 100000 ItemWrapSize = 100000 ) func main() { log.SetFlags(log.Flags() | log.Lshortfile) trackerRedisOptions, err := redis.ParseURL(os.Getenv("REDIS_TRACKER")) if err != nil { log.Panicf("invalid REDIS_TRACKER url: %s", err) } trackerRedisOptions.ReadTimeout = 15 * time.Minute trackerRedisClient := redis.NewClient(trackerRedisOptions) backfeedRedisClient := redis.NewClusterClient(&redis.ClusterOptions{ Addrs: strings.Split(os.Getenv("REDIS_BACKFEED_ADDRS"), ","), Username: os.Getenv("REDIS_BACKFEED_USERNAME"), Password: os.Getenv("REDIS_BACKFEED_PASSWORD"), ReadTimeout: 15 * time.Minute, PoolSize: 256, }) backfeedRedisMetricsHook := redisprom.NewHook( redisprom.WithInstanceName("backfeed"), ) backfeedRedisClient.AddHook(backfeedRedisMetricsHook) trackerRedisMetricsHook := redisprom.NewHook( redisprom.WithInstanceName("tracker"), ) trackerRedisClient.AddHook(trackerRedisMetricsHook) if err := trackerRedisClient.Ping(context.Background()).Err(); err != nil { log.Panicf("unable to ping tracker redis: %s", err) } if err := backfeedRedisClient.Ping(context.Background()).Err(); err != nil { log.Panicf("unable to ping backfeed redis: %s", err) } err = backfeedRedisClient.ForEachShard(context.Background(), func(ctx context.Context, client *redis.Client) error { client.ClientGetName(ctx) return client.Ping(ctx).Err() }) globalBackfeedManager := &GlobalBackfeedManager{ ActiveFeeds: map[string]*ProjectBackfeedManager{}, ActiveSlugs: map[string]string{}, TrackerRedis: trackerRedisClient, BackfeedRedis: backfeedRedisClient, Populated: abool.New(), } globalBackfeedManager.Context, globalBackfeedManager.Cancel = context.WithCancel(context.Background()) defer globalBackfeedManager.CancelAllFeeds() if err := globalBackfeedManager.RefreshFeeds(); err != nil { log.Panicf("unable to set up backfeed projects: %s", err) } r := mux.NewRouter() r.Methods(http.MethodPost).Path("/legacy/{slug}").HandlerFunc(globalBackfeedManager.HandleLegacy) r.Methods(http.MethodGet).Path("/ping").HandlerFunc(globalBackfeedManager.HandlePing) r.Methods(http.MethodGet).Path("/health").HandlerFunc(globalBackfeedManager.HandleHealth) r.Methods(http.MethodGet).Path("/manage/lastaccessstats").HandlerFunc(globalBackfeedManager.HandleLastAccessStats) r.Methods(http.MethodGet).Path("/manage/dump/{key}").HandlerFunc(globalBackfeedManager.HandleDump) r.Methods(http.MethodPut).Path("/manage/load").HandlerFunc(globalBackfeedManager.HandleLoad) r.Methods(http.MethodDelete).Path("/manage/unlink/{key}").HandlerFunc(globalBackfeedManager.HandleUnlink) r.Methods(http.MethodGet).Path("/manage/redisinfo").HandlerFunc(globalBackfeedManager.HandleRedisInfo) rMetrics := mux.NewRouter() rMetrics.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux) rMetrics.Path("/metrics").Handler(promhttp.Handler()) doneChan := make(chan bool) serveErrChan := make(chan error) go func() { s := &http.Server{ Addr: os.Getenv("HTTP_ADDR"), IdleTimeout: 1 * time.Hour, MaxHeaderBytes: 1 * 1024 * 1024, Handler: r, } serveErrChan <- s.ListenAndServe() }() metricsErrChan := make(chan error) go func() { if os.Getenv("METRICS_ADDR") != "" { s := &http.Server{ Addr: os.Getenv("METRICS_ADDR"), IdleTimeout: 1 * time.Hour, MaxHeaderBytes: 1 * 1024 * 1024, Handler: rMetrics, } metricsErrChan <- s.ListenAndServe() } else { <-doneChan metricsErrChan <- nil } }() log.Printf("backfeed listening on %s", os.Getenv("HTTP_ADDR")) if os.Getenv("METRICS_ADDR") != "" { log.Printf("metrics/debug listening on %s", os.Getenv("METRICS_ADDR")) } sc := make(chan os.Signal, 1) signal.Notify(sc, syscall.SIGINT, syscall.SIGTERM, os.Interrupt, os.Kill) ticker := time.NewTicker(1 * time.Second) for { select { case <-sc: return case <-ticker.C: } if err := globalBackfeedManager.RefreshFeeds(); err != nil { log.Printf("unable to refresh backfeed projects: %s", err) } } }