|
- // Copyright 2017 The Go Authors. All rights reserved.
- //
- // Use of this source code is governed by a BSD-style
- // license that can be found in the LICENSE file or at
- // https://developers.google.com/open-source/licenses/bsd.
-
- package main
-
- import (
- "log"
- "time"
-
- "github.com/spf13/viper"
- "google.golang.org/appengine"
-
- "github.com/golang/gddo/database"
- "github.com/golang/gddo/gosrc"
- )
-
- type BackgroundTask struct {
- name string
- fn func() error
- interval time.Duration
- next time.Time
- }
-
- func runBackgroundTasks() {
- defer log.Println("ERROR: Background exiting!")
-
- var backgroundTasks = []BackgroundTask{
- {
- name: "GitHub updates",
- fn: readGitHubUpdates,
- interval: viper.GetDuration(ConfigGithubInterval),
- },
- {
- name: "Crawl",
- fn: doCrawl,
- interval: viper.GetDuration(ConfigCrawlInterval),
- },
- }
-
- sleep := time.Minute
- for _, task := range backgroundTasks {
- if task.interval > 0 && sleep > task.interval {
- sleep = task.interval
- }
- }
-
- for {
- for _, task := range backgroundTasks {
- start := time.Now()
- if task.interval > 0 && start.After(task.next) {
- if err := task.fn(); err != nil {
- log.Printf("Task %s: %v", task.name, err)
- }
- task.next = time.Now().Add(task.interval)
- }
- }
- time.Sleep(sleep)
- }
- }
-
- func doCrawl() error {
- // Look for new package to crawl.
- importPath, hasSubdirs, err := db.PopNewCrawl()
- if err != nil {
- log.Printf("db.PopNewCrawl() returned error %v", err)
- return nil
- }
- if importPath != "" {
- if pdoc, err := crawlDoc("new", importPath, nil, hasSubdirs, time.Time{}); pdoc == nil && err == nil {
- if err := db.AddBadCrawl(importPath); err != nil {
- log.Printf("ERROR db.AddBadCrawl(%q): %v", importPath, err)
- }
- }
- return nil
- }
-
- // Crawl existing doc.
- pdoc, pkgs, nextCrawl, err := db.Get("-")
- if err != nil {
- log.Printf("db.Get(\"-\") returned error %v", err)
- return nil
- }
- if pdoc == nil || nextCrawl.After(time.Now()) {
- return nil
- }
- if _, err = crawlDoc("crawl", pdoc.ImportPath, pdoc, len(pkgs) > 0, nextCrawl); err != nil {
- // Touch package so that crawl advances to next package.
- if err := db.SetNextCrawl(pdoc.ImportPath, time.Now().Add(viper.GetDuration(ConfigMaxAge)/3)); err != nil {
- log.Printf("ERROR db.SetNextCrawl(%q): %v", pdoc.ImportPath, err)
- }
- }
- return nil
- }
-
- func readGitHubUpdates() error {
- const key = "gitHubUpdates"
- var last string
- if err := db.GetGob(key, &last); err != nil {
- return err
- }
- last, names, err := gosrc.GetGitHubUpdates(httpClient, last)
- if err != nil {
- return err
- }
-
- for _, name := range names {
- log.Printf("bump crawl github.com/%s", name)
- if err := db.BumpCrawl("github.com/" + name); err != nil {
- log.Println("ERROR force crawl:", err)
- }
- }
-
- if err := db.PutGob(key, last); err != nil {
- return err
- }
- return nil
- }
-
- func reindex() {
- c := appengine.BackgroundContext()
- if err := db.Reindex(c); err != nil {
- log.Println("reindex:", err)
- }
- }
-
- func purgeIndex() {
- c := appengine.BackgroundContext()
- if err := database.PurgeIndex(c); err != nil {
- log.Println("purgeIndex:", err)
- }
- }
|