|
- // Copyright 2013 The Go Authors. All rights reserved.
- //
- // Use of this source code is governed by a BSD-style
- // license that can be found in the LICENSE file or at
- // https://developers.google.com/open-source/licenses/bsd.
-
- package main
-
- import (
- "fmt"
- "io/ioutil"
- "log"
- "os"
- "strings"
-
- "github.com/garyburd/redigo/redis"
- "github.com/golang/gddo/database"
- )
-
- var crawlCommand = &command{
- name: "crawl",
- run: crawl,
- usage: "crawl [new]",
- }
-
- func crawl(c *command) {
- if len(c.flag.Args()) > 1 {
- c.printUsage()
- os.Exit(1)
- }
- db, err := database.New()
- if err != nil {
- log.Fatal(err)
- }
-
- if len(c.flag.Args()) == 1 {
- p, err := ioutil.ReadFile(c.flag.Args()[0])
- if err != nil {
- log.Fatal(err)
- }
- for _, p := range strings.Fields(string(p)) {
- db.AddNewCrawl(p)
- }
- }
-
- conn := db.Pool.Get()
- defer conn.Close()
- paths, err := redis.Strings(conn.Do("SMEMBERS", "newCrawl"))
- if err != nil {
- log.Fatal(err)
- }
- fmt.Println("NEW")
- for _, path := range paths {
- fmt.Println(path)
- }
-
- paths, err = redis.Strings(conn.Do("SMEMBERS", "badCrawl"))
- if err != nil {
- log.Fatal(err)
- }
- fmt.Println("BAD")
- for _, path := range paths {
- fmt.Println(path)
- }
- }
|