// Copyright 2013 The Go Authors. All rights reserved. // // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file or at // https://developers.google.com/open-source/licenses/bsd. package gosrc import ( "encoding/json" "fmt" "net/http" "net/url" "regexp" "strings" "time" ) func init() { addService(&service{ pattern: regexp.MustCompile(`^github\.com/(?P[a-z0-9A-Z_.\-]+)/(?P[a-z0-9A-Z_.\-]+)(?P/.*)?$`), prefix: "github.com/", get: getGitHubDir, getPresentation: getGitHubPresentation, getProject: getGitHubProject, }) addService(&service{ pattern: regexp.MustCompile(`^gist\.github\.com/(?P[a-z0-9A-Z_.\-]+)\.git$`), prefix: "gist.github.com/", get: getGistDir, }) } var ( gitHubRawHeader = http.Header{"Accept": {"application/vnd.github-blob.raw"}} gitHubPreviewHeader = http.Header{"Accept": {"application/vnd.github.preview"}} ownerRepoPat = regexp.MustCompile(`^https://api.github.com/repos/([^/]+)/([^/]+)/`) ) type githubCommit struct { ID string `json:"sha"` Commit struct { Committer struct { Date time.Time `json:"date"` } `json:"committer"` } `json:"commit"` } func gitHubError(resp *http.Response) error { var e struct { Message string `json:"message"` } if err := json.NewDecoder(resp.Body).Decode(&e); err == nil { return &RemoteError{resp.Request.URL.Host, fmt.Errorf("%d: %s (%s)", resp.StatusCode, e.Message, resp.Request.URL.String())} } return &RemoteError{resp.Request.URL.Host, fmt.Errorf("%d: (%s)", resp.StatusCode, resp.Request.URL.String())} } func getGitHubDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) { c := &httpClient{client: client, errFn: gitHubError} var repo struct { Fork bool `json:"fork"` Stars int `json:"stargazers_count"` CreatedAt time.Time `json:"created_at"` PushedAt time.Time `json:"pushed_at"` } if _, err := c.getJSON(expand("https://api.github.com/repos/{owner}/{repo}", match), &repo); err != nil { return nil, err } status := Active var commits []*githubCommit url := expand("https://api.github.com/repos/{owner}/{repo}/commits", match) if match["dir"] != "" { url += fmt.Sprintf("?path=%s", match["dir"]) } if _, err := c.getJSON(url, &commits); err != nil { return nil, err } if len(commits) == 0 { return nil, NotFoundError{Message: "package directory changed or removed"} } lastCommitted := commits[0].Commit.Committer.Date if lastCommitted.Add(ExpiresAfter).Before(time.Now()) { status = NoRecentCommits } else if repo.Fork { if repo.PushedAt.Before(repo.CreatedAt) { status = DeadEndFork } else if isQuickFork(commits, repo.CreatedAt) { status = QuickFork } } if commits[0].ID == savedEtag { return nil, NotModifiedError{ Since: lastCommitted, Status: status, } } var contents []*struct { Type string Name string GitURL string `json:"git_url"` HTMLURL string `json:"html_url"` } if _, err := c.getJSON(expand("https://api.github.com/repos/{owner}/{repo}/contents{dir}", match), &contents); err != nil { // The GitHub content API returns array values for directories // and object values for files. If there's a type mismatch at // the beginning of the response, then assume that the path is // for a file. if e, ok := err.(*json.UnmarshalTypeError); ok && e.Offset == 1 { return nil, NotFoundError{Message: "Not a directory"} } return nil, err } if len(contents) == 0 { return nil, NotFoundError{Message: "No files in directory."} } // GitHub owner and repo names are case-insensitive. Redirect if requested // names do not match the canonical names in API response. if m := ownerRepoPat.FindStringSubmatch(contents[0].GitURL); m != nil && (m[1] != match["owner"] || m[2] != match["repo"]) { match["owner"] = m[1] match["repo"] = m[2] return nil, NotFoundError{ Message: "Github import path has incorrect case.", Redirect: expand("github.com/{owner}/{repo}{dir}", match), } } var files []*File var dataURLs []string var subdirs []string for _, item := range contents { switch { case item.Type == "dir": if isValidPathElement(item.Name) { subdirs = append(subdirs, item.Name) } case isDocFile(item.Name): files = append(files, &File{Name: item.Name, BrowseURL: item.HTMLURL}) dataURLs = append(dataURLs, item.GitURL) } } c.header = gitHubRawHeader if err := c.getFiles(dataURLs, files); err != nil { return nil, err } browseURL := expand("https://github.com/{owner}/{repo}", match) if match["dir"] != "" { browseURL = expand("https://github.com/{owner}/{repo}/tree{dir}", match) } return &Directory{ BrowseURL: browseURL, Etag: commits[0].ID, Files: files, LineFmt: "%s#L%d", ProjectName: match["repo"], ProjectRoot: expand("github.com/{owner}/{repo}", match), ProjectURL: expand("https://github.com/{owner}/{repo}", match), Subdirectories: subdirs, VCS: "git", Status: status, Fork: repo.Fork, Stars: repo.Stars, }, nil } // isQuickFork reports whether the repository is a "quick fork": // it has fewer than 3 commits, all within a week of the repo creation, createdAt. func isQuickFork(commits []*githubCommit, createdAt time.Time) bool { oneWeekOld := createdAt.Add(7 * 24 * time.Hour) if oneWeekOld.After(time.Now()) { return false // a newborn baby of a repository } n := 0 for _, commit := range commits { if commit.Commit.Committer.Date.After(oneWeekOld) { return false } n++ } return n < 3 } func getGitHubPresentation(client *http.Client, match map[string]string) (*Presentation, error) { c := &httpClient{client: client, header: gitHubRawHeader} p, err := c.getBytes(expand("https://api.github.com/repos/{owner}/{repo}/contents{dir}/{file}", match)) if err != nil { return nil, err } apiBase, err := url.Parse(expand("https://api.github.com/repos/{owner}/{repo}/contents{dir}/", match)) if err != nil { return nil, err } rawBase, err := url.Parse(expand("https://raw.github.com/{owner}/{repo}/master{dir}/", match)) if err != nil { return nil, err } c.header = gitHubRawHeader b := &presBuilder{ data: p, filename: match["file"], fetch: func(fnames []string) ([]*File, error) { var files []*File var dataURLs []string for _, fname := range fnames { u, err := apiBase.Parse(fname) if err != nil { return nil, err } u.RawQuery = apiBase.RawQuery files = append(files, &File{Name: fname}) dataURLs = append(dataURLs, u.String()) } err := c.getFiles(dataURLs, files) return files, err }, resolveURL: func(fname string) string { u, err := rawBase.Parse(fname) if err != nil { return "/notfound" } if strings.HasSuffix(fname, ".svg") { u.Host = "rawgithub.com" } return u.String() }, } return b.build() } // GetGitHubUpdates returns the full names ("owner/repo") of recently pushed GitHub repositories. // by pushedAfter. func GetGitHubUpdates(client *http.Client, pushedAfter string) (maxPushedAt string, names []string, err error) { c := httpClient{client: client, header: gitHubPreviewHeader} if pushedAfter == "" { pushedAfter = time.Now().Add(-24 * time.Hour).UTC().Format("2006-01-02T15:04:05Z") } u := "https://api.github.com/search/repositories?order=asc&sort=updated&q=fork:true+language:Go+pushed:>" + pushedAfter var updates struct { Items []struct { FullName string `json:"full_name"` PushedAt string `json:"pushed_at"` } } _, err = c.getJSON(u, &updates) if err != nil { return pushedAfter, nil, err } maxPushedAt = pushedAfter for _, item := range updates.Items { names = append(names, item.FullName) if item.PushedAt > maxPushedAt { maxPushedAt = item.PushedAt } } return maxPushedAt, names, nil } func getGitHubProject(client *http.Client, match map[string]string) (*Project, error) { c := &httpClient{client: client, errFn: gitHubError} var repo struct { Description string } if _, err := c.getJSON(expand("https://api.github.com/repos/{owner}/{repo}", match), &repo); err != nil { return nil, err } return &Project{ Description: repo.Description, }, nil } func getGistDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) { c := &httpClient{client: client, errFn: gitHubError} var gist struct { Files map[string]struct { Content string } HTMLURL string `json:"html_url"` History []struct { Version string } } if _, err := c.getJSON(expand("https://api.github.com/gists/{gist}", match), &gist); err != nil { return nil, err } if len(gist.History) == 0 { return nil, NotFoundError{Message: "History not found."} } commit := gist.History[0].Version if commit == savedEtag { return nil, NotModifiedError{} } var files []*File for name, file := range gist.Files { if isDocFile(name) { files = append(files, &File{ Name: name, Data: []byte(file.Content), BrowseURL: gist.HTMLURL + "#file-" + strings.Replace(name, ".", "-", -1), }) } } return &Directory{ BrowseURL: gist.HTMLURL, Etag: commit, Files: files, LineFmt: "%s-L%d", ProjectName: match["gist"], ProjectRoot: expand("gist.github.com/{gist}.git", match), ProjectURL: gist.HTMLURL, Subdirectories: nil, VCS: "git", }, nil }