chore(v2): vendor go/pkgsite/source package for remote URL discovery

go.mod

@@ -5,10 +5,14 @@ go 1.16
 require (
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/google/go-cmp v0.5.6
+	github.com/google/go-replayers/httpreplay v1.0.0
 	github.com/google/licenseclassifier v0.0.0-20210722185704-3043a050f148
 	github.com/otiai10/copy v1.6.0
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/spf13/cobra v1.3.0
+	go.opencensus.io v0.23.0
+	golang.org/x/mod v0.5.1
+	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
 	golang.org/x/tools v0.1.5
 	gopkg.in/src-d/go-git.v4 v4.13.1
 )

internal/third_party/pkgsite/LICENSE

@@ -0,0 +1,24 @@
+Copyright (c) 2020 The Go Authors. All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

internal/third_party/pkgsite/PATENTS

@@ -0,0 +1,20 @@
+Additional IP Rights Grant (Patents)
+"This implementation" means the copyrightable works distributed by
+Google as part of the Go project.
+Google hereby grants to You a perpetual, worldwide, non-exclusive,
+no-charge, royalty-free, irrevocable (except as stated in this section)
+patent license to make, have made, use, offer to sell, sell, import,
+transfer and otherwise run, modify and propagate the contents of this
+implementation of Go, where such license applies only to those patent
+claims, both currently owned or controlled by Google and acquired in
+the future, licensable by Google that are necessarily infringed by this
+implementation of Go.  This grant does not include claims that would be
+infringed only as a consequence of further modification of this
+implementation.  If you or your agent or exclusive licensee institute or
+order or agree to the institution of patent litigation against any
+entity (including a cross-claim or counterclaim in a lawsuit) alleging
+that this implementation of Go or any code incorporated within this
+implementation of Go constitutes direct or contributory patent
+infringement, or inducement of patent infringement, then any patent
+rights granted to you under this License for this implementation of Go
+shall terminate as of the date such litigation is filed.

internal/third_party/pkgsite/README.md

@@ -0,0 +1,18 @@
+# go/pkgsite/source
+
+Vendored from <https://go.googlesource.com/pkgsite/+/ef8047b111963f61f5a0b3ae5b464cdc18dc5f74/internal/source>,
+because the source package is internal, and there's no plan to move it out anytime soon: <https://github.com/golang/go/issues/40477#issuecomment-868532845>.
+
+The entire source folder and be downloaded via:
+
+```bash
+curl -LO https://go.googlesource.com/pkgsite/+archive/ef8047b111963f61f5a0b3ae5b464cdc18dc5f74/internal.tar.gz
+```
+
+Local modifications:
+
+- Update import paths.
+- Removed unused functions from pkgsite/internal/stdlib, pkgsite/internal/derrors,
+  pkgsite/internal/version to avoid other dependencies.
+- For pkgsite/internal/source, switched to use go log package, because glog conflicts with a test
+  dependency that also defines the "v" flag.

internal/third_party/pkgsite/derrors/derrors.go

@@ -0,0 +1,47 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package derrors defines internal error values to categorize the different
+// types error semantics we support.
+package derrors
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+)
+
+//lint:file-ignore ST1012 prefixing error values with Err would stutter
+
+var (
+	// NotFound indicates that a requested entity was not found (HTTP 404).
+	NotFound = errors.New("not found")
+
+	// InvalidArgument indicates that the input into the request is invalid in
+	// some way (HTTP 400).
+	InvalidArgument = errors.New("invalid argument")
+)
+
+var codes = []struct {
+	err  error
+	code int
+}{
+	{NotFound, http.StatusNotFound},
+	{InvalidArgument, http.StatusBadRequest},
+}
+
+// Wrap adds context to the error and allows
+// unwrapping the result to recover the original error.
+//
+// Example:
+//
+//	defer derrors.Wrap(&err, "copy(%s, %s)", src, dst)
+//
+// See Add for an equivalent function that does not allow
+// the result to be unwrapped.
+func Wrap(errp *error, format string, args ...interface{}) {
+	if *errp != nil {
+		*errp = fmt.Errorf("%s: %w", fmt.Sprintf(format, args...), *errp)
+	}
+}

internal/third_party/pkgsite/derrors/derrors_test.go

@@ -0,0 +1,29 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package derrors
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestWrap(t *testing.T) {
+	var err error
+	Wrap(&err, "whatever")
+	if err != nil {
+		t.Errorf("got %v, want nil", err)
+	}
+
+	orig := errors.New("bad stuff")
+	err = orig
+	Wrap(&err, "Frob(%d)", 3)
+	want := "Frob(3): bad stuff"
+	if got := err.Error(); got != want {
+		t.Errorf("got %s, want %s", got, want)
+	}
+	if got := errors.Unwrap(err); got != orig {
+		t.Errorf("Unwrap: got %#v, want %#v", got, orig)
+	}
+}

internal/third_party/pkgsite/source/meta-tags.go

@@ -0,0 +1,161 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package source
+
+import (
+	"context"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/google/go-licenses/internal/third_party/pkgsite/derrors"
+)
+
+// This code adapted from https://go.googlesource.com/gddo/+/refs/heads/master/gosrc/gosrc.go.
+
+// sourceMeta represents the values in a go-source meta tag, or as a fallback,
+// values from a go-import meta tag.
+// The go-source spec is at https://github.com/golang/gddo/wiki/Source-Code-Links.
+// The go-import spec is in "go help importpath".
+type sourceMeta struct {
+	repoRootPrefix string // import path prefix corresponding to repo root
+	repoURL        string // URL of the repo root
+	// The next two are only present in a go-source tag.
+	dirTemplate  string // URL template for a directory
+	fileTemplate string // URL template for a file and line
+}
+
+// fetchMeta retrieves go-import and go-source meta tag information, using the import path to construct
+// a URL as described in "go help importpath".
+//
+// The importPath argument, as the name suggests, could be any package import
+// path. But we only pass module paths.
+//
+// The discovery site only cares about linking to source, not fetching it (we
+// already have it in the module zip file). So we merge the go-import and
+// go-source meta tag information, preferring the latter.
+func fetchMeta(ctx context.Context, client *Client, importPath string) (_ *sourceMeta, err error) {
+	defer derrors.Wrap(&err, "fetchMeta(ctx, client, %q)", importPath)
+
+	uri := importPath
+	if !strings.Contains(uri, "/") {
+		// Add slash for root of domain.
+		uri = uri + "/"
+	}
+	uri = uri + "?go-get=1"
+
+	resp, err := client.doURL(ctx, "GET", "https://"+uri, true)
+	if err != nil {
+		resp, err = client.doURL(ctx, "GET", "http://"+uri, false)
+		if err != nil {
+			return nil, err
+		}
+	}
+	defer resp.Body.Close()
+	return parseMeta(importPath, resp.Body)
+}
+
+func parseMeta(importPath string, r io.Reader) (sm *sourceMeta, err error) {
+	errorMessage := "go-import and go-source meta tags not found"
+	// gddo uses an xml parser, and this code is adapted from it.
+	d := xml.NewDecoder(r)
+	d.Strict = false
+metaScan:
+	for {
+		t, tokenErr := d.Token()
+		if tokenErr != nil {
+			break metaScan
+		}
+		switch t := t.(type) {
+		case xml.EndElement:
+			if strings.EqualFold(t.Name.Local, "head") {
+				break metaScan
+			}
+		case xml.StartElement:
+			if strings.EqualFold(t.Name.Local, "body") {
+				break metaScan
+			}
+			if !strings.EqualFold(t.Name.Local, "meta") {
+				continue metaScan
+			}
+			nameAttr := attrValue(t.Attr, "name")
+			if nameAttr != "go-import" && nameAttr != "go-source" {
+				continue metaScan
+			}
+			fields := strings.Fields(attrValue(t.Attr, "content"))
+			if len(fields) < 1 {
+				continue metaScan
+			}
+			repoRootPrefix := fields[0]
+			if !strings.HasPrefix(importPath, repoRootPrefix) ||
+				!(len(importPath) == len(repoRootPrefix) || importPath[len(repoRootPrefix)] == '/') {
+				// Ignore if root is not a prefix of the  path. This allows a
+				// site to use a single error page for multiple repositories.
+				continue metaScan
+			}
+			switch nameAttr {
+			case "go-import":
+				if len(fields) != 3 {
+					errorMessage = "go-import meta tag content attribute does not have three fields"
+					continue metaScan
+				}
+				if fields[1] == "mod" {
+					// We can't make source links from a "mod" vcs type, so skip it.
+					continue
+				}
+				if sm != nil {
+					sm = nil
+					errorMessage = "more than one go-import meta tag found"
+					break metaScan
+				}
+				sm = &sourceMeta{
+					repoRootPrefix: repoRootPrefix,
+					repoURL:        fields[2],
+				}
+				// Keep going in the hope of finding a go-source tag.
+			case "go-source":
+				if len(fields) != 4 {
+					errorMessage = "go-source meta tag content attribute does not have four fields"
+					continue metaScan
+				}
+				if sm != nil && sm.repoRootPrefix != repoRootPrefix {
+					errorMessage = fmt.Sprintf("import path prefixes %q for go-import and %q for go-source disagree", sm.repoRootPrefix, repoRootPrefix)
+					sm = nil
+					break metaScan
+				}
+				// If go-source repo is "_", then default to the go-import repo.
+				repoURL := fields[1]
+				if repoURL == "_" {
+					if sm == nil {
+						errorMessage = `go-source repo is "_", but no previous go-import tag`
+						break metaScan
+					}
+					repoURL = sm.repoURL
+				}
+				sm = &sourceMeta{
+					repoRootPrefix: repoRootPrefix,
+					repoURL:        repoURL,
+					dirTemplate:    fields[2],
+					fileTemplate:   fields[3],
+				}
+				break metaScan
+			}
+		}
+	}
+	if sm == nil {
+		return nil, fmt.Errorf("%s: %w", errorMessage, derrors.NotFound)
+	}
+	return sm, nil
+}
+
+func attrValue(attrs []xml.Attr, name string) string {
+	for _, a := range attrs {
+		if strings.EqualFold(a.Name.Local, name) {
+			return a.Value
+		}
+	}
+	return ""
+}