Skip to content

Commit

Permalink
internal/source: match old templates to find new ones
Browse files Browse the repository at this point in the history 
Previously, when we encountered a repo whose URL doesn't match an
existing pattern, we did not generate any URL templates for it,
meaning we could not render source links in the documentation.

This CL uses the templates in the go-source meta tag to guess the
version-aware templates that are likely to work for the repo.

For golang/go#40477

Change-Id: I2d1978da5a6de1af19284233dbab9ac1ae2cb582
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/285312
Trust: Jonathan Amsterdam <[email protected]>
Run-TryBot: Jonathan Amsterdam <[email protected]>
Reviewed-by: Julie Qiu <[email protected]>
  • Loading branch information
@jba
jba committed Jan 21, 2021
1 parent 7099521 commit a9ff35d
Show file tree
Hide file tree
Showing 3 changed files with 2,509 additions and 1,111 deletions.
118 changes: 93 additions & 25 deletions internal/source/source.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,7 @@ func newStdlibInfo(version string) (_ *Info, err error) {
// example.com/a/b.git/c
// then repo="example.com/a/b" and relativeModulePath="c"; the ".git" is omitted, since it is neither
// part of the repo nor part of the relative path to the module within the repo.
func matchStatic(moduleOrRepoPath string) (repo, relativeModulePath string, _ urlTemplates, transformCommit func(string, bool) string, _ error) {
func matchStatic(moduleOrRepoPath string) (repo, relativeModulePath string, _ urlTemplates, transformCommit transformCommitFunc, _ error) {
for _, pat := range patterns {
matches := pat.re.FindStringSubmatch(moduleOrRepoPath)
if matches == nil {
Expand Down Expand Up @@ -395,7 +395,12 @@ func moduleInfoDynamic(ctx context.Context, client *Client, modulePath, version
var repo string
repo, _, templates, transformCommit, _ = matchStatic(removeHTTPScheme(sourceMeta.dirTemplate))
if templates == (urlTemplates{}) {
log.Infof(ctx, "no templates for repo URL %q from meta tag: err=%v", sourceMeta.repoURL, err)
if err == nil {
templates, transformCommit = matchLegacyTemplates(ctx, sourceMeta)
repoURL = strings.TrimSuffix(repoURL, ".git")
} else {
log.Infof(ctx, "no templates for repo URL %q from meta tag: err=%v", sourceMeta.repoURL, err)
}
} else {
// Use the repo from the template, not the original one.
repoURL = "https://" + repo
Expand All @@ -414,6 +419,63 @@ func moduleInfoDynamic(ctx context.Context, client *Client, modulePath, version
}, nil
}

// List of template regexps and their corresponding likely templates,
// used by matchLegacyTemplates below.
var legacyTemplateMatches = []struct {
fileRegexp *regexp.Regexp
templates urlTemplates
transformCommit transformCommitFunc
}{
{
regexp.MustCompile(`/src/branch/\w+\{/dir\}/\{file\}#L\{line\}$`),
giteaURLTemplates, giteaTransformCommit,
},
{
regexp.MustCompile(`/src/\w+\{/dir\}/\{file\}#L\{line\}$`),
giteaURLTemplates, nil,
},
{
regexp.MustCompile(`/-/blob/\w+\{/dir\}/\{file\}#L\{line\}$`),
gitlab2URLTemplates, nil,
},
{
regexp.MustCompile(`/tree\{/dir\}/\{file\}#n\{line\}$`),
fdioURLTemplates, fdioTransformCommit,
},
}

// matchLegacyTemplates matches the templates from the go-source meta tag
// against some known patterns to guess the version-aware URL templates. If it
// can't find a match, it falls back using the go-source templates with some
// small replacements. These will not be version-aware but will still serve
// source at a fixed commit, which is better than nothing.
func matchLegacyTemplates(ctx context.Context, sm *sourceMeta) (_ urlTemplates, transformCommit transformCommitFunc) {
if sm.fileTemplate == "" {
return urlTemplates{}, nil
}
for _, ltm := range legacyTemplateMatches {
if ltm.fileRegexp.MatchString(sm.fileTemplate) {
return ltm.templates, ltm.transformCommit
}
}
log.Infof(ctx, "matchLegacyTemplates: no matches for repo URL %q; replacing", sm.repoURL)
rep := strings.NewReplacer(
"{/dir}/{file}", "/{file}",
"{dir}/{file}", "{file}",
"{/dir}", "/{dir}")
line := rep.Replace(sm.fileTemplate)
file := line
if i := strings.LastIndexByte(line, '#'); i > 0 {
file = line[:i]
}
return urlTemplates{
Repo: sm.repoURL,
Directory: rep.Replace(sm.dirTemplate),
File: file,
Line: line,
}, nil
}

// adjustVersionedModuleDirectory changes info.moduleDir if necessary to
// correctly reflect the repo structure. info.moduleDir will be wrong if it has
// a suffix "/vN" for N > 1, and the repo uses the "major branch" convention,
Expand Down Expand Up @@ -463,6 +525,8 @@ func removeVersionSuffix(s string) string {
return strings.TrimSuffix(dir, "/")
}

type transformCommitFunc func(commit string, isHash bool) string

// Patterns for determining repo and URL templates from module paths or repo
// URLs. Each regexp must match a prefix of the target string, and must have a
// group named "repo".
Expand All @@ -471,7 +535,7 @@ var patterns = []struct {
templates urlTemplates
re *regexp.Regexp
// transformCommit may alter the commit before substitution
transformCommit func(commit string, isHash bool) string
transformCommit transformCommitFunc
}{
{
pattern: `^(?P<repo>github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)`,
Expand Down Expand Up @@ -504,30 +568,13 @@ var patterns = []struct {
},
},
{
pattern: `^(?P<repo>git\.fd\.io/[a-z0-9A-Z_.\-]+)`,
templates: urlTemplates{
Directory: "{repo}/tree/{dir}?{commit}",
File: "{repo}/tree/{file}?{commit}",
Line: "{repo}/tree/{file}?{commit}#n{line}",
Raw: "{repo}/plain/{file}?{commit}",
},
transformCommit: func(commit string, isHash bool) string {
// hashes use "?id=", tags use "?h="
p := "h"
if isHash {
p = "id"
}
return fmt.Sprintf("%s=%s", p, commit)
},
pattern: `^(?P<repo>git\.fd\.io/[a-z0-9A-Z_.\-]+)`,
templates: fdioURLTemplates,
transformCommit: fdioTransformCommit,
},
{
pattern: `^(?P<repo>git\.pirl\.io/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)`,
templates: urlTemplates{
Directory: "{repo}/-/tree/{commit}/{dir}",
File: "{repo}/-/blob/{commit}/{file}",
Line: "{repo}/-/blob/{commit}/{file}#L{line}",
Raw: "{repo}/-/raw/{commit}/{file}",
},
pattern: `^(?P<repo>git\.pirl\.io/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)`,
templates: gitlab2URLTemplates,
},
{
pattern: `^(?P<repo>gitea\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+)(\.git|$)`,
Expand Down Expand Up @@ -617,6 +664,15 @@ func giteaTransformCommit(commit string, isHash bool) string {
return "tag/" + commit
}

func fdioTransformCommit(commit string, isHash bool) string {
// hashes use "?id=", tags use "?h="
p := "h"
if isHash {
p = "id"
}
return fmt.Sprintf("%s=%s", p, commit)
}

// urlTemplates describes how to build URLs from bits of source information.
// The fields are exported for JSON encoding.
//
Expand Down Expand Up @@ -664,6 +720,18 @@ var (
Line: "{repo}/+/{commit}/{file}#{line}",
// Gitiles has no support for serving raw content at this time.
}
gitlab2URLTemplates = urlTemplates{
Directory: "{repo}/-/tree/{commit}/{dir}",
File: "{repo}/-/blob/{commit}/{file}",
Line: "{repo}/-/blob/{commit}/{file}#L{line}",
Raw: "{repo}/-/raw/{commit}/{file}",
}
fdioURLTemplates = urlTemplates{
Directory: "{repo}/tree/{dir}?{commit}",
File: "{repo}/tree/{file}?{commit}",
Line: "{repo}/tree/{file}?{commit}#n{line}",
Raw: "{repo}/plain/{file}?{commit}",
}
)

// commitFromVersion returns a string that refers to a commit corresponding to version.
Expand Down
102 changes: 97 additions & 5 deletions internal/source/source_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,42 @@ func TestModuleInfo(t *testing.T) {
"https://gotools.org/dmitri.shuralyov.com/gpu/mtl/example/movingtriangle/internal/coreanim?rev=d42048ed14fd#coreanim.go-L1",
"",
},
{
"go-source templates match gitea with transform",
"opendev.org/airship/airshipctl", "v2.0.0-beta.1", "pkg/cluster/command.go",
"https://opendev.org/airship/airshipctl",
"https://opendev.org/airship/airshipctl/src/tag/v2.0.0-beta.1",
"https://opendev.org/airship/airshipctl/src/tag/v2.0.0-beta.1/pkg/cluster/command.go",
"https://opendev.org/airship/airshipctl/src/tag/v2.0.0-beta.1/pkg/cluster/command.go#L1",
"",
},
{
"go-source templates match gitea without transform",
"git.borago.de/Marco/gqltest", "v0.0.18", "go.mod",
"https://git.borago.de/Marco/gqltest",
"https://git.borago.de/Marco/gqltest/src/v0.0.18",
"https://git.borago.de/Marco/gqltest/src/v0.0.18/go.mod",
"https://git.borago.de/Marco/gqltest/src/v0.0.18/go.mod#L1",
"https://git.borago.de/Marco/gqltest/raw/v0.0.18/go.mod",
},
{
"go-source templates match gitlab2",
"git.pluggableideas.com/destrealm/3rdparty/go-yaml", "v2.2.6", "go.mod",
"https://git.pluggableideas.com/destrealm/3rdparty/go-yaml",
"https://git.pluggableideas.com/destrealm/3rdparty/go-yaml/-/tree/v2.2.6",
"https://git.pluggableideas.com/destrealm/3rdparty/go-yaml/-/blob/v2.2.6/go.mod",
"https://git.pluggableideas.com/destrealm/3rdparty/go-yaml/-/blob/v2.2.6/go.mod#L1",
"https://git.pluggableideas.com/destrealm/3rdparty/go-yaml/-/raw/v2.2.6/go.mod",
},
{
"go-source templates match fdio",
"golang.zx2c4.com/wireguard/windows", "v0.3.4", "go.mod",
"https://git.zx2c4.com/wireguard-windows",
"https://git.zx2c4.com/wireguard-windows/tree/?h=v0.3.4",
"https://git.zx2c4.com/wireguard-windows/tree/go.mod?h=v0.3.4",
"https://git.zx2c4.com/wireguard-windows/tree/go.mod?h=v0.3.4#n1",
"https://git.zx2c4.com/wireguard-windows/plain/go.mod?h=v0.3.4",
},
} {
t.Run(test.desc, func(t *testing.T) {
info, err := ModuleInfo(context.Background(), &Client{client}, test.modulePath, test.version)
Expand Down Expand Up @@ -484,12 +520,17 @@ func TestModuleInfoDynamic(t *testing.T) {
},
{
"alice.org/pkg/source",
// Has a go-source tag, but we can't use the templates.
// Has a go-source tag; we try to use the templates.
&Info{
repoURL: "http://alice.org/pkg",
moduleDir: "source",
commit: "source/v1.2.3",
// empty templates
templates: urlTemplates{
Repo: "http://alice.org/pkg",
Directory: "http://alice.org/pkg/{dir}",
File: "http://alice.org/pkg/{dir}?f={file}",
Line: "http://alice.org/pkg/{dir}?f={file}#Line{line}",
},
},
},

Expand All @@ -500,7 +541,12 @@ func TestModuleInfoDynamic(t *testing.T) {
repoURL: "http://alice.org/pkg",
moduleDir: "ignore",
commit: "ignore/v1.2.3",
// empty templates
templates: urlTemplates{
Repo: "http://alice.org/pkg",
Directory: "http://alice.org/pkg/{dir}",
File: "http://alice.org/pkg/{dir}?f={file}",
Line: "http://alice.org/pkg/{dir}?f={file}#Line{line}",
},
},
},
{"alice.org/pkg/multiple", nil},
Expand All @@ -510,7 +556,7 @@ func TestModuleInfoDynamic(t *testing.T) {
&Info{
// The go-import tag's repo root ends in ".git", but according to the spec
// there should not be a .vcs suffix, so we include the ".git" in the repo URL.
repoURL: "https://vcs.net/bob/pkg.git",
repoURL: "https://vcs.net/bob/pkg",
moduleDir: "",
commit: "v1.2.3",
// empty templates
Expand All @@ -519,7 +565,7 @@ func TestModuleInfoDynamic(t *testing.T) {
{
"bob.com/pkg/sub",
&Info{
repoURL: "https://vcs.net/bob/pkg.git",
repoURL: "https://vcs.net/bob/pkg",
moduleDir: "sub",
commit: "sub/v1.2.3",
// empty templates
Expand Down Expand Up @@ -907,3 +953,49 @@ func TestURLTemplates(t *testing.T) {
check(p.templates.Raw, "commit", "file")
}
}

func TestMatchLegacyTemplates(t *testing.T) {
for _, test := range []struct {
sm sourceMeta
wantTemplates urlTemplates
wantTransformCommitNil bool
}{
{
sm: sourceMeta{"", "", "", "https://git.blindage.org/21h/hcloud-dns/src/branch/master{/dir}/{file}#L{line}"},
wantTemplates: giteaURLTemplates,
wantTransformCommitNil: false,
},
{
sm: sourceMeta{"", "", "", "https://git.lastassault.de/sup/networkoverlap/-/blob/master{/dir}/{file}#L{line}"},
wantTemplates: gitlab2URLTemplates,
wantTransformCommitNil: true,
},
{
sm: sourceMeta{"", "", "", "https://git.borago.de/Marco/gqltest/src/master{/dir}/{file}#L{line}"},
wantTemplates: giteaURLTemplates,
wantTransformCommitNil: true,
},
{
sm: sourceMeta{"", "", "", "https://git.zx2c4.com/wireguard-windows/tree{/dir}/{file}#n{line}"},
wantTemplates: fdioURLTemplates,
wantTransformCommitNil: false,
},
{
sm: sourceMeta{"", "", "unknown{/dir}", "unknown{/dir}/{file}#L{line}"},
wantTemplates: urlTemplates{
Repo: "",
Directory: "unknown/{dir}",
File: "unknown/{file}",
Line: "unknown/{file}#L{line}",
},
wantTransformCommitNil: true,
},
} {
gotTemplates, gotTransformCommit := matchLegacyTemplates(context.Background(), &test.sm)
gotNil := gotTransformCommit == nil
if gotTemplates != test.wantTemplates || gotNil != test.wantTransformCommitNil {
t.Errorf("%+v:\ngot (%+v, %t)\nwant (%+v, %t)",
test.sm, gotTemplates, gotNil, test.wantTemplates, test.wantTransformCommitNil)
}
}
}
Loading

0 comments on commit a9ff35d

Please sign in to comment.