From 13e6bcfb4f7d2db7d80af77f1b32b2394abca9da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Mur=C3=A9?= Date: Thu, 14 May 2020 20:35:11 +0200 Subject: [PATCH] feat: wildcard support for public gateways Add support for one or more wildcards in the hostname definition of a public gateway. This is useful for example to support easily multiples environment. Wildcarded hostname are set in the config as for example "*.domain.tld". --- core/corehttp/hostname.go | 98 +++++++++++++++++------ core/corehttp/hostname_test.go | 86 +++++++++++--------- docs/config.md | 6 ++ test/sharness/t0114-gateway-subdomains.sh | 83 +++++++++++++++++++ 4 files changed, 213 insertions(+), 60 deletions(-) diff --git a/core/corehttp/hostname.go b/core/corehttp/hostname.go index f29b4d11f63..04f640e2fa4 100644 --- a/core/corehttp/hostname.go +++ b/core/corehttp/hostname.go @@ -6,6 +6,7 @@ import ( "net" "net/http" "net/url" + "regexp" "strings" cid "github.com/ipfs/go-cid" @@ -24,17 +25,17 @@ import ( var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/", "/version"} -var pathGatewaySpec = config.GatewaySpec{ +var pathGatewaySpec = &config.GatewaySpec{ Paths: defaultPaths, UseSubdomains: false, } -var subdomainGatewaySpec = config.GatewaySpec{ +var subdomainGatewaySpec = &config.GatewaySpec{ Paths: defaultPaths, UseSubdomains: true, } -var defaultKnownGateways = map[string]config.GatewaySpec{ +var defaultKnownGateways = map[string]*config.GatewaySpec{ "localhost": subdomainGatewaySpec, "ipfs.io": pathGatewaySpec, "gateway.ipfs.io": pathGatewaySpec, @@ -58,22 +59,8 @@ func HostnameOption() ServeOption { if err != nil { return nil, err } - knownGateways := make( - map[string]config.GatewaySpec, - len(defaultKnownGateways)+len(cfg.Gateway.PublicGateways), - ) - for hostname, gw := range defaultKnownGateways { - knownGateways[hostname] = gw - } - for hostname, gw := range cfg.Gateway.PublicGateways { - if gw == nil { - // Allows the user to remove gateways but _also_ - // allows us to continuously update the list. - delete(knownGateways, hostname) - } else { - knownGateways[hostname] = *gw - } - } + + knownGateways := prepareKnownGateways(cfg.Gateway.PublicGateways) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { // Unfortunately, many (well, ipfs.io) gateways use @@ -233,22 +220,85 @@ func HostnameOption() ServeOption { } } +type gatewayHosts struct { + exact map[string]*config.GatewaySpec + wildcard []wildcardHost +} + +type wildcardHost struct { + re *regexp.Regexp + spec *config.GatewaySpec +} + +func prepareKnownGateways(publicGateways map[string]*config.GatewaySpec) gatewayHosts { + var hosts gatewayHosts + + if len(publicGateways) == 0 { + hosts.exact = make( + map[string]*config.GatewaySpec, + len(defaultKnownGateways), + ) + for hostname, gw := range defaultKnownGateways { + hosts.exact[hostname] = gw + } + return hosts + } + + hosts.exact = make(map[string]*config.GatewaySpec, len(publicGateways)) + + for hostname, gw := range publicGateways { + if gw == nil { + continue + } + if strings.Contains(hostname, "*") { + // from *.domain.tld, construct a regexp that match any direct subdomain + // of .domain.tld. + // + // Regexp will be in the form of ^[^.]+\.domain.tld(?::\d+)?$ + + escaped := strings.ReplaceAll(hostname, ".", `\.`) + regexed := strings.ReplaceAll(escaped, "*", "[^.]+") + + re, err := regexp.Compile(fmt.Sprintf(`^%s(?::\d+)?$`, regexed)) + if err != nil { + log.Warn("invalid wildcard gateway hostname \"%s\"", hostname) + } + + hosts.wildcard = append(hosts.wildcard, wildcardHost{re: re, spec: gw}) + } else { + hosts.exact[hostname] = gw + } + } + + return hosts +} + // isKnownHostname checks Gateway.PublicGateways and returns matching // GatewaySpec with gracefull fallback to version without port -func isKnownHostname(hostname string, knownGateways map[string]config.GatewaySpec) (gw config.GatewaySpec, ok bool) { +func isKnownHostname(hostname string, knownGateways gatewayHosts) (gw *config.GatewaySpec, ok bool) { // Try hostname (host+optional port - value from Host header as-is) - if gw, ok := knownGateways[hostname]; ok { + if gw, ok := knownGateways.exact[hostname]; ok { + return gw, ok + } + // Also test without port + if gw, ok = knownGateways.exact[stripPort(hostname)]; ok { return gw, ok } - // Fallback to hostname without port - gw, ok = knownGateways[stripPort(hostname)] + + // Wildcard support. Test both with and without port. + for _, host := range knownGateways.wildcard { + if host.re.MatchString(hostname) { + return host.spec, true + } + } + return gw, ok } // Parses Host header and looks for a known subdomain gateway host. // If found, returns GatewaySpec and subdomain components. // Note: hostname is host + optional port -func knownSubdomainDetails(hostname string, knownGateways map[string]config.GatewaySpec) (gw config.GatewaySpec, knownHostname, ns, rootID string, ok bool) { +func knownSubdomainDetails(hostname string, knownGateways gatewayHosts) (gw *config.GatewaySpec, knownHostname, ns, rootID string, ok bool) { labels := strings.Split(hostname, ".") // Look for FQDN of a known gateway hostname. // Example: given "dist.ipfs.io.ipns.dweb.link": diff --git a/core/corehttp/hostname_test.go b/core/corehttp/hostname_test.go index cf00e827121..472dbcf16a2 100644 --- a/core/corehttp/hostname_test.go +++ b/core/corehttp/hostname_test.go @@ -32,6 +32,7 @@ func TestToSubdomainURL(t *testing.T) { {"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://k2k4r8l9ja7hkzynavdqup76ou46tnvuaqegbd04a4o1mpbsey0meucb.ipns.localhost/", nil}, // PeerID: ed25519+identity multihash → CIDv1Base36 {"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://k51qzi5uqu5di608geewp3nqkg0bpujoasmka7ftkyxgcm3fh1aroup0gsdrna.ipns.localhost/", nil}, + {"sub.localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.sub.localhost/", nil}, } { url, err := toSubdomainURL(test.hostname, test.path, r) if url != test.url || !equalError(err, test.err) { @@ -104,60 +105,73 @@ func TestDNSPrefix(t *testing.T) { } func TestKnownSubdomainDetails(t *testing.T) { - gwSpec := config.GatewaySpec{ - UseSubdomains: true, - } - knownGateways := map[string]config.GatewaySpec{ - "localhost": gwSpec, - "dweb.link": gwSpec, - "dweb.ipfs.pvt.k12.ma.us": gwSpec, // note the sneaky ".ipfs." ;-) - } + gwLocalhost := &config.GatewaySpec{} + gwDweb := &config.GatewaySpec{} + gwLong := &config.GatewaySpec{} + gwWildcard1 := &config.GatewaySpec{} + gwWildcard2 := &config.GatewaySpec{} + + knownGateways := prepareKnownGateways(map[string]*config.GatewaySpec{ + "localhost": gwLocalhost, + "dweb.link": gwDweb, + "dweb.ipfs.pvt.k12.ma.us": gwLong, // note the sneaky ".ipfs." ;-) + "*.wildcard1.tld": gwWildcard1, + "*.*.wildcard2.tld": gwWildcard2, + }) for _, test := range []struct { // in: hostHeader string // out: + gw *config.GatewaySpec hostname string ns string rootID string ok bool }{ // no subdomain - {"127.0.0.1:8080", "", "", "", false}, - {"[::1]:8080", "", "", "", false}, - {"hey.look.example.com", "", "", "", false}, - {"dweb.link", "", "", "", false}, + {"127.0.0.1:8080", nil, "", "", "", false}, + {"[::1]:8080", nil, "", "", "", false}, + {"hey.look.example.com", nil, "", "", "", false}, + {"dweb.link", nil, "", "", "", false}, // malformed Host header - {".....dweb.link", "", "", "", false}, - {"link", "", "", "", false}, - {"8080:dweb.link", "", "", "", false}, - {" ", "", "", "", false}, - {"", "", "", "", false}, + {".....dweb.link", nil, "", "", "", false}, + {"link", nil, "", "", "", false}, + {"8080:dweb.link", nil, "", "", "", false}, + {" ", nil, "", "", "", false}, + {"", nil, "", "", "", false}, // unknown gateway host - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.unknown.example.com", "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.unknown.example.com", nil, "", "", "", false}, // cid in subdomain, known gateway - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:8080", "localhost:8080", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.link", "dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:8080", gwLocalhost, "localhost:8080", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.link", gwDweb, "dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, // capture everything before .ipfs. - {"foo.bar.boo-buzz.ipfs.dweb.link", "dweb.link", "ipfs", "foo.bar.boo-buzz", true}, + {"foo.bar.boo-buzz.ipfs.dweb.link", gwDweb, "dweb.link", "ipfs", "foo.bar.boo-buzz", true}, // ipns - {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.localhost:8080", "localhost:8080", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, - {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.link", "dweb.link", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.link", gwDweb, "dweb.link", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, // edge case check: public gateway under long TLD (see: https://publicsuffix.org) - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, - {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, // dnslink in subdomain - {"en.wikipedia-on-ipfs.org.ipns.localhost:8080", "localhost:8080", "ipns", "en.wikipedia-on-ipfs.org", true}, - {"en.wikipedia-on-ipfs.org.ipns.localhost", "localhost", "ipns", "en.wikipedia-on-ipfs.org", true}, - {"dist.ipfs.io.ipns.localhost:8080", "localhost:8080", "ipns", "dist.ipfs.io", true}, - {"en.wikipedia-on-ipfs.org.ipns.dweb.link", "dweb.link", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"en.wikipedia-on-ipfs.org.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"en.wikipedia-on-ipfs.org.ipns.localhost", gwLocalhost, "localhost", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"dist.ipfs.io.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "dist.ipfs.io", true}, + {"en.wikipedia-on-ipfs.org.ipns.dweb.link", gwDweb, "dweb.link", "ipns", "en.wikipedia-on-ipfs.org", true}, // edge case check: public gateway under long TLD (see: https://publicsuffix.org) - {"foo.dweb.ipfs.pvt.k12.ma.us", "", "", "", false}, - {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, - {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"foo.dweb.ipfs.pvt.k12.ma.us", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, // other namespaces - {"api.localhost", "", "", "", false}, - {"peerid.p2p.localhost", "localhost", "p2p", "peerid", true}, + {"api.localhost", nil, "", "", "", false}, + {"peerid.p2p.localhost", gwLocalhost, "localhost", "p2p", "peerid", true}, + // wildcards + {"wildcard1.tld", nil, "", "", "", false}, + {".wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub.wildcard1.tld", gwWildcard1, "sub.wildcard1.tld", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub1.sub2.wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub1.sub2.wildcard2.tld", gwWildcard2, "sub1.sub2.wildcard2.tld", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, } { gw, hostname, ns, rootID, ok := knownSubdomainDetails(test.hostHeader, knownGateways) if ok != test.ok { @@ -172,8 +186,8 @@ func TestKnownSubdomainDetails(t *testing.T) { if hostname != test.hostname { t.Errorf("knownSubdomainDetails(%s): hostname is '%s', expected '%s'", test.hostHeader, hostname, test.hostname) } - if ok && gw.UseSubdomains != gwSpec.UseSubdomains { - t.Errorf("knownSubdomainDetails(%s): gw is %+v, expected %+v", test.hostHeader, gw, gwSpec) + if gw != test.gw { + t.Errorf("knownSubdomainDetails(%s): gw is %+v, expected %+v", test.hostHeader, gw, test.gw) } } diff --git a/docs/config.md b/docs/config.md index 7a5fc9adc3e..2acea031abb 100644 --- a/docs/config.md +++ b/docs/config.md @@ -586,6 +586,12 @@ Type: `array[string]` `PublicGateways` is a dictionary for defining gateway behavior on specified hostnames. +Hostnames can optionally be defined with one or more wildcards. + +Examples: +- `*.example.com` will match requests to `http://foo.example.com/ipfs/*` or `http://{cid}.ipfs.bar.example.com/*`. +- `foo-*.example.com` will match requests to `http://foo-bar.example.com/ipfs/*` or `http://{cid}.ipfs.foo-xyz.example.com/*`. + #### `Gateway.PublicGateways: Paths` Array of paths that should be exposed on the hostname. diff --git a/test/sharness/t0114-gateway-subdomains.sh b/test/sharness/t0114-gateway-subdomains.sh index 5f3020346ef..5c9d459e681 100755 --- a/test/sharness/t0114-gateway-subdomains.sh +++ b/test/sharness/t0114-gateway-subdomains.sh @@ -821,6 +821,89 @@ test_expect_success "request for http://fake.domain.com/ipfs/{CID} with X-Forwar test_should_contain \"Location: https://$CIDv1.ipfs.example.com/\" response " +## ============================================================================ +## Test support for wildcards in gateway config +## ============================================================================ + +# set explicit subdomain gateway config for the hostnames +ipfs config --json Gateway.PublicGateways '{ + "*.example1.com": { + "UseSubdomains": true, + "Paths": ["/ipfs"] + }, + "*.*.example2.com": { + "UseSubdomains": true, + "Paths": ["/ipfs"] + }, + "foo.*.example3.com": { + "UseSubdomains": true, + "Paths": ["/ipfs"] + }, + "foo.bar-*-boo.example4.com": { + "UseSubdomains": true, + "Paths": ["/ipfs"] + } +}' || exit 1 +# restart daemon to apply config changes +test_kill_ipfs_daemon +test_launch_ipfs_daemon --offline + +# *.example1.com + +test_hostname_gateway_response_should_contain \ + "request for foo.example1.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.foo.example1.com" \ + "foo.example1.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ + "Location: http://$CIDv1.ipfs.foo.example1.com/" + +test_hostname_gateway_response_should_contain \ + "request for {CID}.ipfs.foo.example1.com should return expected payload" \ + "${CIDv1}.ipfs.foo.example1.com" \ + "http://127.0.0.1:$GWAY_PORT/" \ + "$CID_VAL" + +# *.*.example2.com + +test_hostname_gateway_response_should_contain \ + "request for foo.bar.example2.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.foo.bar.example2.com" \ + "foo.bar.example2.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ + "Location: http://$CIDv1.ipfs.foo.bar.example2.com/" + +test_hostname_gateway_response_should_contain \ + "request for {CID}.ipfs.foo.bar.example2.com should return expected payload" \ + "${CIDv1}.ipfs.foo.bar.example2.com" \ + "http://127.0.0.1:$GWAY_PORT/" \ + "$CID_VAL" + +# foo.*.example3.com + +test_hostname_gateway_response_should_contain \ + "request for foo.bar.example3.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.foo.bar.example3.com" \ + "foo.bar.example3.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ + "Location: http://$CIDv1.ipfs.foo.bar.example3.com/" + +test_hostname_gateway_response_should_contain \ + "request for {CID}.ipfs.foo.bar.example3.com should return expected payload" \ + "${CIDv1}.ipfs.foo.bar.example3.com" \ + "http://127.0.0.1:$GWAY_PORT/" \ + "$CID_VAL" + +# foo.bar-*-boo.example4.com + +test_hostname_gateway_response_should_contain \ + "request for foo.bar-dev-boo.example4.com/ipfs/{CIDv1} produces redirect to {CIDv1}.ipfs.foo.bar-dev-boo.example4.com" \ + "foo.bar-dev-boo.example4.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1" \ + "Location: http://$CIDv1.ipfs.foo.bar-dev-boo.example4.com/" + +test_hostname_gateway_response_should_contain \ + "request for {CID}.ipfs.foo.bar-dev-boo.example4.com should return expected payload" \ + "${CIDv1}.ipfs.foo.bar-dev-boo.example4.com" \ + "http://127.0.0.1:$GWAY_PORT/" \ + "$CID_VAL" + # ============================================================================= # ensure we end with empty Gateway.PublicGateways ipfs config --json Gateway.PublicGateways '{}'