Skip to content

Commit

Permalink
fix: use Hostname instead of Host in s3utils.GetRegionFromURL (#2046)
Browse files Browse the repository at this point in the history 
This resolves an issue where the regex fails to match the region if the endpoint includes a port.
Reference: https://pkg.go.dev/net/url#URL.Hostname
  • Loading branch information
@huanghaoyuanhhy
huanghaoyuanhhy authored Jan 9, 2025
1 parent ad925db commit 5cf2ee7
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 113 deletions.
20 changes: 10 additions & 10 deletions pkg/s3utils/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,53 +118,53 @@ func GetRegionFromURL(endpointURL url.URL) string {
if endpointURL == sentinelURL {
return ""
}
if endpointURL.Host == "s3-external-1.amazonaws.com" {
if endpointURL.Hostname() == "s3-external-1.amazonaws.com" {
return ""
}

// if elb's are used we cannot calculate which region it may be, just return empty.
if elbAmazonRegex.MatchString(endpointURL.Host) || elbAmazonCnRegex.MatchString(endpointURL.Host) {
if elbAmazonRegex.MatchString(endpointURL.Hostname()) || elbAmazonCnRegex.MatchString(endpointURL.Hostname()) {
return ""
}

// We check for FIPS dualstack matching first to avoid the non-greedy
// regex for FIPS non-dualstack matching a dualstack URL
parts := amazonS3HostFIPSDualStack.FindStringSubmatch(endpointURL.Host)
parts := amazonS3HostFIPSDualStack.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}

parts = amazonS3HostFIPS.FindStringSubmatch(endpointURL.Host)
parts = amazonS3HostFIPS.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}

parts = amazonS3HostDualStack.FindStringSubmatch(endpointURL.Host)
parts = amazonS3HostDualStack.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}

parts = amazonS3HostHyphen.FindStringSubmatch(endpointURL.Host)
parts = amazonS3HostHyphen.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}

parts = amazonS3ChinaHost.FindStringSubmatch(endpointURL.Host)
parts = amazonS3ChinaHost.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}

parts = amazonS3ChinaHostDualStack.FindStringSubmatch(endpointURL.Host)
parts = amazonS3ChinaHostDualStack.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}

parts = amazonS3HostDot.FindStringSubmatch(endpointURL.Host)
parts = amazonS3HostDot.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}

parts = amazonS3HostPrivateLink.FindStringSubmatch(endpointURL.Host)
parts = amazonS3HostPrivateLink.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
return parts[1]
}
Expand Down
151 changes: 48 additions & 103 deletions pkg/s3utils/utils_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,114 +27,59 @@ import (
// Tests get region from host URL.
func TestGetRegionFromURL(t *testing.T) {
testCases := []struct {
u url.URL
u string
expectedRegion string
}{
{
u: url.URL{Host: "storage.googleapis.com"},
expectedRegion: "",
},
{
u: url.URL{Host: "s3.cn-north-1.amazonaws.com.cn"},
expectedRegion: "cn-north-1",
},
{
u: url.URL{Host: "s3.dualstack.cn-north-1.amazonaws.com.cn"},
expectedRegion: "cn-north-1",
},
{
u: url.URL{Host: "s3.cn-northwest-1.amazonaws.com.cn"},
expectedRegion: "cn-northwest-1",
},
{
u: url.URL{Host: "s3.dualstack.cn-northwest-1.amazonaws.com.cn"},
expectedRegion: "cn-northwest-1",
},
{
u: url.URL{Host: "s3-fips-us-gov-west-1.amazonaws.com"},
expectedRegion: "us-gov-west-1",
},
{
u: url.URL{Host: "s3-fips.us-gov-west-1.amazonaws.com"},
expectedRegion: "us-gov-west-1",
},
{
u: url.URL{Host: "s3-fips.us-gov-east-1.amazonaws.com"},
expectedRegion: "us-gov-east-1",
},
{
u: url.URL{Host: "s3-us-gov-west-1.amazonaws.com"},
expectedRegion: "us-gov-west-1",
},
{
u: url.URL{Host: "192.168.1.1"},
expectedRegion: "",
},
{
u: url.URL{Host: "s3-eu-west-1.amazonaws.com"},
expectedRegion: "eu-west-1",
},
{
u: url.URL{Host: "s3.eu-west-1.amazonaws.com"},
expectedRegion: "eu-west-1",
},
{
u: url.URL{Host: "s3.dualstack.eu-west-1.amazonaws.com"},
expectedRegion: "eu-west-1",
},
{
u: url.URL{Host: "s3.amazonaws.com"},
expectedRegion: "",
},
{
u: url.URL{Host: "s3-external-1.amazonaws.com"},
expectedRegion: "",
},
{
u: url.URL{
Host: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.us-west-2.amazonaws.com",
},
expectedRegion: "",
},
{
u: url.URL{
Host: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.amazonaws.com",
},
expectedRegion: "",
},
{
u: url.URL{
Host: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.amazonaws.com.cn",
},
},
{
u: url.URL{
Host: "bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com",
},
expectedRegion: "us-east-1",
},
{
u: url.URL{
Host: "accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com",
},
expectedRegion: "us-east-1",
},
{
u: url.URL{
Host: "s3-fips.us-east-1.amazonaws.com",
},
expectedRegion: "us-east-1",
},
{
u: url.URL{
Host: "s3-fips.dualstack.us-west-1.amazonaws.com",
},
expectedRegion: "us-west-1",
},
{u: "storage.googleapis.com", expectedRegion: ""},
{u: "s3.cn-north-1.amazonaws.com.cn", expectedRegion: "cn-north-1"},
{u: "s3.dualstack.cn-north-1.amazonaws.com.cn", expectedRegion: "cn-north-1"},
{u: "s3.cn-northwest-1.amazonaws.com.cn", expectedRegion: "cn-northwest-1"},
{u: "s3.dualstack.cn-northwest-1.amazonaws.com.cn", expectedRegion: "cn-northwest-1"},
{u: "s3-fips-us-gov-west-1.amazonaws.com", expectedRegion: "us-gov-west-1"},
{u: "s3-fips.us-gov-west-1.amazonaws.com", expectedRegion: "us-gov-west-1"},
{u: "s3-fips.us-gov-east-1.amazonaws.com", expectedRegion: "us-gov-east-1"},
{u: "s3-us-gov-west-1.amazonaws.com", expectedRegion: "us-gov-west-1"},
{u: "192.168.1.1", expectedRegion: ""},
{u: "s3-eu-west-1.amazonaws.com", expectedRegion: "eu-west-1"},
{u: "s3.eu-west-1.amazonaws.com", expectedRegion: "eu-west-1"},
{u: "s3.dualstack.eu-west-1.amazonaws.com", expectedRegion: "eu-west-1"},
{u: "s3.amazonaws.com", expectedRegion: ""},
{u: "s3-external-1.amazonaws.com", expectedRegion: ""},
{u: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.us-west-2.amazonaws.com", expectedRegion: ""},
{u: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.amazonaws.com", expectedRegion: ""},
{u: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.amazonaws.com.cn", expectedRegion: ""},
{u: "bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com", expectedRegion: "us-east-1"},
{u: "accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com", expectedRegion: "us-east-1"},
{u: "s3-fips.us-east-1.amazonaws.com", expectedRegion: "us-east-1"},
{u: "s3-fips.dualstack.us-west-1.amazonaws.com", expectedRegion: "us-west-1"},

// Test cases with port numbers.
{u: "storage.googleapis.com:80", expectedRegion: ""},
{u: "s3.cn-north-1.amazonaws.com.cn:80", expectedRegion: "cn-north-1"},
{u: "s3.dualstack.cn-north-1.amazonaws.com.cn:80", expectedRegion: "cn-north-1"},
{u: "s3.cn-northwest-1.amazonaws.com.cn:80", expectedRegion: "cn-northwest-1"},
{u: "s3.dualstack.cn-northwest-1.amazonaws.com.cn:80", expectedRegion: "cn-northwest-1"},
{u: "s3-fips-us-gov-west-1.amazonaws.com:80", expectedRegion: "us-gov-west-1"},
{u: "s3-fips.us-gov-west-1.amazonaws.com:80", expectedRegion: "us-gov-west-1"},
{u: "s3-fips.us-gov-east-1.amazonaws.com:80", expectedRegion: "us-gov-east-1"},
{u: "s3-us-gov-west-1.amazonaws.com:80", expectedRegion: "us-gov-west-1"},
{u: "192.168.1.1:80", expectedRegion: ""},
{u: "s3-eu-west-1.amazonaws.com:80", expectedRegion: "eu-west-1"},
{u: "s3.eu-west-1.amazonaws.com:80", expectedRegion: "eu-west-1"},
{u: "s3.dualstack.eu-west-1.amazonaws.com:80", expectedRegion: "eu-west-1"},
{u: "s3.amazonaws.com:80", expectedRegion: ""},
{u: "s3-external-1.amazonaws.com:80", expectedRegion: ""},
{u: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.us-west-2.amazonaws.com:80", expectedRegion: ""},
{u: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.amazonaws.com:80", expectedRegion: ""},
{u: "s3.kubernetesfrontendlb-caf78da2b1f7516c.elb.amazonaws.com.cn:80", expectedRegion: ""},
{u: "bucket.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com:80", expectedRegion: "us-east-1"},
{u: "accesspoint.vpce-1a2b3c4d-5e6f.s3.us-east-1.vpce.amazonaws.com:80", expectedRegion: "us-east-1"},
{u: "s3-fips.us-east-1.amazonaws.com:80", expectedRegion: "us-east-1"},
{u: "s3-fips.dualstack.us-west-1.amazonaws.com:80", expectedRegion: "us-west-1"},
}

for i, testCase := range testCases {
region := GetRegionFromURL(testCase.u)
region := GetRegionFromURL(url.URL{Host: testCase.u})
if testCase.expectedRegion != region {
t.Errorf("Test %d: Expected region %s, got %s", i+1, testCase.expectedRegion, region)
}
Expand Down

0 comments on commit 5cf2ee7

Please sign in to comment.