feat: CRL cache (#216)

Feat:
- added a `Cache` interface for CRL
- added a `Fetcher` for fetching from the cache, downloading and setting
the cache
Test:
- added unit test

---------

Signed-off-by: Junjie Gao <[email protected]>

revocation/crl/bundle.go

@@ -0,0 +1,28 @@
+// Copyright The Notary Project Authors.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crl
+
+import "crypto/x509"
+
+// Bundle is a collection of CRLs, including base and delta CRLs
+type Bundle struct {
+	// BaseCRL is the parsed base CRL
+	BaseCRL *x509.RevocationList
+
+	// DeltaCRL is the parsed delta CRL
+	//
+	// TODO: support delta CRL https://github.com/notaryproject/notation-core-go/issues/228
+	// It will always be nil until we support delta CRL
+	DeltaCRL *x509.RevocationList
+}

revocation/crl/cache.go

@@ -0,0 +1,32 @@
+// Copyright The Notary Project Authors.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crl
+
+import "context"
+
+// Cache is an interface that specifies methods used for caching
+type Cache interface {
+	// Get retrieves the CRL bundle with the given url
+	//
+	// url is the key to retrieve the CRL bundle
+	//
+	// if the key does not exist or the content is expired, return ErrCacheMiss.
+	Get(ctx context.Context, url string) (*Bundle, error)
+
+	// Set stores the CRL bundle with the given url
+	//
+	// url is the key to store the CRL bundle
+	// bundle is the CRL collections to store
+	Set(ctx context.Context, url string, bundle *Bundle) error
+}

revocation/internal/crl/errors.go → revocation/crl/errors.go

@@ -15,8 +15,5 @@ package crl
 
 import "errors"
 
-var (
-	// ErrDeltaCRLNotSupported is returned when the CRL contains a delta CRL but
-	// the delta CRL is not supported.
-	ErrDeltaCRLNotSupported = errors.New("delta CRL is not supported")
-)
+// ErrCacheMiss is returned when a cache miss occurs.
+var ErrCacheMiss = errors.New("cache miss")

revocation/crl/fetcher.go

@@ -0,0 +1,167 @@
+// Copyright The Notary Project Authors.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package crl provides Fetcher interface with its implementation, and the
+// Cache interface.
+package crl
+
+import (
+	"context"
+	"crypto/x509"
+	"encoding/asn1"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+// oidFreshestCRL is the object identifier for the distribution point
+// for the delta CRL. (See RFC 5280, Section 5.2.6)
+var oidFreshestCRL = asn1.ObjectIdentifier{2, 5, 29, 46}
+
+// maxCRLSize is the maximum size of CRL in bytes
+//
+// The 32 MiB limit is based on investigation that even the largest CRLs
+// are less than 16 MiB. The limit is set to 32 MiB to prevent
+const maxCRLSize = 32 * 1024 * 1024 // 32 MiB
+
+// Fetcher is an interface that specifies methods used for fetching CRL
+// from the given URL
+type Fetcher interface {
+	// Fetch retrieves the CRL from the given URL.
+	Fetch(ctx context.Context, url string) (*Bundle, error)
+}
+
+// HTTPFetcher is a Fetcher implementation that fetches CRL from the given URL
+type HTTPFetcher struct {
+	// Cache stores fetched CRLs and reuses them until the CRL reaches the
+	// NextUpdate time.
+	// If Cache is nil, no cache is used.
+	Cache Cache
+
+	// DiscardCacheError specifies whether to discard any error on cache.
+	//
+	// ErrCacheMiss is not considered as an failure and will not be returned as
+	// an error if DiscardCacheError is false.
+	DiscardCacheError bool
+
+	httpClient *http.Client
+}
+
+// NewHTTPFetcher creates a new HTTPFetcher with the given HTTP client
+func NewHTTPFetcher(httpClient *http.Client) (*HTTPFetcher, error) {
+	if httpClient == nil {
+		return nil, errors.New("httpClient cannot be nil")
+	}
+
+	return &HTTPFetcher{
+		httpClient: httpClient,
+	}, nil
+}
+
+// Fetch retrieves the CRL from the given URL
+//
+// If cache is not nil, try to get the CRL from the cache first. On failure
+// (e.g. cache miss), it will download the CRL from the URL and store it to the
+// cache.
+func (f *HTTPFetcher) Fetch(ctx context.Context, url string) (*Bundle, error) {
+	if url == "" {
+		return nil, errors.New("CRL URL cannot be empty")
+	}
+
+	if f.Cache != nil {
+		bundle, err := f.Cache.Get(ctx, url)
+		if err == nil {
+			// check expiry
+			nextUpdate := bundle.BaseCRL.NextUpdate
+			if !nextUpdate.IsZero() && !time.Now().After(nextUpdate) {
+				return bundle, nil
+			}
+		} else if !errors.Is(err, ErrCacheMiss) && !f.DiscardCacheError {
+			return nil, fmt.Errorf("failed to retrieve CRL from cache: %w", err)
+		}
+	}
+
+	bundle, err := f.fetch(ctx, url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to retrieve CRL: %w", err)
+	}
+
+	if f.Cache != nil {
+		err = f.Cache.Set(ctx, url, bundle)
+		if err != nil && !f.DiscardCacheError {
+			return nil, fmt.Errorf("failed to store CRL to cache: %w", err)
+		}
+	}
+
+	return bundle, nil
+}
+
+// fetch downloads the CRL from the given URL.
+func (f *HTTPFetcher) fetch(ctx context.Context, url string) (*Bundle, error) {
+	// fetch base CRL
+	base, err := fetchCRL(ctx, url, f.httpClient)
+	if err != nil {
+		return nil, err
+	}
+
+	// check delta CRL
+	// TODO: support delta CRL https://github.com/notaryproject/notation-core-go/issues/228
+	for _, ext := range base.Extensions {
+		if ext.Id.Equal(oidFreshestCRL) {
+			return nil, errors.New("delta CRL is not supported")
+		}
+	}
+
+	return &Bundle{
+		BaseCRL: base,
+	}, nil
+}
+
+func fetchCRL(ctx context.Context, crlURL string, client *http.Client) (*x509.RevocationList, error) {
+	// validate URL
+	parsedURL, err := url.Parse(crlURL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid CRL URL: %w", err)
+	}
+	if parsedURL.Scheme != "http" {
+		return nil, fmt.Errorf("unsupported scheme: %s. Only supports CRL URL in HTTP protocol", parsedURL.Scheme)
+	}
+
+	// download CRL
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, crlURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create CRL request: %w", err)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("request failed: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, fmt.Errorf("failed to download with status code: %d", resp.StatusCode)
+	}
+	// read with size limit
+	data, err := io.ReadAll(io.LimitReader(resp.Body, maxCRLSize))
+	if err != nil {
+		return nil, fmt.Errorf("failed to read CRL response: %w", err)
+	}
+	if len(data) == maxCRLSize {
+		return nil, fmt.Errorf("CRL size exceeds the limit: %d", maxCRLSize)
+	}
+
+	// parse CRL
+	return x509.ParseRevocationList(data)
+}