image/cas: Add a generic CAS interface

And implement that interface for tarballs based on the specs
image-layout.  I plan on adding other backends and methods later, but
this is enough for a proof of concept getter.

Also add a new oci-cas command so folks can access the new read
functionality from the command line.

In a subsequent commit, I'll replace the image/walker.go functionality
with this new API.

The Context interface follows the pattern recommended in [1], allowing
callers to cancel long running actions (e.g. push/pull over the
network for engine implementations that communicate with a remote
store).  Passing a Context instance along to NewEngine gives us a way
to cancel engine initialization which happens to take too long.
That's unlikely to happen for NewTarEngine, but it's easy enough to
pass it on down just in case.

blobPath's separator argument will allow us to use
string(os.PathSeparator)) once we add directory support.

[1]: https://blog.golang.org/context

Signed-off-by: W. Trevor King <[email protected]>

.gitignore

@@ -1,3 +1,4 @@
+/oci-cas
 /oci-create-runtime-bundle
-/oci-unpack
 /oci-image-validate
+/oci-unpack

Makefile

@@ -5,6 +5,7 @@ COMMIT=$(shell git rev-parse HEAD 2> /dev/null || true)
 
 EPOCH_TEST_COMMIT ?= v0.2.0
 TOOLS := \
+	oci-cas \
 	oci-create-runtime-bundle \
 	oci-image-validate \
 	oci-unpack

cmd/oci-cas/get.go

@@ -0,0 +1,99 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-tools/image/cas/layout"
+	"github.com/spf13/cobra"
+	"golang.org/x/net/context"
+)
+
+type getCmd struct {
+	path   string
+	digest digest.Digest
+}
+
+func newGetCmd() *cobra.Command {
+	state := &getCmd{}
+
+	return &cobra.Command{
+		Use:   "get PATH DIGEST",
+		Short: "Retrieve a blob from the store",
+		Long:  "Retrieve a blob from the store and write it to stdout.",
+		Run:   state.Run,
+	}
+}
+
+func (state *getCmd) Run(cmd *cobra.Command, args []string) {
+	if len(args) != 2 {
+		fmt.Fprintln(os.Stderr, "both PATH and DIGEST must be provided")
+		if err := cmd.Usage(); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+		}
+		os.Exit(1)
+	}
+
+	state.path = args[0]
+	var err error
+	state.digest, err = digest.Parse(args[1])
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+
+	err = state.run()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+
+	os.Exit(0)
+}
+
+func (state *getCmd) run() (err error) {
+	ctx := context.Background()
+
+	engine, err := layout.NewEngine(ctx, state.path)
+	if err != nil {
+		return err
+	}
+	defer engine.Close()
+
+	reader, err := engine.Get(ctx, state.digest)
+	if err != nil {
+		return err
+	}
+	defer reader.Close()
+
+	bytes, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return err
+	}
+
+	n, err := os.Stdout.Write(bytes)
+	if err != nil {
+		return err
+	}
+	if n < len(bytes) {
+		return fmt.Errorf("wrote %d of %d bytes", n, len(bytes))
+	}
+
+	return nil
+}

cmd/oci-cas/main.go

@@ -0,0 +1,39 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	_ "crypto/sha256"
+	_ "crypto/sha512"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+func main() {
+	cmd := &cobra.Command{
+		Use:   "oci-cas",
+		Short: "Content-addressable storage manipulation",
+	}
+
+	cmd.AddCommand(newGetCmd())
+
+	err := cmd.Execute()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

image/cas/interface.go

@@ -0,0 +1,34 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package cas implements generic content-addressable storage.
+package cas
+
+import (
+	"io"
+
+	"github.com/opencontainers/go-digest"
+	"golang.org/x/net/context"
+)
+
+// Engine represents a content-addressable storage engine.
+type Engine interface {
+	// Get returns a reader for retrieving a blob from the store.
+	// Returns os.ErrNotExist if the digest is not found.
+	Get(ctx context.Context, digest digest.Digest) (reader io.ReadCloser, err error)
+
+	// Close releases resources held by the engine.  Subsequent engine
+	// method calls will fail.
+	Close() (err error)
+}

image/cas/layout/interface.go

@@ -0,0 +1,25 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package layout
+
+import (
+	"io"
+)
+
+// ReadWriteSeekCloser wraps the Read, Write, Seek, and Close methods.
+type ReadWriteSeekCloser interface {
+	io.ReadWriteSeeker
+	io.Closer
+}

image/cas/layout/main.go

@@ -0,0 +1,51 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package layout implements the cas interface using the image-spec's
+// image-layout [1].
+//
+// [1]: https://github.com/opencontainers/image-spec/blob/master/image-layout.md
+package layout
+
+import (
+	"os"
+	"strings"
+
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-tools/image/cas"
+	"golang.org/x/net/context"
+)
+
+// NewEngine instantiates an engine with the appropriate backend (tar,
+// HTTP, ...).
+func NewEngine(ctx context.Context, path string) (engine cas.Engine, err error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewTarEngine(ctx, file)
+}
+
+// blobPath returns the PATH to the DIGEST blob.  SEPARATOR selects
+// the path separator used between components.
+func blobPath(digest digest.Digest, separator string) (path string, err error) {
+	err = digest.Validate()
+	if err != nil {
+		return "", err
+	}
+	algorithm := digest.Algorithm().String()
+	components := []string{".", "blobs", algorithm, digest.Hex()}
+	return strings.Join(components, separator), nil
+}

image/cas/layout/tar.go

@@ -0,0 +1,79 @@
+// Copyright 2016 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package layout
+
+import (
+	"archive/tar"
+	"errors"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-tools/image/cas"
+	"golang.org/x/net/context"
+)
+
+// TarEngine is a cas.Engine backed by a tar file.
+type TarEngine struct {
+	file ReadWriteSeekCloser
+}
+
+// NewTarEngine returns a new TarEngine.
+func NewTarEngine(ctx context.Context, file ReadWriteSeekCloser) (engine cas.Engine, err error) {
+	engine = &TarEngine{
+		file: file,
+	}
+
+	return engine, nil
+}
+
+// Get returns a reader for retrieving a blob from the store.
+func (engine *TarEngine) Get(ctx context.Context, digest digest.Digest) (reader io.ReadCloser, err error) {
+	targetName, err := blobPath(digest, "/")
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = engine.file.Seek(0, os.SEEK_SET)
+	if err != nil {
+		return nil, err
+	}
+
+	tarReader := tar.NewReader(engine.file)
+	for {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+		}
+
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			return nil, os.ErrNotExist
+		} else if err != nil {
+			return nil, err
+		}
+
+		if header.Name == targetName {
+			return ioutil.NopCloser(tarReader), nil
+		}
+	}
+}
+
+// Close releases resources held by the engine.
+func (engine *TarEngine) Close() (err error) {
+	return engine.file.Close()
+}