From be3d2ac7b0132edff8f3357b81d530e4e4753b68 Mon Sep 17 00:00:00 2001 From: zhouhao Date: Thu, 22 Dec 2016 17:04:00 +0800 Subject: [PATCH] Increase the generate function Signed-off-by: zhouhao --- .gitignore | 1 + Makefile | 3 +- cmd/oci-generate/main.go | 100 +++++++++++++ image/generate.go | 299 +++++++++++++++++++++++++++++++++++++++ image/image.go | 43 ++++++ image/image_test.go | 278 ------------------------------------ 6 files changed, 445 insertions(+), 279 deletions(-) create mode 100644 cmd/oci-generate/main.go create mode 100644 image/generate.go diff --git a/.gitignore b/.gitignore index d897571..64f48fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /oci-create-runtime-bundle /oci-unpack /oci-image-validate +/oci-generate diff --git a/Makefile b/Makefile index f0f35f7..3e36d68 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,8 @@ EPOCH_TEST_COMMIT ?= v0.2.0 TOOLS := \ oci-create-runtime-bundle \ oci-image-validate \ - oci-unpack + oci-unpack \ + oci-generate default: help diff --git a/cmd/oci-generate/main.go b/cmd/oci-generate/main.go new file mode 100644 index 0000000..3b57a3a --- /dev/null +++ b/cmd/oci-generate/main.go @@ -0,0 +1,100 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "log" + "os" + + specs "github.com/opencontainers/image-spec/specs-go" + "github.com/opencontainers/image-tools/image" + "github.com/spf13/cobra" +) + +// gitCommit will be the hash that the binary was built from +// and will be populated by the Makefile +var gitCommit = "" + +type generateCmd struct { + stdout *log.Logger + stderr *log.Logger + version bool +} + +func main() { + stdout := log.New(os.Stdout, "", 0) + stderr := log.New(os.Stderr, "", 0) + + cmd := newGenerateCmd(stdout, stderr) + if err := cmd.Execute(); err != nil { + stderr.Println(err) + os.Exit(1) + } +} + +func newGenerateCmd(stdout, stderr *log.Logger) *cobra.Command { + v := &generateCmd{ + stdout: stdout, + stderr: stderr, + } + + cmd := &cobra.Command{ + Use: "generate [dest]", + Short: "Generate an OCI image", + Long: `Generate the OCI image to the destination directory [dest].`, + Run: v.Run, + } + + cmd.Flags().BoolVarP( + &v.version, "version", "v", false, + `Print version information and exit`, + ) + + origHelp := cmd.HelpFunc() + + cmd.SetHelpFunc(func(c *cobra.Command, args []string) { + origHelp(c, args) + stdout.Println("\nMore information:") + stdout.Printf("\treferences\t%s\n", image.SpecURL) + stdout.Printf("\tbug report\t%s\n", image.IssuesURL) + }) + + return cmd +} + +func (v *generateCmd) Run(cmd *cobra.Command, args []string) { + if v.version { + v.stdout.Printf("commit: %s", gitCommit) + v.stdout.Printf("spec: %s", specs.Version) + os.Exit(0) + } + + if len(args) != 1 { + v.stderr.Print("dest must be provided") + if err := cmd.Usage(); err != nil { + v.stderr.Println(err) + } + os.Exit(1) + } + + err := image.Generate(args[0]) + + if err != nil { + v.stderr.Printf("generating failed: %v", err) + os.Exit(1) + } + + os.Exit(0) +} diff --git a/image/generate.go b/image/generate.go new file mode 100644 index 0000000..80ba895 --- /dev/null +++ b/image/generate.go @@ -0,0 +1,299 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package image + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/opencontainers/image-spec/specs-go/v1" +) + +const ( + refTag = "latest" + + layoutStr = `{"imageLayoutVersion": "1.0.0"}` + + configStr = `{ + "created": "2015-10-31T22:22:56.015925234Z", + "author": "Alyssa P. Hacker ", + "architecture": "amd64", + "os": "linux", + "config": { + "User": "alice", + "Memory": 2048, + "MemorySwap": 4096, + "CpuShares": 8, + "ExposedPorts": { + "8080/tcp": {} + }, + "Env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "FOO=oci_is_a", + "BAR=well_written_spec" + ], + "Entrypoint": [ + "/bin/my-app-binary" + ], + "Cmd": [ + "--foreground", + "--config", + "/etc/my-app.d/default.cfg" + ], + "Volumes": { + "/var/job-result-data": {}, + "/var/log/my-app-logs": {} + }, + "WorkingDir": "/home/alice" + }, + "rootfs": { + "diff_ids": [ + "sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1", + "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" + ], + "type": "layers" + }, + "history": [ + { + "created": "2015-10-31T22:22:54.690851953Z", + "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /" + }, + { + "created": "2015-10-31T22:22:55.613815829Z", + "created_by": "/bin/sh -c #(nop) CMD [\"sh\"]", + "empty_layer": true + } + ] +} +` +) + +var ( + refStr = `{"digest":"","mediaType":"application/vnd.oci.image.manifest.v1+json","size":}` + + manifestStr = `{ + "annotations": null, + "config": { + "digest": "", + "mediaType": "application/vnd.oci.image.config.v1+json", + "size": + }, + "layers": [ + { + "digest": "", + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "size": + } + ], + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "schemaVersion": 2 +} + ` +) + +type tarContent struct { + header *tar.Header + b []byte +} + +type imageLayout struct { + rootDir string + layout string + ref string + manifest string + config string + tarList []tarContent +} + +func createImageLayoutBundle(il imageLayout) error { + err := os.MkdirAll(filepath.Join(il.rootDir, "blobs", "sha256"), 0700) + if err != nil { + return err + } + + err = os.MkdirAll(filepath.Join(il.rootDir, "refs"), 0700) + if err != nil { + return err + } + + // create image layout file + err = createLayoutFile(il.rootDir) + if err != nil { + return err + } + + // create image layer blob file. + desc, err := createImageLayerFile(il.rootDir, il.tarList) + if err != nil { + return err + } + il.manifest = strings.Replace(il.manifest, "", desc.Digest, 1) + il.manifest = strings.Replace(il.manifest, "", strconv.FormatInt(desc.Size, 10), 1) + + desc, err = createConfigFile(il.rootDir, il.config) + if err != nil { + return err + } + il.manifest = strings.Replace(il.manifest, "", desc.Digest, 1) + il.manifest = strings.Replace(il.manifest, "", strconv.FormatInt(desc.Size, 10), 1) + + // create manifest blob file + desc, err = createManifestFile(il.rootDir, il.manifest) + if err != nil { + return err + } + + return createRefFile(il.rootDir, il.ref, desc) +} + +func createLayoutFile(root string) error { + layoutPath := filepath.Join(root, "oci-layout") + f, err := os.Create(layoutPath) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(f, bytes.NewBuffer([]byte(layoutStr))) + return err +} + +func createRefFile(root, ref string, mft descriptor) error { + refpath := filepath.Join(root, "refs", ref) + f, err := os.Create(refpath) + if err != nil { + return err + } + defer f.Close() + refStr = strings.Replace(refStr, "", mft.Digest, -1) + refStr = strings.Replace(refStr, "", strconv.FormatInt(mft.Size, 10), -1) + _, err = io.Copy(f, bytes.NewBuffer([]byte(refStr))) + return err +} + +func createManifestFile(root, str string) (descriptor, error) { + name := filepath.Join(root, "blobs", "sha256", "test-manifest") + f, err := os.Create(name) + if err != nil { + return descriptor{}, err + } + defer f.Close() + + _, err = io.Copy(f, bytes.NewBuffer([]byte(str))) + if err != nil { + return descriptor{}, err + } + + return createHashedBlob(name) +} + +func createConfigFile(root, config string) (descriptor, error) { + name := filepath.Join(root, "blobs", "sha256", "test-config") + f, err := os.Create(name) + if err != nil { + return descriptor{}, err + } + defer f.Close() + + _, err = io.Copy(f, bytes.NewBuffer([]byte(config))) + if err != nil { + return descriptor{}, err + } + + return createHashedBlob(name) +} + +func createImageLayerFile(root string, list []tarContent) (descriptor, error) { + name := filepath.Join(root, "blobs", "sha256", "test-layer") + err := createTarBlob(name, list) + if err != nil { + return descriptor{}, err + } + + desc, err := createHashedBlob(name) + if err != nil { + return descriptor{}, err + } + + desc.MediaType = v1.MediaTypeImageLayer + return desc, nil +} + +func createTarBlob(name string, list []tarContent) error { + file, err := os.Create(name) + if err != nil { + return err + } + defer file.Close() + gzipWriter := gzip.NewWriter(file) + defer gzipWriter.Close() + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + + for _, content := range list { + if err = tarWriter.WriteHeader(content.header); err != nil { + return err + } + if _, err = io.Copy(tarWriter, bytes.NewReader(content.b)); err != nil { + return err + } + } + return nil +} + +func createHashedBlob(name string) (descriptor, error) { + desc, err := newDescriptor(name) + if err != nil { + return descriptor{}, err + } + + // Rename the file to hashed-digest name. + err = os.Rename(name, filepath.Join(filepath.Dir(name), desc.Digest)) + if err != nil { + return descriptor{}, err + } + + //Normalize the hashed digest. + desc.Digest = "sha256:" + desc.Digest + + return desc, nil +} + +func newDescriptor(name string) (descriptor, error) { + file, err := os.Open(name) + if err != nil { + return descriptor{}, err + } + defer file.Close() + + // generate sha256 hash + hash := sha256.New() + size, err := io.Copy(hash, file) + if err != nil { + return descriptor{}, err + } + + return descriptor{ + Digest: fmt.Sprintf("%x", hash.Sum(nil)), + Size: size, + }, nil +} diff --git a/image/image.go b/image/image.go index 1551c1c..123ce35 100644 --- a/image/image.go +++ b/image/image.go @@ -15,8 +15,10 @@ package image import ( + "archive/tar" "encoding/json" "fmt" + "io/ioutil" "log" "os" "path/filepath" @@ -198,3 +200,44 @@ func createRuntimeBundle(w walker, dest, refName, rootfs string) error { return json.NewEncoder(f).Encode(spec) } + +// Generate create an imageLayou that conforms to the OCI specification on a given path +func Generate(dest string) (retErr error) { + // error out if the dest directory is not empty or failed to create the file + s, err := ioutil.ReadDir(dest) + if err != nil { + if os.IsNotExist(err) { + if err2 := os.Mkdir(dest, 0755); err2 != nil { + return errors.Wrap(err2, "unable to create file") + } + } else { + return errors.Wrap(err, "unable to open file") // err contains dest + } + } + if len(s) > 0 { + return fmt.Errorf("%s is not empty", dest) + } + defer func() { + // if we encounter error during generating + // clean up the partially-denerated destination + if retErr != nil { + if err := os.RemoveAll(dest); err != nil { + fmt.Printf("Error: failed to remove partially-unpacked destination %v", err) + } + } + }() + + il := imageLayout{ + rootDir: dest, + layout: layoutStr, + ref: refTag, + manifest: manifestStr, + config: configStr, + tarList: []tarContent{ + tarContent{&tar.Header{Name: "example", Size: 7, Mode: 0600}, []byte("example")}, + }, + } + + // create image layout bundle + return createImageLayoutBundle(il) +} diff --git a/image/image_test.go b/image/image_test.go index e808e7d..2d09d92 100644 --- a/image/image_test.go +++ b/image/image_test.go @@ -17,116 +17,11 @@ package image import ( "archive/tar" "bytes" - "compress/gzip" - "crypto/sha256" - "fmt" - "io" "io/ioutil" "os" - "path/filepath" - "strconv" - "strings" "testing" - - "github.com/opencontainers/image-spec/specs-go/v1" -) - -const ( - refTag = "latest" - - layoutStr = `{"imageLayoutVersion": "1.0.0"}` - - configStr = `{ - "created": "2015-10-31T22:22:56.015925234Z", - "author": "Alyssa P. Hacker ", - "architecture": "amd64", - "os": "linux", - "config": { - "User": "alice", - "Memory": 2048, - "MemorySwap": 4096, - "CpuShares": 8, - "ExposedPorts": { - "8080/tcp": {} - }, - "Env": [ - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", - "FOO=oci_is_a", - "BAR=well_written_spec" - ], - "Entrypoint": [ - "/bin/my-app-binary" - ], - "Cmd": [ - "--foreground", - "--config", - "/etc/my-app.d/default.cfg" - ], - "Volumes": { - "/var/job-result-data": {}, - "/var/log/my-app-logs": {} - }, - "WorkingDir": "/home/alice" - }, - "rootfs": { - "diff_ids": [ - "sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1", - "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" - ], - "type": "layers" - }, - "history": [ - { - "created": "2015-10-31T22:22:54.690851953Z", - "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /" - }, - { - "created": "2015-10-31T22:22:55.613815829Z", - "created_by": "/bin/sh -c #(nop) CMD [\"sh\"]", - "empty_layer": true - } - ] -} -` ) -var ( - refStr = `{"digest":"","mediaType":"application/vnd.oci.image.manifest.v1+json","size":}` - - manifestStr = `{ - "annotations": null, - "config": { - "digest": "", - "mediaType": "application/vnd.oci.image.config.v1+json", - "size": - }, - "layers": [ - { - "digest": "", - "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", - "size": - } - ], - "mediaType": "application/vnd.oci.image.manifest.v1+json", - "schemaVersion": 2 -} - ` -) - -type tarContent struct { - header *tar.Header - b []byte -} - -type imageLayout struct { - rootDir string - layout string - ref string - manifest string - config string - tarList []tarContent -} - func TestValidateLayout(t *testing.T) { root, err := ioutil.TempDir("", "oci-test") if err != nil { @@ -156,176 +51,3 @@ func TestValidateLayout(t *testing.T) { t.Fatal(err) } } - -func createImageLayoutBundle(il imageLayout) error { - err := os.MkdirAll(filepath.Join(il.rootDir, "blobs", "sha256"), 0700) - if err != nil { - return err - } - - err = os.MkdirAll(filepath.Join(il.rootDir, "refs"), 0700) - if err != nil { - return err - } - - // create image layout file - err = createLayoutFile(il.rootDir) - if err != nil { - return err - } - - // create image layer blob file. - desc, err := createImageLayerFile(il.rootDir, il.tarList) - if err != nil { - return err - } - il.manifest = strings.Replace(il.manifest, "", desc.Digest, 1) - il.manifest = strings.Replace(il.manifest, "", strconv.FormatInt(desc.Size, 10), 1) - - desc, err = createConfigFile(il.rootDir, il.config) - if err != nil { - return err - } - il.manifest = strings.Replace(il.manifest, "", desc.Digest, 1) - il.manifest = strings.Replace(il.manifest, "", strconv.FormatInt(desc.Size, 10), 1) - - // create manifest blob file - desc, err = createManifestFile(il.rootDir, il.manifest) - if err != nil { - return err - } - - return createRefFile(il.rootDir, il.ref, desc) -} - -func createLayoutFile(root string) error { - layoutPath := filepath.Join(root, "oci-layout") - f, err := os.Create(layoutPath) - if err != nil { - return err - } - defer f.Close() - _, err = io.Copy(f, bytes.NewBuffer([]byte(layoutStr))) - return err -} - -func createRefFile(root, ref string, mft descriptor) error { - refpath := filepath.Join(root, "refs", ref) - f, err := os.Create(refpath) - if err != nil { - return err - } - defer f.Close() - refStr = strings.Replace(refStr, "", mft.Digest, -1) - refStr = strings.Replace(refStr, "", strconv.FormatInt(mft.Size, 10), -1) - _, err = io.Copy(f, bytes.NewBuffer([]byte(refStr))) - return err -} - -func createManifestFile(root, str string) (descriptor, error) { - name := filepath.Join(root, "blobs", "sha256", "test-manifest") - f, err := os.Create(name) - if err != nil { - return descriptor{}, err - } - defer f.Close() - - _, err = io.Copy(f, bytes.NewBuffer([]byte(str))) - if err != nil { - return descriptor{}, err - } - - return createHashedBlob(name) -} - -func createConfigFile(root, config string) (descriptor, error) { - name := filepath.Join(root, "blobs", "sha256", "test-config") - f, err := os.Create(name) - if err != nil { - return descriptor{}, err - } - defer f.Close() - - _, err = io.Copy(f, bytes.NewBuffer([]byte(config))) - if err != nil { - return descriptor{}, err - } - - return createHashedBlob(name) -} - -func createImageLayerFile(root string, list []tarContent) (descriptor, error) { - name := filepath.Join(root, "blobs", "sha256", "test-layer") - err := createTarBlob(name, list) - if err != nil { - return descriptor{}, err - } - - desc, err := createHashedBlob(name) - if err != nil { - return descriptor{}, err - } - - desc.MediaType = v1.MediaTypeImageLayer - return desc, nil -} - -func createTarBlob(name string, list []tarContent) error { - file, err := os.Create(name) - if err != nil { - return err - } - defer file.Close() - gzipWriter := gzip.NewWriter(file) - defer gzipWriter.Close() - tarWriter := tar.NewWriter(gzipWriter) - defer tarWriter.Close() - - for _, content := range list { - if err = tarWriter.WriteHeader(content.header); err != nil { - return err - } - if _, err = io.Copy(tarWriter, bytes.NewReader(content.b)); err != nil { - return err - } - } - return nil -} - -func createHashedBlob(name string) (descriptor, error) { - desc, err := newDescriptor(name) - if err != nil { - return descriptor{}, err - } - - // Rename the file to hashed-digest name. - err = os.Rename(name, filepath.Join(filepath.Dir(name), desc.Digest)) - if err != nil { - return descriptor{}, err - } - - //Normalize the hashed digest. - desc.Digest = "sha256:" + desc.Digest - - return desc, nil -} - -func newDescriptor(name string) (descriptor, error) { - file, err := os.Open(name) - if err != nil { - return descriptor{}, err - } - defer file.Close() - - // generate sha256 hash - hash := sha256.New() - size, err := io.Copy(hash, file) - if err != nil { - return descriptor{}, err - } - - return descriptor{ - Digest: fmt.Sprintf("%x", hash.Sum(nil)), - Size: size, - }, nil -}