From 25e67b373b5101a111d6a36ddf9f9f40db56e0ba Mon Sep 17 00:00:00 2001 From: Adam Hughes Date: Wed, 23 Jun 2021 03:31:53 +0000 Subject: [PATCH] build(deps): use github.com/sylabs/sif/v2 --- cmd/internal/cli/inspect.go | 4 +- cmd/internal/cli/pgp.go | 40 ++---- cmd/internal/cli/siftool.go | 22 ++- docs/content.go | 18 ++- go.mod | 3 +- go.sum | 34 ++++- internal/app/singularity/overlay_create.go | 77 ++++++----- .../app/singularity/plugin_compile_linux.go | 125 ++++-------------- internal/app/singularity/push.go | 12 +- internal/app/singularity/sign.go | 10 +- internal/app/singularity/sign_test.go | 4 +- internal/app/singularity/verify.go | 23 ++-- internal/app/singularity/verify_test.go | 78 +++++++---- internal/pkg/build/assemblers/sif.go | 97 ++++---------- internal/pkg/plugin/sif.go | 10 +- internal/pkg/syecl/syecl.go | 18 ++- internal/pkg/util/machine/machine.go | 18 +-- pkg/image/sif.go | 107 ++++++--------- pkg/image/sif_test.go | 125 ++++++------------ pkg/util/cryptkey/key.go | 31 +++-- 20 files changed, 366 insertions(+), 490 deletions(-) diff --git a/cmd/internal/cli/inspect.go b/cmd/internal/cli/inspect.go index 0bc88c84ba..57ac5d7920 100644 --- a/cmd/internal/cli/inspect.go +++ b/cmd/internal/cli/inspect.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -20,7 +20,7 @@ import ( "strings" "github.com/spf13/cobra" - "github.com/sylabs/sif/pkg/sif" + "github.com/sylabs/sif/v2/pkg/sif" "github.com/sylabs/singularity/docs" "github.com/sylabs/singularity/internal/pkg/util/env" "github.com/sylabs/singularity/pkg/cmdline" diff --git a/cmd/internal/cli/pgp.go b/cmd/internal/cli/pgp.go index 0a2a0946a8..bc34166d3f 100644 --- a/cmd/internal/cli/pgp.go +++ b/cmd/internal/cli/pgp.go @@ -1,5 +1,5 @@ // Copyright (c) 2020, Control Command Inc. All rights reserved. -// Copyright (c) 2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2020-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the LICENSE.md file // distributed with the sources of this project regarding your rights to use or distribute this // software. @@ -17,8 +17,8 @@ import ( "github.com/sylabs/singularity/internal/pkg/buildcfg" "github.com/fatih/color" - "github.com/sylabs/sif/pkg/integrity" - "github.com/sylabs/sif/pkg/sif" + "github.com/sylabs/sif/v2/pkg/integrity" + "github.com/sylabs/sif/v2/pkg/sif" "github.com/sylabs/singularity/internal/app/singularity" "github.com/sylabs/singularity/internal/pkg/util/interactive" "github.com/sylabs/singularity/pkg/sylog" @@ -175,28 +175,22 @@ func outputVerify(f *sif.FileImage, r integrity.VerifyResult) bool { fmt.Printf("%-4s|%-8s|%-8s|%s\n", "ID", "GROUP", "LINK", "TYPE") fmt.Print("------------------------------------------------\n") } - for _, id := range r.Verified() { - od, _, err := f.GetFromDescrID(id) - if err != nil { - sylog.Errorf("failed to get descriptor: %v", err) - return false - } - + for _, od := range r.Verified() { group := "NONE" - if gid := od.Groupid; gid != sif.DescrUnusedGroup { - group = fmt.Sprintf("%d", gid&^sif.DescrGroupMask) + if gid := od.GroupID(); gid != 0 { + group = fmt.Sprintf("%d", gid) } link := "NONE" - if l := od.Link; l != sif.DescrUnusedLink { - if l&sif.DescrGroupMask == sif.DescrGroupMask { - link = fmt.Sprintf("%d (G)", l&^sif.DescrGroupMask) + if l, isGroup := od.LinkedID(); l != 0 { + if isGroup { + link = fmt.Sprintf("%d (G)", l) } else { link = fmt.Sprintf("%d", l) } } - fmt.Printf("%-4d|%-8s|%-8s|%s\n", id, group, link, od.Datatype) + fmt.Printf("%-4d|%-8s|%-8s|%s\n", od.ID(), group, link, od.DataType()) } if err := r.Error(); err != nil { @@ -246,15 +240,9 @@ func getJSONCallback(kl *keyList) singularity.VerifyCallback { } // For each verified object, append an entry to the list. - for _, id := range r.Verified() { - od, _, err := f.GetFromDescrID(id) - if err != nil { - sylog.Errorf("failed to get descriptor: %v", err) - continue - } - + for _, od := range r.Verified() { ke := keyEntity{ - Partition: od.Datatype.String(), + Partition: od.DataType().String(), Name: name, Fingerprint: fp, KeyLocal: keyLocal, @@ -266,14 +254,14 @@ func getJSONCallback(kl *keyList) singularity.VerifyCallback { var integrityError *integrity.ObjectIntegrityError if errors.As(r.Error(), &integrityError) { - od, _, err := f.GetFromDescrID(integrityError.ID) + od, err := f.GetDescriptor(sif.WithID(integrityError.ID)) if err != nil { sylog.Errorf("failed to get descriptor: %v", err) return false } ke := keyEntity{ - Partition: od.Datatype.String(), + Partition: od.DataType().String(), Name: name, Fingerprint: fp, KeyLocal: keyLocal, diff --git a/cmd/internal/cli/siftool.go b/cmd/internal/cli/siftool.go index 815be6f4a2..36a65a1ace 100644 --- a/cmd/internal/cli/siftool.go +++ b/cmd/internal/cli/siftool.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -6,16 +6,24 @@ package cli import ( - "github.com/sylabs/sif/pkg/siftool" + "github.com/spf13/cobra" + "github.com/sylabs/sif/v2/pkg/siftool" + "github.com/sylabs/singularity/docs" "github.com/sylabs/singularity/pkg/cmdline" ) -// SiftoolCmd is easily set since the sif repo allows the cobra.Command struct to be -// easily accessed with Siftool(), we do not need to do anything but call that function. -var SiftoolCmd = siftool.Siftool() - func init() { addCmdInit(func(cmdManager *cmdline.CommandManager) { - cmdManager.RegisterCmd(SiftoolCmd) + cmd := &cobra.Command{ + Use: docs.SIFUse, + Aliases: []string{docs.SIFAlias}, + Short: docs.SIFShort, + Long: docs.SIFLong, + Example: docs.SIFExample, + DisableFlagsInUseLine: true, + } + siftool.AddCommands(cmd) + + cmdManager.RegisterCmd(cmd) }) } diff --git a/docs/content.go b/docs/content.go index 8a7ce21c2c..3d3b318fba 100644 --- a/docs/content.go +++ b/docs/content.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2017-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -1062,3 +1062,19 @@ Enterprise Performance Computing (EPC)` To create a single EXT3 writable overlay image: $ singularity overlay create --size 1024 /tmp/my_overlay.img` ) + +// Documentation for sif/siftool command. +const ( + SIFUse string = `sif` + SIFAlias string = `siftool` + SIFShort string = `Manipulate Singularity Image Format (SIF) images` + SIFLong string = ` + A set of commands are provided to display elements such as the SIF global + header, the data object descriptors and to dump data objects. It is also + possible to modify a SIF file via this tool via the add/del commands.` + SIFExample string = ` + All sif commands have their own help output: + + $ singularity help sif list + $ singularity sif list --help` +) diff --git a/go.mod b/go.mod index b534aed3cf..183df98d7c 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,6 @@ require ( github.com/pelletier/go-toml v1.9.3 github.com/pkg/errors v0.9.1 github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/satori/go.uuid v1.2.1-0.20180404165556-75cca531ea76 github.com/seccomp/containers-golang v0.6.0 github.com/seccomp/libseccomp-golang v0.9.1 github.com/spf13/cobra v1.2.1 @@ -50,7 +49,7 @@ require ( github.com/sylabs/scs-build-client v0.1.6 github.com/sylabs/scs-key-client v0.6.2 github.com/sylabs/scs-library-client v1.0.5 - github.com/sylabs/sif v1.6.0 + github.com/sylabs/sif/v2 v2.0.0-beta.5 github.com/urfave/cli v1.22.5 // indirect github.com/vbauerster/mpb/v4 v4.12.2 github.com/vbauerster/mpb/v6 v6.0.4 diff --git a/go.sum b/go.sum index 6c7204ba99..31453555c3 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,7 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/Netflix/go-expect v0.0.0-20190729225929-0e00d9168667 h1:l2RCK7mjLhjfZRIcCXTVHI34l67IRtKASBjusViLzQ0= github.com/Netflix/go-expect v0.0.0-20190729225929-0e00d9168667/go.mod h1:oX5x61PbNXchhh0oikYAH+4Pcfw5LKv21+Jnpr6r6Pc= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= @@ -94,6 +95,7 @@ github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1o github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= +github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/adigunhammedolalekan/registry-auth v0.0.0-20200730122110-8cde180a3a60 h1:1IG6ye8dellBRE2uqvG0EzQScRqjsH/n5xOw+n0OGec= github.com/adigunhammedolalekan/registry-auth v0.0.0-20200730122110-8cde180a3a60/go.mod h1:DcXj4IQOoib2b4G2b8JU3VGV3ljXYbIq+PH4CcoAQTI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -103,6 +105,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/alexflint/go-filemutex v0.0.0-20171028004239-d358565f3c3f h1:tbgFqBK8r77y+mT2RKkQ8ukhk/uvPtPZvr3a3166YNw= github.com/alexflint/go-filemutex v0.0.0-20171028004239-d358565f3c3f/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apex/log v1.4.0/go.mod h1:UMNC4vQNC7hb5gyr47r18ylK1n34rV7GO+gb0wpXvcE= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= @@ -115,6 +118,7 @@ github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hC github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -342,6 +346,7 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -354,6 +359,7 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -366,6 +372,12 @@ github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= +github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -544,6 +556,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56 h1:742eGXur0715JMq73aD95/FU0XpVKXqNuTnEfXsLOYQ= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -560,6 +574,7 @@ github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -588,6 +603,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= +github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -595,6 +611,7 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/manifoldco/promptui v0.8.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -660,7 +677,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -789,8 +805,6 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 h1:2c1EFnZHIPCW8qKWgHMH/fX2PkSabFc5mrVzfUNdg5U= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/satori/go.uuid v1.2.1-0.20180404165556-75cca531ea76 h1:ofyVTM1w4iyKwaQIlRR6Ip06mXXx5Cnz7a4mTGYq1hE= -github.com/satori/go.uuid v1.2.1-0.20180404165556-75cca531ea76/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= @@ -798,8 +812,9 @@ github.com/seccomp/containers-golang v0.6.0 h1:VWPMMIDr8pAtNjCX0WvLEEK9EQi5lAm4H github.com/seccomp/containers-golang v0.6.0/go.mod h1:Dd9mONHvW4YdbSzdm23yf2CFw0iqvqLhO0mEFvPIvm4= github.com/seccomp/libseccomp-golang v0.9.1 h1:NJjM5DNFOs0s3kYE1WUOr6G8V97sdt46rlXTMfXGWBo= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -864,8 +879,8 @@ github.com/sylabs/scs-key-client v0.6.2 h1:PTdmkE50rFMrSty9usl8wnzaW+JPK0rjXF4cd github.com/sylabs/scs-key-client v0.6.2/go.mod h1:WoTk47nNRrk0uLfblxUFekxDIIVPY4l2/9evME5fgX0= github.com/sylabs/scs-library-client v1.0.5 h1:u8ueCIQaa38/B0axtKUzdWSQ6uLXL9HvLQGNkB69Oe0= github.com/sylabs/scs-library-client v1.0.5/go.mod h1:ufza0uq4ohT7Fk2Y7vm+uRyMm7gRA+Fde1nxyhGfhTk= -github.com/sylabs/sif v1.6.0 h1:bRcCFnv/DIb4T6rAcaM8NMwBwhnSXfPQn1WIjjbBR7w= -github.com/sylabs/sif v1.6.0/go.mod h1:Q8/gZOgWefU7QNolbLwnUH9sRH4dCaki4kBKepITp04= +github.com/sylabs/sif/v2 v2.0.0-beta.5 h1:7xbKHHRga3//1VT41VRgGnmo27dnKiPME/q10eMHWm8= +github.com/sylabs/sif/v2 v2.0.0-beta.5/go.mod h1:ZfYOz8pvVJv6SrOzvfp6aCaJrE9y4BTwLvf1v0MLPGI= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= @@ -911,6 +926,7 @@ github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3C github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= @@ -1043,6 +1059,7 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210510120150-4163338589ed h1:p9UgmWI9wKpfYmgaV/IZKGdXc5qEK45tDwwwDyjS26I= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1153,6 +1170,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1357,8 +1375,9 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1373,6 +1392,7 @@ gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/app/singularity/overlay_create.go b/internal/app/singularity/overlay_create.go index 6134a20be8..b9cf45d846 100644 --- a/internal/app/singularity/overlay_create.go +++ b/internal/app/singularity/overlay_create.go @@ -1,3 +1,8 @@ +// Copyright (c) 2021, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software.package singularity + package singularity import ( @@ -10,7 +15,7 @@ import ( "runtime" "strings" - "github.com/sylabs/sif/pkg/sif" + "github.com/sylabs/sif/v2/pkg/sif" "github.com/sylabs/singularity/pkg/image" "golang.org/x/sys/unix" ) @@ -20,26 +25,48 @@ const ( ddBinary = "dd" ) -func sifInfo(img *os.File) (string, bool, error) { - fimg, err := sif.LoadContainerFp(img, true) +// isSigned returns true if the SIF in rw contains one or more signature objects. +func isSigned(rw sif.ReadWriter) (bool, error) { + f, err := sif.LoadContainer(rw, + sif.OptLoadWithFlag(os.O_RDONLY), + sif.OptLoadWithCloseOnUnload(false), + ) + if err != nil { + return false, err + } + defer f.UnloadContainer() + + sigs, err := f.GetDescriptors(sif.WithDataType(sif.DataSignature)) + return len(sigs) > 0, err +} + +// addOverlayToImage adds the EXT3 overlay at overlayPath to the SIF image at imagePath. +func addOverlayToImage(imagePath, overlayPath string) error { + f, err := sif.LoadContainerFromPath(imagePath) if err != nil { - return "", false, err + return err } + defer f.UnloadContainer() - arch := string(fimg.Header.Arch[:sif.HdrArchLen-1]) - if arch == sif.HdrArchUnknown { - arch = sif.GetSIFArch(runtime.GOARCH) + tf, err := os.Open(overlayPath) + if err != nil { + return err } + defer tf.Close() - signed := false - for _, desc := range fimg.DescrArr { - if desc.Datatype == sif.DataSignature && desc.Link == sif.DescrDefaultGroup { - signed = true - break - } + arch := f.PrimaryArch() + if arch == "unknown" { + arch = runtime.GOARCH } - return arch, signed, fimg.UnloadContainer() + di, err := sif.NewDescriptorInput(sif.DataPartition, tf, + sif.OptPartitionMetadata(sif.FsExt3, sif.PartOverlay, arch), + ) + if err != nil { + return err + } + + return f.AddObject(di) } func OverlayCreate(size int, imgPath string, overlayDirs ...string) error { @@ -69,7 +96,6 @@ func OverlayCreate(size int, imgPath string, overlayDirs ...string) error { } sifImage := false - sifArch := "" if err := unix.Access(imgPath, unix.W_OK); err == nil { img, err := image.Init(imgPath, false) @@ -89,13 +115,12 @@ func OverlayCreate(size int, imgPath string, overlayDirs ...string) error { if err != nil { return fmt.Errorf("while getting SIF overlay partitions: %s", err) } - arch, signed, err := sifInfo(img.File) + signed, err := isSigned(img.File) if err != nil { return fmt.Errorf("while getting SIF info: %s", err) } else if signed { return fmt.Errorf("SIF image %s is signed: could not add writable overlay", imgPath) } - sifArch = arch img.File.Close() @@ -178,22 +203,8 @@ func OverlayCreate(size int, imgPath string, overlayDirs ...string) error { errBuf.Reset() if sifImage { - self, err := os.Executable() - if err != nil { - return fmt.Errorf("while determining current executable path: %s", err) - } - - args := []string{ - "sif", "add", - "--datatype", "4", "--partfs", "2", - "--parttype", "4", "--partarch", sifArch, - "--groupid", "1", - imgPath, tmpFile, - } - cmd = exec.Command(self, args...) - cmd.Stderr = errBuf - if err := cmd.Run(); err != nil { - return fmt.Errorf("while adding ext3 overlay partition to %s: %s\nCommand error: %s", imgPath, err, errBuf) + if err := addOverlayToImage(imgPath, tmpFile); err != nil { + return fmt.Errorf("while adding ext3 overlay partition to %s: %w", imgPath, err) } } else { if err := os.Rename(tmpFile, imgPath); err != nil { diff --git a/internal/app/singularity/plugin_compile_linux.go b/internal/app/singularity/plugin_compile_linux.go index 82471011e0..f0ecf6f81d 100644 --- a/internal/app/singularity/plugin_compile_linux.go +++ b/internal/app/singularity/plugin_compile_linux.go @@ -10,7 +10,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "io/ioutil" "os" "os/exec" @@ -19,8 +18,7 @@ import ( "runtime/debug" "strings" - uuid "github.com/satori/go.uuid" - "github.com/sylabs/sif/pkg/sif" + "github.com/sylabs/sif/v2/pkg/sif" "github.com/sylabs/singularity/internal/pkg/buildcfg" "github.com/sylabs/singularity/internal/pkg/plugin" pluginapi "github.com/sylabs/singularity/pkg/plugin" @@ -275,49 +273,45 @@ func generateManifest(sourceDir string, bTool buildToolchain) error { // makeSIF takes in two arguments: sourceDir, the path to the plugin source directory; // and sifPath, the path to the final .sif file which is ready to be used. func makeSIF(sourceDir, sifPath string) error { - id, err := uuid.NewV4() - if err != nil { - return err - } + objPath := pluginObjPath(sourceDir) - plCreateInfo := sif.CreateInfo{ - Pathname: sifPath, - Launchstr: sif.HdrLaunch, - Sifversion: sif.HdrVersion, - ID: id, + fp, err := os.Open(objPath) + if err != nil { + return fmt.Errorf("while opening plugin object file %v: %w", objPath, err) } + defer fp.Close() - // create plugin object file descriptor - plObjInput, err := getPluginObjDescr(pluginObjPath(sourceDir)) + plObjInput, err := sif.NewDescriptorInput(sif.DataPartition, fp, + sif.OptObjectName("plugin.so"), + sif.OptPartitionMetadata(sif.FsRaw, sif.PartData, runtime.GOARCH), + ) if err != nil { return err } - if fp, ok := plObjInput.Fp.(io.Closer); ok { - defer fp.Close() - } + // create plugin manifest descriptor + manifestPath := pluginManifestPath(sourceDir) - // add plugin object file descriptor to sif - plCreateInfo.InputDescr = append(plCreateInfo.InputDescr, plObjInput) + fp, err = os.Open(manifestPath) + if err != nil { + return fmt.Errorf("while opening plugin manifest file %v: %w", manifestPath, err) + } + defer fp.Close() - // create plugin manifest descriptor - plManifestInput, err := getPluginManifestDescr(pluginManifestPath(sourceDir)) + plManifestInput, err := sif.NewDescriptorInput(sif.DataGenericJSON, fp, + sif.OptObjectName("plugin.manifest"), + ) if err != nil { return err } - if fp, ok := plManifestInput.Fp.(io.Closer); ok { - defer fp.Close() - } - - // add plugin manifest descriptor to sif - plCreateInfo.InputDescr = append(plCreateInfo.InputDescr, plManifestInput) os.RemoveAll(sifPath) - // create sif file - f, err := sif.CreateContainer(plCreateInfo) + f, err := sif.CreateContainerAtPath(sifPath, + sif.OptCreateWithDescriptors(plObjInput, plManifestInput), + ) if err != nil { - return fmt.Errorf("while creating sif file: %s", err) + return fmt.Errorf("while creating sif file: %w", err) } if err := f.UnloadContainer(); err != nil { @@ -326,74 +320,3 @@ func makeSIF(sourceDir, sifPath string) error { return nil } - -// getPluginObjDescr returns a sif.DescriptorInput which contains the raw -// data of the .so file. -// -// Datatype: sif.DataPartition -// Fstype: sif.FsRaw -// Parttype: sif.PartData -func getPluginObjDescr(objPath string) (sif.DescriptorInput, error) { - var err error - - objInput := sif.DescriptorInput{ - Datatype: sif.DataPartition, - Groupid: sif.DescrDefaultGroup, - Link: sif.DescrUnusedLink, - Fname: objPath, - } - - // open plugin object file - fp, err := os.Open(objInput.Fname) - if err != nil { - return sif.DescriptorInput{}, fmt.Errorf("while opening plugin object file %s: %s", objInput.Fname, err) - } - - // stat file to obtain size - fstat, err := fp.Stat() - if err != nil { - return sif.DescriptorInput{}, fmt.Errorf("while calling stat on plugin object file %s: %s", objInput.Fname, err) - } - - objInput.Fp = fp - objInput.Size = fstat.Size() - - // populate objInput.Extra with appropriate Fstype & Parttype - err = objInput.SetPartExtra(sif.FsRaw, sif.PartData, sif.GetSIFArch(runtime.GOARCH)) - if err != nil { - return sif.DescriptorInput{}, err - } - - return objInput, nil -} - -// getPluginManifestDescr returns a sif.DescriptorInput which contains the manifest -// in JSON form. Grabbing the Manifest is done by loading the .so using the plugin -// package, which is performed inside the container during buildPlugin() function -// -// Datatype: sif.DataGenericJSON -func getPluginManifestDescr(manifestPath string) (sif.DescriptorInput, error) { - manifestInput := sif.DescriptorInput{ - Datatype: sif.DataGenericJSON, - Groupid: sif.DescrDefaultGroup, - Link: sif.DescrUnusedLink, - Fname: manifestPath, - } - - // open plugin object file - fp, err := os.Open(manifestInput.Fname) - if err != nil { - return sif.DescriptorInput{}, fmt.Errorf("while opening plugin object file %s: %s", manifestInput.Fname, err) - } - - // stat file to obtain size - fstat, err := fp.Stat() - if err != nil { - return sif.DescriptorInput{}, fmt.Errorf("while calling stat on plugin object file %s: %s", manifestInput.Fname, err) - } - - manifestInput.Fp = fp - manifestInput.Size = fstat.Size() - - return manifestInput, nil -} diff --git a/internal/app/singularity/push.go b/internal/app/singularity/push.go index b31a74c39b..2b2195072e 100644 --- a/internal/app/singularity/push.go +++ b/internal/app/singularity/push.go @@ -1,5 +1,5 @@ // Copyright (c) 2020, Control Command Inc. All rights reserved. -// Copyright (c) 2019-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -16,7 +16,7 @@ import ( keyclient "github.com/sylabs/scs-key-client/client" "github.com/sylabs/scs-library-client/client" - "github.com/sylabs/sif/pkg/sif" + "github.com/sylabs/sif/v2/pkg/sif" "github.com/sylabs/singularity/internal/pkg/util/fs" "github.com/sylabs/singularity/pkg/sylog" "github.com/vbauerster/mpb/v4" @@ -138,11 +138,13 @@ func LibraryPush(ctx context.Context, pushSpec LibraryPushSpec, libraryConfig *c } func sifArch(filename string) (string, error) { - fimg, err := sif.LoadContainer(filename, true) + f, err := sif.LoadContainerFromPath(filename, sif.OptLoadWithFlag(os.O_RDONLY)) if err != nil { - return "", fmt.Errorf("unable to open: %v: %v", filename, err) + return "", fmt.Errorf("unable to open: %v: %w", filename, err) } - arch := sif.GetGoArch(string(fimg.Header.Arch[:sif.HdrArchLen-1])) + defer f.UnloadContainer() + + arch := f.PrimaryArch() if arch == "unknown" { return arch, fmt.Errorf("unknown architecture in SIF file") } diff --git a/internal/app/singularity/sign.go b/internal/app/singularity/sign.go index e53c403adf..ac22d4cf84 100644 --- a/internal/app/singularity/sign.go +++ b/internal/app/singularity/sign.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2020-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the LICENSE.md file // distributed with the sources of this project regarding your rights to use or distribute this // software. @@ -6,8 +6,8 @@ package singularity import ( - "github.com/sylabs/sif/pkg/integrity" - "github.com/sylabs/sif/pkg/sif" + "github.com/sylabs/sif/v2/pkg/integrity" + "github.com/sylabs/sif/v2/pkg/sif" "github.com/sylabs/singularity/pkg/sypgp" ) @@ -67,14 +67,14 @@ func Sign(path string, opts ...SignOpt) error { } // Load container. - f, err := sif.LoadContainer(path, false) + f, err := sif.LoadContainerFromPath(path) if err != nil { return err } defer f.UnloadContainer() // Apply signature(s). - is, err := integrity.NewSigner(&f, s.opts...) + is, err := integrity.NewSigner(f, s.opts...) if err != nil { return err } diff --git a/internal/app/singularity/sign_test.go b/internal/app/singularity/sign_test.go index 85107b97ff..2be54ea70d 100644 --- a/internal/app/singularity/sign_test.go +++ b/internal/app/singularity/sign_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2020-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the LICENSE.md file // distributed with the sources of this project regarding your rights to use or distribute this // software. @@ -14,7 +14,7 @@ import ( "path/filepath" "testing" - "github.com/sylabs/sif/pkg/integrity" + "github.com/sylabs/sif/v2/pkg/integrity" "github.com/sylabs/singularity/pkg/sypgp" "golang.org/x/crypto/openpgp" ) diff --git a/internal/app/singularity/verify.go b/internal/app/singularity/verify.go index 12a7f82aeb..96d9870e05 100644 --- a/internal/app/singularity/verify.go +++ b/internal/app/singularity/verify.go @@ -1,5 +1,5 @@ // Copyright (c) 2020, Control Command Inc. All rights reserved. -// Copyright (c) 2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2020-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the LICENSE.md file // distributed with the sources of this project regarding your rights to use or distribute this // software. @@ -10,11 +10,12 @@ import ( "context" "encoding/hex" "errors" + "os" "strings" "github.com/sylabs/scs-key-client/client" - "github.com/sylabs/sif/pkg/integrity" - "github.com/sylabs/sif/pkg/sif" + "github.com/sylabs/sif/v2/pkg/integrity" + "github.com/sylabs/sif/v2/pkg/sif" "github.com/sylabs/singularity/internal/pkg/buildcfg" "github.com/sylabs/singularity/pkg/sypgp" "golang.org/x/crypto/openpgp" @@ -150,11 +151,11 @@ func (v verifier) getOpts(ctx context.Context, f *sif.FileImage) ([]integrity.Ve // If no objects explicitly selected, select system partition. if len(v.groupIDs) == 0 && len(v.objectIDs) == 0 { - od, _, err := f.GetPartPrimSys() + od, err := f.GetDescriptor(sif.WithPartitionType(sif.PartPrimSys)) if err != nil { return nil, err } - iopts = append(iopts, integrity.OptVerifyObject(od.ID)) + iopts = append(iopts, integrity.OptVerifyObject(od.ID())) } } } @@ -184,20 +185,20 @@ func Verify(ctx context.Context, path string, opts ...VerifyOpt) error { } // Load container. - f, err := sif.LoadContainer(path, true) + f, err := sif.LoadContainerFromPath(path, sif.OptLoadWithFlag(os.O_RDONLY)) if err != nil { return err } defer f.UnloadContainer() // Get options to validate f. - vopts, err := v.getOpts(ctx, &f) + vopts, err := v.getOpts(ctx, f) if err != nil { return err } // Verify signature(s). - iv, err := integrity.NewVerifier(&f, vopts...) + iv, err := integrity.NewVerifier(f, vopts...) if err != nil { return err } @@ -218,20 +219,20 @@ func VerifyFingerprints(ctx context.Context, path string, fingerprints []string, } // Load container. - f, err := sif.LoadContainer(path, true) + f, err := sif.LoadContainerFromPath(path, sif.OptLoadWithFlag(os.O_RDONLY)) if err != nil { return err } defer f.UnloadContainer() // Get options to validate f. - vopts, err := v.getOpts(ctx, &f) + vopts, err := v.getOpts(ctx, f) if err != nil { return err } // Verify signature(s). - iv, err := integrity.NewVerifier(&f, vopts...) + iv, err := integrity.NewVerifier(f, vopts...) if err != nil { return err } diff --git a/internal/app/singularity/verify_test.go b/internal/app/singularity/verify_test.go index 2164d6d7b9..0a9ec65e99 100644 --- a/internal/app/singularity/verify_test.go +++ b/internal/app/singularity/verify_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2020-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the LICENSE.md file // distributed with the sources of this project regarding your rights to use or distribute this // software. @@ -16,8 +16,8 @@ import ( "testing" "github.com/sylabs/scs-key-client/client" - "github.com/sylabs/sif/pkg/integrity" - "github.com/sylabs/sif/pkg/sif" + "github.com/sylabs/sif/v2/pkg/integrity" + "github.com/sylabs/sif/v2/pkg/sif" "golang.org/x/crypto/openpgp" "golang.org/x/crypto/openpgp/armor" ) @@ -124,13 +124,17 @@ func Test_newVerifier(t *testing.T) { } func Test_verifier_getOpts(t *testing.T) { - emptyImage, err := sif.LoadContainer(filepath.Join("testdata", "images", "empty.sif"), true) + emptyImage, err := sif.LoadContainerFromPath(filepath.Join("testdata", "images", "empty.sif"), + sif.OptLoadWithFlag(os.O_RDONLY), + ) if err != nil { t.Fatal(err) } defer emptyImage.UnloadContainer() - oneGroupImage, err := sif.LoadContainer(filepath.Join("testdata", "images", "one-group.sif"), true) + oneGroupImage, err := sif.LoadContainerFromPath(filepath.Join("testdata", "images", "one-group.sif"), + sif.OptLoadWithFlag(os.O_RDONLY), + ) if err != nil { t.Fatal(err) } @@ -147,7 +151,7 @@ func Test_verifier_getOpts(t *testing.T) { }{ { name: "TLSRequired", - f: &emptyImage, + f: emptyImage, v: verifier{ opts: []client.Option{ client.OptBaseURL("hkp://pool.sks-keyservers.net"), @@ -157,14 +161,14 @@ func Test_verifier_getOpts(t *testing.T) { wantErr: client.ErrTLSRequired, }, { - name: "NotFound", - f: &emptyImage, + name: "NoObjects", + f: emptyImage, v: verifier{legacy: true}, - wantErr: sif.ErrNotFound, + wantErr: sif.ErrNoObjects, }, { name: "Defaults", - f: &oneGroupImage, + f: oneGroupImage, wantOpts: 1, }, { @@ -174,55 +178,55 @@ func Test_verifier_getOpts(t *testing.T) { client.OptBearerToken("token"), }, }, - f: &oneGroupImage, + f: oneGroupImage, wantOpts: 1, }, { name: "Group1", v: verifier{groupIDs: []uint32{1}}, - f: &oneGroupImage, + f: oneGroupImage, wantOpts: 2, }, { name: "Object1", v: verifier{objectIDs: []uint32{1}}, - f: &oneGroupImage, + f: oneGroupImage, wantOpts: 2, }, { name: "All", v: verifier{all: true}, - f: &oneGroupImage, + f: oneGroupImage, wantOpts: 1, }, { name: "Legacy", v: verifier{legacy: true}, - f: &oneGroupImage, + f: oneGroupImage, wantOpts: 3, }, { name: "LegacyGroup1", v: verifier{legacy: true, groupIDs: []uint32{1}}, - f: &oneGroupImage, + f: oneGroupImage, wantOpts: 3, }, { name: "LegacyObject1", v: verifier{legacy: true, objectIDs: []uint32{1}}, - f: &oneGroupImage, + f: oneGroupImage, wantOpts: 3, }, { name: "LegacyAll", v: verifier{legacy: true, all: true}, - f: &oneGroupImage, + f: oneGroupImage, wantOpts: 2, }, { name: "Callcack", v: verifier{cb: cb}, - f: &oneGroupImage, + f: oneGroupImage, wantOpts: 2, }, } @@ -344,14 +348,24 @@ func TestVerify(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { + i := 0 + cb := func(f *sif.FileImage, r integrity.VerifyResult) bool { - if len(tt.wantVerified) == 0 { + defer func() { i++ }() + + if i >= len(tt.wantVerified) { t.Fatalf("wantVerified consumed") } - if got, want := r.Verified(), tt.wantVerified[0]; !reflect.DeepEqual(got, want) { - t.Errorf("got verified %v, want %v", got, want) + + if got, want := r.Verified(), tt.wantVerified[i]; len(got) != len(want) { + t.Errorf("got %v verified, want %v", got, want) + } else { + for i, od := range got { + if got, want := od.ID(), want[i]; got != want { + t.Errorf("got ID %v, want %v", got, want) + } + } } - tt.wantVerified = tt.wantVerified[1:] if got, want := r.Entity().PrimaryKey, tt.wantEntity.PrimaryKey; !reflect.DeepEqual(got, want) { t.Errorf("got entity public key %+v, want %+v", got, want) @@ -506,14 +520,24 @@ func TestVerifyFingerPrint(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { + i := 0 + cb := func(f *sif.FileImage, r integrity.VerifyResult) bool { - if len(tt.wantVerified) == 0 { + defer func() { i++ }() + + if i >= len(tt.wantVerified) { t.Fatalf("wantVerified consumed") } - if got, want := r.Verified(), tt.wantVerified[0]; !reflect.DeepEqual(got, want) { - t.Errorf("got verified %v, want %v", got, want) + + if got, want := r.Verified(), tt.wantVerified[i]; len(got) != len(want) { + t.Errorf("got %v verified, want %v", got, want) + } else { + for i, od := range got { + if got, want := od.ID(), want[i]; got != want { + t.Errorf("got ID %v, want %v", got, want) + } + } } - tt.wantVerified = tt.wantVerified[1:] if got, want := r.Entity().PrimaryKey, tt.wantEntity.PrimaryKey; !reflect.DeepEqual(got, want) { t.Errorf("got entity public key %+v, want %+v", got, want) diff --git a/internal/pkg/build/assemblers/sif.go b/internal/pkg/build/assemblers/sif.go index 4718f50506..7212492fc9 100644 --- a/internal/pkg/build/assemblers/sif.go +++ b/internal/pkg/build/assemblers/sif.go @@ -6,7 +6,7 @@ package assemblers import ( - "encoding/binary" + "bytes" "fmt" "io/ioutil" "os" @@ -16,8 +16,7 @@ import ( "strconv" "syscall" - uuid "github.com/satori/go.uuid" - "github.com/sylabs/sif/pkg/sif" + "github.com/sylabs/sif/v2/pkg/sif" "github.com/sylabs/singularity/internal/pkg/util/crypt" "github.com/sylabs/singularity/internal/pkg/util/machine" "github.com/sylabs/singularity/pkg/build/types" @@ -40,32 +39,16 @@ type encryptionOptions struct { } func createSIF(path string, b *types.Bundle, squashfile string, encOpts *encryptionOptions, arch string) (err error) { - definition := b.Recipe.Raw + var dis []sif.DescriptorInput - id, err := uuid.NewV4() + // data we need to create a definition file descriptor + definput, err := sif.NewDescriptorInput(sif.DataDeffile, bytes.NewReader(b.Recipe.Raw)) if err != nil { return err } - // general info for the new SIF file creation - cinfo := sif.CreateInfo{ - Pathname: path, - Launchstr: sif.HdrLaunch, - Sifversion: sif.HdrVersion, - ID: id, - } - - // data we need to create a definition file descriptor - definput := sif.DescriptorInput{ - Datatype: sif.DataDeffile, - Groupid: sif.DescrDefaultGroup, - Link: sif.DescrUnusedLink, - Data: definition, - } - definput.Size = int64(binary.Size(definput.Data)) - // add this descriptor input element to creation descriptor slice - cinfo.InputDescr = append(cinfo.InputDescr, definput) + dis = append(dis, definput) // add all JSON data object within SIF by alphabetical order sorted := make([]string, 0, len(b.JSONObjects)) @@ -77,56 +60,40 @@ func createSIF(path string, b *types.Bundle, squashfile string, encOpts *encrypt for _, name := range sorted { if len(b.JSONObjects[name]) > 0 { // data we need to create a definition file descriptor - in := sif.DescriptorInput{ - Datatype: sif.DataGenericJSON, - Groupid: sif.DescrDefaultGroup, - Link: sif.DescrUnusedLink, - Data: b.JSONObjects[name], - Fname: name, + in, err := sif.NewDescriptorInput(sif.DataGenericJSON, bytes.NewReader(b.JSONObjects[name]), + sif.OptObjectName(name), + ) + if err != nil { + return err } - in.Size = int64(binary.Size(in.Data)) // add this descriptor input element to creation descriptor slice - cinfo.InputDescr = append(cinfo.InputDescr, in) + dis = append(dis, in) } } - // data we need to create a system partition descriptor - parinput := sif.DescriptorInput{ - Datatype: sif.DataPartition, - Groupid: sif.DescrDefaultGroup, - Link: sif.DescrUnusedLink, - Fname: squashfile, - } // open up the data object file for this descriptor - fp, err := os.Open(parinput.Fname) + fp, err := os.Open(squashfile) if err != nil { return fmt.Errorf("while opening partition file: %s", err) } - defer fp.Close() - fi, err := fp.Stat() - if err != nil { - return fmt.Errorf("while calling stat on partition file: %s", err) - } - - parinput.Fp = fp - parinput.Size = fi.Size() - - sifType := sif.FsSquash - + fs := sif.FsSquash if encOpts != nil { - sifType = sif.FsEncryptedSquashfs + fs = sif.FsEncryptedSquashfs } - err = parinput.SetPartExtra(sifType, sif.PartPrimSys, sif.GetSIFArch(arch)) + // data we need to create a system partition descriptor + parinput, err := sif.NewDescriptorInput(sif.DataPartition, fp, + sif.OptPartitionMetadata(fs, sif.PartPrimSys, arch), + ) if err != nil { - return + return err } // add this descriptor input element to the list - cinfo.InputDescr = append(cinfo.InputDescr, parinput) + dis = append(dis, parinput) if encOpts != nil { data, err := cryptkey.EncryptKey(encOpts.keyInfo, encOpts.plaintext) @@ -135,22 +102,16 @@ func createSIF(path string, b *types.Bundle, squashfile string, encOpts *encrypt } if data != nil { - syspartID := uint32(len(cinfo.InputDescr)) - part := sif.DescriptorInput{ - Datatype: sif.DataCryptoMessage, - Groupid: sif.DescrDefaultGroup, - Link: syspartID, - Data: data, - Size: int64(len(data)), - } - - // extra data needed for the creation of a signature descriptor - err := part.SetCryptoMsgExtra(sif.FormatPEM, sif.MessageRSAOAEP) + syspartID := uint32(len(dis)) + part, err := sif.NewDescriptorInput(sif.DataCryptoMessage, bytes.NewReader(data), + sif.OptLinkedID(syspartID), + sif.OptCryptoMessageMetadata(sif.FormatPEM, sif.MessageRSAOAEP), + ) if err != nil { return err } - cinfo.InputDescr = append(cinfo.InputDescr, part) + dis = append(dis, part) } } @@ -158,9 +119,9 @@ func createSIF(path string, b *types.Bundle, squashfile string, encOpts *encrypt os.RemoveAll(path) // test container creation with two partition input descriptors - f, err := sif.CreateContainer(cinfo) + f, err := sif.CreateContainerAtPath(path, sif.OptCreateWithDescriptors(dis...)) if err != nil { - return fmt.Errorf("while creating container: %s", err) + return fmt.Errorf("while creating container: %w", err) } if err := f.UnloadContainer(); err != nil { diff --git a/internal/pkg/plugin/sif.go b/internal/pkg/plugin/sif.go index c1b60bb3a0..5988b7f787 100644 --- a/internal/pkg/plugin/sif.go +++ b/internal/pkg/plugin/sif.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -10,7 +10,7 @@ import ( "fmt" "io" - "github.com/sylabs/sif/pkg/sif" + "github.com/sylabs/sif/v2/pkg/sif" "github.com/sylabs/singularity/pkg/image" pluginapi "github.com/sylabs/singularity/pkg/plugin" ) @@ -29,9 +29,9 @@ const ( // format: // // DESCR[0]: Sifplugin -// - Datatype: sif.DataPartition -// - Fstype: sif.FsRaw -// - Parttype: sif.PartData +// - DataType: sif.DataPartition +// - FSType: sif.FsRaw +// - PartType: sif.PartData // DESCR[1]: Sifmanifest // - Datatype: sif.DataGenericJSON func isPluginFile(img *image.Image) bool { diff --git a/internal/pkg/syecl/syecl.go b/internal/pkg/syecl/syecl.go index 578929fdc5..01bbb2e7c3 100644 --- a/internal/pkg/syecl/syecl.go +++ b/internal/pkg/syecl/syecl.go @@ -1,5 +1,5 @@ // Copyright (c) 2020, Control Command Inc. All rights reserved. -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -20,8 +20,8 @@ import ( "strings" toml "github.com/pelletier/go-toml" - "github.com/sylabs/sif/pkg/integrity" - "github.com/sylabs/sif/pkg/sif" + "github.com/sylabs/sif/v2/pkg/integrity" + "github.com/sylabs/sif/v2/pkg/sif" "golang.org/x/crypto/openpgp" ) @@ -210,22 +210,26 @@ func shouldRun(ecl *EclConfig, fp *os.File, kr openpgp.KeyRing) (ok bool, err er return false, fmt.Errorf("%s not part of any execgroup", fp.Name()) } - f, err := sif.LoadContainerFp(fp, true) + f, err := sif.LoadContainer(fp, + sif.OptLoadWithFlag(os.O_RDONLY), + sif.OptLoadWithCloseOnUnload(false), + ) if err != nil { return false, err } + defer f.UnloadContainer() opts := []integrity.VerifierOpt{integrity.OptVerifyWithKeyRing(kr)} if ecl.Legacy { // Legacy behavior is to verify the primary partition only. - od, _, err := f.GetPartPrimSys() + od, err := f.GetDescriptor(sif.WithPartitionType(sif.PartPrimSys)) if err != nil { return false, fmt.Errorf("get primary system partition: %v", err) } - opts = append(opts, integrity.OptVerifyLegacy(), integrity.OptVerifyObject(od.ID)) + opts = append(opts, integrity.OptVerifyLegacy(), integrity.OptVerifyObject(od.ID())) } - v, err := integrity.NewVerifier(&f, opts...) + v, err := integrity.NewVerifier(f, opts...) if err != nil { return false, err } diff --git a/internal/pkg/util/machine/machine.go b/internal/pkg/util/machine/machine.go index 5dced41ee2..a06eb0d21e 100644 --- a/internal/pkg/util/machine/machine.go +++ b/internal/pkg/util/machine/machine.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -19,7 +19,6 @@ import ( "runtime" "strings" - "github.com/sylabs/sif/pkg/sif" "github.com/sylabs/singularity/internal/pkg/util/fs" "github.com/sylabs/singularity/pkg/sylog" ) @@ -29,7 +28,6 @@ var ErrUnknownArch = errors.New("architecture not recognized") type format struct { Arch string - Sif string Compatible string Machine elf.Machine Class elf.Class @@ -40,7 +38,6 @@ type format struct { var formats = []format{ { Arch: "386", - Sif: sif.HdrArch386, Compatible: "amd64", Machine: elf.EM_386, Class: elf.ELFCLASS32, @@ -49,7 +46,6 @@ var formats = []format{ }, { Arch: "386", - Sif: sif.HdrArch386, Compatible: "amd64", Machine: elf.EM_486, Class: elf.ELFCLASS32, @@ -58,7 +54,6 @@ var formats = []format{ }, { Arch: "amd64", - Sif: sif.HdrArchAMD64, Machine: elf.EM_X86_64, Class: elf.ELFCLASS64, Endianness: binary.LittleEndian, @@ -66,7 +61,6 @@ var formats = []format{ }, { Arch: "arm", - Sif: sif.HdrArchARM, Compatible: "arm64", Machine: elf.EM_ARM, Class: elf.ELFCLASS32, @@ -75,7 +69,6 @@ var formats = []format{ }, { Arch: "armbe", - Sif: sif.HdrArchARM, // FIXME: add HdrArchARMbe to sif package Compatible: "arm64be", Machine: elf.EM_ARM, Class: elf.ELFCLASS32, @@ -84,7 +77,6 @@ var formats = []format{ }, { Arch: "arm64", - Sif: sif.HdrArchARM64, Machine: elf.EM_AARCH64, Class: elf.ELFCLASS64, Endianness: binary.LittleEndian, @@ -92,7 +84,6 @@ var formats = []format{ }, { Arch: "arm64be", - Sif: sif.HdrArchARM64, // FIXME: add HdrArchARM64be to sif package Machine: elf.EM_AARCH64, Class: elf.ELFCLASS64, Endianness: binary.BigEndian, @@ -100,7 +91,6 @@ var formats = []format{ }, { Arch: "s390x", - Sif: sif.HdrArchS390x, Machine: elf.EM_S390, Class: elf.ELFCLASS64, Endianness: binary.BigEndian, @@ -108,7 +98,6 @@ var formats = []format{ }, { Arch: "ppc64", - Sif: sif.HdrArchPPC64, Machine: elf.EM_PPC64, Class: elf.ELFCLASS32, Endianness: binary.BigEndian, @@ -116,7 +105,6 @@ var formats = []format{ }, { Arch: "ppc64le", - Sif: sif.HdrArchPPC64le, Machine: elf.EM_PPC64, Class: elf.ELFCLASS64, Endianness: binary.LittleEndian, @@ -124,7 +112,6 @@ var formats = []format{ }, { Arch: "mips", - Sif: sif.HdrArchMIPS, Compatible: "mips64", Machine: elf.EM_MIPS, Class: elf.ELFCLASS32, @@ -133,7 +120,6 @@ var formats = []format{ }, { Arch: "mipsle", - Sif: sif.HdrArchMIPSle, Compatible: "mips64le", Machine: elf.EM_MIPS, Class: elf.ELFCLASS32, @@ -142,7 +128,6 @@ var formats = []format{ }, { Arch: "mips64", - Sif: sif.HdrArchMIPS64, Machine: elf.EM_MIPS, Class: elf.ELFCLASS64, Endianness: binary.BigEndian, @@ -150,7 +135,6 @@ var formats = []format{ }, { Arch: "mips64le", - Sif: sif.HdrArchMIPS64le, Machine: elf.EM_MIPS, Class: elf.ELFCLASS64, Endianness: binary.LittleEndian, diff --git a/pkg/image/sif.go b/pkg/image/sif.go index 67c9172a44..281eda88a4 100644 --- a/pkg/image/sif.go +++ b/pkg/image/sif.go @@ -11,7 +11,7 @@ import ( "os" "runtime" - "github.com/sylabs/sif/pkg/sif" + "github.com/sylabs/sif/v2/pkg/sif" "github.com/sylabs/singularity/internal/pkg/util/machine" ) @@ -24,7 +24,7 @@ const ( type sifFormat struct{} -func checkPartitionType(img *Image, fstype sif.Fstype, offset int64) (uint32, error) { +func checkPartitionType(img *Image, fstype sif.FSType, offset int64) (uint32, error) { header := make([]byte, bufferSize) if _, err := img.File.ReadAt(header, offset); err != nil { @@ -59,42 +59,36 @@ func (f *sifFormat) initializer(img *Image, fi os.FileInfo) error { if n, err := img.File.Read(b); err != nil || n != bufferSize { return debugErrorf("can't read first %d bytes: %v", bufferSize, err) } - if !bytes.Contains(b, []byte(sif.HdrMagic)) { + if !bytes.Contains(b, []byte("SIF_MAGIC")) { return debugError("SIF magic not found") } + flag := os.O_RDONLY + if img.Writable { + flag = os.O_RDWR + } + // Load the SIF file - fimg, err := sif.LoadContainerFp(img.File, !img.Writable) + fimg, err := sif.LoadContainer(img.File, + sif.OptLoadWithFlag(flag), + sif.OptLoadWithCloseOnUnload(false), + ) if err != nil { return err } + defer fimg.UnloadContainer() - groupID := -1 + var groupID uint32 // Get the default system partition image - for _, desc := range fimg.DescrArr { - if !desc.Used { - continue - } - ptype, err := desc.GetPartType() + desc, err := fimg.GetDescriptor(sif.WithPartitionType(sif.PartPrimSys)) + if err == nil { + fstype, _, goArch, err := desc.PartitionMetadata() if err != nil { - continue - } - if ptype != sif.PartPrimSys { - continue - } - fstype, err := desc.GetFsType() - if err != nil { - continue + return err } - // checks if the partition length is greater that the file - // size which may reveal a corrupted image (see issue #3996) - if fimg.Filesize < desc.Filelen+desc.Fileoff { - return fmt.Errorf("SIF image %s is corrupted: wrong partition size", img.File.Name()) - } - - htype, err := checkPartitionType(img, fstype, desc.Fileoff) + htype, err := checkPartitionType(img, fstype, desc.Offset()) if err != nil { return fmt.Errorf("while checking system partition header: %s", err) } @@ -103,52 +97,38 @@ func (f *sifFormat) initializer(img *Image, fi os.FileInfo) error { // CompatibleWith call will also check that the current machine // has persistent emulation enabled in /proc/sys/fs/binfmt_misc to // be able to execute container process correctly - sifArch := string(fimg.Header.Arch[:sif.HdrArchLen-1]) - goArch := sif.GetGoArch(sifArch) - if sifArch != sif.HdrArchUnknown && !machine.CompatibleWith(goArch) { + if goArch != "unknown" && !machine.CompatibleWith(goArch) { return fmt.Errorf("the image's architecture (%s) could not run on the host's (%s)", goArch, runtime.GOARCH) } + groupID = desc.GroupID() + img.Partitions = []Section{ { - Offset: uint64(desc.Fileoff), - Size: uint64(desc.Filelen), - ID: desc.ID, + Offset: uint64(desc.Offset()), + Size: uint64(desc.Size()), + ID: desc.ID(), Name: RootFs, Type: htype, AllowedUsage: RootFsUsage, }, } - - groupID = int(desc.Groupid) - break } - for _, desc := range fimg.DescrArr { - if !desc.Used { - continue - } - if ptype, err := desc.GetPartType(); err == nil { + fimg.WithDescriptors(func(desc sif.Descriptor) bool { + if fstype, ptype, _, err := desc.PartitionMetadata(); err == nil { // exclude partitions that are not types data or overlay if ptype != sif.PartData && ptype != sif.PartOverlay { - continue + return false } // ignore overlay partitions not associated to root filesystem group ID if any - if ptype == sif.PartOverlay && groupID > 0 && groupID != int(desc.Groupid) { - continue - } - fstype, err := desc.GetFsType() - if err != nil { - continue + if ptype == sif.PartOverlay && groupID > 0 && groupID != desc.GroupID() { + return false } - if fimg.Filesize < desc.Filelen+desc.Fileoff { - return fmt.Errorf("SIF image %s is corrupted: wrong partition size", img.File.Name()) - } - - htype, err := checkPartitionType(img, fstype, desc.Fileoff) + htype, err := checkPartitionType(img, fstype, desc.Offset()) if err != nil { - return fmt.Errorf("while checking data partition header: %s", err) + return false } var usage Usage @@ -160,27 +140,28 @@ func (f *sifFormat) initializer(img *Image, fi os.FileInfo) error { } partition := Section{ - Offset: uint64(desc.Fileoff), - Size: uint64(desc.Filelen), - ID: desc.ID, - Name: desc.GetName(), + Offset: uint64(desc.Offset()), + Size: uint64(desc.Size()), + ID: desc.ID(), + Name: desc.Name(), Type: htype, AllowedUsage: usage, } img.Partitions = append(img.Partitions, partition) img.Usage |= usage - } else if desc.Datatype != 0 { + } else if desc.DataType() != 0 { data := Section{ - Offset: uint64(desc.Fileoff), - Size: uint64(desc.Filelen), - ID: desc.ID, - Type: uint32(desc.Datatype), - Name: desc.GetName(), + Offset: uint64(desc.Offset()), + Size: uint64(desc.Size()), + ID: desc.ID(), + Type: uint32(desc.DataType()), + Name: desc.Name(), AllowedUsage: DataUsage, } img.Sections = append(img.Sections, data) } - } + return false + }) img.Type = SIF diff --git a/pkg/image/sif_test.go b/pkg/image/sif_test.go index af0fbe79b7..125ff21d22 100644 --- a/pkg/image/sif_test.go +++ b/pkg/image/sif_test.go @@ -11,42 +11,33 @@ import ( "runtime" "testing" - uuid "github.com/satori/go.uuid" - "github.com/sylabs/sif/pkg/sif" + "github.com/sylabs/sif/v2/pkg/sif" "github.com/sylabs/singularity/internal/pkg/util/fs" ) const testSquash = "./testdata/squashfs.v4" -func createSIF(t *testing.T, inputDesc []sif.DescriptorInput, corrupted bool) string { +func createSIF(t *testing.T, corrupted bool, fns ...func() (sif.DescriptorInput, error)) string { sifFile, err := fs.MakeTmpFile("", "sif-", 0o644) if err != nil { t.Fatalf("failed to create temporary file: %s", err) } sifFile.Close() - for _, d := range inputDesc { - if f, ok := d.Fp.(*os.File); ok { - f.Seek(0, 0) - } - } + var opts []sif.CreateOpt - id, err := uuid.NewV4() - if err != nil { - t.Fatal(err) - } + for _, fn := range fns { + di, err := fn() + if err != nil { + t.Fatalf("failed to get DescriptorInput: %v", err) + } - cinfo := sif.CreateInfo{ - Pathname: sifFile.Name(), - Launchstr: sif.HdrLaunch, - Sifversion: sif.HdrVersion, - ID: id, - InputDescr: inputDesc, + opts = append(opts, sif.OptCreateWithDescriptors(di)) } - fp, err := sif.CreateContainer(cinfo) + fp, err := sif.CreateContainerAtPath(sifFile.Name(), opts...) if err != nil { - t.Fatalf("failed to create empty SIF") + t.Fatalf("failed to create SIF: %v", err) } fp.UnloadContainer() @@ -70,73 +61,37 @@ func createSIF(t *testing.T, inputDesc []sif.DescriptorInput, corrupted bool) st } func TestSIFInitializer(t *testing.T) { - fp1, err := os.Open(testSquash) - if err != nil { - t.Fatalf("failed to open %s: %s", testSquash, err) - } - defer fp1.Close() - - fp2, err := os.Open(testSquash) + b, err := os.ReadFile(testSquash) if err != nil { - t.Fatalf("failed to open %s: %s", testSquash, err) + t.Fatalf("failed to read %s: %s", testSquash, err) } - defer fp2.Close() - onePart := sif.DescriptorInput{ - Datatype: sif.DataPartition, - Groupid: sif.DescrDefaultGroup, - Link: sif.DescrUnusedLink, - Fname: "onePart", - Fp: fp1, - Extra: *bytes.NewBuffer([]byte{ - 0x01, 0x00, 0x00, 0x00, // fstype - 0x01, 0x00, 0x00, 0x00, // part type - }), + onePart := func() (sif.DescriptorInput, error) { + return sif.NewDescriptorInput(sif.DataPartition, bytes.NewReader(b), + sif.OptPartitionMetadata(sif.FsSquash, sif.PartSystem, runtime.GOARCH), + ) } - oneSection := sif.DescriptorInput{ - Datatype: sif.DataGeneric, - Groupid: sif.DescrDefaultGroup, - Link: sif.DescrUnusedLink, - Fname: "oneSection", - Fp: fp1, + oneSection := func() (sif.DescriptorInput, error) { + return sif.NewDescriptorInput(sif.DataGeneric, bytes.NewReader(b)) } - primPartNoArch := sif.DescriptorInput{ - Datatype: sif.DataPartition, - Groupid: sif.DescrDefaultGroup, - Link: sif.DescrUnusedLink, - Fname: "primPart", - Fp: fp1, - Extra: *bytes.NewBuffer([]byte{ - 0x01, 0x00, 0x00, 0x00, // fstype - 0x02, 0x00, 0x00, 0x00, // part type - }), + primPartOtherArch := func() (sif.DescriptorInput, error) { + return sif.NewDescriptorInput(sif.DataPartition, bytes.NewReader(b), + sif.OptPartitionMetadata(sif.FsSquash, sif.PartPrimSys, "s390x"), + ) } - primPart := sif.DescriptorInput{ - Datatype: sif.DataPartition, - Groupid: sif.DescrDefaultGroup, - Link: sif.DescrUnusedLink, - Fname: "primPart", - Fp: fp1, - Extra: *bytes.NewBuffer([]byte{ - 0x01, 0x00, 0x00, 0x00, // fstype - 0x02, 0x00, 0x00, 0x00, // part type - }), + primPart := func() (sif.DescriptorInput, error) { + return sif.NewDescriptorInput(sif.DataPartition, bytes.NewReader(b), + sif.OptPartitionMetadata(sif.FsSquash, sif.PartPrimSys, runtime.GOARCH), + ) } - primPart.Extra.WriteString(sif.GetSIFArch(runtime.GOARCH)) - overlayPart := sif.DescriptorInput{ - Datatype: sif.DataPartition, - Groupid: sif.DescrDefaultGroup, - Link: sif.DescrUnusedLink, - Fname: "overlayPart", - Fp: fp2, - Extra: *bytes.NewBuffer([]byte{ - 0x01, 0x00, 0x00, 0x00, // fstype - 0x04, 0x00, 0x00, 0x00, // part type - }), + overlayPart := func() (sif.DescriptorInput, error) { + return sif.NewDescriptorInput(sif.DataPartition, bytes.NewReader(b), + sif.OptPartitionMetadata(sif.FsSquash, sif.PartOverlay, runtime.GOARCH), + ) } tests := []struct { @@ -149,7 +104,7 @@ func TestSIFInitializer(t *testing.T) { }{ { name: "NoPartitionSIF", - path: createSIF(t, nil, false), + path: createSIF(t, false), writable: false, expectedSuccess: true, expectedPartitions: 0, @@ -157,15 +112,15 @@ func TestSIFInitializer(t *testing.T) { }, { name: "UnkownPartitionSIF", - path: createSIF(t, []sif.DescriptorInput{onePart}, false), + path: createSIF(t, false, onePart), writable: false, expectedSuccess: true, expectedPartitions: 0, expectedSections: 0, }, { - name: "PrimaryPartitionNoArchSIF", - path: createSIF(t, []sif.DescriptorInput{primPartNoArch}, false), + name: "PrimaryPartitionOtherArchSIF", + path: createSIF(t, false, primPartOtherArch), writable: false, expectedSuccess: false, expectedPartitions: 0, @@ -173,7 +128,7 @@ func TestSIFInitializer(t *testing.T) { }, { name: "PrimaryPartitionSIF", - path: createSIF(t, []sif.DescriptorInput{primPart}, false), + path: createSIF(t, false, primPart), writable: false, expectedSuccess: true, expectedPartitions: 1, @@ -181,7 +136,7 @@ func TestSIFInitializer(t *testing.T) { }, { name: "PrimaryPartitionCorruptedSIF", - path: createSIF(t, []sif.DescriptorInput{primPart}, true), + path: createSIF(t, true, primPart), writable: false, expectedSuccess: false, expectedPartitions: 0, @@ -189,7 +144,7 @@ func TestSIFInitializer(t *testing.T) { }, { name: "PrimaryAndOverlayPartitionsSIF", - path: createSIF(t, []sif.DescriptorInput{primPart, overlayPart}, false), + path: createSIF(t, false, primPart, overlayPart), writable: false, expectedSuccess: true, expectedPartitions: 2, @@ -197,7 +152,7 @@ func TestSIFInitializer(t *testing.T) { }, { name: "SectionSIF", - path: createSIF(t, []sif.DescriptorInput{oneSection}, false), + path: createSIF(t, false, oneSection), writable: false, expectedSuccess: true, expectedPartitions: 0, @@ -205,7 +160,7 @@ func TestSIFInitializer(t *testing.T) { }, { name: "PartitionAndSectionSIF", - path: createSIF(t, []sif.DescriptorInput{primPart, oneSection}, false), + path: createSIF(t, false, primPart, oneSection), writable: false, expectedSuccess: true, expectedPartitions: 1, diff --git a/pkg/util/cryptkey/key.go b/pkg/util/cryptkey/key.go index 48bbf3b41e..40ba1d6058 100644 --- a/pkg/util/cryptkey/key.go +++ b/pkg/util/cryptkey/key.go @@ -17,8 +17,9 @@ import ( "fmt" "io" "io/ioutil" + "os" - "github.com/sylabs/sif/pkg/sif" + "github.com/sylabs/sif/v2/pkg/sif" ) var ( @@ -201,31 +202,29 @@ func savePEMMessage(w io.Writer, msg []byte) error { } func getEncryptionKeyFromImage(fn string) ([]byte, error) { - img, err := sif.LoadContainer(fn, true) + img, err := sif.LoadContainerFromPath(fn, sif.OptLoadWithFlag(os.O_RDONLY)) if err != nil { - return nil, fmt.Errorf("could not load container: %v", err) + return nil, fmt.Errorf("could not load container: %w", err) } defer img.UnloadContainer() - primDescr, _, err := img.GetPartPrimSys() + primDescr, err := img.GetDescriptor(sif.WithPartitionType(sif.PartPrimSys)) if err != nil { - return nil, fmt.Errorf("could not retrieve primary system partition from '%s'", fn) + return nil, fmt.Errorf("could not retrieve primary system partition from '%s': %w", fn, err) } - descr, _, err := img.GetLinkedDescrsByType(primDescr.ID, sif.DataCryptoMessage) + descr, err := img.GetDescriptors( + sif.WithLinkedID(primDescr.ID()), + sif.WithDataType(sif.DataCryptoMessage), + ) if err != nil { - return nil, fmt.Errorf("could not retrieve linked descriptors for primary system partition from %s", fn) + return nil, fmt.Errorf("could not retrieve linked descriptors for primary system partition from %s: %w", fn, err) } for _, d := range descr { - format, err := d.GetFormatType() + format, message, err := d.CryptoMessageMetadata() if err != nil { - return nil, fmt.Errorf("could not get descriptor format type: %v", err) - } - - message, err := d.GetMessageType() - if err != nil { - return nil, fmt.Errorf("could not get descriptor message type: %v", err) + return nil, fmt.Errorf("could not get crypto message metadata: %w", err) } // currently only support one type of message @@ -236,8 +235,8 @@ func getEncryptionKeyFromImage(fn string) ([]byte, error) { // TODO(ian): For now, assume the first linked message is what we // are looking for. We should consider what we want to do in the // case of multiple linked messages - key := make([]byte, d.Filelen) - if _, err := io.ReadFull(d.GetReader(&img), key); err != nil { + key, err := d.GetData() + if err != nil { return nil, fmt.Errorf("could not retrieve LUKS key data from %s: %w", fn, err) }