From f4d8b551fe0e841b716eea09f763b97db9379a2a Mon Sep 17 00:00:00 2001 From: Matt Wrock Date: Mon, 30 Sep 2024 15:39:33 -0700 Subject: [PATCH] adding pkg-sync tool and update docs to reflect changes in syncing Signed-off-by: Matt Wrock --- README.md | 5 - on-prem-docs/bootstrap-core.md | 44 ++- on-prem-docs/builder-oauth.md | 6 +- on-prem-docs/getting-started.md | 5 - package_seed_lists/README.md | 35 +- .../builder_x86_64-linux_lts_2024 | 8 + .../builder_x86_64-linux_stable | 5 - .../effortless_x86_64-linux_unstable | 1 - .../effortless_x86_64-windows_unstable | 1 - pkg-sync/README.md | 45 +++ pkg-sync/main.go | 311 ++++++++++++++++++ pkg-sync/plan.sh | 19 ++ 12 files changed, 419 insertions(+), 66 deletions(-) create mode 100644 package_seed_lists/builder_x86_64-linux_lts_2024 delete mode 100644 package_seed_lists/effortless_x86_64-linux_unstable delete mode 100644 package_seed_lists/effortless_x86_64-windows_unstable create mode 100644 pkg-sync/README.md create mode 100644 pkg-sync/main.go create mode 100644 pkg-sync/plan.sh diff --git a/README.md b/README.md index 86a4fc7..841f276 100644 --- a/README.md +++ b/README.md @@ -50,11 +50,6 @@ Once installed, the following functionality will be available to users: * Package builds using the `hab` client and Chef Habitat Studio * Ability to import core packages from the upstream Chef Habitat Builder -The following Chef Habitat Builder on-prem functionalities are *NOT* currently available: - -* Automated package builds using Chef Habitat Builder on-prem -* Automated package exports using Chef Habitat Builder on-prem - ## Documentation The documentation for Builder on-prem is located in the [on-prem-docs](on-prem-docs/getting-started.md) directory. diff --git a/on-prem-docs/bootstrap-core.md b/on-prem-docs/bootstrap-core.md index b5cb782..33a438d 100644 --- a/on-prem-docs/bootstrap-core.md +++ b/on-prem-docs/bootstrap-core.md @@ -1,50 +1,46 @@ # Bootstrap Core Packages -## Create the Core Origin - -Create a `core` origin for an initial set of base packages. Uploads will fail unless you first populate your Chef Habitat Builder on-prem with the upstream `core` upstream origin. - -Once you are logged in to the Chef Habitat Builder on-prem UI, select the `New Origin` button and enter in `core` as the origin name. - ## Generate a Personal Access Token -Next, generate a Personal Access Token for bootstrapping the `core` packages, as well as for performing authenticated operations using the `hab` client. +Generate a Personal Access Token for bootstrapping the `core` packages, as well as for performing authenticated operations using the `hab` client. Select your Gravatar icon on the top right corner of the Chef Habitat Builder on-prem web page, and then select **Profile**. This will take you to a page where you can generate your access token. Make sure to save it securely. ## Bootstrap Builder with Habitat Packages Chef Habitat Builder on-prem has no pre-installed package sets. You must populate your Builder instance by uploading packages. -With Habitat *0.88.0*, two new commands were introduced to assist in bootstrapping an on-prem Builder instance with a set of stable packages: +To assist in bootstrapping an on-prem Builder instance with a set of LTS packages, you can install the habitat/pkg-sync package which will download packages from the public [SaaS Builder](https://bldr.habitat.sh) followed by a bulkupload to your on-prem Builder instance(s). + +The following snippet illustrates how to bootstrap the on-prem Builder with a set of LTS-2024 core, chef, and chef-platform packages: -1. *hab pkg download* -1. *hab pkg bulkupload* + ```bash + hab pkg install habitat/pkg-sync + hab pkg exec habitat/pkg-sync pkg-sync --bldr-url https://your-builder.tld --channel LTS-2024 --auth + ``` +This performs a pre-flight check to ensure that you do not have local core, chef, or chef-platform packages in the LTS-2024 channel that are not in the same channel on bldr.habitat.sh. If there are, these local packages must be demoted. Once the pre-flight check passes, pkg-sync will build a list of all the latest LTS-2024 packages that you do not already have and then upload them to your on-prem builder instance. -As you can see from the commands above, the package Bootstrap flow is comprised of two main phases: a download from the public [SaaS Builder](https://bldr.habitat.sh) followed by a bulkupload to your on-prem Builder instance(s). Historically, we bootstrapped on-prem-builders by downloading all the packages in 'core' for all targets. That amounted to ~15GB and was both too much and too little, in that many of the packages weren't needed, and for many patterns (effortless) other origins were needed. +### Airgapped Environments -The [new bootstrap process flow](https://forums.habitat.sh/t/populating-chef-habitat-builder-on-prem/1228) allows you to easily customize your Bootstrap package set or use pre-populated [Package Seed Lists](https://github.com/habitat-sh/on-prem-builder/blob/master/package_seed_lists/README.md) files. +Airgapped builder instances must take an alternative approach because pkg-sync will not be able to transfer packages from the public internet to your instance. Instead you will use the `--generate-airgap-list` flag with pkg-sync to build a list of packages that need to be downloaded. Then you will use `hab pkg download` and `hab pkg upload` to download the packages from bldr.habitat.sh and upload them to your instance. Note that `pkg-sync` and `hab pkg download` must be used on a machine with access to the public internet. This will download a bundle you can archive and transfer to your instance. Finally you will use `hab pkg upload` locally on your builder instance to upload the packages into your instance. -The following section illustrates the steps required to bootstrap the on-prem Builder with the [Effortless Linux](https://github.com/habitat-sh/on-prem-builder/blob/master/package_seed_lists/effortless_x86_64-linux_stable) package seed list. Simply repeat the following download/bulkupload flow for any other package seed lists you think you will need to have in your on-prem Builder, or even create your own custom package seed list file: +The following section illustrates the steps required to bootstrap an airgapped on-prem Builder with a set of LTS-2024 core, chef, and chef-platform packages: -1. Phase 1: download +1. Phase 1: download from a machine with internet connectivity ```bash - export HAB_AUTH_TOKEN= - cd on-prem-builder - hab pkg download --target x86_64-linux --channel stable --file package_seed_lists/effortless_x86_64-linux_stable --download-directory builder_bootstrap + hab pkg install habitat/pkg-sync + hab pkg exec habitat/pkg-sync pkg-sync --generate-airgap-list --channel LTS-2024 + hab pkg download --target x86_64-linux --channel LTS-2024 --file package_list_x86_64-linux.txt --download-directory builder_bootstrap + hab pkg download --target x86_64-windows --channel LTS-2024 --file package_list_x86_64-windows.txt --download-directory builder_bootstrap ``` - > Note: If the on-prem Builder is Airgapped, this phase must be completed on a system with Internet connectivity. The downloaded content will need to be zipped and then transferred to the Builder system for phase 2. - -1. Phase 2: bulkupload - - **Important**: Inspect the contents of the `builder_bootstrap/artifacts` directory created from the download command above. For each of the origins (`core`, `effortless`, etc), create the origin name if one doesn't exist already in the on-prem Builder UI before starting the bulkupload. + Zip up the contents of `builder_bootstrap`. Copy and unzip to the builder instance - > Note: If your on-prem builder's SSL certificate was issued from an internal Public Key Infrastructure and not from a Publicly Trusted Certificate Authority, you will need to copy the SSL public key cert chain into `/hab/cache/ssl` locally on the system that is uploading packages to the on-prem Builder. This is described in more detail [here](https://blog.chef.io/chef-habitat-product-announcement-improved-tls-certificate-management/) +1. Phase 2: bulkupload locally on the builder instance ```bash export HAB_AUTH_TOKEN= - hab pkg bulkupload --url https://your-builder.tld --channel stable builder_bootstrap/ + hab pkg bulkupload --url https://your-builder.tld --channel LTS-2024 --auto-create-origins builder_bootstrap/ ``` ## Configuring a user workstation diff --git a/on-prem-docs/builder-oauth.md b/on-prem-docs/builder-oauth.md index 87a95f2..213566e 100644 --- a/on-prem-docs/builder-oauth.md +++ b/on-prem-docs/builder-oauth.md @@ -83,22 +83,22 @@ In order to install the on-prem Chef Habitat Builder in an airgapped (no direct curl -LO https://github.com/habitat-sh/on-prem-builder/archive/master.zip ``` -1. Download the Chef Habitat [cli tool](https://api.bintray.com/content/habitat/stable/linux/x86_64/hab-%24latest-x86_64-linux.tar.gz?bt_package=hab-x86_64-linux) +1. Download the Chef Habitat [cli tool](https://packages.chef.io/files/stable/habitat/latest/hab-x86_64-linux.tar.gz) ```bash curl -Lo hab.tar.gz https://packages.chef.io/files/stable/habitat/latest/hab-x86_64-linux.tar.gz ``` -1. Create the Habitat Builder package bundle from the [Builder Seed List](https://github.com/habitat-sh/on-prem-builder/blob/master/package_seed_lists/builder_x86_64-linux_stable) package seed list and download it +1. Create the Habitat Builder package bundle from the Builder seed lists and download them ```bash git clone https://github.com/habitat-sh/on-prem-builder.git export DOWNLOAD_DIR=/some/base/download/directory cd on-prem-builder + hab pkg download --target x86_64-linux --channel LTS-2024 --file package_seed_lists/builder_x86_64-linux_lts_2024 --download-directory ${DOWNLOAD_DIR}/builder_packages hab pkg download --target x86_64-linux --channel stable --file package_seed_lists/builder_x86_64-linux_stable --download-directory ${DOWNLOAD_DIR}/builder_packages ``` -1. Create any additional package bundles to upload to Builder from package seed lists as documented in the [Bootstrap Builder](https://github.com/habitat-sh/on-prem-builder/blob/master/README.md#bootstrap-builder-with-habitat-packages-new) section of this README. You can specify `--download-directory ${DOWNLOAD_DIR}/builder_bootstrap` argument to the download command in order to consolidate all bootstrap packages in a single directory 1. Zip up all the above content, transfer and unzip on the Linux system where Builder will be deployed in the Airgapped environment > Note: The following tasks are intended to be completed on the Airgapped system where Builder will be deployed, in advance of the [Installation](https://github.com/habitat-sh/on-prem-builder/blob/master/README.md#Installation). diff --git a/on-prem-docs/getting-started.md b/on-prem-docs/getting-started.md index ebe4084..f93dd34 100644 --- a/on-prem-docs/getting-started.md +++ b/on-prem-docs/getting-started.md @@ -42,11 +42,6 @@ Once installed, the following functionality will be available to users: * Package builds using the `hab` client and Chef Habitat Studio * Ability to import core packages from the upstream Chef Habitat Builder -The following Chef Habitat Builder on-prem functionalities are *NOT* currently available: - -* Automated package builds using Chef Habitat Builder on-prem -* Automated package exports using Chef Habitat Builder on-prem - ### Memory Filesystem Storage Preparing your filesystem (Optional) diff --git a/package_seed_lists/README.md b/package_seed_lists/README.md index bde9fb6..858f992 100644 --- a/package_seed_lists/README.md +++ b/package_seed_lists/README.md @@ -1,23 +1,17 @@ # Habitat seed lists -Historically we bootstrapped on-prem Builders by downloading all the packages in 'core' -for all targets. That amounted to about 15GB, and was both too much and too little, in that many of -the packages weren't needed, and for many patterns (Effortless) other origins were needed. +With the `hab pkg download` command, you can download a list of seed packages along with their transitive dependencies. Of course, the question now becomes 'what do I use for the seed'. -With the creation of the `hab pkg download` command, a different approach is possible; you can start -with a list of seed packages, and download them along with their transitive dependencies. Of course, -the question now becomes 'what do I use for the seed'. - -This directory contains sample 'seed lists' of packages for bootstrapping and syncing on-prem Builder, for +This directory contains sample 'seed lists' of packages for bootstrapping and syncing on-prem Builder for a number of different scenarios. -The basic file naming pattern is TASK\_ARCH\_CHANNEL. The files are newline separated list of +> Note: We are moving away from seed lists. See [Bootstrap Core Origin](../on-prem-docs/bootstrap-core.md) for instructions on using the `pkg-sync` tool to sync packages from public builder to an on-prem instance. + +The basic file naming pattern is TASK\_ARCH\_CHANNEL. The files are a newline separated list of package identifiers. The contents are specific to a particular architecture and channel and the correct architecture and channel must be provided on the command line when running the `hab pkg download` command. The simple file format used doesn't allow for specification of channel or target -architecture inline, so we're using a file naming convention to represent that information. We plan -to improve on that experience, but for now keep it mind when building your own lists. - +architecture inline, so we're using a file naming convention to represent that information. # Scenarios The current scenarios are: @@ -26,21 +20,19 @@ The current scenarios are: * core_full (everything for a particular architecture) * effortless (starter set for the effortless pattern) -Each is broken out by the architecture and channel required; to complete some two downloads, once -from stable and once from unstable will be required. +Each is broken out by the architecture and channel required. -For example, to get the complete Effortless infrastructure for Linux, +For example, to get the complete Builder infrastructure for Linux, ``` -hab pkg download --download-directory download_pkgs --channel=unstable --target x86\_64-linux --file package_seed_lists/effortless_x86_64-linux_unstable -hab pkg download --download-directory download_pkgs --channel=stable --target x86\_64-linux --file package_seed_lists/effortless_x86_64-linux_stable +hab pkg download --download-directory download_pkgs --channel=LTS-2024 --target x86\_64-linux --file package_seed_lists/builder_x86_64-linux_lts_2024 +hab pkg download --download-directory download_pkgs --channel=stable --target x86\_64-linux --file package_seed_lists/builder_x86_64-linux_stable ``` Current scenarios include: ## Effortless (effortless_ARCH_CHANNEL) -These should provide all required packages for the various Effortless patterns. They're broken out -by architecture, and both stable and unstable are required for a complete Effortless infrastructure. +These should provide all required packages for the various Effortless patterns, broken down by architecture. See https://github.com/chef/effortless for more details on the Effortless infrastructure pattern. @@ -57,7 +49,6 @@ Linux kernel2 are about 3.5GB and 1GB respectively. These are the packages listed as a transitive dep or build dep of another package in core. It is intended as good starting point for building packages. -## Builder (builder_x86_64-linux_stable) +## Builder (builder_x86_64-linux_CHANNEL) -This should be just enough packages to get Builder installed on your system. Currently Builder only -supports the x86_64-linux platform. +This should be just enough packages to get Builder installed on your system. Currently Builder only supports the x86_64-linux platform. You will need both the LTS-2024 channel list and also the stable channel list. diff --git a/package_seed_lists/builder_x86_64-linux_lts_2024 b/package_seed_lists/builder_x86_64-linux_lts_2024 new file mode 100644 index 0000000..8d07aef --- /dev/null +++ b/package_seed_lists/builder_x86_64-linux_lts_2024 @@ -0,0 +1,8 @@ +core/hab +core/hab-sup +core/hab-launcher +habitat/builder-api +habitat/builder-api-proxy +habitat/builder-datastore +habitat/builder-minio +habitat/builder-memcached diff --git a/package_seed_lists/builder_x86_64-linux_stable b/package_seed_lists/builder_x86_64-linux_stable index 8d07aef..d514f79 100644 --- a/package_seed_lists/builder_x86_64-linux_stable +++ b/package_seed_lists/builder_x86_64-linux_stable @@ -1,8 +1,3 @@ core/hab core/hab-sup core/hab-launcher -habitat/builder-api -habitat/builder-api-proxy -habitat/builder-datastore -habitat/builder-minio -habitat/builder-memcached diff --git a/package_seed_lists/effortless_x86_64-linux_unstable b/package_seed_lists/effortless_x86_64-linux_unstable deleted file mode 100644 index b124bad..0000000 --- a/package_seed_lists/effortless_x86_64-linux_unstable +++ /dev/null @@ -1 +0,0 @@ -effortless/config-baseline diff --git a/package_seed_lists/effortless_x86_64-windows_unstable b/package_seed_lists/effortless_x86_64-windows_unstable deleted file mode 100644 index dd161bc..0000000 --- a/package_seed_lists/effortless_x86_64-windows_unstable +++ /dev/null @@ -1 +0,0 @@ -chef/effortless-config-baseline diff --git a/pkg-sync/README.md b/pkg-sync/README.md new file mode 100644 index 0000000..329c000 --- /dev/null +++ b/pkg-sync/README.md @@ -0,0 +1,45 @@ +# pkg-sync + +A tool for syncing packages in an on-prem builder instance with the packages in public builder. + +Given a channel (defaults to `LTS-2024`) and an on-prem builder url and token, this tool can download all core, chef, or chef-platform packages in the channel from the piblic Chef Habitat builder that the on-prem instance does not already have and upload them to the on-prem builder. + +This performs a pre-flight check to ensure that you do not have local core, chef, or chef-platform packages in the channel that are not in the same channel on bldr.habitat.sh. If there are, these local packages must be demoted. + +This tool can also be used to generate a list of packages without actually syncing them. + +## Usage + +Install this package with: + +``` +hab pkg install habitat/pkg-sync +``` + +Examples: + +Sync all the latest core, chef, or chef-platform LTS-2024 packages that you do not already have from the public builder and upload them to your on-prem builder instance. + +``` +hab pkg exec habitat/pkg-sync pkg-sync --bldr-url https://your-builder.tld --channel LTS-2024 --auth +``` + +Generate a list of all LTS-2024 packages in the core, chef, and chef-platform origins. + +``` +hab pkg exec habitat/pkg-sync pkg-sync --generate-airgap-list --channel LTS-2024 +``` + +These lists can be used with `hab pkg download` to download the hart files from builder: + +``` +hab pkg download --target x86_64-linux --channel LTS-2024 --file package_list_x86_64-linux.txt --download-directory builder_bootstrap +hab pkg download --target x86_64-windows --channel LTS-2024 --file package_list_x86_64-windows.txt --download-directory builder_bootstrap +``` + +You could then copy the `--download-directory` contents to an airgapped builder and use `hab pkg bulkupload` to upload them: + +``` +export HAB_AUTH_TOKEN= +hab pkg bulkupload --url https://your-builder.tld --channel LTS-2024 --auto-create-origins builder_bootstrap/ +``` \ No newline at end of file diff --git a/pkg-sync/main.go b/pkg-sync/main.go new file mode 100644 index 0000000..5124ddb --- /dev/null +++ b/pkg-sync/main.go @@ -0,0 +1,311 @@ +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "log" + "net/http" + "os" + "os/exec" + "slices" + "strings" +) + +type PackageListResponse struct { + RangeEnd int `json:"range_end"` + TotalCount int `json:"total_count"` + Data []*PackageIdent `json:"data"` +} + +type PackageIdent struct { + Origin string `json:"origin"` + Name string `json:"name"` + Version string `json:"version"` + Release string `json:"release"` +} + +type PackageDetails struct { + Target string `json:"target"` + Type string `json:"package_type"` +} + +var originList = []string{"core", "chef", "chef-platform"} + +// parseArgs parses and validates command-line arguments +func parseArgs() (bldrURL, channel, auth string, generateListOnly, effortlessOnly, help bool) { + flag.StringVar(&bldrURL, "bldr-url", "", "Base URL of your on-prem builder") + flag.StringVar(&channel, "channel", "LTS-2024", "Refresh channel to sync") + flag.StringVar(&auth, "auth", "", "Authorization Token for on-prem builder") + flag.BoolVar(&generateListOnly, "generate-airgap-list", false, "Output list of package identifiers needed to download to a file in the current directory. If specified, --bldr-url is not required and a sync will not be performed.") + // Uncomment once we have LTS-2024 releases for Chef infra and inspec + // flag.BoolVar(&effortlessOnly, "effortless-only", false, "Only sync effortless packages") + flag.BoolVar(&help, "help", false, "Show help message") + flag.Parse() + + return +} + +// fetchJSON makes an HTTP GET request to the specified URL and decodes the JSON response +func fetchJSON(url string, target interface{}) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return json.NewDecoder(resp.Body).Decode(target) + } else { + return fmt.Errorf("Received status %d from %s", resp.StatusCode, url) + } + +} + +// fetchPackages fetches packages for a given origin from the builder service +func fetchPackages(bldrURL, channel string) ([]string, error) { + rangeSize := 50 + var result []string + for _, origin := range originList { + fmt.Printf("\nFinding packages for origin %s in channel %s from %s\n", origin, channel, bldrURL) + packageCount := 0 + for rangeIndex := 0; ; { + fmt.Printf("\rFetching range %d-%d...", rangeIndex, rangeIndex+50) + var response *PackageListResponse + url := fmt.Sprintf("%s/v1/depot/channels/%s/%s/pkgs?range=%d", bldrURL, origin, channel, rangeIndex) + err := fetchJSON(url, &response) + if err != nil { + return nil, fmt.Errorf("\nfailed to fetch package data from %s: %s", url, err.Error()) + } else { + for _, pkg := range response.Data { + result = append(result, fmt.Sprintf("%s/%s/%s/%s", pkg.Origin, pkg.Name, pkg.Version, pkg.Release)) + } + packageCount += len(response.Data) + rangeIndex += rangeSize + if response.TotalCount == 0 || (response.RangeEnd+1) == response.TotalCount { + break + } + } + } + fmt.Printf("\nDiscovered %d packages in origin %s\n", packageCount, origin) + } + slices.Sort(result) + return result, nil +} + +func fetchDetail(bldrURL, ident string) (*PackageDetails, error) { + var response *PackageDetails + url := fmt.Sprintf("%s/v1/depot/pkgs/%s", bldrURL, ident) + err := fetchJSON(url, &response) + if err != nil { + return nil, fmt.Errorf("failed to fetch package detail for %s: %s", ident, err.Error()) + } else { + return response, nil + } +} + +func fetchLatestPackages(channel, target string, localPackages []string) ([]string, error) { + var response *PackageListResponse + var result []string + for _, origin := range originList { + fmt.Printf("\nFetching latest %s packages for origin %s in channel %s from public bldr...\n", target, origin, channel) + url := fmt.Sprintf("https://bldr.habitat.sh/v1/depot/channels/%s/%s/pkgs/_latest?target=%s", origin, channel, target) + err := fetchJSON(url, &response) + if err != nil { + return nil, fmt.Errorf("Failed to fetch latest packages from %s: %s", url, err.Error()) + } else { + fmt.Printf("Found %d latest %s packages for %s\n", len(response.Data), target, origin) + pkgCount := 0 + for _, pkg := range response.Data { + pkgCount = pkgCount + 1 + fmt.Printf("\rFetching detail for %d of %d packages", pkgCount, len(response.Data)) + ident := fmt.Sprintf("%s/%s/%s/%s", pkg.Origin, pkg.Name, pkg.Version, pkg.Release) + if !slices.Contains(localPackages, ident) { + detail, err := fetchDetail("https://bldr.habitat.sh", ident) + if err != nil { + fmt.Println("") + return nil, err + } + // filter out native and bootstrap packages used internally + if !strings.Contains(pkg.Name, "-stage1") && !strings.Contains(pkg.Name, "-stage0") && !strings.HasPrefix(pkg.Name, "build-tools-") && detail.Type != "Native" { + result = append(result, ident) + } + } + } + } + } + fmt.Printf("\nFound %d latest %s packages needed to download\n", len(result), target) + slices.Sort(result) + return result, nil +} + +func ReadYesNo() bool { + r := bufio.NewReader(os.Stdin) + var s string + + for { + s, _ = r.ReadString('\n') + s = strings.TrimSpace(s) + s = strings.ToLower(s) + if s == "y" { + return true + } + if s == "n" { + return false + } + fmt.Println("OOpsies! You did not enter a 'y' or an 'n'. Lets try again.") + } +} + +func executeCommand(command string, args ...string) error { + cmd := exec.Command(command, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + return fmt.Errorf("Error calling %s: %s", strings.Join(append([]string{command}, args...), " "), err.Error()) + } + return nil +} + +func sync(effortlessOnly bool, channel, bldrURL, auth string, localPackages []string, generateListOnly bool) error { + for _, target := range []string{"x86_64-linux", "x86_64-windows"} { + var latestPackages []string + var err error + + // build a list of all packages that need to be downloaded + if effortlessOnly { + latestPackages = []string{"chef/chef-infra-client", "chef/inspec", "chef/scaffolding-chef-inspec", "chef/scaffolding-chef-infra"} + } else { + latestPackages, err = fetchLatestPackages(channel, target, localPackages) + if err != nil { + return err + } + } + + file, err := os.Create("package_list_" + target + ".txt") + if err != nil { + return err + } + + if !generateListOnly { + defer os.Remove(file.Name()) + } + + if len(latestPackages) == 0 { + continue + } + + for _, pkg := range latestPackages { + _, err = fmt.Fprintln(file, pkg) + if err != nil { + return err + } + } + + if !generateListOnly { + dir, err := os.MkdirTemp("", "download_cache") + if err != nil { + return err + } + defer os.RemoveAll(dir) + fmt.Printf("\nDownloading %s packages from http://bldr.habitat.sh to %s", target, dir) + err = executeCommand("hab", "pkg", "download", "--download-directory", dir, "--target", target, "--channel", channel, "--file", file.Name()) + if err != nil { + return err + } + + fmt.Printf("\nUploading %s packages to %s", target, bldrURL) + err = executeCommand("hab", "pkg", "bulkupload", "--url", bldrURL, "--channel", channel, "--auth", auth, "--auto-create-origins", dir) + if err != nil { + return err + } + } else { + fmt.Printf("Generated list of packages needed to download at %s\n", file.Name()) + } + } + return nil +} + +func preflightCheck(channel, bldrURL, auth string) ([]string, []string, error) { + // Get complete list of all packages in channel on sass builder + saasPackages, err := fetchPackages("https://bldr.habitat.sh", channel) + if err != nil { + return nil, nil, err + } + + // Get complete list of all packages in channel on customer builder + localPackages, err := fetchPackages(bldrURL, channel) + if err != nil { + return nil, nil, err + } + + // Look for cases where the customer has promoted a package on their own builder + // that has not been promoted by chef + var foreignPackages []string + for _, pkg := range localPackages { + if !slices.Contains(saasPackages, pkg) { + foreignPackages = append(foreignPackages, pkg) + } + } + + return localPackages, foreignPackages, nil +} + +func main() { + // Dont need errors prefixed with date and time + log.SetFlags(0) + + bldrURL, channel, auth, generateListOnly, effortlessOnly, help := parseArgs() + if help || (bldrURL == "" && !generateListOnly) { + fmt.Printf("Usage: %s [OPTIONS]\n", os.Args[0]) + fmt.Println("Options:") + flag.PrintDefaults() + return + } + + var localPackages, foreignPackages []string + var err error + if !generateListOnly { + fmt.Printf("Starting preflight check for local %s packages not promoted to %s on bldr.habitat.sh...\n", channel, channel) + localPackages, foreignPackages, err = preflightCheck(channel, bldrURL, auth) + if err != nil { + log.Fatalln(err.Error()) + } + // To prevent the possibility of build issues, we now need to demote these packages + if foreignPackages != nil { + fmt.Printf("\nFound the following local packages in the %s channel that are not on bldr.habitat.sh:\n", channel) + for _, pkg := range foreignPackages { + fmt.Println(pkg) + } + fmt.Printf("These packages must be demoted from %s before we can continue. Shall we demote them?(y/n)\n", channel) + + if !ReadYesNo() { + fmt.Println("Exiting sync") + return + } else { + for _, pkg := range foreignPackages { + fmt.Println("Demoting", pkg) + detail, err := fetchDetail(bldrURL, pkg) + if err != nil { + log.Fatalln(err.Error()) + } + err = executeCommand("hab", "pkg", "demote", "-u", bldrURL, "-z", auth, pkg, channel, detail.Target) + if err != nil { + log.Fatalln(err.Error()) + } + } + fmt.Println("Succesfully demoted all conflicting packages.") + } + } else { + fmt.Println("No conflicting packages found.") + } + } + + err = sync(effortlessOnly, channel, bldrURL, auth, localPackages, generateListOnly) + if err != nil { + log.Fatalln(err.Error()) + } + + fmt.Printf("Package sync with %s channel was succesful!\n", channel) +} diff --git a/pkg-sync/plan.sh b/pkg-sync/plan.sh new file mode 100644 index 0000000..05c51d1 --- /dev/null +++ b/pkg-sync/plan.sh @@ -0,0 +1,19 @@ +pkg_name=pkg-sync +pkg_origin=habitat +pkg_version="0.1.0" +pkg_maintainer="The Habitat Maintainers " +pkg_build_deps=(core/go21) +pkg_bin_dirs=(bin) +pkg_description="syncs on prem builder packages with latest saas packages." + +do_build() { + return 0 +} + +do_install() { + CGO_ENABLED=0 go build -installsuffix 'static' -o "$pkg_prefix"/bin/pkg-sync main.go +} + +do_strip() { + return 0 +}