diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c857885 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +test/gradle/demo +test/npm/demo diff --git a/.github/workflows/dip.yml b/.github/workflows/dip.yml index e612900..c7f0b0b 100644 --- a/.github/workflows/dip.yml +++ b/.github/workflows/dip.yml @@ -1,6 +1,6 @@ --- name: DIP -on: [push] +'on': push jobs: dive: runs-on: ubuntu-latest @@ -20,3 +20,8 @@ jobs: GOLANGCI_LINT_VERSION=$(./dip image --name=golangci/golangci-lint --regex=^v1\.[0-9]+\.[0-9]+-alpine$) echo "Check whether the latest GolangCI version: '${GOLANGCI_LINT_VERSION}' is used..." grep "golangci-lint:${GOLANGCI_LINT_VERSION}" ./.github/workflows/go.yml + - name: Check Yamllint + run: | + YAMLLINT_VERSION=$(./dip image --name=pipelinecomponents/yamllint --regex=0\..*) + echo "Check whether the latest yamllint version: '${YAMLLINT_VERSION}' is used..." + grep "pipelinecomponents/yamllint:${YAMLLINT_VERSION}" ./.github/workflows/yamllint.yml diff --git a/.github/workflows/dive.yml b/.github/workflows/dive.yml index 7630ca1..5c5e7fc 100644 --- a/.github/workflows/dive.yml +++ b/.github/workflows/dive.yml @@ -1,6 +1,6 @@ --- name: Dive CI -on: [push] +'on': push jobs: dive: runs-on: ubuntu-latest diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index dffabe4..2006b35 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,6 +1,6 @@ --- name: Docker -on: +'on': push: tags: - '*' diff --git a/.github/workflows/dockle.yml b/.github/workflows/dockle.yml index 990279f..6a288ee 100644 --- a/.github/workflows/dockle.yml +++ b/.github/workflows/dockle.yml @@ -1,6 +1,6 @@ --- name: Dockle -on: [push] +'on': push jobs: dive: runs-on: ubuntu-latest diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 6438417..58cc987 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -1,27 +1,48 @@ --- name: Go -on: [push] +'on': push jobs: build: - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - - os: macos-10.15 - shasum: shasum -a 512 - - os: ubuntu-20.04 - shasum: sha512sum + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: Set up Go uses: actions/setup-go@v2 with: go-version: 1.19.0 + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + - uses: actions/setup-node@v3 + with: + node-version: 14 - name: Unit tests - run: go test ./... -cover - if: ${{ startsWith(matrix.os, 'ubuntu') }} + timeout-minutes: 15 + run: | + go test -short -cover -v -coverprofile=coverage.txt \ + -covermode=atomic ./... + - uses: codecov/codecov-action@v1 + with: + files: ./coverage.txt + flags: unittests + verbose: true + - name: SonarCloud Scan + uses: sonarsource/sonarcloud-github-action@master + with: + args: > + -Dsonar.organization=030-github + -Dsonar.projectKey=030_yaam + -Dsonar.exclusions=internal/goswagger/**,test/gradle/demo/**,test/npm/demo/** + -Dsonar.sources=. + -Dsonar.coverage.exclusions=**/*_test.go,internal/goswagger/**/*,test/gradle/demo/**,test/npm/demo/** + -Dsonar.verbose=true + -Dsonar.go.coverage.reportPaths="coverage.txt" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - name: Set YAAM deliverable environment variable - run: echo "yaam-deliverable=yaam-${{ matrix.os }}" >> $GITHUB_ENV + run: echo "yaam-deliverable=yaam-ubuntu-20.04" >> $GITHUB_ENV - name: Use the value run: | echo "${{ env.yaam-deliverable }}" @@ -29,13 +50,11 @@ jobs: run: ./scripts/build.sh env: YAAM_DELIVERABLE: ${{ env.yaam-deliverable }} - SHA512_CMD: ${{ matrix.shasum }} + SHA512_CMD: sha512sum - name: Quality run: | docker run --rm -v ${PWD}:/data markdownlint/markdownlint:0.11.0 \ README.md -s /data/configs/.markdownlint.rb - docker run --rm -v $(pwd):/data cytopia/yamllint:1.26-0.8 . docker run --rm -v $(pwd):/app -w /app -e GOFLAGS=-buildvcs=false \ - golangci/golangci-lint:v1.48.0-alpine golangci-lint run -v \ + golangci/golangci-lint:v1.49.0-alpine golangci-lint run -v \ --timeout 2m30s - if: ${{ startsWith(matrix.os, 'ubuntu') }} diff --git a/.github/workflows/gosec.yml b/.github/workflows/gosec.yml index 83b101f..51f5e7d 100644 --- a/.github/workflows/gosec.yml +++ b/.github/workflows/gosec.yml @@ -1,6 +1,6 @@ --- name: Run Gosec -on: +'on': push: branches: - main @@ -11,7 +11,7 @@ jobs: tests: runs-on: ubuntu-latest env: - GO111MODULE: on + GO111MODULE: 'on' steps: - name: Checkout Source uses: actions/checkout@v2 diff --git a/.github/workflows/hadolint.yml b/.github/workflows/hadolint.yml index b70ef87..d52368c 100644 --- a/.github/workflows/hadolint.yml +++ b/.github/workflows/hadolint.yml @@ -1,5 +1,6 @@ +--- name: Hadolint -on: [push] +'on': push jobs: dive: runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9ccd1ef..55aee00 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,20 +1,13 @@ --- name: Release -on: +'on': push: tags: - '*' jobs: release: name: Create Release - runs-on: ${{ matrix.os }} - strategy: - matrix: - include: - - os: macos-10.15 - shasum: shasum -a 512 - - os: ubuntu-20.04 - shasum: sha512sum + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: Set up Go @@ -22,9 +15,7 @@ jobs: with: go-version: 1.19.0 - name: Set YAAM deliverable environment variable - run: echo "yaam-deliverable=yaam-${{ matrix.os }}" >> $GITHUB_ENV - if: | - ${{ startsWith(matrix.os, 'mac') || startsWith(matrix.os, 'ubuntu') }} + run: echo "yaam-deliverable=yaam-ubuntu-20.04" >> $GITHUB_ENV - name: Use the value run: | echo "${{ env.yaam-deliverable }}" @@ -34,27 +25,12 @@ jobs: echo "Version: ${version}" echo "Checking README.md..." grep "yaam:${version}" docs/usage/DOCKER.md - # yamllint disable rule:line-length - if: ${{ startsWith(matrix.os, 'mac') || startsWith(matrix.os, 'ubuntu') }} - name: Create release run: ./scripts/build.sh env: YAAM_DELIVERABLE: ${{ env.yaam-deliverable }} GITHUB_TAG: ${{ github.ref }} - SHA512_CMD: ${{ matrix.shasum }} - if: ${{ startsWith(matrix.os, 'mac') || startsWith(matrix.os, 'ubuntu') }} - - name: Create release windows - shell: cmd - run: | - echo "GITHUB_TAG: '${{ github.ref }}'" - echo "YAAM_DELIVERABLE: '${{ env.yaam-deliverable }}'" - cd cmd/yaam - go build -buildvcs=false -ldflags "-X main.Version=${{ github.ref }}" -o "${{ env.yaam-deliverable }}" - sha512sum "${{ env.yaam-deliverable }}" > "${{ env.yaam-deliverable }}.sha512.txt" - chmod +x "${{ env.yaam-deliverable }}" - ls yaam-windows-2019 - if: ${{ startsWith(matrix.os, 'windows') }} - # yamllint enable rule:line-length + SHA512_CMD: sha512sum - name: Upload binaries to release uses: svenstaro/upload-release-action@v2 with: diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml deleted file mode 100644 index 3f24794..0000000 --- a/.github/workflows/sonarcloud.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -name: SonarCloud -on: - push: - branches: - - main - pull_request: - types: [opened, synchronize, reopened] -jobs: - sonarcloud: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.19.0 - - name: Unit test - run: | - go test -short -cover -v -coverprofile=coverage.txt \ - -covermode=atomic ./... - - name: SonarCloud Scan - uses: sonarsource/sonarcloud-github-action@master - with: - args: > - -Dsonar.organization=030-github - -Dsonar.projectKey=030_yaam - -Dsonar.exclusions=internal/goswagger/** - -Dsonar.sources=. - -Dsonar.coverage.exclusions=**/*_test.go,internal/goswagger/**/*,cmd/**/* - -Dsonar.verbose=true - -Dsonar.go.coverage.reportPaths="coverage.txt" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 1a59359..bbc9c2f 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -1,6 +1,6 @@ --- name: Trivy -on: [push] +'on': push jobs: build: name: Build @@ -11,12 +11,23 @@ jobs: - name: Build an image from Dockerfile run: | docker build -t utrecht/yaam:${{ github.sha }} . + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@7b7aa264d83dc58691451798b4d117d53d21edfe + with: + image-ref: 'utrecht/yaam:${{ github.sha }}' + format: sarif + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: 'trivy-results.sarif' - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master with: - image-ref: "utrecht/yaam:${{ github.sha }}" - format: "table" - exit-code: "1" + image-ref: 'utrecht/yaam:${{ github.sha }}' + format: 'table' + exit-code: '1' ignore-unfixed: true - vuln-type: "os,library" - severity: "CRITICAL,HIGH" + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH' diff --git a/.github/workflows/yamllint.yml b/.github/workflows/yamllint.yml new file mode 100644 index 0000000..5059962 --- /dev/null +++ b/.github/workflows/yamllint.yml @@ -0,0 +1,16 @@ +--- +name: Yamllint +'on': push +jobs: + yamllint: + runs-on: ubuntu-latest + container: + image: pipelinecomponents/yamllint:0.20.7 + env: + YAMLLINT_CONFIG_FILE: /code/configs/.yamllint.yaml + options: --cpus 1 + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: run yamllint + run: yamllint . diff --git a/.gitignore b/.gitignore index 6f9fed9..80aac7e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,10 @@ cmd/yaam/yaam + +test/gradle/demo/.gradle +test/gradle/demo/build +test/gradle/demo/build.gradle +test/gradle/demo/settings.gradle + +test/npm/demo/node_modules +test/npm/demo/.npmrc +test/npm/demo/package-lock.json diff --git a/Dockerfile b/Dockerfile index 4973ed6..6a49ed1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.0-alpine3.16 as builder +FROM golang:1.19.1-alpine3.16 as builder ARG VERSION ENV USERNAME=yaam ENV BASE=/opt/${USERNAME} diff --git a/README.md b/README.md index cb14865..8b1ba9b 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=030_yaam&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=030_yaam) [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=030_yaam&metric=security_rating)](https://sonarcloud.io/dashboard?id=030_yaam) [![Technical Debt](https://sonarcloud.io/api/project_badges/measure?project=030_yaam&metric=sqale_index)](https://sonarcloud.io/dashboard?id=030_yaam) +[![codecov](https://codecov.io/gh/030/yaam/branch/main/graph/badge.svg)](https://codecov.io/gh/030/yaam) [![BCH compliance](https://bettercodehub.com/edge/badge/030/yaam?branch=main)](https://bettercodehub.com/results/030/yaam) [![GolangCI](https://golangci.com/badges/github.com/golangci/golangci-web.svg)](https://golangci.com/r/github.com/030/yaam) [![codebeat badge](https://codebeat.co/badges/af6b1a01-df2c-40e7-bfb1-13ec0bb90087)](https://codebeat.co/projects/github-com-030-yaam-main) diff --git a/cmd/yaam/main.go b/cmd/yaam/main.go index d3c2a13..f269f3e 100644 --- a/cmd/yaam/main.go +++ b/cmd/yaam/main.go @@ -1,58 +1,133 @@ package main import ( + "fmt" + "io" "net/http" "os" "strconv" + "time" "github.com/030/yaam/internal/api" - "github.com/030/yaam/pkg/artifact" - "github.com/030/yaam/pkg/artifact/maven" + "github.com/030/yaam/internal/artifact" + "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) -const port = 25213 +const ( + serverLogMsg = "check the server logs" + port = 25213 +) var Version string -func httpInternalServerErrorReadTheLogs(w http.ResponseWriter) { - http.Error(w, "check the server logs", http.StatusInternalServerError) +func httpNotFoundReadTheLogs(w http.ResponseWriter, err error) { + log.Error(err) + http.Error(w, serverLogMsg, http.StatusNotFound) +} + +func httpInternalServerErrorReadTheLogs(w http.ResponseWriter, err error) { + log.Error(err) + http.Error(w, serverLogMsg, http.StatusInternalServerError) +} + +func mavenArtifact(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := r.Body.Close(); err != nil { + panic(err) + } + }() + + if err := api.Validation(r.Method, r, w); err != nil { + httpInternalServerErrorReadTheLogs(w, err) + return + } + + m := artifact.Maven{RequestBody: r.Body, RequestURI: r.RequestURI, ResponseWriter: w} + if r.Method == "PUT" { + var p artifact.Publisher = m + if err := p.Publish(); err != nil { + httpInternalServerErrorReadTheLogs(w, err) + return + } + return + } + + var ap artifact.Preserver = m + if err := ap.Preserve(); err != nil { + httpNotFoundReadTheLogs(w, fmt.Errorf("maven artifact caching failed. Error: '%v'", err)) + return + } + + var ar artifact.Reader = m + if err := ar.Read(); err != nil { + httpNotFoundReadTheLogs(w, fmt.Errorf("cannot read artifact from disk. Error: '%v'. Perhaps it resides in another repository?", err)) + return + } } -func handler(w http.ResponseWriter, r *http.Request) { +func mavenGroup(w http.ResponseWriter, r *http.Request) { defer func() { if err := r.Body.Close(); err != nil { panic(err) } }() - method := r.Method - reqURL := r.URL + if err := api.Validation(r.Method, r, w); err != nil { + httpInternalServerErrorReadTheLogs(w, err) + return + } - if err := api.Validation(method, r, w); err != nil { - log.Errorf("request is invalid. Error: '%v'", err) - httpInternalServerErrorReadTheLogs(w) + vars := mux.Vars(r) + artifactURI := vars["artifact"] + groupName := vars["name"] + log.Debugf("Group: %v, Artifact: %v", groupName, artifactURI) + var p artifact.Unifier = artifact.Maven{ResponseWriter: w, RequestURI: artifactURI} + if err := p.Unify(groupName); err != nil { + log.Error(fmt.Errorf("grouping failed. Error: '%v'", err)) + http.Error(w, serverLogMsg, http.StatusInternalServerError) return } +} - if method == "POST" { - if err := artifact.Publish(r); err != nil { - log.Errorf("publish of an artifact failed. Error: '%v'", err) - httpInternalServerErrorReadTheLogs(w) +func npmArtifact(w http.ResponseWriter, r *http.Request) { + defer func() { + if err := r.Body.Close(); err != nil { + panic(err) + } + }() + + if err := api.Validation(r.Method, r, w); err != nil { + httpInternalServerErrorReadTheLogs(w, err) + return + } + + n := artifact.Maven{RequestBody: r.Body, RequestURI: r.RequestURI, ResponseWriter: w} + if r.Method == "POST" { + var p artifact.Publisher = n + if err := p.Publish(); err != nil { + httpInternalServerErrorReadTheLogs(w, err) return } return } - if err := maven.Cache(w, reqURL); err != nil { - log.Errorf("maven artifact caching failed. Error: '%v'", err) - httpInternalServerErrorReadTheLogs(w) + var ap artifact.Preserver = n + if err := ap.Preserve(); err != nil { + httpNotFoundReadTheLogs(w, err) + return + } + + var ar artifact.Reader = n + if err := ar.Read(); err != nil { + httpNotFoundReadTheLogs(w, err) return } +} - if err := artifact.ReadFromDisk(w, r); err != nil { - log.Warnf("cannot read artifact from disk. Error: '%v'. Perhaps it resides in another repository?", err) - http.Error(w, "check the server logs", http.StatusNotFound) +func status(w http.ResponseWriter, r *http.Request) { + if _, err := io.WriteString(w, "ok"); err != nil { + httpNotFoundReadTheLogs(w, err) return } } @@ -68,10 +143,23 @@ func main() { log.SetLevel(log.DebugLevel) } - http.HandleFunc("/", handler) + r := mux.NewRouter() + r.HandleFunc("/maven/groups/{name}/{artifact:.*}", mavenGroup) + r.HandleFunc("/maven/{repo}/{artifact:.*}", mavenArtifact) + r.HandleFunc("/npm/{repo}/{artifact:.*}", npmArtifact) + r.HandleFunc("/status", status) + + srv := &http.Server{ + Addr: "0.0.0.0:" + strconv.Itoa(port), + // Good practice to set timeouts to avoid Slowloris attacks. + WriteTimeout: time.Second * 120, + ReadTimeout: time.Second * 180, + IdleTimeout: time.Second * 240, + Handler: r, // Pass our instance of gorilla/mux in. + } log.Infof("Starting YAAM version: '%s' on localhost on port: '%d'...", Version, port) - if err := http.ListenAndServe(":"+strconv.Itoa(port), nil); err != nil { + if err := srv.ListenAndServe(); err != nil { log.Fatal(err) } } diff --git a/cmd/yaam/main_test.go b/cmd/yaam/main_test.go new file mode 100644 index 0000000..d604a80 --- /dev/null +++ b/cmd/yaam/main_test.go @@ -0,0 +1,353 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/tj/assert" +) + +const ( + allowedReposMaven = `allowedRepos: + - releases` + allowedReposNpm = `allowedRepos: + - 3rdparty-npm` + gradleHomeDemoProject = "../../test/gradle/demo" + npmHomeDemoProject = "../../test/npm/demo" + + npmrc = `registry=http://localhost:25213/npm/3rdparty-npm/ +always-auth=true +_auth=aGVsbG86d29ybGQ=` + + caches = `mavenReposAndUrls: + 3rdparty-maven: https://repo.maven.apache.org/maven2/ + 3rdparty-maven-gradle-plugins: https://plugins.gradle.org/m2/ + 3rdparty-maven-spring: https://repo.spring.io/release/ + 3rdparty-npm: https://registry.npmjs.org/` + groups = `groups: + hello: + - maven/releases + - maven/3rdparty-maven + - maven/3rdparty-maven-gradle-plugins + - maven/3rdparty-maven-spring` + cmdExitErrMsg = "%v, err: '%v'" + mavenReleasesRepo = "maven/releases" +) + +var ( + mavenRepos = []string{"maven/3rdparty-maven", "maven/3rdparty-maven-gradle-plugins", "maven/3rdparty-maven-spring", mavenReleasesRepo} +) + +func testConfigHelper() error { + os.Setenv("YAAM_HOME", filepath.Join("/tmp", "yaam", "test"+time.Now().Format("20060102150405111"))) + dir := filepath.Join(os.Getenv("YAAM_HOME"), "conf") + reposDir := filepath.Join(dir, "repositories") + if err := os.MkdirAll(reposDir, os.ModePerm); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, "caches.yaml"), []byte(caches), 0600); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, "groups.yaml"), []byte(groups), 0600); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(reposDir, "maven.yaml"), []byte(allowedReposMaven), 0600); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(reposDir, "npm.yaml"), []byte(allowedReposNpm), 0600); err != nil { + return err + } + + os.Setenv("YAAM_DEBUG", "true") + os.Setenv("YAAM_USER", "hello") + + return nil +} + +func testNpmConfigHelper() (int, error) { + os.Setenv("YAAM_PASS", "world") + + if err := os.RemoveAll(filepath.Join(npmHomeDemoProject, "node_modules")); err != nil { + return 1, err + } + packageLockJson := filepath.Join(npmHomeDemoProject, "package-lock.json") + if _, err := os.Stat(packageLockJson); err == nil { + if err := os.Remove(packageLockJson); err != nil { + return 1, err + } + } + + npmrcWithCacheLocation := npmrc + ` +cache=/tmp/yaam/test/npm/cache` + time.Now().Format("20060102150405111") + `` + if err := os.WriteFile(filepath.Join(npmHomeDemoProject, ".npmrc"), []byte(npmrcWithCacheLocation), 0600); err != nil { + return 1, err + } + + cmd := exec.Command("bash", "-c", "npm cache clean --force && npm i") + cmd.Dir = npmHomeDemoProject + co, err := cmd.CombinedOutput() + if err != nil { + return cmd.ProcessState.ExitCode(), fmt.Errorf(cmdExitErrMsg, string(co), err) + } + + return 0, nil +} + +func init() { + if err := testConfigHelper(); err != nil { + panic(err) + } + + go main() +} + +func testMainGradleFile(content []byte, name string) error { + if err := os.WriteFile(filepath.Join(gradleHomeDemoProject, name+".gradle"), content, 0600); err != nil { + return err + } + return nil +} + +func testMainGradleCleanBuildHelper(pass string) (int, error) { + os.Setenv("YAAM_PASS", pass) + + os.Setenv("GRADLE_USER_HOME", "/tmp/yaam/test/gradle"+time.Now().Format("20060102150405111")) + cmd := exec.Command("bash", "-c", "./gradlew clean build --no-daemon") + cmd.Dir = gradleHomeDemoProject + co, err := cmd.CombinedOutput() + if err != nil { + return cmd.ProcessState.ExitCode(), fmt.Errorf(cmdExitErrMsg, string(co), err) + } + + return 0, nil +} + +func testMainGradlePublishHelper(repo string) (int, error) { + if err := testGradleBuildFileHelper(mavenRepos, repo); err != nil { + return 1, err + } + if err := testGradleSettingsFileHelper(mavenRepos); err != nil { + return 1, err + } + + exitCode, err := testMainGradleCleanBuildHelper("world") + if err != nil { + return exitCode, err + } + + cmd := exec.Command("bash", "-c", "./gradlew publish --no-daemon") + cmd.Dir = gradleHomeDemoProject + co, err := cmd.CombinedOutput() + if err != nil { + return cmd.ProcessState.ExitCode(), fmt.Errorf(cmdExitErrMsg, string(co), err) + } + + return 0, nil +} + +func testGradleMavenRepositoriesFileHelper(repos []string) string { + var sb strings.Builder + for _, repo := range repos { + content := ` + maven { + allowInsecureProtocol true + url 'http://localhost:25213/` + repo + `/' + authentication { + basic(BasicAuthentication) + } + credentials { + username "hello" + password "world" + } + }` + sb.WriteString(content) + } + return sb.String() +} + +func testGradlePublishingFileHelper(repo string) string { + repos := []string{repo} + content := ` +publishing { + publications { + mavenJava(MavenPublication) { + versionMapping { + usage('java-api') { + fromResolutionOf('runtimeClasspath') + } + usage('java-runtime') { + fromResolutionResult() + } + } + } + } + + repositories {` + + testGradleMavenRepositoriesFileHelper(repos) + ` + } +} +` + return content +} + +func testGradleBuildFileHelper(repos []string, repoPublish string) error { + content := ` +plugins { + id 'org.springframework.boot' version '2.7.3' + id 'io.spring.dependency-management' version '1.0.13.RELEASE' + id 'java' + id 'maven-publish' +} + +group = 'com.example' +version = '0.0.1-SNAPSHOT' +sourceCompatibility = '17' + +repositories {` + + testGradleMavenRepositoriesFileHelper(repos) + ` +} + +` + testGradlePublishingFileHelper(repoPublish) + ` + +dependencies { + implementation 'org.springframework.boot:spring-boot-starter' + testImplementation 'org.springframework.boot:spring-boot-starter-test' +} + +tasks.named('test') { + useJUnitPlatform() +} +` + + if err := testMainGradleFile([]byte(content), "build"); err != nil { + return err + } + return nil +} + +func testGradleSettingsFileHelper(repos []string) error { + content := ` +pluginManagement { + repositories {` + + testGradleMavenRepositoriesFileHelper(repos) + ` + } +} + +rootProject.name = 'demo' +` + + if err := testMainGradleFile([]byte(content), "settings"); err != nil { + return err + } + return nil +} + +func TestMainNpmBuild(t *testing.T) { + exitCode, err := testNpmConfigHelper() + if err != nil { + t.Error(err) + return + } + + assert.NoError(t, err) + assert.Equal(t, 0, exitCode) +} + +func TestMainGradleCleanBuild(t *testing.T) { + if err := testGradleBuildFileHelper(mavenRepos, mavenReleasesRepo); err != nil { + t.Error(err) + } + if err := testGradleSettingsFileHelper(mavenRepos); err != nil { + t.Error(err) + } + + exitCode, err := testMainGradleCleanBuildHelper("world") + if err != nil { + t.Error(err) + } + + assert.NoError(t, err) + assert.Equal(t, 0, exitCode) +} + +func TestMainGradleCleanBuildFail(t *testing.T) { + if err := testConfigHelper(); err != nil { + t.Error(err) + } + + if err := testGradleBuildFileHelper(mavenRepos, mavenReleasesRepo); err != nil { + t.Error(err) + } + if err := testGradleSettingsFileHelper(mavenRepos); err != nil { + t.Error(err) + } + + exitCode, err := testMainGradleCleanBuildHelper("incorrectPass") + + assert.Regexp(t, "was not found in any of the following sources", err) + assert.Equal(t, 1, exitCode) +} + +func TestMainGradleCleanBuildNonMavenFail(t *testing.T) { + repos := []string{"3rdparty-maven", "3rdparty-maven-gradle-plugins", "3rdparty-maven-spring", "releases"} + if err := testGradleBuildFileHelper(repos, "releases"); err != nil { + t.Error(err) + } + if err := testGradleSettingsFileHelper(repos); err != nil { + t.Error(err) + } + + exitCode, err := testMainGradleCleanBuildHelper("world") + + assert.Regexp(t, "was not found in any of the following sources", err) + assert.Equal(t, 1, exitCode) +} + +func TestMainGradlePublish(t *testing.T) { + exitCode, err := testMainGradlePublishHelper(mavenReleasesRepo) + + assert.NoError(t, err) + assert.Equal(t, 0, exitCode) +} + +func TestMainGradlePublishFail(t *testing.T) { + exitCode, err := testMainGradlePublishHelper("maven/releases-non-existent") + + assert.Regexp(t, `Could not PUT 'http://localhost:25213/maven/releases-non-existent/com/example/demo/0.0.1-SNAPSHOT/maven-metadata.xml'. Received status code 500 from server: Internal Server Error`, err) + assert.Equal(t, 1, exitCode) +} + +func TestMainGradleCleanBuildGroup(t *testing.T) { + repos := []string{"maven/groups/hello"} + if err := testGradleBuildFileHelper(repos, mavenReleasesRepo); err != nil { + t.Error(err) + } + if err := testGradleSettingsFileHelper(repos); err != nil { + t.Error(err) + } + + exitCode, err := testMainGradleCleanBuildHelper("world") + + assert.NoError(t, err) + assert.Equal(t, 0, exitCode) +} + +func TestMainGradleCleanBuildGroupFail(t *testing.T) { + repos := []string{"maven/groups/helloworld"} + if err := testGradleBuildFileHelper(repos, mavenReleasesRepo); err != nil { + t.Error(err) + } + if err := testGradleSettingsFileHelper(mavenRepos); err != nil { + t.Error(err) + } + + exitCode, err := testMainGradleCleanBuildHelper("world") + + assert.Regexp(t, `Could not GET 'http://localhost:25213/maven/groups/helloworld/.*'. Received status code 500 from server: Internal Server Error`, err) + assert.Equal(t, 1, exitCode) +} diff --git a/configs/.yamllint.yaml b/configs/.yamllint.yaml new file mode 100644 index 0000000..807204c --- /dev/null +++ b/configs/.yamllint.yaml @@ -0,0 +1,5 @@ +--- +extends: default + +ignore: | + test/npm/demo/node_modules/ diff --git a/configs/kind.yaml b/configs/kind.yaml new file mode 100644 index 0000000..876fd7d --- /dev/null +++ b/configs/kind.yaml @@ -0,0 +1,22 @@ +--- +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + system-reserved: memory=8Gi + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP + extraMounts: + - hostPath: /tmp/yaam/repositories + containerPath: /repositories diff --git a/deployments/k8s-openshift/deploy.yml b/deployments/k8s-openshift/deploy.yml index cc3bb17..4e68db4 100644 --- a/deployments/k8s-openshift/deploy.yml +++ b/deployments/k8s-openshift/deploy.yml @@ -1,13 +1,39 @@ --- apiVersion: v1 +kind: Namespace +metadata: + name: yaam +--- +apiVersion: v1 kind: ConfigMap metadata: name: conf data: - repositories.yaml: |- + caches.yaml: |- mavenReposAndUrls: - 3rdparty-maven: https://repo1.maven.org/maven2/ + 3rdparty-maven: https://repo.maven.apache.org/maven2/ 3rdparty-maven-gradle-plugins: https://plugins.gradle.org/m2/ + 3rdparty-maven-spring: https://repo.spring.io/release/ + 3rdparty-npm: https://registry.npmjs.org/ + groups.yaml: |- + groups: + hello: + - maven/releases + - maven/3rdparty-maven + - maven/3rdparty-maven-gradle-plugins + - maven/3rdparty-maven-spring +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: conf-repos +data: + maven.yaml: |- + allowRepos: + - releases + npm.yaml: |- + allowRepos: + - 3rdparty-npm --- apiVersion: v1 data: @@ -44,7 +70,6 @@ spec: ports: - port: 25213 name: yaam - clusterIP: None selector: app: yaam --- @@ -53,15 +78,21 @@ kind: HorizontalPodAutoscaler metadata: name: yaam spec: - maxReplicas: 10 + maxReplicas: 20 metrics: - resource: name: cpu target: - averageUtilization: 80 + averageUtilization: 500 type: Utilization type: Resource - minReplicas: 1 + - resource: + name: memory + target: + averageUtilization: 1500 + type: Utilization + type: Resource + minReplicas: 2 scaleTargetRef: apiVersion: apps/v1 kind: StatefulSet @@ -87,6 +118,10 @@ spec: containers: - name: yaam env: + - name: YAAM_DEBUG + value: 'true' + - name: YAAM_HOST + value: yaam.some-domain - name: YAAM_USER valueFrom: secretKeyRef: @@ -97,35 +132,64 @@ spec: secretKeyRef: name: creds key: pass - image: utrecht/yaam:v0.2.1 - readinessProbe: - tcpSocket: - port: 25213 - initialDelaySeconds: 3 - periodSeconds: 2 + image: utrecht/yaam:v0.3.0 livenessProbe: - tcpSocket: + httpGet: + path: /status + port: 25213 + readinessProbe: + httpGet: + path: /status port: 25213 - initialDelaySeconds: 9 - periodSeconds: 4 resources: limits: - memory: 16Mi - cpu: 50m + cpu: 100m + memory: 50Mi requests: - memory: 1Mi cpu: 1m + memory: 1Mi ports: - containerPort: 25213 name: yaam volumeMounts: - name: conf mountPath: /opt/yaam/.yaam/conf + - name: conf-repos + mountPath: /opt/yaam/.yaam/conf/repositories - name: repositories mountPath: /opt/yaam/.yaam/repositories volumes: - name: conf configMap: name: conf + - name: conf-repos + configMap: + name: conf-repos - name: repositories - emptyDir: {} + persistentVolumeClaim: + claimName: repositories +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: repositories +spec: + storageClassName: standard + accessModes: + - ReadWriteOnce + capacity: + storage: 2Gi + hostPath: + path: /repositories/ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: repositories +spec: + volumeName: repositories + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 2d82fca..cb287b3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [v0.3.0] - 2022-09-14 + +### Added + +- integration tests by running gradle commands in a demo project. +- grouping of Maven repositories. +- publishing Maven artifacts. +- NPM and Maven caching. + +### Changed + +- http to mux. + ## [v0.2.1] - 2022-08-23 ### Added @@ -45,7 +58,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Cache all Maven2 artifacts that are required by a gradle project locally. -[Unreleased]: https://github.com/030/yaam/compare/v0.2.1...HEAD +[Unreleased]: https://github.com/030/yaam/compare/v0.3.0...HEAD +[v0.3.0]: https://github.com/030/yaam/compare/v0.2.1...v0.3.0 [v0.2.1]: https://github.com/030/yaam/compare/v0.2.0...v0.2.1 [v0.2.0]: https://github.com/030/yaam/compare/v0.1.0...v0.2.0 [v0.1.0]: https://github.com/030/yaam/releases/tag/v0.1.0 diff --git a/docs/SECURITY.md b/docs/SECURITY.md new file mode 100644 index 0000000..f765cb6 --- /dev/null +++ b/docs/SECURITY.md @@ -0,0 +1,19 @@ +# Security Policy + +## Supported Versions + +Use this section to tell people about which versions of your project are +currently being supported with security updates. + +| Version | Supported | +| ------- | ------------------ | +| 0.3.x | :white_check_mark: | +| < 0.3.0 | :x: | + +## Reporting a Vulnerability + +Use this section to tell people how to report a vulnerability. + +Tell them where to go, how often they can expect to get an update on a +reported vulnerability, what to expect if the vulnerability is accepted or +declined, etc. diff --git a/docs/usage/CONFIG.md b/docs/usage/CONFIG.md index 15f118a..5d38a78 100644 --- a/docs/usage/CONFIG.md +++ b/docs/usage/CONFIG.md @@ -1,10 +1,37 @@ # Config ```bash -mkdir ~/.yaam +mkdir -p ~/.yaam/conf chown 9999 -R ~/.yaam/ ``` +vim ~/.yaam/conf/caches.yaml + +```bash +mavenReposAndUrls: + 3rdparty-maven: https://repo.maven.apache.org/maven2/ + 3rdparty-maven-gradle-plugins: https://plugins.gradle.org/m2/ + 3rdparty-maven-spring: https://repo.spring.io/release/ +``` + +vim ~/.yaam/conf/groups.yaml + +```bash +groups: + hello: + - releases + - 3rdparty-maven + - 3rdparty-maven-gradle-plugins + - 3rdparty-maven-spring +``` + +vim ~/.yaam/conf/repositories.yaml + +```bash +maven: + - releases +``` + ## Gradle Adjust the `build.gradle` and/or `settings.gradle`: @@ -13,7 +40,7 @@ Adjust the `build.gradle` and/or `settings.gradle`: repositories { maven { allowInsecureProtocol true - url 'http://localhost:25213/releases/' + url 'http://localhost:25213/maven/releases/' authentication { basic(BasicAuthentication) } @@ -24,7 +51,7 @@ repositories { } maven { allowInsecureProtocol true - url 'http://localhost:25213/3rdparty-maven/' + url 'http://localhost:25213/maven/3rdparty-maven/' authentication { basic(BasicAuthentication) } @@ -35,7 +62,7 @@ repositories { } maven { allowInsecureProtocol true - url 'http://localhost:25213/3rdparty-maven-gradle-plugins/' + url 'http://localhost:25213/maven/3rdparty-maven-gradle-plugins/' authentication { basic(BasicAuthentication) } diff --git a/docs/usage/DOCKER.md b/docs/usage/DOCKER.md index fd85ffc..2d91309 100644 --- a/docs/usage/DOCKER.md +++ b/docs/usage/DOCKER.md @@ -9,5 +9,5 @@ docker run \ -e YAAM_USER=hello \ -e YAAM_PASS=world \ -p 25213:25213 \ - -it utrecht/yaam:v0.2.1 + -it utrecht/yaam:v0.3.0 ``` diff --git a/docs/usage/PUBLISH.md b/docs/usage/PUBLISH.md index 5d0e037..4a0f658 100644 --- a/docs/usage/PUBLISH.md +++ b/docs/usage/PUBLISH.md @@ -1,5 +1,5 @@ # Publish ```bash -curl -X POST localhost:25213/hello/world/hola.yaml --data-binary @blahblah.json +curl -X PUT localhost:25213/hello/world/hola.yaml --data-binary @blahblah.json ``` diff --git a/go.mod b/go.mod index 70d09d3..9d7c56f 100644 --- a/go.mod +++ b/go.mod @@ -3,24 +3,35 @@ module github.com/030/yaam go 1.19 require ( + github.com/gorilla/mux v1.8.0 + github.com/hashicorp/go-retryablehttp v0.7.1 github.com/mitchellh/go-homedir v1.1.0 github.com/sirupsen/logrus v1.9.0 github.com/spf13/viper v1.12.0 + github.com/tidwall/gjson v1.14.3 + github.com/tj/assert v0.0.3 + golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 ) require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.3 // indirect + github.com/pelletier/go-toml/v2 v2.0.5 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/afero v1.9.2 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/subosito/gotenv v1.4.0 // indirect - golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 // indirect + github.com/stretchr/testify v1.8.0 // indirect + github.com/subosito/gotenv v1.4.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect golang.org/x/text v0.3.7 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index db1d996..6fe4ecd 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= @@ -116,6 +117,15 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= +github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= +github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -133,14 +143,16 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.3 h1:h9JoA60e1dVEOpp0PFwJSmt1Htu057NUq9/bUwaO61s= -github.com/pelletier/go-toml/v2 v2.0.3/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= +github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= +github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -165,12 +177,21 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs= -github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7vWWGaGo= +github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= +github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= +github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -198,6 +219,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E= +golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -307,8 +330,8 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 h1:Sx/u41w+OwrInGdEckYmEuU5gHoGSL4QbDz3S9s6j4U= -golang.org/x/sys v0.0.0-20220818161305-2296e01440c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -470,6 +493,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/api/api.go b/internal/api/api.go index fba91b4..994ab4e 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -4,10 +4,13 @@ import ( "fmt" "net/http" "os" + + log "github.com/sirupsen/logrus" ) func basicAuth(r *http.Request) error { u, p, ok := r.BasicAuth() + log.Debugf("user: '%s', pass: '********', basicAuthUsed?: '%t'", u, ok) if ok { if !(u == os.Getenv("YAAM_USER") && p == os.Getenv("YAAM_PASS")) { return fmt.Errorf("auth failed") @@ -15,16 +18,18 @@ func basicAuth(r *http.Request) error { } else { return fmt.Errorf("request is NOT using basic authentication") } + return nil } func Validation(method string, r *http.Request, w http.ResponseWriter) error { - if !(method == "POST" || method == "GET" || method == "HEAD") { - return fmt.Errorf("only POSTs, GETs and HEADs are supported. Method: '%s'", method) + if !(method == "PUT" || method == "POST" || method == "GET" || method == "HEAD") { + return fmt.Errorf("only PUTs, POSTs, GETs and HEADs are supported. Used method: '%s'", method) } if err := basicAuth(r); err != nil { return fmt.Errorf("basic auth failed. Error: '%v'", err) } + return nil } diff --git a/internal/api/api_test.go b/internal/api/api_test.go new file mode 100644 index 0000000..778f64e --- /dev/null +++ b/internal/api/api_test.go @@ -0,0 +1 @@ +package api diff --git a/internal/artifact/artifact.go b/internal/artifact/artifact.go new file mode 100644 index 0000000..5ea8334 --- /dev/null +++ b/internal/artifact/artifact.go @@ -0,0 +1,37 @@ +package artifact + +import ( + "fmt" + "path/filepath" + + "github.com/030/yaam/internal/pkg/project" + "github.com/spf13/viper" +) + +type artefact struct { + path, url string +} + +func allowedRepos(name string) ([]string, error) { + h, err := project.Home() + if err != nil { + return []string{}, err + } + + viper.SetConfigName("groups") + viper.SetConfigType("yaml") + viper.AddConfigPath(filepath.Join(h, "conf")) + if err := viper.ReadInConfig(); err != nil { + return []string{}, err + } + + groups := viper.GetStringMapStringSlice("groups") + var repos []string + if values, ok := groups[name]; ok { + repos = values + } else { + return []string{}, fmt.Errorf("group: '%s' not found in config file", name) + } + + return repos, nil +} diff --git a/internal/artifact/artifact_test.go b/internal/artifact/artifact_test.go new file mode 100644 index 0000000..2e85109 --- /dev/null +++ b/internal/artifact/artifact_test.go @@ -0,0 +1,49 @@ +package artifact + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/tj/assert" +) + +const ( + groups = `groups: + hello: + - maven/releases + - maven/3rdparty-maven + - maven/3rdparty-maven-gradle-plugins + - maven/3rdparty-maven-spring` +) + +func init() { + os.Setenv("YAAM_HOME", filepath.Join("/tmp", "yaam", "test"+time.Now().Format("20060102150405111"))) + dir := filepath.Join(os.Getenv("YAAM_HOME"), "conf") + reposDir := filepath.Join(dir, "repositories") + if err := os.MkdirAll(reposDir, os.ModePerm); err != nil { + panic(err) + } + if err := os.WriteFile(filepath.Join(dir, "groups.yaml"), []byte(groups), 0600); err != nil { + panic(err) + } +} + +func TestAllowedRepos(t *testing.T) { + expRepos := []string{"maven/releases", "maven/3rdparty-maven", "maven/3rdparty-maven-gradle-plugins", "maven/3rdparty-maven-spring"} + actRrepos, err := allowedRepos("hello") + if err != nil { + t.Error(err) + } + + assert.Equal(t, expRepos, actRrepos) +} + +func TestAllowedReposFail(t *testing.T) { + expRepos := []string{} + actRrepos, err := allowedRepos("world") + + assert.Equal(t, expRepos, actRrepos) + assert.EqualError(t, err, "group: 'world' not found in config file") +} diff --git a/internal/artifact/interface.go b/internal/artifact/interface.go new file mode 100644 index 0000000..5c88478 --- /dev/null +++ b/internal/artifact/interface.go @@ -0,0 +1,30 @@ +package artifact + +// Preserver is the interface that wraps the basic Preserve method. +// +// Preserve downloads an artifact from an external repository and writes it to +// disk. +type Preserver interface { + Preserve(urlStrings ...string) error +} + +// Publisher is the interface that wraps the basic Publish method. +// +// Publish writes an artifact to disk. +type Publisher interface { + Publish() error +} + +// Reader is the interface that wraps the basic Read method. +// +// Reader reads an artifact from disk. +type Reader interface { + Read() error +} + +// Unifier is the interface that wraps the basic Unify method. +// +// Unify groups multiple repositories. +type Unifier interface { + Unify(name string) error +} diff --git a/internal/artifact/maven.go b/internal/artifact/maven.go new file mode 100644 index 0000000..bd796da --- /dev/null +++ b/internal/artifact/maven.go @@ -0,0 +1,141 @@ +package artifact + +import ( + "fmt" + "io" + "net/http" + "path/filepath" + "reflect" + + "github.com/030/yaam/internal/pkg/artifact" + "github.com/030/yaam/internal/pkg/file" + "github.com/030/yaam/internal/pkg/project" + log "github.com/sirupsen/logrus" +) + +type Maven struct { + ResponseWriter http.ResponseWriter + RequestBody io.ReadCloser + RequestURI string +} + +func maven(url string, repoInConfigFile artifact.PublicRepository) (artefact, error) { + h, err := project.RepositoriesHome() + if err != nil { + return artefact{}, err + } + + if err := artifact.Dir(url); err != nil { + return artefact{}, err + } + + du, err := artifact.DownloadUrl(repoInConfigFile.Url, repoInConfigFile.Regex, url) + if err != nil { + return artefact{}, err + } + + completeFile := filepath.Join(h, url) + log.Debugf("completeFile: '%s', downloadUrl: '%s'", completeFile, du) + + return artefact{path: completeFile, url: du}, nil +} + +func (m Maven) downloadAgainIfInvalid(a artefact, resp *http.Response) error { + if resp.StatusCode == http.StatusOK { + if err := file.CreateIfDoesNotExistOrEmpty(a.url, a.path, resp.Body); err != nil { + return err + } + } + + if file.EmptyFile(a.path) { + if err := m.Preserve(); err != nil { + return err + } + } + + return nil +} + +func (m Maven) Preserve(urlStrings ...string) error { + urlString := m.RequestURI + if len(urlStrings) > 0 { + urlString = urlStrings[0] + } + log.Debugf("urlString: '%s'", urlString) + + repoInConfigFile, err := artifact.RepoInConfigFile(m.ResponseWriter, urlString) + if err != nil { + return err + } + + if !reflect.ValueOf(repoInConfigFile).IsZero() { + a, err := maven(urlString, repoInConfigFile) + if err != nil { + return err + } + + resp, err := file.DownloadWithRetries(a.url) + if err != nil { + return err + } + defer func() { + if err := resp.Body.Close(); err != nil { + panic(err) + } + }() + + if err := m.downloadAgainIfInvalid(a, resp); err != nil { + return err + } + } + + return nil +} + +func (m Maven) Publish() error { + if err := artifact.StoreOnDisk(m.RequestURI, m.RequestBody); err != nil { + return err + } + + return nil +} + +func (m Maven) Read() error { + if err := artifact.ReadFromDisk(m.ResponseWriter, m.RequestURI); err != nil { + return fmt.Errorf(file.CannotReadErrMsg, err) + } + + return nil +} + +func (m Maven) Unify(name string) error { + repos, err := allowedRepos(name) + if err != nil { + return err + } + + log.Debugf("repos: '%v'", repos) + for _, repo := range repos { + log.Debugf("repo: '%s'", repo) + urlString := "/" + repo + "/" + m.RequestURI + log.Debugf("urlString: '%s'", urlString) + + h, err := project.RepositoriesHome() + if err != nil { + return err + } + + if err := m.Preserve(urlString); err != nil { + log.Errorf("maven artifact caching failed. Error: '%v'", err) + } + + if _, fileExists := file.Exists(filepath.Join(h, urlString)); fileExists { + if err := artifact.ReadFromDisk(m.ResponseWriter, urlString); err != nil { + log.Warnf(file.CannotReadErrMsg, err) + } + return nil + } + } + + return nil +} diff --git a/internal/artifact/maven_test.go b/internal/artifact/maven_test.go new file mode 100644 index 0000000..c146142 --- /dev/null +++ b/internal/artifact/maven_test.go @@ -0,0 +1 @@ +package artifact diff --git a/internal/artifact/npm.go b/internal/artifact/npm.go new file mode 100644 index 0000000..a126edb --- /dev/null +++ b/internal/artifact/npm.go @@ -0,0 +1,215 @@ +package artifact + +import ( + "crypto/sha1" // #nosec + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "reflect" + "regexp" + "strings" + "time" + + "github.com/030/yaam/internal/pkg/artifact" + "github.com/030/yaam/internal/pkg/file" + "github.com/030/yaam/internal/pkg/project" + log "github.com/sirupsen/logrus" + "github.com/tidwall/gjson" +) + +type Npm struct { + ResponseWriter http.ResponseWriter + RequestBody io.ReadCloser + RequestURI string +} + +func replaceUrlPublicNpmWithYaamHost(f, url string) error { + if filepath.Ext(f) == ".tmp" { + input, err := os.ReadFile(filepath.Clean(f)) + if err != nil { + return err + } + + host := os.Getenv("YAAM_HOST") + if host == "" { + host = "localhost:25213" + } + output := strings.Replace(string(input), "https://registry.npmjs.org", "http://"+host+"/npm/3rdparty-npm", -1) + + var re = regexp.MustCompile(`(/@[a-z]+)(/)`) + s := re.ReplaceAllString(output, `$1%2f`) + err = os.WriteFile(f, []byte(s), 0600) + if err != nil { + return err + } + + b, err := os.ReadFile(filepath.Clean(f)) + if err != nil { + return err + } + if !json.Valid(b) { + return fmt.Errorf("json for file: '%s' is invalid", f) + } + } + return nil +} + +func npm(url string, repoInConfigFile artifact.PublicRepository) (artefact, error) { + h, err := project.RepositoriesHome() + if err != nil { + return artefact{}, err + } + dir := strings.Replace(url, "%2f", "/", -1) + log.Debugf("extension found: '%s', file: '%s'", filepath.Ext(dir), dir) + if filepath.Ext(dir) != ".tgz" { + log.Debugf("file: '%s' does not have an extension", dir) + dir = dir + ".tmp" + } + if err := artifact.Dir(dir); err != nil { + return artefact{}, err + } + + log.Debugf("downloadUrl before entering downloadUrl method: '%s', regex: '%s'", url, repoInConfigFile.Regex) + du, err := artifact.DownloadUrl(repoInConfigFile.Url, repoInConfigFile.Regex, url) + if err != nil { + return artefact{}, err + } + completeFile := filepath.Join(h, dir) + log.Debugf("completeFile: '%s', downloadUrl: '%s'", completeFile, du) + + if err := replaceUrlPublicNpmWithYaamHost(completeFile, url); err != nil { + return artefact{}, err + } + + return artefact{path: completeFile, url: du}, err +} + +func checksum(f string) (bool, error) { + checksumValid := true + _, fileExists := file.Exists(f) + if !fileExists && filepath.Ext(f) == ".tgz" { + re := regexp.MustCompile(`-([0-9]+\.[0-9]+\.[0-9]).tgz$`) + match := re.FindStringSubmatch(f) + log.Debugf("match version length: '%d' for file: '%s'", len(match), f) + version := match[1] + fmt.Println(version) + + re = regexp.MustCompile(`^(/.*/[0-9a-z-/@]+)/-.*$`) + match = re.FindStringSubmatch(f) + log.Debugf("match tmp dir length: '%d' for file: '%s'", len(match), f) + blah := match[1] + fmt.Println(blah) + + blahFile := filepath.Join(blah + ".tmp") + b, err := os.ReadFile(filepath.Clean(blahFile)) + if err != nil { + return checksumValid, err + } + + version = strings.Replace(version, ".", `\.`, -1) + value := gjson.GetBytes(b, `versions.`+version+`.dist.shasum`) + println(value.String()) + + f2, err := os.Open(filepath.Clean(blahFile)) + if err != nil { + return checksumValid, err + } + defer func() { + if err := f2.Close(); err != nil { + panic(err) + } + }() + + /* #nosec */ + h := sha1.New() + if _, err := io.Copy(h, f2); err != nil { + return checksumValid, err + } + + fmt.Printf("%x", h.Sum(nil)) + checksum := fmt.Sprintf("%x", h.Sum(nil)) + if checksum != value.String() { + log.Errorf("file: '%s' checksum on disk: '%s' does not match expected checksum: '%s'", f, checksum, value.String()) + checksumValid = false + log.Warnf(file.WaitMsg, file.RetryDuration) + time.Sleep(file.RetryDuration) + } + } + + return checksumValid, nil +} + +func (n Npm) downloadAgainIfInvalid(a artefact, resp *http.Response) error { + checksumValid, err := checksum(a.path) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusOK || !checksumValid || filepath.Ext(a.path) == ".tmp" { + if err := file.CreateIfDoesNotExistOrEmpty(a.url, a.path, resp.Body); err != nil { + return err + } + } + + if file.EmptyFile(a.path) { + if err := n.Preserve(); err != nil { + return err + } + } + + return nil +} + +func (n Npm) Preserve() error { + repoInConfigFile, err := artifact.RepoInConfigFile(n.ResponseWriter, n.RequestURI) + if err != nil { + return err + } + + if !reflect.ValueOf(repoInConfigFile).IsZero() { + a, err := npm(n.RequestURI, repoInConfigFile) + if err != nil { + return err + } + + resp, err := file.DownloadWithRetries(a.url) + if err != nil { + return err + } + defer func() { + if err := resp.Body.Close(); err != nil { + panic(err) + } + }() + + if err := n.downloadAgainIfInvalid(a, resp); err != nil { + return err + } + } + + return nil +} + +func (n Npm) Publish() error { + if err := artifact.StoreOnDisk(n.RequestURI, n.RequestBody); err != nil { + return err + } + + return nil +} + +func (n Npm) Read() error { + reqUrlString := strings.Replace(n.RequestURI, "%2f", "/", -1) + if filepath.Ext(reqUrlString) != ".tgz" { + log.Debugf("file: '%s' does not have an extension", reqUrlString) + reqUrlString = reqUrlString + ".tmp" + } + if err := artifact.ReadFromDisk(n.ResponseWriter, reqUrlString); err != nil { + return fmt.Errorf(file.CannotReadErrMsg, err) + } + + return nil +} diff --git a/internal/pkg/artifact/artifact.go b/internal/pkg/artifact/artifact.go new file mode 100644 index 0000000..3f7f32f --- /dev/null +++ b/internal/pkg/artifact/artifact.go @@ -0,0 +1,188 @@ +package artifact + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + + "github.com/030/yaam/internal/pkg/file" + "github.com/030/yaam/internal/pkg/project" + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" +) + +func createHomeAndReturnPath(requestURI string) (string, error) { + h, err := project.RepositoriesHome() + if err != nil { + return "", err + } + + artifactPath := filepath.Join(h, requestURI) + artifactHome := filepath.Dir(artifactPath) + if err := os.MkdirAll(artifactHome, os.ModePerm); err != nil { + return "", err + } + + return artifactPath, nil +} + +func createIfDoesNotExist(path string, requestBody io.ReadCloser) error { + if _, fileExists := file.Exists(path); !fileExists { + dst, err := os.Create(filepath.Clean(path)) + if err != nil { + return err + } + + defer func() { + if err := dst.Close(); err != nil { + panic(err) + } + }() + + w, err := io.Copy(dst, requestBody) + if err != nil { + log.Error(err) + } + log.Debugf("file: '%s' created and it contains: '%d' bytes", path, w) + if err := dst.Sync(); err != nil { + return err + } + } else { + log.Debugf("file: '%s' exists already", path) + } + return nil +} + +func StoreOnDisk(requestURI string, requestBody io.ReadCloser) error { + if err := validate(requestURI); err != nil { + return err + } + + path, err := createHomeAndReturnPath(requestURI) + if err != nil { + return err + } + + if err := createIfDoesNotExist(path, requestBody); err != nil { + return err + } + + return nil +} + +func ReadFromDisk(w http.ResponseWriter, reqURL string) error { + prh, err := project.RepositoriesHome() + if err != nil { + return err + } + + f := filepath.Join(prh, reqURL) + log.Debugf("reading file: '%s' from disk...", f) + b, err := os.ReadFile(filepath.Clean(f)) + if err != nil { + return err + } + if _, err := io.WriteString(w, string(b)); err != nil { + return err + } + + return nil +} + +// ReadRepositoriesAndUrlsFromConfigFileAndCacheArtifact reads a repositories +// yaml file that contains repositories and their URLs. If a request is +// attempted to download a file, it will look up the name in the config file +// and find the public URLs so it can download the file from the public maven +// repository and cache it on disk. +func RepoInConfigFile(w http.ResponseWriter, urlString string) (PublicRepository, error) { + yh, err := project.Home() + if err != nil { + return PublicRepository{}, err + } + + viper.SetConfigName("caches") + viper.SetConfigType("yaml") + viper.AddConfigPath(filepath.Join(yh, "conf")) + if err := viper.ReadInConfig(); err != nil { + return PublicRepository{}, err + } + + reposAndUrls := viper.GetStringMapString("mavenReposAndUrls") + for repo, url := range reposAndUrls { + + // if err := pr.cache(n.RequestURL); err != nil { + // return err + // } + // reqURLString := reqURL.String() + log.Debugf("trying to cache artifact from: '%s'...", urlString) + + rr := repoRegex(repo) + log.Debugf("repoRegex: '%s'", rr) + pr := PublicRepository{Name: repo, Regex: rr, Url: url} + riu, err := repoInUrl(rr, urlString) + if err != nil { + return PublicRepository{}, err + } + log.Debugf("repoInUrl: '%t'", riu) + + if riu { + // if err := pr.createDirAndStoreOnDisk(rr, reqURLString); err != nil { + // return err + // } + return pr, nil + } + } + + return PublicRepository{}, nil +} + +type PublicRepository struct { + Name, Regex, Url string +} + +func repoRegex(repo string) string { + return `^/(maven|npm)/` + repo + `/(.*)$` +} + +func repoInUrl(repoRegex, url string) (bool, error) { + log.Debugf("check whether url: '%s' contains repo according to regex: '%s'", url, repoRegex) + match, err := regexp.MatchString(repoRegex, url) + if err != nil { + return false, err + } + log.Debugf("outcome regex check: '%t'", match) + + return match, nil +} + +func DownloadUrl(publicRepoUrl, regex, url string) (string, error) { + log.Debugf("check whether url: '%s' matches regex: '%s'. Params -> publicRepoUrl: '%s', regex: '%s' and url: '%s'", url, regex, publicRepoUrl, regex, url) + re := regexp.MustCompile(regex) + match := re.FindStringSubmatch(url) + log.Debugf("number of matching elements: %d", len(match)) + if len(match) != 3 { + return "", fmt.Errorf("should be 3! publicRepoUrl: '%s', regex: '%s', url: '%s'", publicRepoUrl, regex, url) + } + + u := re.ReplaceAllString(url, publicRepoUrl+`$2`) + + return u, nil +} + +func Dir(path string) error { + h, err := project.RepositoriesHome() + if err != nil { + return err + } + + dir := filepath.Join(h, filepath.Dir(path)) + + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return err + } + + return nil +} diff --git a/internal/pkg/artifact/artifact_test.go b/internal/pkg/artifact/artifact_test.go new file mode 100644 index 0000000..04dc07e --- /dev/null +++ b/internal/pkg/artifact/artifact_test.go @@ -0,0 +1,46 @@ +package artifact + +import ( + "io" + "path/filepath" + "strings" + "testing" + + "github.com/tj/assert" +) + +func TestStoreOnDisk(t *testing.T) { + s := strings.NewReader("Hola mundo!") + rc := io.NopCloser(s) + + err := StoreOnDisk(filepath.Join("/maven/releases/world", "hola.mundo"), rc) + if err != nil { + t.Error(err) + } + + assert.NoError(t, err) +} + +func TestStoreOnDiskFail(t *testing.T) { + err := StoreOnDisk(filepath.Join("/maven/releases-not-allowed/world", "hola.mundo"), nil) + + assert.EqualError(t, err, "repository: 'releases-not-allowed' is not allowed. Allowed repos: '[releases]'") +} + +const testUrl = "/hello/world" + +func TestRepoInUrlTrue(t *testing.T) { + match, err := repoInUrl("hello", testUrl) + if err != nil { + t.Error(err) + } + assert.Equal(t, true, match) +} + +func TestRepoInUrlFalse(t *testing.T) { + match, err := repoInUrl("hello123", testUrl) + if err != nil { + t.Error(err) + } + assert.Equal(t, false, match) +} diff --git a/internal/pkg/artifact/validate.go b/internal/pkg/artifact/validate.go new file mode 100644 index 0000000..ea423fd --- /dev/null +++ b/internal/pkg/artifact/validate.go @@ -0,0 +1,57 @@ +package artifact + +import ( + "fmt" + "path/filepath" + "regexp" + + "github.com/030/yaam/internal/pkg/project" + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "golang.org/x/exp/slices" +) + +func validate(requestURI string) error { + log.Debugf("requestURI: '%s'", requestURI) + dir := filepath.Dir(requestURI) + ext := filepath.Ext(requestURI) + if dir == "/" || ext == "" { + return fmt.Errorf("requestURI: '%s' should start with a: '/' and contain an extension", requestURI) + } + + regex := `^/([a-z]+)/([0-9a-z-]+)/` + re := regexp.MustCompile(regex) + repoTypeAndName := re.FindStringSubmatch(requestURI) + if len(repoTypeAndName) <= 2 { + return fmt.Errorf("no repo type or name detected. Verify whether the regex: '%s' matches the URL: '%s'", regex, requestURI) + } + repoType := repoTypeAndName[1] + repoName := repoTypeAndName[2] + + if err := allowedRepo(repoName, repoType); err != nil { + return err + } + + return nil +} + +func allowedRepo(name, repoType string) error { + h, err := project.Home() + if err != nil { + return err + } + + viper.SetConfigName(repoType) + viper.SetConfigType("yaml") + viper.AddConfigPath(filepath.Join(h, "conf", "repositories")) + if err := viper.ReadInConfig(); err != nil { + return err + } + + repos := viper.GetStringSlice("allowedRepos") + if !slices.Contains(repos, name) { + return fmt.Errorf("repository: '%s' is not allowed. Allowed repos: '%v'", name, repos) + } + + return nil +} diff --git a/internal/pkg/artifact/validate_test.go b/internal/pkg/artifact/validate_test.go new file mode 100644 index 0000000..361e34e --- /dev/null +++ b/internal/pkg/artifact/validate_test.go @@ -0,0 +1,59 @@ +package artifact + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/tj/assert" +) + +const ( + allowedReposMaven = `allowedRepos: + - releases` + allowedReposNpm = `allowedRepos: + - 3rdparty-npm` +) + +func ConfigHelper() error { + os.Setenv("YAAM_HOME", filepath.Join("/tmp", "yaam", "test"+time.Now().Format("20060102150405111"))) + dir := filepath.Join(os.Getenv("YAAM_HOME"), "conf", "repositories") + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, "maven.yaml"), []byte(allowedReposMaven), 0600); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, "npm.yaml"), []byte(allowedReposNpm), 0600); err != nil { + return err + } + + os.Setenv("YAAM_DEBUG", "true") + os.Setenv("YAAM_USER", "hello") + + return nil +} + +func init() { + if err := ConfigHelper(); err != nil { + panic(err) + } +} + +func TestValidate(t *testing.T) { + expDir := filepath.Join("/maven", "releases", "world") + err := validate(filepath.Join(expDir, "hola.mundo")) + if err != nil { + t.Error(err) + } + + assert.NoError(t, err) +} + +func TestValidateFail(t *testing.T) { + expErr := `Config File "something" Not Found in "\[/tmp/yaam/test[0-9]+/conf/repositories\]"` + actErr := validate(filepath.Join("/something", "releases", "world", "hola.mundo")) + + assert.Regexp(t, expErr, actErr) +} diff --git a/internal/pkg/file/file.go b/internal/pkg/file/file.go new file mode 100644 index 0000000..aaca4e1 --- /dev/null +++ b/internal/pkg/file/file.go @@ -0,0 +1,89 @@ +package file + +import ( + "io" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/hashicorp/go-retryablehttp" + log "github.com/sirupsen/logrus" +) + +const ( + RetryDuration = 30 * time.Second + CannotReadErrMsg = "cannot read artifact from disk. Error: '%v'. Perhaps it resides in another repository?" + WaitMsg = "wait: '%v' before retrying" +) + +func DownloadWithRetries(url string) (*http.Response, error) { + retryClient := retryablehttp.NewClient() + + retryClient.Logger = nil + retryClient.RetryMax = 30 + retryClient.RetryWaitMin = 30 * time.Second + retryClient.RetryWaitMax = 60 * time.Second + standardClient := retryClient.StandardClient() + log.Debugf("downloadURL: '%s'", url) + + /* #nosec */ + r, err := standardClient.Get(url) + if err != nil { + return nil, err + } + + return r, nil +} + +func Exists(f string) (int64, bool) { + fi, err := os.Stat(f) + if err != nil { + return 0, false + } + + return fi.Size(), true +} + +func CreateIfDoesNotExistOrEmpty(url, f string, body io.ReadCloser) error { + var written int64 + fileSize, fileExists := Exists(f) + if !fileExists || fileSize == 0 { + dst, err := os.Create(filepath.Clean(f)) + if err != nil { + return err + } + defer func() { + if err := dst.Close(); err != nil { + panic(err) + } + }() + + written, err = io.Copy(dst, body) + if err != nil { + return err + } + if err := dst.Sync(); err != nil { + return err + } + } + log.Debugf("downloaded: '%s' to: '%s'. Wrote: '%d' bytes", url, f, written) + + return nil +} + +func EmptyFile(f string) (emptyFile bool) { + fileSize, fileExists := Exists(f) + if !fileExists { + return false + } + + if fileSize == 0 { + log.Errorf("file: '%s' size is 0", f) + log.Warnf(WaitMsg, RetryDuration) + time.Sleep(RetryDuration) + return true + } + + return emptyFile +} diff --git a/pkg/project/project.go b/internal/pkg/project/project.go similarity index 74% rename from pkg/project/project.go rename to internal/pkg/project/project.go index c6d8c63..810f5aa 100644 --- a/pkg/project/project.go +++ b/internal/pkg/project/project.go @@ -1,9 +1,11 @@ package project import ( + "os" "path/filepath" "github.com/mitchellh/go-homedir" + log "github.com/sirupsen/logrus" ) const hiddenFolderName = ".yaam" @@ -15,6 +17,11 @@ func Home() (string, error) { } yh := filepath.Join(h, hiddenFolderName) + if os.Getenv("YAAM_HOME") != "" { + yh = os.Getenv("YAAM_HOME") + } + log.Debugf("yaam home: '%s'", yh) + return yh, nil } diff --git a/pkg/artifact/artifact.go b/pkg/artifact/artifact.go deleted file mode 100644 index 073f2ae..0000000 --- a/pkg/artifact/artifact.go +++ /dev/null @@ -1,83 +0,0 @@ -package artifact - -import ( - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - - "github.com/030/yaam/pkg/project" - log "github.com/sirupsen/logrus" -) - -func ReadFromDisk(w http.ResponseWriter, r *http.Request) error { - prh, err := project.RepositoriesHome() - if err != nil { - return err - } - - f := filepath.Join(prh, r.URL.String()) - log.Debugf("reading file: '%s' from disk...", f) - b, err := os.ReadFile(filepath.Clean(f)) - if err != nil { - return err - } - if _, err := io.WriteString(w, string(b)); err != nil { - return err - } - - return nil -} -func storeOnDisk(dir, artifactPathUri string, content io.ReadCloser) error { - prh, err := project.RepositoriesHome() - if err != nil { - return err - } - - artifactHome := filepath.Join(prh, dir) - - if err := os.MkdirAll(artifactHome, os.ModePerm); err != nil { - return err - } - - artifactPath := filepath.Join(prh, artifactPathUri) - - if _, err := os.Stat(artifactPath); errors.Is(err, os.ErrNotExist) { - log.Info(artifactPath) - out, err := os.Create(filepath.Clean(artifactPath)) - if err != nil { - return err - } - defer func() { - if err := out.Close(); err != nil { - panic(err) - } - }() - - written, err := io.Copy(out, content) - if err != nil { - log.Error(err) - } - log.Info(written) - } else { - log.Infof("file: '%s' exists already", artifactPath) - } - - return nil -} - -func Publish(r *http.Request) error { - ext := filepath.Ext(r.RequestURI) - dir := filepath.Dir(r.RequestURI) - if ext == "" || dir == "/" { - return fmt.Errorf("should contain a dir and have an extension") - } - - if err := storeOnDisk(dir, r.RequestURI, r.Body); err != nil { - return err - } - - return nil -} diff --git a/pkg/artifact/maven/maven.go b/pkg/artifact/maven/maven.go deleted file mode 100644 index 1b692da..0000000 --- a/pkg/artifact/maven/maven.go +++ /dev/null @@ -1,70 +0,0 @@ -package maven - -import ( - "net/http" - "net/url" - "os" - "path/filepath" - "regexp" - - "github.com/030/yaam/pkg/file" - "github.com/030/yaam/pkg/project" - "github.com/spf13/viper" -) - -func cache(repo, repoURL string, reqURL *url.URL) error { - prh, err := project.RepositoriesHome() - if err != nil { - return err - } - - repoRegex := `^\/` + repo + `\/(.*)$` - reqURLString := reqURL.String() - - match, err := regexp.MatchString(repoRegex, reqURL.Path) - if err != nil { - return err - } - - downloadURL := "" - if match { - re := regexp.MustCompile(repoRegex) - downloadURL = re.ReplaceAllString(reqURLString, repoURL+`$1`) - - dir := filepath.Dir(reqURLString) - completeDir := filepath.Join(prh, dir) - if err := os.MkdirAll(completeDir, os.ModePerm); err != nil { - return err - } - - completeFile := filepath.Join(prh, reqURLString) - if err := file.Download(completeFile, downloadURL); err != nil { - return err - } - } - - return nil -} - -func Cache(w http.ResponseWriter, reqURL *url.URL) error { - yh, err := project.Home() - if err != nil { - return err - } - - viper.SetConfigName("repositories") - viper.SetConfigType("yaml") - viper.AddConfigPath(filepath.Join(yh, "conf")) - if err := viper.ReadInConfig(); err != nil { - return err - } - - reposAndUrls := viper.GetStringMapString("mavenReposAndUrls") - for repo, url := range reposAndUrls { - if err := cache(repo, url, reqURL); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/file/file.go b/pkg/file/file.go deleted file mode 100644 index c9fa2ea..0000000 --- a/pkg/file/file.go +++ /dev/null @@ -1,49 +0,0 @@ -package file - -import ( - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - - log "github.com/sirupsen/logrus" -) - -func Download(f string, url string) (err error) { - if _, err := os.Stat(f); errors.Is(err, os.ErrNotExist) { - out, err := os.Create(filepath.Clean(f)) - if err != nil { - return err - } - defer func() { - if err := out.Close(); err != nil { - panic(err) - } - }() - - /* #nosec */ - r, err := http.Get(url) - if err != nil { - return err - } - defer func() { - if err := r.Body.Close(); err != nil { - panic(err) - } - }() - - if r.StatusCode != http.StatusOK { - return fmt.Errorf("download failed: '%s'", r.Status) - } - - w, err := io.Copy(out, r.Body) - if err != nil { - return err - } - log.Infof("downloaded: '%s' to: '%s'. Wrote: '%d' bytes", url, f, w) - } - - return nil -} diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..f2ede34 --- /dev/null +++ b/test/README.md @@ -0,0 +1,3 @@ +# Test + +## Gradle diff --git a/test/gradle/README.md b/test/gradle/README.md new file mode 100644 index 0000000..c8688fc --- /dev/null +++ b/test/gradle/README.md @@ -0,0 +1,7 @@ +# Gradle + +https://start.spring.io/ + +* Gradle Project +* Generate +* Extract the demo.zip diff --git a/test/gradle/demo/.gitignore b/test/gradle/demo/.gitignore new file mode 100644 index 0000000..c2065bc --- /dev/null +++ b/test/gradle/demo/.gitignore @@ -0,0 +1,37 @@ +HELP.md +.gradle +build/ +!gradle/wrapper/gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ diff --git a/test/gradle/demo/gradle/wrapper/gradle-wrapper.jar b/test/gradle/demo/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..249e583 Binary files /dev/null and b/test/gradle/demo/gradle/wrapper/gradle-wrapper.jar differ diff --git a/test/gradle/demo/gradle/wrapper/gradle-wrapper.properties b/test/gradle/demo/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..8049c68 --- /dev/null +++ b/test/gradle/demo/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/test/gradle/demo/gradlew b/test/gradle/demo/gradlew new file mode 100755 index 0000000..a69d9cb --- /dev/null +++ b/test/gradle/demo/gradlew @@ -0,0 +1,240 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/test/gradle/demo/gradlew.bat b/test/gradle/demo/gradlew.bat new file mode 100644 index 0000000..53a6b23 --- /dev/null +++ b/test/gradle/demo/gradlew.bat @@ -0,0 +1,91 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/test/gradle/demo/src/main/java/com/example/demo/DemoApplication.java b/test/gradle/demo/src/main/java/com/example/demo/DemoApplication.java new file mode 100644 index 0000000..64b538a --- /dev/null +++ b/test/gradle/demo/src/main/java/com/example/demo/DemoApplication.java @@ -0,0 +1,13 @@ +package com.example.demo; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class DemoApplication { + + public static void main(String[] args) { + SpringApplication.run(DemoApplication.class, args); + } + +} diff --git a/test/gradle/demo/src/main/resources/application.properties b/test/gradle/demo/src/main/resources/application.properties new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/test/gradle/demo/src/main/resources/application.properties @@ -0,0 +1 @@ + diff --git a/test/gradle/demo/src/test/java/com/example/demo/DemoApplicationTests.java b/test/gradle/demo/src/test/java/com/example/demo/DemoApplicationTests.java new file mode 100644 index 0000000..2778a6a --- /dev/null +++ b/test/gradle/demo/src/test/java/com/example/demo/DemoApplicationTests.java @@ -0,0 +1,13 @@ +package com.example.demo; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class DemoApplicationTests { + + @Test + void contextLoads() { + } + +} diff --git a/test/npm/README.md b/test/npm/README.md new file mode 100644 index 0000000..2b5c7da --- /dev/null +++ b/test/npm/README.md @@ -0,0 +1,28 @@ +# NPM + +npx create-react-app . + +echo -n 'hello:world' | openssl base64 + +vim ~/.npmrc + +registry=http://localhost:25213/npm/3rdparty-npm/ +always-auth=true +_auth=aGVsbG86d29ybGQ= + +mv ~/.npm/cache{,2} + +cd test/npm/demo +rm -r node_modules + +npm i -d + +validate json using json_pp +for f in $(find /tmp/yaam/testi2/repositories/npm/3rdparty-npm/ -name *.tmp); do cat $f | json_pp; done +for f in $(find /tmp/yaam/testi2/repositories/npm/3rdparty-npm/ -name *.tmp); do du -h $f; done|grep M + + +ENOENT: no such file or directory +enoent x no such file or directory +try remove the package-lock.json +and run npm i -d again \ No newline at end of file diff --git a/test/npm/demo/.gitignore b/test/npm/demo/.gitignore new file mode 100644 index 0000000..4d29575 --- /dev/null +++ b/test/npm/demo/.gitignore @@ -0,0 +1,23 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.js + +# testing +/coverage + +# production +/build + +# misc +.DS_Store +.env.local +.env.development.local +.env.test.local +.env.production.local + +npm-debug.log* +yarn-debug.log* +yarn-error.log* diff --git a/test/npm/demo/README.md b/test/npm/demo/README.md new file mode 100644 index 0000000..58beeac --- /dev/null +++ b/test/npm/demo/README.md @@ -0,0 +1,70 @@ +# Getting Started with Create React App + +This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). + +## Available Scripts + +In the project directory, you can run: + +### `npm start` + +Runs the app in the development mode.\ +Open [http://localhost:3000](http://localhost:3000) to view it in your browser. + +The page will reload when you make changes.\ +You may also see any lint errors in the console. + +### `npm test` + +Launches the test runner in the interactive watch mode.\ +See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information. + +### `npm run build` + +Builds the app for production to the `build` folder.\ +It correctly bundles React in production mode and optimizes the build for the best performance. + +The build is minified and the filenames include the hashes.\ +Your app is ready to be deployed! + +See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. + +### `npm run eject` + +**Note: this is a one-way operation. Once you `eject`, you can't go back!** + +If you aren't satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project. + +Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you're on your own. + +You don't have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn't feel obligated to use this feature. However we understand that this tool wouldn't be useful if you couldn't customize it when you are ready for it. + +## Learn More + +You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started). + +To learn React, check out the [React documentation](https://reactjs.org/). + +### Code Splitting + +This section has moved here: [https://facebook.github.io/create-react-app/docs/code-splitting](https://facebook.github.io/create-react-app/docs/code-splitting) + +### Analyzing the Bundle Size + +This section has moved here: [https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size](https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size) + +### Making a Progressive Web App + +This section has moved here: [https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app](https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app) + +### Advanced Configuration + +This section has moved here: [https://facebook.github.io/create-react-app/docs/advanced-configuration](https://facebook.github.io/create-react-app/docs/advanced-configuration) + +### Deployment + +This section has moved here: [https://facebook.github.io/create-react-app/docs/deployment](https://facebook.github.io/create-react-app/docs/deployment) + +### `npm run build` fails to minify + +This section has moved here: [https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify](https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify) diff --git a/test/npm/demo/package.json b/test/npm/demo/package.json new file mode 100644 index 0000000..94953ea --- /dev/null +++ b/test/npm/demo/package.json @@ -0,0 +1,38 @@ +{ + "name": "demo", + "version": "0.1.0", + "private": true, + "dependencies": { + "@testing-library/jest-dom": "^5.16.5", + "@testing-library/react": "^13.3.0", + "@testing-library/user-event": "^13.5.0", + "react": "^18.2.0", + "react-dom": "^18.2.0", + "react-scripts": "5.0.1", + "web-vitals": "^2.1.4" + }, + "scripts": { + "start": "react-scripts start", + "build": "react-scripts build", + "test": "react-scripts test", + "eject": "react-scripts eject" + }, + "eslintConfig": { + "extends": [ + "react-app", + "react-app/jest" + ] + }, + "browserslist": { + "production": [ + ">0.2%", + "not dead", + "not op_mini all" + ], + "development": [ + "last 1 chrome version", + "last 1 firefox version", + "last 1 safari version" + ] + } +} diff --git a/test/npm/demo/public/favicon.ico b/test/npm/demo/public/favicon.ico new file mode 100644 index 0000000..a11777c Binary files /dev/null and b/test/npm/demo/public/favicon.ico differ diff --git a/test/npm/demo/public/index.html b/test/npm/demo/public/index.html new file mode 100644 index 0000000..aa069f2 --- /dev/null +++ b/test/npm/demo/public/index.html @@ -0,0 +1,43 @@ + + + + + + + + + + + + + React App + + + +
+ + + diff --git a/test/npm/demo/public/logo192.png b/test/npm/demo/public/logo192.png new file mode 100644 index 0000000..fc44b0a Binary files /dev/null and b/test/npm/demo/public/logo192.png differ diff --git a/test/npm/demo/public/logo512.png b/test/npm/demo/public/logo512.png new file mode 100644 index 0000000..a4e47a6 Binary files /dev/null and b/test/npm/demo/public/logo512.png differ diff --git a/test/npm/demo/public/manifest.json b/test/npm/demo/public/manifest.json new file mode 100644 index 0000000..080d6c7 --- /dev/null +++ b/test/npm/demo/public/manifest.json @@ -0,0 +1,25 @@ +{ + "short_name": "React App", + "name": "Create React App Sample", + "icons": [ + { + "src": "favicon.ico", + "sizes": "64x64 32x32 24x24 16x16", + "type": "image/x-icon" + }, + { + "src": "logo192.png", + "type": "image/png", + "sizes": "192x192" + }, + { + "src": "logo512.png", + "type": "image/png", + "sizes": "512x512" + } + ], + "start_url": ".", + "display": "standalone", + "theme_color": "#000000", + "background_color": "#ffffff" +} diff --git a/test/npm/demo/public/robots.txt b/test/npm/demo/public/robots.txt new file mode 100644 index 0000000..e9e57dc --- /dev/null +++ b/test/npm/demo/public/robots.txt @@ -0,0 +1,3 @@ +# https://www.robotstxt.org/robotstxt.html +User-agent: * +Disallow: diff --git a/test/npm/demo/src/App.css b/test/npm/demo/src/App.css new file mode 100644 index 0000000..74b5e05 --- /dev/null +++ b/test/npm/demo/src/App.css @@ -0,0 +1,38 @@ +.App { + text-align: center; +} + +.App-logo { + height: 40vmin; + pointer-events: none; +} + +@media (prefers-reduced-motion: no-preference) { + .App-logo { + animation: App-logo-spin infinite 20s linear; + } +} + +.App-header { + background-color: #282c34; + min-height: 100vh; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + font-size: calc(10px + 2vmin); + color: white; +} + +.App-link { + color: #61dafb; +} + +@keyframes App-logo-spin { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } +} diff --git a/test/npm/demo/src/App.js b/test/npm/demo/src/App.js new file mode 100644 index 0000000..3784575 --- /dev/null +++ b/test/npm/demo/src/App.js @@ -0,0 +1,25 @@ +import logo from './logo.svg'; +import './App.css'; + +function App() { + return ( +
+
+ logo +

+ Edit src/App.js and save to reload. +

+ + Learn React + +
+
+ ); +} + +export default App; diff --git a/test/npm/demo/src/App.test.js b/test/npm/demo/src/App.test.js new file mode 100644 index 0000000..1f03afe --- /dev/null +++ b/test/npm/demo/src/App.test.js @@ -0,0 +1,8 @@ +import { render, screen } from '@testing-library/react'; +import App from './App'; + +test('renders learn react link', () => { + render(); + const linkElement = screen.getByText(/learn react/i); + expect(linkElement).toBeInTheDocument(); +}); diff --git a/test/npm/demo/src/index.css b/test/npm/demo/src/index.css new file mode 100644 index 0000000..ec2585e --- /dev/null +++ b/test/npm/demo/src/index.css @@ -0,0 +1,13 @@ +body { + margin: 0; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', + 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', + sans-serif; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +code { + font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', + monospace; +} diff --git a/test/npm/demo/src/index.js b/test/npm/demo/src/index.js new file mode 100644 index 0000000..d563c0f --- /dev/null +++ b/test/npm/demo/src/index.js @@ -0,0 +1,17 @@ +import React from 'react'; +import ReactDOM from 'react-dom/client'; +import './index.css'; +import App from './App'; +import reportWebVitals from './reportWebVitals'; + +const root = ReactDOM.createRoot(document.getElementById('root')); +root.render( + + + +); + +// If you want to start measuring performance in your app, pass a function +// to log results (for example: reportWebVitals(console.log)) +// or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals +reportWebVitals(); diff --git a/test/npm/demo/src/logo.svg b/test/npm/demo/src/logo.svg new file mode 100644 index 0000000..9dfc1c0 --- /dev/null +++ b/test/npm/demo/src/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/npm/demo/src/reportWebVitals.js b/test/npm/demo/src/reportWebVitals.js new file mode 100644 index 0000000..5253d3a --- /dev/null +++ b/test/npm/demo/src/reportWebVitals.js @@ -0,0 +1,13 @@ +const reportWebVitals = onPerfEntry => { + if (onPerfEntry && onPerfEntry instanceof Function) { + import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { + getCLS(onPerfEntry); + getFID(onPerfEntry); + getFCP(onPerfEntry); + getLCP(onPerfEntry); + getTTFB(onPerfEntry); + }); + } +}; + +export default reportWebVitals; diff --git a/test/npm/demo/src/setupTests.js b/test/npm/demo/src/setupTests.js new file mode 100644 index 0000000..8f2609b --- /dev/null +++ b/test/npm/demo/src/setupTests.js @@ -0,0 +1,5 @@ +// jest-dom adds custom jest matchers for asserting on DOM nodes. +// allows you to do things like: +// expect(element).toHaveTextContent(/react/i) +// learn more: https://github.com/testing-library/jest-dom +import '@testing-library/jest-dom';