diff --git a/.github/workflows/release_droid_prepare_original_checksum.yml b/.github/workflows/release_droid_prepare_original_checksum.yml new file mode 100644 index 0000000..8a4285b --- /dev/null +++ b/.github/workflows/release_droid_prepare_original_checksum.yml @@ -0,0 +1,37 @@ +name: Release Droid - Prepare Original Checksum + +on: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - name: Checkout the repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: 8 + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Run tests and build with Maven + run: mvn --batch-mode --file pom.xml clean verify + - name: Prepare checksum + run: find hadoop-etl-dist/target/ -name *.jar -exec sha256sum "{}" + > original_checksum + - name: Upload checksum to the artifactory + uses: actions/upload-artifact@v2 + with: + name: original_checksum + retention-days: 5 + path: original_checksum diff --git a/.github/workflows/release_droid_print_quick_checksum.yml b/.github/workflows/release_droid_print_quick_checksum.yml new file mode 100644 index 0000000..a411ee2 --- /dev/null +++ b/.github/workflows/release_droid_print_quick_checksum.yml @@ -0,0 +1,32 @@ +name: Release Droid - Print Quick Checksum + +on: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - name: Checkout the repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: 8 + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Build with Maven skipping tests + run: mvn --batch-mode --file pom.xml clean verify -DskipTests + - name: Print checksum + run: echo 'checksum_start==';find hadoop-etl-dist/target -name *.jar -exec sha256sum "{}" + | xargs;echo '==checksum_end' + diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml new file mode 100644 index 0000000..c73424e --- /dev/null +++ b/.github/workflows/release_droid_upload_github_release_assets.yml @@ -0,0 +1,74 @@ +name: Release Droid - Upload GitHub Release Assets + +on: + workflow_dispatch: + inputs: + upload_url: + description: 'Assets upload URL' + required: true + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - maven_opts: -P apache -Dhadoop.version=3.2.2 -Dhive.version=2.3.9 + release_suffix: apache-hadoop-3.2.2-hive-2.3.9 + + - maven_opts: -P cloudera -Dhadoop.version=2.6.0-cdh5.16.2 -Dhive.version=1.1.0-cdh5.16.2 + release_suffix: cloudera-hadoop-2.6.0-cdh5.16.2-hive-1.1.0-cdh5.16.2 + + - maven_opts: -P cloudera6x -Dhadoop.version=3.0.0-cdh6.2.0 -Dhive.version=2.1.1-cdh6.2.0 + release_suffix: cloudera-hadoop-3.0.0-cdh6.2.0-hive-2.1.1-cdh6.2.0 + + - maven_opts: -P cloudera6x -Dhadoop.version=3.1.1.7.1.6.0-297 -Dhive.version=3.1.3000.7.1.6.0-297 + release_suffix: cloudera-hadoop-3.1.1.7.1.6.0-297-hive-3.1.3000.7.1.6.0-297 + + - maven_opts: -P hortonworks -Dhadoop.version=2.7.3.2.6.5.3004-13 -Dhive.version=2.1.0.2.6.5.3004-13 + release_suffix: hortonworks-hadoop-2.7.3.2.6.5.3004-13-hive-2.1.0.2.6.5.3004-13 + + - maven_opts: -P mapr -Dhadoop.version=2.7.0-mapr-1602 -Dhive.version=2.0.0-mapr-1605 + release_suffix: mapr-hadoop-2.7.0-mapr-1602-hive-2.0.0-mapr-1605 + steps: + - name: Checkout the repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: 8 + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - name: Run package using maven skipping tests + run: mvn --batch-mode clean package -DskipTests=true ${{ matrix.maven_opts }} + - name: Prepare jar and generate sha256sum files + run: | + cd hadoop-etl-dist/target/ + rm original-hadoop-etl-udfs*.jar + for f in *.jar; do mv -- "$f" "${f%.jar}-${{ matrix.release_suffix }}.jar"; done + find *.jar -exec bash -c 'sha256sum {} > {}.sha256' \; + - name: Upload assets to the GitHub release draft + uses: shogo82148/actions-upload-release-asset@v1 + with: + upload_url: ${{ github.event.inputs.upload_url }} + asset_path: hadoop-etl-dist/target/*.jar + - name: Upload sha256sum files + uses: shogo82148/actions-upload-release-asset@v1 + with: + upload_url: ${{ github.event.inputs.upload_url }} + asset_path: hadoop-etl-dist/target/*.sha256 + - name: Upload error-code-report + uses: shogo82148/actions-upload-release-asset@v1 + if: ${{ false }} + with: + upload_url: ${{ github.event.inputs.upload_url }} + asset_path: target/error_code_report.json diff --git a/doc/changes/changes_1.1.0.md b/doc/changes/changes_1.1.0.md index 46926dc..c68702b 100644 --- a/doc/changes/changes_1.1.0.md +++ b/doc/changes/changes_1.1.0.md @@ -1,13 +1,19 @@ -# Hadoop ETL UDFs 1.1.0, released 2022-??-?? +# Hadoop ETL UDFs 1.1.0, released 2022-02-15 -Code name: +Code name: Updated Kerberos `auth_to_local` mechanism to use MIT ## Summary +In this release, we have updated Kerberos [`auth_to_local`](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SecureMode.html) mechanism to use MIT configuration file. Additionally, we updated vulnerable dependencies and improved documentation. + ## Bug Fixes * #73: Updated vulnerable Hadoop dependencies +## Features + +* #74: Updated Kerberos `auth_to_local` mechanism property to MIT + ## Documentation * #60: Increased the file length in deployment script @@ -28,4 +34,3 @@ Code name: ### Test Dependency Updates ### Plugin Updates - diff --git a/hadoop-etl-common/src/main/java/com/exasol/hadoop/kerberos/KerberosHadoopUtils.java b/hadoop-etl-common/src/main/java/com/exasol/hadoop/kerberos/KerberosHadoopUtils.java index c6e5d3f..6bce074 100644 --- a/hadoop-etl-common/src/main/java/com/exasol/hadoop/kerberos/KerberosHadoopUtils.java +++ b/hadoop-etl-common/src/main/java/com/exasol/hadoop/kerberos/KerberosHadoopUtils.java @@ -18,6 +18,7 @@ public static UserGroupInformation getKerberosUGI(final KerberosCredentials kerb System.setProperty("java.security.krb5.conf", confPath); Configuration conf = new Configuration(); conf.set("hadoop.security.authentication", "kerberos"); + conf.set("hadoop.security.auth_to_local.mechanism", "mit"); UserGroupInformation.setConfiguration(conf); String keytabPath = writeTempKeytabFile(kerberosCredentials.getKeytabFile(), tmpDir); ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(kerberosCredentials.getPrincipal(), keytabPath); diff --git a/hadoop-etl-dist/pom.xml b/hadoop-etl-dist/pom.xml index 377b738..7b88d38 100644 --- a/hadoop-etl-dist/pom.xml +++ b/hadoop-etl-dist/pom.xml @@ -6,7 +6,7 @@ exa-hadoop-etl-udfs 1.1.0 - hadoop-etl-dist + hadoop-etl-udfs true