From 4c31007fb35be63de2674f2de69c29d647374399 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 30 Jul 2024 18:37:41 +0100 Subject: [PATCH] Release of 2024-07-30 AbfsDiagnostics prints more options * latest manifest committer settings * azure http client options (trunk/branch-3.4 only) --- BUILDING.md | 3 +-- .../fs/store/diag/AbfsDiagnosticsInfo.java | 24 ++++++++++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/BUILDING.md b/BUILDING.md index 555b1d3..2fa1960 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -35,8 +35,7 @@ mvn clean install -Psdk2 Joint build ```bash -mvn clean install -Pextra -mvn install -Psdk2 +mvn clean install -Pextra && mvn install -Psdk2 ``` ## Releasing diff --git a/src/main/java/org/apache/hadoop/fs/store/diag/AbfsDiagnosticsInfo.java b/src/main/java/org/apache/hadoop/fs/store/diag/AbfsDiagnosticsInfo.java index 3bfcc98..4ba3768 100644 --- a/src/main/java/org/apache/hadoop/fs/store/diag/AbfsDiagnosticsInfo.java +++ b/src/main/java/org/apache/hadoop/fs/store/diag/AbfsDiagnosticsInfo.java @@ -172,15 +172,23 @@ public Object[][] getEnvVars() { {"fs.azure.account.oauth2.user.name", false, false}, {"fs.azure.account.oauth2.user.name", false, false}, {"fs.azure.account.oauth2.user.password", true, true}, - {"fs.azure.shellkeyprovider.script", true, true}, + {"fs.azure.account.operation.idle.timeout", false, false}, {"fs.azure.account.throttling.enabled", false, false}, + {"fs.azure.analysis.period", false, false}, + {"fs.azure.shellkeyprovider.script", true, true}, {FS_AZURE_ALWAYS_USE_HTTPS, false, false}, + {"fs.azure.apache.http.client.idle.connection.ttl", false, false}, + {"fs.azure.apache.http.client.max.cache.connection.size", false, false}, + {"fs.azure.apache.http.client.max.io.exception.retries", false, false}, {"fs.azure.appendblob.directories", false, false}, {FS_AZURE_ATOMIC_RENAME_KEY, false, false}, {"fs.azure.block.location.impersonatedhost", false, false}, {"fs.azure.block.size", false, false}, {"fs.azure.buffered.pread.disable", false, false}, {"fs.azure.client-provided-encryption-key", true, true}, + {"fs.azure.client.correlationid", false, false}, + {"fs.azure.cluster.name", false, false}, + {"fs.azure.cluster.type", false, false}, {"fs.azure.concurrentRequestCount.in", false, false}, {"fs.azure.concurrentRequestCount.out", false, false}, {"fs.azure.createRemoteFileSystemDuringInitialization", false, false}, @@ -201,6 +209,9 @@ public Object[][] getEnvVars() { {FS_AZURE_ENABLE_READAHEAD, false, false}, {FS_AZURE_ENABLE_READAHEAD_V2, false, false}, {"fs.azure.enable.rename.resilience", false, false}, + {"fs.azure.encryption.context.provider.type", false, false}, + {"fs.azure.encryption.encoded.client-provided-key", false, true}, + {"fs.azure.encryption.encoded.client-provided-key-sha", true, true}, {FS_AZURE_IDENTITY_TRANSFORMER_CLASS, false, false}, {"fs.azure.identity.transformer.domain.name", false, false}, {"fs.azure.identity.transformer.enable.short.name", false, false}, @@ -218,6 +229,8 @@ public Object[][] getEnvVars() { {"fs.azure.io.retry.min.backoff.interval", false, false}, {FS_AZURE_LEASE_THREADS, false, false}, {"fs.azure.list.max.results", false, false}, + {"fs.azure.metric.analysis.timeout", false, false}, + {"fs.azure.networking.library", false, false}, {"fs.azure.oauth.token.fetch.retry.max.backoff.interval", false, false}, {"fs.azure.oauth.token.fetch.retry.max.retries", false, false}, {"fs.azure.oauth.token.fetch.retry.min.backoff.interval", false, false}, @@ -231,11 +244,13 @@ public Object[][] getEnvVars() { {FS_AZURE_READAHEADQUEUE_DEPTH, false, false}, {"fs.azure.rename.raises.exceptions", false, false}, {FS_AZURE_SAS_TOKEN_PROVIDER_TYPE, false, false}, + {"fs.azure.sas.fixed.token", true, true}, {"fs.azure.sas.token.renew.period.for.streams", false, false}, {"fs.azure.secure.mode", false, false}, {"fs.azure.shellkeyprovider.script", false, false}, {"fs.azure.skipUserGroupMetadataDuringInitialization", false, false}, {"fs.azure.ssl.channel.mode", false, false}, + {"fs.azure.tracingheader.format", false, false}, {"fs.azure.use.upn", false, false}, {"fs.azure.user.agent.prefix", false, false}, {"fs.azure.write.enableappendwithflush", false, false}, @@ -245,16 +260,19 @@ public Object[][] getEnvVars() { /* committer */ {"mapreduce.outputcommitter.factory.scheme.abfs", false, false}, - {"mapreduce.fileoutputcommitter.marksuccessfuljobs", false, false}, {"mapreduce.manifest.committer.cleanup.parallel.delete", false, false}, + {"mapreduce.manifest.committer.cleanup.parallel.delete.base.first", false, false}, {"mapreduce.manifest.committer.delete.target.files", false, false}, {"mapreduce.manifest.committer.diagnostics.manifest.directory", false, false}, {"mapreduce.manifest.committer.io.thread.count", false, false}, + {"mapreduce.manifest.committer.manifest.save.attempts", false, false}, {"mapreduce.manifest.committer.store.operations.classname", false, false}, {"mapreduce.manifest.committer.summary.report.directory", false, false}, {"mapreduce.manifest.committer.validate.output", false, false}, - + {"mapreduce.manifest.committer.writer.queue.capacity", false, false}, + {"mapreduce.fileoutputcommitter.cleanup.skipped", false, false}, + {"mapreduce.fileoutputcommitter.marksuccessfuljobs", false, false}, {"mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.mv.threads", false, false}, {"mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.task.commit", false, false}, {"mapreduce.fileoutputcommitter.algorithm.version.v1.experimental.parallel.rename.recovery", false, false},