From 8053b4981d810b169ccd117e1819fe3b111811c8 Mon Sep 17 00:00:00 2001 From: Anurag Mittal Date: Tue, 9 Jan 2024 13:33:25 +0100 Subject: [PATCH 1/3] OSIS-145: Disable get usage API GetUsage API is disabled to unblock RING v9.3 release. --- .../healthcheck/UtapiHealthIndicator.java | 36 +----- .../osis/resource/ScalityOsisController.java | 8 +- .../osis/model/ScalityOsisConstants.java | 4 +- .../service/impl/ScalityOsisServiceImpl.java | 111 +----------------- .../impl/ScalityOsisServiceMiscTests.java | 22 ++++ 5 files changed, 32 insertions(+), 149 deletions(-) diff --git a/osis-app/src/main/java/com/scality/osis/healthcheck/UtapiHealthIndicator.java b/osis-app/src/main/java/com/scality/osis/healthcheck/UtapiHealthIndicator.java index 944af27..4fb76be 100644 --- a/osis-app/src/main/java/com/scality/osis/healthcheck/UtapiHealthIndicator.java +++ b/osis-app/src/main/java/com/scality/osis/healthcheck/UtapiHealthIndicator.java @@ -5,48 +5,18 @@ package com.scality.osis.healthcheck; -import com.scality.osis.ScalityAppEnv; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.actuate.health.Health; import org.springframework.boot.actuate.health.HealthIndicator; import org.springframework.stereotype.Component; -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.SocketTimeoutException; @Component("utapi") public class UtapiHealthIndicator implements HealthIndicator { - private static final Logger logger = LoggerFactory.getLogger(UtapiHealthIndicator.class); - - @Autowired - private ScalityAppEnv appEnv; @Override public Health health() { - HttpURLConnection connection = null; - try { - // check if Utapi Endpoint is reachable - connection = (HttpURLConnection) new java.net.URL(appEnv.getUtapiEndpoint()).openConnection(); - connection.setConnectTimeout(appEnv.getUtapiHealthCheckTimeout()); - connection.connect(); - } catch (SocketTimeoutException e) { - logger.warn("Failed to connect to Utapi endpoint {} in timeout {}",appEnv.getUtapiEndpoint(), appEnv.getUtapiHealthCheckTimeout()); - return Health.down() - .withDetail("error", e.getMessage()) - .build(); - } catch (IOException e) { - logger.warn("an I/O error occurs while trying to connect {}.",appEnv.getUtapiEndpoint()); - return Health.down() - .withDetail("error", e.getMessage()) - .build(); - } finally { - if (connection != null) { - connection.disconnect(); - } - } - return Health.up().build(); + // We want to make sure healthcheck is false even if it is set in application.properties + // This will be enabled and potentially refactored with S3C-8266 + return Health.down().build(); } } diff --git a/osis-app/src/main/java/com/scality/osis/resource/ScalityOsisController.java b/osis-app/src/main/java/com/scality/osis/resource/ScalityOsisController.java index 4180b68..eb5b33f 100644 --- a/osis-app/src/main/java/com/scality/osis/resource/ScalityOsisController.java +++ b/osis-app/src/main/java/com/scality/osis/resource/ScalityOsisController.java @@ -339,13 +339,13 @@ public OsisTenant getTenant( }, tags = {"usage", "optional"}) @GetMapping(value = "/api/v1/usage", produces = "application/json") + // getUsage API will be re-enabled in S3C-8035 + @NotImplement(name = ScalityOsisConstants.GET_USAGE_API_CODE) public OsisUsage getUsage( @Parameter(description = "The ID of the tenant to get its usage.") @Valid @RequestParam(value = "tenant_id", required = false) Optional tenantId, @Parameter(description = "The ID of the user to get its usage.") @Valid @RequestParam(value = "user_id", required = false) Optional userId) { - if (tenantId.isEmpty() && userId.isPresent()) { - throw new BadRequestException("userId must be specified with associated tenantId!"); - } - return osisService.getOsisUsage(tenantId, userId); + logger.info("Get Osis Usage request received:: tenant ID:{}, user ID:{}", tenantId, userId); + throw new NotImplementedException(); } /* diff --git a/osis-core/src/main/java/com/scality/osis/model/ScalityOsisConstants.java b/osis-core/src/main/java/com/scality/osis/model/ScalityOsisConstants.java index 0a119fd..f485e03 100644 --- a/osis-core/src/main/java/com/scality/osis/model/ScalityOsisConstants.java +++ b/osis-core/src/main/java/com/scality/osis/model/ScalityOsisConstants.java @@ -14,12 +14,12 @@ private ScalityOsisConstants() { } public static final String DELETE_TENANT_API_CODE = "deleteTenant"; - public static final String GET_BUCKET_LIST_API_CODE = "getBucketList"; + public static final String GET_USAGE_API_CODE = "getUsage"; public static final String GET_BUCKET_ID_LOGGING_API_CODE = "getBucketLoggingId"; public static final List API_CODES = Arrays.asList( DELETE_TENANT_API_CODE, - GET_BUCKET_LIST_API_CODE, + GET_USAGE_API_CODE, GET_BUCKET_ID_LOGGING_API_CODE ); } diff --git a/osis-core/src/main/java/com/scality/osis/service/impl/ScalityOsisServiceImpl.java b/osis-core/src/main/java/com/scality/osis/service/impl/ScalityOsisServiceImpl.java index 0e51662..23b5761 100644 --- a/osis-core/src/main/java/com/scality/osis/service/impl/ScalityOsisServiceImpl.java +++ b/osis-core/src/main/java/com/scality/osis/service/impl/ScalityOsisServiceImpl.java @@ -1229,116 +1229,7 @@ public PageOfOsisBucketMeta getBucketList(String tenantId, long offset, long lim @Override public OsisUsage getOsisUsage(Optional tenantId, Optional userId) { - - if (!appEnv.isUtapiEnabled()) { - logger.info("Utapi is not enabled, returning empty usage"); - return new OsisUsage(); - } - - logger.info("Get Osis Usage request received:: tenant ID:{}, user ID:{}", tenantId, userId); - OsisUsage osisUsage = new OsisUsage(); - - if (tenantId.isEmpty()) { - logger.info("tenant ID is empty, getting platform usage at global level"); - - int start = 0; - int limit = 100; - boolean hasMoreTenants = true; - while (hasMoreTenants) { - PageOfTenants pageOfTenants = listTenants(start, limit); - pageOfTenants.getItems().forEach(tenant -> { - logger.info("Getting usage for tenant:{}", tenant.getTenantId()); - OsisUsage tenantOsisUsage = getOsisUsage(Optional.of(tenant.getTenantId()), Optional.empty()); - logger.info("Usage for tenant:{} is:{}", tenant.getTenantId(), new Gson().toJson(tenantOsisUsage)); - - logger.debug("Consolidating usage for tenant:{}", tenant.getTenantId()); - osisUsage.consolidateUsage(tenantOsisUsage); - logger.debug("Consolidated usage:{}", new Gson().toJson(osisUsage)); - }); - start += limit; - hasMoreTenants = pageOfTenants.getPageInfo().getTotal() >= limit; - } - - logger.info("Usage for all tenants is:{}", new Gson().toJson(osisUsage)); - - } else { - - try { - // getting utapi client for tenant - Credentials tempCredentials = getCredentials(tenantId.get()); - final UtapiServiceClient utapiClient = this.utapi.getUtapiServiceClient(tempCredentials, - appEnv.getRegionInfo().get(0)); - - if (userId.isEmpty()) { - - logger.info("tenant ID is specified, getting usage at tenant level, tenant ID:{}", tenantId.get()); - - // getting s3 client for tenant - final AmazonS3 s3Client = this.s3.getS3Client(tempCredentials, - appEnv.getRegionInfo().get(0)); - - logger.debug("Listing buckets for tenant:{}", tenantId.get()); - List buckets = s3Client.listBuckets(); - - // set bucket count by the size of listing buckets - osisUsage.setBucketCount((long) buckets.size()); - - logger.debug("[UTAPI] Listing metrics for tenant:{}", tenantId.get()); - ListMetricsRequestDTO listMetricsRequestDTO = ScalityModelConverter.toScalityListMetricsRequest( - "accounts", - List.of(tenantId.get()), - List.of(getHourTime()) - ); - Response listMetricsResponseDTO = utapiClient.listAccountsMetrics(listMetricsRequestDTO); - MetricsData metricsData = listMetricsResponseDTO.getAwsResponse()[0]; - logger.debug("[UTAPI] List Metrics response:{}", new Gson().toJson(metricsData)); - - // set object count and used bytes by the utapi metrics - osisUsage.setObjectCount(metricsData.getNumberOfObjects().get(1)); - osisUsage.setUsedBytes(metricsData.getStorageUtilized().get(1)); - - AccountData account = vaultAdmin.getAccount(ScalityModelConverter.toGetAccountRequestWithID(tenantId.get())); - if (account.getQuota() > 0) { - osisUsage.setTotalBytes((long) account.getQuota()); // set total bytes by the vault quota - osisUsage.setAvailableBytes(osisUsage.getTotalBytes() - osisUsage.getUsedBytes()); // set available bytes by the difference between total bytes and used bytes - } - logger.info("Usage for tenant:{} is:{}", tenantId.get(), new Gson().toJson(osisUsage)); - - } else { - - logger.info("tenant ID and user ID are specified, getting usage at user level, tenant ID:{}, user ID:{}", tenantId.get(), userId.get()); - - logger.debug("[UTAPI] Listing metrics for user:{}", userId.get()); - ListMetricsRequestDTO listMetricsRequestDTO = ScalityModelConverter.toScalityListMetricsRequest( - "users", - List.of(userId.get()), - List.of(getHourTime()) - ); - Response listMetricsResponseDTO = utapiClient.listUsersMetrics(listMetricsRequestDTO); - MetricsData metricsData = listMetricsResponseDTO.getAwsResponse()[0]; - logger.debug("[UTAPI] List Metrics response:{}", new Gson().toJson(metricsData)); - - // set object count and used bytes by the utapi metrics - osisUsage.setObjectCount(metricsData.getNumberOfObjects().get(1)); - osisUsage.setUsedBytes(metricsData.getStorageUtilized().get(1)); - - logger.info("Usage for user:{} is:{}", userId.get(), new Gson().toJson(osisUsage)); - } - } catch (Exception err) { - if (isAdminPolicyError(err)) { - try { - generateAdminPolicy(tenantId.get()); - return getOsisUsage(tenantId, userId); - } catch (Exception ex) { - err = ex; - } - } - logger.error("GetUsage error. Returning empty list. Error details: ", err); - // For errors, GetUsage should return empty OsisUsage - return osisUsage; - } - } - return osisUsage; + throw new NotImplementedException(); } @Override diff --git a/osis-core/src/test/java/com/scality/osis/service/impl/ScalityOsisServiceMiscTests.java b/osis-core/src/test/java/com/scality/osis/service/impl/ScalityOsisServiceMiscTests.java index 581f260..53984fe 100644 --- a/osis-core/src/test/java/com/scality/osis/service/impl/ScalityOsisServiceMiscTests.java +++ b/osis-core/src/test/java/com/scality/osis/service/impl/ScalityOsisServiceMiscTests.java @@ -14,6 +14,8 @@ import com.scality.osis.utapiclient.dto.MetricsData; import com.scality.osis.vaultadmin.impl.VaultServiceException; import com.scality.vaultclient.dto.*; + +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; @@ -27,6 +29,8 @@ import static com.scality.osis.utils.ScalityConstants.IAM_PREFIX; import static com.scality.osis.utils.ScalityTestUtils.*; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.*; class ScalityOsisServiceMiscTests extends BaseOsisServiceTest { @@ -173,7 +177,19 @@ void testGetBucketListErr() { assertEquals(0L, response.getItems().size()); } + + // test to check if getUsage API throws an error not implemented + // this will be removed as a part of S3C-8266 + @Test + void testGetUsage() { + // Setup + assertThrows(NotImplementedException.class, () -> scalityOsisServiceUnderTest.getOsisUsage(Optional.of(SAMPLE_TENANT_ID), Optional.of(SAMPLE_CD_TENANT_ID)), + NOT_IMPLEMENTED_EXCEPTION_ERR); + } + @Test + @Disabled + // This will be enable with S3C-8266 void testGetOsisUsageForAll() { // Setup @@ -224,6 +240,8 @@ void testGetOsisUsageForAll() { } @Test + @Disabled + // This will be enable with S3C-8266 void testGetOsisUsageForTenant() { // Setup @@ -263,6 +281,8 @@ void testGetOsisUsageForTenant() { } @Test + @Disabled + // This will be enable with S3C-8266 void testGetOsisUsageForUser() { // Setup @@ -281,6 +301,8 @@ void testGetOsisUsageForUser() { } @Test + @Disabled + // This will be enable with S3C-8266 void testGetOsisUsageErr() { // Setup From b1e8b063bf4f18215a72ce1f739b6023fc4ea180 Mon Sep 17 00:00:00 2001 From: Anurag Mittal Date: Tue, 9 Jan 2024 15:04:40 +0100 Subject: [PATCH 2/3] OSIS-145: updated CODEOWNERS file Removed Xin as she is not a part of Scality and changed lead group to object. We still have branch protection rules in place for main branch. This rule does not allow any merges without 2 approvals. --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a4cc01a..c5fa6a3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @scality/object-lead @XinLiScality +* @scality/object From 55c9125a06d26ed061b8448b528aa30511aa5dab Mon Sep 17 00:00:00 2001 From: Anurag Mittal Date: Tue, 9 Jan 2024 16:09:52 +0100 Subject: [PATCH 3/3] OSIS-145: bump-version-to-2.2.2 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index fcb0b4d..7c31823 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,6 @@ buildscript { ext { - osisVersion = '2.2.1' + osisVersion = '2.2.2' vaultclientVersion = '1.1.2' springBootVersion = '2.7.6' }