Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improvement/osis 145 disable get usage api #139

Merged
merged 3 commits into from
Jan 10, 2024
Merged

Improvement/osis 145 disable get usage api #139

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8053b49
OSIS-145: Disable get usage API
anurag4DSB Jan 9, 2024
b1e8b06
OSIS-145: updated CODEOWNERS file
anurag4DSB Jan 9, 2024
55c9125
OSIS-145: bump-version-to-2.2.2
anurag4DSB Jan 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
View file
Open in desktop
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
@@ -1 +1 @@
* @scality/object-lead @XinLiScality
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

😢

Suggested change
* @scality/object-lead @XinLiScality
* @scality/object-lead @XinLiScality

* @scality/object
2 changes: 1 addition & 1 deletion build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
buildscript {
ext {
osisVersion = '2.2.1'
osisVersion = '2.2.2'
vaultclientVersion = '1.1.2'
springBootVersion = '2.7.6'
}
Expand Down
36 changes: 3 additions & 33 deletions osis-app/src/main/java/com/scality/osis/healthcheck/UtapiHealthIndicator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,48 +5,18 @@

package com.scality.osis.healthcheck;

import com.scality.osis.ScalityAppEnv;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.actuate.health.Health;
import org.springframework.boot.actuate.health.HealthIndicator;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;

@Component("utapi")
public class UtapiHealthIndicator implements HealthIndicator {
private static final Logger logger = LoggerFactory.getLogger(UtapiHealthIndicator.class);

@Autowired
private ScalityAppEnv appEnv;

@Override
public Health health() {
HttpURLConnection connection = null;
try {
// check if Utapi Endpoint is reachable
connection = (HttpURLConnection) new java.net.URL(appEnv.getUtapiEndpoint()).openConnection();
connection.setConnectTimeout(appEnv.getUtapiHealthCheckTimeout());
connection.connect();
} catch (SocketTimeoutException e) {
logger.warn("Failed to connect to Utapi endpoint {} in timeout {}",appEnv.getUtapiEndpoint(), appEnv.getUtapiHealthCheckTimeout());
return Health.down()
.withDetail("error", e.getMessage())
.build();
} catch (IOException e) {
logger.warn("an I/O error occurs while trying to connect {}.",appEnv.getUtapiEndpoint());
return Health.down()
.withDetail("error", e.getMessage())
.build();
} finally {
if (connection != null) {
connection.disconnect();
}
}
return Health.up().build();
// We want to make sure healthcheck is false even if it is set in application.properties
// This will be enabled and potentially refactored with S3C-8266
return Health.down().build();
}
}
8 changes: 4 additions & 4 deletions osis-app/src/main/java/com/scality/osis/resource/ScalityOsisController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,13 +339,13 @@ public OsisTenant getTenant(
},
tags = {"usage", "optional"})
@GetMapping(value = "/api/v1/usage", produces = "application/json")
// getUsage API will be re-enabled in S3C-8035
@NotImplement(name = ScalityOsisConstants.GET_USAGE_API_CODE)
public OsisUsage getUsage(
@Parameter(description = "The ID of the tenant to get its usage.") @Valid @RequestParam(value = "tenant_id", required = false) Optional<String> tenantId,
@Parameter(description = "The ID of the user to get its usage.") @Valid @RequestParam(value = "user_id", required = false) Optional<String> userId) {
if (tenantId.isEmpty() && userId.isPresent()) {
throw new BadRequestException("userId must be specified with associated tenantId!");
}
return osisService.getOsisUsage(tenantId, userId);
logger.info("Get Osis Usage request received:: tenant ID:{}, user ID:{}", tenantId, userId);
throw new NotImplementedException();
}

/*
Expand Down
4 changes: 2 additions & 2 deletions osis-core/src/main/java/com/scality/osis/model/ScalityOsisConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@ private ScalityOsisConstants() {
}

public static final String DELETE_TENANT_API_CODE = "deleteTenant";
public static final String GET_BUCKET_LIST_API_CODE = "getBucketList";
public static final String GET_USAGE_API_CODE = "getUsage";
public static final String GET_BUCKET_ID_LOGGING_API_CODE = "getBucketLoggingId";

public static final List<String> API_CODES = Arrays.asList(
DELETE_TENANT_API_CODE,
GET_BUCKET_LIST_API_CODE,
GET_USAGE_API_CODE,
GET_BUCKET_ID_LOGGING_API_CODE
);
}
111 changes: 1 addition & 110 deletions osis-core/src/main/java/com/scality/osis/service/impl/ScalityOsisServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1229,116 +1229,7 @@ public PageOfOsisBucketMeta getBucketList(String tenantId, long offset, long lim

@Override
public OsisUsage getOsisUsage(Optional<String> tenantId, Optional<String> userId) {

if (!appEnv.isUtapiEnabled()) {
logger.info("Utapi is not enabled, returning empty usage");
return new OsisUsage();
}

logger.info("Get Osis Usage request received:: tenant ID:{}, user ID:{}", tenantId, userId);
OsisUsage osisUsage = new OsisUsage();

if (tenantId.isEmpty()) {
logger.info("tenant ID is empty, getting platform usage at global level");

int start = 0;
int limit = 100;
boolean hasMoreTenants = true;
while (hasMoreTenants) {
PageOfTenants pageOfTenants = listTenants(start, limit);
pageOfTenants.getItems().forEach(tenant -> {
logger.info("Getting usage for tenant:{}", tenant.getTenantId());
OsisUsage tenantOsisUsage = getOsisUsage(Optional.of(tenant.getTenantId()), Optional.empty());
logger.info("Usage for tenant:{} is:{}", tenant.getTenantId(), new Gson().toJson(tenantOsisUsage));

logger.debug("Consolidating usage for tenant:{}", tenant.getTenantId());
osisUsage.consolidateUsage(tenantOsisUsage);
logger.debug("Consolidated usage:{}", new Gson().toJson(osisUsage));
});
start += limit;
hasMoreTenants = pageOfTenants.getPageInfo().getTotal() >= limit;
}

logger.info("Usage for all tenants is:{}", new Gson().toJson(osisUsage));

} else {

try {
// getting utapi client for tenant
Credentials tempCredentials = getCredentials(tenantId.get());
final UtapiServiceClient utapiClient = this.utapi.getUtapiServiceClient(tempCredentials,
appEnv.getRegionInfo().get(0));

if (userId.isEmpty()) {

logger.info("tenant ID is specified, getting usage at tenant level, tenant ID:{}", tenantId.get());

// getting s3 client for tenant
final AmazonS3 s3Client = this.s3.getS3Client(tempCredentials,
appEnv.getRegionInfo().get(0));

logger.debug("Listing buckets for tenant:{}", tenantId.get());
List<Bucket> buckets = s3Client.listBuckets();

// set bucket count by the size of listing buckets
osisUsage.setBucketCount((long) buckets.size());

logger.debug("[UTAPI] Listing metrics for tenant:{}", tenantId.get());
ListMetricsRequestDTO listMetricsRequestDTO = ScalityModelConverter.toScalityListMetricsRequest(
"accounts",
List.of(tenantId.get()),
List.of(getHourTime())
);
Response<MetricsData[]> listMetricsResponseDTO = utapiClient.listAccountsMetrics(listMetricsRequestDTO);
MetricsData metricsData = listMetricsResponseDTO.getAwsResponse()[0];
logger.debug("[UTAPI] List Metrics response:{}", new Gson().toJson(metricsData));

// set object count and used bytes by the utapi metrics
osisUsage.setObjectCount(metricsData.getNumberOfObjects().get(1));
osisUsage.setUsedBytes(metricsData.getStorageUtilized().get(1));

AccountData account = vaultAdmin.getAccount(ScalityModelConverter.toGetAccountRequestWithID(tenantId.get()));
if (account.getQuota() > 0) {
osisUsage.setTotalBytes((long) account.getQuota()); // set total bytes by the vault quota
osisUsage.setAvailableBytes(osisUsage.getTotalBytes() - osisUsage.getUsedBytes()); // set available bytes by the difference between total bytes and used bytes
}
logger.info("Usage for tenant:{} is:{}", tenantId.get(), new Gson().toJson(osisUsage));

} else {

logger.info("tenant ID and user ID are specified, getting usage at user level, tenant ID:{}, user ID:{}", tenantId.get(), userId.get());

logger.debug("[UTAPI] Listing metrics for user:{}", userId.get());
ListMetricsRequestDTO listMetricsRequestDTO = ScalityModelConverter.toScalityListMetricsRequest(
"users",
List.of(userId.get()),
List.of(getHourTime())
);
Response<MetricsData[]> listMetricsResponseDTO = utapiClient.listUsersMetrics(listMetricsRequestDTO);
MetricsData metricsData = listMetricsResponseDTO.getAwsResponse()[0];
logger.debug("[UTAPI] List Metrics response:{}", new Gson().toJson(metricsData));

// set object count and used bytes by the utapi metrics
osisUsage.setObjectCount(metricsData.getNumberOfObjects().get(1));
osisUsage.setUsedBytes(metricsData.getStorageUtilized().get(1));

logger.info("Usage for user:{} is:{}", userId.get(), new Gson().toJson(osisUsage));
}
} catch (Exception err) {
if (isAdminPolicyError(err)) {
try {
generateAdminPolicy(tenantId.get());
return getOsisUsage(tenantId, userId);
} catch (Exception ex) {
err = ex;
}
}
logger.error("GetUsage error. Returning empty list. Error details: ", err);
// For errors, GetUsage should return empty OsisUsage
return osisUsage;
}
}
return osisUsage;
throw new NotImplementedException();
}

@Override
Expand Down
22 changes: 22 additions & 0 deletions osis-core/src/test/java/com/scality/osis/service/impl/ScalityOsisServiceMiscTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
import com.scality.osis.utapiclient.dto.MetricsData;
import com.scality.osis.vaultadmin.impl.VaultServiceException;
import com.scality.vaultclient.dto.*;

import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
Expand All @@ -27,6 +29,8 @@
import static com.scality.osis.utils.ScalityConstants.IAM_PREFIX;
import static com.scality.osis.utils.ScalityTestUtils.*;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.*;

class ScalityOsisServiceMiscTests extends BaseOsisServiceTest {
Expand Down Expand Up @@ -173,7 +177,19 @@ void testGetBucketListErr() {
assertEquals(0L, response.getItems().size());
}


// test to check if getUsage API throws an error not implemented
// this will be removed as a part of S3C-8266
@Test
void testGetUsage() {
// Setup
assertThrows(NotImplementedException.class, () -> scalityOsisServiceUnderTest.getOsisUsage(Optional.of(SAMPLE_TENANT_ID), Optional.of(SAMPLE_CD_TENANT_ID)),
NOT_IMPLEMENTED_EXCEPTION_ERR);
}

@Test
@Disabled
// This will be enable with S3C-8266
void testGetOsisUsageForAll() {

// Setup
Expand Down Expand Up @@ -224,6 +240,8 @@ void testGetOsisUsageForAll() {
}

@Test
@Disabled
// This will be enable with S3C-8266
void testGetOsisUsageForTenant() {

// Setup
Expand Down Expand Up @@ -263,6 +281,8 @@ void testGetOsisUsageForTenant() {
}

@Test
@Disabled
// This will be enable with S3C-8266
void testGetOsisUsageForUser() {

// Setup
Expand All @@ -281,6 +301,8 @@ void testGetOsisUsageForUser() {
}

@Test
@Disabled
// This will be enable with S3C-8266
void testGetOsisUsageErr() {

// Setup
Expand Down
Loading