Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Feature] Support count data size of db and table via /api/show_data #41334

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+121 −30
Open

[Feature] Support count data size of db and table via /api/show_data #41334

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 48 additions & 21 deletions fe/fe-core/src/main/java/com/starrocks/http/rest/ShowDataAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,12 @@
import com.starrocks.server.GlobalStateMgr;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpResponseStatus;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.List;
import java.util.Collection;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;

public class ShowDataAction extends RestBaseAction {
private static final Logger LOG = LogManager.getLogger(ShowDataAction.class);

public ShowDataAction(ActionController controller) {
super(controller);
Expand All @@ -64,44 +62,73 @@ public static void registerAction(ActionController controller) throws IllegalArg
}

public long getDataSizeOfDatabase(Database db) {
long totalSize = 0;
Locker locker = new Locker();
locker.lockDatabase(db, LockType.READ);
try {
// sort by table name
List<Table> tables = db.getTables();
for (Table table : tables) {
if (!table.isNativeTableOrMaterializedView()) {
continue;
}
long tableSize = ((OlapTable) table).getDataSize();
totalSize += tableSize;
} // end for tables
return db.getTables().stream()
.filter(Table::isNativeTableOrMaterializedView)
.mapToLong(table -> ((OlapTable) table).getDataSize())
.sum();
} finally {
locker.unLockDatabase(db, LockType.READ);
}
return totalSize;
}

@Override
public void execute(BaseRequest request, BaseResponse response) {
String dbName = request.getSingleParameter("db");
ConcurrentHashMap<String, Database> fullNameToDb = GlobalStateMgr.getCurrentState().getLocalMetastore().getFullNameToDb();
long totalSize = 0;
String tableName = request.getSingleParameter("table");
long totalSize;
if (dbName != null) {
Database db = fullNameToDb.get(dbName);
if (db == null) {
response.getContent().append("database " + dbName + " not found.");
response.getContent().append("database ").append(dbName).append(" not found.");
sendResult(request, response, HttpResponseStatus.NOT_FOUND);
return;
}
totalSize = getDataSizeOfDatabase(db);
if (tableName == null) {
totalSize = getDataSizeOfDatabase(db);
} else {
Optional<Table> tableOptional = db.getTables().stream()
.filter(t -> t.getName().equals(tableName))
.findAny();
if (tableOptional.isEmpty()) {
response.getContent().append("table ").append(tableName).append(" not found.");
sendResult(request, response, HttpResponseStatus.NOT_FOUND);
return;
} else {
totalSize = getDataSizeOfTable(tableOptional.get());
}
}
} else {
for (Database db : fullNameToDb.values()) {
totalSize += getDataSizeOfDatabase(db);
if (tableName == null) {
totalSize = getDataSizeOfDatabases(fullNameToDb.values());
} else {
totalSize = fullNameToDb.values()
.parallelStream()
.map(Database::getTables)
.flatMap(Collection::stream)
.filter(Table::isNativeTableOrMaterializedView)
.filter(t -> t.getName().equals(tableName))
.mapToLong(this::getDataSizeOfTable)
.sum();
}
}
response.getContent().append(String.valueOf(totalSize));
response.getContent().append(totalSize);
sendResult(request, response);
}

private long getDataSizeOfTable(Table table) {
if (!table.isNativeTableOrMaterializedView()) {
return 0;
}
return ((OlapTable) table).getDataSize();
}

private long getDataSizeOfDatabases(Collection<Database> dbs) {
return dbs.parallelStream()
.mapToLong(this::getDataSizeOfDatabase)
.sum();
}
}
82 changes: 73 additions & 9 deletions fe/fe-core/src/test/java/com/starrocks/http/ShowDataActionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,19 +32,33 @@ public class ShowDataActionTest extends StarRocksHttpTestCase {

private static final String SHOW_DATA_URI = "/show_data";
private static final String SHOW_DATA_DB_NAME = "showdatadb";
private long expectedSize = 0;

private static final String SHOW_DATA_TABLE_NAME_SECOND = "ShowDataTable2";

private static final String SHOW_DATA_TABLE_NAME = "ShowDataTable";
private long expectedShowDataTableSize = 0;

@Override
public void doSetUp() {
Database db = new Database(1000 + testDbId, SHOW_DATA_DB_NAME);
OlapTable table = newTable("ShowDataTable");
OlapTable table = newTable(SHOW_DATA_TABLE_NAME);
db.registerTableUnlocked(table);
expectedSize = table.getDataSize();

OlapTable olapTable = newTable(SHOW_DATA_TABLE_NAME_SECOND);
db.registerTableUnlocked(olapTable);

expectedShowDataTableSize = olapTable.getDataSize();

Database db1 = new Database(1000 + testDbId, SHOW_DATA_DB_NAME + "!");

db1.registerTableUnlocked(olapTable);


// inject our test db
ConcurrentHashMap<String, Database> fullNameToDb = GlobalStateMgr.getCurrentState()
.getLocalMetastore().getFullNameToDb();
fullNameToDb.put(SHOW_DATA_DB_NAME, db);
fullNameToDb.put(SHOW_DATA_DB_NAME + "1", db1);
}

@Test
Expand All @@ -55,11 +69,61 @@ public void testGetShowData() throws IOException {
.addHeader("Authorization", rootAuth)
.url(BASE_URL + "/api" + SHOW_DATA_URI + "?db=" + SHOW_DATA_DB_NAME)
.build();
Response response = networkClient.newCall(request).execute();
assertTrue(response.isSuccessful());
Assert.assertNotNull(response.body());
String respStr = response.body().string();
Assert.assertNotNull(respStr);
Assert.assertEquals(String.valueOf(expectedSize), respStr);

try (Response response = networkClient.newCall(request).execute()) {
assertTrue(response.isSuccessful());
Assert.assertNotNull(response.body());
String respStr = response.body().string();
Assert.assertNotNull(respStr);
Assert.assertEquals(String.valueOf(expectedShowDataTableSize), respStr);
}
}

@Test
public void testShowDbAndTableSize() throws IOException {
Request request = new Request.Builder()
.get()
.addHeader("Authorization", rootAuth)
.url(BASE_URL + "/api" + SHOW_DATA_URI + "?db=" + SHOW_DATA_DB_NAME + "&table=" + SHOW_DATA_TABLE_NAME_SECOND)
.build();
try (Response response = networkClient.newCall(request).execute()) {
assertTrue(response.isSuccessful());
Assert.assertNotNull(response.body());
String respStr = response.body().string();
Assert.assertNotNull(respStr);
Assert.assertEquals(String.valueOf(expectedShowDataTableSize), respStr);
}
}

@Test
public void testShowTableSize() throws IOException {
Request request = new Request.Builder()
.get()
.addHeader("Authorization", rootAuth)
.url(BASE_URL + "/api" + SHOW_DATA_URI + "?table=" + SHOW_DATA_TABLE_NAME_SECOND)
.build();
try (Response response = networkClient.newCall(request).execute()) {
assertTrue(response.isSuccessful());
Assert.assertNotNull(response.body());
String respStr = response.body().string();
Assert.assertNotNull(respStr);
Assert.assertEquals(String.valueOf(expectedShowDataTableSize * 2), respStr);
}
}

@Test
public void testShowAllSize() throws IOException {
Request request = new Request.Builder()
.get()
.addHeader("Authorization", rootAuth)
.url(BASE_URL + "/api" + SHOW_DATA_URI + "?db=" + SHOW_DATA_DB_NAME)
.build();
try (Response response = networkClient.newCall(request).execute()) {
assertTrue(response.isSuccessful());
Assert.assertNotNull(response.body());
String respStr = response.body().string();
Assert.assertNotNull(respStr);
Assert.assertEquals(String.valueOf(expectedShowDataTableSize), respStr);
}
}
}
Loading