Skip to content

Commit

Permalink
modify testcases and verify highlevel bug (#592)
Browse files Browse the repository at this point in the history 
Signed-off-by: yongpengli-z <[email protected]>
  • Loading branch information
@yongpengli-z
yongpengli-z committed Aug 23, 2023
1 parent b57a141 commit d594dec
Show file tree
Hide file tree
Showing 16 changed files with 473 additions and 289 deletions.
57 changes: 46 additions & 11 deletions tests/milvustest/src/main/java/com/zilliz/milvustest/common/CommonFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@
import com.zilliz.milvustest.entity.FileBody;
import com.zilliz.milvustest.entity.MilvusEntity;
import com.zilliz.milvustest.util.MathUtil;
import com.zilliz.milvustest.util.PropertyFilesUtil;
import io.milvus.client.MilvusServiceClient;
import io.milvus.grpc.DataType;
import io.milvus.param.*;
import io.milvus.param.alias.CreateAliasParam;
Expand All @@ -18,8 +16,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

import javax.swing.text.FlowView;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
Expand Down Expand Up @@ -485,23 +481,33 @@ public static String provideExtraParam(IndexType indexType) {
case IVF_PQ:
extraParm = "{\"nlist\":128, \"m\":16, \"nbits\":8}";
break;
case ANNOY:
/* case ANNOY:
extraParm = "{\"n_trees\":16}";
break;
break;*/
case HNSW:
extraParm = "{\"M\":16,\"efConstruction\":64}";
break;
case RHNSW_FLAT:
/* case RHNSW_FLAT:
extraParm = "{\"M\":16,\"efConstruction\":64}";
break;
case RHNSW_PQ:
break;*/
/* case RHNSW_PQ:
extraParm = "{\"M\":16,\"efConstruction\":64, \"PQM\":16}";
break;
case RHNSW_SQ:
extraParm = "{\"M\":16,\"efConstruction\":64}";
break;
break;*/
case BIN_IVF_FLAT:
extraParm = "";
extraParm = "{\"nlist\": 128}";
break;
case SCANN:
extraParm="{\"nlist\":1024}";
break;
case GPU_IVF_FLAT:
extraParm="{\"nlist\": 64}";
break;
case GPU_IVF_PQ:
extraParm="{\"nlist\": 64, \"m\": 16, \"nbits\": 8}";
break;
default:
extraParm = "{\"nlist\":128}";
break;
Expand Down Expand Up @@ -686,6 +692,35 @@ public static List<JSONObject> generateDataWithDynamicFiledRow(int num) {
}
return jsonList;
}
public static List<JSONObject> generateVarcharPKDataWithDynamicFiledRow(int num) {
List<JSONObject> jsonList = new ArrayList<>();
Random ran = new Random();
for (int i = 0; i < num; i++) {
JSONObject row = new JSONObject();
row.put("book_name", "StringPK"+i);
row.put("word_count", (long) i);
row.put("extra_field", "String" + i);
row.put("extra_field2", i);
// $innerJson
JSONObject innerJson = new JSONObject();
innerJson.put("int64", (long) i);
innerJson.put("varchar", "varchar"+i);
innerJson.put("int16", i);
innerJson.put("int32", i);
innerJson.put("int8", (short)i);
innerJson.put("float", (float)i);
innerJson.put("double", (double)i);
innerJson.put("bool", i % 2 == 0);
row.put("json_field", innerJson);
List<Float> vector = new ArrayList<>();
for (int k = 0; k < 128; ++k) {
vector.add(ran.nextFloat());
}
row.put("book_intro", vector);
jsonList.add(row);
}
return jsonList;
}
public static void createIndexWithLoad(String collection, IndexType indexType, MetricType metricType, String fieldName){
R<RpcStatus> rpcStatusR =
milvusClient.createIndex(
Expand Down
20 changes: 9 additions & 11 deletions tests/milvustest/src/main/java/com/zilliz/milvustest/service/CustomerListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ public class CustomerListener extends TestListenerAdapter {
@Override
public void onStart(ITestContext iTestContext) {
super.onStart(iTestContext);
startTime=System.currentTimeMillis();
logger.info("===================={}测试开始====================", iTestContext.getName());
}

Expand All @@ -40,7 +39,6 @@ public void onStart(ITestContext iTestContext) {
public void onTestStart(ITestResult iTestResult) {
super.onTestStart(iTestResult);
logger.info("========{}测试开始========", iTestResult.getName());
totalCase++;
}

/**
Expand All @@ -52,7 +50,6 @@ public void onTestStart(ITestResult iTestResult) {
public void onTestSuccess(ITestResult iTestResult) {
super.onTestSuccess(iTestResult);
logger.info("========{}测试通过========", iTestResult.getName());
passCase++;
}

/**
Expand Down Expand Up @@ -87,37 +84,38 @@ public void onTestSkipped(ITestResult iTestResult) {
public void onFinish(ITestContext iTestContext) {
super.onFinish(iTestContext);
logger.info("===================={}测试结束====================", iTestContext.getName());
endTime=System.currentTimeMillis();
// insert result into db
double passRate= passCase*100.00 / totalCase;
DecimalFormat df=new DecimalFormat("0.00");
int costTime= (int) ((endTime-startTime)/1000/60);
double passRate= iTestContext.getPassedTests().size()*100.00 / (iTestContext.getPassedTests().size()+iTestContext.getFailedTests().size()+iTestContext.getSkippedTests().size());
int costTime= (int) ((iTestContext.getEndDate().getTime()-iTestContext.getStartDate().getTime())/1000/60);
String scenarioDesc=System.getProperty("ScenarioDesc") == null
? PropertyFilesUtil.getRunValue("ScenarioDesc")
: System.getProperty("ScenarioDesc");
String BuildId=System.getProperty("BuildId") == null
? PropertyFilesUtil.getRunValue("BuildId")
: System.getProperty("BuildId");
String SDKBranch=System.getProperty("SDKBranch") == null
? PropertyFilesUtil.getRunValue("SDKBranch")
: System.getProperty("SDKBranch");
String jenkinsLink="https://qa-jenkins.milvus.io/job/Java-sdk-test-nightly/"+BuildId+"/";
String githubLink="https://github.com/milvus-io/milvus-sdk-java/actions/workflows/java_sdk_ci_test.yaml";
JSONObject request=new JSONObject();
request.put("Product","Milvus");
request.put("Category","Function");
request.put("Date", LocalDate.now().toString());
request.put("Scenario",scenarioDesc);
request.put("Branch","Master");
request.put("Branch",SDKBranch);
request.put("ImageName","2.2.0");
request.put("SDK","java");
request.put("MilvusMode","standalone");
request.put("MqMode","kafka");
request.put("MqMode","rocksMq");
request.put("TestResult",passRate==100?"pass":"fail");
request.put("PassRate", passRate);
request.put("RunningTime", costTime);
request.put("Link",scenarioDesc.equalsIgnoreCase("CI")?githubLink:jenkinsLink);
String s = HttpClientUtils.doPostJson("http://qtp-server.zilliz.cc/results/insert",request.toJSONString());
logger.info("insert result:"+s);
if(iTestContext.getFailedTests().size()>0){
System.exit(1);
if (iTestContext.getFailedTests().size()>0){
throw new RuntimeException("Case Failed "+iTestContext.getFailedTests().size());
}
}
}
32 changes: 8 additions & 24 deletions tests/milvustest/src/test/java/com/zilliz/milvustest/index/CreateIndexTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,16 +50,13 @@ public void dropCollection() {
@DataProvider(name = "IndexTypes")
public Object[][] provideIndexType() {
return new Object[][] {
{IndexType.FLAT},
{IndexType.IVF_FLAT},
{IndexType.IVF_SQ8},
{IndexType.IVF_PQ},
{IndexType.HNSW},
/* {IndexType.ANNOY},
{IndexType.RHNSW_FLAT},
{IndexType.RHNSW_PQ},
{IndexType.RHNSW_SQ},*/
{IndexType.DISKANN}
{IndexType.IVF_FLAT},
{IndexType.IVF_SQ8},
{IndexType.IVF_PQ},
{IndexType.HNSW},
{IndexType.SCANN},
{IndexType.GPU_IVF_FLAT},
{IndexType.GPU_IVF_PQ}
};
}

Expand All @@ -77,10 +74,7 @@ public Object[][] providerMetricType() {
public Object[][] providerBinaryMetricType() {
return new Object[][] {
{MetricType.HAMMING},
{MetricType.JACCARD},
{MetricType.SUBSTRUCTURE},
{MetricType.SUPERSTRUCTURE},
{MetricType.TANIMOTO}
{MetricType.JACCARD}
};
}

Expand Down Expand Up @@ -150,11 +144,6 @@ public void createIndexAsync(IndexType indexType, MetricType metricType) {
@Issue("https://github.com/milvus-io/milvus-sdk-java/issues/321")
@Test(description = "Create index for collection Async", dataProvider = "BinaryIndex")
public void createBinaryIndexAsync(IndexType indexType, MetricType metricType) {
if (indexType.equals(IndexType.BIN_IVF_FLAT)
&& (metricType.equals(MetricType.SUBSTRUCTURE)
|| metricType.equals(MetricType.SUPERSTRUCTURE))) {
return;
}
R<RpcStatus> rpcStatusR =
milvusClient.createIndex(
CreateIndexParam.newBuilder()
Expand All @@ -180,11 +169,6 @@ public void createBinaryIndexAsync(IndexType indexType, MetricType metricType) {
@Issue("https://github.com/milvus-io/milvus-sdk-java/issues/321")
@Test(description = "Create index for collection sync", dataProvider = "BinaryIndex")
public void createBinaryIndexSync(IndexType indexType, MetricType metricType) {
if (indexType.equals(IndexType.BIN_IVF_FLAT)
&& (metricType.equals(MetricType.SUBSTRUCTURE)
|| metricType.equals(MetricType.SUPERSTRUCTURE))) {
return;
}
R<RpcStatus> rpcStatusR =
milvusClient.createIndex(
CreateIndexParam.newBuilder()
Expand Down
21 changes: 6 additions & 15 deletions tests/milvustest/src/test/java/com/zilliz/milvustest/index/IndexLoadTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,13 @@ public void dropCollection() {
@DataProvider(name = "IndexTypes")
public Object[][] provideIndexType() {
return new Object[][] {
{IndexType.FLAT},
{IndexType.IVF_FLAT},
{IndexType.IVF_SQ8},
{IndexType.IVF_PQ},
{IndexType.HNSW},
/* {IndexType.ANNOY},
{IndexType.RHNSW_FLAT},
{IndexType.RHNSW_PQ},
{IndexType.RHNSW_SQ}*/
{IndexType.SCANN},
{IndexType.GPU_IVF_FLAT},
{IndexType.GPU_IVF_PQ}
};
}

Expand All @@ -71,10 +69,7 @@ public Object[][] providerMetricType() {
public Object[][] providerBinaryMetricType() {
return new Object[][] {
{MetricType.HAMMING},
{MetricType.JACCARD},
{MetricType.SUBSTRUCTURE},
{MetricType.SUPERSTRUCTURE},
{MetricType.TANIMOTO}
{MetricType.JACCARD}
};
}

Expand Down Expand Up @@ -160,13 +155,9 @@ public void createIndexAfterLoadFloatCollection(IndexType indexType, MetricType
}

@Severity(SeverityLevel.NORMAL)
@Test(description = "Test create/drop index when collection is loaded for binary vector", dataProvider = "BinaryIndex",groups = {"Smoke"})
@Test(description = "Test create/drop index when collection is loaded for binary vector",
dataProvider = "BinaryIndex",groups = {"Smoke"})
public void createIndexAfterLoadBinaryCollection(IndexType indexType, MetricType metricType) {
if (indexType.equals(IndexType.BIN_IVF_FLAT)
&& (metricType.equals(MetricType.SUBSTRUCTURE)
|| metricType.equals(MetricType.SUPERSTRUCTURE))) {
return;
}
// 1. create index params
CreateIndexParam createIndexParams = CreateIndexParam.newBuilder()
.withCollectionName(binaryCollection)
Expand Down
2 changes: 1 addition & 1 deletion tests/milvustest/src/test/java/com/zilliz/milvustest/insert/DeleteTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.util.List;

@Epic("Index")
@Epic("Insert")
@Feature("Delete")
public class DeleteTest extends BaseTest {
public String commonCollection;
Expand Down
2 changes: 1 addition & 1 deletion tests/milvustest/src/test/java/com/zilliz/milvustest/insert/InsertAsyncTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import java.util.Random;
import java.util.concurrent.ExecutionException;

@Epic("Index")
@Epic("Insert")
@Feature("InsertAsyncT")
public class InsertAsyncTest extends BaseTest {
public String stringPKAndBinaryCollection;
Expand Down
2 changes: 1 addition & 1 deletion tests/milvustest/src/test/java/com/zilliz/milvustest/insert/InsertTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
import java.util.List;
import java.util.Random;

@Epic("Index")
@Epic("Insert")
@Feature("Insert")
public class InsertTest extends BaseTest {
public String stringPKAndBinaryCollection;
Expand Down
Loading

0 comments on commit d594dec

Please sign in to comment.