Add functionality in benchmarking code to optionally provide a hand-o…

…ptimized DDlog program file. In that case, /tmp/program.dl is not compiled using sql-ddlog compiler. Instead, we just copy over contents of the hand-optimized file into /tmp/program.dl.

Signed-off-by: Amy Tai <[email protected]>

k8s-scheduler/src/main/java/com/vmware/dcm/DDlogDBConnectionPool.java

@@ -9,6 +9,7 @@
 import com.vmware.ddlog.util.sql.H2SqlStatement;
 import ddlogapi.DDlogAPI;
 import ddlogapi.DDlogException;
+import org.apache.commons.io.FileUtils;
 import org.jooq.DSLContext;
 import org.jooq.SQLDialect;
 import org.jooq.conf.ParamCastMode;
@@ -17,10 +18,9 @@
 import org.jooq.impl.DSL;
 import org.jooq.tools.jdbc.MockConnection;
 
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
+import javax.annotation.Nullable;
+import java.io.*;
+import java.nio.Buffer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -33,11 +33,18 @@ public class DDlogDBConnectionPool implements IConnectionPool {
     private final List<String> scopedViews;
 
     private DDlogJooqProvider provider;
+    private final String ddlogFile;
 
     public DDlogDBConnectionPool() {
+        this.ddlogFile = null;
         scopedViews = new ArrayList<>();
     }
 
+    public DDlogDBConnectionPool(String ddlogFile) {
+        this.ddlogFile = ddlogFile;
+        this.scopedViews = new ArrayList<>();
+    }
+
     @Override
     public DSLContext getConnectionToDb() {
         final MockConnection connection = new MockConnection(provider);
@@ -61,19 +68,26 @@ public void addScopedViews(List<String> statements) {
         scopedViews.addAll(statements);
     }
 
-    private static void compileAndLoad(final List<CalciteSqlStatement> ddl, final List<String> createIndexStatements)
+    private static void compileAndLoad(final List<CalciteSqlStatement> ddl, final List<String> createIndexStatements,
+                                       final String ddlogFile)
             throws IOException, DDlogException {
-        final Translator t = new Translator(null);
-        CalciteToPrestoTranslator ctopTranslator = new CalciteToPrestoTranslator();
-        ddl.forEach(x -> t.translateSqlStatement(ctopTranslator.toPresto(x)));
-        createIndexStatements.forEach(t::translateCreateIndexStatement);
-
-        final DDlogProgram dDlogProgram = t.getDDlogProgram();
         final String fileName = "/tmp/program.dl";
-        File tmp = new File(fileName);
-        BufferedWriter bw = new BufferedWriter(new FileWriter(tmp));
-        bw.write(dDlogProgram.toString());
-        bw.close();
+        if (ddlogFile != null) {
+            // don't compile, instead use this ddlogFile
+            // Copy ddlogFile contents to fileName
+            FileUtils.copyFile(new File(ddlogFile), new File(fileName));
+        } else {
+            final Translator t = new Translator(null);
+            CalciteToPrestoTranslator ctopTranslator = new CalciteToPrestoTranslator();
+            ddl.forEach(x -> t.translateSqlStatement(ctopTranslator.toPresto(x)));
+            createIndexStatements.forEach(t::translateCreateIndexStatement);
+
+            final DDlogProgram dDlogProgram = t.getDDlogProgram();
+            File tmp = new File(fileName);
+            BufferedWriter bw = new BufferedWriter(new FileWriter(tmp));
+            bw.write(dDlogProgram.toString());
+            bw.close();
+        }
         DDlogAPI.CompilationResult result = new DDlogAPI.CompilationResult(true);
         final String ddlogHome = System.getenv("DDLOG_HOME");
         assertNotNull(ddlogHome);
@@ -104,7 +118,7 @@ private void setupDDlog(boolean seal) {
 
             DDlogAPI ddlogAPI = null;
             if (seal) {
-                compileAndLoad(tablesInCalcite, createIndexStatements);
+                compileAndLoad(tablesInCalcite, createIndexStatements, ddlogFile);
 
                 ddlogAPI = new DDlogAPI(1, false);
             }

k8s-scheduler/src/main/java/com/vmware/dcm/EmulatedCluster.java

@@ -51,9 +51,9 @@ class EmulatedCluster {
 
     public void runTraceLocally(final int numNodes, final String traceFileName, final int cpuScaleDown,
                                 final int memScaleDown, final int timeScaleDown, final int startTimeCutOff,
-                                final int affinityRequirementsProportion, final boolean scopeOn)
+                                final int affinityRequirementsProportion, final boolean scopeOn, final String ddlogFile)
             throws Exception {
-        final IConnectionPool dbConnectionPool = new DDlogDBConnectionPool(); // new DBConnectionPool();
+        final IConnectionPool dbConnectionPool = new DDlogDBConnectionPool(ddlogFile); // new DBConnectionPool();
         if (dbConnectionPool instanceof DDlogDBConnectionPool) {
             ((DDlogDBConnectionPool) dbConnectionPool).buildDDlog(false);
         }
@@ -184,6 +184,8 @@ public static void runWorkload(final String[] args) throws Exception {
                 "P, from 0 to 100, indicating the proportion of pods that have affinity requirements");
         options.addOption("S", "scopeOn", false,
                 "enable auto-scope in scheduler");
+        options.addOption("df", "ddlogFile", true,
+                "specify which ddlog program.dl to use");
         final CommandLineParser parser = new DefaultParser();
         final CommandLine cmd = parser.parse(options, args);
         final int numNodes = Integer.parseInt(cmd.getOptionValue("numNodes"));
@@ -195,13 +197,15 @@ public static void runWorkload(final String[] args) throws Exception {
         final int affinityRequirementsProportion = Integer.parseInt(cmd.hasOption("proportion") ?
                 cmd.getOptionValue("proportion") : "0");
         final boolean scopeOn = cmd.hasOption("scopeOn");
+        final String ddlogFile = cmd.getOptionValue("ddlogFile");
+
         assert affinityRequirementsProportion >= 0 && affinityRequirementsProportion <= 100;
         LOG.info("Running experiment with parameters: numNodes: {}, traceFile: {}, cpuScaleDown: {}, " +
                         "memScaleDown: {}, timeScaleDown: {}, startTimeCutOff: {}, proportion: {}, scopeOn: {}",
                 numNodes, traceFile, cpuScaleDown, memScaleDown,
                 timeScaleDown, startTimeCutOff, affinityRequirementsProportion, scopeOn);
         emulatedCluster.runTraceLocally(numNodes, traceFile, cpuScaleDown, memScaleDown, timeScaleDown,
-                startTimeCutOff, affinityRequirementsProportion, scopeOn);
+                startTimeCutOff, affinityRequirementsProportion, scopeOn, ddlogFile);
     }
 
     public static void main(final String[] args) throws Exception {

k8s-scheduler/src/test/java/com/vmware/dcm/WorkloadReplayTest.java

@@ -12,7 +12,8 @@ public class WorkloadReplayTest {
     @Test
     public void runTest() throws Exception {
         final String[] args =
-                {"-n", "500", "-f", "test-data.txt", "-c", "100", "-m", "200", "-t", "100", "-s", "1000"};
+                {"-n", "500", "-f", "test-data.txt", "-c", "100", "-m", "200", "-t", "100", "-s", "1000", "-df",
+                "../artifacts/optimized_ddlog_programs/preempt_program.dl.dl"};
         EmulatedCluster.runWorkload(args);
     }