From 3490705922dd252fabcb87f4262af2a0eb581575 Mon Sep 17 00:00:00 2001
From: Sanjiban Sengupta <sanjiban.sg@gmail.com>
Date: Thu, 5 Oct 2023 15:15:59 +0530
Subject: [PATCH] feat: schema to use a JSON format and query should be
 generated in backend

---
 api/app.py                          |  20 +-
 api/backend/duckdb.py               |  27 ++-
 api/test.py                         |   7 +-
 client/src/components/SqlSchema.vue | 356 ++++++++++++++++++++++++----
 4 files changed, 362 insertions(+), 48 deletions(-)

diff --git a/api/app.py b/api/app.py
index 9571da40..f43f2598 100644
--- a/api/app.py
+++ b/api/app.py
@@ -3,6 +3,8 @@
 from fastapi.openapi.utils import get_openapi
 from fastapi_health import health
 
+import json
+
 from duckdb import DuckDBPyConnection
 
 import substrait_validator as sv
@@ -14,7 +16,6 @@
     ParseFromDuckDB,
 )
 
-1
 from loguru import logger
 
 
@@ -83,12 +84,27 @@ async def ValidateFile(file: UploadFile = File(), override_levels: list[int] = F
             status_code=500, detail="Substrait Validator Internal Error: " + str(e)
         )
 
-
 @app.post("/execute/duckdb/")
 def ExecuteBackend(data: dict, db_conn: DuckDBPyConnection = Depends(get_duck_conn)):
     response = ExecuteDuckDb(data, db_conn)
     return response
 
+@app.post("/add_schema/")
+def AddSchema(data: dict, db_conn: DuckDBPyConnection = Depends(get_duck_conn)):
+    schema = data["schema"]
+    query = "CREATE TEMP TABLE "
+    json_data = json.loads(schema["schema"])
+    query += json_data["table"] + "("
+    for field in json_data["fields"]:
+        query += field["name"] + " "
+        query += field["type"] + " "
+        for props in field["properties"]:
+            query += props + " "
+        query += ", "
+    query += ");"
+    response = ExecuteDuckDb(query, db_conn)
+    return response
+
 
 @app.post("/parse/", status_code=status.HTTP_200_OK)
 def ParseToSubstrait(data: dict, db_conn: DuckDBPyConnection = Depends(get_duck_conn)):
diff --git a/api/backend/duckdb.py b/api/backend/duckdb.py
index f7a314a9..9d44f320 100644
--- a/api/backend/duckdb.py
+++ b/api/backend/duckdb.py
@@ -1,5 +1,4 @@
 import duckdb
-from pydantic import BaseModel
 from fastapi import HTTPException
 
 from loguru import logger
@@ -8,12 +7,34 @@
 class DuckDBConnection:
     def __init__(self):
         self.conn_pool = []
+
+        query_lineitem = '''CREATE TABLE IF NOT EXISTS lineitem(
+                        l_orderkey INTEGER NOT NULL, 
+                        l_partkey INTEGER NOT NULL, 
+                        l_suppkey INTEGER NOT NULL, 
+                        l_linenumber INTEGER NOT NULL, 
+                        l_quantity INTEGER NOT NULL, 
+                        l_extendedprice DECIMAL(15,2) NOT NULL, 
+                        l_discount DECIMAL(15,2) NOT NULL, 
+                        l_tax DECIMAL(15,2) NOT NULL, 
+                        l_returnflag VARCHAR NOT NULL, 
+                        l_linestatus VARCHAR NOT NULL, 
+                        l_shipdate DATE NOT NULL, 
+                        l_commitdate DATE NOT NULL, 
+                        l_receiptdate DATE NOT NULL, 
+                        l_shipinstruct VARCHAR NOT NULL, 
+                        l_shipmode VARCHAR NOT NULL, 
+                        l_comment VARCHAR NOT NULL);'''
+        conn = duckdb.connect("duck.db")
+        conn.execute(query=query_lineitem)
+        
         for i in range(5):
             conn = duckdb.connect("duck.db")
             conn.install_extension("substrait")
             conn.load_extension("substrait")
             self.conn_pool.append(conn)
 
+
     def check_pool(self):
         if len(self.conn_pool) == 0:
             print("creating new conn objects")
@@ -44,9 +65,9 @@ def CheckDuckDBConnection(con):
         return status
 
 
-def ExecuteDuckDb(data, con):
+def ExecuteDuckDb(query, con):
     try:
-        con.execute(query=data["query"])
+        con.execute(query = query)
         return {"message": "DuckDB Operation successful"}
     except Exception as e:
         raise HTTPException(
diff --git a/api/test.py b/api/test.py
index a8639e33..0de8625a 100644
--- a/api/test.py
+++ b/api/test.py
@@ -1,6 +1,5 @@
 from fastapi.testclient import TestClient
 from io import BytesIO
-import pytest
 import requests
 from app import app
 
@@ -51,7 +50,8 @@ def test_duckdb_execute():
         res = client.post(
             "/execute/duckdb/",
             json={
-                "query": "CREATE TABLE IF NOT EXISTS weather(city VARCHAR, temp_lo INTEGER);",
+                "query": '''CREATE TABLE IF NOT EXISTS 
+                            weather(city VARCHAR, temp_lo INTEGER);''',
             },
         )
         print(res.json())
@@ -65,7 +65,8 @@ def test_parse_to_substrait():
         client.post(
             "/execute/duckdb/",
             json={
-                "query": "CREATE TABLE IF NOT EXISTS test_fiddle(id INTEGER NOT NULL, key INTEGER NOT NULL);",
+                "query": '''CREATE TABLE IF NOT EXISTS test_fiddle(
+                            id INTEGER NOT NULL, key INTEGER NOT NULL);''',
             },
         )
         response = client.post(
diff --git a/client/src/components/SqlSchema.vue b/client/src/components/SqlSchema.vue
index 674e97d1..21d620bd 100644
--- a/client/src/components/SqlSchema.vue
+++ b/client/src/components/SqlSchema.vue
@@ -21,7 +21,7 @@
         <div class="modal-content">
           <div class="modal-header">
             <h5 class="modal-title text-danger" id="schemaModalLabel">
-              Show/Modify Schema
+              Show/Add Schema
             </h5>
             <button
               type="button"
@@ -31,6 +31,72 @@
             ></button>
           </div>
           <div class="modal-body">
+            <div class="accordion" id="schemaAccordion">
+              <div
+                v-for="(schema, i) in schemas"
+                :key="schema.name"
+                class="accordion-item"
+              >
+                <h2 class="accordion-header" :id="`heading${schema.name}`">
+                  <button
+                    class="accordion-button"
+                    :class="{ collapsed: i !== activeAccordionItem }"
+                    type="button"
+                    @click="toggleAccordion(i)"
+                    :aria-expanded="i === activeAccordionItem"
+                    :aria-controls="`collapse${schema.name}`"
+                  >
+                    {{ schema.name }}
+                  </button>
+                </h2>
+                <div
+                  :id="`collapse${schema.name}`"
+                  class="accordion-collapse collapse"
+                  :class="{ show: i === activeAccordionItem }"
+                  :aria-labelledby="`heading${schema.name}`"
+                  data-bs-parent="#schemaAccordion"
+                >
+                  <pre class="accordion-body">{{ schema.schema }}</pre>
+
+                  <div v-if="i > 0 && !schema.validated">
+                    <textarea
+                      v-model="tempSchemaText"
+                      class="form-control"
+                      rows="4"
+                      placeholder="Enter new schema"
+                    ></textarea>
+                    <div class="d-flex justify-content-end">
+                      <button
+                        class="btn btn-primary mt-2 btn-sm"
+                        style="margin-bottom: 1%; margin-right: 1%"
+                        @click="validateSchema(i)"
+                      >
+                        Validate
+                      </button>
+                    </div>
+                  </div>
+
+                  <span
+                    v-if="i > 0"
+                    class="delete-icon"
+                    @click="deleteSchema(i)"
+                  >
+                    <svg
+                      xmlns="http://www.w3.org/2000/svg"
+                      width="16"
+                      height="16"
+                      fill="currentColor"
+                      class="bi bi-trash3-fill"
+                      viewBox="0 0 16 16"
+                    >
+                      <path
+                        d="M11 1.5v1h3.5a.5.5 0 0 1 0 1h-.538l-.853 10.66A2 2 0 0 1 11.115 16h-6.23a2 2 0 0 1-1.994-1.84L2.038 3.5H1.5a.5.5 0 0 1 0-1H5v-1A1.5 1.5 0 0 1 6.5 0h3A1.5 1.5 0 0 1 11 1.5Zm-5 0v1h4v-1a.5.5 0 0 0-.5-.5h-3a.5.5 0 0 0-.5.5ZM4.5 5.029l.5 8.5a.5.5 0 1 0 .998-.06l-.5-8.5a.5.5 0 1 0-.998.06Zm6.53-.528a.5.5 0 0 0-.528.47l-.5 8.5a.5.5 0 0 0 .998.058l.5-8.5a.5.5 0 0 0-.47-.528ZM8 4.5a.5.5 0 0 0-.5.5v8.5a.5.5 0 0 0 1 0V5a.5.5 0 0 0-.5-.5Z"
+                      />
+                    </svg>
+                  </span>
+                </div>
+              </div>
+            </div>
           </div>
           <div class="modal-footer">
             <button
@@ -40,12 +106,14 @@
             >
               Close
             </button>
+            <button type="button" class="btn btn-success" @click="addNewSchema">
+              Add Schema
+            </button>
             <button
               type="button"
               class="btn btn-primary"
               data-bs-dismiss="modal"
-              @click="saveSchema"
-              disabled
+              @click="saveChanges"
             >
               Save changes
             </button>
@@ -57,14 +125,41 @@
 </template>
 
 <style>
-#schemaTextArea {
+.pre {
+  white-space: pre-wrap;
   font-family: Consolas, Menlo, Monaco, Lucida Console, Liberation Mono,
     DejaVu Sans Mono, Bitstream Vera Sans Mono, Courier New, monospace, serif;
 }
 
+.modal-content {
+  height: 50vh;
+  overflow-y: auto;
+}
+
+.accordion-button {
+  background-color: #f8f9fa !important;
+}
+
+.accordion-body {
+  background-color: #ffffff;
+}
+
+.delete-icon {
+  cursor: pointer;
+  color: red;
+  position: absolute;
+  top: 10px;
+  right: 10px;
+}
+
+.form-control {
+  border: none;
+  resize: none;
+}
 </style>
 
 <script>
+import Ajv from "ajv";
 import axios from "axios";
 import "bootstrap/dist/js/bootstrap.js";
 
@@ -77,57 +172,238 @@ export default {
     return {
       tempSchema: "",
       currSchema: "",
-      defaultSchema: `CREATE TABLE lineitem(
-            l_orderkey INTEGER NOT NULL, 
-            l_partkey INTEGER NOT NULL, 
-            l_suppkey INTEGER NOT NULL, 
-            l_linenumber INTEGER NOT NULL, 
-            l_quantity INTEGER NOT NULL, 
-            l_extendedprice DECIMAL(15,2) NOT NULL, 
-            l_discount DECIMAL(15,2) NOT NULL, 
-            l_tax DECIMAL(15,2) NOT NULL, 
-            l_returnflag VARCHAR NOT NULL, 
-            l_linestatus VARCHAR NOT NULL, 
-            l_shipdate DATE NOT NULL, 
-            l_commitdate DATE NOT NULL, 
-            l_receiptdate DATE NOT NULL, 
-            l_shipinstruct VARCHAR NOT NULL, 
-            l_shipmode VARCHAR NOT NULL, 
-            l_comment VARCHAR NOT NULL);`,
-      otherSchema: [],
+      schemas: [],
+      activeAccordionItem: -1,
+      canSaveChanges: false,
+      newSchemaText: "",
+      tempSchemaText: "",
     };
   },
   mounted() {
     this.currSchema = this.defaultSchema;
     this.tempSchema = this.defaultSchema;
-  },
-  created() {
-    this.getData();
+    this.schemas.push({
+      name: "lineitem",
+      schema: `
+          {
+            table: "lineitem",
+            fields: [
+              {
+                name: "l_orderkey",
+                type: "INTEGER",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_partkey",
+                type: "INTEGER",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_suppkey",
+                type: "INTEGER",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_linenumber",
+                type: "INTEGER",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_quantity",
+                type: "INTEGER",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_extendedprice",
+                type: "DECIMAL(15,2)",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_discount",
+                type: "DECIMAL(15,2)",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_tax",
+                type: "DECIMAL(15,2)",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_returnflag",
+                type: "VARCHAR",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_linestatus",
+                type: "VARCHAR",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_shipdate",
+                type: "DATE",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_commitdate",
+                type: "DATE",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_receiptdate",
+                type: "DATE",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_shipinstruct",
+                type: "VARCHAR",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_shipmode",
+                type: "VARCHAR",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+              {
+                name: "l_comment",
+                type: "VARCHAR",
+                properties: [
+                  "NOT NULL"
+                ]
+              },
+            ]
+          }`,
+    });
   },
   methods: {
+    toggleAccordion(index) {
+      if (this.activeAccordionItem === index) {
+        this.activeAccordionItem = -1;
+      } else {
+        this.activeAccordionItem = index;
+      }
+    },
     getSchema() {
       return this.defaultSchema;
     },
     updateSchema() {
       this.tempSchema = this.defaultSchema;
     },
-    async saveSchema() {
-      if (this.tempSchema != this.defaultSchema) {
-        try {
-          this.defaultSchema = this.tempSchema;
-          await axios.post("/api/execute/duckdb/", [this.schema]);
-          this.$emit(
-            "updateSchemaStatus",
-            "Modified schema executed successfully!"
-          );
-        } catch (error) {
-          console.log(error);
-          this.$emit(
-            "updateSchemaStatus",
-            "Error while executing schema: " + error.response.data["detail"]
-          );
+    addNewSchema() {
+      this.schemas.push({
+        name: "New Schema",
+        schema: "",
+      });
+      this.activeAccordionItem = this.schemas.length - 1;
+      this.canSaveChanges = true;
+    },
+    deleteSchema(index) {
+      if (index >= 0 && index < this.schemas.length) {
+        this.schemas.splice(index, 1);
+        if (this.activeAccordionItem === index) {
+          this.activeAccordionItem = -1;
+        } else if (this.activeAccordionItem > index) {
+          this.activeAccordionItem--;
+        }
+      }
+    },
+    async saveChanges() {
+      if (this.schemas.length > 1) {
+        for (var i = 1; i < this.schemas.length; ++i) {
+          try {
+            await axios.post("/api/add_schema/", {
+              schema: this.schemas[i],
+            });
+          } catch (error) {
+            console.log(error);
+            this.$emit(
+              "updateSchemaStatus",
+              "Error while executing schema: " + error.response.data["detail"]
+            );
+          }
+        }
+      }
+    },
+    validateSchema(index) {
+      const ajv = new Ajv();
+      const format = {
+        type: "object",
+        properties: {
+          table: {
+            type: "string",
+          },
+          fields: {
+            type: "array",
+            items: {
+              type: "object",
+              properties: {
+                name: { type: "string" },
+                type: { type: "string" },
+                properties: {
+                  type: "array",
+                  items: { type: "string" },
+                },
+              },
+              required: ["name", "type", "properties"],
+              additionalProperties: false,
+            },
+          },
+        },
+        required: ["table", "fields"],
+        additionalProperties: false,
+      };
+
+      const validate = ajv.compile(format);
+      try {
+        const jsonData = JSON.parse(this.tempSchemaText);
+        if (jsonData.table === "lineitem") {
+          alert('Invalid schema: Table name cannot be "lineitem"');
+        } else if (ajv.validate(format, jsonData)) {
+          alert("Schema validated!");
+          const schemaToSave = this.schemas[index];
+          schemaToSave.schema = this.tempSchemaText;
+          console.log(jsonData["table"]);
+          schemaToSave.validated = true;
+        } else {
+          const errors = ajv.errorsText(validate.errors, { separator: "\n" });
+          alert(`Invalid schema: \n${errors}`);
         }
+      } catch (error) {
+        alert("Invalid JSON: " + error);
       }
+      this.tempSchemaText = "";
+    },
+    showEnteredText(index) {
+      return index === this.activeAccordionItem;
     },
   },
 };