feat: schema to use a JSON format and query should be generated in ba…

…ckend

api/app.py

@@ -3,6 +3,8 @@
 from fastapi.openapi.utils import get_openapi
 from fastapi_health import health
 
+import json
+
 from duckdb import DuckDBPyConnection
 
 import substrait_validator as sv
@@ -14,7 +16,6 @@
     ParseFromDuckDB,
 )
 
-1
 from loguru import logger
 
 
@@ -83,12 +84,27 @@ async def ValidateFile(file: UploadFile = File(), override_levels: list[int] = F
             status_code=500, detail="Substrait Validator Internal Error: " + str(e)
         )
 
-
 @app.post("/execute/duckdb/")
 def ExecuteBackend(data: dict, db_conn: DuckDBPyConnection = Depends(get_duck_conn)):
     response = ExecuteDuckDb(data, db_conn)
     return response
 
+@app.post("/add_schema/")
+def AddSchema(data: dict, db_conn: DuckDBPyConnection = Depends(get_duck_conn)):
+    schema = data["schema"]
+    query = "CREATE TEMP TABLE "
+    json_data = json.loads(schema["schema"])
+    query += json_data["table"] + "("
+    for field in json_data["fields"]:
+        query += field["name"] + " "
+        query += field["type"] + " "
+        for props in field["properties"]:
+            query += props + " "
+        query += ", "
+    query += ");"
+    response = ExecuteDuckDb(query, db_conn)
+    return response
+
 
 @app.post("/parse/", status_code=status.HTTP_200_OK)
 def ParseToSubstrait(data: dict, db_conn: DuckDBPyConnection = Depends(get_duck_conn)):

api/backend/duckdb.py

@@ -1,5 +1,4 @@
 import duckdb
-from pydantic import BaseModel
 from fastapi import HTTPException
 
 from loguru import logger
@@ -8,12 +7,34 @@
 class DuckDBConnection:
     def __init__(self):
         self.conn_pool = []
+
+        query_lineitem = '''CREATE TABLE IF NOT EXISTS lineitem(
+                        l_orderkey INTEGER NOT NULL, 
+                        l_partkey INTEGER NOT NULL, 
+                        l_suppkey INTEGER NOT NULL, 
+                        l_linenumber INTEGER NOT NULL, 
+                        l_quantity INTEGER NOT NULL, 
+                        l_extendedprice DECIMAL(15,2) NOT NULL, 
+                        l_discount DECIMAL(15,2) NOT NULL, 
+                        l_tax DECIMAL(15,2) NOT NULL, 
+                        l_returnflag VARCHAR NOT NULL, 
+                        l_linestatus VARCHAR NOT NULL, 
+                        l_shipdate DATE NOT NULL, 
+                        l_commitdate DATE NOT NULL, 
+                        l_receiptdate DATE NOT NULL, 
+                        l_shipinstruct VARCHAR NOT NULL, 
+                        l_shipmode VARCHAR NOT NULL, 
+                        l_comment VARCHAR NOT NULL);'''
+        conn = duckdb.connect("duck.db")
+        conn.execute(query=query_lineitem)
+
         for i in range(5):
             conn = duckdb.connect("duck.db")
             conn.install_extension("substrait")
             conn.load_extension("substrait")
             self.conn_pool.append(conn)
 
+
     def check_pool(self):
         if len(self.conn_pool) == 0:
             print("creating new conn objects")
@@ -44,9 +65,9 @@ def CheckDuckDBConnection(con):
         return status
 
 
-def ExecuteDuckDb(data, con):
+def ExecuteDuckDb(query, con):
     try:
-        con.execute(query=data["query"])
+        con.execute(query = query)
         return {"message": "DuckDB Operation successful"}
     except Exception as e:
         raise HTTPException(

api/test.py

@@ -1,6 +1,5 @@
 from fastapi.testclient import TestClient
 from io import BytesIO
-import pytest
 import requests
 from app import app
 
@@ -51,7 +50,8 @@ def test_duckdb_execute():
         res = client.post(
             "/execute/duckdb/",
             json={
-                "query": "CREATE TABLE IF NOT EXISTS weather(city VARCHAR, temp_lo INTEGER);",
+                "query": '''CREATE TABLE IF NOT EXISTS 
+                            weather(city VARCHAR, temp_lo INTEGER);''',
             },
         )
         print(res.json())
@@ -65,7 +65,8 @@ def test_parse_to_substrait():
         client.post(
             "/execute/duckdb/",
             json={
-                "query": "CREATE TABLE IF NOT EXISTS test_fiddle(id INTEGER NOT NULL, key INTEGER NOT NULL);",
+                "query": '''CREATE TABLE IF NOT EXISTS test_fiddle(
+                            id INTEGER NOT NULL, key INTEGER NOT NULL);''',
             },
         )
         response = client.post(