Merge pull request #25 from DataUSA/develop

Cambridge Deployment

.dockerignore

@@ -1,7 +1,8 @@
-.github
+.github/
+build-explorer/
+helm/
 *.md
 *.env
 *.gcp.json
 *.gcp.encoded
 Dockerfile
-kubernetes/

.gcloudignore

@@ -1,5 +1,7 @@
 .git
+.github
+build-explorer
 dist
 node_modules
 vendor
-*.jar
+*.jar

.gitignore

@@ -1,3 +1,5 @@
+node_modules/
+
 *.gcp.json
 *.gcp.encoded
 

Dockerfile

@@ -1,42 +1,35 @@
-FROM python:3.9
+FROM python:3.10 as builder
 
-# Install api pre requirements
-RUN pip install -U pip setuptools wheel
+RUN pip install setuptools wheel poetry==1.8.3
 
-# Define api directory
-ENV APP_HOME /usr/src/app
-WORKDIR $APP_HOME
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
 
-# Allow that statements and log messages appear in the Knative logs
-ENV PYTHONUNBUFFERED True
+WORKDIR /app
 
-# Transfer api requirements
-COPY requirements.txt ./
+COPY pyproject.toml poetry.lock ./
+RUN touch README.md
 
-# Install api requirements
-RUN useradd -m -r tesseract &&\
-    chown tesseract $APP_HOME &&\
-    pip install --no-cache-dir -r requirements.txt
+RUN --mount=type=cache,target=$POETRY_CACHE_DIR poetry install --without dev --no-root
 
-# Transfer app files
-COPY --chown=tesseract:tesseract  . .
-RUN chown -R tesseract $APP_HOME
+FROM python:3.10-slim-buster as runtime
 
-# Define api required env vars
-ARG GIT_HASH
-ENV GIT_HASH=${GIT_HASH:-dev}
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:$PATH"
 
-# Change unix user to tesseract
-USER tesseract
+WORKDIR /app
+
+# create runtime user; install required dependencies
+RUN useradd --system --uid 1001 tesseract &&\
+    chown -R tesseract:tesseract /app
 
-# Expose api port
-ENV PORT 7777
-EXPOSE 7777
+COPY --chown=tesseract --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
 
-# Setup host and port
-# Uncomment this line for a cloudrun instance
-#ENV HOST 0.0.0.0
+COPY --chown=tesseract . /app
+
+# change user to tesseract user
+USER tesseract
 
-# Define startup commands
-CMD ["app.py"]
-ENTRYPOINT ["python"]
+CMD exec granian --interface asgi --host 0.0.0.0 --port 7777 --respawn-failed-workers app:layer

app/__init__.py

@@ -0,0 +1,58 @@
+import logging.config
+import os
+
+from fastapi.responses import RedirectResponse
+from logiclayer import LogicLayer
+from logiclayer_complexity import EconomicComplexityModule
+from tesseract_olap import OlapServer
+from tesseract_olap.logiclayer import TesseractModule
+
+from .debug import DebugModule
+
+
+# PARAMETERS ===================================================================
+
+# These parameters are required and will prevent execution if not set
+olap_backend = os.environ["TESSERACT_BACKEND"]
+olap_schema  = os.environ["TESSERACT_SCHEMA"]
+
+# These parameters are optional
+olap_cache = os.environ.get("TESSERACT_CACHE", ":memory:")
+app_debug = os.environ.get("TESSERACT_DEBUG", None)
+log_filepath = os.environ.get("TESSERACT_LOGGING_CONFIG", "logging.ini")
+commit_hash = os.environ.get("GIT_HASH", "")
+
+app_debug = bool(app_debug)
+
+
+# LOGGING ======================================================================
+# To learn how logging works in python
+# - https://docs.python.org/3.7/howto/logging.html
+# To learn about best practices and the logging.ini file
+# - https://www.datadoghq.com/blog/python-logging-best-practices/
+# - https://guicommits.com/how-to-log-in-python-like-a-pro/
+
+logging.config.fileConfig(log_filepath, disable_existing_loggers=False)
+
+
+# ASGI app =====================================================================
+olap = OlapServer(backend=olap_backend, schema=olap_schema)
+
+mod_tsrc = TesseractModule(olap)
+
+mod_cmplx = EconomicComplexityModule(olap)
+
+mod_debug = DebugModule()
+
+layer = LogicLayer(debug=app_debug)
+
+if app_debug:
+    layer.add_module("/debug", mod_debug)
+
+layer.add_module("/tesseract", mod_tsrc)
+layer.add_module("/complexity", mod_cmplx)
+layer.add_static("/ui", "./explorer/", html=True)
+
+@layer.route("/", response_class=RedirectResponse, status_code=302)
+def route_index():
+    return "/ui/"

app/debug.py

@@ -0,0 +1,22 @@
+import html
+import pathlib
+
+import logiclayer as ll
+from fastapi.responses import StreamingResponse
+
+
+class DebugModule(ll.LogicLayerModule):
+    def log_generator(self):
+        yield "<!DOCTYPE html><html><body>\n<details hidden>"
+        with pathlib.Path("/tmp/oec.log").open() as fp:
+            for line in fp:
+                escaped = html.escape(line.strip())
+                if "oec.auth" in line or "tesseract_olap.server" in line:
+                    yield f"</details>\n<details><summary>{escaped}</summary>"
+                else:
+                    yield f"<p>{escaped}</p>"
+        yield "</details></body></html>"
+
+    @ll.route("GET", "/logs")
+    def route_logs(self):
+        return StreamingResponse(self.log_generator())

build-explorer/index.html

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<title>OEC Tesseract Python Demo</title>
+<meta charset="utf-8" />
+<meta name="viewport" content="width=device-width,initial-scale=1" />
+<link rel="icon" href="data:image/x-icon;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQEAYAAABPYyMiAAAABmJLR0T///////8JWPfcAAAACXBIWXMAAABIAAAASABGyWs+AAAAF0lEQVRIx2NgGAWjYBSMglEwCkbBSAcACBAAAeaR9cIAAAAASUVORK5CYII=" type="image/x-icon" />
+<style type="text/css">html,body{margin:0;padding:0}</style>
+</head>
+<body><div id="app" style="height: 100vh"><p id="warning" style="padding:10% 0;text-align:center">If you see this message, it means the root of the server is set incorrectly.<br/>Tell the system administrator to point the root to the <code>dist/</code> folder.</p></div><script type="module" src="./index.tsx"></script></body>
+</html>

build-explorer/index.tsx

@@ -0,0 +1,32 @@
+//@ts-check
+import { DebugView, Explorer, PivotView, TableView } from "@datawheel/tesseract-explorer";
+import React, { lazy } from "react";
+import { createRoot } from "react-dom/client";
+import "normalize.css";
+
+document.querySelector("p#warning")?.remove();
+const container = document.getElementById("app");
+container && mount(container);
+
+function mount(container) {
+  const VizbuilderPanel = lazy(() => import("./vizbuilder")) ;
+
+  const root = createRoot(container);
+  root.render(
+    <Explorer
+      uiLocale={process.env.__UI_LOCALE__}
+      dataLocale={process.env.__SERVER_LOCALE__}
+      previewLimit={100}
+      panels={[
+        {key: "table", label: "Data Table", component: TableView},
+        {key: "matrix", label: "Pivot Table", component: PivotView},
+        {key: "debug", label: "Raw response", component: DebugView},
+        {key: "vizbuilder", label: "Vizbuilder", component: VizbuilderPanel}
+      ]}
+      source={{url: process.env.__SERVER_URL__}}
+      withinMantineProvider={true}
+      withinReduxProvider={true}
+      withPermalink={true}
+    />
+  );
+}