WIP - Fetch logs when running the ingestion

open-metadata · Oct 18, 2024 · 5e17485 · 5e17485
1 parent 87ac3b5
commit 5e17485
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 19 deletions.
diff --git a/openmetadata-webserver-cli/ui/src/pages/DownloadPage.tsx b/openmetadata-webserver-cli/ui/src/pages/DownloadPage.tsx
@@ -9,6 +9,49 @@ import { ReactComponent as ResumeIcon } from '../assets/svg/ic-play-button.svg';
 
 const DownloadYAML = () => {
     const [yaml, setYaml] = useState<string>('');
+    const [isFetching, setIsFetching] = useState(false);
+    const [logs, setLogs] = useState<string>('');
+
+    const fetchLogs = async () => {
+        setIsFetching(true);
+        try {
+            const response = await runIngestion();
+
+            if (!response.body) {
+                console.error('No response body found');
+                return;
+            }
+
+            const reader = response.body.getReader();
+            const decoder = new TextDecoder('utf-8');
+
+            // Temporary variable to hold fetched logs
+            let receivedLogs = '';
+
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done) break;
+
+                const chunk = decoder.decode(value, { stream: true });
+
+                // Append the chunk to the temporary variable
+                receivedLogs += chunk;
+
+                // Set logs to re-render the LazyLog component with new data
+                // Appends a line break between prevLogs and chunk
+                setLogs((prevLogs) => `${prevLogs}\n${chunk}`);
+            }
+        } catch (error) {
+            setLogs('Logs could not be fetched...');
+            console.error('Error fetching logs:', error);
+        } finally {
+            setIsFetching(false);
+        }
+    };
+
+    const handleRunIngestion = () => {
+        fetchLogs();
+    };
 
     const handleDownload = async () => {
         try {
@@ -35,21 +78,9 @@ const DownloadYAML = () => {
         }
     };
 
-    const fetchFileContent = async () => {
-        fetchYaml()
-            .then(response => setYaml(response.data))
-            .catch(error => setYaml(`Failed to load yaml ${error.message}`));
-    };
-
-    const handleRunIngestion = async () => {
-        runIngestion()
-            .then(response => alert("Run ingestion successfully requested"))
-            .catch(error => alert("The request to run the ingestion failed"));
-    }
-
-    useEffect(() => {
-        fetchFileContent();
-    }, []);
+    // useEffect(() => {
+    //     fetchFileContent();
+    // }, []);
 
     return (
         <PageLayoutV1 pageTitle="View or Download Yaml">
@@ -95,7 +126,8 @@ const DownloadYAML = () => {
                                 enableSearch
                                 selectableLines
                                 extraLines={1}
-                                text={yaml || 'No content to display'}
+                                text={logs || 'No content to display'}
+                                follow
                             />
                         </Col>
                     </Row>
@@ -111,10 +143,11 @@ const DownloadYAML = () => {
                         <Space direction="vertical" className="mt-2 mb-4">
                             <Tooltip title="Run the ingestion with the yaml">
                                 <Button type="primary"
+                                    disabled={isFetching}
                                     icon={<ResumeIcon height={16} width={16} style={{ marginRight: 8, verticalAlign: 'middle' }} />}
                                     onClick={handleRunIngestion}
                                 >
-                                    Run now
+                                    {isFetching ? 'Running...' : 'Run now'}
                                 </Button>
                             </Tooltip>
                         </Space>

diff --git a/openmetadata-webserver-cli/ui/src/utils/APIUtils.ts b/openmetadata-webserver-cli/ui/src/utils/APIUtils.ts
@@ -19,4 +19,9 @@ export const fetchYaml = () => APIClient.get('/api/yaml');
 
 export const downloadYaml = () => APIClient.get('api/yaml/download', { responseType: 'blob' });
 
-export const runIngestion = () => APIClient.post('/api/run');
+// It was quicker to use Fetch API instead of Axios.
+// We can revisit later.
+export const runIngestion = () => fetch(`${APIClient.defaults.baseURL}/api/run`, {
+    method: 'POST',
+});
+
diff --git a/openmetadata-webserver-cli/webserver/routes.py b/openmetadata-webserver-cli/webserver/routes.py
@@ -31,6 +31,7 @@
 from webserver.models import OMetaServerModel
 from webserver.repository import LocalIngestionServer
 from webserver.workflow import workflow_runner, handle_log_queue
+from flask import Response
 
 
 @app.route("/ingestion", methods=["POST"])
@@ -115,4 +116,4 @@ def run_ingestion():
     3. Stream the generator results
     """
     thread, que = workflow_runner()
-    return handle_log_queue(thread=thread, que=que), {"Content-Type": "text/plain"}
+    return Response(handle_log_queue(thread=thread, que=que), content_type="text/plain")
diff --git a/openmetadata-webserver-cli/webserver/workflow.py b/openmetadata-webserver-cli/webserver/workflow.py
@@ -26,6 +26,8 @@
 
 def workflow_runner() -> Tuple[Thread, Queue]:
     """Trigger the ingestion"""
+    LocalIngestionServer().build_workflow()
+
     logger = logging.getLogger(METADATA_LOGGER)
     que = queue.Queue(-1)
     queue_handler = QueueHandler(que)