Develop

agent-backend/src/tools/builtins/firecrawl_loader.py

@@ -1,22 +1,32 @@
 from langchain_community.document_loaders import AsyncChromiumLoader
 from langchain_community.document_loaders import FireCrawlLoader
+from pydantic import PrivateAttr
 from tools.builtins.base import BaseBuiltinTool
+import requests
 
 class FireCrawlLoader(BaseBuiltinTool):
-    loader = FireCrawlLoader
 
     def __init__(self, *args, **kwargs):
-        print(f"KWARGS: {kwargs}")
+        super().__init__(**kwargs)
         parameters: dict = kwargs.get("parameters")
         if parameters is not None:
-            print(f"PARAMETERS: {parameters}")
-            api_key:str = parameters.get("firecrawl_api_key")
-            print(f"API KEY: {api_key}")
-            kwargs["loader"] = FireCrawlLoader(api_key)
-            super().__init__(**kwargs)
+            self.__dict__['_api_key'] = parameters.get("api_key", "")
         else:
             print("Parameters was None!")
 
     def run_tool(self, query: str) -> str:
-        data = FireCrawlLoader(url = query, mode = "scrape")
-        return data
+        if not getattr(self, '_api_key', None):
+            raise ValueError("API key is not set!") #type saftey
+
+        # Use the API key for running the tool logic
+        url = "https://api.firecrawl.dev/v1/scrape"
+        payload = {
+        "url": query
+        }
+        headers = {
+            "Authorization": f"Bearer {self._api_key}",
+            "Content-Type": "application/json"
+        }
+        response = requests.request('POST', url, json=payload, headers=headers)
+        responseJson = response.json()
+        return responseJson

docker-compose.yml

@@ -29,6 +29,8 @@ services:
 
   docker_redis:
     restart: always
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     image: redis:latest
     ports:
       - "127.0.0.1:6379:6379"
@@ -97,6 +99,7 @@ services:
       - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD:-password}
       - AIRBYTE_CLIENT_ID=${AIRBYTE_CLIENT_ID}
       - AIRBYTE_CLIENT_SECRET=${AIRBYTE_CLIENT_SECRET}
+      - NEXT_PUBLIC_IS_AIRBYTE_ENABLED=false
       - GCS_BUCKET_NAME=${GCS_BUCKET_NAME}
       - GCS_BUCKET_LOCATION=${GCS_BUCKET_LOCATION}
       - STRIPE_FREE_PLAN_PRICE_ID=price_1P0zlRDxQ9GZKzvoYUAzWMSv
@@ -122,6 +125,8 @@ services:
       - AIRBYTE_RABBITMQ_HOST=${AIRBYTE_RABBITMQ_HOST}
     volumes:
       - datasource_files:/tmp
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
 
   webapp_syncserver:
     restart: always
@@ -230,13 +235,13 @@ services:
 
   qdrant:
     ports:
-      - '127.0.0.1:6333:6333'
-      - '127.0.0.1:6334:6334'
+      - '0.0.0.0:6333:6333'
+      - '0.0.0.0:6334:6334'
     image: qdrant/qdrant
     environment:
       - QDRANT__LOG_LEVEL=DEBUG
     volumes:
-      - qdrant_data:/qdrant_data
+      - qdrant_data:/qdrant_data 
 
   # minio:
   #   image: minio/minio

install.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 
+# set -e
+# set -o pipefail
+# trap 'echo "An error occurred during installation. Exiting..."; exit 1; echo "Please forward relevant error logs to the Agentcloud team."' ERR SIGINT
 # set -e
 # set -o pipefail
 # trap 'echo "An error occurred during installation. Exiting..."; exit 1; echo "Please forward relevant error logs to the Agentcloud team."' ERR SIGINT
@@ -94,7 +97,9 @@ GCS_BUCKET_NAME=""
 GCS_BUCKET_LOCATION=""
 STRIPE_PRICING_TABLE_ID=""
 STRIPE_PUBLISHABLE_KEY=""
-export AIRBYTE_RABBITMQ_HOST=$(ifconfig | grep -v 127.0.0.1 | grep -F "inet " | awk '{print $2}' | head -n 1)
+export AIRBYTE_RABBITMQ_HOST=$(ip address | grep -v 127.0.0.1 | grep -F "inet " | awk '{print $2}' | cut -d'/' -f1 | head -n 1)
+
+echo "Airbyte RabbitMQ Host: ${AIRBYTE_RABBITMQ_HOST}"
 
 # Initialize variables to indicate whether to kill specific containers
 KILL_WEBAPP_NEXT=0
@@ -211,7 +216,7 @@ docker tag downloads.unstructured.io/unstructured-io/unstructured-api:latest loc
 if [ "$MINIMAL" -eq 1 ]; then
 	docker compose -f docker-compose.minimal.yml up --build -d
 else
-	docker compose up --build -d
+    docker compose up -d
 fi
 
 # At the end of the script, check the variables and kill containers if requested

webapp/.env.example

@@ -36,6 +36,7 @@ AIRBYTE_USERNAME=airbyte
 AIRBYTE_PASSWORD=password
 AIRBYTE_CLIENT_ID=
 AIRBYTE_CLIENT_SECRET=
+NEXT_PUBLIC_IS_AIRBYTE_ENABLED=false
 NEXT_PUBLIC_GCS_BUCKET_NAME_PRIVATE=agentcloud-bucket-dev
 NEXT_PUBLIC_GCS_BUCKET_NAME=agentcloud-public-dev
 GCS_BUCKET_LOCATION=australia-southeast1

webapp/src/api.ts

@@ -704,6 +704,17 @@ export function getDatasourceSchema(body, dispatch, errorCallback, router) {
 	);
 }//@TEST
 
+export function checkAirbyteConnection(body, dispatch, errorCallback, router) {
+	return ApiCall(
+		`/${body.resourceSlug}/airbyte/connection`,
+		'GET',
+		null,
+		dispatch,
+		errorCallback,
+		router
+	);
+}
+
 //Temp datasource stuff
 export function uploadDatasourceFileTemp(body, dispatch, errorCallback, router) {
 	return ApiCall(

webapp/src/components/DatasourceTable.tsx

@@ -13,6 +13,7 @@
 import DevBadge from 'components/DevBadge';
 import { useAccountContext } from 'context/account';
 import { useNotificationContext } from 'context/notifications';
+import cn from 'lib/cn';
 import Link from 'next/link';
 import { useRouter } from 'next/router';
 import { Fragment, useReducer, useState } from 'react';
@@ -22,10 +23,12 @@
 
 export default function DatasourceTable({
 	datasources,
-	fetchDatasources
+	fetchDatasources,
+	isAirbyteEnabled
 }: {
 	datasources: any[];
 	fetchDatasources?: any;
+	isAirbyteEnabled?: boolean;
 }) {
 	const [notificationContext, refreshNotificationContext]: any = useNotificationContext();
 	const [accountContext]: any = useAccountContext();
@@ -37,6 +40,12 @@
 	const [deletingMap, setDeletingMap] = useState({});
 	const [confirmClose, setConfirmClose] = useState(false);
 
+	const goToDatasourcePage = (id: string) => {
+		if (isAirbyteEnabled) {
+			router.push(`/${resourceSlug}/datasource/${id}`);
+		}
+	};
+
 	async function deleteDatasource(datasourceId) {
 		setDeleting({ [datasourceId]: true });
 		try {
@@ -152,13 +161,14 @@
 							return (
 								<tr
 									key={datasource._id}
-									className={`cursor-pointer hover:bg-gray-50 dark:hover:bg-slate-700 dark:text-white dark:!border-slate-700 transition-all opacity-1 duration-700 ${deletingMap[datasource._id] ? 'bg-red-400' : 'cursor-pointer hover:bg-gray-50'}`}
+									className={cn(
+										`hover:bg-gray-50 dark:hover:bg-slate-700 dark:text-white dark:!border-slate-700 transition-all opacity-1 duration-700 ${deletingMap[datasource._id] ? 'bg-red-400' : ' hover:bg-gray-50'}`,
+										{ 'cursor-pointer': isAirbyteEnabled }
+									)}
 									style={{ borderColor: deletingMap[datasource._id] ? 'red' : '' }}
+									onClick={() => goToDatasourcePage(datasource._id)}
 								>
-									<td
-										className='px-6 py-3 whitespace-nowrap flex items-center'
-										onClick={() => router.push(`/${resourceSlug}/datasource/${datasource._id}`)}
-									>
+									<td className='px-6 py-3 whitespace-nowrap flex items-center'>
 										<img
 											src={`https://connectors.airbyte.com/files/metadata/airbyte/source-${datasource.sourceType}/latest/icon.svg`}
 											className='w-6 me-1.5'
@@ -168,34 +178,22 @@
 										</span>
 										<DevBadge value={datasource?._id} />
 									</td>
-									<td
-										className='px-6 py-3 whitespace-nowrap'
-										onClick={() => router.push(`/${resourceSlug}/datasource/${datasource._id}`)}
-									>
+									<td className='px-6 py-3 whitespace-nowrap'>
 										<div className='flex items-center'>
 											<div className='text-sm font-medium text-gray-900 dark:text-white'>
 												{datasource.name}
 											</div>
 										</div>
 									</td>
-									<td
-										className='px-6 py-3 whitespace-nowrap'
-										onClick={() => router.push(`/${resourceSlug}/datasource/${datasource._id}`)}
-									>
+									<td className='px-6 py-3 whitespace-nowrap'>
 										<DatasourceStatusIndicator datasource={datasource} />
 									</td>
-									<td
-										className='px-6 py-3 whitespace-nowrap'
-										onClick={() => router.push(`/${resourceSlug}/datasource/${datasource._id}`)}
-									>
+									<td className='px-6 py-3 whitespace-nowrap'>
 										<span className='px-2 inline-flex text-sm leading-5 rounded-full capitalize'>
 											{datasource?.connectionSettings?.scheduleType || '-'}
 										</span>
 									</td>
-									<td
-										className='px-6 py-3 whitespace-nowrap'
-										onClick={() => router.push(`/${resourceSlug}/datasource/${datasource._id}`)}
-									>
+									<td className='px-6 py-3 whitespace-nowrap'>
 										<div className='text-sm text-gray-900 dark:text-white' suppressHydrationWarning>
 											{datasource.sourceType === 'file'
 												? 'N/A'
@@ -204,10 +202,7 @@
 													: 'Never'}
 										</div>
 									</td>
-									<td
-										className='px-6 py-3 whitespace-nowrap'
-										onClick={() => router.push(`/${resourceSlug}/datasource/${datasource._id}`)}
-									>
+									<td className='px-6 py-3 whitespace-nowrap'>
 										<span
 											suppressHydrationWarning
 											className='text-sm text-gray-900 dark:text-white'
@@ -217,7 +212,8 @@
 									</td>
 									<td className='px-6 py-5 whitespace-nowrap text-right text-sm font-medium flex justify-end space-x-5 items-center'>
 										<button
-											onClick={() => {
+											onClick={e => {
+												e.stopPropagation();
 												// if (datasource.status !== DatasourceStatus.READY) {
 												// 	setConfirmClose(datasource._id);
 												// } else {
@@ -227,7 +223,8 @@
 											disabled={
 												syncing[datasource._id] ||
 												deleting[datasource._id] ||
-												datasource.status !== DatasourceStatus.READY
+												datasource.status !== DatasourceStatus.READY ||
+												!isAirbyteEnabled
 											}
 											className='rounded-md disabled:bg-slate-400 bg-indigo-600 px-2 -my-1 py-1 text-sm font-semibold text-white shadow-sm hover:bg-indigo-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 dark:text-white'
 										>

webapp/src/controllers/airbyte.ts

@@ -14,6 +14,7 @@ import {
 } from 'db/datasource';
 import { addNotification } from 'db/notification';
 import debug from 'debug';
+import * as airbyteSetup from 'lib/airbyte/setup';
 import posthog from 'lib/posthog';
 import { chainValidations } from 'lib/utils/validationutils';
 import toObjectId from 'misc/toobjectid';
@@ -339,3 +340,19 @@ export async function handleSuccessfulEmbeddingWebhook(req, res, next) {
 
 	return dynamicResponse(req, res, 200, {});
 }
+
+export async function checkAirbyteConnection(req, res, next) {
+	const status = await airbyteSetup.checkAirbyteStatus();
+
+	let isEnabled = process.env.NEXT_PUBLIC_IS_AIRBYTE_ENABLED === 'true';
+
+	if (status && !isEnabled) {
+		isEnabled = await airbyteSetup.init();
+	}
+
+	if (!status) {
+		process.env.NEXT_PUBLIC_IS_AIRBYTE_ENABLED = 'false';
+	}
+
+	return dynamicResponse(req, res, 201, { isEnabled });
+}

webapp/src/lib/airbyte/setup.ts

@@ -1,19 +1,16 @@
+import * as dns from 'node:dns';
+import * as util from 'node:util';
+
 import debug from 'debug';
 import dotenv from 'dotenv';
 import fs from 'fs';
-import getGoogleCredentials from 'misc/getgooglecredentials';
 import fetch from 'node-fetch'; // Ensure node-fetch is installed or use a compatible fetch API
-import path from 'path';
-const { GoogleAuth } = require('google-auth-library');
-import * as dns from 'node:dns';
-import * as util from 'node:util';
 const lookup = util.promisify(dns.lookup);
 
 import * as process from 'node:process';
 
 import getAirbyteApi, { AirbyteApiType, getAirbyteAuthToken } from 'airbyte/api';
 import SecretProviderFactory from 'lib/secret';
-import { AIRBYTE_OAUTH_PROVIDERS } from 'struct/oauth';
 
 import getAirbyteInternalApi from './internal';
 
@@ -87,6 +84,30 @@ async function deleteDestination(destinationId: string) {
 	);
 }
 
+export async function checkAirbyteStatus() {
+	try {
+		const response = await fetch(`${process.env.AIRBYTE_API_URL}/api/v1/health`, {
+			method: 'GET'
+		});
+		if (response?.status !== 200) {
+			return false;
+		}
+		const workspaces = await fetch(`${process.env.AIRBYTE_API_URL}/api/v1/workspaces`, {
+			method: 'GET',
+			headers: {
+				accept: 'application/json',
+				authorization: `Bearer ${await getAirbyteAuthToken()}`
+			}
+		});
+		if (response?.status !== 200) {
+			return false;
+		}
+		return true;
+	} catch (error) {
+		console.log('error', error);
+	}
+}
+
 async function getDestinationConfiguration(provider: string) {
 	if (provider === 'rabbitmq') {
 		log(`RabbitMQ HOST: ${process.env.AIRBYTE_RABBITMQ_HOST}`);
@@ -270,7 +291,10 @@ export async function init() {
 		// for (let provider in AIRBYTE_OAUTH_PROVIDERS) {
 		// 	overrideOauthCreds(airbyteAdminWorkspaceId, provider.toLowerCase());
 		// }
+		process.env.NEXT_PUBLIC_IS_AIRBYTE_ENABLED = 'true';
+		return true;
 	} catch (error) {
+		process.env.NEXT_PUBLIC_IS_AIRBYTE_ENABLED = 'false';
 		logerror('Error during Airbyte configuration:', error);
 	}
 }

webapp/src/pages/[resourceSlug]/datasource/[datasourceId]/index.tsx

@@ -35,6 +35,7 @@ export default function Datasource(props) {
 	const router = useRouter();
 	const { resourceSlug, datasourceId } = router.query;
 	const [state, dispatch] = useState(props);
+	const [airbyteState, setAirbyteState] = useState(null);
 	const [jobsList, setJobsList] = useState(null);
 	const [tab, setTab] = useState(0);
 	const [schemaDiscoverState, setSchemaDiscoverState] = useState(null);
@@ -79,6 +80,7 @@ export default function Datasource(props) {
 			setError,
 			router
 		);
+		await API.checkAirbyteConnection({ resourceSlug }, setAirbyteState, setError, router);
 	}
 
 	async function fetchJobsList() {
@@ -264,7 +266,11 @@ export default function Datasource(props) {
 							</button>
 							<button
 								onClick={e => updateStreams(e, true)}
-								disabled={submitting['updateStreamssync'] || submitting['updateStreams']}
+								disabled={
+									submitting['updateStreamssync'] ||
+									submitting['updateStreams'] ||
+									!airbyteState?.isEnabled
+								}
 								type='submit'
 								className='rounded-md disabled:bg-slate-400 bg-indigo-600 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-indigo-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600'
 							>