diff --git a/apps/magic-link/src/lib/ProviderModal.tsx b/apps/magic-link/src/lib/ProviderModal.tsx index d2406c082..a226ded56 100644 --- a/apps/magic-link/src/lib/ProviderModal.tsx +++ b/apps/magic-link/src/lib/ProviderModal.tsx @@ -197,7 +197,6 @@ const ProviderModal = () => { const providerMetadata = CONNECTORS_METADATA[selectedProvider.category][selectedProvider.provider]; if (providerMetadata.authStrategy.strategy === AuthStrategy.oauth2) { - console.log("values are "+ JSON.stringify(values)) setAdditionalParams(values); setStartFlow(true); }else{ diff --git a/apps/webapp/src/app/(Dashboard)/configuration/page.tsx b/apps/webapp/src/app/(Dashboard)/configuration/page.tsx index 966e8b1fa..8c79032f8 100644 --- a/apps/webapp/src/app/(Dashboard)/configuration/page.tsx +++ b/apps/webapp/src/app/(Dashboard)/configuration/page.tsx @@ -39,7 +39,7 @@ import useUpdatePullFrequency from "@/hooks/create/useCreatePullFrequency"; import { toast } from "sonner"; import { useQueryClient } from "@tanstack/react-query"; import { Loader2 } from "lucide-react"; -import { RAGSettingsPage } from "@/components/Configuration/RAGSettings/RAGSettingsPage"; +import RAGSettingsPage from "@/components/Configuration/RAGSettings/RAGSettingsPage"; const frequencyOptions = [ { label: '5 min', value: 300 }, @@ -128,10 +128,17 @@ export default function Page() { - const saveFrequency = async (vertical: string) => { + const handleFrequencyChangeAndSave = async (vertical: string, value: string) => { + handleFrequencyChange(vertical, value); + await saveFrequency(vertical, value); + }; + + const saveFrequency = async (vertical: string, value: string) => { setLoadingStates(prev => ({ ...prev, [vertical]: true })); try { - const updateData = { [vertical]: parseInt(localFrequencies[vertical] || '0', 10) }; + const frequency = parseInt(value, 10); + console.log("frequency being saved: " + frequency); + const updateData = { [vertical]: frequency }; await toast.promise( createPullFrequencyPromise(updateData), @@ -140,9 +147,9 @@ export default function Page() { success: (data: any) => { queryClient.setQueryData(['pull-frequencies'], (oldData: any) => ({ ...oldData, - [vertical]: localFrequencies[vertical], + [vertical]: frequency.toString(), })); - return `Frequency saved`; + return frequency === 0 ? `${vertical} sync deactivated` : `Frequency saved for ${vertical}`; }, error: (err: any) => err.message || 'Error updating frequency', } @@ -238,42 +245,42 @@ export default function Page() { Set the sync frequency for each vertical - {VERTICALS.map(vertical => ( -
- -
- - -
-
- ))} - + {VERTICALS.map(vertical => ( +
+ +
+ + {localFrequencies[vertical] && localFrequencies[vertical] !== '0'&& ( + + )} +
+
+ ))} + diff --git a/apps/webapp/src/components/Configuration/Connector/ConnectorLayout.tsx b/apps/webapp/src/components/Configuration/Connector/ConnectorLayout.tsx index 50dd7a475..44553fa35 100644 --- a/apps/webapp/src/components/Configuration/Connector/ConnectorLayout.tsx +++ b/apps/webapp/src/components/Configuration/Connector/ConnectorLayout.tsx @@ -26,7 +26,7 @@ export function ConnectorLayout({ }: Props) { const [connector] = useConnector() const [selectedVertical, setSelectedVertical] = React.useState("") - + const handleSearchChange = (event: React.ChangeEvent) => { onSearchChange(event.target.value) } diff --git a/apps/webapp/src/components/Configuration/Connector/ConnectorList.tsx b/apps/webapp/src/components/Configuration/Connector/ConnectorList.tsx index 092de227c..59c7d2338 100644 --- a/apps/webapp/src/components/Configuration/Connector/ConnectorList.tsx +++ b/apps/webapp/src/components/Configuration/Connector/ConnectorList.tsx @@ -24,7 +24,7 @@ export function ConnectorList({ items }: ConnectorListProps) { )} onClick={() => setConnector({ - ...connector, + ...connector, selected: `${item.vertical}-${item.name}`, }) } diff --git a/apps/webapp/src/components/Configuration/Connector/CustomConnectorPage.tsx b/apps/webapp/src/components/Configuration/Connector/CustomConnectorPage.tsx index 5f3628174..2dc8b7b96 100644 --- a/apps/webapp/src/components/Configuration/Connector/CustomConnectorPage.tsx +++ b/apps/webapp/src/components/Configuration/Connector/CustomConnectorPage.tsx @@ -17,8 +17,7 @@ export default function CustomConnectorPage() { : providersArray(vertical); const connectors = filteredConnectors.filter((connector: Provider) => { - const matchesSearch = connector.name.toLowerCase().includes(searchQuery.toLowerCase()) || - connector.description?.toLowerCase().includes(searchQuery.toLowerCase()) + const matchesSearch = connector.name.toLowerCase().includes(searchQuery.toLowerCase()) || connector.description?.toLowerCase().includes(searchQuery.toLowerCase()) return matchesSearch }) diff --git a/apps/webapp/src/components/Configuration/RAGSettings/RAGItemDisplay.tsx b/apps/webapp/src/components/Configuration/RAGSettings/RAGItemDisplay.tsx new file mode 100644 index 000000000..99e56567c --- /dev/null +++ b/apps/webapp/src/components/Configuration/RAGSettings/RAGItemDisplay.tsx @@ -0,0 +1,292 @@ +import React, { useEffect, useState } from 'react'; +import { useForm } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useQueryClient } from "@tanstack/react-query"; +import { usePostHog } from 'posthog-js/react'; +import { toast } from "sonner"; +import * as z from "zod"; +import { Button } from "@/components/ui/button"; +import { Separator } from "@/components/ui/separator"; +import { Switch } from "@/components/ui/switch"; +import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import config from "@/lib/config"; +import useProjectStore from "@/state/projectStore"; +import useConnectionStrategies from "@/hooks/get/useConnectionStrategies"; +import useCreateConnectionStrategy from "@/hooks/create/useCreateConnectionStrategy"; +import useUpdateConnectionStrategy from "@/hooks/update/useUpdateConnectionStrategy"; +import useConnectionStrategyAuthCredentials from "@/hooks/get/useConnectionStrategyAuthCredentials"; +import { vectorDatabases, embeddingModels, TabType } from './utils'; + +const formSchema = z.object({ + apiKey: z.string().optional(), + baseUrl: z.string().optional(), + url: z.string().optional(), + indexName: z.string().optional(), + embeddingApiKey: z.string().optional(), +}); + +interface ItemDisplayProps { + item?: typeof vectorDatabases[number] | typeof embeddingModels[number]; + type: string; +} + +export function RAGItemDisplay({ item, type }: ItemDisplayProps) { + const [isActive, setIsActive] = useState(false); + const { idProject } = useProjectStore(); + const queryClient = useQueryClient(); + const posthog = usePostHog(); + + const { data: connectionStrategies } = useConnectionStrategies(); + const { createCsPromise } = useCreateConnectionStrategy(); + const { updateCsPromise } = useUpdateConnectionStrategy(); + const { mutateAsync: fetchCredentials } = useConnectionStrategyAuthCredentials(); + + const form = useForm>({ + resolver: zodResolver(formSchema), + defaultValues: { + apiKey: "", baseUrl: "", url: "", indexName: "", embeddingApiKey: "", + }, + }); + + const currentStrategy = connectionStrategies?.find( + (cs) => cs.type === `${type}.${item?.name.toLowerCase()}` + ); + + useEffect(() => { + if (currentStrategy) { + fetchCredentials({ type: currentStrategy.type, attributes: getAttributesForItem(item?.name) }, { + onSuccess(data) { + setFormValuesFromCredentials(item?.name ?? '', data); + setIsActive(currentStrategy.status === true); + } + }); + } else { + form.reset(); + setIsActive(false); + } + }, [connectionStrategies, item]); + + function onSubmit(values: z.infer) { + const attributes = getAttributesForItem(item?.name); + const attributeValues = attributes.map(attr => values[attr as keyof typeof values]).filter((value): value is string => value !== undefined); + + const promise = currentStrategy + ? updateCsPromise({ + id_cs: currentStrategy.id_connection_strategy, + updateToggle: false, + status: isActive, + attributes, + values: attributeValues, + }) + : createCsPromise({ + type: `${type}.${item?.name.toLowerCase()}`, + attributes, + values: attributeValues, + status: false, + }); + + toast.promise(promise, { + loading: 'Saving changes...', + success: (data: any) => { + queryClient.setQueryData( + ['connection-strategies'], + (oldData = []) => { + if (currentStrategy) { + return oldData.map(cs => + cs.id_connection_strategy === data.id_connection_strategy + ? { ...data, status: isActive } // Ensure status matches isActive + : cs + ); + } else { + return [...oldData, { ...data, status: false }]; // New strategy is inactive + } + } + ); + setIsActive(data.status); + return "Changes saved successfully"; + }, + error: (err: any) => err.message || 'An error occurred', + }); + + posthog?.capture(`RAG_${type}_${currentStrategy ? 'updated' : 'created'}`, { + id_project: idProject, + item_name: item?.name, + mode: config.DISTRIBUTION + }); + } + + const handleActiveChange = (checked: boolean) => { + if (currentStrategy) { + const updatePromises = []; + + // Update current strategy + updatePromises.push( + toast.promise( + updateCsPromise({ + id_cs: currentStrategy.id_connection_strategy, + updateToggle: true, + status: checked + }), + { + loading: `${checked ? 'Activating' : 'Deactivating'} ${item?.name}...`, + success: (data: any) => { + queryClient.setQueryData(['connection-strategies'], (oldData = []) => + oldData.map(cs => cs.id_connection_strategy === data.id_connection_strategy ? data : cs) + ); + setIsActive(checked); + return `${item?.name} ${checked ? 'activated' : 'deactivated'} successfully`; + }, + error: (err: any) => err.message || 'An error occurred', + } + ) + ); + + if (checked) { + // Deactivate other items of the same type + connectionStrategies?.forEach(cs => { + if (cs.type.startsWith(`${type}.`) && cs.type !== `${type}.${item?.name.toLowerCase()}` && cs.status) { + updatePromises.push( + toast.promise( + updateCsPromise({ + id_cs: cs.id_connection_strategy, + updateToggle: true, + status: false + }), + { + loading: `Deactivating ${cs.type.split('.')[1]}...`, + success: (data: any) => { + queryClient.setQueryData(['connection-strategies'], (oldData = []) => + oldData.map(strategy => strategy.id_connection_strategy === data.id_connection_strategy ? data : strategy) + ); + return `${cs.type.split('.')[1]} deactivated successfully`; + }, + error: (err: any) => err.message || 'An error occurred', + } + ) + ); + } + }); + } + + // Wait for all updates to complete + Promise.all(updatePromises).then(() => { + queryClient.invalidateQueries({ queryKey: ['connection-strategies'] }); + }); + } + }; + + const getAttributesForItem = (itemName?: string): string[] => { + switch (itemName?.toLowerCase()) { + case 'turbopuffer': + return ['apiKey']; + case 'pinecone': + return ['apiKey', 'indexName']; + case 'qdrant': + return ['apiKey', 'baseUrl']; + case 'chromadb': + return ['url']; + case 'weaviate': + return ['apiKey', 'url']; + case 'openai_ada_small_1536': + case 'openai_ada_large_3072': + case 'openai_ada_002': + case 'cohere_multilingual_v3': + return ['embeddingApiKey']; + default: + return []; + } + }; + + const setFormValuesFromCredentials = (itemName: string, data: string[]) => { + switch (itemName?.toLowerCase()) { + case 'turbopuffer': + form.setValue("apiKey", data[0]); + break; + case 'pinecone': + form.setValue("apiKey", data[0]); + form.setValue("indexName", data[1]); + break; + case 'qdrant': + form.setValue("apiKey", data[0]); + form.setValue("baseUrl", data[1]); + break; + case 'chromadb': + form.setValue("url", data[0]); + break; + case 'weaviate': + form.setValue("apiKey", data[0]); + form.setValue("url", data[1]); + break; + case 'openai_ada_small_1536': + case 'openai_ada_large_3072': + case 'openai_ada_002': + case 'cohere_multilingual_v3': + form.setValue("embeddingApiKey", data[0]); + break; + } + }; + + const renderInputs = () => { + const attributes = getAttributesForItem(item?.name); + return attributes.map((attr) => ( + } + render={({ field }) => ( + + {attr} + + + + + + )} + /> + )); + }; + + return ( +
+ + {item ? ( +
+
+
+ {item.name} +
+
{item.name}
+
{item.description}
+
+ +
+
+
+
+ +
+
+ + {renderInputs()} + +
+ +
+
+ ) : ( +
+ No item selected +
+ )} +
+ ); +} \ No newline at end of file diff --git a/apps/webapp/src/components/Configuration/RAGSettings/RAGItemList.tsx b/apps/webapp/src/components/Configuration/RAGSettings/RAGItemList.tsx new file mode 100644 index 000000000..d5a8d79c6 --- /dev/null +++ b/apps/webapp/src/components/Configuration/RAGSettings/RAGItemList.tsx @@ -0,0 +1,40 @@ +import * as React from "react" +import { cn } from "@/lib/utils" +import { Button } from "@/components/ui/button" +import { ScrollArea } from "@/components/ui/scroll-area" +import { vectorDatabases, embeddingModels } from "./utils" +import { useRagItem } from "./useRAGItem" + +interface RAGItemListProps { + items: (typeof vectorDatabases[number] | typeof embeddingModels[number])[]; + type: string; +} + +export function RAGItemList({ items, type }: RAGItemListProps) { + const [ragItem, setRagItem] = useRagItem() + + return ( + +
+ {items.map((item) => ( + + ))} +
+ + ) +} \ No newline at end of file diff --git a/apps/webapp/src/components/Configuration/RAGSettings/RAGLayout.tsx b/apps/webapp/src/components/Configuration/RAGSettings/RAGLayout.tsx new file mode 100644 index 000000000..a64a8d279 --- /dev/null +++ b/apps/webapp/src/components/Configuration/RAGSettings/RAGLayout.tsx @@ -0,0 +1,79 @@ +import { Input } from "@/components/ui/input" +import { ResizableHandle, ResizablePanel, ResizablePanelGroup } from "@/components/ui/resizable" +import { Separator } from "@/components/ui/separator" +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs" +import { TooltipProvider } from "@/components/ui/tooltip" +import { Search } from "lucide-react" +import * as React from "react" +import { RAGItemDisplay } from "./RAGItemDisplay" +import { RAGItemList } from "./RAGItemList" +import { embeddingModels, vectorDatabases } from "./utils" +import { useRagItem } from "./useRAGItem" + +interface Props { + items: (typeof vectorDatabases[number] | typeof embeddingModels[number])[]; + defaultLayout: number[] | undefined + onSearchChange: (query: string) => void + activeTab: string; + setActiveTab: (tab: string) => void +} + +export function RAGLayout({ + items, + defaultLayout = [265, 440, 655], + onSearchChange, + activeTab, + setActiveTab +}: Props) { + const [ragItem, setRagItem] = useRagItem() + const handleSearchChange = (event: React.ChangeEvent) => { + onSearchChange(event.target.value) + } + + return ( + + { + document.cookie = `react-resizable-panels:layout=${JSON.stringify( + sizes + )}` + }} + className="h-full max-h-[800px] items-stretch" + > + + setActiveTab(value as "vectorDatabase" | "embeddingModel")}> +
+

RAG Settings

+

Configure your vector databases and embedding models for Retrieval-Augmented Generation.

+
+ + + Vector Databases + Embedding Models + +
+
+
+ + +
+
+
+ + + + + + item.name === ragItem.selected)} + type={activeTab} + /> + + + + ) +} \ No newline at end of file diff --git a/apps/webapp/src/components/Configuration/RAGSettings/RAGSettingsPage.tsx b/apps/webapp/src/components/Configuration/RAGSettings/RAGSettingsPage.tsx index 30b9ed1f0..22b96e7ef 100644 --- a/apps/webapp/src/components/Configuration/RAGSettings/RAGSettingsPage.tsx +++ b/apps/webapp/src/components/Configuration/RAGSettings/RAGSettingsPage.tsx @@ -1,555 +1,31 @@ -import React, { useEffect, useState } from 'react'; -import { useForm } from "react-hook-form"; -import { zodResolver } from "@hookform/resolvers/zod"; -import { useQueryClient } from "@tanstack/react-query"; -import { usePostHog } from 'posthog-js/react'; -import { toast } from "sonner"; -import * as z from "zod"; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { Switch } from "@/components/ui/switch"; -import { Label } from "@/components/ui/label"; -import { Button } from "@/components/ui/button"; -import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage } from "@/components/ui/form"; -import { Input } from "@/components/ui/input"; -import config from "@/lib/config"; -import useProjectStore from "@/state/projectStore"; -import useConnectionStrategies from "@/hooks/get/useConnectionStrategies"; -import useCreateConnectionStrategy from "@/hooks/create/useCreateConnectionStrategy"; -import useUpdateConnectionStrategy from "@/hooks/update/useUpdateConnectionStrategy"; -import useConnectionStrategyAuthCredentials from "@/hooks/get/useConnectionStrategyAuthCredentials"; -import Image from 'next/image'; +import * as React from "react" +import { RAGLayout } from "./RagLayout"; +import { embeddingModels, TabType, vectorDatabases } from "./utils"; -const formSchema = z.object({ - vectorDatabase: z.string(), - embeddingModel: z.string(), - apiKey: z.string().optional(), - baseUrl: z.string().optional(), - url: z.string().optional(), - indexName: z.string().optional(), - embeddingApiKey: z.string().optional(), -}); +export default function RAGSettingsPage() { + const [searchQuery, setSearchQuery] = React.useState("") + const [activeTab, setActiveTab] = React.useState("vectorDatabase") + const defaultLayout = undefined -export function RAGSettingsPage() { - const { idProject } = useProjectStore(); - const queryClient = useQueryClient(); - const posthog = usePostHog(); - - const { data: connectionStrategies } = useConnectionStrategies(); - const { createCsPromise } = useCreateConnectionStrategy(); - const { updateCsPromise } = useUpdateConnectionStrategy(); - const { mutateAsync: fetchCredentials } = useConnectionStrategyAuthCredentials(); - - const [ragModeActive, setRagModeActive] = useState(false); - - const form = useForm>({ - resolver: zodResolver(formSchema), - defaultValues: { - vectorDatabase: "", - embeddingModel: "", - apiKey: "", - baseUrl: "", - indexName: "", - url: "", - embeddingApiKey: "", - }, - }); - - const vectorDbConnectionStrategies = connectionStrategies?.filter( - (cs) => cs.type.startsWith('vector_db.') - ) || []; - - const embeddingModelConnectionStrategies = connectionStrategies?.filter( - (cs) => cs.type.startsWith('embedding_model.') - ) || []; - - useEffect(() => { - const currentVectorDb = form.getValues().vectorDatabase; - const currentEmbeddingModel = form.getValues().embeddingModel; - - const currentStrategy = vectorDbConnectionStrategies.find( - cs => cs.type === `vector_db.${currentVectorDb}` - ); - - const embeddingModelStrategy = embeddingModelConnectionStrategies.find( - cs => cs.type === `embedding_model.${currentEmbeddingModel}` - ); - - if (currentStrategy) { - fetchCredentials({ type: currentStrategy.type, attributes: getAttributesForVectorDb(currentVectorDb) }, { - onSuccess(data) { - setFormValuesFromCredentials(currentVectorDb, data); - setRagModeActive(currentStrategy.status === true); - } - }); - } else { - // Reset form values for the fields specific to vector databases - form.setValue("apiKey", ""); - form.setValue("baseUrl", ""); - form.setValue("url", ""); - form.setValue("indexName", ""); - setRagModeActive(false); - } - - if (embeddingModelStrategy) { - fetchCredentials({ type: embeddingModelStrategy.type, attributes: ['api_key'] }, { - onSuccess(data) { - form.setValue("embeddingApiKey", data[0]); - } - }); - } else { - form.setValue("embeddingApiKey", ""); - } - }, [form.watch("vectorDatabase"), form.watch("embeddingModel"), connectionStrategies]); - - const getAttributesForVectorDb = (vectorDb: string): string[] => { - switch (vectorDb) { - case 'turbopuffer': - return ['api_key']; - case 'pinecone': - return ['api_key', 'index_name']; - case 'qdrant': - return ['api_key', 'base_url']; - case 'chromadb': - return ['url']; - case 'weaviate': - return ['api_key', 'url']; - default: - return []; - } - }; - - const setFormValuesFromCredentials = (vectorDb: string, data: string[]) => { - switch (vectorDb) { - case 'turbopuffer': - form.setValue("apiKey", data[0]); - break; - case 'pinecone': - form.setValue("apiKey", data[0]); - form.setValue("indexName", data[1]); - break; - case 'qdrant': - form.setValue("apiKey", data[0]); - form.setValue("baseUrl", data[1]); - break; - case 'chromadb': - form.setValue("url", data[0]); - break; - case 'weaviate': - form.setValue("apiKey", data[0]); - form.setValue("url", data[1]); - break; - } - }; - - function onSubmit(values: z.infer) { - const { vectorDatabase, apiKey, baseUrl, url, indexName, embeddingModel, embeddingApiKey } = values; - const currentStrategy = vectorDbConnectionStrategies.find( - cs => cs.type === `vector_db.${vectorDatabase}` - ); - const currentEmbeddingModelStrategy = embeddingModelConnectionStrategies.find( - cs => cs.type === `embedding_model.${embeddingModel}` - ); - - const performUpdate = !!currentStrategy; - const performEmbeddingModelUpdate = !!currentEmbeddingModelStrategy; - - let attributes: string[] = []; - let attributeValues: string[] = []; - let embeddingModelAttributes: string[] = ['api_key']; - let embeddingModelAttributeValues: string[] = [embeddingApiKey!]; - - switch (vectorDatabase) { - case 'turbopuffer': - attributes = ['api_key']; - attributeValues = [apiKey!]; - break; - case 'pinecone': - attributes = ['api_key', 'index_name']; - attributeValues = [apiKey!, indexName!]; - break; - case 'qdrant': - attributes = ['api_key', 'base_url']; - attributeValues = [apiKey!, baseUrl!]; - break; - case 'chromadb': - attributes = ['url']; - attributeValues = [url!]; - break; - case 'weaviate': - attributes = ['api_key', 'url']; - attributeValues = [apiKey!, url!]; - break; - } - - const promise = performUpdate - ? updateCsPromise({ - id_cs: currentStrategy!.id_connection_strategy, - updateToggle: false, - status: ragModeActive, - attributes, - values: attributeValues, - }) - : createCsPromise({ - type: `vector_db.${vectorDatabase}`, - attributes, - values: attributeValues, - }); - - const embeddingModelPromise = performEmbeddingModelUpdate - ? updateCsPromise({ - id_cs: currentEmbeddingModelStrategy!.id_connection_strategy, - updateToggle: false, - status: true, - attributes: embeddingModelAttributes, - values: embeddingModelAttributeValues, - }) - : createCsPromise({ - type: `embedding_model.${embeddingModel}`, - attributes: embeddingModelAttributes, - values: embeddingModelAttributeValues, - }); - - toast.promise(Promise.all([promise, embeddingModelPromise]), { - loading: 'Saving RAG settings...', - success: () => { - queryClient.invalidateQueries({ queryKey: ['connection-strategies'] }); - return "RAG settings saved successfully"; - }, - error: (err: any) => err.message || 'An error occurred', - }); - - posthog?.capture(`RAG_settings_${performUpdate ? 'updated' : 'created'}`, { - id_project: idProject, - vector_database: vectorDatabase, - embedding_model: values.embeddingModel, - mode: config.DISTRIBUTION - }); + const handleSearchChange = (query: string) => { + setSearchQuery(query) } - const handleRagModeChange = (checked: boolean) => { - setRagModeActive(checked); - }; - - const renderVectorDbInputs = () => { - const vectorDatabase = form.watch("vectorDatabase"); - switch (vectorDatabase) { - case 'turbopuffer': - return ( - ( - - API Key - - - - - - )} - /> - ); - case 'pinecone': - return ( - <> - ( - - API Key - - - - - - )} - /> - ( - - Index Name - - - - - - )} - /> - - ); - case 'qdrant': - return ( - <> - ( - - API Key - - - - - - )} - /> - ( - - Base URL - - - - - - )} - /> - - ); - case 'chromadb': - return ( - ( - - URL - - - - - - )} - /> - ); - case 'weaviate': - return ( - <> - ( - - API Key - - - - - - )} - /> - ( - - URL - - - - - - )} - /> - - ); - default: - return null; - } - }; - - const renderEmbeddingModelInputs = () => { - const embeddingModel = form.watch("embeddingModel"); - if (embeddingModel === "OPENAI_ADA_SMALL_1536" || embeddingModel === "OPENAI_ADA_LARGE_3072" || embeddingModel === "OPENAI_ADA_002" || embeddingModel === "COHERE_MULTILINGUAL_V3") { - return ( - ( - - API Key - - - - - - )} - /> - ); - } - return null; - }; + const items = activeTab === "vectorDatabase" ? vectorDatabases : embeddingModels + const filteredItems = items.filter((item) => { + const matchesSearch = item.name.toLowerCase().includes(searchQuery.toLowerCase()) || item.description?.toLowerCase().includes(searchQuery.toLowerCase()) + return matchesSearch + }) return ( - - - RAG Settings - Configure your Retrieval-Augmented Generation settings - - -
- - ( - - Vector Database - - - - )} - /> - {renderVectorDbInputs()} - - - ( - - Embedding Model - - - - )} - /> - - {renderEmbeddingModelInputs()} - -
- - -
- - - - -
- - ); -} - +
+ +
+ ) +} \ No newline at end of file diff --git a/apps/webapp/src/components/Configuration/RAGSettings/useRagItem.tsx b/apps/webapp/src/components/Configuration/RAGSettings/useRagItem.tsx new file mode 100644 index 000000000..9d20bddbf --- /dev/null +++ b/apps/webapp/src/components/Configuration/RAGSettings/useRagItem.tsx @@ -0,0 +1,18 @@ +import { atom, useAtom } from "jotai" +import { vectorDatabases, embeddingModels } from "./utils" + +type ItemType = "vectorDatabase" | "embeddingModel" + +type Config = { + selected: string + type: string +} + +const configAtom = atom({ + selected: vectorDatabases[0].name, + type: "vectorDatabase" +}) + +export function useRagItem() { + return useAtom(configAtom) +} \ No newline at end of file diff --git a/apps/webapp/src/components/Configuration/RAGSettings/utils.ts b/apps/webapp/src/components/Configuration/RAGSettings/utils.ts new file mode 100644 index 000000000..770da4909 --- /dev/null +++ b/apps/webapp/src/components/Configuration/RAGSettings/utils.ts @@ -0,0 +1,17 @@ +export const vectorDatabases = [ + { name: "Pinecone", logoPath: "/rag/pinecone.png", description: "Pinecone vector database" }, + { name: "Qdrant", logoPath: "/rag/qdrant.png", description: "Qdrant vector database" }, + { name: "Weaviate", logoPath: "/rag/weaviate.webp", description: "Weaviate vector database" }, + { name: "Chromadb", logoPath: "/rag/chroma.webp", description: "ChromaDB vector database" }, + { name: "Turbopuffer", logoPath: "/rag/turbopuffer.png", description: "TurboPuffer vector database" }, +]; + +export const embeddingModels = [ + { name: "OPENAI_ADA_SMALL_1536", logoPath: "/rag/openai.png", description: "OpenAI - text-embedding-3-small" }, + { name: "OPENAI_ADA_LARGE_3072", logoPath: "/rag/openai.png", description: "OpenAI - text-embedding-3-large" }, + { name: "OPENAI_ADA_002", logoPath: "/rag/openai.png", description: "OpenAI - text-embedding-ada-002" }, + { name: "COHERE_MULTILINGUAL_V3", logoPath: "/rag/cohere.jpeg", description: "Cohere - embed-multilingual-v3.0" }, +]; + +export type TabType = "vectorDatabase" | "embeddingModel"; + \ No newline at end of file diff --git a/apps/webapp/src/hooks/create/useCreateConnectionStrategy.tsx b/apps/webapp/src/hooks/create/useCreateConnectionStrategy.tsx index 735d57527..74702d6da 100644 --- a/apps/webapp/src/hooks/create/useCreateConnectionStrategy.tsx +++ b/apps/webapp/src/hooks/create/useCreateConnectionStrategy.tsx @@ -6,6 +6,7 @@ interface IConnectionStrategyDto { type: string, attributes: string[], values: string[], + status?: boolean } const useCreateConnectionStrategy = () => { diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 70a9045e7..325e8e0f7 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -45,7 +45,7 @@ services: REDIS_PORT: ${REDIS_PORT} WEBAPP_URL: ${NEXT_PUBLIC_WEBAPP_DOMAIN} #REDIS_TLS: 1 # set this variable to 1 when Redis is AWS hosted - REDIS_DB: ${REDIS_DB} + REDIS_DB: ${REDIS_DB} ENCRYPT_CRYPTO_SECRET_KEY: ${ENCRYPT_CRYPTO_SECRET_KEY} HUBSPOT_CRM_CLOUD_CLIENT_ID: ${HUBSPOT_CRM_CLOUD_CLIENT_ID} HUBSPOT_CRM_CLOUD_CLIENT_SECRET: ${HUBSPOT_CRM_CLOUD_CLIENT_SECRET} @@ -339,8 +339,8 @@ services: volumes: - minio_storage:/data environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + MINIO_ROOT_USER: myaccesskey13 + MINIO_ROOT_PASSWORD: mysecretkey12 command: server --console-address ":9001" /data healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] diff --git a/docs/catalog.mdx b/docs/catalog.mdx index 23ebd1de1..8066ff787 100644 --- a/docs/catalog.mdx +++ b/docs/catalog.mdx @@ -44,6 +44,14 @@ icon: album-collection } horizontal> + + } horizontal> + + + } horizontal> + @@ -64,6 +72,10 @@ icon: album-collection } horizontal> + + } horizontal> + @@ -72,6 +84,22 @@ icon: album-collection } horizontal> + + } horizontal> + + + } horizontal> + + + } horizontal> + + + } horizontal> + @@ -97,6 +125,10 @@ icon: album-collection } horizontal> + + } horizontal> + @@ -106,6 +138,14 @@ icon: album-collection } horizontal> + + } horizontal> + + + } horizontal> + diff --git a/docs/integrations-catalog.mdx b/docs/integrations-catalog.mdx index 823ccb4bd..9c7847abb 100644 --- a/docs/integrations-catalog.mdx +++ b/docs/integrations-catalog.mdx @@ -9,7 +9,7 @@ icon: album-collection - + diff --git a/docs/integrations/crm/microsoftdynamicssales/index.mdx b/docs/integrations/crm/microsoftdynamicssales/index.mdx new file mode 100644 index 000000000..be78b93b0 --- /dev/null +++ b/docs/integrations/crm/microsoftdynamicssales/index.mdx @@ -0,0 +1,25 @@ +--- +title: "Microsoft Dynamics Sales" +description: "" +--- + +# Common Objects + + +| Unified Model | Supported | Provider Endpoints | +| -------- | ------------------------------------- | ------------------------------------- | +| Contact | Yes ✅ | /Contacts | +| Company | Yes ✅ | /Accounts | +| Deal | Yes ✅ | /Deals | +| Engagement | Yes ✅| [/Calls , /Events] | +| Note | Yes ✅ | [/Notes, /Notes/:remote_note_id, ] | +| Stage | No 🚫| | +| Task | Yes ✅| /Tasks | +| User | Yes ✅ | /users?type=AllUsers | + + + +| Features | Supported | +| -------- | ------------------------------------- | +| Scopes |ZohoCRM.modules.ALL,ZohoCRM.users.READ | +| Realtime webhook | No 🚫| \ No newline at end of file diff --git a/docs/integrations/crm/salesforce/index.mdx b/docs/integrations/crm/salesforce/index.mdx new file mode 100644 index 000000000..9c60fa3f5 --- /dev/null +++ b/docs/integrations/crm/salesforce/index.mdx @@ -0,0 +1,25 @@ +--- +title: "Salesforce" +description: "" +--- + +# Common Objects + + +| Unified Model | Supported | Provider Endpoints | +| -------- | ------------------------------------- | ------------------------------------- | +| Contact | Yes ✅ | /Contacts | +| Company | Yes ✅ | /Accounts | +| Deal | Yes ✅ | /Deals | +| Engagement | Yes ✅| [/Calls , /Events] | +| Note | Yes ✅ | [/Notes, /Notes/:remote_note_id, ] | +| Stage | No 🚫| | +| Task | Yes ✅| /Tasks | +| User | Yes ✅ | /users?type=AllUsers | + + + +| Features | Supported | +| -------- | ------------------------------------- | +| Scopes |ZohoCRM.modules.ALL,ZohoCRM.users.READ | +| Realtime webhook | No 🚫| \ No newline at end of file diff --git a/docs/integrations/ecommerce/webflow/index.mdx b/docs/integrations/ecommerce/webflow/index.mdx new file mode 100644 index 000000000..d9dd40dea --- /dev/null +++ b/docs/integrations/ecommerce/webflow/index.mdx @@ -0,0 +1,21 @@ +--- +title: "Webflow" +description: "" +--- + +# Common Objects + + +| Unified Model | Supported | Provider Endpoints | +| -------- | ------------------------------------- | ------------------------------------- | +| Customer | Yes ✅ | /admin/api/2024-07/customers.json | +| Order | Yes ✅ | /admin/api/2024-07/orders.json | +| Fulfillment | Yes ✅ | /admin/api/2024-07/orders/:remote_order_id/fulfillments.json| +| Fulfillment Orders | No 🚫| | +| Product | Yes ✅ | /admin/api/2024-07/products.json | + + +| Features | Supported | +| -------- | ------------------------------------- | +| Scopes | | +| Realtime webhook | No 🚫| \ No newline at end of file diff --git a/docs/integrations/filestorage/dropbox/index.mdx b/docs/integrations/filestorage/dropbox/index.mdx new file mode 100644 index 000000000..5d0abf40e --- /dev/null +++ b/docs/integrations/filestorage/dropbox/index.mdx @@ -0,0 +1,24 @@ +--- +title: "Dropbox" +description: "" +--- + +# Common Objects + + +| Unified Model | Supported | Provider Endpoints | +| -------- | ------------------------------------- | ------------------------------------- | +| Drive |No 🚫 | | +| File | Yes ✅ | /folders/:remote_folder_id/items | +| Folder | Yes ✅ | [/folders, /folders/:remote_folder_id/items] | +| Group | Yes ✅| /groups | +| Permission | No 🚫 | Coming Soon | +| Shared Link | No 🚫| Coming Soon | +| User | Yes ✅ | /users | + + + +| Features | Supported | +| -------- | ------------------------------------- | +| Scopes | | +| Realtime webhook | No 🚫| \ No newline at end of file diff --git a/docs/integrations/filestorage/googledrive/index.mdx b/docs/integrations/filestorage/googledrive/index.mdx new file mode 100644 index 000000000..3dda5fc5c --- /dev/null +++ b/docs/integrations/filestorage/googledrive/index.mdx @@ -0,0 +1,24 @@ +--- +title: "Google Drive" +description: "" +--- + +# Common Objects + + +| Unified Model | Supported | Provider Endpoints | +| -------- | ------------------------------------- | ------------------------------------- | +| Drive |No 🚫 | | +| File | Yes ✅ | /folders/:remote_folder_id/items | +| Folder | Yes ✅ | [/folders, /folders/:remote_folder_id/items] | +| Group | Yes ✅| /groups | +| Permission | No 🚫 | Coming Soon | +| Shared Link | No 🚫| Coming Soon | +| User | Yes ✅ | /users | + + + +| Features | Supported | +| -------- | ------------------------------------- | +| Scopes | | +| Realtime webhook | No 🚫| \ No newline at end of file diff --git a/docs/integrations/filestorage/onedrive/index.mdx b/docs/integrations/filestorage/onedrive/index.mdx new file mode 100644 index 000000000..b96f7da10 --- /dev/null +++ b/docs/integrations/filestorage/onedrive/index.mdx @@ -0,0 +1,24 @@ +--- +title: "One Drive" +description: "" +--- + +# Common Objects + + +| Unified Model | Supported | Provider Endpoints | +| -------- | ------------------------------------- | ------------------------------------- | +| Drive |No 🚫 | | +| File | Yes ✅ | /folders/:remote_folder_id/items | +| Folder | Yes ✅ | [/folders, /folders/:remote_folder_id/items] | +| Group | Yes ✅| /groups | +| Permission | No 🚫 | Coming Soon | +| Shared Link | No 🚫| Coming Soon | +| User | Yes ✅ | /users | + + + +| Features | Supported | +| -------- | ------------------------------------- | +| Scopes | | +| Realtime webhook | No 🚫| \ No newline at end of file diff --git a/docs/integrations/filestorage/sharepoint/index.mdx b/docs/integrations/filestorage/sharepoint/index.mdx new file mode 100644 index 000000000..b38fe3741 --- /dev/null +++ b/docs/integrations/filestorage/sharepoint/index.mdx @@ -0,0 +1,24 @@ +--- +title: "Sharepoint" +description: "" +--- + +# Common Objects + + +| Unified Model | Supported | Provider Endpoints | +| -------- | ------------------------------------- | ------------------------------------- | +| Drive |No 🚫 | | +| File | Yes ✅ | /folders/:remote_folder_id/items | +| Folder | Yes ✅ | [/folders, /folders/:remote_folder_id/items] | +| Group | Yes ✅| /groups | +| Permission | No 🚫 | Coming Soon | +| Shared Link | No 🚫| Coming Soon | +| User | Yes ✅ | /users | + + + +| Features | Supported | +| -------- | ------------------------------------- | +| Scopes | | +| Realtime webhook | No 🚫| \ No newline at end of file diff --git a/docs/integrations/hris/deel/index.mdx b/docs/integrations/hris/deel/index.mdx new file mode 100644 index 000000000..d66442495 --- /dev/null +++ b/docs/integrations/hris/deel/index.mdx @@ -0,0 +1,30 @@ +--- +title: "Deel" +description: "" +--- + +# Common Objects + +| Unified Model | Supported | Provider Endpoints | +| ---------------- | --------- | ------------------ | +| Bankinfo | No 🚫 | | +| Benefit | Yes ✅ | /v1/employees/${employee.remote_id}/employee_benefits | +| Company | Yes ✅ | /v1/companies/${id} | +| Dependent | No 🚫 | | +| Employee | Yes ✅ | /v1/companies/${company.remote_id}/employees | +| Employeepayrollrun | No 🚫 | | +| Employerbenefit | Yes ✅ | /v1/companies/${company.remote_id}/company_benefits | +| Employment | Yes ✅ | /v1/companies/${company.remote_id}/employees | +| Group | Yes ✅ | /v1/companies/${company.remote_id}/departments | +| Location | Yes ✅ | [/v1/employees/${employee.remote_id}/work_addresses , /v1/employees/${employee.remote_id}/home_addresses] | +| Paygroup | No 🚫 | | +| Payrollrun | No 🚫 | | +| Timeoff | No 🚫 | | +| Timeoffbalance | No 🚫 | | +| Timesheetentry | No 🚫 | | + + +| Features | Supported | +| -------- | ------------------------------------- | +| Scopes || +| Realtime webhook | No 🚫| \ No newline at end of file diff --git a/docs/integrations/hris/sage/index.mdx b/docs/integrations/hris/sage/index.mdx new file mode 100644 index 000000000..f6e7b0e67 --- /dev/null +++ b/docs/integrations/hris/sage/index.mdx @@ -0,0 +1,30 @@ +--- +title: "Sage" +description: "" +--- + +# Common Objects + +| Unified Model | Supported | Provider Endpoints | +| ---------------- | --------- | ------------------ | +| Bankinfo | No 🚫 | | +| Benefit | Yes ✅ | /v1/employees/${employee.remote_id}/employee_benefits | +| Company | Yes ✅ | /v1/companies/${id} | +| Dependent | No 🚫 | | +| Employee | Yes ✅ | /v1/companies/${company.remote_id}/employees | +| Employeepayrollrun | No 🚫 | | +| Employerbenefit | Yes ✅ | /v1/companies/${company.remote_id}/company_benefits | +| Employment | Yes ✅ | /v1/companies/${company.remote_id}/employees | +| Group | Yes ✅ | /v1/companies/${company.remote_id}/departments | +| Location | Yes ✅ | [/v1/employees/${employee.remote_id}/work_addresses , /v1/employees/${employee.remote_id}/home_addresses] | +| Paygroup | No 🚫 | | +| Payrollrun | No 🚫 | | +| Timeoff | No 🚫 | | +| Timeoffbalance | No 🚫 | | +| Timesheetentry | No 🚫 | | + + +| Features | Supported | +| -------- | ------------------------------------- | +| Scopes || +| Realtime webhook | No 🚫| \ No newline at end of file diff --git a/docs/integrations/ticketing/linear/index.mdx b/docs/integrations/ticketing/linear/index.mdx new file mode 100644 index 000000000..b70296c32 --- /dev/null +++ b/docs/integrations/ticketing/linear/index.mdx @@ -0,0 +1,25 @@ +--- +title: "Linear" +description: "" +--- + +# Common Objects + + +| Unified Model | Supported | Provider Endpoints | +| -------- | ------------------------------------- | ------------------------------------- | +| Account | Yes ✅ | /accounts | +| Attachment | No 🚫 | Coming Soon | +| Collection |No 🚫 | | +| Comment | Yes ✅| /conversations/:remote_ticket_id/comments | +| Contact | Yes ✅ | /contacts | +| Tag | Yes ✅| /tags | +| Team | Yes ✅| /teams | +| Ticket | Yes ✅ | /conversations | +| User | Yes ✅| /teammates| + + +| Features | Supported | +| -------- | ------------------------------------- | +| Scopes | | +| Realtime webhook | No 🚫| \ No newline at end of file diff --git a/docs/mint.json b/docs/mint.json index 9076d1888..d3a8fac54 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -55,7 +55,8 @@ "group": "Documentation", "pages": [ "quick-start", - "catalog" + "catalog", + "what-is-panora" ] }, { @@ -603,158 +604,6 @@ "api-reference/postman" ] }, - { - "group": "CRM Objects", - "pages": [ - { - "group": "CRM", - "pages": [ - { - "group": "Contact", - "pages": [ - "api-reference/crmcontact/list-a-batch-of-crm-contacts", - "api-reference/crmcontact/create-crm-contact", - "api-reference/crmcontact/update-a-crm-contact", - "api-reference/crmcontact/retrieve-a-crm-contact", - "api-reference/crmcontact/add-a-batch-of-crm-contacts" - ] - }, - { - "group": "Deal", - "pages": [ - "api-reference/crmdeal/list-a-batch-of-deals", - "api-reference/crmdeal/create-a-deal", - "api-reference/crmdeal/retrieve-a-deal", - "api-reference/crmdeal/update-a-deal", - "api-reference/crmdeal/add-a-batch-of-deals" - ] - }, - { - "group": "Note", - "pages": [ - "api-reference/crmnote/list-a-batch-of-notes", - "api-reference/crmnote/create-a-note", - "api-reference/crmnote/retrieve-a-note", - "api-reference/crmnote/add-a-batch-of-notes" - ] - }, - { - "group": "Company", - "pages": [ - "api-reference/crmcompany/list-a-batch-of-companies", - "api-reference/crmcompany/create-a-company", - "api-reference/crmcompany/update-a-company", - "api-reference/crmcompany/retrieve-a-company", - "api-reference/crmcompany/add-a-batch-of-companies" - ] - }, - { - "group": "Engagement", - "pages": [ - "api-reference/crmengagement/list-a-batch-of-engagements", - "api-reference/crmengagement/create-a-engagement", - "api-reference/crmengagement/update-a-engagement", - "api-reference/crmengagement/retrieve-a-engagement", - "api-reference/crmengagement/add-a-batch-of-engagements" - ] - }, - { - "group": "Stage", - "pages": [ - "api-reference/crmstage/list-a-batch-of-stages", - "api-reference/crmstage/retrieve-a-stage" - ] - }, - { - "group": "Task", - "pages": [ - "api-reference/crmtask/list-a-batch-of-tasks", - "api-reference/crmtask/create-a-task", - "api-reference/crmtask/update-a-task", - "api-reference/crmtask/retrieve-a-task", - "api-reference/crmtask/add-a-batch-of-tasks" - ] - }, - { - "group": "User", - "pages": [ - "api-reference/crmuser/list-a-batch-of-users", - "api-reference/crmuser/retrieve-a-user" - ] - } - ] - }, - { - "group": "Ticketing", - "pages": [ - { - "group": "Ticket", - "pages": [ - "api-reference/ticketingticket/list-a-batch-of-tickets", - "api-reference/ticketingticket/create-a-ticket", - "api-reference/ticketingticket/update-a-ticket", - "api-reference/ticketingticket/retrieve-a-ticket", - "api-reference/ticketingticket/add-a-batch-of-tickets" - ] - }, - { - "group": "Comment", - "pages": [ - "api-reference/ticketingcomment/list-a-batch-of-comments", - "api-reference/ticketingcomment/create-a-comment", - "api-reference/ticketingcomment/retrieve-a-comment", - "api-reference/ticketingcomment/add-a-batch-of-comments" - ] - }, - { - "group": "User", - "pages": [ - "api-reference/ticketinguser/list-a-batch-of-users", - "api-reference/ticketinguser/retrieve-a-user" - ] - }, - { - "group": "Attachment", - "pages": [ - "api-reference/ticketingattachment/list-a-batch-of-attachments", - "api-reference/ticketingattachment/create-a-attachment", - "api-reference/ticketingattachment/retrieve-a-attachment", - "api-reference/ticketingattachment/download-a-attachment", - "api-reference/ticketingattachment/add-a-batch-of-attachments" - ] - }, - { - "group": "Contact", - "pages": [ - "api-reference/ticketingcontact/list-a-batch-of-contacts", - "api-reference/ticketingcontact/retrieve-a-contact" - ] - }, - { - "group": "Account", - "pages": [ - "api-reference/ticketingaccount/list-a-batch-of-accounts", - "api-reference/ticketingaccount/retrieve-an-account" - ] - }, - { - "group": "Tag", - "pages": [ - "api-reference/ticketingtag/list-a-batch-of-tags", - "api-reference/ticketingtag/retrieve-a-tag" - ] - }, - { - "group": "Team", - "pages": [ - "api-reference/ticketingteam/list-a-batch-of-teams", - "api-reference/ticketingteam/retrieve-a-team" - ] - } - ] - } - ] - }, { "group": "Core Resources", "pages": [ diff --git a/docs/snippets/ats-catalog.mdx b/docs/snippets/ats-catalog.mdx index 907a7c090..f5c2ffe7a 100644 --- a/docs/snippets/ats-catalog.mdx +++ b/docs/snippets/ats-catalog.mdx @@ -1,3 +1,3 @@ | | Activities | Applications | Candidates | Departments | Interviews | Jobs | Offers | Offices | Scorecard | Users | |-------------|:----------:|:------------:|:----------:|:-----------:|:----------:|:----:|:------:|:-------:|:---------:|:-----:| -| Ashby | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | +| [Ashby](https://www.ashbyhq.com/) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | \ No newline at end of file diff --git a/docs/snippets/crm-catalog.mdx b/docs/snippets/crm-catalog.mdx index 87dd52264..fcf2bb5d9 100644 --- a/docs/snippets/crm-catalog.mdx +++ b/docs/snippets/crm-catalog.mdx @@ -6,3 +6,5 @@ | [Zendesk Sell](https://www.zendesk.com/sell/) | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | | | [Freshsales](https://www.freshworks.com/crm/sales/) | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | | | [Attio](https://attio.com) | ✔️ | | | | | | ✔️ | +| [Microsoft Dynamics Sales](https://www.microsoft.com/en-us/dynamics-365/products/sales) | ✔️ | | | | | | ✔️ | +| [Salesforce](https://www.salesforce.com/) | ✔️ | | | | | | ✔️ | diff --git a/docs/snippets/ecommerce-catalog.mdx b/docs/snippets/ecommerce-catalog.mdx index e47b59410..064faf123 100644 --- a/docs/snippets/ecommerce-catalog.mdx +++ b/docs/snippets/ecommerce-catalog.mdx @@ -1,4 +1,7 @@ | | Products | Fulfillments | Fulfillment Orders | Customers | Orders | |-------------|:----------:|:------------:|:----------:|:-----------:|:----------:| -| Shopify | ✔ | ✔ | | ✔ | ✔ | -| WooCommerce | ✔ | | | ✔ | ✔ | +| [Shopify](https://www.shopify.com) | ✔ | ✔ | | ✔ | ✔ | +| [WooCommerce](https://woocommerce.com) | ✔ | | | ✔ | ✔ | +| [Webflow](https://webflow.com) | ✔ | | | ✔ | ✔ | +| [Amazon](https://sell.amazon.com) | ✔ | | | ✔ | ✔ | +| [Squarespace](https://www.squarespace.com) | ✔ | | | ✔ | ✔ | \ No newline at end of file diff --git a/docs/snippets/filestorage-catalog.mdx b/docs/snippets/filestorage-catalog.mdx index 252d04ed5..de00874c6 100644 --- a/docs/snippets/filestorage-catalog.mdx +++ b/docs/snippets/filestorage-catalog.mdx @@ -1,6 +1,7 @@ | File Storage | Drives | Files | Folders | Groups | Users | Permissions | Shared Links | |-----------------------------------------------|:--------:|:-----:|:-----:|:-----------:|:-----:|:-----:|:---------:| -| [Google Drive]() | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | | | -| [Box]() | | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | | -| [Dropbox]() | | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | | -| [OneDrive]() | ✔️ | ✔️ | ✔️| ✔️ | ✔️ | | | +| [Google Drive](https://www.google.com/drive/) | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | | | +| [Box](https://www.box.com/) | | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | | +| [Dropbox](https://www.dropbox.com/) | | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | | +| [OneDrive](https://www.microsoft.com/en-us/microsoft-365/onedrive/online-cloud-storage) | ✔️ | ✔️ | ✔️| ✔️ | ✔️ | | | +| [SharePoint](https://www.microsoft.com/en-us/microsoft-365/sharepoint/collaboration) | ✔️ | ✔️ | ✔️| ✔️ | ✔️ | | | \ No newline at end of file diff --git a/docs/snippets/hris-catalog.mdx b/docs/snippets/hris-catalog.mdx index 9fc70aee0..066788407 100644 --- a/docs/snippets/hris-catalog.mdx +++ b/docs/snippets/hris-catalog.mdx @@ -1,3 +1,5 @@ | | Bankinfo | Benefit | Company | Dependent | Employee | Employeepayrollrun | Employerbenefit | Employment | Group | Location | Paygroup | Payrollrun | Timeoff | Timeoffbalance | Timesheetentry | |-------------|----------|:-------:|:-------:|:---------:|:--------:|:------------------:|:---------------:|:----------:|:-----:|:--------:|:--------:|:----------:|:-------:|:--------------:|:---------------:| -| Gusto | ❌ | ✔ | ✔ | ❌ | ✔ | ❌ | ✔ | ✔ | ✔ | ✔ | ❌ | ❌ | ❌ | ❌ | ❌ | \ No newline at end of file +| [Gusto](https://gusto.com/) | ❌ | ✔ | ✔ | ❌ | ✔ | ❌ | ✔ | ✔ | ✔ | ✔ | ❌ | ❌ | ❌ | ❌ | ❌ | +| [Deel](https://www.deel.com/) | ❌ | ✔ | ✔ | ❌ | ✔ | ❌ | ✔ | ✔ | ✔ | ✔ | ❌ | ❌ | ❌ | ❌ | ❌ | +| [Sage](https://www.sage.com/) | ❌ | ✔ | ✔ | ❌ | ✔ | ❌ | ✔ | ✔ | ✔ | ✔ | ❌ | ❌ | ❌ | ❌ | ❌ | \ No newline at end of file diff --git a/docs/snippets/ticketing-catalog.mdx b/docs/snippets/ticketing-catalog.mdx index fe944c27f..f0a50f5b1 100644 --- a/docs/snippets/ticketing-catalog.mdx +++ b/docs/snippets/ticketing-catalog.mdx @@ -1,6 +1,8 @@ | | Tickets | Comments | Users | Contacts | Accounts | Tags | Teams | Collections | |-------------|:----------:|:-------:|:-------:|:------------:|:-------:|:-------:|:------:|:-------------:| -| Zendesk | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | | -| Front | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | | -| Jira | ✔ | ✔ | ✔ | | | ✔ | ✔ | ✔ | -| Gitlab | ✔ | ✔ | ✔ | | | | | ✔| +| [Zendesk](https://www.zendesk.com/) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | | +| [Front](https://front.com/) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | | +| [Jira](https://www.atlassian.com/software/jira) | ✔ | ✔ | ✔ | | | ✔ | ✔ | ✔ | +| [GitLab](https://about.gitlab.com/) | ✔ | ✔ | ✔ | | | | | ✔| +| [GitHub](https://github.com/) | ✔ | ✔ | ✔ | | | | | ✔| +| [Linear](https://linear.app/) | ✔ | ✔ | ✔ | | | | | ✔| \ No newline at end of file diff --git a/docs/what-is-panora.mdx b/docs/what-is-panora.mdx new file mode 100644 index 000000000..cedb4af7a --- /dev/null +++ b/docs/what-is-panora.mdx @@ -0,0 +1,15 @@ +--- +title: "What is Panora?" +description: "Integrate with the Panora API to easily collect data across your CRM, File Storage and Ticketing systems in real time." +icon: "star" +--- +We are a universal API that allows you to sync data from multiple third-party platforms. +We maintain standardized models across all verticals. +Panora works by enabling your customers to seamlessly connect their diverse data sources (CRM systems, File Storage platforms, Ticketing tools, etc.) to your application.
+ +This integration process is streamlined through [Panora Magic Link](/core-concepts/magic-links), our user-friendly interface that provides a secure and elegant authorization flow. Here, your customers can select their preferred providers and authorize access to their various data systems.
+Upon successful connection, Panora issues a connection token to your application, which you can use to make API requests to Panora's endpoints. This token grants you access to a wealth of standardized data for a sepcific third-party.
+ + +What sets Panora apart is its advanced RAG (Retrieval-Augmented Generation) pipeline capabilities. As part of our service, you can leverage Panora's chunking and embedding functionalities for enhanced retrieval search. This feature allows you to efficiently process and search through large volumes of connected data, enabling more accurate and context-aware information retrieval in your applications.
+By integrating Panora, you're not just connecting data sources – you're unlocking the power of intelligent data processing and retrieval, all through a single, unified API. \ No newline at end of file diff --git a/packages/api/prisma/schema.prisma b/packages/api/prisma/schema.prisma index 45ad8e9b0..d9889a819 100644 --- a/packages/api/prisma/schema.prisma +++ b/packages/api/prisma/schema.prisma @@ -1346,6 +1346,11 @@ model connector_sets { ats_ashby Boolean? ecom_webflow Boolean? crm_microsoftdynamicssales Boolean? + fs_dropbox Boolean? + fs_googledrive Boolean? + fs_sharepoint Boolean? + fs_onedrive Boolean? + crm_salesforce Boolean? projects projects[] } diff --git a/packages/api/src/@core/@core-services/queues/shared.service.ts b/packages/api/src/@core/@core-services/queues/shared.service.ts index 08c38289f..b059e2529 100644 --- a/packages/api/src/@core/@core-services/queues/shared.service.ts +++ b/packages/api/src/@core/@core-services/queues/shared.service.ts @@ -36,19 +36,20 @@ export class BullQueueService { return this.ragDocumentQueue; } - async queueSyncJob(jobName: string, jobData: any, cron: string) { + async removeRepeatableJob(jobName: string) { const jobs = await this.syncJobsQueue.getRepeatableJobs(); for (const job of jobs) { if (job.name === jobName) { await this.syncJobsQueue.removeRepeatableByKey(job.key); } } - //await this.syncJobsQueue.add('health-check', {}, { attempts: 1 }); + } - // Add new job with the job name and data + async queueSyncJob(jobName: string, jobData: any, cron: string) { + await this.removeRepeatableJob(jobName); const res = await this.syncJobsQueue.add(jobName, jobData, { repeat: { cron }, - jobId: jobName, // Using jobId to identify repeatable jobs + jobId: jobName, }); console.log('job is ' + JSON.stringify(res)); } diff --git a/packages/api/src/@core/connections-strategies/connections-strategies.controller.ts b/packages/api/src/@core/connections-strategies/connections-strategies.controller.ts index 209bd5445..0047dbd90 100644 --- a/packages/api/src/@core/connections-strategies/connections-strategies.controller.ts +++ b/packages/api/src/@core/connections-strategies/connections-strategies.controller.ts @@ -47,12 +47,13 @@ export class ConnectionsStrategiesController { @Body() connectionStrategyCreateDto: CreateConnectionStrategyDto, ) { const { id_project } = req.user; - const { type, attributes, values } = connectionStrategyCreateDto; + const { type, attributes, values, status } = connectionStrategyCreateDto; return await this.connectionsStrategiesService.createConnectionStrategy( id_project, type, attributes, values, + status, ); } diff --git a/packages/api/src/@core/connections-strategies/connections-strategies.service.ts b/packages/api/src/@core/connections-strategies/connections-strategies.service.ts index 4f95145e2..ebae9e14d 100644 --- a/packages/api/src/@core/connections-strategies/connections-strategies.service.ts +++ b/packages/api/src/@core/connections-strategies/connections-strategies.service.ts @@ -59,6 +59,7 @@ export class ConnectionsStrategiesService { type: string, attributes: string[], values: string[], + status?: boolean, ) { try { const checkCSDuplicate = @@ -79,7 +80,7 @@ export class ConnectionsStrategiesService { id_connection_strategy: uuidv4(), id_project: projectId, type: type, - status: true, + status: status || true, }, }); const entity = await this.prisma.cs_entities.create({ diff --git a/packages/api/src/@core/connections-strategies/dto/create-connections-strategies.dto.ts b/packages/api/src/@core/connections-strategies/dto/create-connections-strategies.dto.ts index 8f79fbbf0..ff31d042b 100644 --- a/packages/api/src/@core/connections-strategies/dto/create-connections-strategies.dto.ts +++ b/packages/api/src/@core/connections-strategies/dto/create-connections-strategies.dto.ts @@ -1,4 +1,4 @@ -import { ApiProperty } from '@nestjs/swagger'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; export class CreateConnectionStrategyDto { @ApiProperty() @@ -7,4 +7,6 @@ export class CreateConnectionStrategyDto { attributes: string[]; @ApiProperty() values: string[]; + @ApiPropertyOptional() + status?: boolean; } diff --git a/packages/api/src/@core/rag/embedding/embedding.credentials.service.ts b/packages/api/src/@core/rag/embedding/embedding.credentials.service.ts index c15626f09..4561375b7 100644 --- a/packages/api/src/@core/rag/embedding/embedding.credentials.service.ts +++ b/packages/api/src/@core/rag/embedding/embedding.credentials.service.ts @@ -14,11 +14,12 @@ export class EmbeddingCredentialsService { projectId: string, embeddingModel: EmbeddingModelType, ): Promise { - const type = `embedding_model.${embeddingModel.toLowerCase()}`; - const isCustom = await this.connectionsStrategiesService.isCustomCredentials( - projectId, - type, - ); + const type = `embeddingModel.${embeddingModel.toLowerCase()}`; + const isCustom = + await this.connectionsStrategiesService.isCustomCredentials( + projectId, + type, + ); if (isCustom) { return this.getCustomCredentials(projectId, type); @@ -54,4 +55,4 @@ export class EmbeddingCredentialsService { throw new Error(`Unsupported embedding model: ${embeddingModel}`); } } -} \ No newline at end of file +} diff --git a/packages/api/src/@core/rag/embedding/embedding.service.ts b/packages/api/src/@core/rag/embedding/embedding.service.ts index fbe32d6d9..df96355fa 100644 --- a/packages/api/src/@core/rag/embedding/embedding.service.ts +++ b/packages/api/src/@core/rag/embedding/embedding.service.ts @@ -37,14 +37,24 @@ export class EmbeddingService implements OnModuleInit { let apiKey: string; if (projectId) { - const activeStrategies = await this.connectionsStrategiesService.getConnectionStrategiesForProject(projectId); + const activeStrategies = + await this.connectionsStrategiesService.getConnectionStrategiesForProject( + projectId, + ); const activeEmbeddingStrategy = activeStrategies.find( - (strategy) => strategy.type.startsWith('embedding_model.') && strategy.status, + (strategy) => + strategy.type.startsWith('embeddingModel.') && strategy.status, ); if (activeEmbeddingStrategy) { - embeddingType = activeEmbeddingStrategy.type.split('.')[1].toUpperCase() as EmbeddingModelType; - [apiKey] = await this.embeddingCredentialsService.getEmbeddingCredentials(projectId, embeddingType); + embeddingType = activeEmbeddingStrategy.type + .split('.')[1] + .toUpperCase() as EmbeddingModelType; + [apiKey] = + await this.embeddingCredentialsService.getEmbeddingCredentials( + projectId, + embeddingType, + ); } else { embeddingType = 'OPENAI_ADA_002'; apiKey = this.envService.getOpenAIApiKey(); @@ -104,4 +114,4 @@ export class EmbeddingService implements OnModuleInit { // await this.initializeEmbeddings(projectId); return this.embeddings.embedQuery(query); } -} \ No newline at end of file +} diff --git a/packages/api/src/@core/rag/vecdb/vecdb.credentials.service.ts b/packages/api/src/@core/rag/vecdb/vecdb.credentials.service.ts index 8e97e7eee..b574f4fb4 100644 --- a/packages/api/src/@core/rag/vecdb/vecdb.credentials.service.ts +++ b/packages/api/src/@core/rag/vecdb/vecdb.credentials.service.ts @@ -13,7 +13,7 @@ export class VectorDbCredentialsService { projectId: string, vectorDb: string, ): Promise { - const type = `vector_db.${vectorDb}`; + const type = `vectorDatabase.${vectorDb}`; const isCustom = await this.connectionsStrategiesService.isCustomCredentials( projectId, @@ -65,15 +65,15 @@ export class VectorDbCredentialsService { private getAttributesForVectorDb(vectorDb: string): string[] { switch (vectorDb) { case 'pinecone': - return ['api_key', 'index_name']; + return ['apiKey', 'indexName']; case 'turbopuffer': - return ['api_key']; + return ['apiKey']; case 'qdrant': - return ['api_key', 'base_url']; + return ['apiKey', 'baseUrl']; case 'chromadb': return ['url']; case 'weaviate': - return ['api_key', 'url']; + return ['apiKey', 'url']; default: throw new Error(`Unsupported vector database: ${vectorDb}`); } diff --git a/packages/api/src/@core/rag/vecdb/vecdb.service.ts b/packages/api/src/@core/rag/vecdb/vecdb.service.ts index 1d3340e6e..7ddbb398e 100644 --- a/packages/api/src/@core/rag/vecdb/vecdb.service.ts +++ b/packages/api/src/@core/rag/vecdb/vecdb.service.ts @@ -43,7 +43,8 @@ export class VectorDatabaseService implements OnModuleInit { projectId, ); const activeVectorDbStrategy = activeStrategies.find( - (strategy) => strategy.type.startsWith('vector_db.') && strategy.status, + (strategy) => + strategy.type.startsWith('vectorDatabase.') && strategy.status, ); let dbType: string; diff --git a/packages/api/src/@core/s3/s3.service.ts b/packages/api/src/@core/s3/s3.service.ts index 333bbc350..3aacbd2d5 100644 --- a/packages/api/src/@core/s3/s3.service.ts +++ b/packages/api/src/@core/s3/s3.service.ts @@ -1,8 +1,10 @@ import { EnvironmentService } from '@@core/@core-services/environment/environment.service'; import { FileInfo } from '@@core/rag/types'; import { + CreateBucketCommand, GetObjectCommand, - S3Client + HeadBucketCommand, + S3Client, } from '@aws-sdk/client-s3'; import { ServiceRegistry } from '@filestorage/file/services/registry.service'; import { IFileService } from '@filestorage/file/types'; @@ -14,7 +16,10 @@ import { BUCKET_NAME } from './constants'; export class S3Service { private s3: S3Client; - constructor(private envService: EnvironmentService, private fileServiceRegistry: ServiceRegistry) { + constructor( + private envService: EnvironmentService, + private fileServiceRegistry: ServiceRegistry, + ) { // const creds = this.envService.getAwsCredentials(); const creds = this.envService.getMinioCredentials(); this.s3 = new S3Client({ @@ -23,30 +28,48 @@ export class S3Service { forcePathStyle: true, //region: creds.region, credentials: { - accessKeyId: creds.accessKeyId, - secretAccessKey: creds.secretAccessKey, + accessKeyId: creds.accessKeyId || 'myaccesskey13', + secretAccessKey: creds.secretAccessKey || 'mysecretkey12', }, }); } + async ensureBucketExists(s3Client: S3Client, bucketName: string) { + try { + await s3Client.send(new HeadBucketCommand({ Bucket: bucketName })); + } catch (error) { + if (error.name === 'NotFound') { + try { + await s3Client.send(new CreateBucketCommand({ Bucket: bucketName })); + console.log(`Bucket ${bucketName} created successfully`); + } catch (createError) { + console.error(`Error creating bucket ${bucketName}:`, createError); + throw createError; + } + } else { + // Some other error occurred + console.error(`Error checking bucket ${bucketName}:`, error); + throw error; + } + } + } + async uploadFilesFromUrls( urlsWithKeys: FileInfo[], - linkedUserId: string + linkedUserId: string, ): Promise { + await this.ensureBucketExists(this.s3, BUCKET_NAME); const batchSize = 10; for (let i = 0; i < urlsWithKeys.length; i += batchSize) { const batch = urlsWithKeys.slice(i, i + batchSize); await Promise.all( batch.map(async ({ id, url, s3Key, provider }) => { try { - const service: IFileService = this.fileServiceRegistry.getService(provider.toLowerCase().trim()); - if (!service) return; - await service.streamFileToS3( - id, - linkedUserId, - this.s3, - s3Key, + const service: IFileService = this.fileServiceRegistry.getService( + provider.toLowerCase().trim(), ); + if (!service) return; + await service.streamFileToS3(id, linkedUserId, this.s3, s3Key); console.log(`Successfully uploaded ${s3Key} from ${provider}`); } catch (error) { console.error( diff --git a/packages/api/src/@core/sync/sync.service.ts b/packages/api/src/@core/sync/sync.service.ts index d0e9c2acb..2233df697 100644 --- a/packages/api/src/@core/sync/sync.service.ts +++ b/packages/api/src/@core/sync/sync.service.ts @@ -63,82 +63,91 @@ export class CoreSyncService { for (const [vertical, interval] of Object.entries( syncIntervals, )) { - const now = new Date(); - const lastSyncEvent = await this.prisma.events.findFirst({ - where: { - id_project: project.id_project, - type: `${vertical}.batchSyncStart`, - }, - orderBy: { - timestamp: 'desc', - }, - }); - - const lastSyncTime = lastSyncEvent - ? lastSyncEvent.timestamp - : new Date(0); - - const secondsSinceLastSync = - Number(now.getTime() - lastSyncTime.getTime()) / 1000; - - if (interval && secondsSinceLastSync >= interval) { - await this.prisma.events.create({ - data: { + if (interval && interval > 0) { + const now = new Date(); + const lastSyncEvent = await this.prisma.events.findFirst({ + where: { id_project: project.id_project, - id_event: uuidv4(), - status: 'success', type: `${vertical}.batchSyncStart`, - method: 'GET', - url: '', - provider: '', - direction: '0', - timestamp: new Date(), + }, + orderBy: { + timestamp: 'desc', }, }); - const commonObjects = - getCommonObjectsForVertical(vertical); - for (const commonObject of commonObjects) { - try { - const service = this.registry.getService( - vertical, - commonObject, - ); - if (service) { - try { - const cronExpression = this.convertIntervalToCron( - Number(interval), - ); - await this.bullQueueService.queueSyncJob( - `${vertical}-sync-${commonObject}s`, - { - projectId: project.id_project, - vertical, - commonObject, - }, - cronExpression, - ); - this.logger.log( - `Synced ${vertical}.${commonObject} for project ${project.id_project}`, - ); - } catch (error) { - this.logger.error( - `Error syncing ${vertical}.${commonObject} for project ${project.id_project}: ${error.message}`, - error, + const lastSyncTime = lastSyncEvent + ? lastSyncEvent.timestamp + : new Date(0); + + const secondsSinceLastSync = + Number(now.getTime() - lastSyncTime.getTime()) / 1000; + + if (interval && secondsSinceLastSync >= interval) { + await this.prisma.events.create({ + data: { + id_project: project.id_project, + id_event: uuidv4(), + status: 'success', + type: `${vertical}.batchSyncStart`, + method: 'GET', + url: '', + provider: '', + direction: '0', + timestamp: new Date(), + }, + }); + const commonObjects = + getCommonObjectsForVertical(vertical); + for (const commonObject of commonObjects) { + try { + const service = this.registry.getService( + vertical, + commonObject, + ); + if (service) { + try { + const cronExpression = + this.convertIntervalToCron(Number(interval)); + + await this.bullQueueService.queueSyncJob( + `${vertical}-sync-${commonObject}s`, + { + projectId: project.id_project, + vertical, + commonObject, + }, + cronExpression, + ); + this.logger.log( + `Synced ${vertical}.${commonObject} for project ${project.id_project}`, + ); + } catch (error) { + this.logger.error( + `Error syncing ${vertical}.${commonObject} for project ${project.id_project}: ${error.message}`, + error, + ); + } + } else { + this.logger.warn( + `No service found for ${vertical}.${commonObject}`, ); } - } else { - this.logger.warn( - `No service found for ${vertical}.${commonObject}`, + } catch (error) { + this.logger.error( + `Error processing ${vertical}.${commonObject} for project ${project.id_project}: ${error.message}`, + error, ); } - } catch (error) { - this.logger.error( - `Error processing ${vertical}.${commonObject} for project ${project.id_project}: ${error.message}`, - error, - ); } } + } else { + const commonObjects = + getCommonObjectsForVertical(vertical); + for (const commonObject of commonObjects) { + await this.bullQueueService.removeRepeatableJob( + `${vertical}-sync-${commonObject}s`, + ); + } } } }