Merge pull request #4750 from alkem-io/develop

Release: Async VCs

.build/ory/oathkeeper/access-rules.yml

@@ -23,6 +23,30 @@
               - unauthorized
               - forbidden
 
+- id: 'alkemio:graphiql:protected'
+  upstream:
+    preserve_host: true
+    url: 'http://host.docker.internal:4000'
+  match:
+    url: 'http://localhost:3000/graphiql'
+    methods:
+      - POST
+      - GET
+  authenticators:
+    - handler: cookie_session
+    - handler: noop
+  authorizer:
+    handler: allow
+  mutators:
+    - handler: id_token
+  errors:
+    - handler: redirect
+      config:
+        to: http://localhost:3000/login
+        when:
+          - error:
+              - unauthorized
+              - forbidden
 
 - id: 'alkemio:api:private:rest:storage'
   upstream:

.build/traefik/http.yml

@@ -115,6 +115,12 @@ http:
       entryPoints:
         - 'web'
 
+    graphiql:
+      rule: 'PathPrefix(`/graphiql`)'
+      service: 'oathkeeper-proxy'
+      entryPoints:
+        - 'web'
+
     api-public-graphql:
       rule: 'PathPrefix(`/api/public/graphql`)'
       service: 'alkemio-server'

.coderabbit.yaml

@@ -11,7 +11,26 @@ reviews:
     - "!**/dist/**"          # Exclude build files
   path_instructions:
     - path: "src/**/*.{ts,js}"
-      instructions: "Review the TypeScript/JavaScript code for NestJS best practices, dependency injection, module structure, and potential bugs. Ensure that the code adheres to TypeScript's typing system and modern standards."
+      instructions: |
+        Review the TypeScript/JavaScript code for NestJS best practices, dependency injection, module structure, and potential bugs.
+
+        **Context Files (Do Not Review):**
+        - `docs/Design.md` - Design overview of the project
+        - `docs/Pagination.md` - Pagination design overview
+        - `docs/Developing.md` - Development setup overview
+        - `docs/graphql-typeorm-usage.md` - overview of GraphQL and TypeORM usage and how they are used together with NestJS in the project
+        - `docs/database-definitions.md` - guidelines for creating TypeORM entity defnitions
+        - `src/core/error-handling/graphql.exception.filter.ts` - GraphQL error handling
+        - `src/core/error-handling/http.exception.filter.ts` - HTTP error handling
+        - `src/core/error-handling/rest.error.response.ts` - REST error response
+        - `src/core/error-handling/unhandled.exception.filter.ts` - Global exception handler
+
+        **Guidelines:**
+        - Our project uses global exception handlers (`UnhandledExceptionFilter`), so avoid suggesting additional `try/catch` blocks unless handling specific cases.
+        - Use NestJS latest documentation from `https://docs.nestjs.com/` for reference on NestJS best practices.
+        - Use TypeORM latest documentation from `https://typeorm.io/` for reference on TypeORM best practices.
+        - Refer to the design overview in the context files for better understanding.
+
     - path: "src/**/*.spec.ts"
       instructions: "Review the unit tests, ensuring proper NestJS testing techniques (using TestingModule, mocks, etc.). Check for completeness and coverage."
     - path: "manifests/**/*.{yaml,yml}"

.env.docker

@@ -30,6 +30,9 @@ RABBITMQ_INGEST_SPACE_QUEUE=virtual-contributor-ingest-space
 RABBITMQ_INGEST_SPACE_RESULT_QUEUE=virtual-contributor-ingest-space-result
 RABBITMQ_EVENT_BUS_EXCHANGE=event-bus
 
+RABBITMQ_RESULT_QUEUE=virtual-contributor-invoke-engine-result
+RABBITMQ_RESULT_ROUTING_KEY=invoke-engine-result
+
 ALKEMIO_SERVER_ENDPOINT=http://host.docker.internal:4000/graphql
 KRATOS_API_PUBLIC_ENDPOINT=http://kratos:4433/
 [email protected]
@@ -46,16 +49,21 @@ AI_LOCAL_PATH=/home/alkemio/data
 OPENAI_API_VERSION=2023-05-15
 AZURE_OPENAI_ENDPOINT=https://alkemio-gpt.openai.azure.com
 AZURE_OPENAI_API_KEY=your-openai-key
+AZURE_MISTRAL_ENDPOINT=https://Mistral-small-alkemio-serverless.swedencentral.inference.ai.azure.com
+AZURE_MISTRAL_API_KEY=mistral-api-key
 LLM_DEPLOYMENT_NAME=deploy-gpt-35-turbo
 EMBEDDINGS_DEPLOYMENT_NAME=embedding
 AI_MODEL_TEMPERATURE=0.3
 AI_SOURCE_WEBSITE=https://www.alkemio.org
 AI_SOURCE_WEBSITE2=https://welcome.alkem.io
+
 AI_WEBSITE_REPO=github.com/alkem-io/website.git
 AI_WEBSITE_REPO2=github.com/alkem-io/welcome-site.git
+
 AI_GITHUB_USER=your-github-user
 AI_GITHUB_PAT=your-github-pat
-LANGCHAIN_TRACING_V2=true
+
+LANGCHAIN_TRACING_V2=false
 LANGCHAIN_ENDPOINT=https://api.smith.langchain.com
 LANGCHAIN_API_KEY=your-langchain-key
 LANGCHAIN_PROJECT=guidance-engine

package.json

@@ -1,6 +1,6 @@
 {
   "name": "alkemio-server",
-  "version": "0.96.3",
+  "version": "0.97.0",
   "description": "Alkemio server, responsible for managing the shared Alkemio platform",
   "author": "Alkemio Foundation",
   "private": false,
@@ -90,6 +90,8 @@
     "dataloader": "^2.2.2",
     "graphql": "^16.9.0",
     "graphql-amqp-subscriptions": "^2.0.0",
+    "graphql-helix": "^1.13.0",
+    "graphql-http": "^1.22.3",
     "graphql-subscriptions": "^2.0.0",
     "graphql-type-json": "^0.3.2",
     "graphql-upload": "^13.0.0",

quickstart-services-ai.yml

@@ -323,7 +323,7 @@ services:
   virtual_contributor_engine_guidance:
     container_name: alkemio_dev_virtual_contributor_engine_guidance
     hostname: virtual-contributor-engine-guidance
-    image: alkemio/virtual-contributor-engine-guidance:v0.7.1
+    image: alkemio/virtual-contributor-engine-guidance:v0.8.2
     platform: linux/x86_64
     restart: always
     volumes:
@@ -339,6 +339,11 @@ services:
       - RABBITMQ_USER
       - RABBITMQ_PASSWORD
       - RABBITMQ_QUEUE=virtual-contributor-engine-guidance
+      - RABBITMQ_EVENT_BUS_EXCHANGE
+      - RABBITMQ_RESULT_QUEUE
+      - RABBITMQ_RESULT_ROUTING_KEY
+      - AZURE_MISTRAL_ENDPOINT
+      - AZURE_MISTRAL_API_KEY
       - ENVIRONMENT=dev
       - EMBEDDINGS_DEPLOYMENT_NAME
       - AZURE_OPENAI_API_KEY
@@ -396,7 +401,7 @@ services:
       - 'host.docker.internal:host-gateway'
     container_name: alkemio_dev_virtual_contributor_engine_expert
     hostname: virtual-contributor-engine-expert
-    image: alkemio/virtual-contributor-engine-expert:v0.5.0
+    image: alkemio/virtual-contributor-engine-expert:v0.10.0
     platform: linux/x86_64
     restart: always
     volumes:
@@ -412,6 +417,11 @@ services:
       - RABBITMQ_PASSWORD
       - RABBITMQ_PORT
       - RABBITMQ_QUEUE=virtual-contributor-engine-expert
+      - RABBITMQ_EVENT_BUS_EXCHANGE
+      - RABBITMQ_RESULT_QUEUE
+      - RABBITMQ_RESULT_ROUTING_KEY
+      - AZURE_MISTRAL_ENDPOINT
+      - AZURE_MISTRAL_API_KEY
       - API_ENDPOINT_PRIVATE_GRAPHQL=http://host.docker.internal:3000/api/private/non-interactive/graphql
       - AUTH_ORY_KRATOS_PUBLIC_BASE_URL=http://host.docker.internal:3000/ory/kratos/public
       - AUTH_ADMIN_EMAIL

src/common/enums/ai.persona.invocation.operation.ts

@@ -0,0 +1,4 @@
+export enum InvocationOperation {
+  QUERY = 'query',
+  INGEST = 'ingest',
+}

src/common/enums/room.type.ts

@@ -5,4 +5,5 @@ export enum RoomType {
   DISCUSSION_FORUM = 'discussion_forum',
   UPDATES = 'updates',
   CALLOUT = 'callout',
+  GUIDANCE = 'guidance',
 }

src/common/enums/template.type.ts

@@ -5,7 +5,6 @@ export enum TemplateType {
   POST = 'post',
   WHITEBOARD = 'whiteboard',
   COMMUNITY_GUIDELINES = 'community-guidelines',
-  INNOVATION_FLOW = 'innovation-flow',
   COLLABORATION = 'collaboration',
 }
 

src/core/bootstrap/bootstrap.module.ts

@@ -22,10 +22,12 @@ import { TemplateDefaultModule } from '@domain/template/template-default/templat
 import { LicenseModule } from '@domain/common/license/license.module';
 import { LicensePlanModule } from '@platform/license-plan/license.plan.module';
 import { LicensingFrameworkModule } from '@platform/licensing-framework/licensing.framework.module';
+import { AiPersonaServiceModule } from '@services/ai-server/ai-persona-service/ai.persona.service.module';
 
 @Module({
   imports: [
     AiServerModule,
+    AiPersonaServiceModule,
     AgentModule,
     AuthorizationPolicyModule,
     LicenseModule,

src/core/bootstrap/bootstrap.service.ts

@@ -59,6 +59,10 @@ import { LicenseService } from '@domain/common/license/license.service';
 import { AccountLicenseService } from '@domain/space/account/account.service.license';
 import { LicensePlanService } from '@platform/license-plan/license.plan.service';
 import { LicensingFrameworkService } from '@platform/licensing-framework/licensing.framework.service';
+import { AiPersonaServiceService } from '@services/ai-server/ai-persona-service/ai.persona.service.service';
+import { AiPersonaEngine } from '@common/enums/ai.persona.engine';
+import { AiPersonaBodyOfKnowledgeType } from '@common/enums/ai.persona.body.of.knowledge.type';
+import { AiPersonaDataAccessMode } from '@common/enums/ai.persona.data.access.mode';
 
 @Injectable()
 export class BootstrapService {
@@ -82,6 +86,7 @@ export class BootstrapService {
     @Inject(WINSTON_MODULE_NEST_PROVIDER)
     private readonly logger: LoggerService,
     private aiServer: AiServerService,
+    private aiPersonaServiceService: AiPersonaServiceService,
     private aiServerAuthorizationService: AiServerAuthorizationService,
     private templatesManagerService: TemplatesManagerService,
     private templatesSetService: TemplatesSetService,
@@ -113,6 +118,7 @@ export class BootstrapService {
       await this.ensurePlatformTemplatesArePresent();
       await this.ensureOrganizationSingleton();
       await this.ensureSpaceSingleton();
+      await this.ensureGuidanceChat();
       await this.ensureSsiPopulated();
       // reset auth as last in the actions
       // await this.ensureSpaceNamesInElastic();
@@ -524,4 +530,44 @@ export class BootstrapService {
       return this.spaceService.getSpaceOrFail(space.id);
     }
   }
+
+  private async ensureGuidanceChat() {
+    const platform = await this.platformService.getPlatformOrFail({
+      relations: { guidanceVirtualContributor: true },
+    });
+    if (!platform.guidanceVirtualContributor?.id) {
+      const aiPersonaService =
+        await this.aiPersonaServiceService.createAiPersonaService({
+          bodyOfKnowledgeID: '',
+          bodyOfKnowledgeType: AiPersonaBodyOfKnowledgeType.NONE,
+          engine: AiPersonaEngine.GUIDANCE,
+          dataAccessMode: AiPersonaDataAccessMode.NONE,
+          prompt: [],
+          externalConfig: undefined,
+        });
+
+      // Get admin account:
+      const hostOrganization =
+        await this.organizationService.getOrganizationOrFail(
+          DEFAULT_HOST_ORG_NAMEID
+        );
+      const account =
+        await this.organizationService.getAccount(hostOrganization);
+
+      // Create the VC
+      const vc = await this.accountService.createVirtualContributorOnAccount({
+        accountID: account.id,
+        aiPersona: {
+          aiPersonaServiceID: aiPersonaService.id,
+        },
+        profileData: {
+          displayName: 'Guidance',
+          description: 'Guidance Virtual Contributor',
+        },
+      });
+
+      platform.guidanceVirtualContributor = vc;
+      await this.platformService.savePlatform(platform);
+    }
+  }
 }

src/core/microservices/client.proxy.factory.ts

@@ -6,12 +6,6 @@ import { AlkemioConfig } from '@src/types';
 
 const QUEUE_CONTEXT_MAP: { [key in MessagingQueue]?: LogContext } = {
   [MessagingQueue.AUTH_RESET]: LogContext.AUTH,
-  [MessagingQueue.VIRTUAL_CONTRIBUTOR_ENGINE_OPENAI_ASSISTANT]:
-    LogContext.VIRTUAL_CONTRIBUTOR_ENGINE_OPENAI_ASSISTANT,
-  [MessagingQueue.VIRTUAL_CONTRIBUTOR_ENGINE_GENERIC]:
-    LogContext.VIRTUAL_CONTRIBUTOR_ENGINE_GENERIC,
-  [MessagingQueue.VIRTUAL_CONTRIBUTOR_ENGINE_EXPERT]:
-    LogContext.VIRTUAL_CONTRIBUTOR_ENGINE_EXPERT,
 };
 
 export const clientProxyFactory = (queue: MessagingQueue, durable = true) => {

src/core/microservices/microservices.module.ts

@@ -99,7 +99,7 @@ const excalidrawPubSubFactoryProvider = subscriptionFactoryProvider(
       provide: VIRTUAL_CONTRIBUTOR_ENGINE_GUIDANCE,
       useFactory: clientProxyFactory(
         MessagingQueue.VIRTUAL_CONTRIBUTOR_ENGINE_GUIDANCE,
-        false
+        true
       ),
 
       inject: [WINSTON_MODULE_NEST_PROVIDER, ConfigService],

src/domain/common/profile/profile.service.ts

@@ -273,6 +273,24 @@ export class ProfileService {
     return await this.referenceService.save(newReference);
   }
 
+  async deleteAllReferencesFromProfile(profileId: string): Promise<void> {
+    const profile = await this.getProfileOrFail(profileId, {
+      relations: { references: true },
+    });
+
+    if (!profile.references)
+      throw new EntityNotInitializedException(
+        'References not defined',
+        LogContext.COMMUNITY
+      );
+
+    for (const reference of profile.references) {
+      await this.referenceService.deleteReference({
+        ID: reference.id,
+      });
+    }
+  }
+
   async getProfileOrFail(
     profileID: string,
     options?: FindOneOptions<Profile>