Merge pull request #291 from Anush008/feat-287-sync-with-qdrant-bun-ts

feat: sync-with-qdrant template - Bun

bun/sync-with-qdrant/.gitignore

@@ -0,0 +1,145 @@
+### Node ###
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+.pnpm-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# Snowpack dependency directory (https://snowpack.dev/)
+web_modules/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Optional stylelint cache
+.stylelintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+out
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# vuepress v2.x temp and cache directory
+.temp
+
+# Docusaurus cache and generated files
+.docusaurus
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz
+.pnp.*
+
+### Node Patch ###
+# Serverless Webpack directories
+.webpack/
+
+# Optional stylelint cache
+
+# SvelteKit build / generate output
+.svelte-kit
+
+# End of https://www.toptal.com/developers/gitignore/api/node
+
+# OS
+## Mac
+.DS_Store

bun/sync-with-qdrant/README.md

@@ -0,0 +1,100 @@
+# ⚡ Bun Sync with Qdrant Function
+
+Syncs documents in an Appwrite database collection to a [Qdrant](https://qdrant.tech/) collection.
+
+## 🧰 Usage
+
+### GET /
+
+Returns an HTML page where a search can be performed to test the sync of the documents.
+
+### POST /
+
+Triggers sync of the Appwrite database collection to Qdrant.
+
+## ⚙️ Configuration
+
+| Setting           | Value         |
+| ----------------- | ------------- |
+| Runtime           | Node (18.0)   |
+| Entrypoint        | `src/main.js` |
+| Build Commands    | `bun install` |
+| Permissions       | `any`         |
+| Timeout (Seconds) | 15            |
+
+## 🔒 Environment Variables
+
+### APPWRITE_API_KEY
+
+API Key to talk to Appwrite backend APIs.
+
+| Question      | Answer                                                                                             |
+| ------------- | -------------------------------------------------------------------------------------------------- |
+| Required      | Yes                                                                                                |
+| Sample Value  | `d1efb...aec35`                                                                                    |
+| Documentation | [Appwrite: Getting Started for Server](https://appwrite.io/docs/getting-started-for-server#apiKey) |
+
+### APPWRITE_DATABASE_ID
+
+The ID of the Appwrite database that contains the collection to sync.
+
+| Question      | Answer                                                    |
+| ------------- | --------------------------------------------------------- |
+| Required      | Yes                                                       |
+| Sample Value  | `612a3...5b6c9`                                           |
+| Documentation | [Appwrite: Databases](https://appwrite.io/docs/databases) |
+
+### APPWRITE_COLLECTION_ID
+
+The ID of the collection in the Appwrite database to sync.
+
+| Question      | Answer                                                                 |
+| ------------- | ---------------------------------------------------------------------- |
+| Required      | Yes                                                                    |
+| Sample Value  | `7c3e8...2a9f1`                                                        |
+| Documentation | [Appwrite: Collections](https://appwrite.io/docs/databases#collection) |
+
+### APPWRITE_ENDPOINT
+
+The URL endpoint of the Appwrite server. If not provided, it defaults to the Appwrite Cloud server: `https://cloud.appwrite.io/v1`.
+
+| Question     | Answer                         |
+| ------------ | ------------------------------ |
+| Required     | No                             |
+| Sample Value | `https://cloud.appwrite.io/v1` |
+
+### QDRANT_URL
+
+The URL of the Qdrant server.
+
+| Question      | Answer                                                                                             |
+| ------------- | -------------------------------------------------------------------------------------------------- |
+| Required      | Yes                                                                                                |
+| Documentation | [Qdrant Cloud](https://qdrant.tech/documentation/cloud/#getting-started-with-qdrant-managed-cloud) |
+
+### QDRANT_API_KEY
+
+The API key to authenticate requests to Qdrant cloud.
+
+| Question      | Answer                                                                     |
+| ------------- | -------------------------------------------------------------------------- |
+| Required      | Yes                                                                        |
+| Documentation | [Qdrant: API Keys](https://qdrant.tech/documentation/cloud/authentication) |
+
+### QDRANT_COLLECTION_NAME
+
+The name of the Qdrant collection to sync the documents to.
+
+| Question      | Answer                                                                 |
+| ------------- | ---------------------------------------------------------------------- |
+| Required      | Yes                                                                    |
+| Documentation | [Collections](https://qdrant.tech/documentation/concepts/collections/) |
+
+### OPENAI_API_KEY
+
+API Key for OpenAI API. Used to generate the embeddings for the documents.
+
+| Question      | Answer                                                   |
+| ------------- | -------------------------------------------------------- |
+| Required      | Yes                                                      |
+| Documentation | [OpenAI: API Keys](https://platform.openai.com/api-keys) |

bun/sync-with-qdrant/env.d.ts

@@ -0,0 +1,15 @@
+declare module "bun" {
+  interface Env {
+    APPWRITE_ENDPOINT?: string;
+    APPWRITE_API_KEY: string;
+    APPWRITE_FUNCTION_PROJECT_ID: string;
+    APPWRITE_DATABASE_ID: string;
+    APPWRITE_COLLECTION_ID: string;
+    MEILISEARCH_ENDPOINT: string;
+    MEILISEARCH_INDEX_NAME: string;
+    MEILISEARCH_ADMIN_API_KEY: string;
+    MEILISEARCH_SEARCH_API_KEY: string;
+  }
+}
+
+export {};

bun/sync-with-qdrant/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "sync-with-qdrant",
+  "version": "1.0.0",
+  "description": "",
+  "main": "src/main.ts",
+  "type": "module",
+  "scripts": {
+    "format": "prettier --write ."
+  },
+  "dependencies": {
+    "@qdrant/js-client-rest": "^1.9.0",
+    "bun-types": "^1.0.7",
+    "node-appwrite": "^11.0.0",
+    "openai": "^4.47.1",
+    "path": "^0.12.7",
+    "url": "^0.11.3"
+  },
+  "devDependencies": {
+    "prettier": "^3.0.0"
+  }
+}

bun/sync-with-qdrant/src/appwrite.ts

@@ -0,0 +1,55 @@
+import { Client, Databases, Query } from 'node-appwrite';
+
+class AppwriteService {
+  databases: Databases;
+
+  constructor() {
+    const client = new Client();
+    client
+      .setEndpoint(
+        process.env.APPWRITE_ENDPOINT ?? 'https://cloud.appwrite.io/v1'
+      )
+      .setProject(process.env.APPWRITE_FUNCTION_PROJECT_ID)
+      .setKey(process.env.APPWRITE_API_KEY);
+
+    this.databases = new Databases(client);
+  }
+
+  /**
+   * Retrieves the entire collection of documents from an Appwrite database.
+   *
+   * @param {string} databaseId
+   * @param {string} collectionId
+   * @returns {Promise<import('node-appwrite').Models.Document[]>}
+   */
+  async getAllDocuments(databaseId: string, collectionId: string) {
+    const cumulative = [];
+
+    let cursor = null;
+    do {
+      const queries = [Query.limit(100)];
+
+      if (cursor) {
+        queries.push(Query.cursorAfter(cursor));
+      }
+
+      const { documents } = await this.databases.listDocuments(
+        databaseId,
+        collectionId,
+        queries
+      );
+
+      if (documents.length === 0) {
+        break;
+      }
+
+      cursor = documents[documents.length - 1].$id;
+
+      cumulative.push(...documents);
+    } while (cursor);
+
+    return cumulative;
+  }
+}
+
+export default AppwriteService;