-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
added :cargo-qdrant module to install/start/stop Qdrant Vector DB docker
containers
- Loading branch information
Showing
5 changed files
with
295 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
59 changes: 59 additions & 0 deletions
59
.../com/github/jlangch/venice/impl/docgen/cheatsheet/modules/ModuleCargoQdrantDBSection.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| /* __ __ _ | ||
| * \ \ / /__ _ __ (_) ___ ___ | ||
| * \ \/ / _ \ '_ \| |/ __/ _ \ | ||
| * \ / __/ | | | | (_| __/ | ||
| * \/ \___|_| |_|_|\___\___| | ||
| * | ||
| * | ||
| * Copyright 2017-2024 Venice | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package com.github.jlangch.venice.impl.docgen.cheatsheet.modules; | ||
|
|
||
| import com.github.jlangch.venice.impl.docgen.cheatsheet.DocItemBuilder; | ||
| import com.github.jlangch.venice.impl.docgen.cheatsheet.DocSection; | ||
| import com.github.jlangch.venice.impl.docgen.cheatsheet.ISectionBuilder; | ||
|
|
||
|
|
||
| public class ModuleCargoQdrantDBSection implements ISectionBuilder { | ||
|
|
||
| public ModuleCargoQdrantDBSection(final DocItemBuilder diBuilder) { | ||
| this.diBuilder = diBuilder; | ||
| } | ||
|
|
||
| @Override | ||
| public DocSection section() { | ||
| final DocSection section = new DocSection( | ||
| "Cargo Qdrant Vector DB", | ||
| "Qdrant Testcontainers", | ||
| "modules.cargo-qdrant"); | ||
|
|
||
| final DocSection all = new DocSection("(load-module :cargo-qdrant)", id()); | ||
| section.addSection(all); | ||
|
|
||
| final DocSection docker = new DocSection("Lifecycle", id()); | ||
| all.addSection(docker); | ||
| docker.addItem(diBuilder.getDocItem("cargo-qdrant/start", false)); | ||
| docker.addItem(diBuilder.getDocItem("cargo-qdrant/stop", false)); | ||
| docker.addItem(diBuilder.getDocItem("cargo-qdrant/running?", false)); | ||
|
|
||
| return section; | ||
| } | ||
|
|
||
| private String id() { | ||
| return diBuilder.id(); | ||
| } | ||
|
|
||
| private final DocItemBuilder diBuilder; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
221 changes: 221 additions & 0 deletions
221
src/main/resources/com/github/jlangch/venice/cargo-qdrant.venice
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,221 @@ | ||
| ;;;; __ __ _ | ||
| ;;;; \ \ / /__ _ __ (_) ___ ___ | ||
| ;;;; \ \/ / _ \ '_ \| |/ __/ _ \ | ||
| ;;;; \ / __/ | | | | (_| __/ | ||
| ;;;; \/ \___|_| |_|_|\___\___| | ||
| ;;;; | ||
| ;;;; | ||
| ;;;; Copyright 2017-2024 Venice | ||
| ;;;; | ||
| ;;;; Licensed under the Apache License, Version 2.0 (the "License"); | ||
| ;;;; you may not use this file except in compliance with the License. | ||
| ;;;; You may obtain a copy of the License at | ||
| ;;;; | ||
| ;;;; http://www.apache.org/licenses/LICENSE-2.0 | ||
| ;;;; | ||
| ;;;; Unless required by applicable law or agreed to in writing, software | ||
| ;;;; distributed under the License is distributed on an "AS IS" BASIS, | ||
| ;;;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| ;;;; See the License for the specific language governing permissions and | ||
| ;;;; limitations under the License. | ||
|
|
||
| ;;;; Docker Qdrant cargo testcontainer | ||
|
|
||
| ;;;; This is just a configuration wrapper on top of the :cargo module to | ||
| ;;;; simplify using Qdrant testcontainers | ||
|
|
||
| (ns cargo-qdrant) | ||
|
|
||
| (load-module :docker) | ||
| (load-module :cargo) | ||
|
|
||
| (import :com.github.jlangch.venice.ShellException) | ||
|
|
||
| ;; Qdrant is now accessible through these APIs | ||
| ;; REST API: localhost:6333 | ||
| ;; Web UI: localhost:6333/dashboard | ||
| ;; GRPC API: localhost:6334 | ||
| (defonce internal-rest-port 6333) | ||
| (defonce internal-grpc-port 6334) | ||
|
|
||
|
|
||
| ;; --------------------------------------------------------------------------- | ||
| ;; Qdrant start & stop container | ||
| ;; --------------------------------------------------------------------------- | ||
|
|
||
| (defn | ||
| ^{ :arglists '( | ||
| "(cargo-qdrant/start cname version storage-dir)") | ||
| :doc """ | ||
| Starts a Qdrant container. | ||
|
|
||
| Qdrant is vector database often used for LLM embeddings. | ||
|
|
||
| Start rules: | ||
|
|
||
| * If a container with another version exists for the container | ||
| name remove the container and the image | ||
| * Pull the image if not yet locally available | ||
| * If the container already runs - use it | ||
| * If the container is available but does not run - start it `(docker/start ...)` | ||
| * If the container is not available - run it `(docker/run ...)` | ||
| * Finally check for a successful startup. The container log must | ||
| contain the string ".*is ready for business. Have fun.*" on the | ||
| last line. | ||
|
|
||
| Args: | ||
|
|
||
| | [![width: 20%]] | [![width: 85%]] | | ||
| | cname | A unique container name | | ||
| | version | The Qdrant version to use. E.g.: 1.8.3 | | ||
| | mapped-rest-port | The published (mapped) Qdrant REST port on the \ | ||
| host. Defaults to 6333 | | ||
| | mapped-grpc-port | The published (mapped) Qdrant GRPC port on the \ | ||
| host. Defaults to 6334 | | ||
| | storage-dir | Directory where Qdrant persists all the data. | | ||
| | config-file | An optional custom configuration yaml file | | ||
| | log | A log function, may be *nil*. \ | ||
| E.g: `(fn [s] (println "Qdrant:" s))`| | ||
| """ | ||
| :examples '( | ||
| """ | ||
| (do | ||
| (load-module :cargo-qdrant ['cargo-qdrant :as 'cq]) | ||
|
|
||
| ;; Run a Qdrant container labeled as "qdrant" | ||
| (cq/start "qdrant" "1.8.3" "./qdrant-storage")) | ||
| """) | ||
| :see-also '( | ||
| "cargo-qdrant/stop" | ||
| "cargo-qdrant/running?") } | ||
|
|
||
| start | ||
|
|
||
| ([cname version storage-dir config-file log] | ||
| (start cname | ||
| version | ||
| internal-rest-port ;; default REST port | ||
| internal-grpc-port ;; default GRPC port | ||
| storage-dir | ||
| config-file | ||
| log)) | ||
|
|
||
| ([cname version mapped-rest-port mapped-grpc-port storage-dir config-file log] | ||
| { :pre [(string? cname) | ||
| (string? version) | ||
| (or (int? mapped-rest-port) (long? mapped-rest-port)) | ||
| (< 0 mapped-rest-port 65536) | ||
| (or (int? mapped-grpc-port) (long? mapped-grpc-port)) | ||
| (< 0 mapped-grpc-port 65536) | ||
| (or (string? storage-dir) | ||
| (io/file? storage-dir)) | ||
| (or (nil? config-file) | ||
| (string? config-file) | ||
| (io/file? config-file))] } | ||
| (when-not (io/exists-dir? storage-dir) | ||
| (throw :VncException | ||
| "The Qdrant storage directory \"~{storage-dir}\" does not exist!")) | ||
| (when-not (and (some? config-file) (io/exists-file? config-file)) | ||
| (throw :VncException | ||
| "The Qdrant config file \"~{config-file}\" does not exist!")) | ||
| (let [publish ["~{mapped-rest-port}:~{internal-rest-port}" | ||
| "~{mapped-grpc-port}:~{internal-grpc-port}"] | ||
| envs [] | ||
| volumes (if (nil? config-file) | ||
| ["~{storage-dir}:/qdrant/storage"] | ||
| ["~{storage-dir}:/qdrant/storage" | ||
| "~{config-file}:/qdrant/config/production.yaml"]) | ||
| args [] | ||
| log (or log default-log) | ||
| wait-after-start-secs 3 | ||
| ready-check-max-secs 30] | ||
| (log "starting ~{version} as \"~{cname}\" @ port ~{publish} ...") | ||
| (cargo/start cname "qdrant/qdrant" version | ||
| publish | ||
| envs | ||
| volumes | ||
| args | ||
| qdrant-ready? | ||
| log | ||
| wait-after-start-secs | ||
| ready-check-max-secs)))) | ||
|
|
||
|
|
||
| (defn | ||
| ^{ :arglists '("(cargo-qdrant/stop cname log)") | ||
| :doc """ | ||
| Stops an Qdrant container | ||
|
|
||
| Args: | ||
|
|
||
| | cname | A unique container name | | ||
| """ | ||
| :examples '( | ||
| """ | ||
| (do | ||
| (load-module :cargo-qdrant ['cargo-qdrant :as 'cq]) | ||
|
|
||
| ;; Stop the Qdrant container labeled as "qdrant" | ||
| (cq/stop "qdrant")) | ||
| """) | ||
| :see-also '( | ||
| "cargo-qdrant/start" | ||
| "cargo-qdrant/running?") } | ||
|
|
||
| stop [cname log] | ||
|
|
||
| { :pre [(string? cname)] } | ||
|
|
||
| (let [log (or log default-log)] | ||
| (cargo/stop cname log))) | ||
|
|
||
|
|
||
| (defn | ||
| ^{ :arglists '("(cargo-qdrant/running? cname)") | ||
| :doc """ | ||
| Returns true if a container with the specified name is running. | ||
|
|
||
| Args: | ||
|
|
||
| | cname | A unique container name | | ||
| """ | ||
| :examples '( | ||
| """ | ||
| ;; Test if Qdrant container is running | ||
| (do | ||
| (load-module :cargo-qdrant ['cargo-qdrant :as 'cq]) | ||
| (cq/running? "qdrant")) | ||
| """) | ||
| :see-also '( | ||
| "cargo-qdrant/start", | ||
| "cargo-qdrant/stop") } | ||
|
|
||
| running? [cname] | ||
|
|
||
| { :pre [(string? cname)] } | ||
|
|
||
| (docker/container-running-with-name? cname)) | ||
|
|
||
|
|
||
|
|
||
| ;; --------------------------------------------------------------------------- | ||
| ;; utils | ||
| ;; --------------------------------------------------------------------------- | ||
|
|
||
| (defn default-log [s] | ||
| (println "Qdrant:" s)) | ||
|
|
||
| (defn qdrant-ready? [cname] | ||
| (-> (docker/container-logs cname :tail 1) | ||
| (str/trim) | ||
| (match? #".*is ready for business. Have fun.*"))) | ||
|
|
||
| (defn dir-empty? [dir] | ||
| (zero? (count (io/list-files dir)))) | ||
|
|
||
| (defn dir-not-empty? [dir] | ||
| (pos? (count (io/list-files dir)))) | ||
|
|
||
|
|
||
| (defn admin-url [port] | ||
| "http://localhost:6333/dashboard") |