Release/v0.25.0

.github/actions/init-environment/action.yml

@@ -35,3 +35,7 @@ runs:
           source .venv/bin/activate
           echo PATH=$PATH >> $GITHUB_ENV
         shell: bash
+
+      - name: Install playwright
+        run: playwright install --with-deps
+        shell: bash

.github/workflows/integration-tests.yml → .github/workflows/docs-integration-tests.yml

@@ -1,9 +1,11 @@
-name: Integration Tests
-
-on: workflow_dispatch
+name: Docs Integration Tests
 
+on:
+  pull_request_review:
+    types: [submitted]
 jobs:
   test:
+    if: github.event.review.state == 'APPROVED'
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -38,6 +40,15 @@ jobs:
       MONGODB_DATABASE_NAME: ${{ secrets.INTEG_MONGODB_DATABASE_NAME }}
       MONGODB_COLLECTION_NAME: ${{ secrets.INTEG_MONGODB_COLLECTION_NAME }}
       MONGODB_PASSWORD: ${{ secrets.INTEG_MONGODB_PASSWORD }}
+      MONGODB_INDEX_NAME: ${{ secrets.INTEG_MONGODB_INDEX_NAME }}
+      MONGODB_VECTOR_PATH: ${{ secrets.INTEG_MONGODB_VECTOR_PATH }}
+      AZURE_MONGODB_HOST: ${{ secrets.INTEG_AZURE_MONGODB_HOST }}
+      AZURE_MONGODB_USERNAME: ${{ secrets.INTEG_AZURE_MONGODB_USERNAME }}
+      AZURE_MONGODB_PASSWORD: ${{ secrets.INTEG_AZURE_MONGODB_PASSWORD }}
+      AZURE_MONGODB_DATABASE_NAME: ${{ secrets.INTEG_AZURE_MONGODB_DATABASE_NAME }}
+      AZURE_MONGODB_COLLECTION_NAME: ${{ secrets.INTEG_AZURE_MONGODB_COLLECTION_NAME }}
+      AZURE_MONGODB_INDEX_NAME: ${{ secrets.INTEG_AZURE_MONGODB_INDEX_NAME }}
+      AZURE_MONGODB_VECTOR_PATH: ${{ secrets.INTEG_AZURE_MONGODB_VECTOR_PATH }}
       AMAZON_OPENSEARCH_HOST: ${{ secrets.INTEG_AMAZON_OPENSEARCH_HOST }}
       AMAZON_OPENSEARCH_INDEX_NAME: ${{ secrets.INTEG_AMAZON_OPENSEARCH_INDEX_NAME }}
       GOOGLE_API_KEY: ${{ secrets.INTEG_GOOGLE_API_KEY }}
@@ -52,19 +63,43 @@ jobs:
       GOOGLE_AUTH_URI: ${{ secrets.INTEG_GOOGLE_AUTH_URI }}
       GOOGLE_TOKEN_URI: ${{ secrets.INTEG_GOOGLE_TOKEN_URI }}
       GOOGLE_AUTH_PROVIDER_X509_CERT_URL: ${{ secrets.INTEG_GOOGLE_AUTH_PROVIDER_X509_CERT_URL }}
+      GRIPTAPE_CLOUD_API_KEY: ${{ secrets.INTEG_GRIPTAPE_CLOUD_API_KEY }}
+      GRIPTAPE_CLOUD_STRUCTURE_ID: ${{ secrets.INTEG_GRIPTAPE_CLOUD_STRUCTURE_ID }}
+      GRIPTAPE_CLOUD_BASE_URL: ${{ secrets.INTEG_GRIPTAPE_CLOUD_BASE_URL }}
+      OPENWEATHER_API_KEY: ${{ secrets.INTEG_OPENWEATHER_API_KEY }}
       ANTHROPIC_API_KEY: ${{ secrets.INTEG_ANTHROPIC_API_KEY }}
       SAGEMAKER_LLAMA_ENDPOINT_NAME: ${{ secrets.INTEG_LLAMA_ENDPOINT_NAME }}
       SAGEMAKER_FALCON_ENDPOINT_NAME: ${{ secrets.INTEG_FALCON_ENDPOINT_NAME }}
-      HUGGINGFACE_ACCESS_TOKEN: ${{ secrets.INTEG_HUGGINGFACE_ACCESS_TOKEN }}
-      AZURE_OPENAI_API_BASE_1: ${{ secrets.INTEG_AZURE_OPENAI_API_BASE_1 }}
+      HUGGINGFACE_HUB_ACCESS_TOKEN: ${{ secrets.INTEG_HUGGINGFACE_HUB_ACCESS_TOKEN }}
+      AZURE_OPENAI_ENDPOINT_1: ${{ secrets.INTEG_AZURE_OPENAI_ENDPOINT_1 }}
       AZURE_OPENAI_API_KEY_1: ${{ secrets.INTEG_AZURE_OPENAI_API_KEY_1 }}
-      AZURE_OPENAI_API_BASE_2: ${{ secrets.INTEG_AZURE_OPENAI_API_BASE_2 }}
+      AZURE_OPENAI_ENDPOINT_2: ${{ secrets.INTEG_AZURE_OPENAI_ENDPOINT_2 }}
       AZURE_OPENAI_API_KEY_2: ${{ secrets.INTEG_AZURE_OPENAI_API_KEY_2 }}
-      AZURE_OPENAI_35_16k_DEPLOYMENT_ID: ${{ secrets.INTEG_OPENAI_35_16k_DEPLOYMENT_ID }}
+      AZURE_OPENAI_35_TURBO_16K_DEPLOYMENT_ID: ${{ secrets.INTEG_OPENAI_35_TURBO_16K_DEPLOYMENT_ID }}
       AZURE_OPENAI_DAVINCI_DEPLOYMENT_ID: ${{ secrets.INTEG_OPENAI_DAVINCI_DEPLOYMENT_ID }}
       AZURE_OPENAI_4_DEPLOYMENT_ID: ${{ secrets.INTEG_OPENAI_4_DEPLOYMENT_ID }}
-      AZURE_OPENAI_4_32k_DEPLOYMENT_ID: ${{ secrets.INTEG_OPENAI_4_32k_DEPLOYMENT_ID }}
+      AZURE_OPENAI_4_32K_DEPLOYMENT_ID: ${{ secrets.INTEG_OPENAI_4_32K_DEPLOYMENT_ID }}
+      AZURE_OPENAI_DALL_E_3_DEPLOYMENT_ID: ${{ secrets.INTEG_AZURE_OPENAI_DALL_E_3_DEPLOYMENT_ID }}
+      AZURE_OPENAI_DEV_2_API_BASE: ${{ secrets.INTEG_AZURE_OPENAI_DEV_2_API_BASE }}
+      AZURE_OPENAI_DEV_2_API_KEY: ${{ secrets.INTEG_AZURE_OPENAI_DEV_2_API_KEY }}
+      AZURE_OPENAI_API_BASE: ${{ secrets.INTEG_AZURE_OPENAI_API_BASE }}
       COHERE_API_KEY: ${{ secrets.INTEG_COHERE_API_KEY }}
+      GOOGLE_OWNER_EMAIL: ${{ secrets.INTEG_GOOGLE_OWNER_EMAIL }}
+      LEONARDO_API_KEY: ${{ secrets.INTEG_LEONARDO_API_KEY }}
+      LEONARDO_MODEL_ID: ${{ secrets.INTEG_LEONARDO_MODEL_ID }}
+      SAGEMAKER_TENSORFLOW_HUB_MODEL: ${{ secrets.INTEG_SAGEMAKER_TENSORFLOW_HUB_MODEL }}
+      SAGEMAKER_HUGGINGFACE_MODEL: ${{ secrets.INTEG_SAGEMAKER_HUGGINGFACE_MODEL }}
+      POSTGRES_USER: ${{ secrets.INTEG_POSTGRES_USER }}
+      POSTGRES_PASSWORD: ${{ secrets.INTEG_POSTGRES_PASSWORD }}
+      POSTGRES_DB: ${{ secrets.INTEG_POSTGRES_DB }}
+      POSTGRES_HOST: ${{ secrets.INTEG_POSTGRES_HOST }}
+      POSTGRES_PORT: ${{ secrets.INTEG_POSTGRES_PORT }}
+      VOYAGE_API_KEY: ${{ secrets.INTEG_VOYAGE_API_KEY }}
+      WEBHOOK_URL: ${{ secrets.INTEG_WEBHOOK_URL }}
+      AMAZON_SQS_QUEUE_URL: ${{ secrets.INTEG_AMAZON_SQS_QUEUE_URL }}
+      GT_CLOUD_STRUCTURE_RUN_ID: ${{ secrets.INTEG_GT_CLOUD_STRUCTURE_RUN_ID }}
+      AWS_IOT_CORE_ENDPOINT: ${{ secrets.INTEG_AWS_IOT_CORE_ENDPOINT }}
+      AWS_IOT_CORE_TOPIC: ${{ secrets.INTEG_AWS_IOT_CORE_TOPIC }}
     services:
       postgres:
         image: ankane/pgvector:v0.5.0
@@ -84,5 +119,21 @@ jobs:
         uses: actions/checkout@v3
       - name: Init environment
         uses: ./.github/actions/init-environment
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v44
+        with:
+          files: |
+             **.md
+      - name: List all changed files
+        env:
+          ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+        run: |
+          for file in ${ALL_CHANGED_FILES}; do
+            echo "$file was changed"
+          done
       - name: Run integration tests
-        run: pytest -vs tests/integration
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: pytest -n auto tests/integration/test_code_blocks.py
+        env:
+          DOCS_ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}

.github/workflows/pull-request-links.yml

@@ -0,0 +1,20 @@
+name: Read the Docs Preview
+on:
+  pull_request_target:
+    types:
+      - opened
+    paths:
+      - "docs/**"
+
+permissions:
+  pull-requests: write
+
+jobs:
+  pull-request-links:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: readthedocs/actions/preview@v1
+        with:
+          project-slug: ${{ secrets.READTHEDOCS_PROJECT_SLUG }} 
+          single-version: 'false'
+          project-language: ""

.gitignore

@@ -53,3 +53,7 @@ dist/**
 
 # asdf
 .tool-versions
+
+# mkdocs build output
+site
+reference

.readthedocs.yml

@@ -0,0 +1,16 @@
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+  jobs:
+    post_create_environment:
+      - pip install poetry
+      - poetry config virtualenvs.create false
+    post_install:
+      - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with docs
+
+mkdocs:
+  configuration: mkdocs.yml
+  fail_on_warning: true

CHANGELOG.md

@@ -6,6 +6,48 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## [0.25.0] - 2024-05-06
+
+### Added
+- `list_files_from_disk` activity to `FileManager` Tool.
+- Support for Drivers in `EventListener`.
+- `AmazonSqsEventListenerDriver` for sending events to an Amazon SQS queue.
+- `AwsIotCoreEventListenerDriver` for sending events to a topic on AWS IoT Core.
+- `GriptapeCloudEventListenerDriver` for sending events to Griptape Cloud.
+- `WebhookEventListenerDriver` for sending events to a webhook.
+- `BaseFileManagerDriver` to abstract file management operations.
+- `LocalFileManagerDriver` for managing files on the local file system.
+- Optional `BaseLoader.encoding` field.
+- `BlobLoader` for loading arbitrary binary data as a `BlobArtifact`.
+- `model` field to `StartPromptEvent` and `FinishPromptEvent`.
+- `input_task_input` and `input_task_output` fields to `StartStructureRunEvent`.
+- `output_task_input` and `output_task_output` fields to `FinishStructureRunEvent`.
+- `AmazonS3FileManagerDriver` for managing files on Amazon S3.
+- `MediaArtifact` as a base class for `ImageArtifact` and future media Artifacts.
+- Optional `exception` field to `ErrorArtifact`.
+- `StructureRunClient` for running other Structures via a Tool.
+- `StructureRunTask` for running Structures as a Task from within another Structure.
+- `GriptapeCloudStructureRunDriver` for running Structures in Griptape Cloud.
+- `LocalStructureRunDriver` for running Structures in the same run-time environment as the code that is running the Structure.
+
+### Changed
+- **BREAKING**: Secret fields (ex: api_key) removed from serialized Drivers.
+- **BREAKING**: Remove `FileLoader`.
+- **BREAKING**: `CsvLoader` no longer accepts `str` file paths as a source. It will now accept the content of the CSV file as a `str` or `bytes` object.
+- **BREAKING**: `PdfLoader` no longer accepts `str` file content, `Path` file paths or `IO` objects as sources. Instead, it will only accept the content of the PDF file as a `bytes` object.
+- **BREAKING**: `TextLoader` no longer accepts `Path` file paths as a source. It will now accept the content of the text file as a `str` or `bytes` object.
+- **BREAKING**: `FileManager.default_loader` is now `None` by default.
+- **BREAKING** Bumped `pinecone` from `^2` to `^3`.
+- **BREAKING**: Removed `workdir`, `loaders`, `default_loader`, and `save_file_encoding` fields from `FileManager` and added `file_manager_driver`.
+- **BREAKING**: Removed `mime_type` field from `ImageArtifact`. `mime_type` is now a property constructed using the Artifact type and `format` field.
+- Improved RAG performance in `VectorQueryEngine`.
+- Moved [Griptape Docs](https://github.com/griptape-ai/griptape-docs) to this repository.
+- Updated `EventListener.handler`'s behavior so that the return value will be passed to the `EventListenerDriver.try_publish_event_payload`'s `event_payload` parameter.
+
+### Fixed
+- Type hint for parameter `azure_ad_token_provider` on Azure OpenAI drivers to `Optional[Callable[[], str]]`.
+- Missing parameters `azure_ad_token` and `azure_ad_token_provider` on the default client for `AzureOpenAiCompletionPromptDriver`.
+
 ## [0.24.2] - 2024-04-04
 
 - Fixed FileManager.load_files_from_disk schema.
@@ -86,13 +128,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `ImageQueryTask` and `ImageQueryEngine`.
 
 ### Fixed 
-- `BedrockStableDiffusionImageGenerationModelDriver` request parameters for SDXLv1.
+- `BedrockStableDiffusionImageGenerationModelDriver` request parameters for SDXLv1 (`stability.stable-diffusion-xl-v1`).
 - `BedrockStableDiffusionImageGenerationModelDriver` correctly handles the CONTENT_FILTERED response case.
 
 ### Changed
 - **BREAKING**: Make `index_name` on `MongoDbAtlasVectorStoreDriver` a required field.
 - **BREAKING**: Remove `create_index()` from `MarqoVectorStoreDriver`, `OpenSearchVectorStoreDriver`, `PineconeVectorStoreDriver`, `RedisVectorStoreDriver`.
 - **BREAKING**: `ImageLoader().load()` now accepts image bytes instead of a file path.
+- **BREAKING**: Request parameters for `BedrockStableDiffusionImageGenerationModelDriver` have been updated for `stability.stable-diffusion-xl-v1`. Use this over the now deprecated `stability.stable-diffusion-xl-v0`.
 - Deprecated `Structure.prompt_driver` in favor of `Structure.config.global_drivers.prompt_driver`.
 - Deprecated `Structure.embedding_driver` in favor of `Structure.config.global_drivers.embedding_driver`.
 - Deprecated `Structure.stream` in favor of `Structure.config.global_drivers.prompt_driver.stream`.