research_notebooks/explain_documentation/01_load_hummingbot_docs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "## Loading Hummingbot\n",
    "\n",
    "We are going to use a Generic Loader and get the files from the scripts folder of the Hummingbot repository.\n",
    "Also, we are using a LanguageParser to parse the files as Python code."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import sys\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "# Assuming the root directory is one level up from 'research_notebooks'\n",
    "root_path = os.path.abspath(os.path.join(os.getcwd(), '../..'))\n",
    "\n",
    "# Path to 'custom_loaders'\n",
    "loaders_path = os.path.join(root_path)\n",
    "\n",
    "# Add 'custom_loaders' to the Python path\n",
    "sys.path.append(loaders_path)\n",
    "load_dotenv()\n",
    "\n",
    "os.environ[\"OPENAI_API_KEY\"]\n",
    "# you should see the api key if the .env file is loaded correctly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "from custom_loaders.mkdocs_site_loader import MkDocsSiteLoader\n",
    "\n",
    "\n",
    "url = \"https://hummingbot.org/\"\n",
    "sections_filter = [\"blog\", \"release-notes\", \"botcamp\", \"academy\", \"academy-content\", \"exchanges\", \"chain\"]\n",
    "metadata_filter = [\"related_paths\", \"related_urls\"]\n",
    "\n",
    "loader = MkDocsSiteLoader(site_url=url, sections_filter=sections_filter, metadata_filter=metadata_filter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "documents = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "[doc.metadata for doc in documents[:10]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "len(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "print(documents[0].page_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "r_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=1500,\n",
    "    chunk_overlap=0,\n",
    "    separators=[\"\\n\\n\", \"\\n\", \" \", \"\"]\n",
    ")\n",
    "splitted_docs = r_splitter.split_documents([documents[0]])\n",
    "len(splitted_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "splitted_docs[0].page_content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "splitted_docs[1].page_content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "splitted_docs[2].page_content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "splitted_docs[3].page_content"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "# Splitting the documents\n",
    "\n",
    "Now we are going to split the documents using RecursiveCharacterTextSplitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "chunk_size = 2000\n",
    "chunk_overlap = 0\n",
    "persist_directory = os.environ.get(\"PERSIST_DIRECTORY\", \"/tmp\")\n",
    "\n",
    "# Docs: https://python.langchain.com/docs/modules/data_connection/document_transformers/text_splitters/recursive_text_splitter\n",
    "r_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=chunk_size,\n",
    "    chunk_overlap=chunk_overlap,\n",
    "    separators=[\"\\n\\n\", \"\\n\", \" \", \"\"]\n",
    ")\n",
    "splitted_documents = r_splitter.split_documents(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "len(splitted_documents)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "# Creating the embeddings and store in chroma db\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "from langchain.vectorstores.chroma import Chroma\n",
    "from langchain.embeddings import OpenAIEmbeddings\n",
    "\n",
    "def split_list(input_list, size):\n",
    "    return [input_list[i:i + size] for i in range(0, len(input_list), size)]\n",
    "\n",
    "\n",
    "embedding = OpenAIEmbeddings()\n",
    "\n",
    "batch_size = 150\n",
    "document_batches = split_list(splitted_documents, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "len(document_batches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "for batch in document_batches:\n",
    "    vectordb = Chroma.from_documents(\n",
    "        collection_name=\"hummingbot_documentation\",\n",
    "        collection_metadata={\n",
    "            \"Description\": \"This are the files from the Hummingbot docs.\"\n",
    "        },\n",
    "        documents=batch,\n",
    "        embedding=embedding,\n",
    "        persist_directory=persist_directory\n",
    "    )\n",
    "    time.sleep(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "vectordb.similarity_search(\"how can I configure a pure market making strategy?\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}