notebooks/llama_index.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "# %pip install -r ../requirements-open-llms.txt"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721661843911
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# %pip install llama-index llama-index-llms-azure-openai llama-index-embeddings-azure-openai llama-index-llms-huggingface llama-index-embeddings-huggingface"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "gather": {
          "logged": 1721661844856
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!bash /home/azureuser/cloudfiles/code/blobfuse/blobfuse_raadsinformatie.sh"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "sys.path.append(\"..\")"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662503857
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Select where to run notebook: \"azure\" or \"local\"\n",
        "my_run = \"azure\""
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662504191
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import my_secrets as sc\n",
        "import settings as st\n",
        "\n",
        "if my_run == \"azure\":\n",
        "    import config_azure as cf\n",
        "elif my_run == \"local\":\n",
        "    import config as cf"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662505005
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "if my_run == \"azure\":\n",
        "    if not os.path.exists(cf.HUGGING_CACHE):\n",
        "        os.mkdir(cf.HUGGING_CACHE)\n",
        "\n",
        "    os.environ[\"TRANSFORMERS_CACHE\"] = cf.HUGGING_CACHE"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662505278
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from pathlib import Path\n",
        "\n",
        "comparison_folder = f\"{cf.raadsinformatie_out_folder}/comparison\"\n",
        "Path(comparison_folder).mkdir(parents=True, exist_ok=True)"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662505571
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import glob\n",
        "\n",
        "woo_dirs = \\\n",
        "        [f\"{cf.woo_sources['openamsterdam']}/{folder}\" for folder in os.listdir(cf.woo_sources['openamsterdam'])] + \\\n",
        "        [f\"{cf.woo_sources['raadsinformatie']}/{folder}\" for folder in os.listdir(cf.woo_sources['raadsinformatie'])] + \\\n",
        "        [f\"{cf.woo_sources['amsterdam.nl']}/{folder}\" for folder in os.listdir(cf.woo_sources['amsterdam.nl'])]\n",
        "\n",
        "woo_files = sum([glob.glob(f\"{folder}/*.ocr\") for folder in woo_dirs], [])"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662506793
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "len(woo_files)"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662506977
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Set up llm and embed model"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.llms.azure_openai import AzureOpenAI\n",
        "from llama_index.llms.huggingface import HuggingFaceLLM\n",
        "from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding\n",
        "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
        "# from llama_index.text_splitter import TokenTextSplitter\n",
        "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, Settings, StorageContext, load_index_from_storage\n",
        "from llama_index.core import PromptTemplate\n",
        "import tiktoken\n",
        "from transformers import AutoModel, AutoTokenizer\n",
        "import logging\n",
        "import sys\n",
        "\n",
        "logging.basicConfig(\n",
        "    stream=sys.stdout, level=logging.WARNING,\n",
        ")  # logging.DEBUG for more verbose output\n",
        "logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))\n",
        "\n",
        "\n",
        "API_KEY = sc.AZURE_OPENAI_API_KEY\n",
        "RESOURCE_ENDPOINT = st.AZURE_OPENAI_ENDPOINT\n",
        "\n",
        "llama_query_wrapper = PromptTemplate(\"<s>[INST] {query_str} [/INST] </s>\\n\")\n",
        "\n",
        "default_llm_params = {\n",
        "    \"do_sample\": True,\n",
        "    \"temperature\": 0.6,\n",
        "    \"top_k\": 25,\n",
        "    \"top_p\": 0.65\n",
        "}\n",
        "\n",
        "max_new_tokens = 256\n",
        "\n",
        "def get_llm(model_name):\n",
        "    if model_name == \"gpt\":\n",
        "        model = AzureOpenAI(\n",
        "            # defaults to os.environ.get(\"OPENAI_API_KEY\")\n",
        "            # engine=\"LC-csv\",\n",
        "            model=\"gpt-35-turbo\",\n",
        "            deployment_name =\"gpt-35-turbo\",\n",
        "            api_key=API_KEY,\n",
        "            api_version=\"2023-05-15\",\n",
        "            azure_endpoint=RESOURCE_ENDPOINT,\n",
        "            max_new_tokens=max_new_tokens,\n",
        "            **default_llm_params\n",
        "        )\n",
        "        context_window = 8192\n",
        "\n",
        "    elif model_name == \"mistral\":   \n",
        "        model = HuggingFaceLLM(\n",
        "            model_name=\"mistralai/Mistral-7B-Instruct-v0.1\",\n",
        "            tokenizer_name=\"mistralai/Mistral-7B-Instruct-v0.1\",\n",
        "            query_wrapper_prompt=llama_query_wrapper,\n",
        "            context_window=3900,\n",
        "            max_new_tokens=max_new_tokens,\n",
        "            generate_kwargs=default_llm_params,\n",
        "            device_map=\"auto\",\n",
        "        )\n",
        "        context_window = 3900\n",
        "\n",
        "    elif model_name == \"llama\":\n",
        "        model = HuggingFaceLLM(\n",
        "            model_name=\"meta-llama/Llama-2-13b-chat-hf\",\n",
        "            tokenizer_name=\"meta-llama/Llama-2-13b-chat-hf\",\n",
        "            query_wrapper_prompt=llama_query_wrapper,\n",
        "            context_window=3900,\n",
        "            max_new_tokens=max_new_tokens,\n",
        "            generate_kwargs=default_llm_params,\n",
        "            device_map=\"auto\",\n",
        "        )\n",
        "        context_window = 3900\n",
        "\n",
        "    else:\n",
        "        raise ValueError(f\"Unknown model {model_name}. Known models: gpt, mistral, llama\")\n",
        "\n",
        "    return model, context_window\n",
        "\n",
        "def get_embed_model(model_name):\n",
        "    if model_name == \"ada\":\n",
        "        model = AzureOpenAIEmbedding(\n",
        "            model=\"text-embedding-ada-002\",\n",
        "            # engine=\"text-embedding-ada-002\",\n",
        "            deployment_name=\"text-embedding-ada-002\",\n",
        "            api_key=API_KEY,\n",
        "            azure_endpoint=RESOURCE_ENDPOINT,\n",
        "            api_version=\"2023-05-15\",\n",
        "        )\n",
        "        # chunk_size = 8191\n",
        "        # chunk_size = 1024 # -> way too slow\n",
        "        # chunk_size = 4096\n",
        "        # tokenizer = tiktoken.get_encoding(\"cl100k_base\").encode\n",
        "        chunk_size = 512\n",
        "\n",
        "    elif model_name == \"bert\":\n",
        "        model_id = \"jegormeister/bert-base-dutch-cased-snli\"\n",
        "        model = HuggingFaceEmbedding(model_id)\n",
        "        # tokenizer = AutoTokenizer.from_pretrained(model_id).encode\n",
        "        chunk_size = 512\n",
        "        \n",
        "\n",
        "    elif model_name == \"robbert\":\n",
        "        model_id = \"NetherlandsForensicInstitute/robbert-2022-dutch-sentence-transformers\"\n",
        "        model = HuggingFaceEmbedding(model_id)\n",
        "        # tokenizer = AutoTokenizer.from_pretrained(model_id).encode\n",
        "        chunk_size = 128\n",
        "\n",
        "    elif model_name == \"cohere\":\n",
        "        model_id = \"Cohere/Cohere-embed-multilingual-v3.0\"\n",
        "        model = HuggingFaceEmbedding(model_id)\n",
        "        # tokenizer = AutoTokenizer.from_pretrained(model_id).encode\n",
        "        chunk_size = 512\n",
        "\n",
        "    elif model_name == \"me5\":\n",
        "        model_id = \"intfloat/multilingual-e5-large-instruct\"\n",
        "        automodel = AutoModel.from_pretrained(model_id)\n",
        "        autotokenizer = AutoTokenizer.from_pretrained(model_id)\n",
        "        model = HuggingFaceEmbedding(\n",
        "            model_name=model_id,\n",
        "            tokenizer_name=model_id,\n",
        "            model=automodel,\n",
        "            tokenizer=autotokenizer,\n",
        "            max_length=512,\n",
        "        )\n",
        "        # tokenizer = tokenizer_model.encode\n",
        "        chunk_size = 512\n",
        "\n",
        "    else:\n",
        "        raise ValueError(f\"Unknown model {model_name}. Known models: ada, bert, robbert, cohere, me5\")\n",
        "\n",
        "    # tokenizer = tiktoken.get_encoding(\"cl100k_base\").encode\n",
        "\n",
        "    # return model, tokenizer, chunk_size\n",
        "    return model, chunk_size\n",
        "\n",
        "llm_model = \"gpt\"\n",
        "embed_model = \"ada\"\n",
        "\n",
        "Settings.llm, Settings.context_window = get_llm(llm_model)\n",
        "# Settings.embed_model, Settings.tokenizer, Settings.chunk_size = get_embed_model(embed_model)\n",
        "Settings.embed_model, Settings.chunk_size = get_embed_model(embed_model)\n",
        "Settings.chunk_overlap = 25\n",
        "\n",
        "# Settings.text_splitter = TokenTextSplitter(\n",
        "#     chunk_size=Settings.chunk_size,\n",
        "#     tokenizer=Settings.tokenizer,\n",
        "# )"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662509778
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "try:\n",
        "    print(Settings.llm.model)\n",
        "except Exception as e:\n",
        "    print(Settings.llm.model_name)\n",
        "\n",
        "print(Settings.embed_model)"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662509972
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Load / Create Index"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Can move with loading, preserving outside to manually inspect documents\n",
        "documents = SimpleDirectoryReader(input_files=woo_files).load_data()"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662520440
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "from llama_index.core import (\n",
        "    VectorStoreIndex,\n",
        "    SimpleDirectoryReader,\n",
        "    StorageContext,\n",
        "    load_index_from_storage,\n",
        ")\n",
        "\n",
        "import pathlib\n",
        "\n",
        "llama_index_folder = f\"{cf.raadsinformatie_out_folder}/llama-indices\"\n",
        "pathlib.Path(llama_index_folder).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "index_persist_path = f\"{cf.raadsinformatie_out_folder}/llama-indices/all_docs-{embed_model}-{Settings.chunk_size}-{Settings.chunk_overlap}\"\n",
        "\n",
        "\n",
        "if not os.path.exists(index_persist_path):\n",
        "    # documents = SimpleDirectoryReader(\"./data/some_data_path\").load_data()\n",
        "    print(\"Creating index...\")\n",
        "    index = VectorStoreIndex.from_documents(documents, show_progress=True)\n",
        "    index.storage_context.persist(persist_dir=index_persist_path)\n",
        "else:\n",
        "    print(\"Loading index...\")\n",
        "    index = load_index_from_storage(\n",
        "        StorageContext.from_defaults(persist_dir=index_persist_path),\n",
        "    )"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "query_engine = index.as_query_engine(similarity_top_k=10)\n",
        "retriever = index.as_retriever(verbose=True, similarity_top_k=10)"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662584751
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Check and adjust prompts\n",
        "https://github.com/run-llama/llama_index/blob/main/docs/examples/prompts/prompts_rag.ipynb"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import Markdown, display\n",
        "\n",
        "def display_prompt_dict(prompts_dict):\n",
        "    for k, p in prompts_dict.items():\n",
        "        text_md = f\"**Prompt Key**: {k}<br>\" f\"**Text:** <br>\"\n",
        "        display(Markdown(text_md))\n",
        "        print(p.get_template())\n",
        "        display(Markdown(\"<br><br>\"))"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662593501
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "display_prompt_dict(query_engine.get_prompts())"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1710499319376
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Check Prompts"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def check_prompt(query_engine, prompt, retriever=None):\n",
        "    print(prompt)\n",
        "\n",
        "    if retriever:\n",
        "        print(\"===== Retrieved =====\")\n",
        "        retrieved = retriever.retrieve(prompt)\n",
        "        print(10*\"-\")\n",
        "        print(\"\\n\".join([node.metadata[\"file_path\"] for node in retrieved]))\n",
        "\n",
        "    try:\n",
        "        response = query_engine.query(prompt)\n",
        "        print(\"----- Source Nodes -----\")\n",
        "        print(\"\\n\".join([node.metadata[\"file_path\"] for node in response.source_nodes]))\n",
        "    except Exception as e:\n",
        "        print(e)\n",
        "\n",
        "    for i in range(5):\n",
        "        print(f\"{10*'-'} Answer {i+1} {10*'-'}\")\n",
        "        try:\n",
        "            answer = query_engine.query(prompt)\n",
        "            print(answer.response)\n",
        "        except Exception as e:\n",
        "            print(e)\n",
        "    \n"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662640131
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def check_prompt_simple(query_engine, prompt):\n",
        "    print(prompt)\n",
        "    answer = query_engine.query(prompt)\n",
        "    print(\"----- Source Nodes -----\")\n",
        "    print(\"\\n\".join([node.metadata[\"file_path\"] for node in answer.source_nodes]))\n",
        "    print(answer.response)"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662643628
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for prompt in st.TEST_PROMPTS:\n",
        "    check_prompt(query_engine, prompt)\n",
        "    # check_prompt_simple(query_engine, prompt)\n",
        "    print(20*\"=\")"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662713403
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for prompt in st.CONFUSION_PROMPTS:\n",
        "    check_prompt(query_engine, prompt)"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1710499465872
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Streaming"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "streaming_query_engine = index.as_query_engine(streaming=True)\n",
        "streaming_response = streaming_query_engine.query(st.TEST_PROMPTS[-1])\n",
        "streaming_response.print_response_stream()"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1721662714591
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Citation"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.core.query_engine import CitationQueryEngine\n",
        "\n",
        "citation_query_engine = CitationQueryEngine.from_args(\n",
        "    index,\n",
        "    similarity_top_k=10,\n",
        "    citation_chunk_size=Settings.chunk_size,\n",
        ")"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1710499468023
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "display_prompt_dict(citation_query_engine.get_prompts())"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1710499468271
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for prompt in st.TEST_PROMPTS:\n",
        "    check_prompt(citation_query_engine, prompt)\n",
        "    print(20*\"=\")\n"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1710499586925
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Save retrieved per prompt"
      ],
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "similarity_top_k = 20\n",
        "retriever_20 = index.as_retriever(verbose=True, similarity_top_k=similarity_top_k, search_kwargs={\"score_threshold\": 0.5})"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1710499587152
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "experiment_name = f\"llama-index-{embed_model}-{Settings.chunk_size}-{Settings.chunk_overlap}\"\n",
        "\n",
        "for ind, prompt in enumerate(st.TEST_PROMPTS):\n",
        "    retrieval_file = Path(comparison_folder, f\"retrieval_{ind}.csv\")\n",
        "\n",
        "    if retrieval_file.exists():\n",
        "        df = pd.read_csv(retrieval_file, index_col=0)\n",
        "    else:\n",
        "        df = pd.DataFrame(index=range(similarity_top_k))\n",
        "\n",
        "    retrieved = retriever_20.retrieve(prompt)\n",
        "    missing = [None] * (20 - len(retrieved))\n",
        "\n",
        "    df[f\"{experiment_name}-file\"] = [node.metadata[\"file_path\"].removeprefix(cf.raadsinformatie_in_folder) for node in retrieved] + missing\n",
        "    df[f\"{experiment_name}-score\"] = [node.score for node in retrieved] + missing\n",
        "    df[f\"{experiment_name}-start\"] = [node.node.start_char_idx for node in retrieved] + missing\n",
        "    df[f\"{experiment_name}-end\"] = [node.node.end_char_idx for node in retrieved] + missing\n",
        "    df[f\"{experiment_name}-text\"] = [node.text for node in retrieved] + missing\n",
        "\n",
        "    df.to_csv(retrieval_file)\n",
        "    "
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1710499590619
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "outputs": [],
      "execution_count": null,
      "metadata": {
        "gather": {
          "logged": 1710499590797
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      }
    }
  ],
  "metadata": {
    "kernel_info": {
      "name": "woo-py39"
    },
    "kernelspec": {
      "name": "woo-py39",
      "language": "python",
      "display_name": "woo-py39"
    },
    "language_info": {
      "name": "python",
      "version": "3.9.18",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "microsoft": {
      "host": {
        "AzureML": {
          "notebookHasBeenCompleted": true
        }
      },
      "ms_spell_check": {
        "ms_spell_check_language": "en"
      }
    },
    "nteract": {
      "version": "nteract-front-end@1.0.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 2
}