claude3opus_eval.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This is an eval on Friedman #2, Original #1, and Original #3\n",
    "\n",
    "Claude 3 Opus compared with (1) Linear Regression, (2) Multi-Layer Perceptron, (3) Gradient Boosting, (4) Random Forests."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "##############################\n",
    "## Step 1: Set up your API key\n",
    "##############################\n",
    "\n",
    "import os\n",
    "import tqdm\n",
    "\n",
    "# This key will not be active by the time you see this :)\n",
    "os.environ['OPENROUTER_API_KEY'] = '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'\n",
    "\n",
    "###############################################\n",
    "## Step 2: create an llm object to call Claude 3 Opus \n",
    "###############################################\n",
    "from langchain.chat_models import ChatOpenAI\n",
    "from typing import Optional\n",
    "\n",
    "class ChatOpenRouter(ChatOpenAI):\n",
    "    \"\"\"\n",
    "    OpenRouter uses same API as OpenAI\n",
    "    See: https://medium.com/@gal.peretz/openrouter-langchain-leverage-opensource-models-without-the-ops-hassle-9ffbf0016da7\n",
    "    \"\"\"\n",
    "    openai_api_base: str\n",
    "    openai_api_key: str\n",
    "    model_name: str\n",
    "\n",
    "    def __init__(self,\n",
    "                 model_name: str,\n",
    "                 openai_api_key: Optional[str] = None,\n",
    "                 openai_api_base: str = \"https://openrouter.ai/api/v1\",\n",
    "                 **kwargs):\n",
    "        openai_api_key = openai_api_key or os.getenv('OPENROUTER_API_KEY')\n",
    "        super().__init__(openai_api_base=openai_api_base,\n",
    "                         openai_api_key=openai_api_key,\n",
    "                         model_name=model_name, **kwargs)\n",
    "\n",
    "llm = ChatOpenRouter(model_name='anthropic/claude-3-opus', temperature=0, max_retries=5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#############################\n",
    "## Step 3: Prepare the prompt\n",
    "#############################\n",
    "from langchain import PromptTemplate, FewShotPromptTemplate\n",
    "\n",
    "def prepare_prompt(x_train, y_train, x_test):\n",
    "    \"\"\"\n",
    "    Prepare the prompt\n",
    "    \"\"\"\n",
    "    suffix = [feature + \": {\" + f\"{feature}\" + \"}\" for feature in x_train.columns] + [y_train.name + \":\"]\n",
    "    suffix = \"\\n\".join(suffix)\n",
    "\n",
    "    input_variables=x_train.columns.to_list()\n",
    "\n",
    "    # The template for the in-context examples. Here, you also give the expected output\n",
    "    template = [feature + \": {\" + f\"{feature}\" + \"}\" for feature in x_train.columns] + [y_train.name + \": {\" + f\"{y_train.name}\" + \"}\"]\n",
    "    template = \"\\n\".join(template)\n",
    "    example_prompt = PromptTemplate(\n",
    "        template=template,\n",
    "        input_variables=x_train.columns.to_list() + [y_train.name],\n",
    "    )\n",
    "\n",
    "\n",
    "    # Create the few-shot prompt template\n",
    "    fspt = FewShotPromptTemplate(\n",
    "        examples        =  [{**x1, y_train.name: x2} for x1, x2 in zip(x_train.to_dict('records'), y_train)],\n",
    "        example_prompt  =  example_prompt,\n",
    "        suffix          =  suffix,\n",
    "        input_variables = input_variables,\n",
    "    )\n",
    "\n",
    "    # An instruction to prevent the model from generating explanations.\n",
    "    prefix_instruction = 'The task is to provide your best estimate for \"Output\". Please provide that and only that, without any additional text.\\n\\n\\n\\n\\n'\n",
    "\n",
    "    return prefix_instruction + fspt.format(**x_test.to_dict('records')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\n",
    "\n",
    "\n",
    "\n",
    "def linear_regression(x_train, x_test, y_train, y_test, random_state=1):\n",
    "    model = LinearRegression()\n",
    "    model.fit(x_train, y_train)\n",
    "    y_predict = model.predict(x_test)\n",
    "    y_test    = y_test.to_numpy()\n",
    "\n",
    "    return y_predict\n",
    "\n",
    "def mlp(x_train, x_test, y_train, y_test, random_state=1):\n",
    "    \"\"\"\n",
    "    Multi-Layer Perceptron\n",
    "    \"\"\"\n",
    "    model = MLPRegressor(hidden_layer_sizes=(100, ), activation='relu', solver='lbfgs', random_state=random_state)\n",
    "    model.fit(x_train, y_train)\n",
    "    y_predict = model.predict(x_test)\n",
    "    y_test    = y_test.to_numpy()\n",
    "\n",
    "    return y_predict\n",
    "\n",
    "def gradient_boosting(x_train, x_test, y_train, y_test, random_state=1):\n",
    "    \"\"\"\n",
    "    Gradient Boosting Regressor\n",
    "    \"\"\"\n",
    "    model = GradientBoostingRegressor(random_state=random_state)\n",
    "    model.fit(x_train, y_train)\n",
    "    y_predict = model.predict(x_test)\n",
    "    y_test    = y_test.to_numpy()\n",
    "\n",
    "    return y_predict\n",
    "\n",
    "def random_forest(x_train, x_test, y_train, y_test, random_state=1):\n",
    "    \"\"\"\n",
    "    Random Forest Regressor\n",
    "    \"\"\"\n",
    "    model = RandomForestRegressor(max_depth=3, random_state=random_state)\n",
    "    model.fit(x_train, y_train)\n",
    "    y_predict = model.predict(x_test)\n",
    "    y_test    = y_test.to_numpy()\n",
    "\n",
    "    return y_predict\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (1) Friedman #2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#############################\n",
    "## Friedman #2 Dataset\n",
    "##############################\n",
    "# Here, we will use Friedman #2\n",
    "from sklearn.datasets import make_friedman2\n",
    "from sklearn.model_selection import train_test_split\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "def get_dataset1(random_state=1):\n",
    "\n",
    "    # The data from sklearn\n",
    "    r_data, r_values = make_friedman2(n_samples=51, noise=0, random_state=random_state)\n",
    "\n",
    "    # Create a dataframe; Not mandatory, but makes things easier\n",
    "    df = pd.DataFrame({**{f'Feature {i}': r_data[:, i] for i in range(r_data.shape[1])}, 'Output': r_values})\n",
    "    x = df.drop(['Output'], axis=1)\n",
    "    y = df['Output']\n",
    "\n",
    "    # Round the values to 2 decimal places\n",
    "    # Not mandatory, but helps to: (1) Keep the costs low, (2) Work with the same numbers of examples with models that have a smaller context (e.g., Yi, Llama, etc)\n",
    "    x = np.round(x, 2)\n",
    "    y = np.round(y, 2)\n",
    "\n",
    "    # Do a random split\n",
    "    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=1, random_state=random_state)\n",
    "\n",
    "\n",
    "    x_train = x_train.iloc[:50]\n",
    "    y_train = y_train.iloc[:50]\n",
    "    x_test  = x_test.iloc[:1]\n",
    "    y_test  = y_test.iloc[:1]\n",
    "\n",
    "    return x_train, y_train, x_test, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/10 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/storage/rvacareanu/apps/miniconda3/envs/llm4r_v2/lib/python3.12/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The function `call_as_llm` was deprecated in LangChain 0.1.7 and will be removed in 0.2.0. Use invoke instead.\n",
      "  warn_deprecated(\n",
      " 50%|█████     | 5/10 [00:27<00:26,  5.36s/it]/storage/rvacareanu/apps/miniconda3/envs/llm4r_v2/lib/python3.12/site-packages/sklearn/neural_network/_multilayer_perceptron.py:546: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "  self.n_iter_ = _check_optimize_result(\"lbfgs\", opt_res, self.max_iter)\n",
      "100%|██████████| 10/10 [00:51<00:00,  5.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "Claude 3 Opus MAE    : 8.084999999999999\n",
      "Linear Regression MAE: 91.16681885157396\n",
      "MLP MAE              : 229.09094047542962\n",
      "Gradient Boosting MAE: 25.45407523722286\n",
      "Random Forest MAE    : 58.116755484567776\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "predictions = []\n",
    "for random_seed in tqdm.tqdm(range(1,11)):\n",
    "    (x_train, y_train, x_test, y_test) = get_dataset1(random_state=random_seed)\n",
    "    claude3opus_prediction = llm.call_as_llm(prepare_prompt(x_train, y_train, x_test))\n",
    "    linear_regression_prediction   = linear_regression(x_train, x_test, y_train, y_test)\n",
    "    mlp_prediction                 = mlp(x_train, x_test, y_train, y_test)\n",
    "    gradient_boosting_prediction   = gradient_boosting(x_train, x_test, y_train, y_test)\n",
    "    random_forest_prediction       = random_forest(x_train, x_test, y_train, y_test)\n",
    "\n",
    "    gold = y_test.values[0]\n",
    "\n",
    "    predictions.append({\n",
    "        'gold'             : gold,\n",
    "        'claude3opus'      : float(claude3opus_prediction.strip()), # Slightly risky\n",
    "        'linear_regression': linear_regression_prediction[0],\n",
    "        'mlp'              : mlp_prediction[0],\n",
    "        'gradient_boosting': gradient_boosting_prediction[0],\n",
    "        'random_forest'    : random_forest_prediction[0],\n",
    "        'y_test'           : y_test.values[0],\n",
    "        'random_seed'      : random_seed,\n",
    "    })\n",
    "\n",
    "claude3opus_predictions       = np.array([x['claude3opus'] for x in predictions])\n",
    "linear_regression_predictions = np.array([x['linear_regression'] for x in predictions])\n",
    "mlp_predictions               = np.array([x['mlp'] for x in predictions])\n",
    "gradient_boosting_predictions = np.array([x['gradient_boosting'] for x in predictions])\n",
    "random_forest_predictions     = np.array([x['random_forest'] for x in predictions])\n",
    "gold                          = np.array([x['gold'] for x in predictions])\n",
    "\n",
    "print(\"\\n\\n\")\n",
    "print(\"Claude 3 Opus MAE    :\", np.abs(claude3opus_predictions - gold).mean())\n",
    "print(\"Linear Regression MAE:\", np.abs(linear_regression_predictions - gold).mean())\n",
    "print(\"MLP MAE              :\", np.abs(mlp_predictions - gold).mean())\n",
    "print(\"Gradient Boosting MAE:\", np.abs(gradient_boosting_predictions - gold).mean())\n",
    "print(\"Random Forest MAE    :\", np.abs(random_forest_predictions - gold).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'gold': 146.8, 'claude3opus': 129.01, 'linear_regression': 195.35132048426374, 'mlp': 529.9781317360549, 'gradient_boosting': 148.00856845600683, 'random_forest': 113.27732924425248, 'y_test': 146.8, 'random_seed': 1}\n",
      "{'gold': 434.54, 'claude3opus': 436.01, 'linear_regression': 395.6242666518726, 'mlp': 354.32758619807396, 'gradient_boosting': 432.0025598775895, 'random_forest': 490.04636230677875, 'y_test': 434.54, 'random_seed': 2}\n",
      "{'gold': 180.52, 'claude3opus': 184.11, 'linear_regression': 178.7166833783375, 'mlp': 450.2954607848463, 'gradient_boosting': 190.23431534096437, 'random_forest': 180.9916758140389, 'y_test': 180.52, 'random_seed': 3}\n",
      "{'gold': 516.99, 'claude3opus': 516.28, 'linear_regression': 574.6307789050409, 'mlp': 740.3805611370773, 'gradient_boosting': 559.4804491537382, 'random_forest': 540.308340646571, 'y_test': 516.99, 'random_seed': 4}\n",
      "{'gold': 88.66, 'claude3opus': 61.31, 'linear_regression': -79.39015826482563, 'mlp': 268.01716488267596, 'gradient_boosting': 80.96517538073554, 'random_forest': 102.30794596523576, 'y_test': 88.66, 'random_seed': 5}\n",
      "{'gold': 1158.02, 'claude3opus': 1155.98, 'linear_regression': 965.6361648056395, 'mlp': 807.0994606859491, 'gradient_boosting': 1043.92497196549, 'random_forest': 1025.143878029017, 'y_test': 1158.02, 'random_seed': 6}\n",
      "{'gold': 130.88, 'claude3opus': 140.96, 'linear_regression': 102.73845148607376, 'mlp': 389.0850118923067, 'gradient_boosting': 160.53997255287425, 'random_forest': 208.7535060212275, 'y_test': 130.88, 'random_seed': 7}\n",
      "{'gold': 20.57, 'claude3opus': 14.36, 'linear_regression': -253.55641892004377, 'mlp': 288.8384008097797, 'gradient_boosting': 27.93912841891627, 'random_forest': 118.35665875277904, 'y_test': 20.57, 'random_seed': 8}\n",
      "{'gold': 115.4, 'claude3opus': 126.38, 'linear_regression': 32.95234738781335, 'mlp': 348.98069695900256, 'gradient_boosting': 149.90226194013132, 'random_forest': 258.95571771510294, 'y_test': 115.4, 'random_seed': 9}\n",
      "{'gold': 315.39, 'claude3opus': 314.76, 'linear_regression': 295.7825743486977, 'mlp': 359.41102343657604, 'gradient_boosting': 320.65876373341234, 'random_forest': 312.7814451027865, 'y_test': 315.39, 'random_seed': 10}\n"
     ]
    }
   ],
   "source": [
    "for line in predictions:\n",
    "    print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The task is to provide your best estimate for \"Output\". Please provide that and only that, without any additional text.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Feature 0: 50.7\n",
      "Feature 1: 1463.66\n",
      "Feature 2: 0.09\n",
      "Feature 3: 9.0\n",
      "Output: 141.29\n",
      "\n",
      "Feature 0: 48.4\n",
      "Feature 1: 1505.08\n",
      "Feature 2: 0.17\n",
      "Feature 3: 1.15\n",
      "Output: 267.52\n",
      "\n",
      "Feature 0: 6.43\n",
      "Feature 1: 1724.69\n",
      "Feature 2: 0.34\n",
      "Feature 3: 5.95\n",
      "Output: 585.93\n",
      "\n",
      "Feature 0: 64.51\n",
      "Feature 1: 205.1\n",
      "Feature 2: 0.25\n",
      "Feature 3: 6.42\n",
      "Output: 82.23\n",
      "\n",
      "Feature 0: 60.56\n",
      "Feature 1: 964.48\n",
      "Feature 2: 0.6\n",
      "Feature 3: 3.62\n",
      "Output: 579.77\n",
      "\n",
      "Feature 0: 0.39\n",
      "Feature 1: 962.4\n",
      "Feature 2: 0.81\n",
      "Feature 3: 7.13\n",
      "Output: 782.06\n",
      "\n",
      "Feature 0: 46.98\n",
      "Feature 1: 1102.99\n",
      "Feature 2: 0.15\n",
      "Feature 3: 2.84\n",
      "Output: 169.47\n",
      "\n",
      "Feature 0: 30.09\n",
      "Feature 1: 167.16\n",
      "Feature 2: 0.3\n",
      "Feature 3: 3.42\n",
      "Output: 58.92\n",
      "\n",
      "Feature 0: 7.71\n",
      "Feature 1: 462.11\n",
      "Feature 2: 0.57\n",
      "Feature 3: 3.94\n",
      "Output: 265.26\n",
      "\n",
      "Feature 0: 65.04\n",
      "Feature 1: 1107.54\n",
      "Feature 2: 0.81\n",
      "Feature 3: 6.22\n",
      "Output: 894.18\n",
      "\n",
      "Feature 0: 44.18\n",
      "Feature 1: 834.68\n",
      "Feature 2: 0.62\n",
      "Feature 3: 6.13\n",
      "Output: 517.53\n",
      "\n",
      "Feature 0: 29.59\n",
      "Feature 1: 187.02\n",
      "Feature 2: 0.03\n",
      "Feature 3: 5.53\n",
      "Output: 30.14\n",
      "\n",
      "Feature 0: 16.91\n",
      "Feature 1: 269.98\n",
      "Feature 2: 0.69\n",
      "Feature 3: 10.53\n",
      "Output: 185.8\n",
      "\n",
      "Feature 0: 74.49\n",
      "Feature 1: 1036.08\n",
      "Feature 2: 0.39\n",
      "Feature 3: 2.68\n",
      "Output: 405.9\n",
      "\n",
      "Feature 0: 56.67\n",
      "Feature 1: 997.16\n",
      "Feature 2: 0.01\n",
      "Feature 3: 10.78\n",
      "Output: 58.58\n",
      "\n",
      "Feature 0: 54.76\n",
      "Feature 1: 1464.07\n",
      "Feature 2: 0.2\n",
      "Feature 3: 9.57\n",
      "Output: 296.38\n",
      "\n",
      "Feature 0: 69.39\n",
      "Feature 1: 296.57\n",
      "Feature 2: 0.44\n",
      "Feature 3: 2.66\n",
      "Output: 147.69\n",
      "\n",
      "Feature 0: 57.3\n",
      "Feature 1: 1419.1\n",
      "Feature 2: 0.56\n",
      "Feature 3: 9.77\n",
      "Output: 798.96\n",
      "\n",
      "Feature 0: 31.49\n",
      "Feature 1: 1165.45\n",
      "Feature 2: 0.35\n",
      "Feature 3: 1.43\n",
      "Output: 404.88\n",
      "\n",
      "Feature 0: 72.18\n",
      "Feature 1: 602.48\n",
      "Feature 2: 0.92\n",
      "Feature 3: 8.15\n",
      "Output: 557.63\n",
      "\n",
      "Feature 0: 49.85\n",
      "Feature 1: 492.9\n",
      "Feature 2: 0.2\n",
      "Feature 3: 8.61\n",
      "Output: 109.62\n",
      "\n",
      "Feature 0: 32.55\n",
      "Feature 1: 395.24\n",
      "Feature 2: 0.39\n",
      "Feature 3: 1.93\n",
      "Output: 158.52\n",
      "\n",
      "Feature 0: 60.32\n",
      "Feature 1: 1401.63\n",
      "Feature 2: 0.62\n",
      "Feature 3: 1.21\n",
      "Output: 865.14\n",
      "\n",
      "Feature 0: 78.19\n",
      "Feature 1: 1299.59\n",
      "Feature 2: 0.59\n",
      "Feature 3: 1.37\n",
      "Output: 765.59\n",
      "\n",
      "Feature 0: 21.88\n",
      "Feature 1: 1354.96\n",
      "Feature 2: 0.11\n",
      "Feature 3: 8.45\n",
      "Output: 146.46\n",
      "\n",
      "Feature 0: 84.88\n",
      "Feature 1: 1338.92\n",
      "Feature 2: 0.46\n",
      "Feature 3: 5.17\n",
      "Output: 617.34\n",
      "\n",
      "Feature 0: 54.25\n",
      "Feature 1: 357.92\n",
      "Feature 2: 0.37\n",
      "Feature 3: 7.74\n",
      "Output: 144.22\n",
      "\n",
      "Feature 0: 82.11\n",
      "Feature 1: 372.59\n",
      "Feature 2: 0.38\n",
      "Feature 3: 10.44\n",
      "Output: 165.0\n",
      "\n",
      "Feature 0: 55.76\n",
      "Feature 1: 1049.49\n",
      "Feature 2: 0.48\n",
      "Feature 3: 3.93\n",
      "Output: 501.75\n",
      "\n",
      "Feature 0: 87.99\n",
      "Feature 1: 1372.52\n",
      "Feature 2: 0.88\n",
      "Feature 3: 5.18\n",
      "Output: 1208.41\n",
      "\n",
      "Feature 0: 33.07\n",
      "Feature 1: 1389.81\n",
      "Feature 2: 0.04\n",
      "Feature 3: 5.29\n",
      "Output: 64.64\n",
      "\n",
      "Feature 0: 98.76\n",
      "Feature 1: 871.1\n",
      "Feature 2: 0.83\n",
      "Feature 3: 3.51\n",
      "Output: 726.38\n",
      "\n",
      "Feature 0: 83.83\n",
      "Feature 1: 1104.29\n",
      "Feature 2: 0.78\n",
      "Feature 3: 9.49\n",
      "Output: 868.4\n",
      "\n",
      "Feature 0: 22.68\n",
      "Feature 1: 748.75\n",
      "Feature 2: 0.92\n",
      "Feature 3: 10.25\n",
      "Output: 690.89\n",
      "\n",
      "Feature 0: 97.71\n",
      "Feature 1: 845.72\n",
      "Feature 2: 0.32\n",
      "Feature 3: 6.2\n",
      "Output: 286.36\n",
      "\n",
      "Feature 0: 35.17\n",
      "Feature 1: 1358.48\n",
      "Feature 2: 0.3\n",
      "Feature 3: 9.84\n",
      "Output: 403.59\n",
      "\n",
      "Feature 0: 56.51\n",
      "Feature 1: 1088.44\n",
      "Feature 2: 0.2\n",
      "Feature 3: 5.36\n",
      "Output: 222.9\n",
      "\n",
      "Feature 0: 3.93\n",
      "Feature 1: 709.17\n",
      "Feature 2: 0.08\n",
      "Feature 3: 4.05\n",
      "Output: 56.6\n",
      "\n",
      "Feature 0: 65.57\n",
      "Feature 1: 1438.4\n",
      "Feature 2: 0.35\n",
      "Feature 3: 1.93\n",
      "Output: 509.42\n",
      "\n",
      "Feature 0: 11.67\n",
      "Feature 1: 678.94\n",
      "Feature 2: 0.09\n",
      "Feature 3: 8.16\n",
      "Output: 65.32\n",
      "\n",
      "Feature 0: 81.33\n",
      "Feature 1: 1407.84\n",
      "Feature 2: 0.39\n",
      "Feature 3: 9.64\n",
      "Output: 559.81\n",
      "\n",
      "Feature 0: 90.86\n",
      "Feature 1: 647.18\n",
      "Feature 2: 0.09\n",
      "Feature 3: 4.01\n",
      "Output: 108.09\n",
      "\n",
      "Feature 0: 75.05\n",
      "Feature 1: 413.25\n",
      "Feature 2: 0.46\n",
      "Feature 3: 6.13\n",
      "Output: 203.8\n",
      "\n",
      "Feature 0: 10.81\n",
      "Feature 1: 417.92\n",
      "Feature 2: 0.89\n",
      "Feature 3: 4.65\n",
      "Output: 370.37\n",
      "\n",
      "Feature 0: 57.81\n",
      "Feature 1: 1520.67\n",
      "Feature 2: 0.07\n",
      "Feature 3: 5.65\n",
      "Output: 118.6\n",
      "\n",
      "Feature 0: 67.35\n",
      "Feature 1: 385.73\n",
      "Feature 2: 0.05\n",
      "Feature 3: 4.38\n",
      "Output: 70.1\n",
      "\n",
      "Feature 0: 77.13\n",
      "Feature 1: 159.56\n",
      "Feature 2: 0.63\n",
      "Feature 3: 8.49\n",
      "Output: 127.17\n",
      "\n",
      "Feature 0: 59.74\n",
      "Feature 1: 1600.56\n",
      "Feature 2: 0.53\n",
      "Feature 3: 6.9\n",
      "Output: 857.67\n",
      "\n",
      "Feature 0: 58.42\n",
      "Feature 1: 1283.66\n",
      "Feature 2: 0.15\n",
      "Feature 3: 5.28\n",
      "Output: 199.42\n",
      "\n",
      "Feature 0: 11.4\n",
      "Feature 1: 1479.42\n",
      "Feature 2: 0.05\n",
      "Feature 3: 7.26\n",
      "Output: 70.31\n",
      "\n",
      "Feature 0: 35.07\n",
      "Feature 1: 1045.71\n",
      "Feature 2: 0.3\n",
      "Feature 3: 6.12\n",
      "Output:\n"
     ]
    }
   ],
   "source": [
    "print(prepare_prompt(x_train, y_train, x_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (2) Original #1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataset2(random_state=1):\n",
    "\n",
    "    generator = np.random.RandomState(random_state)\n",
    "    \n",
    "    x = generator.uniform(size=(51, 1), low=0, high=100)\n",
    "    x = np.round(x, 2)\n",
    "    y_fn = lambda x: np.round(x[0] + 10*np.sin(x[0]/100 * np.pi * 5) + 10*np.cos(x[0]/100 * np.pi * 6), 2)\n",
    "    y = np.array([y_fn(point) for point in x])\n",
    "\n",
    "    r_data   = x\n",
    "    r_values = y\n",
    "\n",
    "    # Create a dataframe; Not mandatory, but makes things easier\n",
    "    df = pd.DataFrame({**{f'Feature {i}': r_data[:, i] for i in range(r_data.shape[1])}, 'Output': r_values})\n",
    "    x = df.drop(['Output'], axis=1)\n",
    "    y = df['Output']\n",
    "\n",
    "    # Round the values to 2 decimal places\n",
    "    # Not mandatory, but helps to: (1) Keep the costs low, (2) Work with the same numbers of examples with models that have a smaller context (e.g., Yi, Llama, etc)\n",
    "    x = np.round(x, 2)\n",
    "    y = np.round(y, 2)\n",
    "\n",
    "    # Do a random split\n",
    "    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=1, random_state=random_state)\n",
    "\n",
    "\n",
    "    x_train = x_train.iloc[:50]\n",
    "    y_train = y_train.iloc[:50]\n",
    "    x_test  = x_test.iloc[:1]\n",
    "    y_test  = y_test.iloc[:1]\n",
    "\n",
    "    return x_train, y_train, x_test, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/10 [00:00<?, ?it/s]/storage/rvacareanu/apps/miniconda3/envs/llm4r_v2/lib/python3.12/site-packages/sklearn/neural_network/_multilayer_perceptron.py:546: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "  self.n_iter_ = _check_optimize_result(\"lbfgs\", opt_res, self.max_iter)\n",
      " 20%|██        | 2/10 [00:06<00:24,  3.09s/it]/storage/rvacareanu/apps/miniconda3/envs/llm4r_v2/lib/python3.12/site-packages/sklearn/neural_network/_multilayer_perceptron.py:546: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "  self.n_iter_ = _check_optimize_result(\"lbfgs\", opt_res, self.max_iter)\n",
      " 80%|████████  | 8/10 [00:27<00:06,  3.25s/it]/storage/rvacareanu/apps/miniconda3/envs/llm4r_v2/lib/python3.12/site-packages/sklearn/neural_network/_multilayer_perceptron.py:546: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "  self.n_iter_ = _check_optimize_result(\"lbfgs\", opt_res, self.max_iter)\n",
      " 90%|█████████ | 9/10 [00:31<00:03,  3.46s/it]/storage/rvacareanu/apps/miniconda3/envs/llm4r_v2/lib/python3.12/site-packages/sklearn/neural_network/_multilayer_perceptron.py:546: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "  self.n_iter_ = _check_optimize_result(\"lbfgs\", opt_res, self.max_iter)\n",
      "100%|██████████| 10/10 [00:36<00:00,  3.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "Claude 3 Opus MAE    : 0.49899999999999894\n",
      "Linear Regression MAE: 3.6857252327603915\n",
      "MLP MAE              : 3.1900151823279295\n",
      "Gradient Boosting MAE: 1.625270198927847\n",
      "Random Forest MAE    : 2.242188367868139\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "predictions = []\n",
    "for random_seed in tqdm.tqdm(range(1,11)):\n",
    "    (x_train, y_train, x_test, y_test) = get_dataset2(random_state=random_seed)\n",
    "    claude3opus_prediction = llm.call_as_llm(prepare_prompt(x_train, y_train, x_test))\n",
    "    linear_regression_prediction   = linear_regression(x_train, x_test, y_train, y_test)\n",
    "    mlp_prediction                 = mlp(x_train, x_test, y_train, y_test)\n",
    "    gradient_boosting_prediction   = gradient_boosting(x_train, x_test, y_train, y_test)\n",
    "    random_forest_prediction       = random_forest(x_train, x_test, y_train, y_test)\n",
    "\n",
    "    gold = y_test.values[0]\n",
    "\n",
    "    predictions.append({\n",
    "        'gold'             : gold,\n",
    "        'claude3opus'      : float(claude3opus_prediction.strip()), # Slightly risky\n",
    "        'linear_regression': linear_regression_prediction[0],\n",
    "        'mlp'              : mlp_prediction[0],\n",
    "        'gradient_boosting': gradient_boosting_prediction[0],\n",
    "        'random_forest'    : random_forest_prediction[0],\n",
    "        'y_test'           : y_test.values[0],\n",
    "        'random_seed'      : random_seed,\n",
    "    })\n",
    "\n",
    "claude3opus_predictions       = np.array([x['claude3opus'] for x in predictions])\n",
    "linear_regression_predictions = np.array([x['linear_regression'] for x in predictions])\n",
    "mlp_predictions               = np.array([x['mlp'] for x in predictions])\n",
    "gradient_boosting_predictions = np.array([x['gradient_boosting'] for x in predictions])\n",
    "random_forest_predictions     = np.array([x['random_forest'] for x in predictions])\n",
    "gold                          = np.array([x['gold'] for x in predictions])\n",
    "\n",
    "print(\"\\n\\n\")\n",
    "print(\"Claude 3 Opus MAE    :\", np.abs(claude3opus_predictions - gold).mean())\n",
    "print(\"Linear Regression MAE:\", np.abs(linear_regression_predictions - gold).mean())\n",
    "print(\"MLP MAE              :\", np.abs(mlp_predictions - gold).mean())\n",
    "print(\"Gradient Boosting MAE:\", np.abs(gradient_boosting_predictions - gold).mean())\n",
    "print(\"Random Forest MAE    :\", np.abs(random_forest_predictions - gold).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'gold': 30.86, 'claude3opus': 30.91, 'linear_regression': 33.47658919729503, 'mlp': 28.512980570223135, 'gradient_boosting': 31.340051809689232, 'random_forest': 29.40768928571427, 'y_test': 30.86, 'random_seed': 1}\n",
      "{'gold': 93.91, 'claude3opus': 90.38, 'linear_regression': 89.28822640661915, 'mlp': 92.37802978727407, 'gradient_boosting': 83.6462745756705, 'random_forest': 82.10056480047724, 'y_test': 93.91, 'random_seed': 2}\n",
      "{'gold': 67.4, 'claude3opus': 67.39, 'linear_regression': 66.4348281383472, 'mlp': 66.05952143176427, 'gradient_boosting': 67.58913956918009, 'random_forest': 67.58561282523031, 'y_test': 67.4, 'random_seed': 3}\n",
      "{'gold': 11.62, 'claude3opus': 11.64, 'linear_regression': 16.851732568630606, 'mlp': 12.217127752930132, 'gradient_boosting': 11.782856504820295, 'random_forest': 13.111334702203107, 'y_test': 11.62, 'random_seed': 4}\n",
      "{'gold': 69.51, 'claude3opus': 69.91, 'linear_regression': 80.13828704971182, 'mlp': 80.3822931694284, 'gradient_boosting': 68.83991745351756, 'random_forest': 68.67691437798315, 'y_test': 69.51, 'random_seed': 5}\n",
      "{'gold': 68.47, 'claude3opus': 68.71, 'linear_regression': 77.21535535894135, 'mlp': 77.21532795420033, 'gradient_boosting': 68.83689929038782, 'random_forest': 67.70937003019014, 'y_test': 68.47, 'random_seed': 6}\n",
      "{'gold': 59.42, 'claude3opus': 60.11, 'linear_regression': 58.2734957834262, 'mlp': 57.53729841684258, 'gradient_boosting': 55.97931934364044, 'random_forest': 55.185238854176895, 'y_test': 59.42, 'random_seed': 7}\n",
      "{'gold': 45.98, 'claude3opus': 46.01, 'linear_regression': 47.14708753616836, 'mlp': 43.581619079462556, 'gradient_boosting': 46.519492167948954, 'random_forest': 45.973917809317875, 'y_test': 45.98, 'random_seed': 8}\n",
      "{'gold': 49.59, 'claude3opus': 49.6, 'linear_regression': 51.11926125663588, 'mlp': 48.61791775328912, 'gradient_boosting': 49.65775592601827, 'random_forest': 49.249387816072804, 'y_test': 49.59, 'random_seed': 9}\n",
      "{'gold': 51.4, 'claude3opus': 51.41, 'linear_regression': 51.60548968861342, 'mlp': 50.18723001442383, 'gradient_boosting': 51.32798190593768, 'random_forest': 52.70801912518036, 'y_test': 51.4, 'random_seed': 10}\n"
     ]
    }
   ],
   "source": [
    "for line in predictions:\n",
    "    print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The task is to provide your best estimate for \"Output\". Please provide that and only that, without any additional text.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Feature 0: 4.69\n",
      "Output: 17.75\n",
      "\n",
      "Feature 0: 75.46\n",
      "Output: 68.05\n",
      "\n",
      "Feature 0: 67.41\n",
      "Output: 68.13\n",
      "\n",
      "Feature 0: 90.86\n",
      "Output: 99.25\n",
      "\n",
      "Feature 0: 54.25\n",
      "Output: 55.14\n",
      "\n",
      "Feature 0: 74.88\n",
      "Output: 67.9\n",
      "\n",
      "Feature 0: 52.16\n",
      "Output: 52.41\n",
      "\n",
      "Feature 0: 14.22\n",
      "Output: 13.15\n",
      "\n",
      "Feature 0: 32.55\n",
      "Output: 33.23\n",
      "\n",
      "Feature 0: 76.05\n",
      "Output: 68.27\n",
      "\n",
      "Feature 0: 19.81\n",
      "Output: 11.81\n",
      "\n",
      "Feature 0: 54.76\n",
      "Output: 55.85\n",
      "\n",
      "Feature 0: 63.36\n",
      "Output: 66.44\n",
      "\n",
      "Feature 0: 81.93\n",
      "Output: 75.26\n",
      "\n",
      "Feature 0: 9.05\n",
      "Output: 17.59\n",
      "\n",
      "Feature 0: 68.54\n",
      "Output: 68.18\n",
      "\n",
      "Feature 0: 82.87\n",
      "Output: 77.27\n",
      "\n",
      "Feature 0: 30.07\n",
      "Output: 28.24\n",
      "\n",
      "Feature 0: 91.78\n",
      "Output: 101.61\n",
      "\n",
      "Feature 0: 49.85\n",
      "Output: 49.85\n",
      "\n",
      "Feature 0: 2.08\n",
      "Output: 14.53\n",
      "\n",
      "Feature 0: 0.39\n",
      "Output: 10.98\n",
      "\n",
      "Feature 0: 85.69\n",
      "Output: 84.46\n",
      "\n",
      "Feature 0: 61.78\n",
      "Output: 65.07\n",
      "\n",
      "Feature 0: 80.52\n",
      "Output: 72.71\n",
      "\n",
      "Feature 0: 29.6\n",
      "Output: 27.24\n",
      "\n",
      "Feature 0: 22.48\n",
      "Output: 14.11\n",
      "\n",
      "Feature 0: 51.22\n",
      "Output: 51.3\n",
      "\n",
      "Feature 0: 37.33\n",
      "Output: 40.55\n",
      "\n",
      "Feature 0: 71.46\n",
      "Output: 67.91\n",
      "\n",
      "Feature 0: 29.19\n",
      "Output: 26.37\n",
      "\n",
      "Feature 0: 81.26\n",
      "Output: 73.98\n",
      "\n",
      "Feature 0: 19.89\n",
      "Output: 11.85\n",
      "\n",
      "Feature 0: 31.92\n",
      "Output: 32.02\n",
      "\n",
      "Feature 0: 44.18\n",
      "Output: 45.72\n",
      "\n",
      "Feature 0: 95.34\n",
      "Output: 108.41\n",
      "\n",
      "Feature 0: 62.63\n",
      "Output: 65.86\n",
      "\n",
      "Feature 0: 72.18\n",
      "Output: 67.83\n",
      "\n",
      "Feature 0: 16.5\n",
      "Output: 11.73\n",
      "\n",
      "Feature 0: 88.39\n",
      "Output: 92.28\n",
      "\n",
      "Feature 0: 39.25\n",
      "Output: 42.47\n",
      "\n",
      "Feature 0: 16.91\n",
      "Output: 11.59\n",
      "\n",
      "Feature 0: 35.17\n",
      "Output: 37.7\n",
      "\n",
      "Feature 0: 60.1\n",
      "Output: 63.21\n",
      "\n",
      "Feature 0: 43.4\n",
      "Output: 45.28\n",
      "\n",
      "Feature 0: 65.04\n",
      "Output: 67.46\n",
      "\n",
      "Feature 0: 77.13\n",
      "Output: 68.87\n",
      "\n",
      "Feature 0: 61.25\n",
      "Output: 64.52\n",
      "\n",
      "Feature 0: 11.4\n",
      "Output: 15.7\n",
      "\n",
      "Feature 0: 8.83\n",
      "Output: 17.73\n",
      "\n",
      "Feature 0: 51.31\n",
      "Output:\n"
     ]
    }
   ],
   "source": [
    "print(prepare_prompt(x_train, y_train, x_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Original #3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataset3(random_state=1):\n",
    "\n",
    "    generator = np.random.RandomState(random_state)\n",
    "    \n",
    "    x = generator.uniform(size=(51, 4))\n",
    "    x[:, 0] *= 2\n",
    "    x[:, 0] += 1\n",
    "    x[:, 1] *= 9\n",
    "    x[:, 1] += 1\n",
    "    x[:, 2] *= 10\n",
    "    x[:, 3] *= 19\n",
    "    x[:, 3] += 1\n",
    "\n",
    "    x = np.round(x, 2)\n",
    "    y_fn = lambda x: np.round(np.e ** x[0] + (x[1] * x[2]) / np.sqrt(x[3]) + ((x[3] * x[0]) ** 1.5), 2)\n",
    "    y = np.array([y_fn(point) for point in x])\n",
    "\n",
    "    r_data   = x\n",
    "    r_values = y\n",
    "\n",
    "    # Create a dataframe; Not mandatory, but makes things easier\n",
    "    df = pd.DataFrame({**{f'Feature {i}': r_data[:, i] for i in range(r_data.shape[1])}, 'Output': r_values})\n",
    "    x = df.drop(['Output'], axis=1)\n",
    "    y = df['Output']\n",
    "\n",
    "    # Round the values to 2 decimal places\n",
    "    # Not mandatory, but helps to: (1) Keep the costs low, (2) Work with the same numbers of examples with models that have a smaller context (e.g., Yi, Llama, etc)\n",
    "    x = np.round(x, 2)\n",
    "    y = np.round(y, 2)\n",
    "\n",
    "    # Do a random split\n",
    "    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=1, random_state=random_state)\n",
    "\n",
    "\n",
    "    x_train = x_train.iloc[:50]\n",
    "    y_train = y_train.iloc[:50]\n",
    "    x_test  = x_test.iloc[:1]\n",
    "    y_test  = y_test.iloc[:1]\n",
    "\n",
    "    return x_train, y_train, x_test, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/10 [00:00<?, ?it/s]/storage/rvacareanu/apps/miniconda3/envs/llm4r_v2/lib/python3.12/site-packages/sklearn/neural_network/_multilayer_perceptron.py:546: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "  self.n_iter_ = _check_optimize_result(\"lbfgs\", opt_res, self.max_iter)\n",
      " 20%|██        | 2/10 [00:08<00:32,  4.00s/it]/storage/rvacareanu/apps/miniconda3/envs/llm4r_v2/lib/python3.12/site-packages/sklearn/neural_network/_multilayer_perceptron.py:546: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "  self.n_iter_ = _check_optimize_result(\"lbfgs\", opt_res, self.max_iter)\n",
      " 80%|████████  | 8/10 [00:32<00:08,  4.21s/it]/storage/rvacareanu/apps/miniconda3/envs/llm4r_v2/lib/python3.12/site-packages/sklearn/neural_network/_multilayer_perceptron.py:546: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "  self.n_iter_ = _check_optimize_result(\"lbfgs\", opt_res, self.max_iter)\n",
      " 90%|█████████ | 9/10 [00:35<00:03,  3.85s/it]/storage/rvacareanu/apps/miniconda3/envs/llm4r_v2/lib/python3.12/site-packages/sklearn/neural_network/_multilayer_perceptron.py:546: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "  self.n_iter_ = _check_optimize_result(\"lbfgs\", opt_res, self.max_iter)\n",
      "100%|██████████| 10/10 [00:38<00:00,  3.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "Claude 3 Opus MAE    : 0.49899999999999894\n",
      "Linear Regression MAE: 3.6857252327603915\n",
      "MLP MAE              : 3.1900151823279295\n",
      "Gradient Boosting MAE: 1.625270198927847\n",
      "Random Forest MAE    : 2.242188367868139\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "predictions = []\n",
    "for random_seed in tqdm.tqdm(range(1,11)):\n",
    "    (x_train, y_train, x_test, y_test) = get_dataset2(random_state=random_seed)\n",
    "    claude3opus_prediction = llm.call_as_llm(prepare_prompt(x_train, y_train, x_test))\n",
    "    linear_regression_prediction   = linear_regression(x_train, x_test, y_train, y_test)\n",
    "    mlp_prediction                 = mlp(x_train, x_test, y_train, y_test)\n",
    "    gradient_boosting_prediction   = gradient_boosting(x_train, x_test, y_train, y_test)\n",
    "    random_forest_prediction       = random_forest(x_train, x_test, y_train, y_test)\n",
    "\n",
    "    gold = y_test.values[0]\n",
    "\n",
    "    predictions.append({\n",
    "        'gold'             : gold,\n",
    "        'claude3opus'      : float(claude3opus_prediction.strip()), # Slightly risky\n",
    "        'linear_regression': linear_regression_prediction[0],\n",
    "        'mlp'              : mlp_prediction[0],\n",
    "        'gradient_boosting': gradient_boosting_prediction[0],\n",
    "        'random_forest'    : random_forest_prediction[0],\n",
    "        'y_test'           : y_test.values[0],\n",
    "        'random_seed'      : random_seed,\n",
    "    })\n",
    "\n",
    "claude3opus_predictions       = np.array([x['claude3opus'] for x in predictions])\n",
    "linear_regression_predictions = np.array([x['linear_regression'] for x in predictions])\n",
    "mlp_predictions               = np.array([x['mlp'] for x in predictions])\n",
    "gradient_boosting_predictions = np.array([x['gradient_boosting'] for x in predictions])\n",
    "random_forest_predictions     = np.array([x['random_forest'] for x in predictions])\n",
    "gold                          = np.array([x['gold'] for x in predictions])\n",
    "\n",
    "print(\"\\n\\n\")\n",
    "print(\"Claude 3 Opus MAE    :\", np.abs(claude3opus_predictions - gold).mean())\n",
    "print(\"Linear Regression MAE:\", np.abs(linear_regression_predictions - gold).mean())\n",
    "print(\"MLP MAE              :\", np.abs(mlp_predictions - gold).mean())\n",
    "print(\"Gradient Boosting MAE:\", np.abs(gradient_boosting_predictions - gold).mean())\n",
    "print(\"Random Forest MAE    :\", np.abs(random_forest_predictions - gold).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'gold': 30.86, 'claude3opus': 30.91, 'linear_regression': 33.47658919729503, 'mlp': 28.512980570223135, 'gradient_boosting': 31.340051809689232, 'random_forest': 29.40768928571427, 'y_test': 30.86, 'random_seed': 1}\n",
      "{'gold': 93.91, 'claude3opus': 90.38, 'linear_regression': 89.28822640661915, 'mlp': 92.37802978727407, 'gradient_boosting': 83.6462745756705, 'random_forest': 82.10056480047724, 'y_test': 93.91, 'random_seed': 2}\n",
      "{'gold': 67.4, 'claude3opus': 67.39, 'linear_regression': 66.4348281383472, 'mlp': 66.05952143176427, 'gradient_boosting': 67.58913956918009, 'random_forest': 67.58561282523031, 'y_test': 67.4, 'random_seed': 3}\n",
      "{'gold': 11.62, 'claude3opus': 11.64, 'linear_regression': 16.851732568630606, 'mlp': 12.217127752930132, 'gradient_boosting': 11.782856504820295, 'random_forest': 13.111334702203107, 'y_test': 11.62, 'random_seed': 4}\n",
      "{'gold': 69.51, 'claude3opus': 69.91, 'linear_regression': 80.13828704971182, 'mlp': 80.3822931694284, 'gradient_boosting': 68.83991745351756, 'random_forest': 68.67691437798315, 'y_test': 69.51, 'random_seed': 5}\n",
      "{'gold': 68.47, 'claude3opus': 68.71, 'linear_regression': 77.21535535894135, 'mlp': 77.21532795420033, 'gradient_boosting': 68.83689929038782, 'random_forest': 67.70937003019014, 'y_test': 68.47, 'random_seed': 6}\n",
      "{'gold': 59.42, 'claude3opus': 60.11, 'linear_regression': 58.2734957834262, 'mlp': 57.53729841684258, 'gradient_boosting': 55.97931934364044, 'random_forest': 55.185238854176895, 'y_test': 59.42, 'random_seed': 7}\n",
      "{'gold': 45.98, 'claude3opus': 46.01, 'linear_regression': 47.14708753616836, 'mlp': 43.581619079462556, 'gradient_boosting': 46.519492167948954, 'random_forest': 45.973917809317875, 'y_test': 45.98, 'random_seed': 8}\n",
      "{'gold': 49.59, 'claude3opus': 49.6, 'linear_regression': 51.11926125663588, 'mlp': 48.61791775328912, 'gradient_boosting': 49.65775592601827, 'random_forest': 49.249387816072804, 'y_test': 49.59, 'random_seed': 9}\n",
      "{'gold': 51.4, 'claude3opus': 51.41, 'linear_regression': 51.60548968861342, 'mlp': 50.18723001442383, 'gradient_boosting': 51.32798190593768, 'random_forest': 52.70801912518036, 'y_test': 51.4, 'random_seed': 10}\n"
     ]
    }
   ],
   "source": [
    "for line in predictions:\n",
    "    print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The task is to provide your best estimate for \"Output\". Please provide that and only that, without any additional text.\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Feature 0: 4.69\n",
      "Output: 17.75\n",
      "\n",
      "Feature 0: 75.46\n",
      "Output: 68.05\n",
      "\n",
      "Feature 0: 67.41\n",
      "Output: 68.13\n",
      "\n",
      "Feature 0: 90.86\n",
      "Output: 99.25\n",
      "\n",
      "Feature 0: 54.25\n",
      "Output: 55.14\n",
      "\n",
      "Feature 0: 74.88\n",
      "Output: 67.9\n",
      "\n",
      "Feature 0: 52.16\n",
      "Output: 52.41\n",
      "\n",
      "Feature 0: 14.22\n",
      "Output: 13.15\n",
      "\n",
      "Feature 0: 32.55\n",
      "Output: 33.23\n",
      "\n",
      "Feature 0: 76.05\n",
      "Output: 68.27\n",
      "\n",
      "Feature 0: 19.81\n",
      "Output: 11.81\n",
      "\n",
      "Feature 0: 54.76\n",
      "Output: 55.85\n",
      "\n",
      "Feature 0: 63.36\n",
      "Output: 66.44\n",
      "\n",
      "Feature 0: 81.93\n",
      "Output: 75.26\n",
      "\n",
      "Feature 0: 9.05\n",
      "Output: 17.59\n",
      "\n",
      "Feature 0: 68.54\n",
      "Output: 68.18\n",
      "\n",
      "Feature 0: 82.87\n",
      "Output: 77.27\n",
      "\n",
      "Feature 0: 30.07\n",
      "Output: 28.24\n",
      "\n",
      "Feature 0: 91.78\n",
      "Output: 101.61\n",
      "\n",
      "Feature 0: 49.85\n",
      "Output: 49.85\n",
      "\n",
      "Feature 0: 2.08\n",
      "Output: 14.53\n",
      "\n",
      "Feature 0: 0.39\n",
      "Output: 10.98\n",
      "\n",
      "Feature 0: 85.69\n",
      "Output: 84.46\n",
      "\n",
      "Feature 0: 61.78\n",
      "Output: 65.07\n",
      "\n",
      "Feature 0: 80.52\n",
      "Output: 72.71\n",
      "\n",
      "Feature 0: 29.6\n",
      "Output: 27.24\n",
      "\n",
      "Feature 0: 22.48\n",
      "Output: 14.11\n",
      "\n",
      "Feature 0: 51.22\n",
      "Output: 51.3\n",
      "\n",
      "Feature 0: 37.33\n",
      "Output: 40.55\n",
      "\n",
      "Feature 0: 71.46\n",
      "Output: 67.91\n",
      "\n",
      "Feature 0: 29.19\n",
      "Output: 26.37\n",
      "\n",
      "Feature 0: 81.26\n",
      "Output: 73.98\n",
      "\n",
      "Feature 0: 19.89\n",
      "Output: 11.85\n",
      "\n",
      "Feature 0: 31.92\n",
      "Output: 32.02\n",
      "\n",
      "Feature 0: 44.18\n",
      "Output: 45.72\n",
      "\n",
      "Feature 0: 95.34\n",
      "Output: 108.41\n",
      "\n",
      "Feature 0: 62.63\n",
      "Output: 65.86\n",
      "\n",
      "Feature 0: 72.18\n",
      "Output: 67.83\n",
      "\n",
      "Feature 0: 16.5\n",
      "Output: 11.73\n",
      "\n",
      "Feature 0: 88.39\n",
      "Output: 92.28\n",
      "\n",
      "Feature 0: 39.25\n",
      "Output: 42.47\n",
      "\n",
      "Feature 0: 16.91\n",
      "Output: 11.59\n",
      "\n",
      "Feature 0: 35.17\n",
      "Output: 37.7\n",
      "\n",
      "Feature 0: 60.1\n",
      "Output: 63.21\n",
      "\n",
      "Feature 0: 43.4\n",
      "Output: 45.28\n",
      "\n",
      "Feature 0: 65.04\n",
      "Output: 67.46\n",
      "\n",
      "Feature 0: 77.13\n",
      "Output: 68.87\n",
      "\n",
      "Feature 0: 61.25\n",
      "Output: 64.52\n",
      "\n",
      "Feature 0: 11.4\n",
      "Output: 15.7\n",
      "\n",
      "Feature 0: 8.83\n",
      "Output: 17.73\n",
      "\n",
      "Feature 0: 51.31\n",
      "Output:\n"
     ]
    }
   ],
   "source": [
    "print(prepare_prompt(x_train, y_train, x_test))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llm4r_v2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}