01_data_transform.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e94bb81a-1c46-4545-bea3-2fe6b6032b75",
   "metadata": {},
   "source": [
    "This notebook with transform the output of Paraview into something feedable to ML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "08a16d58-85e1-40bc-ae5e-bd735e10c6d4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-03T00:25:59.346757Z",
     "iopub.status.busy": "2022-07-03T00:25:59.346237Z",
     "iopub.status.idle": "2022-07-03T00:26:02.129663Z",
     "shell.execute_reply": "2022-07-03T00:26:02.128305Z",
     "shell.execute_reply.started": "2022-07-03T00:25:59.346704Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6fb23918-5f12-4db7-922c-59c0f977c968",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-03T00:26:02.132476Z",
     "iopub.status.busy": "2022-07-03T00:26:02.132082Z",
     "iopub.status.idle": "2022-07-03T00:26:02.137651Z",
     "shell.execute_reply": "2022-07-03T00:26:02.136460Z",
     "shell.execute_reply.started": "2022-07-03T00:26:02.132419Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pathlib import PurePath"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c106c356-6072-4252-8a22-ce640070f0ad",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-03T00:26:02.141251Z",
     "iopub.status.busy": "2022-07-03T00:26:02.140687Z",
     "iopub.status.idle": "2022-07-03T00:26:02.149083Z",
     "shell.execute_reply": "2022-07-03T00:26:02.147668Z",
     "shell.execute_reply.started": "2022-07-03T00:26:02.141197Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "NUM_FRAMES = 999   # number of timestep. Before was 9! 344!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c4d9873-2166-44f4-a351-04598731ae6d",
   "metadata": {},
   "source": [
    "Our simulation managed to create `NUM_FRAMES` in csv forms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4840ba9f-559d-4568-bd8f-717dd579f18e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-03T00:26:02.153396Z",
     "iopub.status.busy": "2022-07-03T00:26:02.152941Z",
     "iopub.status.idle": "2022-07-03T00:26:02.168858Z",
     "shell.execute_reply": "2022-07-03T00:26:02.166763Z",
     "shell.execute_reply.started": "2022-07-03T00:26:02.153344Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "data_paths = [PurePath(\"new_data\", f\"data_{i}.csv\") for i in range(0,NUM_FRAMES)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9fbd1e92-7626-4f88-a334-e38526d7b56a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-03T00:26:02.171580Z",
     "iopub.status.busy": "2022-07-03T00:26:02.170802Z",
     "iopub.status.idle": "2022-07-03T00:26:02.255550Z",
     "shell.execute_reply": "2022-07-03T00:26:02.254512Z",
     "shell.execute_reply.started": "2022-07-03T00:26:02.171497Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time</th>\n",
       "      <th>Points:0</th>\n",
       "      <th>Points:1</th>\n",
       "      <th>Points:2</th>\n",
       "      <th>U:0</th>\n",
       "      <th>U:1</th>\n",
       "      <th>U:2</th>\n",
       "      <th>k</th>\n",
       "      <th>nut</th>\n",
       "      <th>omega</th>\n",
       "      <th>p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.002</td>\n",
       "      <td>0.024731</td>\n",
       "      <td>-0.001165</td>\n",
       "      <td>0.097489</td>\n",
       "      <td>-0.059273</td>\n",
       "      <td>0.062762</td>\n",
       "      <td>10.573</td>\n",
       "      <td>0.006028</td>\n",
       "      <td>0.000032</td>\n",
       "      <td>278.280</td>\n",
       "      <td>3517.600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.002</td>\n",
       "      <td>0.004980</td>\n",
       "      <td>-0.016126</td>\n",
       "      <td>4.835800</td>\n",
       "      <td>-0.025099</td>\n",
       "      <td>0.018351</td>\n",
       "      <td>10.484</td>\n",
       "      <td>0.004322</td>\n",
       "      <td>0.000032</td>\n",
       "      <td>44.067</td>\n",
       "      <td>80.626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.002</td>\n",
       "      <td>0.001506</td>\n",
       "      <td>-0.028538</td>\n",
       "      <td>4.913500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.018508</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6961.300</td>\n",
       "      <td>42.379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.002</td>\n",
       "      <td>0.026603</td>\n",
       "      <td>-0.010438</td>\n",
       "      <td>4.934800</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.018638</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6983.000</td>\n",
       "      <td>31.822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.002</td>\n",
       "      <td>0.001855</td>\n",
       "      <td>0.024689</td>\n",
       "      <td>0.173850</td>\n",
       "      <td>0.040683</td>\n",
       "      <td>0.017779</td>\n",
       "      <td>10.479</td>\n",
       "      <td>0.006059</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>287.620</td>\n",
       "      <td>3480.800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7577</th>\n",
       "      <td>0.002</td>\n",
       "      <td>-0.013326</td>\n",
       "      <td>0.025280</td>\n",
       "      <td>0.124640</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.018415</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6969.100</td>\n",
       "      <td>3504.500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578</th>\n",
       "      <td>0.002</td>\n",
       "      <td>0.017788</td>\n",
       "      <td>0.021330</td>\n",
       "      <td>0.096825</td>\n",
       "      <td>-0.065950</td>\n",
       "      <td>0.078987</td>\n",
       "      <td>10.243</td>\n",
       "      <td>0.016613</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>4931.500</td>\n",
       "      <td>3517.800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7579</th>\n",
       "      <td>0.002</td>\n",
       "      <td>0.005357</td>\n",
       "      <td>-0.015784</td>\n",
       "      <td>0.196900</td>\n",
       "      <td>0.017935</td>\n",
       "      <td>0.000499</td>\n",
       "      <td>10.545</td>\n",
       "      <td>0.004153</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>24.108</td>\n",
       "      <td>3469.100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7580</th>\n",
       "      <td>0.002</td>\n",
       "      <td>0.024119</td>\n",
       "      <td>0.015329</td>\n",
       "      <td>0.657600</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.006105</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>468.020</td>\n",
       "      <td>3139.400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7581</th>\n",
       "      <td>0.002</td>\n",
       "      <td>0.007922</td>\n",
       "      <td>-0.025925</td>\n",
       "      <td>0.221750</td>\n",
       "      <td>-0.009533</td>\n",
       "      <td>0.329060</td>\n",
       "      <td>10.762</td>\n",
       "      <td>0.010709</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>1508.700</td>\n",
       "      <td>3456.700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7582 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Time  Points:0  Points:1  Points:2       U:0       U:1     U:2  \\\n",
       "0     0.002  0.024731 -0.001165  0.097489 -0.059273  0.062762  10.573   \n",
       "1     0.002  0.004980 -0.016126  4.835800 -0.025099  0.018351  10.484   \n",
       "2     0.002  0.001506 -0.028538  4.913500  0.000000  0.000000   0.000   \n",
       "3     0.002  0.026603 -0.010438  4.934800  0.000000  0.000000   0.000   \n",
       "4     0.002  0.001855  0.024689  0.173850  0.040683  0.017779  10.479   \n",
       "...     ...       ...       ...       ...       ...       ...     ...   \n",
       "7577  0.002 -0.013326  0.025280  0.124640  0.000000  0.000000   0.000   \n",
       "7578  0.002  0.017788  0.021330  0.096825 -0.065950  0.078987  10.243   \n",
       "7579  0.002  0.005357 -0.015784  0.196900  0.017935  0.000499  10.545   \n",
       "7580  0.002  0.024119  0.015329  0.657600  0.000000  0.000000   0.000   \n",
       "7581  0.002  0.007922 -0.025925  0.221750 -0.009533  0.329060  10.762   \n",
       "\n",
       "             k       nut     omega         p  \n",
       "0     0.006028  0.000032   278.280  3517.600  \n",
       "1     0.004322  0.000032    44.067    80.626  \n",
       "2     0.018508  0.000000  6961.300    42.379  \n",
       "3     0.018638  0.000000  6983.000    31.822  \n",
       "4     0.006059  0.000025   287.620  3480.800  \n",
       "...        ...       ...       ...       ...  \n",
       "7577  0.018415  0.000000  6969.100  3504.500  \n",
       "7578  0.016613  0.000004  4931.500  3517.800  \n",
       "7579  0.004153  0.000026    24.108  3469.100  \n",
       "7580  0.006105  0.000000   468.020  3139.400  \n",
       "7581  0.010709  0.000008  1508.700  3456.700  \n",
       "\n",
       "[7582 rows x 11 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs_0 = pd.read_csv(str(data_paths[0]))\n",
    "dfs_0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f45c236-1ec0-4c9a-a533-b51d24013010",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-04T15:29:47.274551Z",
     "iopub.status.busy": "2022-06-04T15:29:47.272665Z",
     "iopub.status.idle": "2022-06-04T15:29:47.297229Z",
     "shell.execute_reply": "2022-06-04T15:29:47.294649Z",
     "shell.execute_reply.started": "2022-06-04T15:29:47.274379Z"
    },
    "tags": []
   },
   "source": [
    "There are five values to choose from:\n",
    "\n",
    "| Terms | Meaning                                               |   \n",
    "|-------|-------------------------------------------------------|\n",
    "| u     | speed vector                                          |\n",
    "| k     | rate of dissipation from kinematic turbulence to heat |                                                                     \n",
    "| omega | omega is the kinematic turbulence energy              |                                                                 \n",
    "| nut   | kinematic turbulent viscosity                         |\n",
    "| p     | Pressure                                              |"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "602ac698-27ac-4e7d-8366-926958a1c8ce",
   "metadata": {},
   "source": [
    "Matrix is a long one, has the shape of `(m=time, n=space)`\n",
    "$$\\begin{bmatrix} C_{1,1} & C_{1,2} & \\cdots & C_{1,n} \\\\ C_{2,1} & C_{2,2} & \\cdots & C_{2,n} \\\\ \\vdots & \\vdots & \\vdots & \\vdots \\\\ C_{m,1} & C_{m,2} & \\cdots & C_{m,n} \\end{bmatrix}  (1)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1cc6f1a2-3c7b-46f7-a0de-1509c782a0dc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-03T00:26:02.258192Z",
     "iopub.status.busy": "2022-07-03T00:26:02.257569Z",
     "iopub.status.idle": "2022-07-03T00:26:02.317624Z",
     "shell.execute_reply": "2022-07-03T00:26:02.316377Z",
     "shell.execute_reply.started": "2022-07-03T00:26:02.258143Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Time</th>\n",
       "      <th>Points:0</th>\n",
       "      <th>Points:1</th>\n",
       "      <th>Points:2</th>\n",
       "      <th>U:0</th>\n",
       "      <th>U:1</th>\n",
       "      <th>U:2</th>\n",
       "      <th>k</th>\n",
       "      <th>nut</th>\n",
       "      <th>omega</th>\n",
       "      <th>p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.004</td>\n",
       "      <td>0.024731</td>\n",
       "      <td>-0.001165</td>\n",
       "      <td>0.097489</td>\n",
       "      <td>-0.067505</td>\n",
       "      <td>0.026594</td>\n",
       "      <td>11.308</td>\n",
       "      <td>0.005933</td>\n",
       "      <td>4.079100e-05</td>\n",
       "      <td>279.510</td>\n",
       "      <td>485.5100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.004</td>\n",
       "      <td>0.004980</td>\n",
       "      <td>-0.016126</td>\n",
       "      <td>4.835800</td>\n",
       "      <td>0.114840</td>\n",
       "      <td>0.041690</td>\n",
       "      <td>11.286</td>\n",
       "      <td>0.004493</td>\n",
       "      <td>3.888800e-05</td>\n",
       "      <td>73.945</td>\n",
       "      <td>5.6927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.004</td>\n",
       "      <td>0.001506</td>\n",
       "      <td>-0.028538</td>\n",
       "      <td>4.913500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.561540</td>\n",
       "      <td>3.605000e-07</td>\n",
       "      <td>10798.000</td>\n",
       "      <td>3.0672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.004</td>\n",
       "      <td>0.026603</td>\n",
       "      <td>-0.010438</td>\n",
       "      <td>4.934800</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.552950</td>\n",
       "      <td>2.581800e-07</td>\n",
       "      <td>10772.000</td>\n",
       "      <td>2.3105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.004</td>\n",
       "      <td>0.001855</td>\n",
       "      <td>0.024689</td>\n",
       "      <td>0.173850</td>\n",
       "      <td>-0.012177</td>\n",
       "      <td>-0.022595</td>\n",
       "      <td>11.120</td>\n",
       "      <td>0.005981</td>\n",
       "      <td>4.250800e-05</td>\n",
       "      <td>294.770</td>\n",
       "      <td>482.9800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7577</th>\n",
       "      <td>0.004</td>\n",
       "      <td>-0.013326</td>\n",
       "      <td>0.025280</td>\n",
       "      <td>0.124640</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.522590</td>\n",
       "      <td>3.248700e-08</td>\n",
       "      <td>10586.000</td>\n",
       "      <td>484.6300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7578</th>\n",
       "      <td>0.004</td>\n",
       "      <td>0.017788</td>\n",
       "      <td>0.021330</td>\n",
       "      <td>0.096825</td>\n",
       "      <td>-0.070005</td>\n",
       "      <td>0.075416</td>\n",
       "      <td>10.398</td>\n",
       "      <td>0.309730</td>\n",
       "      <td>3.451100e-05</td>\n",
       "      <td>6743.200</td>\n",
       "      <td>485.4800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7579</th>\n",
       "      <td>0.004</td>\n",
       "      <td>0.005357</td>\n",
       "      <td>-0.015784</td>\n",
       "      <td>0.196900</td>\n",
       "      <td>0.011529</td>\n",
       "      <td>-0.017054</td>\n",
       "      <td>11.499</td>\n",
       "      <td>0.004344</td>\n",
       "      <td>5.775900e-05</td>\n",
       "      <td>39.388</td>\n",
       "      <td>481.9700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7580</th>\n",
       "      <td>0.004</td>\n",
       "      <td>0.024119</td>\n",
       "      <td>0.015329</td>\n",
       "      <td>0.657600</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.014172</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>545.820</td>\n",
       "      <td>430.9400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7581</th>\n",
       "      <td>0.004</td>\n",
       "      <td>0.007922</td>\n",
       "      <td>-0.025925</td>\n",
       "      <td>0.221750</td>\n",
       "      <td>-0.036254</td>\n",
       "      <td>0.270560</td>\n",
       "      <td>11.226</td>\n",
       "      <td>0.109330</td>\n",
       "      <td>2.787500e-05</td>\n",
       "      <td>2287.600</td>\n",
       "      <td>481.0200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7582 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Time  Points:0  Points:1  Points:2       U:0       U:1     U:2  \\\n",
       "0     0.004  0.024731 -0.001165  0.097489 -0.067505  0.026594  11.308   \n",
       "1     0.004  0.004980 -0.016126  4.835800  0.114840  0.041690  11.286   \n",
       "2     0.004  0.001506 -0.028538  4.913500  0.000000  0.000000   0.000   \n",
       "3     0.004  0.026603 -0.010438  4.934800  0.000000  0.000000   0.000   \n",
       "4     0.004  0.001855  0.024689  0.173850 -0.012177 -0.022595  11.120   \n",
       "...     ...       ...       ...       ...       ...       ...     ...   \n",
       "7577  0.004 -0.013326  0.025280  0.124640  0.000000  0.000000   0.000   \n",
       "7578  0.004  0.017788  0.021330  0.096825 -0.070005  0.075416  10.398   \n",
       "7579  0.004  0.005357 -0.015784  0.196900  0.011529 -0.017054  11.499   \n",
       "7580  0.004  0.024119  0.015329  0.657600  0.000000  0.000000   0.000   \n",
       "7581  0.004  0.007922 -0.025925  0.221750 -0.036254  0.270560  11.226   \n",
       "\n",
       "             k           nut      omega         p  \n",
       "0     0.005933  4.079100e-05    279.510  485.5100  \n",
       "1     0.004493  3.888800e-05     73.945    5.6927  \n",
       "2     0.561540  3.605000e-07  10798.000    3.0672  \n",
       "3     0.552950  2.581800e-07  10772.000    2.3105  \n",
       "4     0.005981  4.250800e-05    294.770  482.9800  \n",
       "...        ...           ...        ...       ...  \n",
       "7577  0.522590  3.248700e-08  10586.000  484.6300  \n",
       "7578  0.309730  3.451100e-05   6743.200  485.4800  \n",
       "7579  0.004344  5.775900e-05     39.388  481.9700  \n",
       "7580  0.014172  0.000000e+00    545.820  430.9400  \n",
       "7581  0.109330  2.787500e-05   2287.600  481.0200  \n",
       "\n",
       "[7582 rows x 11 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs_1 = pd.read_csv(str(data_paths[1]))\n",
    "dfs_1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40dc48db-4d18-412d-98bf-f106b70d927a",
   "metadata": {},
   "source": [
    "### Start ETL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1ed406d9-42b1-4aa9-bf63-9d53d3c620a3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-03T00:40:03.153776Z",
     "iopub.status.busy": "2022-07-03T00:40:03.153320Z",
     "iopub.status.idle": "2022-07-03T00:47:54.578455Z",
     "shell.execute_reply": "2022-07-03T00:47:54.576943Z",
     "shell.execute_reply.started": "2022-07-03T00:40:03.153726Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Working with U:0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "191f1d0c20ae45b793538aae7dcac155",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/998 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    shape of U:0 (7582, 999)\n",
      "    stats: 6.237, -5.4987, 0.0, 0.48821991224659234\n",
      "Working with U:1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "971b2539025b4632ba76789adfeb3f2c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/998 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    shape of U:1 (7582, 999)\n",
      "    stats: 4.6267, -4.7924, 0.0, 0.43915575060251183\n",
      "Working with U:2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac0ba815344e48d8b9694c4e3a72150b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/998 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    shape of U:2 (7582, 999)\n",
      "    stats: 39.224, -0.771, 10.494, 6.971926740615387\n",
      "Working with k\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f939df8373794863914d2762882fdbed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/998 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    shape of k (7582, 999)\n",
      "    stats: 8.549, 0.00375, 0.78422, 0.9294222341152129\n",
      "Working with nut\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86c620ee07424d5682cb105cd5eb3e4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/998 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    shape of nut (7582, 999)\n",
      "    stats: 0.001675, 0.0, 0.00010224, 0.00029039667686375916\n",
      "Working with omega\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ab33ea3b93a45fdab94bcb9bcdaec6d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/998 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    shape of omega (7582, 999)\n",
      "    stats: 23856.0, 3.375, 3013.1, 4774.403157469514\n",
      "Working with p\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba2c0821ad684d038e138d39fb5c1c78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/998 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    shape of p (7582, 999)\n",
      "    stats: 3566.0, -320.15, 0.44396, 209.10706385625687\n"
     ]
    }
   ],
   "source": [
    "chosen_columns = ['U:0', 'U:1', 'U:2', 'k', 'nut', 'omega', 'p']\n",
    "tables = {}\n",
    "for term in chosen_columns:\n",
    "    print(f\"Working with {term}\")\n",
    "    dfs_0 = pd.read_csv(str(data_paths[0]))\n",
    "    p_data = dfs_0[term].copy()\n",
    "    for i in tqdm(range(1,NUM_FRAMES)):\n",
    "        _ = pd.read_csv(str(data_paths[i]))\n",
    "        pd.testing.assert_frame_equal(dfs_0[[\"Points:0\", \"Points:1\", \"Points:2\"]], _[[\"Points:0\", \"Points:1\", \"Points:2\"]]) # Make sure all points coordinates are the same accross time\n",
    "        p_data = pd.concat([p_data, _[term]], axis=1)        \n",
    "    print(f\"    shape of {term} {p_data.shape}\")\n",
    "    tmp = p_data.values.flatten()\n",
    "    if term == 'U:0':\n",
    "        max_u0 = np.max(tmp)\n",
    "    if term == 'U:1':\n",
    "        max_u1 = np.max(tmp)\n",
    "    if term == 'U:2':\n",
    "        max_u2 = np.max(tmp)\n",
    "    print(f\"    stats: {np.max(tmp)}, {np.min(tmp)}, {np.median(tmp)}, {np.std(tmp)}\")\n",
    "    tables[term] = p_data\n",
    "assert max_u2 > max_u1 and max_u2 > max_u0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6941563b-aec1-41ee-92c5-5d37d81bccf5",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "959ee121-837d-44c3-8cf9-1a50d40fb0b3",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-07-03T00:27:28.797779Z",
     "iopub.status.idle": "2022-07-03T00:27:28.799449Z",
     "shell.execute_reply": "2022-07-03T00:27:28.798671Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tables['p']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50ac6ea9-c6c6-4d4c-8bf4-9640830c6572",
   "metadata": {},
   "source": [
    "To make it consistent with (1), we need to transopose"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d2e3c3c-e1c8-4e3a-a75a-086b07f89f0c",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-07-03T00:27:28.803362Z",
     "iopub.status.idle": "2022-07-03T00:27:28.805802Z",
     "shell.execute_reply": "2022-07-03T00:27:28.805261Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tables['p'].values.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8ebf7ed-d020-40a7-ad49-4e22c87d173c",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-07-03T00:27:28.809509Z",
     "iopub.status.idle": "2022-07-03T00:27:28.810854Z",
     "shell.execute_reply": "2022-07-03T00:27:28.810148Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key in tables:\n",
    "    tables[key] = tables[key].values.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d13cfd99-07b9-4add-ad23-869efd3c6a6f",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-07-03T00:27:28.813669Z",
     "iopub.status.idle": "2022-07-03T00:27:28.814308Z",
     "shell.execute_reply": "2022-07-03T00:27:28.814016Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tables['p'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4650d37-eb40-42bf-86bc-010efd980ad1",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-07-03T00:27:28.816498Z",
     "iopub.status.idle": "2022-07-03T00:27:28.817090Z",
     "shell.execute_reply": "2022-07-03T00:27:28.816799Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_index = int(p_data.shape[1] * .5)\n",
    "val_index = train_index + int(p_data.shape[1] * .1)\n",
    "print(f\"train_index: {train_index}, val_index: {val_index}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77731bc8-d343-441e-ba94-dfad24693d1b",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-07-03T00:27:28.819030Z",
     "iopub.status.idle": "2022-07-03T00:27:28.820036Z",
     "shell.execute_reply": "2022-07-03T00:27:28.819531Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "assert(tables['p'].shape[0] < tables['p'].shape[1])   # Time is smaller than number of cells in mesh"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58848c7d-9551-439d-bfe8-73378c3d5e12",
   "metadata": {},
   "source": [
    "### Splitting to train val test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5e71377-f8cf-49eb-b812-7c67b4796aa4",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-07-03T00:27:28.822183Z",
     "iopub.status.idle": "2022-07-03T00:27:28.822965Z",
     "shell.execute_reply": "2022-07-03T00:27:28.822638Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_data = tables['p'][:train_index, :]\n",
    "val_data = tables['p'][train_index:val_index, :]\n",
    "test_data = tables['p'][val_index:, :]\n",
    "print(train_data.shape, val_data.shape, test_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a58c19ed-dd96-496a-a8b4-6ae6dd16ae05",
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-07-03T00:27:28.825418Z",
     "iopub.status.idle": "2022-07-03T00:27:28.826153Z",
     "shell.execute_reply": "2022-07-03T00:27:28.825835Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for key in tables:\n",
    "    print(key)\n",
    "    import pdb; pdb.set_trace()\n",
    "    train_data = tables[key][:train_index, :]\n",
    "    val_data = tables[key][train_index:val_index, :]\n",
    "    test_data = tables[key][val_index:, :]\n",
    "    print(train_data.shape, val_data.shape, test_data.shape)\n",
    "    with open(f'train_{key}.pkl', 'wb') as f:\n",
    "        pickle.dump(train_data, f)\n",
    "    with open(f'val_{key}.pkl', 'wb') as f:\n",
    "        pickle.dump(val_data, f)\n",
    "    with open(f'test_{key}.pkl', 'wb') as f:\n",
    "        pickle.dump(test_data, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}