pols_and_fixed_effect_and_random_eff.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TLN9Xg43hGdj",
        "outputId": "fe05980b-3e95-4b88-da2d-e83dcc938e0d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting linearmodels\n",
            "  Downloading linearmodels-6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (7.9 kB)\n",
            "Requirement already satisfied: numpy<3,>=1.22.3 in /usr/local/lib/python3.10/dist-packages (from linearmodels) (1.25.2)\n",
            "Requirement already satisfied: pandas>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from linearmodels) (2.0.3)\n",
            "Requirement already satisfied: scipy>=1.8.0 in /usr/local/lib/python3.10/dist-packages (from linearmodels) (1.11.4)\n",
            "Requirement already satisfied: statsmodels>=0.13.0 in /usr/local/lib/python3.10/dist-packages (from linearmodels) (0.14.2)\n",
            "Collecting mypy-extensions>=0.4 (from linearmodels)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl.metadata (1.1 kB)\n",
            "Requirement already satisfied: Cython>=3.0.10 in /usr/local/lib/python3.10/dist-packages (from linearmodels) (3.0.10)\n",
            "Collecting pyhdfe>=0.1 (from linearmodels)\n",
            "  Downloading pyhdfe-0.2.0-py3-none-any.whl.metadata (4.0 kB)\n",
            "Collecting formulaic>=1.0.0 (from linearmodels)\n",
            "  Downloading formulaic-1.0.2-py3-none-any.whl.metadata (6.8 kB)\n",
            "Collecting setuptools-scm<9.0.0,>=8.0.0 (from setuptools-scm[toml]<9.0.0,>=8.0.0->linearmodels)\n",
            "  Downloading setuptools_scm-8.1.0-py3-none-any.whl.metadata (6.6 kB)\n",
            "Collecting interface-meta>=1.2.0 (from formulaic>=1.0.0->linearmodels)\n",
            "  Downloading interface_meta-1.3.0-py3-none-any.whl.metadata (6.7 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from formulaic>=1.0.0->linearmodels) (4.12.2)\n",
            "Requirement already satisfied: wrapt>=1.0 in /usr/local/lib/python3.10/dist-packages (from formulaic>=1.0.0->linearmodels) (1.14.1)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.4.0->linearmodels) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.4.0->linearmodels) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.4.0->linearmodels) (2024.1)\n",
            "Requirement already satisfied: packaging>=20 in /usr/local/lib/python3.10/dist-packages (from setuptools-scm<9.0.0,>=8.0.0->setuptools-scm[toml]<9.0.0,>=8.0.0->linearmodels) (24.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from setuptools-scm<9.0.0,>=8.0.0->setuptools-scm[toml]<9.0.0,>=8.0.0->linearmodels) (71.0.4)\n",
            "Requirement already satisfied: tomli>=1 in /usr/local/lib/python3.10/dist-packages (from setuptools-scm<9.0.0,>=8.0.0->setuptools-scm[toml]<9.0.0,>=8.0.0->linearmodels) (2.0.1)\n",
            "Requirement already satisfied: patsy>=0.5.6 in /usr/local/lib/python3.10/dist-packages (from statsmodels>=0.13.0->linearmodels) (0.5.6)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from patsy>=0.5.6->statsmodels>=0.13.0->linearmodels) (1.16.0)\n",
            "Downloading linearmodels-6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.5 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.5/1.5 MB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading formulaic-1.0.2-py3-none-any.whl (94 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m94.5/94.5 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Downloading pyhdfe-0.2.0-py3-none-any.whl (19 kB)\n",
            "Downloading setuptools_scm-8.1.0-py3-none-any.whl (43 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.7/43.7 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hDownloading interface_meta-1.3.0-py3-none-any.whl (14 kB)\n",
            "Installing collected packages: setuptools-scm, mypy-extensions, interface-meta, pyhdfe, formulaic, linearmodels\n",
            "Successfully installed formulaic-1.0.2 interface-meta-1.3.0 linearmodels-6.0 mypy-extensions-1.0.0 pyhdfe-0.2.0 setuptools-scm-8.1.0\n"
          ]
        }
      ],
      "source": [
        "pip install linearmodels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Yquj-TRLhNZ5"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import statsmodels.api as sm\n",
        "from google.colab import drive\n",
        "from linearmodels import PanelOLS, RandomEffects, FirstDifferenceOLS\n",
        "from linearmodels.panel import compare"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Accessing drive"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-SS4ohMthQqq",
        "outputId": "fd812529-7316-4da6-c3a4-385ec60479d0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Reading data from the excel file"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "84qnZN0xhSdt"
      },
      "outputs": [],
      "source": [
        "dfr = pd.read_excel(r'/content/drive/My Drive/CMIE_PROJECT/panel.xlsx', sheet_name='mainpanel')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Dropping irrelevant variables"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fOsQzN5FhZ96"
      },
      "outputs": [],
      "source": [
        "dfr=dfr.drop(columns=[ 'Size', 'Net fixed assets', 'IT/ITES & other professional services','Additions to computers and IT systems during the year','Gross computers and IT systems','Total assets','Export / Sales (%)','Gross software','Net profit margin','Net cash inflow or (outflow) from investing activities','Total forex earnings / Total income (%)','Raw material imports / Raw material purchases (%)'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "df=dfr.copy()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Filling missing values, linearly"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9Z1u5ycY3UXW"
      },
      "outputs": [],
      "source": [
        "def fill_nan_linearly(column):\n",
        "    return column.interpolate(method='linear', limit_direction='both')\n",
        "\n",
        "filled_df = df.groupby('Companies').apply(lambda group: group.apply(fill_nan_linearly, axis=0))\n",
        "filled_df.reset_index(drop=True, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HLKmCcY1h7Vn"
      },
      "outputs": [],
      "source": [
        "df=filled_df.copy()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Converting Year to categorical"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a1YisKWlhrxR"
      },
      "outputs": [],
      "source": [
        "year = pd.Categorical(df.Year)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CUBPAg7OhxB9"
      },
      "outputs": [],
      "source": [
        "df = df.set_index([\"Companies\", \"Year\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xm9lnMUNhy8u"
      },
      "outputs": [],
      "source": [
        "df[\"Year\"] = year"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ERtQWksxkvN7"
      },
      "outputs": [],
      "source": [
        "df.columns = df.columns.str.replace(' ', '_')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lpC7qWfrmZyj"
      },
      "source": [
        "# **Fixed Effect**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JGDWisCYm7X-"
      },
      "source": [
        "**Time Invariant**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E3IcTKn6muVN"
      },
      "outputs": [],
      "source": [
        "exo=['Computer_IT,_net_addition_in_year', 'Net_computers_and_IT_systems',\n",
        "       'Net_plant_&_machinery,_computers_and_electrical_installations',\n",
        "       'Net_software', 'Profit_after_tax', 'Total_expenses', 'Year']\n",
        "exog = sm.add_constant(df[exo])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "L19Pu8_NnBM8"
      },
      "outputs": [],
      "source": [
        "modfeti = PanelOLS(df.Total_income, exog, entity_effects=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GDoKf4gknDNs",
        "outputId": "3f464c88-e2f2-4a2a-a3e8-3f5dc0fd255c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "                          PanelOLS Estimation Summary                           \n",
            "================================================================================\n",
            "Dep. Variable:           Total_income   R-squared:                        0.9885\n",
            "Estimator:                   PanelOLS   R-squared (Between):              0.9997\n",
            "No. Observations:               56126   R-squared (Within):               0.9885\n",
            "Date:                Thu, Jul 25 2024   R-squared (Overall):              0.9978\n",
            "Time:                        12:38:45   Log-likelihood                -5.121e+05\n",
            "Cov. Estimator:            Unadjusted                                           \n",
            "                                        F-statistic:                   2.366e+05\n",
            "Entities:                        4009   P-value                           0.0000\n",
            "Avg Obs:                       14.000   Distribution:                F(19,52098)\n",
            "Min Obs:                       14.000                                           \n",
            "Max Obs:                       14.000   F-statistic (robust):          2.366e+05\n",
            "                                        P-value                           0.0000\n",
            "Time periods:                      14   Distribution:                F(19,52098)\n",
            "Avg Obs:                       4009.0                                           \n",
            "Min Obs:                       4009.0                                           \n",
            "Max Obs:                       4009.0                                           \n",
            "                                                                                \n",
            "                                                       Parameter Estimates                                                       \n",
            "=================================================================================================================================\n",
            "                                                               Parameter  Std. Err.     T-stat    P-value    Lower CI    Upper CI\n",
            "---------------------------------------------------------------------------------------------------------------------------------\n",
            "const                                                             94.703     36.531     2.5924     0.0095      23.102      166.30\n",
            "Computer_IT,_net_addition_in_year                                -0.0743     0.0514    -1.4471     0.1479     -0.1750      0.0263\n",
            "Net_computers_and_IT_systems                                      0.2438     0.0628     3.8816     0.0001      0.1207      0.3668\n",
            "Net_plant_&_machinery,_computers_and_electrical_installations     0.0026     0.0010     2.6141     0.0089      0.0006      0.0045\n",
            "Net_software                                                      0.1066     0.0816     1.3072     0.1912     -0.0532      0.2665\n",
            "Profit_after_tax                                                  0.9896     0.0022     446.62     0.0000      0.9852      0.9939\n",
            "Total_expenses                                                    0.9946     0.0006     1768.2     0.0000      0.9935      0.9957\n",
            "Year.2011                                                        -51.665     51.432    -1.0045     0.3151     -152.47      49.142\n",
            "Year.2012                                                        -103.58     51.437    -2.0137     0.0440     -204.40     -2.7618\n",
            "Year.2013                                                        -143.42     51.443    -2.7880     0.0053     -244.25     -42.593\n",
            "Year.2014                                                        -69.502     51.452    -1.3508     0.1768     -170.35      31.343\n",
            "Year.2015                                                        -123.76     51.465    -2.4047     0.0162     -224.63     -22.888\n",
            "Year.2016                                                        -81.481     51.481    -1.5827     0.1135     -182.38      19.422\n",
            "Year.2017                                                        -123.85     51.503    -2.4046     0.0162     -224.79     -22.899\n",
            "Year.2018                                                        -111.41     51.526    -2.1621     0.0306     -212.40     -10.414\n",
            "Year.2019                                                        -140.18     51.579    -2.7177     0.0066     -241.27     -39.083\n",
            "Year.2020                                                        -160.05     51.585    -3.1027     0.0019     -261.16     -58.946\n",
            "Year.2021                                                        -9.1878     51.566    -0.1782     0.8586     -110.26      91.882\n",
            "Year.2022                                                        -140.45     51.704    -2.7165     0.0066     -241.79     -39.113\n",
            "Year.2023                                                        -175.58     51.860    -3.3858     0.0007     -277.23     -73.939\n",
            "=================================================================================================================================\n",
            "\n",
            "F-test for Poolability: 1.5276\n",
            "P-value: 0.0000\n",
            "Distribution: F(4008,52098)\n",
            "\n",
            "Included effects: Entity\n"
          ]
        }
      ],
      "source": [
        "fe_res_ti = modfeti.fit()\n",
        "print(fe_res_ti)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fqmNMY8jnYUN"
      },
      "source": [
        "**Time Effect**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2CyxWiryjcPl"
      },
      "outputs": [],
      "source": [
        "exo=['Computer_IT,_net_addition_in_year', 'Net_computers_and_IT_systems',\n",
        "       'Net_plant_&_machinery,_computers_and_electrical_installations',\n",
        "       'Net_software', 'Profit_after_tax', 'Total_expenses']\n",
        "exog = sm.add_constant(df[exo])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OgUJ22LIlOIY"
      },
      "outputs": [],
      "source": [
        "modfete = PanelOLS(df.Total_income, exog, entity_effects=True, time_effects=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UM2IiR8XlvCM",
        "outputId": "330d2c5f-a9b8-41ae-ff1b-0d88b20a853c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "                          PanelOLS Estimation Summary                           \n",
            "================================================================================\n",
            "Dep. Variable:           Total_income   R-squared:                        0.9881\n",
            "Estimator:                   PanelOLS   R-squared (Between):              0.9997\n",
            "No. Observations:               56126   R-squared (Within):               0.9885\n",
            "Date:                Thu, Jul 25 2024   R-squared (Overall):              0.9978\n",
            "Time:                        12:38:46   Log-likelihood                -5.121e+05\n",
            "Cov. Estimator:            Unadjusted                                           \n",
            "                                        F-statistic:                   7.223e+05\n",
            "Entities:                        4009   P-value                           0.0000\n",
            "Avg Obs:                       14.000   Distribution:                 F(6,52098)\n",
            "Min Obs:                       14.000                                           \n",
            "Max Obs:                       14.000   F-statistic (robust):          7.223e+05\n",
            "                                        P-value                           0.0000\n",
            "Time periods:                      14   Distribution:                 F(6,52098)\n",
            "Avg Obs:                       4009.0                                           \n",
            "Min Obs:                       4009.0                                           \n",
            "Max Obs:                       4009.0                                           \n",
            "                                                                                \n",
            "                                                       Parameter Estimates                                                       \n",
            "=================================================================================================================================\n",
            "                                                               Parameter  Std. Err.     T-stat    P-value    Lower CI    Upper CI\n",
            "---------------------------------------------------------------------------------------------------------------------------------\n",
            "const                                                            -7.7341     11.325    -0.6829     0.4946     -29.931      14.462\n",
            "Computer_IT,_net_addition_in_year                                -0.0743     0.0514    -1.4471     0.1479     -0.1750      0.0263\n",
            "Net_computers_and_IT_systems                                      0.2438     0.0628     3.8816     0.0001      0.1207      0.3668\n",
            "Net_plant_&_machinery,_computers_and_electrical_installations     0.0026     0.0010     2.6141     0.0089      0.0006      0.0045\n",
            "Net_software                                                      0.1066     0.0816     1.3072     0.1912     -0.0532      0.2665\n",
            "Profit_after_tax                                                  0.9896     0.0022     446.62     0.0000      0.9852      0.9939\n",
            "Total_expenses                                                    0.9946     0.0006     1768.2     0.0000      0.9935      0.9957\n",
            "=================================================================================================================================\n",
            "\n",
            "F-test for Poolability: 1.5273\n",
            "P-value: 0.0000\n",
            "Distribution: F(4021,52098)\n",
            "\n",
            "Included effects: Entity, Time\n"
          ]
        }
      ],
      "source": [
        "fe_res_te = modfete.fit()\n",
        "print(fe_res_te)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Owjr4byJoBRq"
      },
      "source": [
        "# **Random Effect**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DKVcfFyIoFUX"
      },
      "outputs": [],
      "source": [
        "exo=['Computer_IT,_net_addition_in_year', 'Net_computers_and_IT_systems',\n",
        "       'Net_plant_&_machinery,_computers_and_electrical_installations',\n",
        "       'Net_software', 'Profit_after_tax', 'Total_expenses', 'Year']\n",
        "exog = sm.add_constant(df[exo])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KTmB3TDslyXG",
        "outputId": "0a871dc4-54d8-48bf-837e-5ec931fd783b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "                        RandomEffects Estimation Summary                        \n",
            "================================================================================\n",
            "Dep. Variable:           Total_income   R-squared:                        0.9973\n",
            "Estimator:              RandomEffects   R-squared (Between):              0.9997\n",
            "No. Observations:               56126   R-squared (Within):               0.9885\n",
            "Date:                Thu, Jul 25 2024   R-squared (Overall):              0.9979\n",
            "Time:                        12:38:47   Log-likelihood                -5.144e+05\n",
            "Cov. Estimator:            Unadjusted                                           \n",
            "                                        F-statistic:                   1.095e+06\n",
            "Entities:                        4009   P-value                           0.0000\n",
            "Avg Obs:                       14.000   Distribution:                F(19,56106)\n",
            "Min Obs:                       14.000                                           \n",
            "Max Obs:                       14.000   F-statistic (robust):          1.095e+06\n",
            "                                        P-value                           0.0000\n",
            "Time periods:                      14   Distribution:                F(19,56106)\n",
            "Avg Obs:                       4009.0                                           \n",
            "Min Obs:                       4009.0                                           \n",
            "Max Obs:                       4009.0                                           \n",
            "                                                                                \n",
            "                                                       Parameter Estimates                                                       \n",
            "=================================================================================================================================\n",
            "                                                               Parameter  Std. Err.     T-stat    P-value    Lower CI    Upper CI\n",
            "---------------------------------------------------------------------------------------------------------------------------------\n",
            "const                                                             118.13     37.081     3.1856     0.0014      45.447      190.81\n",
            "Computer_IT,_net_addition_in_year                                -0.0980     0.0481    -2.0374     0.0416     -0.1923     -0.0037\n",
            "Net_computers_and_IT_systems                                      0.4955     0.0389     12.741     0.0000      0.4193      0.5718\n",
            "Net_plant_&_machinery,_computers_and_electrical_installations     0.0111     0.0006     19.461     0.0000      0.0100      0.0122\n",
            "Net_software                                                      0.3513     0.0605     5.8066     0.0000      0.2327      0.4698\n",
            "Profit_after_tax                                                  1.0014     0.0017     593.22     0.0000      0.9981      1.0047\n",
            "Total_expenses                                                    0.9854     0.0003     3571.0     0.0000      0.9849      0.9860\n",
            "Year.2011                                                        -48.739     51.627    -0.9441     0.3451     -149.93      52.449\n",
            "Year.2012                                                        -92.942     51.629    -1.8002     0.0718     -194.13      8.2504\n",
            "Year.2013                                                        -127.92     51.630    -2.4777     0.0132     -229.12     -26.727\n",
            "Year.2014                                                        -50.547     51.631    -0.9790     0.3276     -151.74      50.651\n",
            "Year.2015                                                        -100.67     51.635    -1.9497     0.0512     -201.88      0.5308\n",
            "Year.2016                                                        -57.647     51.640    -1.1163     0.2643     -158.86      43.569\n",
            "Year.2017                                                        -94.903     51.644    -1.8376     0.0661     -196.13      6.3197\n",
            "Year.2018                                                        -77.422     51.648    -1.4990     0.1339     -178.65      23.809\n",
            "Year.2019                                                        -92.727     51.659    -1.7950     0.0727     -193.98      8.5254\n",
            "Year.2020                                                        -111.48     51.662    -2.1578     0.0309     -212.73     -10.219\n",
            "Year.2021                                                         32.526     51.655     0.6297     0.5289     -68.718      133.77\n",
            "Year.2022                                                        -83.294     51.685    -1.6116     0.1071     -184.60      18.010\n",
            "Year.2023                                                        -100.46     51.717    -1.9425     0.0521     -201.83      0.9054\n",
            "=================================================================================================================================\n"
          ]
        }
      ],
      "source": [
        "mod = RandomEffects(df.Total_income, exog)\n",
        "re_res = mod.fit()\n",
        "print(re_res)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G8Bg0vaEper0"
      },
      "source": [
        "# **First Difference OLS**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lA73PClmpjdp"
      },
      "outputs": [],
      "source": [
        "exo=['Computer_IT,_net_addition_in_year', 'Net_computers_and_IT_systems',\n",
        "       'Net_plant_&_machinery,_computers_and_electrical_installations',\n",
        "       'Net_software', 'Profit_after_tax', 'Total_expenses']\n",
        "exog = df[exo]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Sl6mBcqGpwAN"
      },
      "outputs": [],
      "source": [
        "mod = FirstDifferenceOLS(df.Total_income, exog)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-c0YOczopzIy",
        "outputId": "0a8e6204-7269-43a4-a03e-4a227c947b68"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "                     FirstDifferenceOLS Estimation Summary                      \n",
            "================================================================================\n",
            "Dep. Variable:           Total_income   R-squared:                        0.9234\n",
            "Estimator:         FirstDifferenceOLS   R-squared (Between):              0.9994\n",
            "No. Observations:               52117   R-squared (Within):               0.9879\n",
            "Date:                Thu, Jul 25 2024   R-squared (Overall):              0.9976\n",
            "Time:                        12:38:48   Log-likelihood                -4.944e+05\n",
            "Cov. Estimator:            Unadjusted                                           \n",
            "                                        F-statistic:                   1.048e+05\n",
            "Entities:                        4009   P-value                           0.0000\n",
            "Avg Obs:                       14.000   Distribution:                 F(6,52111)\n",
            "Min Obs:                       14.000                                           \n",
            "Max Obs:                       14.000   F-statistic (robust):          1.048e+05\n",
            "                                        P-value                           0.0000\n",
            "Time periods:                      14   Distribution:                 F(6,52111)\n",
            "Avg Obs:                       4009.0                                           \n",
            "Min Obs:                       4009.0                                           \n",
            "Max Obs:                       4009.0                                           \n",
            "                                                                                \n",
            "                                                       Parameter Estimates                                                       \n",
            "=================================================================================================================================\n",
            "                                                               Parameter  Std. Err.     T-stat    P-value    Lower CI    Upper CI\n",
            "---------------------------------------------------------------------------------------------------------------------------------\n",
            "Computer_IT,_net_addition_in_year                                -0.0888     0.0524    -1.6951     0.0901     -0.1915      0.0139\n",
            "Net_computers_and_IT_systems                                      0.3811     0.1444     2.6398     0.0083      0.0981      0.6640\n",
            "Net_plant_&_machinery,_computers_and_electrical_installations     0.0068     0.0025     2.6943     0.0071      0.0018      0.0117\n",
            "Net_software                                                      0.1266     0.1365     0.9274     0.3537     -0.1409      0.3941\n",
            "Profit_after_tax                                                  0.9849     0.0030     323.75     0.0000      0.9790      0.9909\n",
            "Total_expenses                                                    0.9663     0.0013     746.87     0.0000      0.9637      0.9688\n",
            "=================================================================================================================================\n"
          ]
        }
      ],
      "source": [
        "fd_res = mod.fit()\n",
        "print(fd_res)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KbgbbxbYp5Lx",
        "outputId": "486b64d5-6a3e-416b-c994-f7d2f4515486"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "                                                              Model Comparison                                                              \n",
            "============================================================================================================================================\n",
            "                                                                             FE_TI            FE_TE                RE                     FD\n",
            "--------------------------------------------------------------------------------------------------------------------------------------------\n",
            "Dep. Variable                                                         Total_income     Total_income      Total_income           Total_income\n",
            "Estimator                                                                 PanelOLS         PanelOLS     RandomEffects     FirstDifferenceOLS\n",
            "No. Observations                                                             56126            56126             56126                  52117\n",
            "Cov. Est.                                                               Unadjusted       Unadjusted        Unadjusted             Unadjusted\n",
            "R-squared                                                                   0.9885           0.9881            0.9973                 0.9234\n",
            "R-Squared (Within)                                                          0.9885           0.9885            0.9885                 0.9879\n",
            "R-Squared (Between)                                                         0.9997           0.9997            0.9997                 0.9994\n",
            "R-Squared (Overall)                                                         0.9978           0.9978            0.9979                 0.9976\n",
            "F-statistic                                                              2.366e+05        7.223e+05         1.095e+06              1.048e+05\n",
            "P-value (F-stat)                                                            0.0000           0.0000            0.0000                 0.0000\n",
            "===============================================================     ==============   ==============   ===============   ====================\n",
            "const                                                                       94.703          -7.7341            118.13                       \n",
            "                                                                          (2.5924)        (-0.6829)          (3.1856)                       \n",
            "Computer_IT,_net_addition_in_year                                          -0.0743          -0.0743           -0.0980                -0.0888\n",
            "                                                                         (-1.4471)        (-1.4471)         (-2.0374)              (-1.6951)\n",
            "Net_computers_and_IT_systems                                                0.2438           0.2438            0.4955                 0.3811\n",
            "                                                                          (3.8816)         (3.8816)          (12.741)               (2.6398)\n",
            "Net_plant_&_machinery,_computers_and_electrical_installations               0.0026           0.0026            0.0111                 0.0068\n",
            "                                                                          (2.6141)         (2.6141)          (19.461)               (2.6943)\n",
            "Net_software                                                                0.1066           0.1066            0.3513                 0.1266\n",
            "                                                                          (1.3072)         (1.3072)          (5.8066)               (0.9274)\n",
            "Profit_after_tax                                                            0.9896           0.9896            1.0014                 0.9849\n",
            "                                                                          (446.62)         (446.62)          (593.22)               (323.75)\n",
            "Total_expenses                                                              0.9946           0.9946            0.9854                 0.9663\n",
            "                                                                          (1768.2)         (1768.2)          (3571.0)               (746.87)\n",
            "Year.2011                                                                  -51.665                            -48.739                       \n",
            "                                                                         (-1.0045)                          (-0.9441)                       \n",
            "Year.2012                                                                  -103.58                            -92.942                       \n",
            "                                                                         (-2.0137)                          (-1.8002)                       \n",
            "Year.2013                                                                  -143.42                            -127.92                       \n",
            "                                                                         (-2.7880)                          (-2.4777)                       \n",
            "Year.2014                                                                  -69.502                            -50.547                       \n",
            "                                                                         (-1.3508)                          (-0.9790)                       \n",
            "Year.2015                                                                  -123.76                            -100.67                       \n",
            "                                                                         (-2.4047)                          (-1.9497)                       \n",
            "Year.2016                                                                  -81.481                            -57.647                       \n",
            "                                                                         (-1.5827)                          (-1.1163)                       \n",
            "Year.2017                                                                  -123.85                            -94.903                       \n",
            "                                                                         (-2.4046)                          (-1.8376)                       \n",
            "Year.2018                                                                  -111.41                            -77.422                       \n",
            "                                                                         (-2.1621)                          (-1.4990)                       \n",
            "Year.2019                                                                  -140.18                            -92.727                       \n",
            "                                                                         (-2.7177)                          (-1.7950)                       \n",
            "Year.2020                                                                  -160.05                            -111.48                       \n",
            "                                                                         (-3.1027)                          (-2.1578)                       \n",
            "Year.2021                                                                  -9.1878                             32.526                       \n",
            "                                                                         (-0.1782)                           (0.6297)                       \n",
            "Year.2022                                                                  -140.45                            -83.294                       \n",
            "                                                                         (-2.7165)                          (-1.6116)                       \n",
            "Year.2023                                                                  -175.58                            -100.46                       \n",
            "                                                                         (-3.3858)                          (-1.9425)                       \n",
            "================================================================= ================ ================ ================= ======================\n",
            "Effects                                                                     Entity           Entity                                         \n",
            "                                                                                               Time                                         \n",
            "--------------------------------------------------------------------------------------------------------------------------------------------\n",
            "\n",
            "T-stats reported in parentheses\n"
          ]
        }
      ],
      "source": [
        "print(compare({\"FE_TI\": fe_res_ti, \"FE_TE\": fe_res_te, \"RE\": re_res, \"FD\":fd_res}))"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}