ix-jupyter-visualisierung-2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install eurostat geopandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install seaborn==0.11.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget https://github.com/datanizing/ix-jupyter-visualisierung/raw/main/europe.geo.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import eurostat\n",
    "realnames_i = eurostat.get_dic('indic')\n",
    "df = eurostat.get_data_df(\"ei_bsco_m\")\n",
    "df = df.rename(columns={\"geo\\\\time\": \"country\"})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Spalten aufräumen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "df.columns = [datetime.strptime(f.split(\"M\")[0] + \"-\" + f.split(\"M\")[1] + \"-01\", \"%Y-%m-%d\")\n",
    "                if f.startswith(\"20\") or f.startswith(\"19\") else f for f in df.columns]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Korrelationen der Indikatoren miteinander vergleichen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.stats as stats\n",
    "indicators = df[\"indic\"].unique()\n",
    "corr = []\n",
    "for i1 in indicators:\n",
    "    res = []\n",
    "    d1 = df[(df[\"indic\"] == i1) & (df[\"s_adj\"] == \"NSA\") & (df[\"country\"] == \"DE\")].transpose()\n",
    "    for i2 in indicators:\n",
    "        d2 = df[(df[\"indic\"] == i2) & (df[\"s_adj\"] == \"NSA\") & (df[\"country\"] == \"DE\")].transpose()\n",
    "        r, p = stats.pearsonr(d1[[isinstance(d, datetime) and d.year>=2000 for d in d1.index]].iloc[:,0].values, \n",
    "                              d2[[isinstance(d, datetime) and d.year>=2000 for d in d2.index]].iloc[:,0].values)\n",
    "        res.append(r)\n",
    "    corr.append(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y20 = [datetime(y, m, 1) for y in range(2000, 2021) for m in range(1, 13)][:-2]\n",
    "de20 = df[(df[\"s_adj\"] == \"NSA\") & (df[\"country\"] == \"DE\")].set_index(\"indic\")[y20].transpose()\n",
    "de20.index = pd.DatetimeIndex(de20.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "de20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.stats as stats\n",
    "corr = []\n",
    "indicators = de20.columns\n",
    "# Korrelationen berechnen\n",
    "for i1 in indicators:\n",
    "    res = []\n",
    "    for i2 in indicators:\n",
    "        r, p = stats.pearsonr(de20[i1].values, de20[i2].values)\n",
    "        res.append(r)\n",
    "    corr.append(res)\n",
    "\n",
    "# in Dataframe mit richtigen Spalte und Zeilen wandeln\n",
    "real_indicators = [realnames_i[i] for i in indicators]\n",
    "ihm = pd.DataFrame(corr, index=real_indicators, columns=real_indicators)\n",
    "ihm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Heatmaps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "plt.figure(figsize=(12,12))\n",
    "sns.heatmap(ihm, cmap=\"viridis\", vmin=-1, vmax=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,12))\n",
    "sns.heatmap(ihm, cmap=\"RdBu\", vmin=-1, vmax=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scatterplots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "de20.plot.scatter(x=\"BS-SFSH\", y=\"BS-CSMCI\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import scipy.stats as stats\n",
    "sns.jointplot(x=de20[\"BS-SFSH\"], y=de20[\"BS-CSMCI\"], kind=\"reg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "g = sns.jointplot(x=de20[\"BS-SFSH\"], y=de20[\"BS-CSMCI\"], scatter=False, kind=\"reg\")\n",
    "# aktuelle Ersparnisse als Größe der Bubbles\n",
    "# take only first month of year\n",
    "de20s = de20[de20.index.month==1].copy()\n",
    "# take every other year\n",
    "de20s = de20s[::2]\n",
    "# remove M01 from index\n",
    "de20s.index = de20s.index.map(str).str.replace(\"-01-01 00:00:00\", \"\")\n",
    "sns.scatterplot(x=de20s[\"BS-SFSH\"], y=de20s[\"BS-CSMCI\"], s=de20s[\"BS-SV-PR\"],\n",
    "                hue=de20s.index, legend=True)\n",
    "# Detailplot ohne Legende\n",
    "sns.scatterplot(x=de20[\"BS-SFSH\"], y=de20[\"BS-CSMCI\"], s=de20[\"BS-SV-PR\"],\n",
    "                hue=de20.index, legend=False)\n",
    "plt.gca().annotate(\"pearsonr=%0.2f p=%e\"%stats.pearsonr(de20[\"BS-SFSH\"], de20[\"BS-CSMCI\"]), xy=(5, -25))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.stats\n",
    "scipy.stats.linregress(de20[\"BS-SFSH\"], de20[\"BS-CSMCI\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GeoPandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import geopandas\n",
    "bl_geo = geopandas.read_file(\"europe.geo.json\")\n",
    "# die EU verwendet UK als Name, richtig ist aber GB\n",
    "bl_geo.loc[bl_geo[\"iso_a2\"] == \"GB\", \"iso_a2\"] = \"UK\"\n",
    "bl_geo[[\"iso_a2\", \"geometry\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bl_geo[~bl_geo[\"iso_a2\"].isin([\"RU\", \"IS\", \"UA\", \"BY\", \"MD\"])].plot(figsize=(10,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "hm = df[(df[\"indic\"] == \"BS-CSMCI\") & (df[\"s_adj\"] == \"NSA\")]\n",
    "ghm = geopandas.GeoDataFrame(pd.merge(hm, bl_geo, left_on=\"country\", right_on='iso_a2', how=\"outer\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ghm.plot(column=datetime(2020, 8, 1), legend=True, legend_kwds={'orientation': \"horizontal\"}, figsize=(10,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ghm[~ghm[\"iso_a2\"].isin([\"RU\", \"IS\", \"UA\", \"BY\", \"MD\"])].plot(column=datetime(2020, 8, 1), \n",
    "         legend=True, \n",
    "         legend_kwds={'orientation': \"horizontal\"}, \n",
    "         missing_kwds={\n",
    "           \"color\": \"lightgrey\",\n",
    "    },\n",
    "         figsize=(10,10))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}