1_text-stats.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Stats\n",
    "\n",
    "## Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import statements\n",
    "\n",
    "from functions.basic import read_file\n",
    "from functions.processing import split_sents\n",
    "from functions.stats import get_stats\n",
    "\n",
    "from functions.vis import write_vis, box_plot, scatter_plot\n",
    "import plotly.graph_objects as go\n",
    "import plotly.express as px\n",
    "\n",
    "from scipy import stats\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read data\n",
    "\n",
    "j_all = read_file(\"data\\\\0_extraction_data\\\\j_all.pkl\")\n",
    "k_all = read_file(\"data\\\\0_extraction_data\\\\k_all.pkl\")\n",
    "\n",
    "#subsets\n",
    "j_cited = j_all.loc[j_all['passage_type'] == \"cited\"] \n",
    "j_not_cited = j_all.loc[j_all['passage_type'] == \"not_cited\"] \n",
    "k_cited = k_all.loc[k_all['passage_type'] == \"cited\"] \n",
    "k_not_cited = k_all.loc[k_all['passage_type'] == \"not_cited\"]\n",
    "\n",
    "#define indizes to merge dataframes later on\n",
    "j_cited_index = j_cited.index\n",
    "j_not_cited_index = j_not_cited.index\n",
    "k_cited_index = k_cited.index\n",
    "k_not_cited_index = k_not_cited.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# text processing\n",
    "# this might take a while (should be less than 1 min.)\n",
    "\n",
    "#subsets \n",
    "\n",
    "#k cited\n",
    "k_c_text = [x for x in k_cited[\"text\"]]\n",
    "k_c_citnum = [x for x in k_cited[\"frequency\"]]\n",
    "k_c_sents = split_sents(k_c_text)\n",
    "\n",
    "#k not cited\n",
    "k_nc_text = [x for x in k_not_cited[\"text\"]]\n",
    "k_nc_citnum = [x for x in k_not_cited[\"frequency\"]]\n",
    "k_nc_sents = split_sents(k_nc_text)\n",
    "\n",
    "#j cited\n",
    "j_c_text = [x for x in j_cited[\"text\"]]\n",
    "j_c_citnum = [x for x in j_cited[\"frequency\"]]\n",
    "j_c_sents = split_sents(j_c_text)\n",
    "\n",
    "#j not cited\n",
    "j_nc_text = [x for x in j_not_cited[\"text\"]]\n",
    "j_nc_citnum = [x for x in j_not_cited[\"frequency\"]]\n",
    "j_nc_sents = split_sents(j_nc_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Textual Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply text stats and summary statistics\n",
    "\n",
    "j_c_stats, j_c_sumstats = get_stats(j_c_text, j_c_sents, j_c_citnum)\n",
    "j_nc_stats, j_nc_sumstats = get_stats(j_nc_text, j_nc_sents, j_nc_citnum)\n",
    "k_c_stats, k_c_sumstats = get_stats(k_c_text, k_c_sents, k_c_citnum)\n",
    "k_nc_stats, k_nc_sumstats = get_stats(k_nc_text, k_nc_sents, k_nc_citnum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "k_nc_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# append all stats data to dataframes and save them\n",
    "\n",
    "# J cited\n",
    "j_c_stats.index = j_cited_index\n",
    "j_cited_1 = pd.concat([j_cited, j_c_stats], axis=1)\n",
    "\n",
    "# J not cited\n",
    "j_nc_stats.index = j_not_cited_index\n",
    "j_not_cited_1 = pd.concat([j_not_cited, j_nc_stats], axis=1)\n",
    "\n",
    "# K cited\n",
    "k_c_stats.index = k_cited_index\n",
    "k_cited_1 = pd.concat([k_cited, k_c_stats], axis=1)\n",
    "\n",
    "# K not cited\n",
    "k_nc_stats.index = k_not_cited_index\n",
    "k_not_cited_1 = pd.concat([k_not_cited, k_nc_stats], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save data for the next step\n",
    "\n",
    "new_dfs = [j_cited_1, j_not_cited_1, k_cited_1, k_not_cited_1]\n",
    "paths = [\"data\\\\1_text-stats_data\\\\j_cited_1.pkl\", \"data\\\\1_text-stats_data\\\\j_not_cited_1.pkl\", \"data\\\\1_text-stats_data\\\\k_cited_1.pkl\", \"data\\\\1_text-stats_data\\\\k_not_cited_1.pkl\"]\n",
    "\n",
    "for df, path in zip(new_dfs, paths):\n",
    "    df.to_pickle(path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bar plot, number of cited/not cited passages\n",
    "\n",
    "#get overall length of all subsets\n",
    "j_c_length = j_c_stats.char_len.sum()\n",
    "k_c_length = k_c_stats.char_len.sum()\n",
    "j_nc_length = j_nc_stats.char_len.sum()\n",
    "k_nc_length = k_nc_stats.char_len.sum()\n",
    "\n",
    "# plot them\n",
    "fig = go.Figure(data=[\n",
    "    go.Bar(name='cited', x=[\"J\", \"K\"], y=[j_c_length, k_c_length], marker_color=\"#509F98\", opacity=.7),\n",
    "    go.Bar(name='not cited', x=[\"J\", \"K\"], y=[j_nc_length, k_nc_length], marker_color=\"#C16152\", opacity=.7)\n",
    "])\n",
    "fig.update_layout(barmode='group', template=\"plotly_dark\", font=dict(family = \"CMU Serif\", size=20))\n",
    "fig.show()\n",
    "\n",
    "#write_vis([\"pdf\", \"png\", \"html\"],\"all_charlen_bar\", fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create box_plots for all text stats variables in one step\n",
    "\n",
    "cols_listed = list(j_c_stats) #list of all col names to loop through\n",
    "for i in cols_listed:\n",
    "    if i != \"id\":\n",
    "        cols = [j_c_stats[i], j_nc_stats[i], k_c_stats[i], k_nc_stats[i]]\n",
    "        names = [\"J (C)\", \"J (NC)\",\"K (C)\", \"K (NC)\"]\n",
    "        box = box_plot(cols, names, i)\n",
    "\n",
    "    #write_vis([\"pdf\", \"png\", \"html\"],\"all_\" + i + \"_box\", box)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scatter plot, comparing cited/not cited passages by their frequency and over time\n",
    "\n",
    "fig = px.scatter(j_all, x=\"startpos\", y=\"passage_type\",color=\"frequency\", color_continuous_scale=[\"#C16152\", \"#509F98\", \"#1a3331\"], category_orders={\"passage_type\": [\"cited\", \"not_cited\"]},opacity=.6)\n",
    "fig.update_traces(marker=dict(size=((j_all.endpos-j_all.startpos)*0.05)))\n",
    "fig.update_layout(template=\"plotly_dark\", font=dict(family = \"CMU Serif\", size=20))\n",
    "fig.show()\n",
    "#write_vis([\"pdf\", \"png\", \"html\"],\"j_all_scatter\", fig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scatter plots, to compare relationships\n",
    "\n",
    "j_all_stats = j_c_stats.append(j_nc_stats)\n",
    "k_all_stats = k_c_stats.append(k_nc_stats)\n",
    "all_stats = j_all_stats.append(k_all_stats)\n",
    "print(all_stats)\n",
    "\n",
    "scatter = scatter_plot(all_stats, \"J & K\", \"token_len\", \"sent_len\")\n",
    "#write_vis([\"pdf\"],\"all_scatter_token-len_sent-len\", scatter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# in addition, get correlation values using scipy.stats\n",
    "\n",
    "corr = stats.pearsonr(all_stats.token_len, all_stats.cit_num)\n",
    "for i in corr:\n",
    "    print(round(i, 2))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "df9a18be808e588d487c38d3f3e8ec685bd0e8cad3f7cb7ebe63d95a7f7d79f3"
  },
  "kernelspec": {
   "display_name": "Python 3.7.9 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}