bibtex_thesis_cleanup.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#imports\n",
    "import os, re, glob, copy\n",
    "from collections import defaultdict\n",
    "\n",
    "try:\n",
    "    import bibtexparser\n",
    "except:\n",
    "    ! conda install -c conda-forge bibtexparse\n",
    "    \n",
    "from bibtexparser.bibdatabase import BibDatabase\n",
    "from bibtexparser.bparser import BibTexParser\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parsing and fixing of general things\n",
    "\n",
    "**Warning:** No content checks and no author name rearrangments! (either you use bibtexparser crossref functionality for that or you do it manually)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RULES\n",
    "more latex signs can be found here:\n",
    "https://de.wikibooks.org/wiki/LaTeX/_Akzente_und_Sonderzeichen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to /home/bschroed/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from reffix.entry_rules import titelcasing_fields, journal_iso4, double_minus, capitalize_keys\n",
    "from reffix.entry_rules import check_misc_fields, check_incollection_fields, check_inproceedings_fields, check_inbook_fields, check_article_fields, check_book_fields\n",
    "from reffix.entry_rules import fixLatexChars\n",
    "from bibtexparser.customization import type as bibtexType, homogenize_latex_encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bibtexparser.bparser import BibTexParser\n",
    "from collections import OrderedDict\n",
    "\n",
    "\n",
    "#feel free to add new rules :) or modify them in the entry_rule file.\n",
    "def titelcasing_journal(record):\n",
    "    if(\"journal\" in record):\n",
    "        record.update({\"journal\": \"{\"+titlecase(record['jounal'])+\"}\"})\n",
    "    return record\n",
    "\n",
    "def build_standardParser():\n",
    "    myParser = BibTexParser()\n",
    "\n",
    "    def rulez(record):\n",
    "        record = bibtexType(record)\n",
    "        \n",
    "        #check entry fields\n",
    "        record = check_article_fields(record)\n",
    "        record = check_book_fields(record)\n",
    "        record = check_incollection_fields(record)\n",
    "        record = check_inproceedings_fields(record)\n",
    "        record = check_inbook_fields(record)\n",
    "\n",
    "        record = titelcasing_fields(record)\n",
    "        record = capitalize_keys(record)\n",
    "        record = homogenize_latex_encoding(record)\n",
    "\n",
    "        #check field values\n",
    "        record = fixLatexChars(record)\n",
    "        record = double_minus(record)\n",
    "        record = journal_iso4(record)\n",
    "\n",
    "        return record\n",
    "\n",
    "    myParser.customization = rulez\n",
    "    return myParser\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  DO "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#PATHs\n",
    "root_dir = \"..\"\n",
    "\n",
    "##INPATHS\n",
    "self_path = root_dir+\"/1_frontback\"\n",
    "chapter1_path = root_dir+\"/2_chapter_intro\"\n",
    "chapter2_path = root_dir+\"/3_chapter_1\"\n",
    "chapter3_path = root_dir+\"/4_chapter_2\"\n",
    "chapter4_path = root_dir+\"/5_chapter_3\"\n",
    "chapter5_path = root_dir+\"/6_chapter_4\"\n",
    "chapter6_path = root_dir+\"/7_chapter_5\"\n",
    "chapter7_path = root_dir+\"/8_chapter_outlook\"\n",
    "\n",
    "chapter_paths = [chapter1_path, chapter2_path, chapter3_path, chapter4_path, chapter5_path, chapter6_path, chapter7_path]\n",
    "\n",
    "    \n",
    "chapters = {\n",
    "    \"chapter_\"+str(i):{\n",
    "        \"path\": chapter_path,\n",
    "        \"bib_path\": chapter_path+\"/ref/ref.bib\",\n",
    "        \"tex_paths\": list(filter(lambda x: not \"fragment\" in x, glob.glob(chapter_path+\"/*tex\")+glob.glob(chapter_path+\"/*/*tex\")))\n",
    "    }\n",
    "    for i, chapter_path in enumerate(chapter_paths) }\n",
    "\n",
    "chapters.update({\n",
    "    \"self\": {\n",
    "        \"path\": self_path,\n",
    "        \"bib_path\": self_path+\"/publications.bib\", \n",
    "    }\n",
    "})\n",
    "\n",
    "##OUTPaths\n",
    "out_dir = \"./fixed_ref\"\n",
    "if(not os.path.exists(out_dir)): os.mkdir(out_dir)\n",
    "out_all_ref = out_dir+\"/mergedReferences.bib\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data & Clean Entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chapter:  chapter_0\n",
      "Filter DB for duplicates\n",
      "\tBefore Elements in DB:  142\n",
      "\tAfter Elements in DB:  142\n",
      "Filter DB for used in TEX\n",
      "\tBefore Elements in DB:  142\n",
      "\tAfter Elements in DB:  140\n",
      "\n",
      "Chapter:  chapter_1\n",
      "Filter DB for duplicates\n",
      "\tBefore Elements in DB:  60\n",
      "\tAfter Elements in DB:  60\n",
      "Filter DB for used in TEX\n",
      "\tBefore Elements in DB:  60\n",
      "\tAfter Elements in DB:  60\n",
      "\n",
      "Chapter:  chapter_2\n",
      "Filter DB for duplicates\n",
      "\tBefore Elements in DB:  89\n",
      "\tAfter Elements in DB:  89\n",
      "Filter DB for used in TEX\n",
      "\tBefore Elements in DB:  89\n",
      "\tAfter Elements in DB:  89\n",
      "\n",
      "Chapter:  chapter_3\n",
      "Filter DB for duplicates\n",
      "\tBefore Elements in DB:  55\n",
      "\tAfter Elements in DB:  55\n",
      "Filter DB for used in TEX\n",
      "\tBefore Elements in DB:  55\n",
      "\tAfter Elements in DB:  55\n",
      "\n",
      "Chapter:  chapter_4\n",
      "Filter DB for duplicates\n",
      "\tBefore Elements in DB:  76\n",
      "\tAfter Elements in DB:  76\n",
      "Filter DB for used in TEX\n",
      "\tBefore Elements in DB:  76\n",
      "\tAfter Elements in DB:  76\n",
      "\n",
      "Chapter:  chapter_5\n",
      "Filter DB for duplicates\n",
      "\tBefore Elements in DB:  78\n",
      "\tAfter Elements in DB:  78\n",
      "Filter DB for used in TEX\n",
      "\tBefore Elements in DB:  78\n",
      "\tAfter Elements in DB:  78\n",
      "\n",
      "Chapter:  chapter_6\n",
      "Filter DB for duplicates\n",
      "\tBefore Elements in DB:  43\n",
      "\tAfter Elements in DB:  43\n",
      "Filter DB for used in TEX\n",
      "\tBefore Elements in DB:  43\n",
      "\tAfter Elements in DB:  43\n",
      "\n",
      "Chapter:  self\n",
      "Filter DB for duplicates\n",
      "\tBefore Elements in DB:  10\n",
      "\tAfter Elements in DB:  10\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from reffix.database_functions import remove_duplicates, get_used_citations, filter_database_for_required_citations\n",
    "\n",
    "for chapter in chapters:\n",
    "    print(\"Chapter: \", chapter, )\n",
    "    chapter_data = chapters[chapter]\n",
    "    #print(chapter_data['bib_path'],  chapter_data['tex_paths'])\n",
    "\n",
    "    bib_file = open(chapter_data['bib_path'], \"r\", encoding=\"utf-8\")\n",
    "    db = bibtexparser.load(bib_file, parser=build_standardParser())\n",
    "    \n",
    "    db = remove_duplicates(db)\n",
    "    \n",
    "    if(\"tex_paths\" in chapter_data):\n",
    "        used_cit = []\n",
    "        for tex_path in chapter_data['tex_paths']:\n",
    "            used_cit.extend(list(set(get_used_citations(tex_path))))\n",
    "        chapters[chapter].update({\"tex_cit\":used_cit})\n",
    "        db = filter_database_for_required_citations(db, used_cit)\n",
    "    \n",
    "    chapters[chapter].update({\"bib\":db})\n",
    "\n",
    "    bib_file.close()\n",
    "    print()\n",
    "\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of citations\n",
      "\t chapter_0 \t\t 140\n",
      "\t chapter_1 \t\t 60\n",
      "\t chapter_2 \t\t 89\n",
      "\t chapter_3 \t\t 55\n",
      "\t chapter_4 \t\t 76\n",
      "\t chapter_5 \t\t 78\n",
      "\t chapter_6 \t\t 43\n",
      "\t self \t\t 10\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of citations\")\n",
    "for chapter, chapter_data in chapters.items():\n",
    "    print(\"\\t\",chapter, \"\\t\\t\", len(chapter_data['bib'].entries_dict))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge & Write out BibTex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chapter_0 ../2_chapter_intro\n",
      "chapter_1 ../3_chapter_1\n",
      "chapter_2 ../4_chapter_2\n",
      "chapter_3 ../5_chapter_3\n",
      "chapter_4 ../6_chapter_4\n",
      "chapter_5 ../7_chapter_5\n",
      "chapter_6 ../8_chapter_outlook\n",
      "self ../1_frontback\n",
      "\n",
      "Clean Merged Citations: \n",
      "Filter DB for duplicates\n",
      "\tBefore Elements in DB:  551\n",
      "\tAfter Elements in DB:  402\n"
     ]
    }
   ],
   "source": [
    "#Merge Lib:\n",
    "combined_db = BibDatabase()\n",
    "\n",
    "##Write out and merge all entries\n",
    "for chapter, chapter_data in chapters.items():\n",
    "    print(chapter, chapter_data['path'])\n",
    "    chapter_out = out_dir+\"/\"+chapter\n",
    "    if(not os.path.exists(chapter_out)): os.mkdir(chapter_out)\n",
    "    \n",
    "    with open(chapter_out+\"/\"+\"ref.bib\", 'w', encoding=\"utf-8\") as bibtex_file:\n",
    "        bibtexparser.dump(chapter_data['bib'], bibtex_file)\n",
    "        \n",
    "    combined_db.entries.extend(chapter_data['bib'].entries)\n",
    "    \n",
    "##Clean Merged lib\n",
    "print(\"\\nClean Merged Citations: \")\n",
    "combined_db = remove_duplicates(combined_db, verbose=False)\n",
    "combined_db.entries = list(combined_db.entries_dict.values())\n",
    "\n",
    "## Write out merged lib\n",
    "with open(out_dir+\"/\"+\"mergedReferences.bib\", 'w', encoding=\"utf-8\") as bibtex_file:\n",
    "    bibtexparser.dump(combined_db, bibtex_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}