Yemen News Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "import os\n",
    "import nltk.corpus\n",
    "\n",
    "#DOWNLOAD once\n",
    "#nltk.download('punkt')\n",
    "#nltk.download('wordnet')\n",
    "#nltk.download('stopwords')\n",
    "#nltk.download('words')\n",
    "#nltk.download('averaged_perceptron_tagger')\n",
    "#nltk.download('maxent_ne_chunker')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "#RETRIEVE text\n",
    "#library https://newspaper.readthedocs.io/en/latest/user_guide/quickstart.html\n",
    "from newspaper import Article\n",
    "url = 'https://www.nrc.no/news/2020/september/yemen-900-airstrike-and-shelling-hits-on-farms-in-three-years/'\n",
    "article = Article(url)\n",
    "article.download()\n",
    "article.parse()\n",
    "bod = article.text\n",
    "bodOneString = bod.replace(\"\\n\",\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6, 13, 15]\n"
     ]
    }
   ],
   "source": [
    "#CHECK for governorates with p-codes\n",
    "pUrl = 'https://www.humanitarianresponse.info/sites/www.humanitarianresponse.info/files/documents/files/yemen_pcodes_2015_08_29.xlsx'\n",
    "data = pd.read_excel(pUrl, sheet_name=\"Governorates\",usecols=[0,1],skiprows=[0])\n",
    "govinbodIndex = []\n",
    "govcodes = data[\"Governorate Pcode\"]\n",
    "govnames = data[\"Governorate English\"]\n",
    "#print(data)\n",
    "for key, value in govnames.iteritems():\n",
    "    if value in bodOneString:\n",
    "        govinbodIndex.append(key)\n",
    "print(govinbodIndex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['system', 'give', 'council', 'crisis', 'be', 'shatter', 'show', 'total', 'gun', 'hajjah', 'safe', 'functional', 'restore', 'water', 'bombing', 'also', 'currency', 'time', 'risk', 'week', 'well', 'child', 'nations', 'attack', 'gulf', 'country', 'part', 'keep', 'refugee', 'onion', 'mill', 'yemeni', 'settlement', 'feature', 'family', 'baby', 'get', 'secretary', 'senseless', 'far', 'farmer', 'side', 'refer', 'village', 'few', 'many', 'happen', 'produce', 'destroy', 'crowd', 'flee', 'marib', 'yeman', 'lifeline', 'bomb', 'power', 'dependent', 'abdullah', 'abyss', 'interview', 'state', 'government', 'military', 'underfund', 'programme', 'item', 'starvation', 'higher', 'hassan', 'irrigation', 'ceasefire', 'neighbouring', 'political', 'home', 'civilian', 'no', 'month', 'vegetable', 'gunfire', 'fighting', 'field', 'father', 'wife', 'labourer', 'scale', 'member', 'funding', 'conflict', 'involve', 'organisation', 'generation', 'case', 'used', 'jan', 'long', 'displace', 'glad', 'decimate', 'event', 'step', 'dark', 'reach', 'abandon', 'sunk', 'document', 'only', 'feed', 'other', 'striking', 'buy', 'donor', 'uk', 'impact', 'immediate', 'lain', 'worker', 'fishing', 'last', 'land', 'ability', 'muhsen', 'norwegian', 'name', 'monitor', 'western', 'market', 'violence', 'tomato', 'camp', 'moham', 'day', 'war', 'here', 'back', 'world', 'ali', 'project', 'barren', 'real', 'iran', 'senior', 'united', 'not', 'uae', 'goat', 'most', 'arabia', 'release', 'basic', 'as', 'famine', 'authority', 'include', 'erupt', 'yemenis', 'food', 'now', 'cimp', 'fear', 'january', 'work', 'person', 'hit', 'farm', 'leave', 'cent', 'old', 'stay', 'response', 'general', 'support', 'flour', 'rest', 'push', 'mahyoub', 'sheep', 'left', 'shell', 'incessant', 'shelling', 'almost', 'nrc', 'help', 'multiple', 'still', 'end', 'airstrike', 'district', 'larger', 'plant', 'say', 'kill', 'earlier', 'make', 'price', 'affect', 'incident', 'have', 'year', 'man', 'average', 'pattern', 'see', 'think', 'ground', 'lifesave', 'amend', 'provide', 'tent', 'injured', 'saana', 'engulf', 'barley', 'partially', 'address', 'number', 'hut', 'anymore', 'eat', 'low', 'wheat', 'life', 'concrete', 'sell', 'nowadays', 'saudi', 'just', 'recent', 'abs', 'datum', 'shelter', 'aden', 'live', 'fall', 'city', 'tree', 'level', 'cousin', 'corn', 'collect', 'grow', 'area', 'slashed', 'ahm', 'front', 'visit', 'mahmoud', 'record', 'burnt', 'france', 'desert', 'loss', 'million', 'aid', 'cut', 'drive', 'inherit', 'go', 'boat', 'property']\n"
     ]
    }
   ],
   "source": [
    "#LEMMING (skipping steps: remove punctuation, stop words, uppercase)\n",
    "from gensim.utils import lemmatize \n",
    "\n",
    "lemm = set([word.decode('utf-8').split('/')[0] for word in lemmatize(bodOneString)]) \n",
    "lemmatized = list(lemm)\n",
    "\n",
    "print(lemmatized) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ruin', 'demolish', 'destroy', 'destruct', 'put_down'}\n"
     ]
    }
   ],
   "source": [
    "#SYNONYMS for indication terms\n",
    "from nltk.corpus import wordnet \n",
    "\n",
    "#destroy\n",
    "destroysyn = []   \n",
    "for syn in wordnet.synsets(\"destroy\"): \n",
    "    for l in syn.lemmas(): \n",
    "        destroysyn.append(l.name()) \n",
    "print(set(destroysyn))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['destroy']\n"
     ]
    }
   ],
   "source": [
    "#FIND indication terms\n",
    "\n",
    "#level 1\n",
    "levelonewords = destroysyn.copy() + ['raze','burn','damage']\n",
    "levelonematch = []\n",
    "for term in levelonewords:\n",
    "    if term in lemmatized and term not in levelonematch:\n",
    "        levelonematch.append(term)   \n",
    "\n",
    "print(levelonematch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['village', 'market']\n"
     ]
    }
   ],
   "source": [
    "#FIND infrastructure terms\n",
    "infr = ['building','village','house','school','hospital','market','road','bridge']\n",
    "infrmatch = []\n",
    "\n",
    "for term in infr:\n",
    "    if term in lemmatized and term not in infrmatch:\n",
    "        infrmatch.append(term)\n",
    "        \n",
    "print(infrmatch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Yemen p-codes & cities: \n",
      "17 Hajjah\n",
      "24 Aden\n",
      "26 Marib\n",
      "\n",
      "Indication terms: destroy \n",
      "Infrastructure affected: village market \n",
      "Infrastructure damage intensity (0-1): 1\n"
     ]
    }
   ],
   "source": [
    "#OUTPUT\n",
    "if len(govinbodIndex) != 0 and len(levelonematch) != 0 and len(infrmatch) != 0:\n",
    "    print(\"Yemen p-codes & cities: \")\n",
    "    for i in govinbodIndex:\n",
    "        print(govcodes[i], end=\" \")\n",
    "        print(govnames[i])\n",
    "\n",
    "    print(\"\\nIndication terms: \",end=\"\")\n",
    "    for x in range(len(levelonematch)):\n",
    "        print(levelonematch[x], end=\" \")\n",
    "\n",
    "    print(\"\\nInfrastructure affected: \",end=\"\")\n",
    "    for x in range(len(infrmatch)):\n",
    "        print(infrmatch[x], end=\" \")\n",
    "\n",
    "    print(\"\\nInfrastructure damage intensity (0-1): \", end=\"\")\n",
    "    if len(infrmatch) != 0:\n",
    "        print(1)\n",
    "else:\n",
    "    print(\"Infrastructure damage intensity (0-1): 0\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}