ANERCorp_Data_Cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec\n",
    "word2vec_model = '/home/yan/eventData/website/wiki_arabic_prebuild/wiki_ar_word2vec.model'\n",
    "prebuilt = Word2Vec.load(word2vec_model)\n",
    "vocab_set = set(prebuilt.vocab.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trainingpath='/home/yan/eventData/website/aqmar_trainingdata/ANERCorp'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "newpath='/home/yan/eventData/website/aqmar_trainingdata/ANERCorp_conll_new.ner'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(trainingpath,encoding=\"utf-8\") as f:\n",
    "    content = f.readlines()\n",
    "#content = [x.strip() for x in content] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "#correct way to write line to a file\n",
    "with open(newpath,'a') as f:\n",
    "    f.write(\"hello!\\n\")\n",
    "    f.write(\"world!\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "150285"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "#strip the character return out\n",
    "st=content[0].strip()\n",
    "index=st.index('B')\n",
    "#strip the space\n",
    "word=st[:index].strip()\n",
    "nertag=st[index:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "substring not found",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-50-176b63a4c48c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"heyhello\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'x'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m: substring not found"
     ]
    }
   ],
   "source": [
    "test=\"heyhello\"\n",
    "test.con"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['فرانكفورت B-LOC\\n',\n",
       " '(د O\\n',\n",
       " 'ب  O\\n',\n",
       " 'أ) O\\n',\n",
       " 'أعلن O\\n',\n",
       " 'اتحاد B-ORG\\n',\n",
       " 'صناعة I-ORG\\n',\n",
       " 'السيارات I-ORG\\n',\n",
       " 'في O\\n',\n",
       " 'ألمانيا B-LOC\\n']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "content[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "#concatenate english and arabic together\n",
    "def arabic(str):\n",
    "    return '\\u202B' + str + '\\u202C';\n",
    "def english(str):\n",
    "    return '\\u202A' + str + '\\u202C';"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " np np O\n",
      "\n",
      " np np O\n",
      "\n",
      " np np O\n",
      "\n",
      " np np O\n",
      "\n",
      " np np O\n",
      "\n",
      "الأمن‏1706 ‏ np np I-ORG\n",
      "\n",
      ". np np Oالإصابة O\n",
      "\n",
      "I np np BM B-ORG\n",
      "\n",
      "I np np BM B-ORG\n",
      "\n",
      "I np np BM B-ORG\n",
      "\n",
      "I np np BM B-ORG\n",
      "\n",
      "I np np BM B-ORG\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "with open(newpath,'a') as f:\n",
    "    for con in content:\n",
    "        #also try to solve the no space problem\n",
    "        st=con.strip();\n",
    "        index=0;\n",
    "        if 'I-ORG' in st:\n",
    "            index=st.index('I');\n",
    "        elif 'B-ORG' in st:\n",
    "            index=st.index('B');\n",
    "        elif 'I-LOC' in st:\n",
    "            index=st.index('I');\n",
    "        elif 'B-LOC' in st:\n",
    "            index=st.index('B');\n",
    "        elif 'I-PERS' in st:\n",
    "            index=st.index('I');\n",
    "        elif 'B-PERS' in st:\n",
    "            index=st.index('B');\n",
    "        elif 'O' in st:\n",
    "            index=st.index('O');\n",
    "        word=st[:index].strip()\n",
    "        nertag=st[index:].strip()\n",
    "        strToWrite=(word+\" np\"+\" np\"+\" \"+nertag+\"\\n\")\n",
    "        #print(strToWrite)\n",
    "        if(len(strToWrite.split())==4):\n",
    "            f.write(strToWrite)\n",
    "        else:\n",
    "            print(strToWrite)\n",
    "            \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['np', 'np', 'في', 'O']"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test=\"np np في O\"\n",
    "test.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def changeDataFormat():\n",
    "    dic={}\n",
    "    index=0;\n",
    "    #I-ORG=1,B-ORG=2,I-LOC=3, B-LOC=4,I-PERS=5,B-PERS=6\n",
    "    for i in content:\n",
    "        if index not in dic:\n",
    "            dic[index]={}\n",
    "        if \"sen\" not in dic[index]:\n",
    "            dic[index][\"sen\"]=\"\";\n",
    "            dic[index][\"tag\"]=[];\n",
    "        #print(i)\n",
    "        line=str(i).split(\" \")\n",
    "        #print(line[0:length-1][0])\n",
    "        length=len(line)\n",
    "        dic[index][\"sen\"]=dic[index][\"sen\"]+(line[0:length-1][0])+\" \"\n",
    "        if(line[length-1]==\"O\\n\"):\n",
    "            dic[index][\"tag\"].append(0)\n",
    "        elif (line[length-1]==\"I-ORG\\n\"):\n",
    "            dic[index][\"tag\"].append(1)\n",
    "        elif (line[length-1]==\"B-ORG\\n\"):\n",
    "            dic[index][\"tag\"].append(2)\n",
    "        elif (line[length-1]==\"I-LOC\\n\"):\n",
    "            dic[index][\"tag\"].append(3)\n",
    "        elif (line[length-1]==\"B-LOC\\n\"):\n",
    "            dic[index][\"tag\"].append(4)\n",
    "        elif (line[length-1]==\"I-PERS\\n\"):\n",
    "            dic[index][\"tag\"].append(5)\n",
    "        elif (line[length-1]==\"B-PERS\\n\"):\n",
    "            dic[index][\"tag\"].append(6)    \n",
    "        if(i[0]==\".\"):\n",
    "            index=index+1\n",
    "            #print(\"end of the sentence\")\n",
    "    return dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dic=changeDataFormat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4938"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sen=dic[1]['sen']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sen=\"فرانكفورت B-LOC\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sen.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test=zip(*[sen.split()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "new_vir",
   "language": "python",
   "name": "new_vir"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}