all_level_tagger.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"all_level_tagger.ipynb","provenance":[],"collapsed_sections":[],"mount_file_id":"1tBnltcCpz5mn84yfUUSlK7OcXBu53zFi","authorship_tag":"ABX9TyNdQPY+cjVjx+xZBxYrJYcR"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"yS1OR1z3SRZY","colab_type":"code","colab":{}},"source":["import spacy\n","import numpy as np\n","import os\n","import glob\n","import string\n","import re\n","from gensim.models import Word2Vec\n","from gensim.models import FastText\n","import pandas as pd"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"jgDhE4feRv0n","colab_type":"code","colab":{}},"source":["df=pd.read_csv('/content/drive/My Drive/All Spacy/word_pos.csv')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"85v9Kt-jkJ_h","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":407},"executionInfo":{"status":"ok","timestamp":1593065893386,"user_tz":-360,"elapsed":1211,"user":{"displayName":"Tanvir Ahmmed","photoUrl":"","userId":"06405482776379779141"}},"outputId":"812e3a17-c2cf-41aa-bf89-b109301396e0"},"source":["df.head(2000)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Word</th>\n","      <th>POS</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>বাংলায়</td>\n","      <td>noun</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>সুন্দরবন</td>\n","      <td>noun</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>বনভূমি</td>\n","      <td>noun</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>জন্মায়</td>\n","      <td>verb</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>হয়তো</td>\n","      <td>adv</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>1995</th>\n","      <td>অমরাবতী</td>\n","      <td>noun</td>\n","    </tr>\n","    <tr>\n","      <th>1996</th>\n","      <td>অমরালয়</td>\n","      <td>noun</td>\n","    </tr>\n","    <tr>\n","      <th>1997</th>\n","      <td>অমরেশ</td>\n","      <td>noun</td>\n","    </tr>\n","    <tr>\n","      <th>1998</th>\n","      <td>অমর্ত্য</td>\n","      <td>adv</td>\n","    </tr>\n","    <tr>\n","      <th>1999</th>\n","      <td>অমর্ত্যলোক</td>\n","      <td>noun</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>2000 rows × 2 columns</p>\n","</div>"],"text/plain":["            Word   POS\n","0        বাংলায়  noun\n","1       সুন্দরবন  noun\n","2         বনভূমি  noun\n","3        জন্মায়  verb\n","4          হয়তো   adv\n","...          ...   ...\n","1995     অমরাবতী  noun\n","1996      অমরালয়  noun\n","1997       অমরেশ  noun\n","1998     অমর্ত্য   adv\n","1999  অমর্ত্যলোক  noun\n","\n","[2000 rows x 2 columns]"]},"metadata":{"tags":[]},"execution_count":3}]},{"cell_type":"code","metadata":{"id":"3yLtVNcfSio3","colab_type":"code","colab":{}},"source":["word = []\n","pos = []\n","assign = {}"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"atHFEVmnSUnB","colab_type":"code","colab":{}},"source":["for i in df['Word']:\n","  word.append(i)\n","for i in df['POS']:\n","  pos.append(i)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"mo-ZijcEj6oj","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":54},"executionInfo":{"status":"ok","timestamp":1593065897192,"user_tz":-360,"elapsed":1155,"user":{"displayName":"Tanvir Ahmmed","photoUrl":"","userId":"06405482776379779141"}},"outputId":"4eab85d1-f7e2-44e2-947a-e65b199328cc"},"source":["np.unique(pos)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array(['\\n\\t', '\\n\\t ', '\\n\\t  ', '\\n    ', 'adj', 'adv', 'conj', 'nan',\n","       'noun', 'pron', 'verb'], dtype='<U5')"]},"metadata":{"tags":[]},"execution_count":6}]},{"cell_type":"code","metadata":{"id":"T0V_72zTkBm1","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1593065897757,"user_tz":-360,"elapsed":1134,"user":{"displayName":"Tanvir Ahmmed","photoUrl":"","userId":"06405482776379779141"}},"outputId":"eea861d8-2c5d-4753-9caa-610006c789e0"},"source":["np.unique(word)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array(['nan', 'অ', 'অংশ', ..., 'হয়েছেন', 'হয়েন', 'হয়ো'], dtype='<U23')"]},"metadata":{"tags":[]},"execution_count":7}]},{"cell_type":"code","metadata":{"id":"pQnmPo8um4FZ","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1593065119584,"user_tz":-360,"elapsed":1162,"user":{"displayName":"Tanvir Ahmmed","photoUrl":"","userId":"06405482776379779141"}},"outputId":"46639c2d-07ee-48ec-8f1c-7f762716d47f"},"source":["print(len(pos), ' ',len(word))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["112483   112483\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"clAEhhbzk3YR","colab_type":"code","colab":{}},"source":["temp_pos = pos\n","temp_word = word"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"QT-j4w6ckSsu","colab_type":"code","colab":{}},"source":["for i in range(len(pos)):\n","  #if str(pos[i]) != '\\n\\t' or pos[i] != '\\n\\t ' or pos[i] != '\\n\\t  ' or pos[i] != '\\n    ' or str(pos[i]) != 'nan':\n","  try:\n","    if str(pos[i]) == '\\n\\t' or pos[i] == '\\n\\t ' or pos[i] == '\\n\\t  ' or pos[i] == '\\n    ' or str(pos[i]) == str('nan'):\n","      del temp_pos[i]\n","      del temp_word[i]\n","      #temp_pos.append(pos[i])\n","      #temp_word.append(word[i])\n","  except:\n","    continue"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"aQsAS44-lAm8","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":54},"executionInfo":{"status":"ok","timestamp":1593065902280,"user_tz":-360,"elapsed":859,"user":{"displayName":"Tanvir Ahmmed","photoUrl":"","userId":"06405482776379779141"}},"outputId":"20e47a84-dac9-4f71-ddbe-175d9f65cf9f"},"source":["print(np.unique(temp_word),'\\n',np.unique(temp_pos))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["['nan' 'অ' 'অংশ' ... 'হয়েছেন' 'হয়েন' 'হয়ো'] \n"," ['adj' 'adv' 'conj' 'nan' 'noun' 'pron' 'verb']\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"XKD6B3TmpkUw","colab_type":"code","colab":{}},"source":["adj = ADJ == JJ\n","adv = ADV = RBR\n","conj = CCONJ == CC\n","noun = NOUN == NN\n","pron = PRON == PRP\n","verb = VERB == VBZ\n","unknown = XX ==\tX"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ewGGXY_LqeG6","colab_type":"code","colab":{}},"source":["pos_tuple={}\n","pos_tuple['adj'] = \"JJ\"\n","pos_tuple['adv'] = \"RBR\"\n","pos_tuple['conj'] = \"CC\"\n","pos_tuple['noun'] = \"NN\"\n","pos_tuple['pron'] = \"PRP\"\n","pos_tuple['verb'] = \"VBZ\"\n","pos_tuple[''] = \"X\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"MNpjLTgqrCNU","colab_type":"code","colab":{}},"source":["TAG_MAP = {\"NN\": {\"pos\": \"NOUN\"}, \"VBZ\": {\"pos\": \"VERB\"}, \"RBR\": {\"pos\": \"ADV\"}, \"PRP\": {\"pos\": \"PRON\"}, \"CC\": {\"pos\": \"CCONJ\"}, \"JJ\": {\"pos\": \"ADJ\"}, \"X\": {\"pos\": \"XX\"}}"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"zS2Lkn6am9QS","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1593065907278,"user_tz":-360,"elapsed":775,"user":{"displayName":"Tanvir Ahmmed","photoUrl":"","userId":"06405482776379779141"}},"outputId":"947918a4-aa71-49c5-9727-1ff6517cac5e"},"source":["print(len(temp_pos), ' ',len(temp_word))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["112425   112425\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"M5ZgL22FnfK7","colab_type":"code","colab":{}},"source":["for i in range(len(temp_word)):\n","  assign[temp_word[i]] = temp_pos[i]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"wEDk04LyoPJ8","colab_type":"code","colab":{}},"source":["df=pd.read_csv('/content/drive/My Drive/Google Colab/Bangla_Taging/Data/all_line_update.csv')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"HN8lAWExoRck","colab_type":"code","colab":{}},"source":["line1 = []\n","for i in df['0']:\n","  line1.append(i)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Zd4P8MVSoXhf","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1593065912759,"user_tz":-360,"elapsed":1495,"user":{"displayName":"Tanvir Ahmmed","photoUrl":"","userId":"06405482776379779141"}},"outputId":"c90ff0cb-f290-4c6c-cb4a-2f121d7244c3"},"source":["line1[12]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'উন্নয়নে ঈর্ষান্বিত হয়ে স্বৈরতন্ত্রের তকমা আ লীগ'"]},"metadata":{"tags":[]},"execution_count":17}]},{"cell_type":"code","metadata":{"id":"pqLoqa5ioXef","colab_type":"code","colab":{}},"source":["i = 0\n","final_sent_list = []\n","for word in range(0,5001):\n","  j = 0\n","  for sent in line1:\n","    snt=str(sent)\n","    temp_sent = snt.split()\n","    if temp_word[word] in temp_sent:\n","      j += 1\n","      final_sent_list.append(snt)\n","      if j==1:\n","        break"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"02AzFrlBnD2Q","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1593059279874,"user_tz":-360,"elapsed":1176,"user":{"displayName":"Tanvir Ahmmed","photoUrl":"","userId":"06405482776379779141"}},"outputId":"514f874f-1b61-49c8-8827-36eb73b50a97"},"source":["len(final_sent_list)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["530"]},"metadata":{"tags":[]},"execution_count":48}]},{"cell_type":"code","metadata":{"id":"Hvd7AXbLx4rH","colab_type":"code","colab":{}},"source":["df = pd.DataFrame(final_sent_list)\n","csv_data = df.to_csv(index=False)\n","df.to_csv('/content/drive/My Drive/All Spacy/tagger_sent.csv', index=False)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"obXjLx1BwFDm","colab_type":"code","colab":{}},"source":["len(final_sent_list)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"s1edM8So-npl","colab_type":"code","colab":{}},"source":["def make_data():\n","  for l in range(0,15):\n","  #for l in range(len(sentence)):\n","  #for l in range(0,10000):\n","    print(l)\n","    try:\n","      stri = line[l]\n","      index_info = []\n","      mp={}\n","      for i in range(0,1001):\n","        mp[i]=0\n","      for i in range(0,5001):\n","          pos=[]\n","          f=0\n","          pos.append(0)\n","          s=stri[0]\n","          for j in range(len(stri)):\n","            if stri[j] == ' ':\n","              pos.append(j)\n","          for j in pos:\n","            for k in range(j+1,len(stri)):\n","              s=s+stri[k]\n","              if s == temp_word[i]:\n","                temp_str = s\n","                #print(s,' ',i)\n","                ind=0\n","                for comma in range(k+1,len(stri)):\n","                  if stri[comma] == ' ':\n","                    ind=comma\n","                    break\n","                  else: s=s+stri[comma]\n","                if ind == 0: ind=len(stri)\n","                temp_st=str(\"\")\n","                if j==0 and stri[0]!=' ':\n","                  if mp[j] == 0 and mp[ind] == 0:\n","                    print(stri,' ',s,' ',temp_str)\n","                    #print(s,' ',j,' ',ind)\n","                    assign[s] = assign[temp_str]\n","                  for mapping in range(j,ind+1):\n","                    mp[mapping]=1\n","                else:\n","                  if mp[j+1] == 0 and mp[ind] == 0:\n","                    print(stri,' ',s,' ',temp_str)\n","                    assign[s] = assign[temp_str]\n","                  for mapping in range(j+1,ind+1):\n","                    mp[mapping]=1\n","                f=1\n","                break\n","              elif len(s) == len(temp_word[i]):\n","                s=\"\"\n","                break\n","            if f == 1:\n","              s=\"\"\n","              f=0\n","    except:\n","      continue "],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"a9-Cn_rijnAu","colab_type":"code","colab":{}},"source":["def make_data():\n","  #tag_list = []\n","  for l in range(0,5005):\n","  #for l in range(len(sentence)):\n","  #for l in range(0,10000):\n","    \n","    stri = line1[l]\n","    \n","    s = \"\"\n","    try:\n","      for j in range(0,5001):\n","        for i in range(len(stri)):\n","          #print(stri)\n","          if stri[i] != ' ':\n","            s = s + stri[i]\n","          if s == temp_word[j]:\n","            tmp_s = s\n","            for k in range(i+1,len(stri)):\n","              if stri[k] == ' ':\n","                break\n","              tmp_s = tmp_s+stri[k]\n","            #tag_list.append(assign[s])\n","            assign[tmp_s] = assign[s] \n","            print(stri,' ',s,' ',tmp_s)\n","            s = \"\"\n","          if stri[i] == ' ' and s != temp_word[j]:\n","            s = \"\"\n","    except:\n","      continue"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"doQzfvfGuqrf","colab_type":"code","colab":{}},"source":["make_data()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"JM4tVQ9Tq9yx","colab_type":"code","colab":{}},"source":["train_data = []\n","for i in range(0,5001):\n","  pos_list = []\n","  temp = line1[i].split()\n","  for j in temp:\n","    try:\n","      if j in temp_word:\n","        pos_list.append(pos_tuple[assign[j]]) \n","      else:\n","        pos_list.append(\"X\")\n","    except:\n","      pos_list.append(\"X\")\n","      continue\n","  aa = (line1[i], {\"tags\": pos_list})\n","  train_data.append(aa)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"OfkHkOmsotPh","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1593066239868,"user_tz":-360,"elapsed":26206,"user":{"displayName":"Tanvir Ahmmed","photoUrl":"","userId":"06405482776379779141"}},"outputId":"d277060e-578c-4a16-ea6c-032481b9870b"},"source":["len(train_data)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["5001"]},"metadata":{"tags":[]},"execution_count":23}]},{"cell_type":"code","metadata":{"id":"AzejbIJBvyGS","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":490},"executionInfo":{"status":"ok","timestamp":1593066239869,"user_tz":-360,"elapsed":26195,"user":{"displayName":"Tanvir Ahmmed","photoUrl":"","userId":"06405482776379779141"}},"outputId":"396368b3-a477-4a52-fd5e-72493e35b0e1"},"source":["train_data[:15]"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[('এসএমই খাতে ঋণ প্রদানের আহ্বান গভর্নরের',\n","  {'tags': ['X', 'X', 'NN', 'X', 'NN', 'X']}),\n"," ('এজলাসে হাস্যোজ্জ্বল আসিফ', {'tags': ['X', 'X', 'X']}),\n"," (' বছরেও হয়নি নিজস্ব ভবন ভাড়াও বাকি',\n","  {'tags': ['X', 'X', 'RBR', 'NN', 'X', 'RBR']}),\n"," (' সমুদ্রজয় নিয়ে সরকারকে বির্তকে ডাকলেন ফখরুল',\n","  {'tags': ['X', 'VBZ', 'X', 'X', 'VBZ', 'X']}),\n"," ('কৃষিশিক্ষা', {'tags': ['NN']}),\n"," ('মেয়র পদপ্রার্থীর প্রচারণায় বাধা ও হয়রানির অভিযোগ',\n","  {'tags': ['X', 'X', 'X', 'VBZ', 'JJ', 'X', 'X']}),\n"," ('কোনো অস্ত্র প্রতিযোগিতা নয় নেটো',\n","  {'tags': ['PRP', 'VBZ', 'NN', 'VBZ', 'X']}),\n"," ('চট্টগ্রামে আজ', {'tags': ['X', 'JJ']}),\n"," ('শিশু সাঈদ হত্যা মামলার রায় কাল',\n","  {'tags': ['RBR', 'X', 'NN', 'X', 'RBR', 'RBR']}),\n"," (' সাঁওতালদের ওপর হামলায় সাংসদ ও চেয়ারম্যান জড়িত ',\n","  {'tags': ['X', 'JJ', 'X', 'X', 'JJ', 'NN', 'RBR']}),\n"," ('ব্রহ্মপুত্রের দুর্গম চরে জঙ্গি আস্তানা ',\n","  {'tags': ['X', 'RBR', 'VBZ', 'X', 'NN']}),\n"," ('সংঘর্ষ', {'tags': ['NN']}),\n"," ('উন্নয়নে ঈর্ষান্বিত হয়ে স্বৈরতন্ত্রের তকমা আ লীগ',\n","  {'tags': ['X', 'RBR', 'VBZ', 'X', 'NN', 'X', 'NN']}),\n"," ('উৎখাত নয় বঙ্গবন্ধু পরিবার হত্যাই ছিল লক্ষ্য আনিসুল',\n","  {'tags': ['X', 'VBZ', 'X', 'VBZ', 'X', 'VBZ', 'RBR', 'X']}),\n"," ('জিয়ার মৃত্যুবার্ষিকীর অনুষ্ঠানে হামলা অভিযোগ ছাত্রলীগের দিকে',\n","  {'tags': ['X', 'X', 'X', 'NN', 'X', 'X', 'X']})]"]},"metadata":{"tags":[]},"execution_count":24}]},{"cell_type":"code","metadata":{"id":"RcNTZDrN5Amz","colab_type":"code","colab":{}},"source":["TRAIN_DATA = train_data"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"wiiQDEeXKVtD","colab_type":"code","colab":{}},"source":["TRAIN_DATA = [('মাছ মাংসসহ অধিকাংশ পণ্যের দাম চড়া',\n","  {'tags': ['NN', 'X', 'RBR', 'X', 'NN', 'VBZ']}),]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ZMGDndLyawaL","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"status":"error","timestamp":1593066516196,"user_tz":-360,"elapsed":3586,"user":{"displayName":"Tanvir Ahmmed","photoUrl":"","userId":"06405482776379779141"}},"outputId":"dda87530-0807-4c5f-fbee-23194b808e68"},"source":["#!/usr/bin/env python\n","# coding: utf8\n","\"\"\"\n","A simple example for training a part-of-speech tagger with a custom tag map.\n","To allow us to update the tag map with our custom one, this example starts off\n","with a blank Language class and modifies its defaults. For more details, see\n","the documentation:\n","* Training: https://spacy.io/usage/training\n","* POS Tagging: https://spacy.io/usage/linguistic-features#pos-tagging\n","Compatible with: spaCy v2.0.0+\n","Last tested with: v2.1.0\n","\"\"\"\n","from __future__ import unicode_literals, print_function\n","\n","import plac\n","import random\n","from pathlib import Path\n","import spacy\n","from spacy.util import minibatch, compounding\n","output_dir='/content/drive/My Drive/All Spacy/all_tagger'\n","\n","# You need to define a mapping from your data's part-of-speech tag names to the\n","# Universal Part-of-Speech tag set, as spaCy includes an enum of these tags.\n","# See here for the Universal Tag Set:\n","# http://universaldependencies.github.io/docs/u/pos/index.html\n","# You may also specify morphological features for your tags, from the universal\n","# scheme.\n","#TAG_MAP = {\"N\": {\"pos\": \"NOUN\"}, \"VB\": {\"pos\": \"VERB\"}, \"RB\": {\"pos\": \"ADV\"}, \"PRP\": {\"pos\": \"PRON\"}, \"CC\": {\"pos\": \"CCONJ\"}, \"JJ\": {\"pos\": \"ADJ\"}}\n","TAG_MAP = {\"NN\": {\"pos\": \"NOUN\"}, \"VBZ\": {\"pos\": \"VERB\"}, \"RBR\": {\"pos\": \"ADV\"}, \"PRP\": {\"pos\": \"PRON\"}, \"CC\": {\"pos\": \"CCONJ\"}, \"JJ\": {\"pos\": \"ADJ\"}, \"X\": {\"pos\": \"XX\"}}\n","\n","# Usually you'll read this in, of course. Data formats vary. Ensure your\n","# strings are unicode and that the number of tags assigned matches spaCy's\n","# tokenization. If not, you can always add a 'words' key to the annotations\n","# that specifies the gold-standard tokenization, e.g.:\n","# (\"Eatblueham\", {'words': ['Eat', 'blue', 'ham'], 'tags': ['V', 'J', 'N']})\n","'''\n","TRAIN_DATA = [\n","    (\"আমি নাবানা গ্রুপ তৈরি করব\", {\"tags\": [\"N\", \"N\", \"N\", \"V\", \"V\"]}),\n","    (\"আমি আকিজ গ্রুপের মালিক\", {\"tags\": [\"N\", \"N\", \"N\", \"N\"]}),\n","    (\"বাংলাদেশের সাংবিধানিক নাম গণপ্রজাতন্ত্রী বাংলাদেশ\", {\"tags\": [\"N\", \"N\", \"N\", \"N\", \"N\", ]}),\n","    (\"ভৌগলিকভাবে বাংলাদেশের পশ্চিম উত্তর ও পূর্ব সীমান্তে ভারত ও দক্ষিণ সীমান্তে মায়ানমার অবস্থিত\", {\"tags\": [\"N\", \"N\", \"N\", \"N\", \"N\", \"N\", \"N\", \"N\", \"N\", \"N\", \"N\", \"N\", \"N\", ]}),\n","    #(\"Eat blue ham\", {\"tags\": [\"V\", \"J\", \"N\"]}),\n","]\n","'''\n","\n","\n","@plac.annotations(\n","    lang=(\"ISO Code of language to use\", \"option\", \"l\", str),\n","    output_dir=(\"Optional output directory\", \"option\", \"o\", Path),\n","    n_iter=(\"Number of training iterations\", \"option\", \"n\", int),\n",")\n","def main1(lang, output_dir, n_iter):\n","    \"\"\"Create a new model, set up the pipeline and train the tagger. In order to\n","    train the tagger with a custom tag map, we're creating a new Language\n","    instance with a custom vocab.\n","    \"\"\"\n","\n","    #n_iter=25\n","    nlp = spacy.blank(lang)\n","    # add the tagger to the pipeline\n","    # nlp.create_pipe works for built-ins that are registered with spaCy\n","    tagger = nlp.create_pipe(\"tagger\")\n","    # Add the tags. This needs to be done before you start training.\n","    for tag, values in TAG_MAP.items():\n","        tagger.add_label(tag, values)\n","    nlp.add_pipe(tagger)\n","\n","    optimizer = nlp.begin_training()\n","    for i in range(n_iter):\n","        random.shuffle(TRAIN_DATA)\n","        losses = {}\n","        # batch up the examples using spaCy's minibatch\n","        batches = minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001))\n","        for batch in batches:\n","            texts, annotations = zip(*batch)\n","            nlp.update(texts, annotations, sgd=optimizer, losses=losses)\n","        print(\"Losses\", losses)\n","\n","    # test the trained model\n","    test_text = \"গণসংগ্রামের হয়েছে যার ধারাবাহিকতা অদ্যাবধি বর্তমান\"\n","    doc = nlp(test_text)\n","    print(\"Tags\", [(t.text, t.tag_, t.pos_) for t in doc])\n","\n","    # save model to output directory\n","    if output_dir is not None:\n","        output_dir = Path(output_dir)\n","        if not output_dir.exists():\n","            output_dir.mkdir()\n","        nlp.to_disk(output_dir)\n","        print(\"Saved model to\", output_dir)\n","\n","        # test the save model\n","        print(\"Loading from\", output_dir)\n","        nlp2 = spacy.load(output_dir)\n","        doc = nlp2(test_text)\n","        print(\"Tags\", [(t.text, t.tag_, t.pos_) for t in doc])\n","\n","main1(\"bn\",output_dir,100)\n","#if __name__ == \"__main__\":\n","    #plac.call(main)\n","\n","    # Expected output:\n","    # [\n","    #   ('I', 'N', 'NOUN'),\n","    #   ('like', 'V', 'VERB'),\n","    #   ('blue', 'J', 'ADJ'),\n","    #   ('eggs', 'N', 'NOUN')\n","    # ]"],"execution_count":null,"outputs":[{"output_type":"stream","text":["/usr/lib/python3.6/runpy.py:193: UserWarning: [W022] Training a new part-of-speech tagger using a model with no lemmatization rules or data. This means that the trained model may not be able to lemmatize correctly. If this is intentional or the language you're using doesn't have lemmatization data, you can ignore this warning by setting SPACY_WARNING_IGNORE=W022. If this is surprising, make sure you have the spacy-lookups-data package installed.\n","  \"__main__\", mod_spec)\n"],"name":"stderr"},{"output_type":"stream","text":["Losses {'tagger': 5.911764144897461}\n","Losses {'tagger': 5.869943141937256}\n","Losses {'tagger': 5.753739833831787}\n","Losses {'tagger': 5.529888153076172}\n","Losses {'tagger': 5.1157732009887695}\n","Losses {'tagger': 4.415450096130371}\n","Losses {'tagger': 3.4391300678253174}\n","Losses {'tagger': 2.3848373889923096}\n","Losses {'tagger': 1.4581477642059326}\n","Losses {'tagger': 0.7734574675559998}\n","Losses {'tagger': 0.3578754961490631}\n","Losses {'tagger': 0.15062350034713745}\n","Losses {'tagger': 0.059641171246767044}\n","Losses {'tagger': 0.022910429164767265}\n","Losses {'tagger': 0.008708316832780838}\n","Losses {'tagger': 0.0032888264395296574}\n","Losses {'tagger': 0.0012417593970894814}\n","Losses {'tagger': 0.0004711601650342345}\n","Losses {'tagger': 0.00017945931176654994}\n","Losses {'tagger': 6.98830554028973e-05}\n","Losses {'tagger': 2.8320142519078217e-05}\n","Losses {'tagger': 1.2055462320859078e-05}\n","Losses {'tagger': 5.43062651559012e-06}\n","Losses {'tagger': 2.5868921511573717e-06}\n","Losses {'tagger': 1.3060126775599201e-06}\n","Losses {'tagger': 6.973290282985545e-07}\n","Losses {'tagger': 3.918780180356407e-07}\n","Losses {'tagger': 2.3053975439779606e-07}\n","Losses {'tagger': 1.4206248977188807e-07}\n","Losses {'tagger': 9.111899856861783e-08}\n","Losses {'tagger': 6.043116229648149e-08}\n","Losses {'tagger': 4.180970591960431e-08}\n","Losses {'tagger': 2.9678549395839582e-08}\n","Losses {'tagger': 2.16665014818318e-08}\n","Losses {'tagger': 1.6241518352444473e-08}\n","Losses {'tagger': 1.251029946303106e-08}\n","Losses {'tagger': 9.885194884873272e-09}\n","Losses {'tagger': 7.97292187826315e-09}\n","Losses {'tagger': 6.4894623008626695e-09}\n","Losses {'tagger': 5.437741368297111e-09}\n","Losses {'tagger': 4.623323945907032e-09}\n","Losses {'tagger': 3.974180984300801e-09}\n","Losses {'tagger': 3.472521381553406e-09}\n","Losses {'tagger': 3.052841979211962e-09}\n","Losses {'tagger': 2.695512035799652e-09}\n","Losses {'tagger': 2.4119841679492993e-09}\n","Losses {'tagger': 2.1557884366529834e-09}\n","Losses {'tagger': 1.9878083623581233e-09}\n","Losses {'tagger': 1.8314343375180897e-09}\n","Losses {'tagger': 1.7004608832138501e-09}\n","Losses {'tagger': 1.5787464668903795e-09}\n","Losses {'tagger': 1.491913037554582e-09}\n","Losses {'tagger': 1.4161971595427758e-09}\n","Losses {'tagger': 1.3327534631457638e-09}\n","Losses {'tagger': 1.2719156838869594e-09}\n","Losses {'tagger': 1.2257082016020604e-09}\n","Losses {'tagger': 1.1769110130899207e-09}\n","Losses {'tagger': 1.1371219521549847e-09}\n","Losses {'tagger': 1.0892036161891383e-09}\n","Losses {'tagger': 1.0553571350158109e-09}\n","Losses {'tagger': 1.023992557414033e-09}\n","Losses {'tagger': 1.0006921957739223e-09}\n","Losses {'tagger': 9.66561053417081e-10}\n","Losses {'tagger': 9.441274428922952e-10}\n","Losses {'tagger': 9.291579727843668e-10}\n","Losses {'tagger': 9.082062324416995e-10}\n","Losses {'tagger': 8.903748849320436e-10}\n","Losses {'tagger': 8.778087146055213e-10}\n","Losses {'tagger': 8.643710192046683e-10}\n","Losses {'tagger': 8.463635348121556e-10}\n","Losses {'tagger': 8.392825323610964e-10}\n","Losses {'tagger': 8.242844740102839e-10}\n","Losses {'tagger': 8.22687751256268e-10}\n","Losses {'tagger': 8.113266725118251e-10}\n","Losses {'tagger': 8.003032125891707e-10}\n","Losses {'tagger': 7.899882970008321e-10}\n","Losses {'tagger': 7.840660343205741e-10}\n","Losses {'tagger': 7.797926193653382e-10}\n","Losses {'tagger': 7.719997974220405e-10}\n","Losses {'tagger': 7.629474829684568e-10}\n","Losses {'tagger': 7.593343731571167e-10}\n","Losses {'tagger': 7.517157452063827e-10}\n","Losses {'tagger': 7.488408781952671e-10}\n","Losses {'tagger': 7.404742929928432e-10}\n","Losses {'tagger': 7.36531835521248e-10}\n","Losses {'tagger': 7.356920628254215e-10}\n","Losses {'tagger': 7.303243565459638e-10}\n","Losses {'tagger': 7.264692181152554e-10}\n","Losses {'tagger': 7.256903411523297e-10}\n","Losses {'tagger': 7.193329265575699e-10}\n","Losses {'tagger': 7.109942634642152e-10}\n","Losses {'tagger': 7.03417102343451e-10}\n","Losses {'tagger': 7.00182800628113e-10}\n","Losses {'tagger': 6.977650124362356e-10}\n","Losses {'tagger': 6.926219597858108e-10}\n","Losses {'tagger': 6.712945199716103e-10}\n","Losses {'tagger': 6.681583064604979e-10}\n","Losses {'tagger': 6.627198789743716e-10}\n","Losses {'tagger': 6.595543000642579e-10}\n","Losses {'tagger': 6.560597620719477e-10}\n"],"name":"stdout"},{"output_type":"error","ename":"ValueError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)","\u001b[0;32m<ipython-input-30-1715fe9886b3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     96\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Tags\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpos_\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdoc\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 98\u001b[0;31m \u001b[0mmain1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"bn\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0moutput_dir\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;31m#if __name__ == \"__main__\":\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m     \u001b[0;31m#plac.call(main)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m<ipython-input-30-1715fe9886b3>\u001b[0m in \u001b[0;36mmain1\u001b[0;34m(lang, output_dir, n_iter)\u001b[0m\n\u001b[1;32m     79\u001b[0m     \u001b[0;31m# test the trained model\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m     \u001b[0mtest_text\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"গণসংগ্রামের হয়েছে যার ধারাবাহিকতা অদ্যাবধি বর্তমান\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m     \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnlp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_text\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     82\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Tags\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpos_\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdoc\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.6/dist-packages/spacy/language.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, text, disable, component_cfg)\u001b[0m\n\u001b[1;32m    437\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"__call__\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    438\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mE003\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcomponent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 439\u001b[0;31m             \u001b[0mdoc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mproc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mcomponent_cfg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    440\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mdoc\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    441\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mE005\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32mpipes.pyx\u001b[0m in \u001b[0;36mspacy.pipeline.pipes.Tagger.__call__\u001b[0;34m()\u001b[0m\n","\u001b[0;32mpipes.pyx\u001b[0m in \u001b[0;36mspacy.pipeline.pipes.Tagger.set_annotations\u001b[0;34m()\u001b[0m\n","\u001b[0;32mmorphology.pyx\u001b[0m in \u001b[0;36mspacy.morphology.Morphology.assign_tag_id\u001b[0;34m()\u001b[0m\n","\u001b[0;32mmorphology.pyx\u001b[0m in \u001b[0;36mspacy.morphology.Morphology.add\u001b[0;34m()\u001b[0m\n","\u001b[0;31mValueError\u001b[0m: [E167] Unknown morphological feature: 'Person' (2313063860588076218). This can happen if the tagger was trained with a different set of morphological features. If you're using a pretrained model, make sure that your models are up to date:\npython -m spacy validate"]}]},{"cell_type":"code","metadata":{"id":"birdpzsq3tdR","colab_type":"code","colab":{}},"source":["nn"],"execution_count":null,"outputs":[]}]}