named_entities script.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "12ef96c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"PERSON\": [\"UN Secretary General\"], \"ORG\": [\"UNGA77\"], \"GPE\": [], \"EVENTS\": [\"The Common Agenda\", \"Summit of the Future\", \"Global Digital Compact\"], \"DATE\": [\"September 2024\"], \"LAW\": []}\n",
      "[('PERSON', 'UN Secretary General'), ('ORG', 'UNGA77'), ('EVENTS', 'The Common Agenda'), ('EVENTS', 'Summit of the Future'), ('EVENTS', 'Global Digital Compact'), ('DATE', 'September 2024')]\n"
     ]
    }
   ],
   "source": [
    "# Used try method to check if the response of chatgpt is a dictionary format\n",
    "# Used spacy to extract NER in case the chat gpt response are not in dictionary format\n",
    "# Also used spacy if chat gpt returns and an empty list for all named entities \n",
    "# for ex 'PERSON': [], 'ORG': [], 'GPE': [], 'EVENTS': [], 'DATE': [], 'LAW': []\n",
    "# as I encountered a case once.\n",
    "\n",
    "\n",
    "from llama_index.llms import OpenAI as LlamaOpenAI\n",
    "import spacy\n",
    "import ast\n",
    "\n",
    "def perform_spacy_ner(input_string):\n",
    "    # Initialize spaCy NER model\n",
    "    NER = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "    # Process the input string with spaCy NER\n",
    "    ner = NER(input_string)\n",
    "    exclude_entities = [ \"TIME\", \"PERCENT\", \"MONEY\", \"QUANTITY\", \"ORDINAL\", \"CARDINAL\"]\n",
    "\n",
    "    named_entities=[]\n",
    "    for ent in ner.ents:\n",
    "        if ent.label_ not in exclude_entities:\n",
    "            tup=(ent.label_, ent.text )\n",
    "            named_entities.append(tup)\n",
    "    \n",
    "    return named_entities\n",
    "\n",
    "\n",
    "def response_parse(data: dict) -> list:\n",
    "    \"\"\"\n",
    "    Takes in the dictionary from OpenAI Response.\n",
    "    Note - Each Key in this dictionary is a list of word.\n",
    "    Output - List of tuples, Each tuple is a entity and word pair.\n",
    "    \"\"\"\n",
    "    entities = []\n",
    "\n",
    "    # Check if all lists in the dictionary are empty\n",
    "    if all(not data[key] for key in data):\n",
    "        return perform_spacy_ner(data)\n",
    "    \n",
    "    return [(key, item) if data[key] is not None else (key, None) for key in data for item in data[key]]\n",
    "\n",
    "\n",
    "def run_llama_ner_task(input_text:str, labels, model:str='gpt-3.5'):\n",
    "    llm=LlamaOpenAI(temperature=0, model=model)\n",
    "    response= llm.complete(\n",
    "        f\"\"\"You are an expert in Natural Language Processing. Your task is to identify common Named Entities (NER) in a given text: {input_text}.\n",
    "        The possible common Named Entities (NER) types are exclusively: ({\", \".join(labels)}).\n",
    "        --------\n",
    "        Follow the Entity defination as below:\n",
    "        1. PERSON: Short name or full name of a person from any geographic regions.\n",
    "        2. ORG: Organizations, Companies, Agencies or Institutions.\n",
    "        3. GPE: Geopolitical entities like name of countries, continents, cities, states, district.\n",
    "        4. EVENTS: Scientific milestones, historical events, future events.\n",
    "        5. DATE: Any format of dates. Dates can also be in natural language.\n",
    "        6. LAW : Mention of any article, treaty, law, agenda, roadmap.\n",
    "        --------\n",
    "        Output Format:\n",
    "        {{\"PERSON\": [list of entities present], \"ORG\": [list of entities present], \"GPE\": [list of entities present], \"EVENTS\": [list of entities present], \"DATE\": [list of entities present], \"LAW\": [list of entities present]}}\n",
    "        If no entities are presented in any categories keep it an empty list.\n",
    "        Your output response MUST only contain a Dictionary object as specified in output format. If it is not in the format strip the string until you encounter a dictionary in the format specified.\\n\n",
    "        --------\n",
    "        EXAMPLE:\n",
    "        Text: 'To this end, internet restrictions should be considered not merely as technical measures affecting the logical or physical layers of the networks, but also as a tool used to impair the means of expression of individuals (see below, Avoid internet fragmentation). Therefore, internet shutdowns and content blocking may constitute violations of the principles contained in several international human rights instruments, in particular article 19 (right to freedom of expression, the scope of which is clarified by General Comment 34 of the Human Rights Committee) and article 21 (right of assembly) of the International Covenant on Civil and Political Rights (ICCPR). Regional conventions for the protection of human rights generally include similar provisions.\n",
    "        This perspective is supported by the practice of several organs and international institutions, notably resolutions adopted by the Human Rights Council on 'The promotion, protection and fulfillment of human rights on the Internet' (A/HRC/32/L.20, June 27th, 2016). The Council condemns measures that seek to ' intentionally prevent or disrupt access to or dissemination of information online in violation of international human rights law, and calls upon all States to refrain from and cease such measures ', relying on a substantial number of secondary legislation not specifically related to the issue of Internet access. Several UN Special Rapporteurs have recognized the fundamental nature of the right of access to the Internet through the prism of freedom of expression or the right to peaceful assembly. However, this right of access to the internet should not be construed as a duty of States to provide access without any charge.'\n",
    "        {{\"PERSON\": [], \"ORG\": [\"Human Rights Committee\", \"International Covenant on Civil and Political Rights (ICCPR)\", \"Human Rights Council\", \"UN\", \"Special Rapporteurs\"], \"GPE\": [\"States\"],  \"EVENTS\": [], \"DATE\": [\"June 27th, 2016\"], \"LAW\": [\"article 19 (right to freedom of expression\", \"article 21 (right of assembly)\", \"A/HRC/32/L.20\", \"international human rights law\"] }}\n",
    "        \n",
    "        \"\"\"\n",
    "    )\n",
    "    return response\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "\n",
    "    from dotenv import load_dotenv, find_dotenv\n",
    "\n",
    "    # needs an openai api key\n",
    "    load_dotenv(find_dotenv())\n",
    "\n",
    "    input_text = \"\"\"\n",
    "        \n",
    "    Under The Common Agenda, the UN Secretary General called for a Global Digital Compact to be agreed at the Summit of the Future (September 2024) to “outline \n",
    "    shared principles for an open, free and secure digital future for all.” Ensuring that \n",
    "    women’s rights organizations in all their diversity feed into this process is critical to \n",
    "    shape a vision of digital cooperation and governance based on human rights and \n",
    "    feminist principles.  In his report to UNGA77, on the Intensification of efforts to eliminate all forms of violence against women and girls, the Secretary General proposed a high level consultation on violence against women and girls in digital \n",
    "    contexts to inform the Global Digital Compact.\n",
    "        \"\"\"\n",
    "\n",
    "    LABELS = [\n",
    "        \"PERSON\",   \n",
    "        \"ORG\",         \n",
    "        \"GPE\",        \n",
    "        \"EVENTS\",     \n",
    "        \"DATE\",      \n",
    "        \"LAW\",        \n",
    "    ]\n",
    "\n",
    "    NER_openai_llama=run_llama_ner_task(input_text, LABELS, \"gpt-4-1106-preview\")\n",
    "    \n",
    "    try:\n",
    "        extracted_dict = ast.literal_eval(NER_openai_llama.text)\n",
    "        \n",
    "        if isinstance(extracted_dict, dict):\n",
    "            result= response_parse(extracted_dict)\n",
    "            \n",
    "        else:\n",
    "            raise ValueError(\"Could not extract a valid dictionary using eval.\")\n",
    "    except (SyntaxError, ValueError) as e:\n",
    "        print(f\"Error with eval: {e}\")\n",
    "        result= perform_spacy_ner(input_string)\n",
    "    \n",
    "    print(NER_openai_llama)\n",
    "\n",
    "    print(result)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chat_applications",
   "language": "python",
   "name": "chat_applications"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}