query_db.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\hp\\AppData\\Local\\pypoetry\\Cache\\virtualenvs\\llm-data-generator-vX9nrdqr-py3.10\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModel, AutoTokenizer \n",
    "from qdrant_client import QdrantClient\n",
    "from dotenv import dotenv_values\n",
    "import yaml\n",
    "\n",
    "DBPATH = \"data/db/\"\n",
    "with open(\"config.yml\") as yml:\n",
    "    conf = yaml.safe_load(yml)\n",
    "ENV = dotenv_values(\".env\")\n",
    "MODEL = \"sts/paraphrase-multilingual-MiniLM-L12-v2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = QdrantClient(url = ENV[\"QDRANT_URL\"], location=DBPATH, timeout=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load model and tokenizer..\n",
    "tokenizer = AutoTokenizer.from_pretrained(MODEL)\n",
    "model = AutoModel.from_pretrained(MODEL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = \"كرة قدم\"\n",
    "inputs = tokenizer(\n",
    "    text=query,\n",
    "    padding=True,\n",
    "    truncation=True,\n",
    "    return_tensors=\"pt\",\n",
    "    max_length=conf[\"vector_size\"]\n",
    ")\n",
    "results = model(**inputs)\n",
    "embeddings = results.last_hidden_state[:, 0, :].cpu().detach().numpy()\n",
    "embeddings = embeddings.flatten().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[ScoredPoint(id='f0b6282b-6f0c-20d5-6c60-a8d1510dbfc7', version=10, score=0.656389, payload={'title': 'ما قيمة الكرة الذهبية.. وهل مصنوعة من الذهب الخالص؟'}, vector=None),\n",
       " ScoredPoint(id='34f6f516-d061-16be-6300-b58926f6c17a', version=10, score=0.5459675, payload={'title': 'أغلى نجوم الكرة الذهبية قبل الكشف عن الفائز بنسخة 2023.. إنفو جراف'}, vector=None),\n",
       " ScoredPoint(id='f4f482a2-83b6-18a1-8799-05ee7e17e6b0', version=0, score=0.53724855, payload={'title': 'تولوز يصعق ليفربول 3-2 ومحمد صلاح ينجو من إصابة قوية فى الدوري الأوروبي'}, vector=None)]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client.search(\n",
    "    collection_name=\"prod\",\n",
    "    query_vector=embeddings,\n",
    "    with_payload=[\"title\"],\n",
    "    limit=3\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[ScoredPoint(id='6b3a77d4-887c-0a06-0713-3354d54f8c96', version=7, score=0.60518914, payload={'title': 'أحد الجرحى الفلسطينيين بمستشفى العريش العام: أحظى برعاية خاصة فى مصر.. فيديو'}, vector=None),\n",
       " ScoredPoint(id='3563fa4b-ccf4-24f3-71bd-51674512ec92', version=4, score=0.57786846, payload={'title': 'وزيرة الصحة الفلسطينية: نحتاج إلى مستشفيات ميدانية في غزة'}, vector=None),\n",
       " ScoredPoint(id='bfe1d453-d38d-1efc-9bb9-9a56b55651e4', version=2, score=0.5717842, payload={'title': 'وزير الدفاع الإسرائيلي: إسرائيل \"ستحتفظ بحرية العمل الكاملة\" في غزة بعد انتهاء الحرب'}, vector=None)]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"فلسطين\"\n",
    "inputs = tokenizer(\n",
    "    text=query,\n",
    "    padding=True,\n",
    "    truncation=True,\n",
    "    return_tensors=\"pt\",\n",
    "    max_length=conf[\"vector_size\"]\n",
    ")\n",
    "results = model(**inputs)\n",
    "embeddings = results.last_hidden_state[:, 0, :].cpu().detach().numpy()\n",
    "embeddings = embeddings.flatten().tolist()\n",
    "\n",
    "client.search(\n",
    "    collection_name=\"prod\",\n",
    "    query_vector=embeddings,\n",
    "    with_payload=[\"title\"],\n",
    "    limit=3\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[ScoredPoint(id='76dadd0e-4c10-cefe-bcca-b4f8810d9742', version=6, score=0.7872704, payload={'title': 'حظك اليوم وتوقعات الأبراج السبت 4 نوفمبر على الصعيد المهنى والعاطفى والصحى'}, vector=None),\n",
       " ScoredPoint(id='d4fdd71d-69b3-8b2f-4e65-75e527c45e78', version=1, score=0.78645945, payload={'title': 'أبو عبيدة يكشف خسائر الاحتلال بالمدرعات... ويؤكد: لا حل لملف الأسرى إلا بالتبادل'}, vector=None),\n",
       " ScoredPoint(id='8caa37af-2849-7f04-c58a-05e957e5f553', version=7, score=0.6133511, payload={'title': 'برج الحمل.. حظك اليوم الجمعة 3 نوفمبر: بداية جديدة'}, vector=None)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"حظك اليوم\"\n",
    "inputs = tokenizer(\n",
    "    text=query,\n",
    "    padding=True,\n",
    "    truncation=True,\n",
    "    return_tensors=\"pt\",\n",
    "    max_length=conf[\"vector_size\"]\n",
    ")\n",
    "results = model(**inputs)\n",
    "embeddings = results.last_hidden_state[:, 0, :].cpu().detach().numpy()\n",
    "embeddings = embeddings.flatten().tolist()\n",
    "\n",
    "client.search(\n",
    "    collection_name=\"prod\",\n",
    "    query_vector=embeddings,\n",
    "    with_payload=[\"title\"],\n",
    "    limit=3\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[ScoredPoint(id='010f5d24-3131-426a-921a-63398c916b9e', version=7, score=0.47930846, payload={'title': 'لماذا قرر البنك المركزى الإبقاء على تثبيت سعر الفائدة دون تغيير؟'}, vector=None),\n",
       " ScoredPoint(id='1237d46d-e08f-0a9e-ca5c-1e5e317df123', version=11, score=0.4693601, payload={'title': 'سعر الذهب اليوم في مصر بالتعاملات المسائية'}, vector=None),\n",
       " ScoredPoint(id='e9528e71-44a5-d4a1-7367-cc643fc6eff7', version=8, score=0.43027937, payload={'title': 'بعد تثبيت الفائدة الأمريكية.. تعرف على أسعار الذهب في البورصة العالمية'}, vector=None)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"أسعار العملات\"\n",
    "inputs = tokenizer(\n",
    "    text=query,\n",
    "    padding=True,\n",
    "    truncation=True,\n",
    "    return_tensors=\"pt\",\n",
    "    max_length=conf[\"vector_size\"]\n",
    ")\n",
    "results = model(**inputs)\n",
    "embeddings = results.last_hidden_state[:, 0, :].cpu().detach().numpy()\n",
    "embeddings = embeddings.flatten().tolist()\n",
    "\n",
    "client.search(\n",
    "    collection_name=\"prod\",\n",
    "    query_vector=embeddings,\n",
    "    with_payload=[\"title\"],\n",
    "    limit=3\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llm-data-generator-vX9nrdqr-py3.10",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}