notes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run -n main.py\n",
    "USER = input('login')\n",
    "PASS = get_pass('pass')\n",
    "AUTH = (USER, PASS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "q = ' OR '.join(\n",
    "    template.format(name)\n",
    "    for name in ['corus']  # natasha, yargy, slovnet, razdel, navec, corus, ipymarkup\n",
    "    for template in ['\"from {}\"', '\"import {}\"']\n",
    ")\n",
    "q = gh_q(\n",
    "    q,\n",
    "    extensions=('py', 'ipynb'),\n",
    ")\n",
    "url = gh_search_code_url(q)\n",
    "response, data = call_gh(url, AUTH, TEXT_MATCH)\n",
    "total = parse_serp_total(data)\n",
    "print('Total:', total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "serp = []\n",
    "pages = list(get_pages(min(total, 1000)))\n",
    "while pages:\n",
    "    page = pages.pop(0)\n",
    "    print('Page #:', page, 'To do:', len(pages))\n",
    "    url = gh_search_code_url(q, page=page)\n",
    "    response, data = call_gh(url, AUTH, headers=TEXT_MATCH)\n",
    "    if is_broken(data):\n",
    "        print('Ban')\n",
    "        pages.insert(0, page)\n",
    "        sleep(60)\n",
    "    else:\n",
    "        serp.extend(parse_serp(data))\n",
    "        sleep(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cache = set(load_lines(URLS))\n",
    "urls = {get_serp_record_url(_) for _ in serp}\n",
    "print('In cache:', len(cache & urls), 'New:', len(urls - cache))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "pattern = '(natasha|yargy|slovnet|razdel|navec|corus|ipymarkup)'\n",
    "for record in serp:\n",
    "    url = get_serp_record_url(record)\n",
    "    if url in cache:\n",
    "        continue\n",
    "    print(url)\n",
    "    for text in record.matches:\n",
    "        spans = get_spans(text, pattern)\n",
    "        show_markup(text, spans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dump_lines(sorted(cache | urls), URLS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}