tls_covid19.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "tls-covid19.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "beHjsXYxY6nH",
        "U9a0GmnoZFHE",
        "GmAjDyvljbW0",
        "8LFqFHy_ZyfM",
        "qAIF-0tfdsPI",
        "ikFM71jRwCM5",
        "wiigCoINjfB3",
        "aUio2zmo6BZR",
        "myp4tEGsjfq6",
        "TAhfRgb5ERdf",
        "tZD9ZpnyEjl6",
        "-WNmrgi1Ee9X",
        "rRDR4K08Ehil",
        "VbMtC0hgDywb",
        "GiLZLn5ZxyBY",
        "Ip5cib5ZxyBa",
        "lNt3kJ81LRHK",
        "3FPbAg_Cj0Tj",
        "TLfg2Bpg8ujT",
        "7rFfc3lO9gfH",
        "Y6NC4xgf_6ZW",
        "wD0b2RG_ARiR",
        "o_VCMaHOATFb",
        "A3PmMv9b4x6D",
        "ZsJgalm33e4L",
        "IEXo4fCR3e4t",
        "1W1zWoQG3e4v",
        "SOYWtRM3oNJe",
        "ICzvgwm53e4w",
        "slqqZG9r3e4x",
        "EqC62npXwBPv",
        "vXHXUimjzKcU",
        "N8t_wXYnzMWJ",
        "eEbVeFPgel2a",
        "wqzBycl9el2j",
        "JYZfDKPUel2k",
        "NmDUfdXEel2k",
        "fsiJvhvcel2k",
        "9On2JT2wel2l",
        "RB0LMHtKel2l",
        "QMoEGFBVel2m",
        "VEI92DRuel2n",
        "kkQKctw-el2n",
        "ZvDMaz7jel2o",
        "dxxB4E7Jel2p",
        "1OXOuRhBpQ6n",
        "5UUq-KNg0Gg6",
        "nd65s7-eqLuf",
        "kJzyqz2lqJai",
        "K-P4d6ZdpKae",
        "eD81AnX86Nlg",
        "mCU7YuJd7TNT",
        "-c-kmyRRtX1U",
        "gBeIYgkXt_y_",
        "n-91lzPZt_zj",
        "mCqqLIUVt_zl",
        "CR0izI6_t_zn",
        "pb6feWi2H-b7",
        "Cfd31UqhGq40"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4FqgpxXzY97p"
      },
      "source": [
        "# TLS-COVID19 Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "beHjsXYxY6nH"
      },
      "source": [
        "## Collect data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U9a0GmnoZFHE"
      },
      "source": [
        "### Requirements"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ApZlVoD9Y0Xd"
      },
      "source": [
        "# Selenium (required to collect CNN news)\n",
        "!apt update\n",
        "!apt install chromium-chromedriver\n",
        "\n",
        "!pip install selenium\n",
        "\n",
        "# Date handling\n",
        "!pip install dateparser"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EPWVuNlDZeFw"
      },
      "source": [
        "# Packages from Python standard library\n",
        "import os\n",
        "import time\n",
        "import re\n",
        "import json\n",
        "import html\n",
        "from datetime import datetime\n",
        "from pathlib import Path\n",
        "from urllib.parse import urlencode\n",
        "\n",
        "# Third-party packages\n",
        "import requests\n",
        "import dateparser\n",
        "from bs4 import BeautifulSoup\n",
        "from selenium import webdriver"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GmAjDyvljbW0"
      },
      "source": [
        "### Sources"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1XYd4I9qZtNt"
      },
      "source": [
        "# The root directory where the datasets will be placed\n",
        "DATA_DIR = 'data/'\n",
        "# Create dir if does not exist\n",
        "Path(DATA_DIR).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "# Used in requests\n",
        "HEADERS = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'}\n",
        "\n",
        "# Functions used across all liveblog collectors\n",
        "\n",
        "def clean_html(text):\n",
        "  text = html.unescape(text)\n",
        "  tags_regex = re.compile('<.*?>')\n",
        "  text = re.sub(tags_regex, '', str(text))\n",
        "  return text\n",
        "\n",
        "def pre_proc(text):\n",
        "  text = clean_html(text)\n",
        "  # Some news contain special characters like \\r, \\t, \\n\n",
        "  text = ' '.join(text.split())\n",
        "  return text\n",
        "\n",
        "# Returns date as string in format yyyy-mm-dd hh:mm\n",
        "def format_date(date_str):\n",
        "  return dateparser.parse(date_str).strftime('%Y-%m-%d %H:%M')\n",
        "\n",
        "def write_json(file_path, json_data):\n",
        "  print('Writing ' + file_path)\n",
        "  with open(file_path, 'w', encoding='utf8') as fp:\n",
        "      json.dump(json_data, fp, ensure_ascii=False, indent=4)\n",
        "\n",
        "# Store data collection information as the number of liveblogs, number of news, number of key moments and first and last date\n",
        "collection_stats = {}\n",
        "\n",
        "def get_source_stats(num_lbs, num_news, num_kms, first_date, last_date):\n",
        "  stats = {\n",
        "      'num_lbs': num_lbs,\n",
        "      'num_news': num_news,\n",
        "      'num_kms': num_kms,\n",
        "      'first_date': first_date,\n",
        "      'last_date': last_date\n",
        "  }\n",
        "  return stats"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8LFqFHy_ZyfM"
      },
      "source": [
        "#### Publico"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HN6AWi8zZ3Er"
      },
      "source": [
        "# Place publico data under data/publico\n",
        "DATA_DIR_PUBLICO = os.path.join(DATA_DIR, 'publico/')\n",
        "# Create dir if does not exist\n",
        "Path(DATA_DIR_PUBLICO).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "\n",
        "# Traverse all liveblog api entries and collect their ids, dates and urls\n",
        "# Returns a list of dictionaries. Each dict contains the date, id and url of a liveblog\n",
        "def get_blogs_publico():\n",
        "\n",
        "    print('===== Collecting Publico liveblogs =====')\n",
        "\n",
        "    url_lbs_listing = 'https://www.publico.pt/api/list/coronavirus-ao-minuto'\n",
        "    payload = {'page': 0}\n",
        "\n",
        "    list_blogs = []\n",
        "    while True:\n",
        "\n",
        "        r = requests.get(url_lbs_listing, headers=HEADERS, params=payload)\n",
        "        response_json = r.json()\n",
        "\n",
        "        # Exit if there are no more entries\n",
        "        if len(response_json) < 1:\n",
        "            break\n",
        "\n",
        "        # For each liveblog collect its date, id and url\n",
        "        for lb in response_json:\n",
        "            lb_dict = {}\n",
        "            lb_dict['date'] = format_date(lb['data']).split()[0]\n",
        "            lb_dict['id'] = lb['id']\n",
        "            lb_dict['url'] = lb['shareUrl']\n",
        "            print(lb_dict['url'])\n",
        "            list_blogs.append(lb_dict)\n",
        "\n",
        "        payload['page'] += 1\n",
        "\n",
        "    # 287 lbs as of 18/12\n",
        "    print('# lbs:', len(list_blogs))\n",
        "    print()\n",
        "\n",
        "    return list_blogs\n",
        "\n",
        "# Use the liveblog ids to request the liveblog api endpoint and collect the news under them\n",
        "# Returns a list of dicts of news and a list of dicts of keymoments. Each dict contains the title, text, date, url and keymoment flag of the news\n",
        "def get_news_publico():\n",
        "\n",
        "    lbs = get_blogs_publico()\n",
        "\n",
        "    print('===== Collecting Publico liveblogs news =====')\n",
        "\n",
        "    url_liveblog_api = 'https://api.publico.pt/liveblog/'\n",
        "\n",
        "    list_news, list_kms = [], []\n",
        "    for i, lb in enumerate(lbs):\n",
        "        lb_url = lb['url']\n",
        "\n",
        "        print(str(i+1) + '/' + str(len(lbs)))\n",
        "        print(lb_url)\n",
        "\n",
        "        lb_api_url = url_liveblog_api + str(lb['id']) \n",
        "        r = requests.get(lb_api_url, headers=HEADERS)\n",
        "        response_json = r.json()\n",
        "\n",
        "        # For each liveblog article collect its title, text, date and url\n",
        "        list_news_lb, list_kms_lb = [], []\n",
        "        for article in response_json:\n",
        "            news = {}\n",
        "            news['title'] = clean_html(article['titulo'])\n",
        "            news['text'] = pre_proc(article['texto'])\n",
        "            news['date'] = format_date(article['data'])\n",
        "            news['is_km'] = str(article['isDestaque'])\n",
        "            news['url'] = lb_url + '#' + str(article['id'])\n",
        "            list_news_lb.append(news)\n",
        "\n",
        "            # If article is key moment (isDestaque) append it to key moments list also\n",
        "            if article['isDestaque'] == True:\n",
        "                list_kms_lb.append(news)\n",
        "\n",
        "        list_news.extend(list_news_lb)\n",
        "        list_kms.extend(list_kms_lb)\n",
        "\n",
        "        print('# news:', len(list_news_lb))\n",
        "        print('# kms:', len(list_kms_lb))\n",
        "        print()\n",
        "    \n",
        "    # Sort news by descending date\n",
        "    list_news.sort(key=lambda item:item['date'], reverse=True)\n",
        "    list_kms.sort(key=lambda item:item['date'], reverse=True)\n",
        "\n",
        "    # Generate and store stats of the collection\n",
        "    source_stats = get_source_stats(len(lbs), len(list_news), len(list_kms), list_news[-1]['date'].split()[0], list_news[0]['date'].split()[0])\n",
        "    collection_stats['publico'] = source_stats\n",
        "\n",
        "    print('Stats: ')\n",
        "    for k, v in source_stats.items():\n",
        "      print(k + ': ' + str(v))\n",
        "    print()\n",
        "    \n",
        "    return list_news, list_kms\n",
        "\n",
        "news_publico, kms_publico = get_news_publico()\n",
        "\n",
        "# Creates a json file with the liveblog news under data/publico/\n",
        "publico_news_path = os.path.join(DATA_DIR_PUBLICO, 'news_publico.json')\n",
        "write_json(publico_news_path, news_publico)\n",
        "\n",
        "# Creates a json file with the keymoments news under data/publico/\n",
        "publico_kms_path = os.path.join(DATA_DIR_PUBLICO, 'kms_publico.json')\n",
        "write_json(publico_kms_path, kms_publico)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qAIF-0tfdsPI"
      },
      "source": [
        "#### Observador"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CMffnmcJdusW"
      },
      "source": [
        "# Place observador data under data/observador\n",
        "DATA_DIR_OBSERVADOR = os.path.join(DATA_DIR, 'observador/')\n",
        "# Create dir if does not exist\n",
        "Path(DATA_DIR_OBSERVADOR).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "\n",
        "# Use observador api to collect the ids of all coronavirus liveblogs\n",
        "def get_blogs_ids_observador():\n",
        "\n",
        "    print('===== Collecting Observador liveblogs ids =====')\n",
        "\n",
        "    # 47748 is the id of the section/topic \"coronavirus\" with the associated url \"seccao/saude/saude-publica-saude/coronavirus-saude-publica-saude/\"\n",
        "    url_grid = 'https://observador.pt/wp-json/obs_api/v4/grids/filter/category/47748'\n",
        "\n",
        "    date_format = '%Y%m%d'\n",
        "\n",
        "    first_date_str = '20200129'\n",
        "    today_date_str = datetime.today().strftime(date_format)\n",
        "\n",
        "    lb_ids = []\n",
        "    offset = today_date_str\n",
        "    while True:\n",
        "        \n",
        "        payload = {'render': 'JSON', 'offset': offset}\n",
        "\n",
        "        r = requests.get(url_grid, headers=HEADERS, params=payload)\n",
        "        response_json = r.json()\n",
        "\n",
        "        interval_lbs = []\n",
        "        for entry in response_json['rendered']['modules']:\n",
        "            for module in entry['modules']:\n",
        "                if module['module'] == 'obs_liveblog-1':\n",
        "                    interval_lbs.append(module['meta-id'])\n",
        "\n",
        "        lb_ids.extend(interval_lbs)\n",
        "\n",
        "        next_offset = response_json['rendered']['offset']\n",
        "\n",
        "        print('From: ' + next_offset + ' To: ' + offset)\n",
        "        # print('# days: ' + str((datetime.strptime(offset, date_format) - datetime.strptime(next_offset, date_format)).days))\n",
        "        # print('# lbs: ' + str(len(interval_lbs)))\n",
        "\n",
        "        offset = next_offset\n",
        "\n",
        "        if offset < first_date_str:\n",
        "            break\n",
        "    \n",
        "    lb_ids = sorted(list(set(lb_ids)))\n",
        "\n",
        "    # 316 lbs as of 18/12\n",
        "    print('# lbs:', len(lb_ids))\n",
        "    print()\n",
        "\n",
        "    return lb_ids\n",
        "\n",
        "# Use observador api to collect the urls of all coronavirus liveblogs\n",
        "def get_blogs_urls_observador():\n",
        "\n",
        "    blogs_ids = get_blogs_ids_observador()\n",
        "\n",
        "    print('===== Collecting Observador liveblogs urls =====')\n",
        "\n",
        "    url_news_endpoint = 'https://api.observador.pt/wp/items/id/'\n",
        "    # Alternative\n",
        "    # url_news_endpoint = 'https://observador.pt/observador_api/req/3_0/items/id/'\n",
        "\n",
        "    # List of dicts. Each dict contains id, url and date\n",
        "    lbs = []\n",
        "    for i, id in enumerate(blogs_ids):\n",
        "        lb_url = url_news_endpoint + str(id)\n",
        "\n",
        "        print(str(i+1) + '/' + str(len(blogs_ids)))\n",
        "        print(lb_url)\n",
        "\n",
        "        r = requests.get(lb_url, headers=HEADERS)\n",
        "        response_json = r.json()\n",
        "\n",
        "        news_dict = {\n",
        "            'id': id,\n",
        "            'url': response_json['links']['webUri'],\n",
        "            'date': format_date(response_json['pubDate'])\n",
        "        }\n",
        "        lbs.append(news_dict)\n",
        "\n",
        "    # 316 lbs as of 18/12\n",
        "    print('# lbs:', len(lbs))\n",
        "    print()\n",
        "\n",
        "    return lbs\n",
        "\n",
        "def get_news_observador():\n",
        "\n",
        "    lbs = get_blogs_urls_observador()\n",
        "\n",
        "    print('===== Collecting Observador liveblogs news =====')\n",
        "\n",
        "    list_news, list_kms = [], []\n",
        "    for i, lb in enumerate(lbs):\n",
        "        lb_url = lb['url']\n",
        "\n",
        "        print(str(i+1) + '/' + str(len(lbs)))\n",
        "        print(lb_url)\n",
        "\n",
        "        r = requests.get(lb_url, headers=HEADERS)\n",
        "\n",
        "        soup = BeautifulSoup(r.text, 'html.parser')\n",
        "\n",
        "        # Liveblog\n",
        "        lb_data = soup.find('script', id='liveblog-data', type='application/json').find(text=True)\n",
        "\n",
        "        lb_json = json.loads(lb_data)\n",
        "\n",
        "        list_news_lb = []\n",
        "        for entry in lb_json:\n",
        "            article = {\n",
        "                'title': pre_proc(entry['title']),\n",
        "                'text': pre_proc(entry['content']),\n",
        "                'date': format_date(entry['date']['datetime']),\n",
        "                'is_km': 'False',\n",
        "                'url': lb_url + '#liveblog-entry-' + str(entry['id'])\n",
        "            }\n",
        "            list_news_lb.append(article)\n",
        "\n",
        "        # Key moments\n",
        "        key_moments_div = soup.find('details', {'class': 'liveblog-highlights-wrapper'}).find('ul', {'class': 'liveblog-highlights-list'}).find_all('li')\n",
        "\n",
        "        list_kms_lb = []\n",
        "        for km in key_moments_div:\n",
        "            article = {\n",
        "                'title': pre_proc(km.find('span', {'class': 'liveblog-highlights-item-title'})),\n",
        "                'date': format_date(km.find('time')['datetime']),\n",
        "                'url': lb_url + '#liveblog-entry-' + str(km['data-id'])\n",
        "            }\n",
        "            list_kms_lb.append(article)\n",
        "\n",
        "        # Flag key moments in news\n",
        "        for km in list_kms_lb:\n",
        "            for news in list_news_lb:\n",
        "                if news['title'] == km['title'] and news['date'] == km['date'] and news['url'] == km['url']:\n",
        "                    news['is_km'] = 'True'\n",
        "        \n",
        "        # Check key moments were correctly merged\n",
        "        # print('# kms:', len(kms))\n",
        "        # print('# kms in news:', len([x for x in news if x['is_km'] == 'True']))\n",
        "        # print()\n",
        "\n",
        "        list_news.extend(list_news_lb)\n",
        "        list_kms.extend(list_kms_lb)\n",
        "\n",
        "        print('# news:', len(list_news_lb))\n",
        "        print('# kms:', len(list_kms_lb))\n",
        "        print()\n",
        "\n",
        "    # Sort news by descending date\n",
        "    list_news.sort(key=lambda item:item['date'], reverse=True)\n",
        "    list_kms.sort(key=lambda item:item['date'], reverse=True)\n",
        "\n",
        "    # Generate and store stats of the collection\n",
        "    source_stats = get_source_stats(len(lbs), len(list_news), len(list_kms), list_news[-1]['date'].split()[0], list_news[0]['date'].split()[0])\n",
        "    collection_stats['observador'] = source_stats\n",
        "\n",
        "    print('Stats: ')\n",
        "    for k, v in source_stats.items():\n",
        "      print(k + ': ' + str(v))\n",
        "    print()\n",
        "    \n",
        "    return list_news, list_kms\n",
        "\n",
        "\n",
        "news_observador, kms_observador = get_news_observador()\n",
        "\n",
        "# Creates a json file with the liveblog news under data/observador/\n",
        "observador_news_path = os.path.join(DATA_DIR_OBSERVADOR, 'news_observador.json')\n",
        "write_json(observador_news_path, news_observador)\n",
        "\n",
        "# Creates a json file with the keymoments news under data/observador/\n",
        "observador_kms_path = os.path.join(DATA_DIR_OBSERVADOR, 'kms_observador.json')\n",
        "write_json(observador_kms_path, kms_observador)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ikFM71jRwCM5"
      },
      "source": [
        "#### CNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dY8m8Bn5xre_"
      },
      "source": [
        "# Place cnn data under data/cnn\n",
        "DATA_DIR_CNN = os.path.join(DATA_DIR, 'cnn/')\n",
        "# Create dir if does not exist\n",
        "Path(DATA_DIR_CNN).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "# Setup selenium Chrome driver\n",
        "def get_chrome_driver():\n",
        "  options = webdriver.ChromeOptions()\n",
        "  options.add_argument('--headless')\n",
        "  options.add_argument('--no-sandbox')\n",
        "  options.add_argument('--disable-dev-shm-usage')\n",
        "\n",
        "  driver = webdriver.Chrome('chromedriver', options=options)\n",
        "  driver.implicitly_wait(30)\n",
        "\n",
        "  return driver\n",
        "\n",
        "\n",
        "def get_blogs_urls_cnn(browser):\n",
        "\n",
        "    print('===== Collecting CNN liveblogs urls =====')\n",
        "\n",
        "    search_url = 'https://edition.cnn.com/search?'\n",
        "    payload = {'q': '\"coronavirus news\"', 'size':20, 'page': 1, 'from': 0}\n",
        "\n",
        "    lb_urls = set()\n",
        "    while True:\n",
        "\n",
        "        url = search_url + urlencode(payload)\n",
        "        print(url)\n",
        "\n",
        "        browser.get(url)\n",
        "\n",
        "        time.sleep(3)\n",
        "\n",
        "        soup = BeautifulSoup(browser.page_source, 'html.parser')\n",
        "\n",
        "        results_list = soup.find('div', {'class': 'cnn-search__results-list'})\n",
        "\n",
        "        for result in results_list:\n",
        "            try:\n",
        "                result_content = result.find_next('div', {'class': 'cnn-search__result-contents'})\n",
        "                headline = result_content.find_next('h3', {'class': 'cnn-search__result-headline'})\n",
        "                a = headline.find_next('a')\n",
        "                lb_url = 'https:' + a['href']\n",
        "                if 'coronavirus news' in a.get_text() and 'live-news' in lb_url:\n",
        "                    lb_urls.add(lb_url)\n",
        "\n",
        "            except AttributeError as ae:\n",
        "                # print(ae)\n",
        "                pass\n",
        "                \n",
        "        # Stop if last page (pagination did not display maximum number of contents)\n",
        "        if len(results_list) < 2*payload['size']:\n",
        "            browser.close()\n",
        "            break\n",
        "\n",
        "        payload['page'] += 1\n",
        "        payload['from'] = (payload['page']*payload['size']) - payload['size']\n",
        "\n",
        "    lb_urls = sorted(list(lb_urls))\n",
        "\n",
        "    # 266 lbs as of 18/12\n",
        "    print('# lbs:', len(lb_urls))\n",
        "    print()\n",
        "    \n",
        "    return lb_urls\n",
        "\n",
        "def get_news_cnn(browser):\n",
        "\n",
        "    lbs = get_blogs_urls_cnn(browser)\n",
        "\n",
        "    print('===== Collecting CNN liveblogs news =====')\n",
        "\n",
        "    list_news, list_kms = [], []\n",
        "    for i, lb_url in enumerate(lbs):\n",
        "\n",
        "        print(str(i+1) + '/' + str(len(lbs)))\n",
        "        print(lb_url)\n",
        "\n",
        "        r = requests.get(lb_url, headers=HEADERS)\n",
        "\n",
        "        soup = BeautifulSoup(r.text, 'html.parser')\n",
        "        try:\n",
        "          lb_data = soup.find('script', id='liveBlog-schema', type='application/ld+json').find(text=True)\n",
        "        except:\n",
        "          print('Could not collect liveblog for this url\\n')\n",
        "          continue\n",
        "\n",
        "        lb_json = json.loads(lb_data)\n",
        "\n",
        "        list_news_lb = []\n",
        "        for entry in lb_json['liveBlogUpdate']:\n",
        "            \n",
        "            try:\n",
        "                title = pre_proc(entry['headline'])\n",
        "            except:\n",
        "                title = ''\n",
        "\n",
        "            article = {\n",
        "                'title': title,\n",
        "                'text': pre_proc(entry['articleBody']),\n",
        "                'date': format_date(entry['datePublished']),\n",
        "                'is_km': 'False',\n",
        "                'url': entry['url']\n",
        "            }\n",
        "            list_news_lb.append(article)\n",
        "\n",
        "        list_news.extend(list_news_lb)\n",
        "\n",
        "        try:\n",
        "            kms_data = soup.find('aside', id='ls-rail').find('div', class_='sc-dnqmqq render-stellar-contentstyles__List-sc-9v7nwy-1 eUPcFX').find_next('ul')\n",
        "        except:\n",
        "            print('There are no key moments for this url')\n",
        "            kms_data = []\n",
        "\n",
        "        list_kms_lb = []\n",
        "        for km in kms_data:\n",
        "            clean_text = pre_proc(km)\n",
        "            article = {\n",
        "                'title': clean_text,\n",
        "                'text': clean_text,\n",
        "                # Date is an approximation as key moments do not refer to any date\n",
        "                'date': list_news_lb[len(list_news_lb)//2]['date'],\n",
        "                'is_km': 'True',\n",
        "                'url': lb_url\n",
        "            }\n",
        "            list_kms_lb.append(article)\n",
        "\n",
        "        list_kms.extend(list_kms_lb)\n",
        "\n",
        "        print('# news:', len(list_news_lb))\n",
        "        print('# kms:', len(list_kms_lb))\n",
        "        print()\n",
        "\n",
        "    # Sort news by descending date\n",
        "    list_news.sort(key=lambda item:item['date'], reverse=True)\n",
        "    list_kms.sort(key=lambda item:item['date'], reverse=True)\n",
        "\n",
        "    # Generate and store stats of the collection\n",
        "    source_stats = get_source_stats(len(lbs), len(list_news), len(list_kms), list_news[-1]['date'].split()[0], list_news[0]['date'].split()[0])\n",
        "    collection_stats['cnn'] = source_stats\n",
        "\n",
        "    print('Stats: ')\n",
        "    for k, v in source_stats.items():\n",
        "      print(k + ': ' + str(v))\n",
        "    print()\n",
        "    \n",
        "    return list_news, list_kms\n",
        "\n",
        "\n",
        "news_cnn, kms_cnn = get_news_cnn(get_chrome_driver())\n",
        "\n",
        "# Creates a json file with the liveblog news under data/cnn/\n",
        "cnn_news_path = os.path.join(DATA_DIR_CNN, 'news_cnn.json')\n",
        "write_json(cnn_news_path, news_cnn)\n",
        "\n",
        "# Creates a json file with the keymoments news under data/cnn/\n",
        "cnn_kms_path = os.path.join(DATA_DIR_CNN, 'kms_cnn.json')\n",
        "write_json(cnn_kms_path, kms_cnn)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wiigCoINjfB3"
      },
      "source": [
        "#### Guardian"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZSENm555jhjQ"
      },
      "source": [
        "# Place guardian data under data/guardian\n",
        "DATA_DIR_GUARDIAN = os.path.join(DATA_DIR, 'guardian/')\n",
        "# Create dir if does not exist\n",
        "Path(DATA_DIR_GUARDIAN).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "\n",
        "def get_blogs_urls_guardian():\n",
        "    \n",
        "    print('===== Collecting Guardian liveblogs urls =====')\n",
        "\n",
        "    # This url contains a listing of all coronavirus liveblogs\n",
        "    search_url = 'https://www.theguardian.com/world/series/coronavirus-live'\n",
        "    payload = {'page': 1}\n",
        "    \n",
        "    lb_urls = set()\n",
        "    while True:\n",
        "        r = requests.get(search_url, headers=HEADERS, params=payload)\n",
        "\n",
        "        # If we try to get a page that does not exist the url returns to the search_url\n",
        "        if r.url == search_url:\n",
        "            break\n",
        "        \n",
        "        soup = BeautifulSoup(r.text, 'html.parser')\n",
        "\n",
        "        lbs_list = soup.find('div', class_='u-cf index-page', role='main')\n",
        "\n",
        "        lbs_alinks = lbs_list.find_all('a', class_='fc-item__link')\n",
        "\n",
        "        for a in lbs_alinks:\n",
        "            url = a['href']\n",
        "            lb_urls.add(url)\n",
        "            print(a['href'])\n",
        "        \n",
        "        payload['page'] += 1\n",
        "\n",
        "    lb_urls = sorted(list(lb_urls))\n",
        "\n",
        "    # 335 lbs as of 18/12\n",
        "    print('# lbs:', len(lb_urls))\n",
        "    print()\n",
        "    \n",
        "    return lb_urls\n",
        "\n",
        "def get_news_guardian():\n",
        "\n",
        "    lbs = get_blogs_urls_guardian()\n",
        "\n",
        "    print('===== Collecting Guardian liveblogs news =====')\n",
        "\n",
        "    base_url_guardian = 'https://www.theguardian.com'\n",
        "\n",
        "    list_news, list_kms = [], []\n",
        "    # Iterate over liveblogs\n",
        "    for i, lb_url in enumerate(lbs):\n",
        "\n",
        "        print(str(i+1) + '/' + str(len(lbs)))\n",
        "        print(lb_url)\n",
        "\n",
        "        list_news_lb, list_kms_lb = [], []\n",
        "        url = lb_url\n",
        "        # Iterate over each liveblog page\n",
        "        while True:\n",
        "            r = requests.get(url, headers=HEADERS)\n",
        "\n",
        "            soup = BeautifulSoup(r.text, 'html.parser')\n",
        "\n",
        "            lb = soup.find('div', class_='js-article__container')\n",
        "\n",
        "            posts = lb.find_all('div', itemprop='liveBlogUpdate')\n",
        "            for p in posts:\n",
        "\n",
        "                title = pre_proc(p.find('meta', itemprop='headline')['content'])\n",
        "\n",
        "                content_pars = p.find('div', class_='block-elements', itemprop='articleBody').find_all('p')\n",
        "                # Clean paragraphs and join them into the same string\n",
        "                text = ' '.join([pre_proc(cp) for cp in content_pars])\n",
        "\n",
        "                date = format_date(p.find('time', class_='js-timestamp')['datetime'])\n",
        "                \n",
        "                is_km = 'False'\n",
        "                \n",
        "                news_url = base_url_guardian + p.find('a', class_='block-time__link', itemprop='url')['href']\n",
        "\n",
        "                article = {\n",
        "                    'title': title,\n",
        "                    'text': text,\n",
        "                    'date': date,\n",
        "                    'is_km': is_km,\n",
        "                    'url': news_url\n",
        "                }\n",
        "\n",
        "                list_news_lb.append(article)\n",
        "\n",
        "            # Key moments are the same for all pages, inside the same liveblog\n",
        "            # Only non first pages contain '#liveblog-navigation' in the url\n",
        "            if '#liveblog-navigation' not in url:\n",
        "                # key_moments_data = soup.find('ul', class_='timeline js-live-blog__timeline u-unstyled')\n",
        "                key_moments_data = soup.find_all('li', class_='timeline__item')\n",
        "\n",
        "                for km in key_moments_data:\n",
        "\n",
        "                    title = pre_proc(km.find('span', class_='timeline__title u-underline').get_text())\n",
        "\n",
        "                    date = format_date(km.find('time', class_='js-timestamp')['datetime'])\n",
        "\n",
        "                    news_url = base_url_guardian + km.find('a', class_='timeline__link')['href']\n",
        "\n",
        "                    is_km = 'True'\n",
        "\n",
        "                    article = {\n",
        "                        'title': title,\n",
        "                        'date': date,\n",
        "                        'is_km': is_km,\n",
        "                        'url': news_url\n",
        "                    }\n",
        "                    list_kms_lb.append(article)\n",
        "\n",
        "            pagination = soup.find('div', id='liveblog-navigation')\n",
        "\n",
        "            # If there is no pagination (only one page) exit loop\n",
        "            if not pagination:\n",
        "                break\n",
        "\n",
        "            pagination_older = pagination.find('div', class_='liveblog-navigation__older')\n",
        "\n",
        "            next_page_a = pagination_older.find('a', class_='liveblog-navigation__link liveblog-navigation__link--primary')\n",
        "\n",
        "            # If there is no next page exit loop\n",
        "            if not next_page_a:\n",
        "                break\n",
        "\n",
        "            url = base_url_guardian + next_page_a['href']\n",
        "\n",
        "        # Flag key moments in news\n",
        "        # kms and news with same url have is_km=True\n",
        "        for km in list_kms_lb:\n",
        "            for news in list_news_lb:\n",
        "                if news['url'] == km['url']:\n",
        "                    news['is_km'] = 'True'\n",
        "\n",
        "        list_news.extend(list_news_lb)\n",
        "        list_kms.extend(list_kms_lb)\n",
        "\n",
        "        print('# news:', len(list_news_lb))\n",
        "        print('# kms:', len(list_kms_lb))\n",
        "        print()\n",
        "    \n",
        "    # Sort news by descending date\n",
        "    list_news.sort(key=lambda item:item['date'], reverse=True)\n",
        "    list_kms.sort(key=lambda item:item['date'], reverse=True)\n",
        "\n",
        "    # Generate and store stats of the collection\n",
        "    source_stats = get_source_stats(len(lbs), len(list_news), len(list_kms), list_news[-1]['date'].split()[0], list_news[0]['date'].split()[0])\n",
        "    collection_stats['guardian'] = source_stats\n",
        "\n",
        "    print('Stats: ')\n",
        "    for k, v in source_stats.items():\n",
        "      print(k + ': ' + str(v))\n",
        "    print()\n",
        "\n",
        "    return list_news, list_kms\n",
        "\n",
        "\n",
        "news_guardian, kms_guardian = get_news_guardian()\n",
        "\n",
        "# Creates a json file with the liveblog news under data/guardian/\n",
        "guradian_news_path = os.path.join(DATA_DIR_GUARDIAN, 'news_guardian.json')\n",
        "write_json(guradian_news_path, news_guardian)\n",
        "\n",
        "# Creates a json file with the keymoments news under data/guardian/\n",
        "guardian_kms_path = os.path.join(DATA_DIR_GUARDIAN, 'kms_guardian.json')\n",
        "write_json(guardian_kms_path, kms_guardian)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aUio2zmo6BZR"
      },
      "source": [
        "### Stats"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gWqhEutc8i0V"
      },
      "source": [
        "import seaborn as sns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BIJQpOM26Dho"
      },
      "source": [
        "for source, stats in collection_stats.items():\n",
        "  print(source)\n",
        "  print(stats)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WaE6TZOc8l-4"
      },
      "source": [
        "sources = list(collection_stats.keys())\n",
        "n_lbs = [v['num_lbs'] for _, v in collection_stats.items()]\n",
        "\n",
        "ax = sns.barplot(x=sources, y=n_lbs)\n",
        "ax.set_title('Number of liveblogs per source');"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jX4I06aW_2z2"
      },
      "source": [
        "sns.set_theme(style=\"whitegrid\")\n",
        "\n",
        "sources = list(collection_stats.keys())\n",
        "\n",
        "sns.set_color_codes(\"pastel\")\n",
        "n_news = [v['num_news'] for _, v in collection_stats.items()]\n",
        "ax = sns.barplot(x=sources, y=n_news, label=\"Total news\", color=\"b\")\n",
        "\n",
        "sns.set_color_codes(\"muted\")\n",
        "n_kms = [v['num_kms'] for _, v in collection_stats.items()]\n",
        "ax = sns.barplot(x=sources, y=n_kms, label=\"Key moments\", color=\"b\")\n",
        "\n",
        "ax.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\n",
        "ax.set_title('Number of news and key moments per source');"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "myp4tEGsjfq6"
      },
      "source": [
        "### Zip data and save to GDrive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "51T6e0x1jlro"
      },
      "source": [
        "zip_file_name = 'data_' + datetime.today().strftime(\"%Y-%m-%d\") + '.zip'\n",
        "\n",
        "!zip -r $zip_file_name $DATA_DIR"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IzX8Led3lKGG"
      },
      "source": [
        "from google.colab import drive\n",
        "\n",
        "DRIVE_MOUNT_DIR = '/content/drive'\n",
        "drive.mount(DRIVE_MOUNT_DIR)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OC0mVTWUlQX4"
      },
      "source": [
        "DATA_STORE_DIR = os.path.join(DRIVE_MOUNT_DIR, 'MyDrive/inesc/tls_covid')\n",
        "\n",
        "!cp -r $zip_file_name $DATA_STORE_DIR"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1vG92Ujtnfqi"
      },
      "source": [
        "!cp -r drive/MyDrive/inesc/tls_covid/data_2021-01-07.zip .\n",
        "!unzip data_2021-01-07.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TAhfRgb5ERdf"
      },
      "source": [
        "## Preprocess datasets"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tZD9ZpnyEjl6"
      },
      "source": [
        "### Requirements"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fncExDXSEyTQ"
      },
      "source": [
        "!pip install -U pandas"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yMY2-snfEmIW"
      },
      "source": [
        "import os\n",
        "from pathlib import Path\n",
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7iSEmQaOK227"
      },
      "source": [
        "pd.set_option('display.max_colwidth', 2000)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K7qBSzsOFcd2"
      },
      "source": [
        "# The root directory where the datasets are placed\n",
        "DATA_DIR = 'data/'\n",
        "\n",
        "# Create python vars with the datasets location\n",
        "DATA_DIR_PUBLICO = os.path.join(DATA_DIR, 'publico/')\n",
        "NEWS_PUBLICO = os.path.join(DATA_DIR_PUBLICO, 'news_publico.json')\n",
        "\n",
        "DATA_DIR_OBSERVADOR = os.path.join(DATA_DIR, 'observador/')\n",
        "NEWS_OBSERVADOR = os.path.join(DATA_DIR_OBSERVADOR, 'news_observador.json')\n",
        "\n",
        "DATA_DIR_CNN = os.path.join(DATA_DIR, 'cnn/')\n",
        "NEWS_CNN = os.path.join(DATA_DIR_CNN, 'news_cnn.json')\n",
        "KMS_CNN = os.path.join(DATA_DIR_CNN, 'kms_cnn.json')\n",
        "\n",
        "DATA_DIR_GUARDIAN = os.path.join(DATA_DIR, 'guardian/')\n",
        "NEWS_GUARDIAN = os.path.join(DATA_DIR_GUARDIAN, 'news_guardian.json')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bf2-ZPOhtHy0"
      },
      "source": [
        "def pre_proc(df_news, noisy_strs_title, noisy_strs_text):\n",
        "  # Exception handling because CNN KMs do not have text column. Only have title\n",
        "\n",
        "  # Remove rows with NaN in title or text\n",
        "  df_news = df_news.dropna(subset=['title'])\n",
        "  try:\n",
        "    df_news = df_news.dropna(subset=['text'])\n",
        "  except KeyError as ke:\n",
        "    pass\n",
        "\n",
        "  # Remove rows containing containing noisy strings in title\n",
        "  df_news = df_news[~df_news['title'].str.contains('|'.join(noisy_strs_title), case=False)]\n",
        "\n",
        "  # Remove noisy strings from news content\n",
        "  try:\n",
        "    df_news['text'] = df_news['text'].str.replace('|'.join(noisy_strs_text), '')\n",
        "  except KeyError as ke:\n",
        "    pass\n",
        "\n",
        "  # Uniformize quotation marks\n",
        "  df_news['title'] = df_news['title'].str.replace('“|”', '\"')\n",
        "\n",
        "  try:\n",
        "    df_news['text'] = df_news['text'].str.replace('“|”', '\"')\n",
        "  except KeyError as ke:\n",
        "    pass\n",
        "\n",
        "  # Remove rows with empty title or text\n",
        "  df_news = df_news[df_news['title'] != '']\n",
        "  try:\n",
        "    df_news = df_news[df_news['text'] != '']\n",
        "  except KeyError as ke:\n",
        "    pass\n",
        "\n",
        "  # Drop duplicated news with same title, text and date, keeping first occurence\n",
        "  try:\n",
        "    df_news = df_news.drop_duplicates(subset=['title', 'text', 'date'])\n",
        "  except KeyError as ke:\n",
        "    df_news = df_news.drop_duplicates(subset=['title', 'date'])\n",
        "\n",
        "  \n",
        "  # Reorder index\n",
        "  df_news = df_news.reset_index(drop=True)\n",
        "\n",
        "  return df_news\n",
        "\n",
        "\n",
        "def write_df_to_json(df, file_path):\n",
        "  df['date'] = df['date'].dt.strftime('%Y-%m-%d %H:%M')\n",
        "  df.to_json(file_path, orient='records', force_ascii=False, indent=4)\n",
        "\n",
        "\n",
        "# The root directory where the prepocessed datasets will be placed\n",
        "DATA_CLEAN_DIR = 'data_clean/'\n",
        "# Create dir if does not exist\n",
        "Path(DATA_CLEAN_DIR).mkdir(parents=True, exist_ok=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-WNmrgi1Ee9X"
      },
      "source": [
        "### PT\n",
        "\n",
        "In the data collection step we generated two .json files (news and kms) for the sake of separation. The news contains all the liveblog standard news as well as the keymoments. So news.json contains all the news articles. In order to standardize and avoid redundancy, in the pre processing step we will only work with the news.json."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rRDR4K08Ehil"
      },
      "source": [
        "#### Publico\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r96oK_PguPji"
      },
      "source": [
        "noisy_strs_title_publico = [\n",
        "                        'A primeira página do PÚBLICO',\n",
        "                        'Fim do acompanhamento',\n",
        "                        'Quatro opiniões para ler',\n",
        "                        'Ponto de situação',\n",
        "                        'Encerramento da cobertura',\n",
        "                        'Os números da pandemia',\n",
        "                        'Encerramento do acompanhamento',\n",
        "                        'Os destaques',\n",
        "                        'para começar o dia',\n",
        "                        'a ler',\n",
        "                        'Vale a pena ler',\n",
        "                        'o que precisa de saber',\n",
        "                        'o que deve saber',\n",
        "                        'Os números actualizados da pandemia',\n",
        "                        'Os números mundiais da pandemia',\n",
        "                        'Os últimos números da pandemia',\n",
        "                        'O número de casos do novo coronavírus',\n",
        "                        'Acompanhe em directo a conferência',\n",
        "                        'Veja a conferência de imprensa',\n",
        "                        'Acompanhe a conferência de imprensa',\n",
        "                        'Veja em directo a conferência',\n",
        "                        'Bom dia',\n",
        "                        'Boa noite',\n",
        "                        'Resumo dos acontecimentos',\n",
        "                        'resumo do dia',\n",
        "                        'resumo da manhã',\n",
        "                        '^resumo$',\n",
        "                        '^encerramento$'\n",
        "]\n",
        "\n",
        "noisy_strs_text_publico = [\n",
        "                        r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',\n",
        "                        r'<span(.*?)>',\n",
        "                        r'Normal 0 21(.*)',\n",
        "\n",
        "                        r'^Leia(.*)aqui(.*)',\n",
        "                        r' Leia(.*)aqui(.*)',\n",
        "                        r' Leia a entrevista(.*)',\n",
        "                        r' Leia a reportagem(.*)',\n",
        "                        r' Leia o resto da reportagem(.*)',\n",
        "                        r' Leia o artigo(.*)',\n",
        "                        r' Leia a notícia(.*)',\n",
        "                        r' Leia o texto(.*)',\n",
        "                        r' Leia mais(.*)',\n",
        "                        r' Saiba mais(.*)',\n",
        "                        r' Ler mais(.*)',\n",
        "                        r' Leia também(.*)',\n",
        "                        r'pic.twitter.com/[a-zA-Z0-9]+(\\s|$)'\n",
        "]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NR073UkFHot2"
      },
      "source": [
        "# Load json dataset into pandas dataframe\n",
        "df_publico_lb = pd.read_json(NEWS_PUBLICO)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uqQtSTCxFil3"
      },
      "source": [
        "# Preprocess dataset\n",
        "df_publico_lb = pre_proc(df_publico_lb, noisy_strs_title_publico, noisy_strs_text_publico)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8NWMLpkFOBRY"
      },
      "source": [
        "df_publico_lb.describe()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sTNypGMLUSj-"
      },
      "source": [
        "df_publico_lb.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f4jceOZ-ciMB"
      },
      "source": [
        "# Place publico data under data_clean/publico\n",
        "DATA_CLEAN_DIR_PUBLICO = os.path.join(DATA_CLEAN_DIR, 'publico/')\n",
        "# Create dir if does not exist\n",
        "Path(DATA_CLEAN_DIR_PUBLICO).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "publico_news_clean_path = os.path.join(DATA_CLEAN_DIR_PUBLICO, 'news_publico.json')\n",
        "write_df_to_json(df_publico_lb, publico_news_clean_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VbMtC0hgDywb"
      },
      "source": [
        "#### Observador"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NhcViKJdDywm"
      },
      "source": [
        "noisy_strs_title_observador = [\n",
        "                        'Ponto da situação',\n",
        "                        'Ponto de situação no mundo',\n",
        "                        'Os pontos mais importantes do dia até ao momento'\n",
        "]\n",
        "\n",
        "noisy_strs_text_observador = [\n",
        "                        r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',\n",
        "                        r' Leia a notícia(.*)',\n",
        "                        r' Pode ler mais(.*)',\n",
        "                        r' Ler mais(.*)',\n",
        "                        r' Saiba mais(.*)',\n",
        "                        r'Leia mais aqui(.)*',\n",
        "                        r'\\(?([A-Z]+(\\s)*)*/OBSERVADOR\\)?',\n",
        "                        r'(\\(?Agência\\)?\\s*)?Lusa\\)?$',\n",
        "                        r'pic.twitter.com/[a-zA-Z0-9]+(\\s|$)'\n",
        "]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kz7B5bxeDywo"
      },
      "source": [
        "# Load json dataset into pandas dataframe\n",
        "df_observador_lb = pd.read_json(NEWS_OBSERVADOR)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pt3flBftDywp"
      },
      "source": [
        "# Preprocess dataset\n",
        "df_observador_lb = pre_proc(df_observador_lb, noisy_strs_title_observador, noisy_strs_text_observador)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oUU_CBiBDywq"
      },
      "source": [
        "df_observador_lb.describe()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZUzpAb2yDyws"
      },
      "source": [
        "df_observador_lb.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "90NB8IDFDywu"
      },
      "source": [
        "# Place observador data under data_clean/observador\n",
        "DATA_CLEAN_DIR_OBSERVADOR = os.path.join(DATA_CLEAN_DIR, 'observador/')\n",
        "# Create dir if does not exist\n",
        "Path(DATA_CLEAN_DIR_OBSERVADOR).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "observador_news_clean_path = os.path.join(DATA_CLEAN_DIR_OBSERVADOR, 'news_observador.json')\n",
        "write_df_to_json(df_observador_lb, observador_news_clean_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GiLZLn5ZxyBY"
      },
      "source": [
        "### EN"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ip5cib5ZxyBa"
      },
      "source": [
        "#### CNN\n",
        "Here we have to deal with the two jsons separately as the news_cnn.json does not contain the news in kms_cnn.json"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "veb4HBGLxyBZ"
      },
      "source": [
        "noisy_strs_title_cnn = [\n",
        "                        'Follow live updates',\n",
        "                        'Go here for latest updates',\n",
        "                        'What you need to know',\n",
        "                        'the latest on the pandemic',\n",
        "                        'Watch the entire CNN coronavirus town hall',\n",
        "                        'coronavirus town hall has ended',\n",
        "                        'global town hall on coronavirus will start soon',\n",
        "                        'the latest coronavirus update',\n",
        "                        'the latest coronavirus numbers',\n",
        "                        'what you may have missed',\n",
        "                        'Catch up:'\n",
        "]\n",
        "\n",
        "noisy_strs_text_cnn = [\n",
        "                        r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',\n",
        "                        r'pic.twitter.com/[a-zA-Z0-9]+(\\s|$)'\n",
        "]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UWvPBH-6WhYe"
      },
      "source": [
        "# Load json datasets into pandas dataframes\n",
        "df_cnn_km = pd.read_json(KMS_CNN)\n",
        "df_cnn_lb = pd.read_json(NEWS_CNN)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oIwlKN0IxyBa"
      },
      "source": [
        "# Preprocess datasets\n",
        "df_cnn_km = pre_proc(df_cnn_km, noisy_strs_title_cnn, noisy_strs_text_cnn)\n",
        "df_cnn_lb = pre_proc(df_cnn_lb, noisy_strs_title_cnn, noisy_strs_text_cnn)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DbmGRLaHUyeF"
      },
      "source": [
        "df_cnn_km.describe()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WcGP-V7WU1PF"
      },
      "source": [
        "df_cnn_km.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5koPmPgOU3l8"
      },
      "source": [
        "df_cnn_lb.describe()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_flLBqprU4CW"
      },
      "source": [
        "df_cnn_lb.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1hVIGsWHhnZF"
      },
      "source": [
        "# Place cnn data under data_clean/cnn\n",
        "DATA_CLEAN_DIR_CNN = os.path.join(DATA_CLEAN_DIR, 'cnn/')\n",
        "# Create dir if does not exist\n",
        "Path(DATA_CLEAN_DIR_CNN).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "cnn_news_clean_path = os.path.join(DATA_CLEAN_DIR_CNN, 'news_cnn.json')\n",
        "write_df_to_json(df_cnn_lb, cnn_news_clean_path)\n",
        "\n",
        "cnn_km_clean_path = os.path.join(DATA_CLEAN_DIR_CNN, 'kms_cnn.json')\n",
        "write_df_to_json(df_cnn_km, cnn_km_clean_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lNt3kJ81LRHK"
      },
      "source": [
        "#### Guardian"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LsAfqpgSLRHM"
      },
      "source": [
        "noisy_strs_title_guardian = [\n",
        "                        'Summary',\n",
        "                        'Key developments in the global coronavirus',\n",
        "                        'What we know so far'\n",
        "]\n",
        "\n",
        "noisy_strs_text_guardian = [\n",
        "                        r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',\n",
        "                        r'Hi everyone, this is(.*)',\n",
        "                        r'Hi, Helen Sullivan(.*)',\n",
        "                        r'Good evening from(.*)',\n",
        "                        r'We’ve launched a(.*)',\n",
        "                        r'We’ve fired up a(.*)',\n",
        "                        r'That’s it for this blog(.*)',\n",
        "                        r'Read more.*$',\n",
        "                        r'More info.*$',\n",
        "                        r'pic.twitter.com/[a-zA-Z0-9]+(\\s|$)'\n",
        "]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MP4-6RKgLRHO"
      },
      "source": [
        "# Load json datasets into pandas dataframes\n",
        "df_guardian_lb = pd.read_json(NEWS_GUARDIAN)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KR0-087SLRHP"
      },
      "source": [
        "# Preprocess datasets\n",
        "df_guardian_lb = pre_proc(df_guardian_lb, noisy_strs_title_guardian, noisy_strs_text_guardian)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pEfTxULhLRHT"
      },
      "source": [
        "df_guardian_lb.describe()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "afjF5ebQLRHU"
      },
      "source": [
        "df_guardian_lb.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UR0P3AJMLRHV"
      },
      "source": [
        "# Place guardian data under data_clean/guardian\n",
        "DATA_CLEAN_DIR_GUARDIAN = os.path.join(DATA_CLEAN_DIR, 'guardian/')\n",
        "# Create dir if does not exist\n",
        "Path(DATA_CLEAN_DIR_GUARDIAN).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "guardian_news_clean_path = os.path.join(DATA_CLEAN_DIR_GUARDIAN, 'news_guardian.json')\n",
        "write_df_to_json(df_guardian_lb, guardian_news_clean_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3FPbAg_Cj0Tj"
      },
      "source": [
        "### Zip data and save to GDrive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TdXj-SZCiTzr"
      },
      "source": [
        "zip_file_name = 'data-clean_' + datetime.today().strftime(\"%Y-%m-%d\") + '.zip'\n",
        "\n",
        "!zip -r $zip_file_name $DATA_CLEAN_DIR"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UrNELp3wj8Dg"
      },
      "source": [
        "from google.colab import drive\n",
        "\n",
        "DRIVE_MOUNT_DIR = '/content/drive'\n",
        "drive.mount(DRIVE_MOUNT_DIR)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-lP9oyqdjEfX"
      },
      "source": [
        "DATA_STORE_DIR = os.path.join(DRIVE_MOUNT_DIR, 'MyDrive/inesc/tls_covid')\n",
        "\n",
        "!cp -r $zip_file_name $DATA_STORE_DIR"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2OLYdyhSpxGx"
      },
      "source": [
        "!cp -r drive/MyDrive/inesc/tls_covid/data-clean_2020-12-31.zip .\n",
        "!unzip data-clean_2020-12-31.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TLfg2Bpg8ujT"
      },
      "source": [
        "## Generate TLS dataset\n",
        "\n",
        "[Timeline17](http://www.l3s.de/~gtran/timeline/) structure\n",
        "\n",
        "Use key moments to identify entities and keywords of interest. We refer to the set of entities and keywords as topics."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7rFfc3lO9gfH"
      },
      "source": [
        "### Requirements"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cJlkVbCe9nW4"
      },
      "source": [
        "# Not necessary to run this cell\n",
        "# Just to check which GPU is allocated (P100 > T4 > P4 > K80) and CUDA version\n",
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q2XRLoZe-V3f"
      },
      "source": [
        "!pip install -U pandas\n",
        "!pip install -U spacy[cuda101]\n",
        "!python -m spacy download en_core_web_sm\n",
        "!python -m spacy download pt_core_news_sm\n",
        "!pip install git+https://github.com/LIAAD/yake\n",
        "!pip install unidecode\n",
        "!pip install syntok"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q51MJlrR-5Y2"
      },
      "source": [
        "import os\n",
        "import json\n",
        "import re\n",
        "from pathlib import Path\n",
        "\n",
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "tqdm.pandas()\n",
        "\n",
        "import spacy\n",
        "spacy_gpu = spacy.prefer_gpu()\n",
        "print('spaCy GPU enabled:', spacy_gpu)\n",
        "import pt_core_news_sm\n",
        "import en_core_web_sm\n",
        "\n",
        "import yake\n",
        "\n",
        "import unidecode\n",
        "\n",
        "import syntok.segmenter as segmenter"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wkco8aBWKNFJ"
      },
      "source": [
        "pd.set_option('display.max_colwidth', 512)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yvGnSseY_k2S"
      },
      "source": [
        "YAKE_PARAMS = {\n",
        "    'language': '',\n",
        "    'max_ngram_size': 4,\n",
        "    'deduplication_thresold': 0.9,\n",
        "    'deduplication_algo': 'seqm',\n",
        "    'windowSize': 1,\n",
        "    'numOfKeywords': 3\n",
        "}\n",
        "\n",
        "LAN_PT = 'pt'\n",
        "LAN_EN = 'en'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BrZkp5yI_px-"
      },
      "source": [
        "# The root directory where the datasets are placed\n",
        "DATA_DIR = 'data_clean/'\n",
        "\n",
        "# Create python vars with the datasets location\n",
        "DATA_DIR_PUBLICO = os.path.join(DATA_DIR, 'publico/')\n",
        "NEWS_PUBLICO = os.path.join(DATA_DIR_PUBLICO, 'news_publico.json')\n",
        "\n",
        "DATA_DIR_OBSERVADOR = os.path.join(DATA_DIR, 'observador/')\n",
        "NEWS_OBSERVADOR = os.path.join(DATA_DIR_OBSERVADOR, 'news_observador.json')\n",
        "\n",
        "DATA_DIR_CNN = os.path.join(DATA_DIR, 'cnn/')\n",
        "NEWS_CNN = os.path.join(DATA_DIR_CNN, 'news_cnn.json')\n",
        "KMS_CNN = os.path.join(DATA_DIR_CNN, 'kms_cnn.json')\n",
        "\n",
        "DATA_DIR_GUARDIAN = os.path.join(DATA_DIR, 'guardian/')\n",
        "NEWS_GUARDIAN = os.path.join(DATA_DIR_GUARDIAN, 'news_guardian.json')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y6NC4xgf_6ZW"
      },
      "source": [
        "### Functions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jw0PnZ7t_8Hf"
      },
      "source": [
        "def concatenate_title_text(df_news):\n",
        "  try:\n",
        "    df_news['title_and_text'] = df_news['title'] + '. ' + df_news['text']\n",
        "  except KeyError:\n",
        "    df_news['title_and_text'] = df_news['title']\n",
        "\n",
        "  # Drop duplicates by title_and_text, keeping first occurence\n",
        "  df_news = df_news.drop_duplicates(subset=['title_and_text'])\n",
        "\n",
        "  return df_news\n",
        "\n",
        "# Return a list set (without duplicates)\n",
        "# as we are not interested in how many times an entity appears\n",
        "# but rather if it appears or not\n",
        "def get_ents(nlp, text):\n",
        "  \n",
        "  try:\n",
        "    doc = nlp(text)\n",
        "\n",
        "    ents = [ent.text.lower() for ent in doc.ents]\n",
        "  \n",
        "    return list(set(ents))\n",
        "\n",
        "  except TypeError as te:\n",
        "    return []\n",
        "\n",
        "def get_kws(kw_extractor, text):\n",
        "\n",
        "  try: \n",
        "    keywords = kw_extractor.extract_keywords(text)\n",
        "    kws_without_scores = [kw[0] for kw in keywords]\n",
        "\n",
        "    return list(set(kws_without_scores))\n",
        "  \n",
        "  except ValueError:\n",
        "    return []\n",
        "  except AttributeError:\n",
        "    return []\n",
        "\n",
        "def parse_name(text):\n",
        "  \n",
        "  text = text.replace(' ', '_')\n",
        "\n",
        "  text = text.lower()\n",
        "  text = unidecode.unidecode(text)\n",
        "\n",
        "  return text\n",
        "\n",
        "def write_json(file_path, json_data):\n",
        "    #print('Writing ' + file_path)\n",
        "    with open(file_path, 'w', encoding='utf8') as fp:\n",
        "        json.dump(json_data, fp, ensure_ascii=False, indent=4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gEZenugMnzG5"
      },
      "source": [
        "# Fucntions related to writing the datasets \n",
        "\n",
        "# Valid languages\n",
        "LANGUAGES = ['pt', 'en']\n",
        "\n",
        "\n",
        "def get_sents(text):\n",
        "  sentences = []\n",
        "  for paragraph in segmenter.process(text):\n",
        "    for sentence in paragraph:\n",
        "      s_sentence = ''\n",
        "      for token in sentence:\n",
        "        s_sentence += token.value + ' '\n",
        "      sentences.append(s_sentence)\n",
        "  \n",
        "  return sentences\n",
        "\n",
        "\n",
        "def get_inputDocs(df, topics):\n",
        "  gt = []\n",
        "  for i, topic in enumerate(topics):\n",
        "    print(str(i+1) + '/' + str(len(topics)))\n",
        "    print(topic)\n",
        "    inner_dict = {}\n",
        "    topic_name = topic[0]\n",
        "    # If single topic, without synonyms\n",
        "    if len(topic) == 1:\n",
        "      # topic_name = topic[0].replace('-', '_')\n",
        "      df_loc = df[df.apply(lambda r: topic_name in r['topics'], axis=1)][['date', 'title', 'text', 'url']].iloc[::-1]\n",
        "    # If topic with synonyms\n",
        "    else:\n",
        "      dfs = []\n",
        "      for st in topic:\n",
        "        df_sub = df[df.apply(lambda r: st in r['topics'], axis=1)][['date', 'title', 'text', 'url']].iloc[::-1]\n",
        "        dfs.append(df_sub)\n",
        "      # Merge topics\n",
        "      df_loc = pd.concat(dfs)\n",
        "      # Remove possible duplicates from merging topics\n",
        "      df_loc = df_loc.drop_duplicates(subset=['date', 'title', 'text', 'url'])\n",
        "      # topic_name = '-'.join([t.replace('-','_') for t in topic])\n",
        "    \n",
        "    inner_dict[topic_name] = df_loc.to_dict(orient='records')\n",
        "    gt.append(inner_dict)\n",
        "\n",
        "  # Convert to format {ent: {date: [list_news]}}\n",
        "  input_docs = {}\n",
        "  for topic in gt:\n",
        "    for k, v in topic.items():\n",
        "      inner_dict = {}\n",
        "      for news in v:\n",
        "        inner_dict.setdefault(news['date'],[]).append((news['title'], news['text'], news['url']))\n",
        "\n",
        "      input_docs[k] = inner_dict\n",
        "\n",
        "  return input_docs\n",
        "\n",
        "\n",
        "def write_inputDocs(dataset, lan, source_name, with_json=False):\n",
        "  \n",
        "  if lan not in LANGUAGES:\n",
        "    print('Please provide a valid language')\n",
        "    return\n",
        "\n",
        "  dataset_dir = 'dataset_' + lan\n",
        "\n",
        "  dataset_dir_txt = os.path.join(dataset_dir, 'txt/')\n",
        "  Path(dataset_dir_txt).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "  if with_json:\n",
        "    dataset_dir_json = os.path.join(dataset_dir, 'json/')\n",
        "    Path(dataset_dir_json).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "  source_name_suffix = '_' + source_name\n",
        "\n",
        "  for i, topic in enumerate(dataset):\n",
        "    print(str(i+1) + '/' + str(len(dataset)))\n",
        "    print(topic)\n",
        "\n",
        "    file_name = parse_name(topic)\n",
        "\n",
        "    # TXT\n",
        "    ent_dir_txt = os.path.join(dataset_dir_txt, file_name + source_name_suffix)\n",
        "    Path(ent_dir_txt).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "    ent_inputdocs_txt = os.path.join(ent_dir_txt, 'input_docs/')\n",
        "    Path(ent_inputdocs_txt).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "    if with_json:\n",
        "      # JSON\n",
        "      ent_dir_json = os.path.join(dataset_dir_json, file_name + source_name_suffix)\n",
        "      Path(ent_dir_json).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "      ent_inputdocs_json = os.path.join(ent_dir_json, 'input_docs/')\n",
        "      Path(ent_inputdocs_json).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "    for date, news in dataset[topic].items():\n",
        "\n",
        "      # TXT\n",
        "      ent_inputdocs_liveblog_day_txt = os.path.join(ent_inputdocs_txt, date)\n",
        "      Path(ent_inputdocs_liveblog_day_txt).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "      counter = 0\n",
        "      for n in news:\n",
        "        fn = str(counter) + '.txt'\n",
        "        file_path_txt_inputdocs = os.path.join(ent_inputdocs_liveblog_day_txt, fn)\n",
        "        with open(file_path_txt_inputdocs, 'a') as txt_file:\n",
        "          # n[1] is text\n",
        "          news_sents = get_sents(n[1])\n",
        "          for ns in news_sents:\n",
        "            txt_file.write(ns + '\\n')\n",
        "        counter += 1\n",
        "\n",
        "      if with_json:\n",
        "        # JSON\n",
        "        ent_inputdocs_liveblog_day_json = os.path.join(ent_inputdocs_json, date)\n",
        "        Path(ent_inputdocs_liveblog_day_json).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "        counter = 0\n",
        "        for n in news:\n",
        "          fn = str(counter) + '.json'\n",
        "          file_path_json_inputdocs = os.path.join(ent_inputdocs_liveblog_day_json, fn)\n",
        "          # n[1] is text\n",
        "          news_sents = get_sents(n[1])\n",
        "          write_json(file_path_json_inputdocs, news_sents)\n",
        "          counter += 1\n",
        "        \n",
        "\n",
        "def get_timelines(df, topics):\n",
        "  gt = []\n",
        "\n",
        "  for i, topic in enumerate(topics):\n",
        "    print(str(i+1) + '/' + str(len(topics)))\n",
        "    print(topic)\n",
        "    inner_dict = {}\n",
        "    topic_name = topic[0]\n",
        "    # If single topic, without synonyms\n",
        "    if len(topic) == 1:\n",
        "      # topic_name = topic[0].replace('-', '_')\n",
        "      df_loc = df[df.apply(lambda r: topic_name in r['topics'], axis=1)][['date', 'title', 'text', 'url']].iloc[::-1]\n",
        "    # If topic with synonyms\n",
        "    else:\n",
        "      dfs = []\n",
        "      for st in topic:\n",
        "        df_loc = df[df.apply(lambda r: st in r['topics'], axis=1)][['date', 'title', 'text', 'url']].iloc[::-1]\n",
        "        dfs.append(df_loc)\n",
        "      # Merge topics\n",
        "      df_loc = pd.concat(dfs)\n",
        "      # Remove possible duplicates from merging topics\n",
        "      df_loc = df_loc.drop_duplicates(subset=['date', 'title', 'text', 'url'])\n",
        "      # topic_name = '-'.join([t.replace('-','_') for t in topic])\n",
        "\n",
        "    inner_dict[topic_name] = df_loc.to_dict(orient='records')\n",
        "    gt.append(inner_dict)\n",
        "\n",
        "  # Convert to format {ent: {date: [list_news]}}\n",
        "  timelines = {}\n",
        "  for topic in gt:\n",
        "    for k, v in topic.items():\n",
        "      inner_dict = {}\n",
        "      for news in v:\n",
        "        inner_dict.setdefault(news['date'],[]).append((news['title'], news['text'], news['url']))\n",
        "\n",
        "      timelines[k] = inner_dict\n",
        "\n",
        "  return timelines\n",
        "\n",
        "\n",
        "def write_timelines(dataset, lan, source_name, with_json=False):\n",
        "\n",
        "  if lan not in LANGUAGES:\n",
        "    print('Please provide a valid language')\n",
        "    return\n",
        "\n",
        "  dataset_dir = 'dataset_' + lan\n",
        "\n",
        "  dataset_dir_txt = os.path.join(dataset_dir, 'txt/')\n",
        "  Path(dataset_dir_txt).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "  if with_json:\n",
        "    dataset_dir_json = os.path.join(dataset_dir, 'json/')\n",
        "    Path(dataset_dir_json).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "  source_name_suffix = '_' + source_name\n",
        "\n",
        "  txt_delimiter = '--------------------------------'\n",
        "\n",
        "  for i, topic in enumerate(dataset):\n",
        "    print(str(i+1) + '/' + str(len(dataset)))\n",
        "    print(topic)\n",
        "\n",
        "    file_name = parse_name(topic)\n",
        "\n",
        "    # TXT\n",
        "    file_name_txt = file_name + '.txt'\n",
        "\n",
        "    ent_timelines_keymoments_txt = os.path.join(dataset_dir_txt, file_name + source_name_suffix, 'timelines')\n",
        "    Path(ent_timelines_keymoments_txt).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "    file_path_txt = os.path.join(ent_timelines_keymoments_txt, file_name_txt)\n",
        "\n",
        "    for k, v in dataset[topic].items():\n",
        "      with open(file_path_txt, 'a') as txt_file:\n",
        "        txt_file.write(k + '\\n')\n",
        "        for news in v:\n",
        "          # news[0] is title\n",
        "          txt_file.write(news[0] + '\\n')\n",
        "        txt_file.write(txt_delimiter + '\\n')\n",
        "    \n",
        "    if with_json:\n",
        "      # JSON\n",
        "      file_name_json = file_name + '.json'\n",
        "\n",
        "      ent_timelines_keymoments_json = os.path.join(dataset_dir_json, file_name + source_name_suffix, 'timelines')\n",
        "      Path(ent_timelines_keymoments_json).mkdir(parents=True, exist_ok=True)\n",
        "\n",
        "      file_path_json = os.path.join(ent_timelines_keymoments_json, file_name_json)\n",
        "      \n",
        "      json_data_to_write = {}\n",
        "      for date, news_list in dataset[topic].items():\n",
        "        json_data_to_write[date] = [news[0] for news in news_list]\n",
        "\n",
        "      write_json(file_path_json, json_data_to_write)\n",
        "\n",
        "\n",
        "# Function to write data in bulk to json\n",
        "def data_to_cmh(data, source, lan):\n",
        "  news_list = []\n",
        "\n",
        "  for topic, content in data.items():\n",
        "    for date, news in content.items():\n",
        "      for n in news:\n",
        "        news_dict = {}\n",
        "        news_dict['source'] = source\n",
        "        news_dict['lan'] = lan\n",
        "        news_dict['topic'] = topic\n",
        "        news_dict['date'] = str(date)\n",
        "        news_dict['title'] = n[0]\n",
        "        news_dict['news'] = n[1]\n",
        "        news_dict['url'] = n[2]\n",
        "        news_list.append(news_dict)\n",
        "\n",
        "  return news_list"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wD0b2RG_ARiR"
      },
      "source": [
        "### PT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XvScaP9-AVBV"
      },
      "source": [
        "# Load Spacy pt ner model\n",
        "nlp_ner_pt = pt_core_news_sm.load(disable=['tagger', 'parser'])\n",
        "\n",
        "# YAKE pt\n",
        "YAKE_PARAMS['language'] = LAN_PT\n",
        "\n",
        "kw_extractor_pt = yake.KeywordExtractor(\n",
        "            lan=YAKE_PARAMS['language'], top=YAKE_PARAMS['numOfKeywords'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o_VCMaHOATFb"
      },
      "source": [
        "#### Publico"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZKXIkOwLxQZe"
      },
      "source": [
        "df_publico_lb = pd.read_json(NEWS_PUBLICO)\n",
        "\n",
        "# Create a new column with the concatenation of title and text (used to find topics)\n",
        "df_publico_lb = concatenate_title_text(df_publico_lb)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A3PmMv9b4x6D"
      },
      "source": [
        "##### Get topics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0wj6q11Q52q_"
      },
      "source": [
        "print('Getting spaCy entities')\n",
        "df_publico_lb['ents'] = df_publico_lb['title_and_text'].progress_apply(lambda t: get_ents(nlp_ner_pt, t))\n",
        "\n",
        "print('Getting yake keywords')\n",
        "df_publico_lb['kws'] = df_publico_lb['title_and_text'].progress_apply(lambda t: get_kws(kw_extractor_pt, t))\n",
        "\n",
        "print('Merging entities and keywords')\n",
        "df_publico_lb['topics'] = df_publico_lb.progress_apply(lambda r: list(set(r['ents'] + r['kws'])), axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZsJgalm33e4L"
      },
      "source": [
        "#### Observador"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rqRYAUJe3e4q"
      },
      "source": [
        "df_observador_lb = pd.read_json(NEWS_OBSERVADOR)\n",
        "\n",
        "# Create a new column with the concatenation of title and text (used to find topics)\n",
        "df_observador_lb = concatenate_title_text(df_observador_lb)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IEXo4fCR3e4t"
      },
      "source": [
        "##### Get topics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VGpm3g6X3e4u"
      },
      "source": [
        "print('Getting spaCy entities')\n",
        "df_observador_lb['ents'] = df_observador_lb['title_and_text'].progress_apply(lambda t: get_ents(nlp_ner_pt, t))\n",
        "\n",
        "print('Getting yake keywords')\n",
        "df_observador_lb['kws'] = df_observador_lb['title_and_text'].progress_apply(lambda t: get_kws(kw_extractor_pt, t))\n",
        "\n",
        "print('Merging entities and keywords')\n",
        "df_observador_lb['topics'] = df_observador_lb.progress_apply(lambda r: list(set(r['ents'] + r['kws'])), axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1W1zWoQG3e4v"
      },
      "source": [
        "#### Topics selection"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SOYWtRM3oNJe"
      },
      "source": [
        "##### Count topics occurences"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k_aXWHKnV7x9"
      },
      "source": [
        "# Topics are chosen based on their occurence in key moments\n",
        "\n",
        "df_publico_km = df_publico_lb.loc[df_publico_lb['is_km'] == 'True']\n",
        "\n",
        "df_observador_km = df_observador_lb.loc[df_observador_lb['is_km'] == 'True']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kY3xmp19SDZu"
      },
      "source": [
        "# For each source create a list of lists with topics. Each inner list corresponds to a news. Used to keep track of the number of news in which each topic appears\n",
        "\n",
        "publico_topics_per_news_km = df_publico_km.topics.tolist()\n",
        "\n",
        "observador_topics_per_news_km = df_observador_km.topics.tolist()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4EhHSfEV6Epx"
      },
      "source": [
        "# List topics for Portuguese\n",
        "\n",
        "publico_km_topics = set([a for b in publico_topics_per_news_km for a in b])\n",
        "\n",
        "observador_km_topics = set([a for b in observador_topics_per_news_km for a in b])\n",
        "\n",
        "pt_km_topics = list(set(publico_km_topics | observador_km_topics))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YIQSw5rYADIA"
      },
      "source": [
        "# For each topic count the number of key moments per source in which it appears\n",
        "\n",
        "pt_km_topics_count = {}\n",
        "\n",
        "for topic in tqdm(pt_km_topics):\n",
        "    \n",
        "  source_topic_count = {}\n",
        "\n",
        "  source_topic_count['publico'] = len([list_of_topics for list_of_topics in publico_topics_per_news_km if topic in list_of_topics])\n",
        "  \n",
        "  source_topic_count['observador'] = len([list_of_topics for list_of_topics in observador_topics_per_news_km if topic in list_of_topics])\n",
        "\n",
        "  pt_km_topics_count[topic] = source_topic_count\n",
        "\n",
        "len(pt_km_topics_count)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p9CNDd-X3e4w"
      },
      "source": [
        "# Filter topics by minimum number of occurrences\n",
        "\n",
        "MIN_NUM_KEY_MOMENTS = 5\n",
        "\n",
        "for topic, counts in list(pt_km_topics_count.items()):\n",
        "\n",
        "  # delete topic if there are less than MIN_NUM_KEY_MOMENTS key moments in one of the sources\n",
        "  if (counts['publico'] < MIN_NUM_KEY_MOMENTS or counts['observador'] < MIN_NUM_KEY_MOMENTS):\n",
        "    del pt_km_topics_count[topic]\n",
        "\n",
        "len(pt_km_topics_count)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ICzvgwm53e4w"
      },
      "source": [
        "##### Keymoments/liveblog ratio"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nrqm4CIjCA41"
      },
      "source": [
        "# For each source create a list of lists with topics. Each inner list corresponds ta a news. Used to keep track of the number of news in which each topic appears\n",
        "\n",
        "publico_topics_per_news_lb = df_publico_lb.topics.tolist()\n",
        "\n",
        "observador_topics_per_news_lb = df_observador_lb.topics.tolist()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RP5b3Hnb3e4x"
      },
      "source": [
        "# For each topic copute a ratio of occurences between key moments and liveblog per source\n",
        "\n",
        "ratio_dict_pt = {}\n",
        "\n",
        "for topic in tqdm(pt_km_topics_count):\n",
        "\n",
        "  # Count occurences in liveblog\n",
        "  publico_lb_count = len([list_of_topics for list_of_topics in publico_topics_per_news_lb if topic in list_of_topics])\n",
        "  observador_lb_count = len([list_of_topics for list_of_topics in observador_topics_per_news_lb if topic in list_of_topics])\n",
        "\n",
        "  publico_km_count = pt_km_topics_count[topic]['publico']\n",
        "  observador_km_count = pt_km_topics_count[topic]['observador']\n",
        "\n",
        "  inner_dict = {}\n",
        "\n",
        "  inner_dict['publico_lb_count'] = publico_lb_count\n",
        "  inner_dict['publico_km_count'] = publico_km_count\n",
        "  if publico_lb_count < 1:\n",
        "    inner_dict['publico_ratio_km_lb'] = 0\n",
        "  else:\n",
        "    inner_dict['publico_ratio_km_lb'] = publico_km_count/publico_lb_count\n",
        "\n",
        "  inner_dict['observador_lb_count'] = observador_lb_count\n",
        "  inner_dict['observador_km_count'] = observador_km_count\n",
        "  if observador_lb_count < 1:\n",
        "    inner_dict['observador_ratio_km_lb'] = 0\n",
        "  else:\n",
        "    inner_dict['observador_ratio_km_lb'] = observador_km_count/observador_lb_count\n",
        "\n",
        "  ratio_dict_pt[topic] = inner_dict\n",
        "\n",
        "len(ratio_dict_pt)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vk8gQ4a43e4x"
      },
      "source": [
        "RATIO = 0.5\n",
        "\n",
        "for topic, counts in list(ratio_dict_pt.items()):\n",
        "  if counts['publico_ratio_km_lb'] > RATIO or counts['observador_ratio_km_lb'] > RATIO:\n",
        "    del ratio_dict_pt[topic]\n",
        "\n",
        "len(ratio_dict_pt)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8S_yjnYEXHIR"
      },
      "source": [
        "rdp = pd.DataFrame.from_dict(ratio_dict_pt, orient='index')\n",
        "rdp = rdp.sort_index(ascending=True)\n",
        "rdp.to_excel(\"ratio_dict_pt.xlsx\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pdGmhM-x6jw6"
      },
      "source": [
        "# Sort ratio dict by key (topic)\n",
        "ratio_dict_pt = {k: ratio_dict_pt[k] for k in sorted(ratio_dict_pt)}\n",
        "\n",
        "# Use list of lists to handle synonyms\n",
        "topics_pt = [[t] for t in list(ratio_dict_pt.keys())]\n",
        "print(sum(len(t) for t in topics_pt))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fLabW79rDy3D"
      },
      "source": [
        "write_json('ratio_dict_pt.json', ratio_dict_pt)\n",
        "\n",
        "with open('topics_pt.txt', 'w') as f:\n",
        "  for topic in topics_pt:\n",
        "    f.write(\"%s\\n\" % topic[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "slqqZG9r3e4x"
      },
      "source": [
        "##### Filter queries manually"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7m5VNegI3e4y"
      },
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ub70fKm93e4z"
      },
      "source": [
        "with open('topics_pt.txt', 'r') as fp:\n",
        "  topics_pt = [line.rstrip() for line in fp]\n",
        "\n",
        "# There may be one topic per line, or multiple topics (synonyms) per line separated by comma\n",
        "topics_pt = [t.split(',') for t in topics_pt]\n",
        "\n",
        "# Check if all topics exist in the original topics identification\n",
        "for topics in topics_pt:\n",
        "  for t in topics:\n",
        "    if t not in list(ratio_dict_pt.keys()):\n",
        "      print(t + ' does not exist')\n",
        "\n",
        "print(sum(len(t) for t in topics_pt))\n",
        "print(len(topics_pt))\n",
        "print(topics_pt)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EqC62npXwBPv"
      },
      "source": [
        "#### Write dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vXHXUimjzKcU"
      },
      "source": [
        "##### Input Docs\n",
        "\n",
        "News texts from whole liveblog"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9kgEr0Flw_L9"
      },
      "source": [
        "# Remove time from date and convert it to string\n",
        "\n",
        "df_publico_lb['date'] = pd.to_datetime(df_publico_lb['date']).dt.date.astype(str)\n",
        "\n",
        "df_observador_lb['date'] = pd.to_datetime(df_observador_lb['date']).dt.date.astype(str)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PfR8b-zTxBJg"
      },
      "source": [
        "inputDocs_publico = get_inputDocs(df_publico_lb, topics_pt)\n",
        "\n",
        "inputDocs_observador = get_inputDocs(df_observador_lb, topics_pt)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "754PU6e2Pj7n"
      },
      "source": [
        "# Generate TLS dataset\n",
        "\n",
        "print('Writing publico input docs dataset')\n",
        "write_inputDocs(inputDocs_publico, 'pt', 'publico', with_json=False)\n",
        "\n",
        "print('Writing observador input docs dataset')\n",
        "write_inputDocs(inputDocs_observador, 'pt', 'observador', with_json=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QTAPrLpjxELG"
      },
      "source": [
        "# Generate CMH dataset\n",
        "\n",
        "publico_lb_json = data_to_cmh(inputDocs_publico, 'publico', 'pt')\n",
        "write_json('publico_lb.json', publico_lb_json)\n",
        "\n",
        "observador_lb_json = data_to_cmh(inputDocs_observador, 'observador', 'pt')\n",
        "write_json('observador_lb.json', observador_lb_json)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N8t_wXYnzMWJ"
      },
      "source": [
        "##### Timelines\n",
        "\n",
        "News titles from key moments"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bq-ZS8lczOZR"
      },
      "source": [
        "# Remove time from date and convert it to string\n",
        "\n",
        "df_publico_km['date'] = pd.to_datetime(df_publico_km['date']).dt.date.astype(str)\n",
        "\n",
        "df_observador_km['date'] = pd.to_datetime(df_observador_km['date']).dt.date.astype(str)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Er3s2rYmzQ9A"
      },
      "source": [
        "timelines_publico = get_timelines(df_publico_km, topics_pt)\n",
        "\n",
        "timelines_observador = get_timelines(df_observador_km, topics_pt)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yq-784iLUMYO"
      },
      "source": [
        "# Generate TLS dataset\n",
        "\n",
        "print('Writing publico timeline dataset')\n",
        "write_timelines(timelines_publico, 'pt', 'publico', with_json=False)\n",
        "\n",
        "print('Writing observador timeline dataset')\n",
        "write_timelines(timelines_observador, 'pt', 'observador', with_json=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cvGZ3pCFzTE6"
      },
      "source": [
        "# Generate CMH dataset\n",
        "\n",
        "publico_km_json = data_to_cmh(timelines_publico, 'publico', 'pt')\n",
        "write_json('publico_km.json', publico_km_json)\n",
        "\n",
        "observador_km_json = data_to_cmh(timelines_observador, 'observador', 'pt')\n",
        "write_json('observador_km.json', observador_km_json)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JCm7h1AWZh69"
      },
      "source": [
        "from datetime import datetime\n",
        "\n",
        "# Save data to drive\n",
        "TLS_DATA_DIR = 'dataset_pt/'\n",
        "\n",
        "zip_file_name = 'dataset_pt-' + datetime.today().strftime(\"%Y-%m-%d\") + '.zip'\n",
        "\n",
        "!zip -r $zip_file_name $TLS_DATA_DIR\n",
        "\n",
        "\n",
        "DATA_STORE_DIR = os.path.join(DRIVE_MOUNT_DIR, 'MyDrive/inesc/tls_covid')\n",
        "\n",
        "!cp -r $zip_file_name $DATA_STORE_DIR"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eEbVeFPgel2a"
      },
      "source": [
        "### EN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jf4i_-ljel2j"
      },
      "source": [
        "# Load Spacy en ner model\n",
        "nlp_ner_en = en_core_web_sm.load(disable=['tagger', 'parser'])\n",
        "\n",
        "# YAKE en\n",
        "YAKE_PARAMS['language'] = LAN_EN\n",
        "\n",
        "kw_extractor_en = yake.KeywordExtractor(\n",
        "            lan=YAKE_PARAMS['language'], top=YAKE_PARAMS['numOfKeywords'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wqzBycl9el2j"
      },
      "source": [
        "#### CNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O8yo4fDqel2j"
      },
      "source": [
        "df_cnn_lb = pd.read_json(NEWS_CNN)\n",
        "\n",
        "# Create a new column with the concatenation of title and text (used to find topics)\n",
        "df_cnn_lb = concatenate_title_text(df_cnn_lb)\n",
        "\n",
        "df_cnn_km = pd.read_json(KMS_CNN)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JYZfDKPUel2k"
      },
      "source": [
        "##### Get topics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "95tvYpwUel2k"
      },
      "source": [
        "print('Getting spaCy entities')\n",
        "df_cnn_km['ents'] = df_cnn_km['title'].progress_apply(lambda t: get_ents(nlp_ner_en, t))\n",
        "\n",
        "print('Getting yake keywords')\n",
        "df_cnn_km['kws'] = df_cnn_km['title'].progress_apply(lambda t: get_kws(kw_extractor_en, t))\n",
        "\n",
        "print('Merging entities and keywords')\n",
        "df_cnn_km['topics'] = df_cnn_km.progress_apply(lambda r: list(set(r['ents'] + r['kws'])), axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qL9ek9VDhG1K"
      },
      "source": [
        "print('Getting spaCy entities')\n",
        "df_cnn_lb['ents'] = df_cnn_lb['title_and_text'].progress_apply(lambda t: get_ents(nlp_ner_en, t))\n",
        "\n",
        "print('Getting yake keywords')\n",
        "df_cnn_lb['kws'] = df_cnn_lb['title_and_text'].progress_apply(lambda t: get_kws(kw_extractor_en, t))\n",
        "\n",
        "print('Merging entities and keywords')\n",
        "df_cnn_lb['topics'] = df_cnn_lb.progress_apply(lambda r: list(set(r['ents'] + r['kws'])), axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NmDUfdXEel2k"
      },
      "source": [
        "#### Guardian"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W3X5ZtN1el2k"
      },
      "source": [
        "df_guardian_lb = pd.read_json(NEWS_GUARDIAN)\n",
        "\n",
        "# Create a new column with the concatenation of title and text (used to find topics)\n",
        "df_guardian_lb = concatenate_title_text(df_guardian_lb)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fsiJvhvcel2k"
      },
      "source": [
        "##### Get topics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-DoaSfKCel2l"
      },
      "source": [
        "print('Getting spaCy entities')\n",
        "df_guardian_lb['ents'] = df_guardian_lb['title_and_text'].progress_apply(lambda t: get_ents(nlp_ner_en, t))\n",
        "\n",
        "print('Getting yake keywords')\n",
        "df_guardian_lb['kws'] = df_guardian_lb['title_and_text'].progress_apply(lambda t: get_kws(kw_extractor_en, t))\n",
        "\n",
        "print('Merging entities and keywords')\n",
        "df_guardian_lb['topics'] = df_guardian_lb.progress_apply(lambda r: list(set(r['ents'] + r['kws'])), axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9On2JT2wel2l"
      },
      "source": [
        "#### Topics selection"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RB0LMHtKel2l"
      },
      "source": [
        "##### Count topics occurences"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L8NeMGxPel2l"
      },
      "source": [
        "# Topics are chosen based on their occurence in key moments\n",
        "\n",
        "df_cnn_km = df_cnn_km\n",
        "\n",
        "df_guardian_km = df_guardian_lb.loc[df_guardian_lb['is_km'] == 'True']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aK16nRrRel2l"
      },
      "source": [
        "# For each source create a list of lists with topics. Each inner list corresponds to a news. Used to keep track of the number of news in which each topic appears\n",
        "\n",
        "cnn_topics_per_news_km = df_cnn_km.topics.tolist()\n",
        "\n",
        "guardian_topics_per_news_km = df_guardian_km.topics.tolist()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Pw2UcS-el2m"
      },
      "source": [
        "# List topics for English\n",
        "\n",
        "cnn_km_topics = set([a for b in cnn_topics_per_news_km for a in b])\n",
        "\n",
        "guardian_km_topics = set([a for b in guardian_topics_per_news_km for a in b])\n",
        "\n",
        "en_km_topics = list(set(cnn_km_topics | guardian_km_topics))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b99bqEV-el2m"
      },
      "source": [
        "# For each topic count the number of key moments per source in which it appears\n",
        "\n",
        "en_km_topics_count = {}\n",
        "\n",
        "for topic in tqdm(en_km_topics):\n",
        "    \n",
        "  source_topic_count = {}\n",
        "\n",
        "  source_topic_count['cnn'] = len([list_of_topics for list_of_topics in cnn_topics_per_news_km if topic in list_of_topics])\n",
        "  \n",
        "  source_topic_count['guardian'] = len([list_of_topics for list_of_topics in guardian_topics_per_news_km if topic in list_of_topics])\n",
        "\n",
        "  en_km_topics_count[topic] = source_topic_count\n",
        "\n",
        "len(en_km_topics_count)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BlUp3TjRel2m"
      },
      "source": [
        "# Filter topics by minimum number of occurrences in key moments\n",
        "\n",
        "MIN_NUM_KEY_MOMENTS = 5\n",
        "\n",
        "for topic, counts in list(en_km_topics_count.items()):\n",
        "\n",
        "  # delete topic if there are less than MIN_NUM_KEY_MOMENTS key moments in one of the sources\n",
        "  if (counts['cnn'] < MIN_NUM_KEY_MOMENTS or counts['guardian'] < MIN_NUM_KEY_MOMENTS):\n",
        "    del en_km_topics_count[topic]\n",
        "\n",
        "len(en_km_topics_count)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QMoEGFBVel2m"
      },
      "source": [
        "##### Keymoments/liveblog ratio"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rut1ddQIel2m"
      },
      "source": [
        "# For each source create a list of lists with topics. Each inner list corresponds ta a news. Used to keep track of the number of news in which each topic appears\n",
        "\n",
        "cnn_topics_per_news_lb = df_cnn_lb.topics.tolist()\n",
        "\n",
        "guardian_topics_per_news_lb = df_guardian_lb.topics.tolist()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V8rG8EhQel2m"
      },
      "source": [
        "# For each topic copute a ratio of occurences between key moments and liveblog per source\n",
        "\n",
        "ratio_dict_en = {}\n",
        "\n",
        "for topic in tqdm(en_km_topics_count):\n",
        "\n",
        "  # Count occurences in liveblog\n",
        "  cnn_lb_count = len([list_of_topics for list_of_topics in cnn_topics_per_news_lb if topic in list_of_topics])\n",
        "  guardian_lb_count = len([list_of_topics for list_of_topics in guardian_topics_per_news_lb if topic in list_of_topics])\n",
        "\n",
        "  cnn_km_count = en_km_topics_count[topic]['cnn']\n",
        "  guardian_km_count = en_km_topics_count[topic]['guardian']\n",
        "\n",
        "  inner_dict = {}\n",
        "\n",
        "  inner_dict['cnn_lb_count'] = cnn_lb_count\n",
        "  inner_dict['cnn_km_count'] = cnn_km_count\n",
        "  if cnn_lb_count < 1:\n",
        "    inner_dict['cnn_ratio_km_lb'] = 0\n",
        "  else:\n",
        "    inner_dict['cnn_ratio_km_lb'] = cnn_km_count/cnn_lb_count\n",
        "\n",
        "  inner_dict['guardian_lb_count'] = guardian_lb_count\n",
        "  inner_dict['guardian_km_count'] = guardian_km_count\n",
        "  if guardian_lb_count < 1:\n",
        "    inner_dict['guardian_ratio_km_lb'] = 0\n",
        "  else:\n",
        "    inner_dict['guardian_ratio_km_lb'] = guardian_km_count/guardian_lb_count\n",
        "\n",
        "  ratio_dict_en[topic] = inner_dict\n",
        "\n",
        "len(ratio_dict_en)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PFJO6eLbel2m"
      },
      "source": [
        "RATIO = 0.5\n",
        "\n",
        "for topic, counts in list(ratio_dict_en.items()):\n",
        "  if counts['cnn_ratio_km_lb'] > RATIO or counts['guardian_ratio_km_lb'] > RATIO:\n",
        "    del ratio_dict_en[topic]\n",
        "\n",
        "len(ratio_dict_en)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xp7nFbWTD2LU"
      },
      "source": [
        "rde = pd.DataFrame.from_dict(ratio_dict_en, orient='index')\n",
        "rde = rde.sort_index(ascending=True)\n",
        "rde.to_excel(\"ratio_dict_en.xlsx\") "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wNZbX659el2n"
      },
      "source": [
        "# Sort ratio dict by key (topic)\n",
        "ratio_dict_en = {k: ratio_dict_en[k] for k in sorted(ratio_dict_en)}\n",
        "\n",
        "# Use list of lists to be prepared to synonyms\n",
        "topics_en = [[t] for t in list(ratio_dict_en.keys())]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CbnWW9v4el2n"
      },
      "source": [
        "write_json('ratio_dict_en.json', ratio_dict_en)\n",
        "\n",
        "with open('topics_en.txt', 'w') as f:\n",
        "  for topic in topics_en:\n",
        "    f.write(\"%s\\n\" % topic[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VEI92DRuel2n"
      },
      "source": [
        "##### Filter queries manually"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qHKiVFEVel2n"
      },
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SxXcWufSel2n"
      },
      "source": [
        "with open('topics_en.txt', 'r') as fp:\n",
        "  topics_en = [line.rstrip() for line in fp]\n",
        "\n",
        "# There may be one topic per line, or multiple topics (synonyms) per line separated by comma\n",
        "topics_en = [t.split(',') for t in topics_en]\n",
        "\n",
        "# Check if all topics exist in the original topics identification\n",
        "for topics in topics_en:\n",
        "  for t in topics:\n",
        "    if t not in list(ratio_dict_en.keys()):\n",
        "      print(t + ' does not exist')\n",
        "\n",
        "print(sum(len(t) for t in topics_en))\n",
        "print(len(topics_en))\n",
        "print(topics_en)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kkQKctw-el2n"
      },
      "source": [
        "#### Write dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZvDMaz7jel2o"
      },
      "source": [
        "##### Input Docs\n",
        "\n",
        "News texts from whole liveblog"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LdOlHjbjel2o"
      },
      "source": [
        "# Remove time from date and convert it to string\n",
        "\n",
        "df_cnn_lb['date'] = pd.to_datetime(df_cnn_lb['date']).dt.date.astype(str)\n",
        "\n",
        "df_guardian_lb['date'] = pd.to_datetime(df_guardian_lb['date']).dt.date.astype(str)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nD2h0pivel2o"
      },
      "source": [
        "inputDocs_cnn = get_inputDocs(df_cnn_lb, topics_en)\n",
        "\n",
        "inputDocs_guardian = get_inputDocs(df_guardian_lb, topics_en)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ACMCVnBhel2o"
      },
      "source": [
        "# Generate TLS dataset\n",
        "\n",
        "print('Writing cnn input docs dataset')\n",
        "write_inputDocs(inputDocs_cnn, 'en', 'cnn', with_json=False)\n",
        "\n",
        "print('Writing guardian input docs dataset')\n",
        "write_inputDocs(inputDocs_guardian, 'en', 'guardian', with_json=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D4VX-nS9el2p"
      },
      "source": [
        "# Generate CMH dataset\n",
        "\n",
        "cnn_lb_json = data_to_cmh(inputDocs_cnn, 'cnn', 'en')\n",
        "write_json('cnn_lb.json', cnn_lb_json)\n",
        "\n",
        "guardian_lb_json = data_to_cmh(inputDocs_guardian, 'guardian', 'en')\n",
        "write_json('guardian_lb.json', guardian_lb_json)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dxxB4E7Jel2p"
      },
      "source": [
        "##### Timelines\n",
        "\n",
        "News titles from key moments"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wDO907Z0el2p"
      },
      "source": [
        "# Remove time from date and convert it to string\n",
        "\n",
        "df_cnn_km['date'] = pd.to_datetime(df_cnn_km['date']).dt.date.astype(str)\n",
        "\n",
        "df_guardian_km['date'] = pd.to_datetime(df_guardian_km['date']).dt.date.astype(str)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v2O7A6Leel2p"
      },
      "source": [
        "timelines_cnn = get_timelines(df_cnn_km, topics_en)\n",
        "\n",
        "timelines_guardian = get_timelines(df_guardian_km, topics_en)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0iNlbwapel2p"
      },
      "source": [
        "# Generate TLS dataset\n",
        "\n",
        "print('Writing cnn timeline dataset')\n",
        "write_timelines(timelines_cnn, 'en', 'cnn', with_json=False)\n",
        "\n",
        "print('Writing guardian timeline dataset')\n",
        "write_timelines(timelines_guardian, 'en', 'guardian', with_json=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dRoCudKxel2p"
      },
      "source": [
        "# Generate CMH dataset\n",
        "\n",
        "cnn_km_json = data_to_cmh(timelines_cnn, 'cnn', 'en')\n",
        "write_json('cnn_km.json', cnn_km_json)\n",
        "\n",
        "guardian_km_json = data_to_cmh(timelines_guardian, 'guardian', 'en')\n",
        "write_json('guardian_km.json', guardian_km_json)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kN4xgrH7el2p"
      },
      "source": [
        "from datetime import datetime\n",
        "\n",
        "# Save data to drive\n",
        "TLS_DATA_DIR = 'dataset_en/'\n",
        "\n",
        "zip_file_name = 'dataset_en-' + datetime.today().strftime(\"%Y-%m-%d\") + '.zip'\n",
        "\n",
        "!zip -r $zip_file_name $TLS_DATA_DIR\n",
        "\n",
        "\n",
        "# DATA_STORE_DIR = os.path.join(DRIVE_MOUNT_DIR, 'MyDrive/inesc/tls_covid')\n",
        "\n",
        "# !cp -r $zip_file_name $DATA_STORE_DIR"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1OXOuRhBpQ6n"
      },
      "source": [
        "## Statistics\n",
        "\n",
        "Statistics provided in paper"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZCro9cVXpbJW"
      },
      "source": [
        "stats_dict = {}\n",
        "\n",
        "def get_topics_freqs(list_of_topics, ratio_dict, lan):\n",
        "  topic_freq = {}\n",
        "\n",
        "  for topic in list_of_topics:\n",
        "    # Aggregate all synonym topics into first one\n",
        "    topic_name = topic[0]\n",
        "    freq = 0\n",
        "    for t in topic:\n",
        "      if lan == 'pt':\n",
        "        freq += ratio_dict[t]['publico_km_count'] + ratio_dict[t]['publico_lb_count'] + ratio_dict[t]['observador_km_count'] + ratio_dict[t]['observador_lb_count']\n",
        "      elif lan == 'en':\n",
        "        freq += ratio_dict[t]['cnn_km_count'] + ratio_dict[t]['cnn_lb_count'] + ratio_dict[t]['guardian_km_count'] + ratio_dict[t]['guardian_lb_count']\n",
        "\n",
        "    topic_freq[topic_name] = freq\n",
        "\n",
        "  return topic_freq\n",
        "\n",
        "def get_num_sents(text):\n",
        "  sentences = []\n",
        "  for paragraph in segmenter.process(text):\n",
        "    for sentence in paragraph:\n",
        "      s_sentence = ''\n",
        "      for token in sentence:\n",
        "        s_sentence += token.value + ' '\n",
        "      sentences.append(s_sentence)\n",
        "  \n",
        "  return len(sentences)\n",
        "\n",
        "def count_sents_timelines(news_dict):\n",
        "  count = 0\n",
        "  for _, n in news_dict.items():\n",
        "    for _, l in n.items():\n",
        "      for news in l:\n",
        "        count += get_num_sents(news[0])\n",
        "  return count\n",
        "\n",
        "def count_sents_inputDocs(news_dict):\n",
        "  count = 0\n",
        "  for _, n in news_dict.items():\n",
        "    for _, l in n.items():\n",
        "      for news in l:\n",
        "        count += get_num_sents(news[1])\n",
        "  return count\n",
        "\n",
        "def get_dates_list(news_dict):\n",
        "  dates = []\n",
        "  for _, n in news_dict.items():\n",
        "    for d in n:\n",
        "      dates.append(d)\n",
        "  \n",
        "  return dates"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5UUq-KNg0Gg6"
      },
      "source": [
        "### WordClouds"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KDRrrEcB0Tgd"
      },
      "source": [
        "from wordcloud import WordCloud\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nd65s7-eqLuf"
      },
      "source": [
        "#### PT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I_h2zd9FnQlz"
      },
      "source": [
        "topic_freq_pt = get_topics_freqs(topics_pt, ratio_dict_pt, 'pt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tFL8Qo4Tzu-Q"
      },
      "source": [
        "wordcloud = WordCloud(background_color='white', max_font_size=100, width=512, height=512).generate_from_frequencies(topic_freq_pt)\n",
        "plt.figure(figsize=(16,9))\n",
        "plt.imshow(wordcloud)\n",
        "plt.axis(\"off\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kJzyqz2lqJai"
      },
      "source": [
        "#### EN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jYC6HfTmxuaF"
      },
      "source": [
        "topic_freq_en = get_topics_freqs(topics_en, ratio_dict_en, 'en')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KplyAaT3zc5z"
      },
      "source": [
        "wordcloud = WordCloud(background_color='white', min_font_size=20, relative_scaling=0, max_font_size=100, prefer_horizontal=1, scale=10, width=512, height=512).generate_from_frequencies(topic_freq_en)\n",
        "\n",
        "plt.figure(figsize=(16,9))\n",
        "plt.imshow(wordcloud)\n",
        "plt.axis(\"off\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K-P4d6ZdpKae"
      },
      "source": [
        "### PT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dwOWToJGtuhJ"
      },
      "source": [
        "# Number of topics\n",
        "num_topics_pt = len(topics_pt)\n",
        "num_topics_pt_expanded = (sum(len(t) for t in topics_pt))\n",
        "\n",
        "print('Number of topics in portuguese: ' + str(num_topics_pt) + ' (' + str(num_topics_pt_expanded) + ')')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eD81AnX86Nlg"
      },
      "source": [
        "#### Publico"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q5qO-gAzwPPm"
      },
      "source": [
        "print('Publico first date:', df_publico_lb.iloc[-1].date)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ha8gUthW6Prf"
      },
      "source": [
        "# Sentences Liveblog\n",
        "num_sents_lb_publico = count_sents_inputDocs(inputDocs_publico)\n",
        "\n",
        "avg_sents_topic_lb_publico = num_sents_lb_publico / num_topics_pt\n",
        "\n",
        "print('Average sentences per topic in liveblog publico: ' + str(round(avg_sents_topic_lb_publico,2)))\n",
        "\n",
        "# Dates Liveblog\n",
        "num_dates_lb_publico = len(get_dates_list(inputDocs_publico))\n",
        "\n",
        "avg_dates_topic_lb_publico = num_dates_lb_publico / num_topics_pt\n",
        "\n",
        "print('Average dates per topic in liveblog publico: ' + str(round(avg_dates_topic_lb_publico,2)))\n",
        "\n",
        "# Senteces/Dates Liveblog\n",
        "avg_sentences_dates_lb_publico = avg_sents_topic_lb_publico/avg_dates_topic_lb_publico\n",
        "\n",
        "print('Average senteces/dates per topic in liveblog publico: ' + str(round(avg_sentences_dates_lb_publico,2)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EM9fESLU9HYt"
      },
      "source": [
        "# Sentences Timeline\n",
        "num_sents_tl_publico = count_sents_timelines(timelines_publico)\n",
        "\n",
        "avg_sents_topic_tl_publico = num_sents_tl_publico / num_topics_pt\n",
        "\n",
        "print('Average sentences per topic in timeline publico: ' + str(round(avg_sents_topic_tl_publico,2)))\n",
        "\n",
        "# Dates Timeline\n",
        "num_dates_tl_publico = len(get_dates_list(timelines_publico))\n",
        "\n",
        "avg_dates_topic_tl_publico = num_dates_tl_publico / num_topics_pt\n",
        "\n",
        "print('Average dates per topic in timeline publico: ' + str(round(avg_dates_topic_tl_publico,2)))\n",
        "\n",
        "# Senteces/Dates Liveblog\n",
        "avg_sentences_dates_tl_publico = avg_sents_topic_tl_publico/avg_dates_topic_tl_publico\n",
        "\n",
        "print('Average senteces/dates per topic in timeline publico: ' + str(round(avg_sentences_dates_tl_publico,2)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BOUfDXrD7_CJ"
      },
      "source": [
        "# Compression Sents\n",
        "compression_sents_publico = (avg_sents_topic_tl_publico/avg_sents_topic_lb_publico)*100\n",
        "\n",
        "print('Compression of sentences publico: ' + str(round(compression_sents_publico,2)))\n",
        "\n",
        "# Compression Dates\n",
        "compression_dates_publico = (avg_dates_topic_tl_publico/avg_dates_topic_lb_publico)*100\n",
        "\n",
        "print('Compression of dates publico: ' + str(round(compression_dates_publico,2)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r13q4JOexTNV"
      },
      "source": [
        "# Number of input docs\n",
        "\n",
        "n_inputDocs_publico = 0\n",
        "for topic, date_news in inputDocs_publico.items():\n",
        "  for date, news_list in date_news.items():\n",
        "    n_inputDocs_publico += len(news_list)\n",
        "\n",
        "print('Number of input docs in publico: ' + str(n_inputDocs_publico))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xWRUx5ofzBiN"
      },
      "source": [
        "# Number of timeline entries\n",
        "\n",
        "n_timelines_publico = 0\n",
        "for topic, date_news in timelines_publico.items():\n",
        "  for date, news_list in date_news.items():\n",
        "    n_timelines_publico += len(news_list)\n",
        "\n",
        "print('Number of timeline entries in publico: ' + str(n_timelines_publico))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ckzs8TcKFSDc"
      },
      "source": [
        "stats_publico = {\n",
        "    'n_topics': num_topics_pt,\n",
        "    'lang': 'pt',\n",
        "    'n_inputDocs': n_inputDocs_publico,\n",
        "    'inputDocs_avg_sents': round(avg_sents_topic_lb_publico,2),\n",
        "    'inputDocs_avg_dates': round(avg_dates_topic_lb_publico,2),\n",
        "    'inputdocs_avg_sents_per_dates': round(avg_sentences_dates_lb_publico,2),\n",
        "    'timeline_avg_sents': round(avg_sents_topic_tl_publico,2),\n",
        "    'timeline_avg_dates': round(avg_dates_topic_tl_publico,2),\n",
        "    'timeline_avg_sents_per_dates': round(avg_sentences_dates_tl_publico,2),\n",
        "    'compression_sents': round(compression_sents_publico,2),\n",
        "    'compression_dates': round(compression_dates_publico,2)\n",
        "}\n",
        "\n",
        "stats_dict['publico'] = stats_publico"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mCU7YuJd7TNT"
      },
      "source": [
        "#### Observador"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vNYNYlKJwjaQ"
      },
      "source": [
        "print('Observador first date:', df_observador_lb.iloc[-1].date)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uoYLeABS7TNX"
      },
      "source": [
        "# Sentences Liveblog\n",
        "num_sents_lb_observador = count_sents_inputDocs(inputDocs_observador)\n",
        "\n",
        "avg_sents_topic_lb_observador = num_sents_lb_observador / num_topics_pt\n",
        "\n",
        "print('Average sentences per topic in liveblog observador: ' + str(round(avg_sents_topic_lb_observador,2)))\n",
        "\n",
        "# Dates Liveblog\n",
        "num_dates_lb_observador = len(get_dates_list(inputDocs_observador))\n",
        "\n",
        "avg_dates_topic_lb_observador = num_dates_lb_observador / num_topics_pt\n",
        "\n",
        "print('Average dates per topic in liveblog observador: ' + str(round(avg_dates_topic_lb_observador,2)))\n",
        "\n",
        "# Senteces/Dates Liveblog\n",
        "avg_sentences_dates_lb_observador = avg_sents_topic_lb_observador/avg_dates_topic_lb_observador\n",
        "\n",
        "print('Average senteces/dates per topic in liveblog observador: ' + str(round(avg_sentences_dates_lb_observador,2)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lT_lV7jX7TNo"
      },
      "source": [
        "# Sentences Timeline\n",
        "num_sents_tl_observador = count_sents_timelines(timelines_observador)\n",
        "\n",
        "avg_sents_topic_tl_observador = num_sents_tl_observador / num_topics_pt\n",
        "\n",
        "print('Average sentences per topic in timeline observador: ' + str(round(avg_sents_topic_tl_observador,2)))\n",
        "\n",
        "# Dates Timeline\n",
        "num_dates_tl_observador = len(get_dates_list(timelines_observador))\n",
        "\n",
        "avg_dates_topic_tl_observador = num_dates_tl_observador / num_topics_pt\n",
        "\n",
        "print('Average dates per topic in timeline observador: ' + str(round(avg_dates_topic_tl_observador,2)))\n",
        "\n",
        "# Senteces/Dates Liveblog\n",
        "avg_sentences_dates_tl_observador = avg_sents_topic_tl_observador/avg_dates_topic_tl_observador\n",
        "\n",
        "print('Average senteces/dates per topic in timeline observador: ' + str(round(avg_sentences_dates_tl_observador,2)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wcr5oisr85K9"
      },
      "source": [
        "# Compression Sents\n",
        "compression_sents_observador = (avg_sents_topic_tl_observador/avg_sents_topic_lb_observador)*100\n",
        "\n",
        "print('Compression of sentences observador: ' + str(round(compression_sents_observador,2)))\n",
        "\n",
        "# Compression Dates\n",
        "compression_dates_observador = (avg_dates_topic_tl_observador/avg_dates_topic_lb_observador)*100\n",
        "\n",
        "print('Compression of dates observador: ' + str(round(compression_dates_observador,2)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c391PZyks2PA"
      },
      "source": [
        "# Number of input docs\n",
        "\n",
        "n_inputDocs_observador = 0\n",
        "for topic, date_news in inputDocs_observador.items():\n",
        "  for date, news_list in date_news.items():\n",
        "    n_inputDocs_observador += len(news_list)\n",
        "\n",
        "print('Number of input docs in observador: ' + str(n_inputDocs_observador))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "13OeNgCps4Kw"
      },
      "source": [
        "# Number of timeline entries\n",
        "\n",
        "n_timelines_observador = 0\n",
        "for topic, date_news in timelines_observador.items():\n",
        "  for date, news_list in date_news.items():\n",
        "    n_timelines_observador += len(news_list)\n",
        "\n",
        "print('Number of timeline entries in observador: ' + str(n_timelines_observador))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b3ElnNSBHSGw"
      },
      "source": [
        "stats_observador = {\n",
        "    'n_topics': num_topics_pt,\n",
        "    'lang': 'pt',\n",
        "    'n_inputDocs': n_inputDocs_observador,\n",
        "    'inputDocs_avg_sents': round(avg_sents_topic_lb_observador,2),\n",
        "    'inputDocs_avg_dates': round(avg_dates_topic_lb_observador,2),\n",
        "    'inputdocs_avg_sents_per_dates': round(avg_sentences_dates_lb_observador,2),\n",
        "    'timeline_avg_sents': round(avg_sents_topic_tl_observador,2),\n",
        "    'timeline_avg_dates': round(avg_dates_topic_tl_observador,2),\n",
        "    'timeline_avg_sents_per_dates': round(avg_sentences_dates_tl_observador,2),\n",
        "    'compression_sents': round(compression_sents_observador,2),\n",
        "    'compression_dates': round(compression_dates_observador,2)\n",
        "}\n",
        "\n",
        "stats_dict['observador'] = stats_observador"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-c-kmyRRtX1U"
      },
      "source": [
        "#### Total"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3gL2GYowta5S"
      },
      "source": [
        "print('Number of input docs in portuguese: ' + str(n_inputDocs_publico + n_inputDocs_observador))\n",
        "print('Number of timelines in portuguese: ' + str(n_timelines_publico + n_timelines_observador))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gBeIYgkXt_y_"
      },
      "source": [
        "### EN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0SD10FSit_zh"
      },
      "source": [
        "num_topics_en = len(topics_en)\n",
        "num_topics_en_expanded = (sum(len(t) for t in topics_en))\n",
        "\n",
        "print('Number of topics in english: ' + str(num_topics_en) + ' (' + str(num_topics_en_expanded) + ')')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n-91lzPZt_zj"
      },
      "source": [
        "#### CNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pBTOsT-twtsA"
      },
      "source": [
        "print('CNN first date:', df_cnn_lb.iloc[-1].date)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WUqeI9o9t_zj"
      },
      "source": [
        "# Sentences Liveblog\n",
        "num_sents_lb_cnn = count_sents_inputDocs(inputDocs_cnn)\n",
        "\n",
        "avg_sents_topic_lb_cnn = num_sents_lb_cnn / num_topics_en\n",
        "\n",
        "print('Average sentences per topic in liveblog cnn: ' + str(round(avg_sents_topic_lb_cnn,2)))\n",
        "\n",
        "# Dates Liveblog\n",
        "num_dates_lb_cnn = len(get_dates_list(inputDocs_cnn))\n",
        "\n",
        "avg_dates_topic_lb_cnn = num_dates_lb_cnn / num_topics_en\n",
        "\n",
        "print('Average dates per topic in liveblog cnn: ' + str(round(avg_dates_topic_lb_cnn,2)))\n",
        "\n",
        "# Senteces/Dates Liveblog\n",
        "avg_sentences_dates_lb_cnn = avg_sents_topic_lb_cnn/avg_dates_topic_lb_cnn\n",
        "\n",
        "print('Average senteces/dates per topic in liveblog cnn: ' + str(round(avg_sentences_dates_lb_cnn,2)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rX9QHo5gt_zj"
      },
      "source": [
        "# Sentences Timeline\n",
        "num_sents_tl_cnn = count_sents_timelines(timelines_cnn)\n",
        "\n",
        "avg_sents_topic_tl_cnn = num_sents_tl_cnn / num_topics_en\n",
        "\n",
        "print('Average sentences per topic in timeline cnn: ' + str(round(avg_sents_topic_tl_cnn,2)))\n",
        "\n",
        "# Dates Timeline\n",
        "num_dates_tl_cnn = len(get_dates_list(timelines_cnn))\n",
        "\n",
        "avg_dates_topic_tl_cnn = num_dates_tl_cnn / num_topics_en\n",
        "\n",
        "print('Average dates per topic in timeline cnn: ' + str(round(avg_dates_topic_tl_cnn,2)))\n",
        "\n",
        "# Senteces/Dates Liveblog\n",
        "avg_sentences_dates_tl_cnn = avg_sents_topic_tl_cnn/avg_dates_topic_tl_cnn\n",
        "\n",
        "print('Average senteces/dates per topic in timeline cnn: ' + str(round(avg_sentences_dates_tl_cnn,2)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mMwTUu8H9PRz"
      },
      "source": [
        "# Compression Sents\n",
        "compression_sents_cnn = (avg_sents_topic_tl_cnn/avg_sents_topic_lb_cnn)*100\n",
        "\n",
        "print('Compression of sentences cnn: ' + str(round(compression_sents_cnn,2)))\n",
        "\n",
        "# Compression Dates\n",
        "compression_dates_cnn = (avg_dates_topic_tl_cnn/avg_dates_topic_lb_cnn)*100\n",
        "\n",
        "print('Compression of dates cnn: ' + str(round(compression_dates_cnn,2)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ycdausIpt_zk"
      },
      "source": [
        "# Number of input docs\n",
        "\n",
        "n_inputDocs_cnn = 0\n",
        "for topic, date_news in inputDocs_cnn.items():\n",
        "  for date, news_list in date_news.items():\n",
        "    n_inputDocs_cnn += len(news_list)\n",
        "\n",
        "print('Number of input docs in cnn: ' + str(n_inputDocs_cnn))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6zAqMFuet_zk"
      },
      "source": [
        "# Number of timeline entries\n",
        "\n",
        "n_timelines_cnn = 0\n",
        "for topic, date_news in timelines_cnn.items():\n",
        "  for date, news_list in date_news.items():\n",
        "    n_timelines_cnn += len(news_list)\n",
        "\n",
        "print('Number of timeline entries in cnn: ' + str(n_timelines_cnn))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PiZ-23sWHcMg"
      },
      "source": [
        "stats_cnn = {\n",
        "    'n_topics': num_topics_en,\n",
        "    'lang': 'en',\n",
        "    'n_inputDocs': n_inputDocs_cnn,\n",
        "    'inputDocs_avg_sents': round(avg_sents_topic_lb_cnn,2),\n",
        "    'inputDocs_avg_dates': round(avg_dates_topic_lb_cnn,2),\n",
        "    'inputdocs_avg_sents_per_dates': round(avg_sentences_dates_lb_cnn,2),\n",
        "    'timeline_avg_sents': round(avg_sents_topic_tl_cnn,2),\n",
        "    'timeline_avg_dates': round(avg_dates_topic_tl_cnn,2),\n",
        "    'timeline_avg_sents_per_dates': round(avg_sentences_dates_tl_cnn,2),\n",
        "    'compression_sents': round(compression_sents_cnn,2),\n",
        "    'compression_dates': round(compression_dates_cnn,2)\n",
        "}\n",
        "\n",
        "stats_dict['cnn'] = stats_cnn"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mCqqLIUVt_zl"
      },
      "source": [
        "#### Guardian"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aE9g0vrHwx6X"
      },
      "source": [
        "print('Guardian first date:', df_guardian_lb.iloc[-1].date)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_3Yy8T2yt_zl"
      },
      "source": [
        "# Sentences Liveblog\n",
        "num_sents_lb_guardian = count_sents_inputDocs(inputDocs_guardian)\n",
        "\n",
        "avg_sents_topic_lb_guardian = num_sents_lb_guardian / num_topics_en\n",
        "\n",
        "print('Average sentences per topic in liveblog guardian: ' + str(round(avg_sents_topic_lb_guardian,2)))\n",
        "\n",
        "# Dates Liveblog\n",
        "num_dates_lb_guardian = len(get_dates_list(inputDocs_guardian))\n",
        "\n",
        "avg_dates_topic_lb_guardian = num_dates_lb_guardian / num_topics_en\n",
        "\n",
        "print('Average dates per topic in liveblog guardian: ' + str(round(avg_dates_topic_lb_guardian,2)))\n",
        "\n",
        "# Senteces/Dates Liveblog\n",
        "avg_sentences_dates_lb_guardian = avg_sents_topic_lb_guardian/avg_dates_topic_lb_guardian\n",
        "\n",
        "print('Average senteces/dates per topic in liveblog guardian: ' + str(round(avg_sentences_dates_lb_guardian,2)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BiQmwYKnt_zm"
      },
      "source": [
        "# Sentences Timeline\n",
        "num_sents_tl_guardian = count_sents_timelines(timelines_guardian)\n",
        "\n",
        "avg_sents_topic_tl_guardian = num_sents_tl_guardian / num_topics_en\n",
        "\n",
        "print('Average sentences per topic in timeline guardian: ' + str(round(avg_sents_topic_tl_guardian,2)))\n",
        "\n",
        "# Dates Timeline\n",
        "num_dates_tl_guardian = len(get_dates_list(timelines_guardian))\n",
        "\n",
        "avg_dates_topic_tl_guardian = num_dates_tl_guardian / num_topics_en\n",
        "\n",
        "print('Average dates per topic in timeline guardian: ' + str(round(avg_dates_topic_tl_guardian,2)))\n",
        "\n",
        "# Senteces/Dates Liveblog\n",
        "avg_sentences_dates_tl_guardian = avg_sents_topic_tl_guardian/avg_dates_topic_tl_guardian\n",
        "\n",
        "print('Average senteces/dates per topic in timeline guardian: ' + str(round(avg_sentences_dates_tl_guardian,2)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NUMzE0EM9aR2"
      },
      "source": [
        "# Compression Sents\n",
        "compression_sents_guardian = (avg_sents_topic_tl_guardian/avg_sents_topic_lb_guardian)*100\n",
        "\n",
        "print('Compression of sentences guardian: ' + str(round(compression_sents_guardian,2)))\n",
        "\n",
        "# Compression Dates\n",
        "compression_dates_guardian = (avg_dates_topic_tl_guardian/avg_dates_topic_lb_guardian)*100\n",
        "\n",
        "print('Compression of dates guardian: ' + str(round(compression_dates_guardian,2)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lCkSsCaPt_zm"
      },
      "source": [
        "# Number of input docs\n",
        "\n",
        "n_inputDocs_guardian = 0\n",
        "for topic, date_news in inputDocs_guardian.items():\n",
        "  for date, news_list in date_news.items():\n",
        "    n_inputDocs_guardian += len(news_list)\n",
        "\n",
        "print('Number of input docs in guardian: ' + str(n_inputDocs_guardian))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qwXZiShYt_zm"
      },
      "source": [
        "# Number of timeline entries\n",
        "\n",
        "n_timelines_guardian = 0\n",
        "for topic, date_news in timelines_guardian.items():\n",
        "  for date, news_list in date_news.items():\n",
        "    n_timelines_guardian += len(news_list)\n",
        "\n",
        "print('Number of timeline entries in guardian: ' + str(n_timelines_guardian))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IA0HdTkrHiZI"
      },
      "source": [
        "stats_guardian = {\n",
        "    'n_topics': num_topics_en,\n",
        "    'lang': 'en',\n",
        "    'n_inputDocs': n_inputDocs_guardian,\n",
        "    'inputDocs_avg_sents': round(avg_sents_topic_lb_guardian,2),\n",
        "    'inputDocs_avg_dates': round(avg_dates_topic_lb_guardian,2),\n",
        "    'inputdocs_avg_sents_per_dates': round(avg_sentences_dates_lb_guardian,2),\n",
        "    'timeline_avg_sents': round(avg_sents_topic_tl_guardian,2),\n",
        "    'timeline_avg_dates': round(avg_dates_topic_tl_guardian,2),\n",
        "    'timeline_avg_sents_per_dates': round(avg_sentences_dates_tl_guardian,2),\n",
        "    'compression_sents': round(compression_sents_guardian,2),\n",
        "    'compression_dates': round(compression_dates_guardian,2)\n",
        "}\n",
        "\n",
        "stats_dict['guardian'] = stats_guardian"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CR0izI6_t_zn"
      },
      "source": [
        "#### Total"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CuQzFEJOt_zn"
      },
      "source": [
        "print('Number of input docs in english: ' + str(n_inputDocs_cnn + n_inputDocs_guardian))\n",
        "print('Number of timelines in english: ' + str(n_timelines_cnn + n_timelines_guardian))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pb6feWi2H-b7"
      },
      "source": [
        "### Export stats"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HsLxnNkMH9jV"
      },
      "source": [
        "stats_df = pd.DataFrame.from_dict(stats_dict, orient='index')\n",
        "stats_df.to_excel(\"dataset_stats.xlsx\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0UB4vBXSIFDY"
      },
      "source": [
        "stats_df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cfd31UqhGq40"
      },
      "source": [
        "### Timeline chart (**ToDo**)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_KA31_-RGstR"
      },
      "source": [
        "def gt_to_list(gt):\n",
        "  news_list = []\n",
        "  for _, n in gt.items():\n",
        "    for d, l in n.items():\n",
        "      for news in l:\n",
        "        news_list.append((d, news))\n",
        "  \n",
        "  return news_list"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z6I4RXmqHKdP"
      },
      "source": [
        "p_l = gt_to_list(gt_publico)\n",
        "dfp = pd.DataFrame(p_l, columns=['date', 'news'])\n",
        "dfp['source'] = 'publico'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YOb9A978HcWy"
      },
      "source": [
        "o_l = gt_to_list(gt_observador)\n",
        "dfo = pd.DataFrame(o_l, columns=['date', 'news'])\n",
        "dfo['source'] = 'observador'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nMEJM2r6H1UM"
      },
      "source": [
        "c_l = gt_to_list(gt_cnn)\n",
        "dfc = pd.DataFrame(c_l, columns=['date', 'news'])\n",
        "dfc['source'] = 'cnn'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uGx2qq8hH4NF"
      },
      "source": [
        "g_l = gt_to_list(gt_guardian)\n",
        "dfg = pd.DataFrame(g_l, columns=['date', 'news'])\n",
        "dfg['source'] = 'guardian'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "smj5u0bCJNfQ"
      },
      "source": [
        "dfs = [dfp, dfo, dfc, dfg]\n",
        "\n",
        "df_total = pd.concat(dfs)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EEEH47clKerl"
      },
      "source": [
        "p_vc = dfp['date'].value_counts().rename_axis('dates').reset_index(name='counts')\n",
        "p_vc['source'] = 'publico'\n",
        "\n",
        "o_vc = dfo['date'].value_counts().rename_axis('dates').reset_index(name='counts')\n",
        "o_vc['source'] = 'observador'\n",
        "\n",
        "c_vc = dfc['date'].value_counts().rename_axis('dates').reset_index(name='counts')\n",
        "c_vc['source'] = 'cnn'\n",
        "\n",
        "g_vc = dfg['date'].value_counts().rename_axis('dates').reset_index(name='counts')\n",
        "g_vc['source'] = 'guardian'\n",
        "\n",
        "dfs_vc = [p_vc, o_vc, c_vc, g_vc]\n",
        "\n",
        "df_vc_total = pd.concat(dfs_vc)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wmuf9ZNZKjTX"
      },
      "source": [
        "df_vc_total = df_vc_total.sort_values(by='dates', ascending=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oy8WAPO2IT8R"
      },
      "source": [
        "import seaborn as sns; sns.set()\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "#xxx = dfp['date'].value_counts().plot()\n",
        "ax = sns.lineplot(x=\"dates\", y=\"counts\", hue='source', data=df_vc_total)\n",
        "#ax.set_xticklabels(ax.get_xticklabels(), rotation=45)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LahtUueKH9dt"
      },
      "source": [
        "dfp = dfp.sort_values(by='date', ascending=True)\n",
        "\n",
        "dfp['date'].value_counts().plot(rot=90);"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NegZrmmeOlOZ"
      },
      "source": [
        "dfp"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RoyfAe0AMz1n"
      },
      "source": [
        "dfo = dfo.sort_values(by='date', ascending=True)\n",
        "\n",
        "dfo['date'].value_counts().plot(rot=90);"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0KIxrHAuPnic"
      },
      "source": [
        "dfo"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j1YYbRUjM2Mn"
      },
      "source": [
        "dfc = dfc.sort_values(by='date', ascending=True)\n",
        "\n",
        "dfc['date'].value_counts().plot(rot=90);"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qHSBOTGaPwEl"
      },
      "source": [
        "dfc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ea_35wXTM4bl"
      },
      "source": [
        "dfg = dfg.sort_values(by='date', ascending=True)\n",
        "\n",
        "dfg['date'].value_counts().plot(rot=90);"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4NsKpjHbPyES"
      },
      "source": [
        "dfg"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}