From 268eef9b434ed8c492f97c984fc78c4fd63f6c4a Mon Sep 17 00:00:00 2001
From: emielvdveen <emiel@applike.nl>
Date: Tue, 12 Apr 2022 00:09:01 +0200
Subject: [PATCH 01/42] Some adaptations to better support new research studies

- Integrated latest pyworker.js from Eyra platform
- Created support for an index.html for every study (script)
---
 .DS_Store                                     |  Bin 0 -> 6148 bytes
 data_extractor/data_extractor/__init__.py     |   17 -
 data_extractor/example/__init__.py            |   20 +
 data_extractor/example/index.html             |   11 +
 .../google_search_history/index.html          |   11 +
 .../index.html                                |   11 +
 data_extractor/poetry.lock                    | 1390 ++++++++++++++---
 data_extractor/pyprocess.js                   |   66 +
 data_extractor/pyproject.toml                 |    1 +
 data_extractor/pyworker.js                    |   42 +
 data_extractor/tests/.DS_Store                |  Bin 0 -> 6148 bytes
 data_extractor/tests/data/.DS_Store           |  Bin 0 -> 6148 bytes
 data_extractor/tests/test_data_extractor.py   |    9 -
 data_extractor/tests/test_example.py          |   22 +
 data_extractor/tests/test_whatsapp.py         |   22 +
 data_extractor/whatsapp/__init__.py           |   20 +
 data_extractor/whatsapp/index.html            |   11 +
 index.html                                    |   63 -
 pyworker.js                                   |  105 --
 19 files changed, 1442 insertions(+), 379 deletions(-)
 create mode 100644 .DS_Store
 delete mode 100644 data_extractor/data_extractor/__init__.py
 create mode 100644 data_extractor/example/__init__.py
 create mode 100644 data_extractor/example/index.html
 create mode 100644 data_extractor/google_search_history/index.html
 create mode 100644 data_extractor/google_semantic_location_history/index.html
 create mode 100644 data_extractor/pyprocess.js
 create mode 100644 data_extractor/pyworker.js
 create mode 100644 data_extractor/tests/.DS_Store
 create mode 100644 data_extractor/tests/data/.DS_Store
 delete mode 100644 data_extractor/tests/test_data_extractor.py
 create mode 100644 data_extractor/tests/test_example.py
 create mode 100644 data_extractor/tests/test_whatsapp.py
 create mode 100644 data_extractor/whatsapp/__init__.py
 create mode 100644 data_extractor/whatsapp/index.html
 delete mode 100644 index.html
 delete mode 100644 pyworker.js

diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6
GIT binary patch
literal 6148
zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3
zem<@ulZcFPQ@L2!n>{z**<q8>++&mCkOWA81W14cNZ<zv;LbK1Poaz?KmsK2CSc!(
z0ynLxE!0092;Krf2c+FF_Fe*7ECH>lEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ
zLs35+`xjp>T0<F0fCPF1$Cyrb|F7^5{eNG?83~ZUUlGt@xh*qZDeu<Z%US-OSsOPv
j)R!Z4KLME7ReXlK;d!wEw5GODWMKRea10D2@KpjYNUI8I

literal 0
HcmV?d00001

diff --git a/data_extractor/data_extractor/__init__.py b/data_extractor/data_extractor/__init__.py
deleted file mode 100644
index 50e2e49..0000000
--- a/data_extractor/data_extractor/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-__version__ = '0.1.0'
-
-import zipfile
-
-
-def process(file_data):
-    names = []
-    data = []
-    zfile = zipfile.ZipFile(file_data)
-    for name in zfile.namelist():
-        names.append(name)
-        data.append((name, zfile.read(name).decode("utf8")))
-
-    return {
-        "summary": f"The following files where read: {', '.join(names)}.",
-        "data": data
-    }
diff --git a/data_extractor/example/__init__.py b/data_extractor/example/__init__.py
new file mode 100644
index 0000000..63f5f31
--- /dev/null
+++ b/data_extractor/example/__init__.py
@@ -0,0 +1,20 @@
+__version__ = '0.2.0'
+
+import zipfile
+import pandas as pd
+
+
+def process(file_data):
+    names = []
+    zfile = zipfile.ZipFile(file_data)
+    data = []
+    for name in zfile.namelist():
+        names.append(name)
+        info = zfile.getinfo(name)
+        data.append((name, info.compress_size, info.file_size))
+
+    return [{
+        "id": "overview",
+        "title": "The following files where read:",
+        "data_frame": pd.DataFrame(data, columns=["filename", "compressed size", "size"])
+    }]
diff --git a/data_extractor/example/index.html b/data_extractor/example/index.html
new file mode 100644
index 0000000..ab1ef7e
--- /dev/null
+++ b/data_extractor/example/index.html
@@ -0,0 +1,11 @@
+<script src="../pyprocess.js" data-script="./example/__init__.py"></script>
+<div id="controls">
+  <p id="loading-indicator">Loading...</p>
+  <input type="file" id="fileItem" disabled onChange="toggleProcessButton()" />
+  <button onClick="process()" id="process" disabled>Process</button>
+</div>
+
+<div id="results" style="display: none">
+  <p id="summary" />
+  <div id="html" />
+</div>
diff --git a/data_extractor/google_search_history/index.html b/data_extractor/google_search_history/index.html
new file mode 100644
index 0000000..52fb51a
--- /dev/null
+++ b/data_extractor/google_search_history/index.html
@@ -0,0 +1,11 @@
+<script src="../pyprocess.js" data-script="./google_search_history/__init__.py"></script>
+<div id="controls">
+  <p id="loading-indicator">Loading...</p>
+  <input type="file" id="fileItem" disabled onChange="toggleProcessButton()" />
+  <button onClick="process()" id="process" disabled>Process</button>
+</div>
+
+<div id="results" style="display: none">
+  <p id="summary" />
+  <div id="html" />
+</div>
diff --git a/data_extractor/google_semantic_location_history/index.html b/data_extractor/google_semantic_location_history/index.html
new file mode 100644
index 0000000..b01100d
--- /dev/null
+++ b/data_extractor/google_semantic_location_history/index.html
@@ -0,0 +1,11 @@
+<script src="../pyprocess.js" data-script="./google_semantic_location_history/__init__.py"></script>
+<div id="controls">
+  <p id="loading-indicator">Loading...</p>
+  <input type="file" id="fileItem" disabled onChange="toggleProcessButton()" />
+  <button onClick="process()" id="process" disabled>Process</button>
+</div>
+
+<div id="results" style="display: none">
+  <p id="summary" />
+  <div id="html" />
+</div>
diff --git a/data_extractor/poetry.lock b/data_extractor/poetry.lock
index 659d3d7..762c91a 100644
--- a/data_extractor/poetry.lock
+++ b/data_extractor/poetry.lock
@@ -1,14 +1,30 @@
+[[package]]
+name = "advertools"
+version = "0.13.0"
+description = "Productivity and analysis tools for online marketing"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+pandas = "*"
+pyarrow = "*"
+pyasn1 = "*"
+scrapy = "*"
+twython = "*"
+
 [[package]]
 name = "astroid"
-version = "2.5.6"
+version = "2.11.2"
 description = "An abstract syntax tree for Python with inference support."
 category = "main"
 optional = false
-python-versions = "~=3.6"
+python-versions = ">=3.6.2"
 
 [package.dependencies]
 lazy-object-proxy = ">=1.4.0"
-wrapt = ">=1.11,<1.13"
+typing-extensions = {version = ">=3.10", markers = "python_version < \"3.10\""}
+wrapt = ">=1.11,<2"
 
 [[package]]
 name = "atomicwrites"
@@ -20,17 +36,62 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 
 [[package]]
 name = "attrs"
-version = "21.2.0"
+version = "21.4.0"
 description = "Classes Without Boilerplate"
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 
 [package.extras]
-dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit"]
+dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"]
 docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"]
-tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface"]
-tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins"]
+tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"]
+tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"]
+
+[[package]]
+name = "automat"
+version = "20.2.0"
+description = "Self-service finite-state machines for the programmer on the go."
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+attrs = ">=19.2.0"
+six = "*"
+
+[package.extras]
+visualize = ["graphviz (>0.5.1)", "Twisted (>=16.1.1)"]
+
+[[package]]
+name = "certifi"
+version = "2021.10.8"
+description = "Python package for providing Mozilla's CA Bundle."
+category = "main"
+optional = false
+python-versions = "*"
+
+[[package]]
+name = "cffi"
+version = "1.15.0"
+description = "Foreign Function Interface for Python calling C code."
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+pycparser = "*"
+
+[[package]]
+name = "charset-normalizer"
+version = "2.0.12"
+description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
+category = "main"
+optional = false
+python-versions = ">=3.5.0"
+
+[package.extras]
+unicode_backport = ["unicodedata2"]
 
 [[package]]
 name = "colorama"
@@ -40,20 +101,66 @@ category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 
+[[package]]
+name = "constantly"
+version = "15.1.0"
+description = "Symbolic constants in Python"
+category = "main"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "coverage"
-version = "5.5"
+version = "6.3.2"
 description = "Code coverage measurement for Python"
 category = "dev"
 optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
+python-versions = ">=3.7"
+
+[package.extras]
+toml = ["tomli"]
+
+[[package]]
+name = "cryptography"
+version = "36.0.2"
+description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+cffi = ">=1.12"
+
+[package.extras]
+docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"]
+docstest = ["pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"]
+pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"]
+sdist = ["setuptools_rust (>=0.11.4)"]
+ssh = ["bcrypt (>=3.1.5)"]
+test = ["pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"]
+
+[[package]]
+name = "cssselect"
+version = "1.1.0"
+description = "cssselect parses CSS3 Selectors and translates them to XPath 1.0"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+
+[[package]]
+name = "dill"
+version = "0.3.4"
+description = "serialize all of python"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*"
 
 [package.extras]
-toml = ["toml"]
+graph = ["objgraph (>=1.7.2)"]
 
 [[package]]
 name = "faker"
-version = "8.11.0"
+version = "8.16.0"
 description = "Faker is a Python package that generates fake data for you."
 category = "main"
 optional = false
@@ -74,6 +181,18 @@ python-versions = "*"
 [package.dependencies]
 Faker = ">=0.7.17"
 
+[[package]]
+name = "filelock"
+version = "3.6.0"
+description = "A platform independent file lock."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.extras]
+docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"]
+testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"]
+
 [[package]]
 name = "flake8"
 version = "3.9.2"
@@ -123,26 +242,101 @@ dev-test = ["async-generator", "coverage", "pytest-aiohttp", "pytest (>=3.10)",
 requests = ["urllib3 (>=1.24.2)", "requests (>=2.16.2)"]
 timezone = ["pytz"]
 
+[[package]]
+name = "hyperlink"
+version = "21.0.0"
+description = "A featureful, immutable, and correct URL for Python."
+category = "main"
+optional = false
+python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+
+[package.dependencies]
+idna = ">=2.5"
+
+[[package]]
+name = "idna"
+version = "3.3"
+description = "Internationalized Domain Names in Applications (IDNA)"
+category = "main"
+optional = false
+python-versions = ">=3.5"
+
+[[package]]
+name = "incremental"
+version = "21.3.0"
+description = "A small library that versions your Python projects."
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.extras]
+scripts = ["click (>=6.0)", "twisted (>=16.4.0)"]
+
 [[package]]
 name = "isort"
-version = "5.8.0"
+version = "5.10.1"
 description = "A Python utility / library to sort Python imports."
 category = "main"
 optional = false
-python-versions = ">=3.6,<4.0"
+python-versions = ">=3.6.1,<4.0"
 
 [package.extras]
 pipfile_deprecated_finder = ["pipreqs", "requirementslib"]
 requirements_deprecated_finder = ["pipreqs", "pip-api"]
 colors = ["colorama (>=0.4.3,<0.5.0)"]
+plugins = ["setuptools"]
+
+[[package]]
+name = "itemadapter"
+version = "0.5.0"
+description = "Common interface for data container classes"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[[package]]
+name = "itemloaders"
+version = "1.0.4"
+description = "Base library for scrapy's ItemLoader"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+itemadapter = ">=0.1.0"
+jmespath = ">=0.9.5"
+parsel = ">=1.5.0"
+w3lib = ">=1.17.0"
+
+[[package]]
+name = "jmespath"
+version = "1.0.0"
+description = "JSON Matching Expressions"
+category = "main"
+optional = false
+python-versions = ">=3.7"
 
 [[package]]
 name = "lazy-object-proxy"
-version = "1.6.0"
+version = "1.7.1"
 description = "A fast and thorough lazy object proxy."
 category = "main"
 optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
+python-versions = ">=3.6"
+
+[[package]]
+name = "lxml"
+version = "4.8.0"
+description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*"
+
+[package.extras]
+cssselect = ["cssselect (>=0.7)"]
+html5 = ["html5lib"]
+htmlsoup = ["beautifulsoup4"]
+source = ["Cython (>=0.29.7)"]
 
 [[package]]
 name = "mccabe"
@@ -167,7 +361,7 @@ test = ["pytest (<5.4)", "pytest-cov"]
 
 [[package]]
 name = "more-itertools"
-version = "8.8.0"
+version = "8.12.0"
 description = "More routines for operating on iterables, beyond itertools"
 category = "dev"
 optional = false
@@ -175,38 +369,82 @@ python-versions = ">=3.5"
 
 [[package]]
 name = "numpy"
-version = "1.21.1"
+version = "1.22.3"
 description = "NumPy is the fundamental package for array computing with Python."
 category = "main"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
+
+[[package]]
+name = "oauthlib"
+version = "3.2.0"
+description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.extras]
+rsa = ["cryptography (>=3.0.0)"]
+signals = ["blinker (>=1.4.0)"]
+signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"]
 
 [[package]]
 name = "packaging"
-version = "20.9"
+version = "21.3"
 description = "Core utilities for Python packages"
 category = "dev"
 optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+python-versions = ">=3.6"
 
 [package.dependencies]
-pyparsing = ">=2.0.2"
+pyparsing = ">=2.0.2,<3.0.5 || >3.0.5"
 
 [[package]]
 name = "pandas"
-version = "1.2.4"
+version = "1.4.2"
 description = "Powerful data structures for data analysis, time series, and statistics"
 category = "main"
 optional = false
-python-versions = ">=3.7.1"
+python-versions = ">=3.8"
 
 [package.dependencies]
-numpy = ">=1.16.5"
-python-dateutil = ">=2.7.3"
-pytz = ">=2017.3"
+numpy = [
+    {version = ">=1.18.5", markers = "platform_machine != \"aarch64\" and platform_machine != \"arm64\" and python_version < \"3.10\""},
+    {version = ">=1.19.2", markers = "platform_machine == \"aarch64\" and python_version < \"3.10\""},
+    {version = ">=1.20.0", markers = "platform_machine == \"arm64\" and python_version < \"3.10\""},
+    {version = ">=1.21.0", markers = "python_version >= \"3.10\""},
+]
+python-dateutil = ">=2.8.1"
+pytz = ">=2020.1"
 
 [package.extras]
-test = ["pytest (>=5.0.1)", "pytest-xdist", "hypothesis (>=3.58)"]
+test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"]
+
+[[package]]
+name = "parsel"
+version = "1.6.0"
+description = "Parsel is a library to extract data from HTML and XML using XPath and CSS selectors"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+cssselect = ">=0.9"
+lxml = "*"
+six = ">=1.6.0"
+w3lib = ">=1.19.0"
+
+[[package]]
+name = "platformdirs"
+version = "2.5.1"
+description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.extras]
+docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"]
+test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"]
 
 [[package]]
 name = "pluggy"
@@ -219,13 +457,54 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 [package.extras]
 dev = ["pre-commit", "tox"]
 
+[[package]]
+name = "protego"
+version = "0.2.1"
+description = "Pure-Python robots.txt parser with support for modern conventions"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+
+[package.dependencies]
+six = "*"
+
 [[package]]
 name = "py"
-version = "1.10.0"
+version = "1.11.0"
 description = "library with cross-python path, ini-parsing, io, code, log facilities"
 category = "dev"
 optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+
+[[package]]
+name = "pyarrow"
+version = "7.0.0"
+description = "Python library for Apache Arrow"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+numpy = ">=1.16.6"
+
+[[package]]
+name = "pyasn1"
+version = "0.4.8"
+description = "ASN.1 types and codecs"
+category = "main"
+optional = false
+python-versions = "*"
+
+[[package]]
+name = "pyasn1-modules"
+version = "0.2.8"
+description = "A collection of ASN.1-based protocols modules."
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+pyasn1 = ">=0.4.6,<0.5.0"
 
 [[package]]
 name = "pycodestyle"
@@ -235,6 +514,22 @@ category = "dev"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 
+[[package]]
+name = "pycparser"
+version = "2.21"
+description = "C parser in Python"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+
+[[package]]
+name = "pydispatcher"
+version = "2.0.5"
+description = "Multi-producer-multi-consumer signal dispatching mechanism"
+category = "main"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "pyflakes"
 version = "2.3.1"
@@ -245,26 +540,58 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 
 [[package]]
 name = "pylint"
-version = "2.8.3"
+version = "2.13.5"
 description = "python code static checker"
 category = "main"
 optional = false
-python-versions = "~=3.6"
+python-versions = ">=3.6.2"
 
 [package.dependencies]
-astroid = "2.5.6"
+astroid = ">=2.11.2,<=2.12.0-dev0"
 colorama = {version = "*", markers = "sys_platform == \"win32\""}
+dill = ">=0.2"
 isort = ">=4.2.5,<6"
-mccabe = ">=0.6,<0.7"
-toml = ">=0.7.1"
+mccabe = ">=0.6,<0.8"
+platformdirs = ">=2.2.0"
+tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""}
+typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""}
+
+[package.extras]
+testutil = ["gitpython (>3)"]
+
+[[package]]
+name = "pyopenssl"
+version = "22.0.0"
+description = "Python wrapper module around the OpenSSL library"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+cryptography = ">=35.0"
+
+[package.extras]
+docs = ["sphinx", "sphinx-rtd-theme"]
+test = ["flaky", "pretend", "pytest (>=3.0.1)"]
 
 [[package]]
 name = "pyparsing"
-version = "2.4.7"
-description = "Python parsing module"
+version = "3.0.8"
+description = "pyparsing module - Classes and methods to define and execute parsing grammars"
 category = "dev"
 optional = false
-python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
+python-versions = ">=3.6.8"
+
+[package.extras]
+diagrams = ["railroad-diagrams", "jinja2"]
+
+[[package]]
+name = "pypydispatcher"
+version = "2.1.2"
+description = "Multi-producer-multi-consumer signal dispatching mechanism"
+category = "main"
+optional = false
+python-versions = "*"
 
 [[package]]
 name = "pytest"
@@ -306,7 +633,7 @@ testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtuale
 
 [[package]]
 name = "python-dateutil"
-version = "2.8.1"
+version = "2.8.2"
 description = "Extensions to the standard Python datetime module"
 category = "main"
 optional = false
@@ -317,12 +644,115 @@ six = ">=1.5"
 
 [[package]]
 name = "pytz"
-version = "2021.1"
+version = "2022.1"
 description = "World timezone definitions, modern and historical"
 category = "main"
 optional = false
 python-versions = "*"
 
+[[package]]
+name = "queuelib"
+version = "1.6.2"
+description = "Collection of persistent (disk-based) and non-persistent (memory-based) queues"
+category = "main"
+optional = false
+python-versions = ">=3.5"
+
+[[package]]
+name = "requests"
+version = "2.27.1"
+description = "Python HTTP for Humans."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
+
+[package.dependencies]
+certifi = ">=2017.4.17"
+charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""}
+idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""}
+urllib3 = ">=1.21.1,<1.27"
+
+[package.extras]
+socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
+use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"]
+
+[[package]]
+name = "requests-file"
+version = "1.5.1"
+description = "File transport adapter for Requests"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+requests = ">=1.0.0"
+six = "*"
+
+[[package]]
+name = "requests-oauthlib"
+version = "1.3.1"
+description = "OAuthlib authentication support for Requests."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+
+[package.dependencies]
+oauthlib = ">=3.0.0"
+requests = ">=2.0.0"
+
+[package.extras]
+rsa = ["oauthlib[signedtoken] (>=3.0.0)"]
+
+[[package]]
+name = "scrapy"
+version = "2.6.1"
+description = "A high-level Web Crawling and Web Scraping framework"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+cryptography = ">=2.0"
+cssselect = ">=0.9.1"
+itemadapter = ">=0.1.0"
+itemloaders = ">=1.0.1"
+lxml = [
+    {version = ">=3.5.0", markers = "platform_python_implementation == \"CPython\""},
+    {version = ">=4.0.0", markers = "platform_python_implementation == \"PyPy\""},
+]
+parsel = ">=1.5.0"
+protego = ">=0.1.15"
+PyDispatcher = {version = ">=2.0.5", markers = "platform_python_implementation == \"CPython\""}
+pyOpenSSL = ">=16.2.0"
+PyPyDispatcher = {version = ">=2.1.0", markers = "platform_python_implementation == \"PyPy\""}
+queuelib = ">=1.4.2"
+service-identity = ">=16.0.0"
+tldextract = "*"
+Twisted = ">=17.9.0"
+w3lib = ">=1.17.0"
+"zope.interface" = ">=4.1.3"
+
+[[package]]
+name = "service-identity"
+version = "21.1.0"
+description = "Service identity verification for pyOpenSSL & cryptography."
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+attrs = ">=19.1.0"
+cryptography = "*"
+pyasn1 = "*"
+pyasn1-modules = "*"
+six = "*"
+
+[package.extras]
+dev = ["coverage[toml] (>=5.0.2)", "pytest", "sphinx", "furo", "idna", "pyopenssl"]
+docs = ["sphinx", "furo"]
+idna = ["idna"]
+tests = ["coverage[toml] (>=5.0.2)", "pytest"]
+
 [[package]]
 name = "six"
 version = "1.16.0"
@@ -339,14 +769,122 @@ category = "main"
 optional = false
 python-versions = "*"
 
+[[package]]
+name = "tldextract"
+version = "3.2.0"
+description = "Accurately separate the TLD from the registered domain and subdomains of a URL, using the Public Suffix List. By default, this includes the public ICANN TLDs and their exceptions. You can optionally support the Public Suffix List's private domains as well."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+filelock = ">=3.0.8"
+idna = "*"
+requests = ">=2.1.0"
+requests-file = ">=1.4"
+
 [[package]]
 name = "toml"
 version = "0.10.2"
 description = "Python Library for Tom's Obvious, Minimal Language"
-category = "main"
+category = "dev"
 optional = false
 python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
 
+[[package]]
+name = "tomli"
+version = "2.0.1"
+description = "A lil' TOML parser"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "twisted"
+version = "22.4.0"
+description = "An asynchronous networking framework written in Python"
+category = "main"
+optional = false
+python-versions = ">=3.6.7"
+
+[package.dependencies]
+attrs = ">=19.2.0"
+Automat = ">=0.8.0"
+constantly = ">=15.1"
+hyperlink = ">=17.1.1"
+incremental = ">=21.3.0"
+twisted-iocpsupport = {version = ">=1.0.2,<2", markers = "platform_system == \"Windows\""}
+typing-extensions = ">=3.6.5"
+"zope.interface" = ">=4.4.2"
+
+[package.extras]
+all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
+conch = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)"]
+conch_nacl = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pynacl"]
+contextvars = ["contextvars (>=2.4,<3)"]
+dev = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "python-subunit (>=1.4,<2.0)", "pydoctor (>=21.9.0,<21.10.0)"]
+dev_release = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pydoctor (>=21.9.0,<21.10.0)"]
+http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"]
+macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
+mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pynacl", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"]
+osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
+serial = ["pyserial (>=3.0)", "pywin32 (!=226)"]
+test = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)"]
+tls = ["pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)"]
+windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
+
+[[package]]
+name = "twisted-iocpsupport"
+version = "1.0.2"
+description = "An extension for use in the twisted I/O Completion Ports reactor."
+category = "main"
+optional = false
+python-versions = "*"
+
+[[package]]
+name = "twython"
+version = "3.9.1"
+description = "Actively maintained, pure Python wrapper for the Twitter API. Supports both normal and streaming Twitter APIs"
+category = "main"
+optional = false
+python-versions = ">=3.5"
+
+[package.dependencies]
+requests = ">=2.1.0"
+requests-oauthlib = ">=0.4.0"
+
+[[package]]
+name = "typing-extensions"
+version = "4.1.1"
+description = "Backported and Experimental Type Hints for Python 3.6+"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[[package]]
+name = "urllib3"
+version = "1.26.9"
+description = "HTTP library with thread-safe connection pooling, file post, and more."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
+
+[package.extras]
+brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"]
+secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"]
+socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
+
+[[package]]
+name = "w3lib"
+version = "1.22.0"
+description = "Library of web-related functions"
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+six = ">=1.4.1"
+
 [[package]]
 name = "wcwidth"
 version = "0.2.5"
@@ -357,95 +895,203 @@ python-versions = "*"
 
 [[package]]
 name = "wrapt"
-version = "1.12.1"
+version = "1.14.0"
 description = "Module for decorators, wrappers and monkey patching."
 category = "main"
 optional = false
-python-versions = "*"
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
+
+[[package]]
+name = "zope.interface"
+version = "5.4.0"
+description = "Interfaces for Python"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+
+[package.extras]
+docs = ["sphinx", "repoze.sphinx.autointerface"]
+test = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
+testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8"
-content-hash = "8597663fbeccd8ceaec3136f843eaa2bd57a519f95f1515bba8f1e8e3bd953af"
+content-hash = "e7da351c090cb9853d848b744eca29c940436d1de37f24867604ba18cf7c793a"
 
 [metadata.files]
+advertools = [
+    {file = "advertools-0.13.0-py2.py3-none-any.whl", hash = "sha256:9eae08ce45e5166d7809f2a5baf1f3a1cb00bc6b67029115b097bd9ce6b77f8c"},
+    {file = "advertools-0.13.0.tar.gz", hash = "sha256:97dd3e7a14c4552fbcb0582cdec071484930a324a131f4847e16c3691934e7f4"},
+]
 astroid = [
-    {file = "astroid-2.5.6-py3-none-any.whl", hash = "sha256:4db03ab5fc3340cf619dbc25e42c2cc3755154ce6009469766d7143d1fc2ee4e"},
-    {file = "astroid-2.5.6.tar.gz", hash = "sha256:8a398dfce302c13f14bab13e2b14fe385d32b73f4e4853b9bdfb64598baa1975"},
+    {file = "astroid-2.11.2-py3-none-any.whl", hash = "sha256:cc8cc0d2d916c42d0a7c476c57550a4557a083081976bf42a73414322a6411d9"},
+    {file = "astroid-2.11.2.tar.gz", hash = "sha256:8d0a30fe6481ce919f56690076eafbb2fb649142a89dc874f1ec0e7a011492d0"},
 ]
 atomicwrites = [
     {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"},
     {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"},
 ]
 attrs = [
-    {file = "attrs-21.2.0-py2.py3-none-any.whl", hash = "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1"},
-    {file = "attrs-21.2.0.tar.gz", hash = "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"},
+    {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"},
+    {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"},
+]
+automat = [
+    {file = "Automat-20.2.0-py2.py3-none-any.whl", hash = "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111"},
+    {file = "Automat-20.2.0.tar.gz", hash = "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33"},
+]
+certifi = [
+    {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"},
+    {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"},
+]
+cffi = [
+    {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"},
+    {file = "cffi-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0"},
+    {file = "cffi-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14"},
+    {file = "cffi-1.15.0-cp27-cp27m-win32.whl", hash = "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474"},
+    {file = "cffi-1.15.0-cp27-cp27m-win_amd64.whl", hash = "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6"},
+    {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27"},
+    {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023"},
+    {file = "cffi-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2"},
+    {file = "cffi-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e"},
+    {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7"},
+    {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3"},
+    {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c"},
+    {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962"},
+    {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382"},
+    {file = "cffi-1.15.0-cp310-cp310-win32.whl", hash = "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55"},
+    {file = "cffi-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0"},
+    {file = "cffi-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e"},
+    {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39"},
+    {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc"},
+    {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032"},
+    {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8"},
+    {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605"},
+    {file = "cffi-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e"},
+    {file = "cffi-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc"},
+    {file = "cffi-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636"},
+    {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4"},
+    {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997"},
+    {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b"},
+    {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2"},
+    {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7"},
+    {file = "cffi-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66"},
+    {file = "cffi-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029"},
+    {file = "cffi-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880"},
+    {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20"},
+    {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024"},
+    {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e"},
+    {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728"},
+    {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6"},
+    {file = "cffi-1.15.0-cp38-cp38-win32.whl", hash = "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c"},
+    {file = "cffi-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443"},
+    {file = "cffi-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a"},
+    {file = "cffi-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37"},
+    {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a"},
+    {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e"},
+    {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796"},
+    {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df"},
+    {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8"},
+    {file = "cffi-1.15.0-cp39-cp39-win32.whl", hash = "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a"},
+    {file = "cffi-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139"},
+    {file = "cffi-1.15.0.tar.gz", hash = "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954"},
+]
+charset-normalizer = [
+    {file = "charset-normalizer-2.0.12.tar.gz", hash = "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597"},
+    {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"},
 ]
 colorama = [
     {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"},
     {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"},
 ]
+constantly = [
+    {file = "constantly-15.1.0-py2.py3-none-any.whl", hash = "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d"},
+    {file = "constantly-15.1.0.tar.gz", hash = "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35"},
+]
 coverage = [
-    {file = "coverage-5.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf"},
-    {file = "coverage-5.5-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:b7895207b4c843c76a25ab8c1e866261bcfe27bfaa20c192de5190121770672b"},
-    {file = "coverage-5.5-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:c2723d347ab06e7ddad1a58b2a821218239249a9e4365eaff6649d31180c1669"},
-    {file = "coverage-5.5-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:900fbf7759501bc7807fd6638c947d7a831fc9fdf742dc10f02956ff7220fa90"},
-    {file = "coverage-5.5-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:004d1880bed2d97151facef49f08e255a20ceb6f9432df75f4eef018fdd5a78c"},
-    {file = "coverage-5.5-cp27-cp27m-win32.whl", hash = "sha256:06191eb60f8d8a5bc046f3799f8a07a2d7aefb9504b0209aff0b47298333302a"},
-    {file = "coverage-5.5-cp27-cp27m-win_amd64.whl", hash = "sha256:7501140f755b725495941b43347ba8a2777407fc7f250d4f5a7d2a1050ba8e82"},
-    {file = "coverage-5.5-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:372da284cfd642d8e08ef606917846fa2ee350f64994bebfbd3afb0040436905"},
-    {file = "coverage-5.5-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:8963a499849a1fc54b35b1c9f162f4108017b2e6db2c46c1bed93a72262ed083"},
-    {file = "coverage-5.5-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:869a64f53488f40fa5b5b9dcb9e9b2962a66a87dab37790f3fcfb5144b996ef5"},
-    {file = "coverage-5.5-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:4a7697d8cb0f27399b0e393c0b90f0f1e40c82023ea4d45d22bce7032a5d7b81"},
-    {file = "coverage-5.5-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:8d0a0725ad7c1a0bcd8d1b437e191107d457e2ec1084b9f190630a4fb1af78e6"},
-    {file = "coverage-5.5-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:51cb9476a3987c8967ebab3f0fe144819781fca264f57f89760037a2ea191cb0"},
-    {file = "coverage-5.5-cp310-cp310-win_amd64.whl", hash = "sha256:c0891a6a97b09c1f3e073a890514d5012eb256845c451bd48f7968ef939bf4ae"},
-    {file = "coverage-5.5-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:3487286bc29a5aa4b93a072e9592f22254291ce96a9fbc5251f566b6b7343cdb"},
-    {file = "coverage-5.5-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:deee1077aae10d8fa88cb02c845cfba9b62c55e1183f52f6ae6a2df6a2187160"},
-    {file = "coverage-5.5-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f11642dddbb0253cc8853254301b51390ba0081750a8ac03f20ea8103f0c56b6"},
-    {file = "coverage-5.5-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:6c90e11318f0d3c436a42409f2749ee1a115cd8b067d7f14c148f1ce5574d701"},
-    {file = "coverage-5.5-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:30c77c1dc9f253283e34c27935fded5015f7d1abe83bc7821680ac444eaf7793"},
-    {file = "coverage-5.5-cp35-cp35m-win32.whl", hash = "sha256:9a1ef3b66e38ef8618ce5fdc7bea3d9f45f3624e2a66295eea5e57966c85909e"},
-    {file = "coverage-5.5-cp35-cp35m-win_amd64.whl", hash = "sha256:972c85d205b51e30e59525694670de6a8a89691186012535f9d7dbaa230e42c3"},
-    {file = "coverage-5.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:af0e781009aaf59e25c5a678122391cb0f345ac0ec272c7961dc5455e1c40066"},
-    {file = "coverage-5.5-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:74d881fc777ebb11c63736622b60cb9e4aee5cace591ce274fb69e582a12a61a"},
-    {file = "coverage-5.5-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:92b017ce34b68a7d67bd6d117e6d443a9bf63a2ecf8567bb3d8c6c7bc5014465"},
-    {file = "coverage-5.5-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:d636598c8305e1f90b439dbf4f66437de4a5e3c31fdf47ad29542478c8508bbb"},
-    {file = "coverage-5.5-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:41179b8a845742d1eb60449bdb2992196e211341818565abded11cfa90efb821"},
-    {file = "coverage-5.5-cp36-cp36m-win32.whl", hash = "sha256:040af6c32813fa3eae5305d53f18875bedd079960822ef8ec067a66dd8afcd45"},
-    {file = "coverage-5.5-cp36-cp36m-win_amd64.whl", hash = "sha256:5fec2d43a2cc6965edc0bb9e83e1e4b557f76f843a77a2496cbe719583ce8184"},
-    {file = "coverage-5.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:18ba8bbede96a2c3dde7b868de9dcbd55670690af0988713f0603f037848418a"},
-    {file = "coverage-5.5-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:2910f4d36a6a9b4214bb7038d537f015346f413a975d57ca6b43bf23d6563b53"},
-    {file = "coverage-5.5-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:f0b278ce10936db1a37e6954e15a3730bea96a0997c26d7fee88e6c396c2086d"},
-    {file = "coverage-5.5-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:796c9c3c79747146ebd278dbe1e5c5c05dd6b10cc3bcb8389dfdf844f3ead638"},
-    {file = "coverage-5.5-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:53194af30d5bad77fcba80e23a1441c71abfb3e01192034f8246e0d8f99528f3"},
-    {file = "coverage-5.5-cp37-cp37m-win32.whl", hash = "sha256:184a47bbe0aa6400ed2d41d8e9ed868b8205046518c52464fde713ea06e3a74a"},
-    {file = "coverage-5.5-cp37-cp37m-win_amd64.whl", hash = "sha256:2949cad1c5208b8298d5686d5a85b66aae46d73eec2c3e08c817dd3513e5848a"},
-    {file = "coverage-5.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:217658ec7187497e3f3ebd901afdca1af062b42cfe3e0dafea4cced3983739f6"},
-    {file = "coverage-5.5-cp38-cp38-manylinux1_i686.whl", hash = "sha256:1aa846f56c3d49205c952d8318e76ccc2ae23303351d9270ab220004c580cfe2"},
-    {file = "coverage-5.5-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:24d4a7de75446be83244eabbff746d66b9240ae020ced65d060815fac3423759"},
-    {file = "coverage-5.5-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d1f8bf7b90ba55699b3a5e44930e93ff0189aa27186e96071fac7dd0d06a1873"},
-    {file = "coverage-5.5-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:970284a88b99673ccb2e4e334cfb38a10aab7cd44f7457564d11898a74b62d0a"},
-    {file = "coverage-5.5-cp38-cp38-win32.whl", hash = "sha256:01d84219b5cdbfc8122223b39a954820929497a1cb1422824bb86b07b74594b6"},
-    {file = "coverage-5.5-cp38-cp38-win_amd64.whl", hash = "sha256:2e0d881ad471768bf6e6c2bf905d183543f10098e3b3640fc029509530091502"},
-    {file = "coverage-5.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d1f9ce122f83b2305592c11d64f181b87153fc2c2bbd3bb4a3dde8303cfb1a6b"},
-    {file = "coverage-5.5-cp39-cp39-manylinux1_i686.whl", hash = "sha256:13c4ee887eca0f4c5a247b75398d4114c37882658300e153113dafb1d76de529"},
-    {file = "coverage-5.5-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:52596d3d0e8bdf3af43db3e9ba8dcdaac724ba7b5ca3f6358529d56f7a166f8b"},
-    {file = "coverage-5.5-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:2cafbbb3af0733db200c9b5f798d18953b1a304d3f86a938367de1567f4b5bff"},
-    {file = "coverage-5.5-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:44d654437b8ddd9eee7d1eaee28b7219bec228520ff809af170488fd2fed3e2b"},
-    {file = "coverage-5.5-cp39-cp39-win32.whl", hash = "sha256:d314ed732c25d29775e84a960c3c60808b682c08d86602ec2c3008e1202e3bb6"},
-    {file = "coverage-5.5-cp39-cp39-win_amd64.whl", hash = "sha256:13034c4409db851670bc9acd836243aeee299949bd5673e11844befcb0149f03"},
-    {file = "coverage-5.5-pp36-none-any.whl", hash = "sha256:f030f8873312a16414c0d8e1a1ddff2d3235655a2174e3648b4fa66b3f2f1079"},
-    {file = "coverage-5.5-pp37-none-any.whl", hash = "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4"},
-    {file = "coverage-5.5.tar.gz", hash = "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c"},
+    {file = "coverage-6.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9b27d894748475fa858f9597c0ee1d4829f44683f3813633aaf94b19cb5453cf"},
+    {file = "coverage-6.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:37d1141ad6b2466a7b53a22e08fe76994c2d35a5b6b469590424a9953155afac"},
+    {file = "coverage-6.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9987b0354b06d4df0f4d3e0ec1ae76d7ce7cbca9a2f98c25041eb79eec766f1"},
+    {file = "coverage-6.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:26e2deacd414fc2f97dd9f7676ee3eaecd299ca751412d89f40bc01557a6b1b4"},
+    {file = "coverage-6.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4dd8bafa458b5c7d061540f1ee9f18025a68e2d8471b3e858a9dad47c8d41903"},
+    {file = "coverage-6.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:46191097ebc381fbf89bdce207a6c107ac4ec0890d8d20f3360345ff5976155c"},
+    {file = "coverage-6.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6f89d05e028d274ce4fa1a86887b071ae1755082ef94a6740238cd7a8178804f"},
+    {file = "coverage-6.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:58303469e9a272b4abdb9e302a780072c0633cdcc0165db7eec0f9e32f901e05"},
+    {file = "coverage-6.3.2-cp310-cp310-win32.whl", hash = "sha256:2fea046bfb455510e05be95e879f0e768d45c10c11509e20e06d8fcaa31d9e39"},
+    {file = "coverage-6.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:a2a8b8bcc399edb4347a5ca8b9b87e7524c0967b335fbb08a83c8421489ddee1"},
+    {file = "coverage-6.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f1555ea6d6da108e1999b2463ea1003fe03f29213e459145e70edbaf3e004aaa"},
+    {file = "coverage-6.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5f4e1edcf57ce94e5475fe09e5afa3e3145081318e5fd1a43a6b4539a97e518"},
+    {file = "coverage-6.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a15dc0a14008f1da3d1ebd44bdda3e357dbabdf5a0b5034d38fcde0b5c234b7"},
+    {file = "coverage-6.3.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21b7745788866028adeb1e0eca3bf1101109e2dc58456cb49d2d9b99a8c516e6"},
+    {file = "coverage-6.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8ce257cac556cb03be4a248d92ed36904a59a4a5ff55a994e92214cde15c5bad"},
+    {file = "coverage-6.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b0be84e5a6209858a1d3e8d1806c46214e867ce1b0fd32e4ea03f4bd8b2e3359"},
+    {file = "coverage-6.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:acf53bc2cf7282ab9b8ba346746afe703474004d9e566ad164c91a7a59f188a4"},
+    {file = "coverage-6.3.2-cp37-cp37m-win32.whl", hash = "sha256:8bdde1177f2311ee552f47ae6e5aa7750c0e3291ca6b75f71f7ffe1f1dab3dca"},
+    {file = "coverage-6.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b31651d018b23ec463e95cf10070d0b2c548aa950a03d0b559eaa11c7e5a6fa3"},
+    {file = "coverage-6.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:07e6db90cd9686c767dcc593dff16c8c09f9814f5e9c51034066cad3373b914d"},
+    {file = "coverage-6.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2c6dbb42f3ad25760010c45191e9757e7dce981cbfb90e42feef301d71540059"},
+    {file = "coverage-6.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c76aeef1b95aff3905fb2ae2d96e319caca5b76fa41d3470b19d4e4a3a313512"},
+    {file = "coverage-6.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cf5cfcb1521dc3255d845d9dca3ff204b3229401994ef8d1984b32746bb45ca"},
+    {file = "coverage-6.3.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fbbdc8d55990eac1b0919ca69eb5a988a802b854488c34b8f37f3e2025fa90d"},
+    {file = "coverage-6.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ec6bc7fe73a938933d4178c9b23c4e0568e43e220aef9472c4f6044bfc6dd0f0"},
+    {file = "coverage-6.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9baff2a45ae1f17c8078452e9e5962e518eab705e50a0aa8083733ea7d45f3a6"},
+    {file = "coverage-6.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd9e830e9d8d89b20ab1e5af09b32d33e1a08ef4c4e14411e559556fd788e6b2"},
+    {file = "coverage-6.3.2-cp38-cp38-win32.whl", hash = "sha256:f7331dbf301b7289013175087636bbaf5b2405e57259dd2c42fdcc9fcc47325e"},
+    {file = "coverage-6.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:68353fe7cdf91f109fc7d474461b46e7f1f14e533e911a2a2cbb8b0fc8613cf1"},
+    {file = "coverage-6.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b78e5afb39941572209f71866aa0b206c12f0109835aa0d601e41552f9b3e620"},
+    {file = "coverage-6.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4e21876082ed887baed0146fe222f861b5815455ada3b33b890f4105d806128d"},
+    {file = "coverage-6.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34626a7eee2a3da12af0507780bb51eb52dca0e1751fd1471d0810539cefb536"},
+    {file = "coverage-6.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1ebf730d2381158ecf3dfd4453fbca0613e16eaa547b4170e2450c9707665ce7"},
+    {file = "coverage-6.3.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd6fe30bd519694b356cbfcaca9bd5c1737cddd20778c6a581ae20dc8c04def2"},
+    {file = "coverage-6.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:96f8a1cb43ca1422f36492bebe63312d396491a9165ed3b9231e778d43a7fca4"},
+    {file = "coverage-6.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:dd035edafefee4d573140a76fdc785dc38829fe5a455c4bb12bac8c20cfc3d69"},
+    {file = "coverage-6.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5ca5aeb4344b30d0bec47481536b8ba1181d50dbe783b0e4ad03c95dc1296684"},
+    {file = "coverage-6.3.2-cp39-cp39-win32.whl", hash = "sha256:f5fa5803f47e095d7ad8443d28b01d48c0359484fec1b9d8606d0e3282084bc4"},
+    {file = "coverage-6.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:9548f10d8be799551eb3a9c74bbf2b4934ddb330e08a73320123c07f95cc2d92"},
+    {file = "coverage-6.3.2-pp36.pp37.pp38-none-any.whl", hash = "sha256:18d520c6860515a771708937d2f78f63cc47ab3b80cb78e86573b0a760161faf"},
+    {file = "coverage-6.3.2.tar.gz", hash = "sha256:03e2a7826086b91ef345ff18742ee9fc47a6839ccd517061ef8fa1976e652ce9"},
+]
+cryptography = [
+    {file = "cryptography-36.0.2-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:4e2dddd38a5ba733be6a025a1475a9f45e4e41139d1321f412c6b360b19070b6"},
+    {file = "cryptography-36.0.2-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:4881d09298cd0b669bb15b9cfe6166f16fc1277b4ed0d04a22f3d6430cb30f1d"},
+    {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ea634401ca02367c1567f012317502ef3437522e2fc44a3ea1844de028fa4b84"},
+    {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:7be666cc4599b415f320839e36367b273db8501127b38316f3b9f22f17a0b815"},
+    {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8241cac0aae90b82d6b5c443b853723bcc66963970c67e56e71a2609dc4b5eaf"},
+    {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b2d54e787a884ffc6e187262823b6feb06c338084bbe80d45166a1cb1c6c5bf"},
+    {file = "cryptography-36.0.2-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:c2c5250ff0d36fd58550252f54915776940e4e866f38f3a7866d92b32a654b86"},
+    {file = "cryptography-36.0.2-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ec6597aa85ce03f3e507566b8bcdf9da2227ec86c4266bd5e6ab4d9e0cc8dab2"},
+    {file = "cryptography-36.0.2-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ca9f686517ec2c4a4ce930207f75c00bf03d94e5063cbc00a1dc42531511b7eb"},
+    {file = "cryptography-36.0.2-cp36-abi3-win32.whl", hash = "sha256:f64b232348ee82f13aac22856515ce0195837f6968aeaa94a3d0353ea2ec06a6"},
+    {file = "cryptography-36.0.2-cp36-abi3-win_amd64.whl", hash = "sha256:53e0285b49fd0ab6e604f4c5d9c5ddd98de77018542e88366923f152dbeb3c29"},
+    {file = "cryptography-36.0.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:32db5cc49c73f39aac27574522cecd0a4bb7384e71198bc65a0d23f901e89bb7"},
+    {file = "cryptography-36.0.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b3d199647468d410994dbeb8cec5816fb74feb9368aedf300af709ef507e3e"},
+    {file = "cryptography-36.0.2-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:da73d095f8590ad437cd5e9faf6628a218aa7c387e1fdf67b888b47ba56a17f0"},
+    {file = "cryptography-36.0.2-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:0a3bf09bb0b7a2c93ce7b98cb107e9170a90c51a0162a20af1c61c765b90e60b"},
+    {file = "cryptography-36.0.2-pp38-pypy38_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8897b7b7ec077c819187a123174b645eb680c13df68354ed99f9b40a50898f77"},
+    {file = "cryptography-36.0.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82740818f2f240a5da8dfb8943b360e4f24022b093207160c77cadade47d7c85"},
+    {file = "cryptography-36.0.2-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:1f64a62b3b75e4005df19d3b5235abd43fa6358d5516cfc43d87aeba8d08dd51"},
+    {file = "cryptography-36.0.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e167b6b710c7f7bc54e67ef593f8731e1f45aa35f8a8a7b72d6e42ec76afd4b3"},
+    {file = "cryptography-36.0.2.tar.gz", hash = "sha256:70f8f4f7bb2ac9f340655cbac89d68c527af5bb4387522a8413e841e3e6628c9"},
+]
+cssselect = [
+    {file = "cssselect-1.1.0-py2.py3-none-any.whl", hash = "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf"},
+    {file = "cssselect-1.1.0.tar.gz", hash = "sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc"},
+]
+dill = [
+    {file = "dill-0.3.4-py2.py3-none-any.whl", hash = "sha256:7e40e4a70304fd9ceab3535d36e58791d9c4a776b38ec7f7ec9afc8d3dca4d4f"},
+    {file = "dill-0.3.4.zip", hash = "sha256:9f9734205146b2b353ab3fec9af0070237b6ddae78452af83d2fca84d739e675"},
 ]
 faker = [
-    {file = "Faker-8.11.0-py3-none-any.whl", hash = "sha256:3e737576ff50cd98dfed643d6b3fd63194eca9df00e7f595960fe7da5220723d"},
-    {file = "Faker-8.11.0.tar.gz", hash = "sha256:b9e81e9da3dda3ac54189e034cfb943de576a259caeb226ccab43fcbcf6a7891"},
+    {file = "Faker-8.16.0-py3-none-any.whl", hash = "sha256:bb10913b9d3ac2aa37180f816c82040e81f9e0c32cb08445533f293cec8930bf"},
+    {file = "Faker-8.16.0.tar.gz", hash = "sha256:d70b375d0af0e4c3abd594003691a1055a96281a414884e623d27bccc7d781da"},
 ]
 faker-schema = [
     {file = "faker-schema-0.1.4.tar.gz", hash = "sha256:8eb3b5c26b2535d2cc905a0bdd7c9d8423efa43e1d9685643a8f2749fa7e809d"},
 ]
+filelock = [
+    {file = "filelock-3.6.0-py3-none-any.whl", hash = "sha256:f8314284bfffbdcfa0ff3d7992b023d4c628ced6feb957351d4c48d059f56bc0"},
+    {file = "filelock-3.6.0.tar.gz", hash = "sha256:9cd540a9352e432c7246a48fe4e8712b10acb1df2ad1f30e8c070b82ae1fed85"},
+]
 flake8 = [
     {file = "flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907"},
     {file = "flake8-3.9.2.tar.gz", hash = "sha256:07528381786f2a6237b061f6e96610a4167b226cb926e2aa2b6b1d78057c576b"},
@@ -461,33 +1107,135 @@ geopy = [
     {file = "geopy-2.2.0-py3-none-any.whl", hash = "sha256:8f1f949082b964385de61fcc3a667a6a9a6e242beb1ae8972449f164b2ba0e89"},
     {file = "geopy-2.2.0.tar.gz", hash = "sha256:58b7edf526b8c32e33126570b5f4fcdfaa29d4416506064777ae8d84cd103fdd"},
 ]
+hyperlink = [
+    {file = "hyperlink-21.0.0-py2.py3-none-any.whl", hash = "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4"},
+    {file = "hyperlink-21.0.0.tar.gz", hash = "sha256:427af957daa58bc909471c6c40f74c5450fa123dd093fc53efd2e91d2705a56b"},
+]
+idna = [
+    {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"},
+    {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"},
+]
+incremental = [
+    {file = "incremental-21.3.0-py2.py3-none-any.whl", hash = "sha256:92014aebc6a20b78a8084cdd5645eeaa7f74b8933f70fa3ada2cfbd1e3b54321"},
+    {file = "incremental-21.3.0.tar.gz", hash = "sha256:02f5de5aff48f6b9f665d99d48bfc7ec03b6e3943210de7cfc88856d755d6f57"},
+]
 isort = [
-    {file = "isort-5.8.0-py3-none-any.whl", hash = "sha256:2bb1680aad211e3c9944dbce1d4ba09a989f04e238296c87fe2139faa26d655d"},
-    {file = "isort-5.8.0.tar.gz", hash = "sha256:0a943902919f65c5684ac4e0154b1ad4fac6dcaa5d9f3426b732f1c8b5419be6"},
+    {file = "isort-5.10.1-py3-none-any.whl", hash = "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7"},
+    {file = "isort-5.10.1.tar.gz", hash = "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951"},
+]
+itemadapter = [
+    {file = "itemadapter-0.5.0-py3-none-any.whl", hash = "sha256:cbe58de9fd64422132610e32e0f603a5a85b0cfc062bbee116782d0b4f90529a"},
+    {file = "itemadapter-0.5.0.tar.gz", hash = "sha256:05b6a79dd31a7a993dfb2e8386a91c97e3b8c6cf28b72553e808e625e0b87c20"},
+]
+itemloaders = [
+    {file = "itemloaders-1.0.4-py3-none-any.whl", hash = "sha256:4cb46a0f8915e910c770242ae3b60b1149913ed37162804f1e40e8535d6ec497"},
+    {file = "itemloaders-1.0.4.tar.gz", hash = "sha256:1277cd8ca3e4c02dcdfbc1bcae9134ad89acfa6041bd15b4561c6290203a0c96"},
+]
+jmespath = [
+    {file = "jmespath-1.0.0-py3-none-any.whl", hash = "sha256:e8dcd576ed616f14ec02eed0005c85973b5890083313860136657e24784e4c04"},
+    {file = "jmespath-1.0.0.tar.gz", hash = "sha256:a490e280edd1f57d6de88636992d05b71e97d69a26a19f058ecf7d304474bf5e"},
 ]
 lazy-object-proxy = [
-    {file = "lazy-object-proxy-1.6.0.tar.gz", hash = "sha256:489000d368377571c6f982fba6497f2aa13c6d1facc40660963da62f5c379726"},
-    {file = "lazy_object_proxy-1.6.0-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:c6938967f8528b3668622a9ed3b31d145fab161a32f5891ea7b84f6b790be05b"},
-    {file = "lazy_object_proxy-1.6.0-cp27-cp27m-win32.whl", hash = "sha256:ebfd274dcd5133e0afae738e6d9da4323c3eb021b3e13052d8cbd0e457b1256e"},
-    {file = "lazy_object_proxy-1.6.0-cp27-cp27m-win_amd64.whl", hash = "sha256:ed361bb83436f117f9917d282a456f9e5009ea12fd6de8742d1a4752c3017e93"},
-    {file = "lazy_object_proxy-1.6.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d900d949b707778696fdf01036f58c9876a0d8bfe116e8d220cfd4b15f14e741"},
-    {file = "lazy_object_proxy-1.6.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:5743a5ab42ae40caa8421b320ebf3a998f89c85cdc8376d6b2e00bd12bd1b587"},
-    {file = "lazy_object_proxy-1.6.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:bf34e368e8dd976423396555078def5cfc3039ebc6fc06d1ae2c5a65eebbcde4"},
-    {file = "lazy_object_proxy-1.6.0-cp36-cp36m-win32.whl", hash = "sha256:b579f8acbf2bdd9ea200b1d5dea36abd93cabf56cf626ab9c744a432e15c815f"},
-    {file = "lazy_object_proxy-1.6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:4f60460e9f1eb632584c9685bccea152f4ac2130e299784dbaf9fae9f49891b3"},
-    {file = "lazy_object_proxy-1.6.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d7124f52f3bd259f510651450e18e0fd081ed82f3c08541dffc7b94b883aa981"},
-    {file = "lazy_object_proxy-1.6.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:22ddd618cefe54305df49e4c069fa65715be4ad0e78e8d252a33debf00f6ede2"},
-    {file = "lazy_object_proxy-1.6.0-cp37-cp37m-win32.whl", hash = "sha256:9d397bf41caad3f489e10774667310d73cb9c4258e9aed94b9ec734b34b495fd"},
-    {file = "lazy_object_proxy-1.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:24a5045889cc2729033b3e604d496c2b6f588c754f7a62027ad4437a7ecc4837"},
-    {file = "lazy_object_proxy-1.6.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:17e0967ba374fc24141738c69736da90e94419338fd4c7c7bef01ee26b339653"},
-    {file = "lazy_object_proxy-1.6.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:410283732af311b51b837894fa2f24f2c0039aa7f220135192b38fcc42bd43d3"},
-    {file = "lazy_object_proxy-1.6.0-cp38-cp38-win32.whl", hash = "sha256:85fb7608121fd5621cc4377a8961d0b32ccf84a7285b4f1d21988b2eae2868e8"},
-    {file = "lazy_object_proxy-1.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:d1c2676e3d840852a2de7c7d5d76407c772927addff8d742b9808fe0afccebdf"},
-    {file = "lazy_object_proxy-1.6.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:b865b01a2e7f96db0c5d12cfea590f98d8c5ba64ad222300d93ce6ff9138bcad"},
-    {file = "lazy_object_proxy-1.6.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:4732c765372bd78a2d6b2150a6e99d00a78ec963375f236979c0626b97ed8e43"},
-    {file = "lazy_object_proxy-1.6.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:9698110e36e2df951c7c36b6729e96429c9c32b3331989ef19976592c5f3c77a"},
-    {file = "lazy_object_proxy-1.6.0-cp39-cp39-win32.whl", hash = "sha256:1fee665d2638491f4d6e55bd483e15ef21f6c8c2095f235fef72601021e64f61"},
-    {file = "lazy_object_proxy-1.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:f5144c75445ae3ca2057faac03fda5a902eff196702b0a24daf1d6ce0650514b"},
+    {file = "lazy-object-proxy-1.7.1.tar.gz", hash = "sha256:d609c75b986def706743cdebe5e47553f4a5a1da9c5ff66d76013ef396b5a8a4"},
+    {file = "lazy_object_proxy-1.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bb8c5fd1684d60a9902c60ebe276da1f2281a318ca16c1d0a96db28f62e9166b"},
+    {file = "lazy_object_proxy-1.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a57d51ed2997e97f3b8e3500c984db50a554bb5db56c50b5dab1b41339b37e36"},
+    {file = "lazy_object_proxy-1.7.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd45683c3caddf83abbb1249b653a266e7069a09f486daa8863fb0e7496a9fdb"},
+    {file = "lazy_object_proxy-1.7.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:8561da8b3dd22d696244d6d0d5330618c993a215070f473b699e00cf1f3f6443"},
+    {file = "lazy_object_proxy-1.7.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fccdf7c2c5821a8cbd0a9440a456f5050492f2270bd54e94360cac663398739b"},
+    {file = "lazy_object_proxy-1.7.1-cp310-cp310-win32.whl", hash = "sha256:898322f8d078f2654d275124a8dd19b079080ae977033b713f677afcfc88e2b9"},
+    {file = "lazy_object_proxy-1.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:85b232e791f2229a4f55840ed54706110c80c0a210d076eee093f2b2e33e1bfd"},
+    {file = "lazy_object_proxy-1.7.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:46ff647e76f106bb444b4533bb4153c7370cdf52efc62ccfc1a28bdb3cc95442"},
+    {file = "lazy_object_proxy-1.7.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12f3bb77efe1367b2515f8cb4790a11cffae889148ad33adad07b9b55e0ab22c"},
+    {file = "lazy_object_proxy-1.7.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c19814163728941bb871240d45c4c30d33b8a2e85972c44d4e63dd7107faba44"},
+    {file = "lazy_object_proxy-1.7.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:e40f2013d96d30217a51eeb1db28c9ac41e9d0ee915ef9d00da639c5b63f01a1"},
+    {file = "lazy_object_proxy-1.7.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2052837718516a94940867e16b1bb10edb069ab475c3ad84fd1e1a6dd2c0fcfc"},
+    {file = "lazy_object_proxy-1.7.1-cp36-cp36m-win32.whl", hash = "sha256:6a24357267aa976abab660b1d47a34aaf07259a0c3859a34e536f1ee6e76b5bb"},
+    {file = "lazy_object_proxy-1.7.1-cp36-cp36m-win_amd64.whl", hash = "sha256:6aff3fe5de0831867092e017cf67e2750c6a1c7d88d84d2481bd84a2e019ec35"},
+    {file = "lazy_object_proxy-1.7.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6a6e94c7b02641d1311228a102607ecd576f70734dc3d5e22610111aeacba8a0"},
+    {file = "lazy_object_proxy-1.7.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4ce15276a1a14549d7e81c243b887293904ad2d94ad767f42df91e75fd7b5b6"},
+    {file = "lazy_object_proxy-1.7.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e368b7f7eac182a59ff1f81d5f3802161932a41dc1b1cc45c1f757dc876b5d2c"},
+    {file = "lazy_object_proxy-1.7.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6ecbb350991d6434e1388bee761ece3260e5228952b1f0c46ffc800eb313ff42"},
+    {file = "lazy_object_proxy-1.7.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:553b0f0d8dbf21890dd66edd771f9b1b5f51bd912fa5f26de4449bfc5af5e029"},
+    {file = "lazy_object_proxy-1.7.1-cp37-cp37m-win32.whl", hash = "sha256:c7a683c37a8a24f6428c28c561c80d5f4fd316ddcf0c7cab999b15ab3f5c5c69"},
+    {file = "lazy_object_proxy-1.7.1-cp37-cp37m-win_amd64.whl", hash = "sha256:df2631f9d67259dc9620d831384ed7732a198eb434eadf69aea95ad18c587a28"},
+    {file = "lazy_object_proxy-1.7.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:07fa44286cda977bd4803b656ffc1c9b7e3bc7dff7d34263446aec8f8c96f88a"},
+    {file = "lazy_object_proxy-1.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4dca6244e4121c74cc20542c2ca39e5c4a5027c81d112bfb893cf0790f96f57e"},
+    {file = "lazy_object_proxy-1.7.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:91ba172fc5b03978764d1df5144b4ba4ab13290d7bab7a50f12d8117f8630c38"},
+    {file = "lazy_object_proxy-1.7.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:043651b6cb706eee4f91854da4a089816a6606c1428fd391573ef8cb642ae4f7"},
+    {file = "lazy_object_proxy-1.7.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b9e89b87c707dd769c4ea91f7a31538888aad05c116a59820f28d59b3ebfe25a"},
+    {file = "lazy_object_proxy-1.7.1-cp38-cp38-win32.whl", hash = "sha256:9d166602b525bf54ac994cf833c385bfcc341b364e3ee71e3bf5a1336e677b55"},
+    {file = "lazy_object_proxy-1.7.1-cp38-cp38-win_amd64.whl", hash = "sha256:8f3953eb575b45480db6568306893f0bd9d8dfeeebd46812aa09ca9579595148"},
+    {file = "lazy_object_proxy-1.7.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dd7ed7429dbb6c494aa9bc4e09d94b778a3579be699f9d67da7e6804c422d3de"},
+    {file = "lazy_object_proxy-1.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70ed0c2b380eb6248abdef3cd425fc52f0abd92d2b07ce26359fcbc399f636ad"},
+    {file = "lazy_object_proxy-1.7.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7096a5e0c1115ec82641afbdd70451a144558ea5cf564a896294e346eb611be1"},
+    {file = "lazy_object_proxy-1.7.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f769457a639403073968d118bc70110e7dce294688009f5c24ab78800ae56dc8"},
+    {file = "lazy_object_proxy-1.7.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:39b0e26725c5023757fc1ab2a89ef9d7ab23b84f9251e28f9cc114d5b59c1b09"},
+    {file = "lazy_object_proxy-1.7.1-cp39-cp39-win32.whl", hash = "sha256:2130db8ed69a48a3440103d4a520b89d8a9405f1b06e2cc81640509e8bf6548f"},
+    {file = "lazy_object_proxy-1.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:677ea950bef409b47e51e733283544ac3d660b709cfce7b187f5ace137960d61"},
+    {file = "lazy_object_proxy-1.7.1-pp37.pp38-none-any.whl", hash = "sha256:d66906d5785da8e0be7360912e99c9188b70f52c422f9fc18223347235691a84"},
+]
+lxml = [
+    {file = "lxml-4.8.0-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:e1ab2fac607842ac36864e358c42feb0960ae62c34aa4caaf12ada0a1fb5d99b"},
+    {file = "lxml-4.8.0-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28d1af847786f68bec57961f31221125c29d6f52d9187c01cd34dc14e2b29430"},
+    {file = "lxml-4.8.0-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b92d40121dcbd74831b690a75533da703750f7041b4bf951befc657c37e5695a"},
+    {file = "lxml-4.8.0-cp27-cp27m-win32.whl", hash = "sha256:e01f9531ba5420838c801c21c1b0f45dbc9607cb22ea2cf132844453bec863a5"},
+    {file = "lxml-4.8.0-cp27-cp27m-win_amd64.whl", hash = "sha256:6259b511b0f2527e6d55ad87acc1c07b3cbffc3d5e050d7e7bcfa151b8202df9"},
+    {file = "lxml-4.8.0-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1010042bfcac2b2dc6098260a2ed022968dbdfaf285fc65a3acf8e4eb1ffd1bc"},
+    {file = "lxml-4.8.0-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:fa56bb08b3dd8eac3a8c5b7d075c94e74f755fd9d8a04543ae8d37b1612dd170"},
+    {file = "lxml-4.8.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:31ba2cbc64516dcdd6c24418daa7abff989ddf3ba6d3ea6f6ce6f2ed6e754ec9"},
+    {file = "lxml-4.8.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:31499847fc5f73ee17dbe1b8e24c6dafc4e8d5b48803d17d22988976b0171f03"},
+    {file = "lxml-4.8.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:5f7d7d9afc7b293147e2d506a4596641d60181a35279ef3aa5778d0d9d9123fe"},
+    {file = "lxml-4.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a3c5f1a719aa11866ffc530d54ad965063a8cbbecae6515acbd5f0fae8f48eaa"},
+    {file = "lxml-4.8.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6268e27873a3d191849204d00d03f65c0e343b3bcb518a6eaae05677c95621d1"},
+    {file = "lxml-4.8.0-cp310-cp310-win32.whl", hash = "sha256:330bff92c26d4aee79c5bc4d9967858bdbe73fdbdbacb5daf623a03a914fe05b"},
+    {file = "lxml-4.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:b2582b238e1658c4061ebe1b4df53c435190d22457642377fd0cb30685cdfb76"},
+    {file = "lxml-4.8.0-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a2bfc7e2a0601b475477c954bf167dee6d0f55cb167e3f3e7cefad906e7759f6"},
+    {file = "lxml-4.8.0-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a1547ff4b8a833511eeaceacbcd17b043214fcdb385148f9c1bc5556ca9623e2"},
+    {file = "lxml-4.8.0-cp35-cp35m-win32.whl", hash = "sha256:a9f1c3489736ff8e1c7652e9dc39f80cff820f23624f23d9eab6e122ac99b150"},
+    {file = "lxml-4.8.0-cp35-cp35m-win_amd64.whl", hash = "sha256:530f278849031b0eb12f46cca0e5db01cfe5177ab13bd6878c6e739319bae654"},
+    {file = "lxml-4.8.0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:078306d19a33920004addeb5f4630781aaeabb6a8d01398045fcde085091a169"},
+    {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:86545e351e879d0b72b620db6a3b96346921fa87b3d366d6c074e5a9a0b8dadb"},
+    {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24f5c5ae618395ed871b3d8ebfcbb36e3f1091fd847bf54c4de623f9107942f3"},
+    {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:bbab6faf6568484707acc052f4dfc3802bdb0cafe079383fbaa23f1cdae9ecd4"},
+    {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7993232bd4044392c47779a3c7e8889fea6883be46281d45a81451acfd704d7e"},
+    {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6d6483b1229470e1d8835e52e0ff3c6973b9b97b24cd1c116dca90b57a2cc613"},
+    {file = "lxml-4.8.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:ad4332a532e2d5acb231a2e5d33f943750091ee435daffca3fec0a53224e7e33"},
+    {file = "lxml-4.8.0-cp36-cp36m-win32.whl", hash = "sha256:db3535733f59e5605a88a706824dfcb9bd06725e709ecb017e165fc1d6e7d429"},
+    {file = "lxml-4.8.0-cp36-cp36m-win_amd64.whl", hash = "sha256:5f148b0c6133fb928503cfcdfdba395010f997aa44bcf6474fcdd0c5398d9b63"},
+    {file = "lxml-4.8.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:8a31f24e2a0b6317f33aafbb2f0895c0bce772980ae60c2c640d82caac49628a"},
+    {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:719544565c2937c21a6f76d520e6e52b726d132815adb3447ccffbe9f44203c4"},
+    {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:c0b88ed1ae66777a798dc54f627e32d3b81c8009967c63993c450ee4cbcbec15"},
+    {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fa9b7c450be85bfc6cd39f6df8c5b8cbd76b5d6fc1f69efec80203f9894b885f"},
+    {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e9f84ed9f4d50b74fbc77298ee5c870f67cb7e91dcdc1a6915cb1ff6a317476c"},
+    {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:1d650812b52d98679ed6c6b3b55cbb8fe5a5460a0aef29aeb08dc0b44577df85"},
+    {file = "lxml-4.8.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:80bbaddf2baab7e6de4bc47405e34948e694a9efe0861c61cdc23aa774fcb141"},
+    {file = "lxml-4.8.0-cp37-cp37m-win32.whl", hash = "sha256:6f7b82934c08e28a2d537d870293236b1000d94d0b4583825ab9649aef7ddf63"},
+    {file = "lxml-4.8.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e1fd7d2fe11f1cb63d3336d147c852f6d07de0d0020d704c6031b46a30b02ca8"},
+    {file = "lxml-4.8.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:5045ee1ccd45a89c4daec1160217d363fcd23811e26734688007c26f28c9e9e7"},
+    {file = "lxml-4.8.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0c1978ff1fd81ed9dcbba4f91cf09faf1f8082c9d72eb122e92294716c605428"},
+    {file = "lxml-4.8.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cbf2ff155b19dc4d4100f7442f6a697938bf4493f8d3b0c51d45568d5666b5"},
+    {file = "lxml-4.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:ce13d6291a5f47c1c8dbd375baa78551053bc6b5e5c0e9bb8e39c0a8359fd52f"},
+    {file = "lxml-4.8.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e11527dc23d5ef44d76fef11213215c34f36af1608074561fcc561d983aeb870"},
+    {file = "lxml-4.8.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:60d2f60bd5a2a979df28ab309352cdcf8181bda0cca4529769a945f09aba06f9"},
+    {file = "lxml-4.8.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:62f93eac69ec0f4be98d1b96f4d6b964855b8255c345c17ff12c20b93f247b68"},
+    {file = "lxml-4.8.0-cp38-cp38-win32.whl", hash = "sha256:20b8a746a026017acf07da39fdb10aa80ad9877046c9182442bf80c84a1c4696"},
+    {file = "lxml-4.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:891dc8f522d7059ff0024cd3ae79fd224752676447f9c678f2a5c14b84d9a939"},
+    {file = "lxml-4.8.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:b6fc2e2fb6f532cf48b5fed57567ef286addcef38c28874458a41b7837a57807"},
+    {file = "lxml-4.8.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:74eb65ec61e3c7c019d7169387d1b6ffcfea1b9ec5894d116a9a903636e4a0b1"},
+    {file = "lxml-4.8.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:627e79894770783c129cc5e89b947e52aa26e8e0557c7e205368a809da4b7939"},
+    {file = "lxml-4.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:545bd39c9481f2e3f2727c78c169425efbfb3fbba6e7db4f46a80ebb249819ca"},
+    {file = "lxml-4.8.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5a58d0b12f5053e270510bf12f753a76aaf3d74c453c00942ed7d2c804ca845c"},
+    {file = "lxml-4.8.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:ec4b4e75fc68da9dc0ed73dcdb431c25c57775383fec325d23a770a64e7ebc87"},
+    {file = "lxml-4.8.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5804e04feb4e61babf3911c2a974a5b86f66ee227cc5006230b00ac6d285b3a9"},
+    {file = "lxml-4.8.0-cp39-cp39-win32.whl", hash = "sha256:aa0cf4922da7a3c905d000b35065df6184c0dc1d866dd3b86fd961905bbad2ea"},
+    {file = "lxml-4.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:dd10383f1d6b7edf247d0960a3db274c07e96cf3a3fc7c41c8448f93eac3fb1c"},
+    {file = "lxml-4.8.0-pp37-pypy37_pp73-macosx_10_14_x86_64.whl", hash = "sha256:2403a6d6fb61c285969b71f4a3527873fe93fd0abe0832d858a17fe68c8fa507"},
+    {file = "lxml-4.8.0-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:986b7a96228c9b4942ec420eff37556c5777bfba6758edcb95421e4a614b57f9"},
+    {file = "lxml-4.8.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6fe4ef4402df0250b75ba876c3795510d782def5c1e63890bde02d622570d39e"},
+    {file = "lxml-4.8.0-pp38-pypy38_pp73-macosx_10_14_x86_64.whl", hash = "sha256:f10ce66fcdeb3543df51d423ede7e238be98412232fca5daec3e54bcd16b8da0"},
+    {file = "lxml-4.8.0-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:730766072fd5dcb219dd2b95c4c49752a54f00157f322bc6d71f7d2a31fecd79"},
+    {file = "lxml-4.8.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8b99ec73073b37f9ebe8caf399001848fced9c08064effdbfc4da2b5a8d07b93"},
+    {file = "lxml-4.8.0.tar.gz", hash = "sha256:f63f62fc60e6228a4ca9abae28228f35e1bd3ce675013d1dfb828688d50c6e23"},
 ]
 mccabe = [
     {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"},
@@ -498,84 +1246,174 @@ mock = [
     {file = "mock-4.0.3.tar.gz", hash = "sha256:7d3fbbde18228f4ff2f1f119a45cdffa458b4c0dee32eb4d2bb2f82554bac7bc"},
 ]
 more-itertools = [
-    {file = "more-itertools-8.8.0.tar.gz", hash = "sha256:83f0308e05477c68f56ea3a888172c78ed5d5b3c282addb67508e7ba6c8f813a"},
-    {file = "more_itertools-8.8.0-py3-none-any.whl", hash = "sha256:2cf89ec599962f2ddc4d568a05defc40e0a587fbc10d5989713638864c36be4d"},
+    {file = "more-itertools-8.12.0.tar.gz", hash = "sha256:7dc6ad46f05f545f900dd59e8dfb4e84a4827b97b3cfecb175ea0c7d247f6064"},
+    {file = "more_itertools-8.12.0-py3-none-any.whl", hash = "sha256:43e6dd9942dffd72661a2c4ef383ad7da1e6a3e968a927ad7a6083ab410a688b"},
 ]
 numpy = [
-    {file = "numpy-1.21.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50"},
-    {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a"},
-    {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062"},
-    {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1"},
-    {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671"},
-    {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e"},
-    {file = "numpy-1.21.1-cp37-cp37m-win32.whl", hash = "sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172"},
-    {file = "numpy-1.21.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8"},
-    {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16"},
-    {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267"},
-    {file = "numpy-1.21.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6"},
-    {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63"},
-    {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af"},
-    {file = "numpy-1.21.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5"},
-    {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68"},
-    {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8"},
-    {file = "numpy-1.21.1-cp38-cp38-win32.whl", hash = "sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd"},
-    {file = "numpy-1.21.1-cp38-cp38-win_amd64.whl", hash = "sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214"},
-    {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f"},
-    {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b"},
-    {file = "numpy-1.21.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac"},
-    {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1"},
-    {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1"},
-    {file = "numpy-1.21.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a"},
-    {file = "numpy-1.21.1-cp39-cp39-win32.whl", hash = "sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2"},
-    {file = "numpy-1.21.1-cp39-cp39-win_amd64.whl", hash = "sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33"},
-    {file = "numpy-1.21.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4"},
-    {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"},
+    {file = "numpy-1.22.3-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:92bfa69cfbdf7dfc3040978ad09a48091143cffb778ec3b03fa170c494118d75"},
+    {file = "numpy-1.22.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8251ed96f38b47b4295b1ae51631de7ffa8260b5b087808ef09a39a9d66c97ab"},
+    {file = "numpy-1.22.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48a3aecd3b997bf452a2dedb11f4e79bc5bfd21a1d4cc760e703c31d57c84b3e"},
+    {file = "numpy-1.22.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3bae1a2ed00e90b3ba5f7bd0a7c7999b55d609e0c54ceb2b076a25e345fa9f4"},
+    {file = "numpy-1.22.3-cp310-cp310-win32.whl", hash = "sha256:f950f8845b480cffe522913d35567e29dd381b0dc7e4ce6a4a9f9156417d2430"},
+    {file = "numpy-1.22.3-cp310-cp310-win_amd64.whl", hash = "sha256:08d9b008d0156c70dc392bb3ab3abb6e7a711383c3247b410b39962263576cd4"},
+    {file = "numpy-1.22.3-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:201b4d0552831f7250a08d3b38de0d989d6f6e4658b709a02a73c524ccc6ffce"},
+    {file = "numpy-1.22.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f8c1f39caad2c896bc0018f699882b345b2a63708008be29b1f355ebf6f933fe"},
+    {file = "numpy-1.22.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:568dfd16224abddafb1cbcce2ff14f522abe037268514dd7e42c6776a1c3f8e5"},
+    {file = "numpy-1.22.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ca688e1b9b95d80250bca34b11a05e389b1420d00e87a0d12dc45f131f704a1"},
+    {file = "numpy-1.22.3-cp38-cp38-win32.whl", hash = "sha256:e7927a589df200c5e23c57970bafbd0cd322459aa7b1ff73b7c2e84d6e3eae62"},
+    {file = "numpy-1.22.3-cp38-cp38-win_amd64.whl", hash = "sha256:07a8c89a04997625236c5ecb7afe35a02af3896c8aa01890a849913a2309c676"},
+    {file = "numpy-1.22.3-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:2c10a93606e0b4b95c9b04b77dc349b398fdfbda382d2a39ba5a822f669a0123"},
+    {file = "numpy-1.22.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fade0d4f4d292b6f39951b6836d7a3c7ef5b2347f3c420cd9820a1d90d794802"},
+    {file = "numpy-1.22.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bfb1bb598e8229c2d5d48db1860bcf4311337864ea3efdbe1171fb0c5da515d"},
+    {file = "numpy-1.22.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97098b95aa4e418529099c26558eeb8486e66bd1e53a6b606d684d0c3616b168"},
+    {file = "numpy-1.22.3-cp39-cp39-win32.whl", hash = "sha256:fdf3c08bce27132395d3c3ba1503cac12e17282358cb4bddc25cc46b0aca07aa"},
+    {file = "numpy-1.22.3-cp39-cp39-win_amd64.whl", hash = "sha256:639b54cdf6aa4f82fe37ebf70401bbb74b8508fddcf4797f9fe59615b8c5813a"},
+    {file = "numpy-1.22.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c34ea7e9d13a70bf2ab64a2532fe149a9aced424cd05a2c4ba662fd989e3e45f"},
+    {file = "numpy-1.22.3.zip", hash = "sha256:dbc7601a3b7472d559dc7b933b18b4b66f9aa7452c120e87dfb33d02008c8a18"},
+]
+oauthlib = [
+    {file = "oauthlib-3.2.0-py3-none-any.whl", hash = "sha256:6db33440354787f9b7f3a6dbd4febf5d0f93758354060e802f6c06cb493022fe"},
+    {file = "oauthlib-3.2.0.tar.gz", hash = "sha256:23a8208d75b902797ea29fd31fa80a15ed9dc2c6c16fe73f5d346f83f6fa27a2"},
 ]
 packaging = [
-    {file = "packaging-20.9-py2.py3-none-any.whl", hash = "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"},
-    {file = "packaging-20.9.tar.gz", hash = "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5"},
+    {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"},
+    {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"},
 ]
 pandas = [
-    {file = "pandas-1.2.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c601c6fdebc729df4438ec1f62275d6136a0dd14d332fc0e8ce3f7d2aadb4dd6"},
-    {file = "pandas-1.2.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:8d4c74177c26aadcfb4fd1de6c1c43c2bf822b3e0fc7a9b409eeaf84b3e92aaa"},
-    {file = "pandas-1.2.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:b730add5267f873b3383c18cac4df2527ac4f0f0eed1c6cf37fcb437e25cf558"},
-    {file = "pandas-1.2.4-cp37-cp37m-win32.whl", hash = "sha256:2cb7e8f4f152f27dc93f30b5c7a98f6c748601ea65da359af734dd0cf3fa733f"},
-    {file = "pandas-1.2.4-cp37-cp37m-win_amd64.whl", hash = "sha256:2111c25e69fa9365ba80bbf4f959400054b2771ac5d041ed19415a8b488dc70a"},
-    {file = "pandas-1.2.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:167693a80abc8eb28051fbd184c1b7afd13ce2c727a5af47b048f1ea3afefff4"},
-    {file = "pandas-1.2.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:612add929bf3ba9d27b436cc8853f5acc337242d6b584203f207e364bb46cb12"},
-    {file = "pandas-1.2.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:971e2a414fce20cc5331fe791153513d076814d30a60cd7348466943e6e909e4"},
-    {file = "pandas-1.2.4-cp38-cp38-win32.whl", hash = "sha256:68d7baa80c74aaacbed597265ca2308f017859123231542ff8a5266d489e1858"},
-    {file = "pandas-1.2.4-cp38-cp38-win_amd64.whl", hash = "sha256:bd659c11a4578af740782288cac141a322057a2e36920016e0fc7b25c5a4b686"},
-    {file = "pandas-1.2.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9db70ffa8b280bb4de83f9739d514cd0735825e79eef3a61d312420b9f16b758"},
-    {file = "pandas-1.2.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:298f0553fd3ba8e002c4070a723a59cdb28eda579f3e243bc2ee397773f5398b"},
-    {file = "pandas-1.2.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:52d2472acbb8a56819a87aafdb8b5b6d2b3386e15c95bde56b281882529a7ded"},
-    {file = "pandas-1.2.4-cp39-cp39-win32.whl", hash = "sha256:d0877407359811f7b853b548a614aacd7dea83b0c0c84620a9a643f180060950"},
-    {file = "pandas-1.2.4-cp39-cp39-win_amd64.whl", hash = "sha256:2b063d41803b6a19703b845609c0b700913593de067b552a8b24dd8eeb8c9895"},
-    {file = "pandas-1.2.4.tar.gz", hash = "sha256:649ecab692fade3cbfcf967ff936496b0cfba0af00a55dfaacd82bdda5cb2279"},
+    {file = "pandas-1.4.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:be67c782c4f1b1f24c2f16a157e12c2693fd510f8df18e3287c77f33d124ed07"},
+    {file = "pandas-1.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5a206afa84ed20e07603f50d22b5f0db3fb556486d8c2462d8bc364831a4b417"},
+    {file = "pandas-1.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0010771bd9223f7afe5f051eb47c4a49534345dfa144f2f5470b27189a4dd3b5"},
+    {file = "pandas-1.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3228198333dd13c90b6434ddf61aa6d57deaca98cf7b654f4ad68a2db84f8cfe"},
+    {file = "pandas-1.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b79af3a69e5175c6fa7b4e046b21a646c8b74e92c6581a9d825687d92071b51"},
+    {file = "pandas-1.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:5586cc95692564b441f4747c47c8a9746792e87b40a4680a2feb7794defb1ce3"},
+    {file = "pandas-1.4.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:061609334a8182ab500a90fe66d46f6f387de62d3a9cb9aa7e62e3146c712167"},
+    {file = "pandas-1.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b8134651258bce418cb79c71adeff0a44090c98d955f6953168ba16cc285d9f7"},
+    {file = "pandas-1.4.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:df82739e00bb6daf4bba4479a40f38c718b598a84654cbd8bb498fd6b0aa8c16"},
+    {file = "pandas-1.4.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:385c52e85aaa8ea6a4c600a9b2821181a51f8be0aee3af6f2dcb41dafc4fc1d0"},
+    {file = "pandas-1.4.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:295872bf1a09758aba199992c3ecde455f01caf32266d50abc1a073e828a7b9d"},
+    {file = "pandas-1.4.2-cp38-cp38-win32.whl", hash = "sha256:95c1e422ced0199cf4a34385ff124b69412c4bc912011ce895582bee620dfcaa"},
+    {file = "pandas-1.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:5c54ea4ef3823108cd4ec7fb27ccba4c3a775e0f83e39c5e17f5094cb17748bc"},
+    {file = "pandas-1.4.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c072c7f06b9242c855ed8021ff970c0e8f8b10b35e2640c657d2a541c5950f59"},
+    {file = "pandas-1.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f549097993744ff8c41b5e8f2f0d3cbfaabe89b4ae32c8c08ead6cc535b80139"},
+    {file = "pandas-1.4.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ff08a14ef21d94cdf18eef7c569d66f2e24e0bc89350bcd7d243dd804e3b5eb2"},
+    {file = "pandas-1.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c5bf555b6b0075294b73965adaafb39cf71c312e38c5935c93d78f41c19828a"},
+    {file = "pandas-1.4.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51649ef604a945f781105a6d2ecf88db7da0f4868ac5d45c51cb66081c4d9c73"},
+    {file = "pandas-1.4.2-cp39-cp39-win32.whl", hash = "sha256:d0d4f13e4be7ce89d7057a786023c461dd9370040bdb5efa0a7fe76b556867a0"},
+    {file = "pandas-1.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:09d8be7dd9e1c4c98224c4dfe8abd60d145d934e9fc1f5f411266308ae683e6a"},
+    {file = "pandas-1.4.2.tar.gz", hash = "sha256:92bc1fc585f1463ca827b45535957815b7deb218c549b7c18402c322c7549a12"},
+]
+parsel = [
+    {file = "parsel-1.6.0-py2.py3-none-any.whl", hash = "sha256:9e1fa8db1c0b4a878bf34b35c043d89c9d1cbebc23b4d34dbc3c0ec33f2e087d"},
+    {file = "parsel-1.6.0.tar.gz", hash = "sha256:70efef0b651a996cceebc69e55a85eb2233be0890959203ba7c3a03c72725c79"},
+]
+platformdirs = [
+    {file = "platformdirs-2.5.1-py3-none-any.whl", hash = "sha256:bcae7cab893c2d310a711b70b24efb93334febe65f8de776ee320b517471e227"},
+    {file = "platformdirs-2.5.1.tar.gz", hash = "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d"},
 ]
 pluggy = [
     {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"},
     {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"},
 ]
+protego = [
+    {file = "Protego-0.2.1-py2.py3-none-any.whl", hash = "sha256:04419b18f20e8909f1691c6b678392988271cc2a324a72f9663cb3af838b4bf7"},
+    {file = "Protego-0.2.1.tar.gz", hash = "sha256:df666d4304dab774e2dc9feb208bb1ac8d71ea5ceec12f4c99eba30fbd642ff2"},
+]
 py = [
-    {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"},
-    {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"},
+    {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"},
+    {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"},
+]
+pyarrow = [
+    {file = "pyarrow-7.0.0-cp310-cp310-macosx_10_13_universal2.whl", hash = "sha256:0f15213f380539c9640cb2413dc677b55e70f04c9e98cfc2e1d8b36c770e1036"},
+    {file = "pyarrow-7.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:29c4e3b3be0b94d07ff4921a5e410fc690a3a066a850a302fc504de5fc638495"},
+    {file = "pyarrow-7.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8a9bfc8a016bcb8f9a8536d2fa14a890b340bc7a236275cd60fd4fb8b93ff405"},
+    {file = "pyarrow-7.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:49d431ed644a3e8f53ae2bbf4b514743570b495b5829548db51610534b6eeee7"},
+    {file = "pyarrow-7.0.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:aa6442a321c1e49480b3d436f7d631c895048a16df572cf71c23c6b53c45ed66"},
+    {file = "pyarrow-7.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6b01a23cb401750092c6f7c4dcae67cd8fd6b99ae710e26f654f23508f25f25"},
+    {file = "pyarrow-7.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f10928745c6ff66e121552731409803bed86c66ac79c64c90438b053b5242c5"},
+    {file = "pyarrow-7.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:759090caa1474cafb5e68c93a9bd6cb45d8bb8e4f2cad2f1a0cc9439bae8ae88"},
+    {file = "pyarrow-7.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:e3fe34bcfc28d9c4a747adc3926d2307a04c5c50b89155946739515ccfe5eab0"},
+    {file = "pyarrow-7.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:040dce5345603e4e621bcf4f3b21f18d557852e7b15307e559bb14c8951c8714"},
+    {file = "pyarrow-7.0.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ed4b647c3345ae3463d341a9d28d0260cd302fb92ecf4e2e3e0f1656d6e0e55c"},
+    {file = "pyarrow-7.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7fecd5d5604f47e003f50887a42aee06cb8b7bf8e8bf7dc543a22331d9ba832"},
+    {file = "pyarrow-7.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f2d00b892fe865e43346acb78761ba268f8bb1cbdba588816590abcb780ee3d"},
+    {file = "pyarrow-7.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:f439f7d77201681fd31391d189aa6b1322d27c9311a8f2fce7d23972471b02b6"},
+    {file = "pyarrow-7.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:3e06b0e29ce1e32f219c670c6b31c33d25a5b8e29c7828f873373aab78bf30a5"},
+    {file = "pyarrow-7.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:13dc05bcf79dbc1bd2de1b05d26eb64824b85883d019d81ca3c2eca9b68b5a44"},
+    {file = "pyarrow-7.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:06183a7ff2b0c030ec0413fc4dc98abad8cf336c78c280a0b7f4bcbebb78d125"},
+    {file = "pyarrow-7.0.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:702c5a9f960b56d03569eaaca2c1a05e8728f05ea1a2138ef64234aa53cd5884"},
+    {file = "pyarrow-7.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7313038203df77ec4092d6363dbc0945071caa72635f365f2b1ae0dd7469865"},
+    {file = "pyarrow-7.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e87d1f7dc7a0b2ecaeb0c7a883a85710f5b5626d4134454f905571c04bc73d5a"},
+    {file = "pyarrow-7.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:ba69488ae25c7fde1a2ae9ea29daf04d676de8960ffd6f82e1e13ca945bb5861"},
+    {file = "pyarrow-7.0.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:11a591f11d2697c751261c9d57e6e5b0d38fdc7f0cc57f4fd6edc657da7737df"},
+    {file = "pyarrow-7.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:6183c700877852dc0f8a76d4c0c2ffd803ba459e2b4a452e355c2d58d48cf39f"},
+    {file = "pyarrow-7.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d1748154714b543e6ae8452a68d4af85caf5298296a7e5d4d00f1b3021838ac6"},
+    {file = "pyarrow-7.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcc8f934c7847a88f13ec35feecffb61fe63bb7a3078bd98dd353762e969ce60"},
+    {file = "pyarrow-7.0.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:759f59ac77b84878dbd54d06cf6df74ff781b8e7cf9313eeffbb5ec97b94385c"},
+    {file = "pyarrow-7.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d3e3f93ac2993df9c5e1922eab7bdea047b9da918a74e52145399bc1f0099a3"},
+    {file = "pyarrow-7.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:306120af554e7e137895254a3b4741fad682875a5f6403509cd276de3fe5b844"},
+    {file = "pyarrow-7.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:087769dac6e567d58d59b94c4f866b3356c00d3db5b261387ece47e7324c2150"},
+    {file = "pyarrow-7.0.0.tar.gz", hash = "sha256:da656cad3c23a2ebb6a307ab01d35fce22f7850059cffafcb90d12590f8f4f38"},
+]
+pyasn1 = [
+    {file = "pyasn1-0.4.8-py2.4.egg", hash = "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3"},
+    {file = "pyasn1-0.4.8-py2.5.egg", hash = "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf"},
+    {file = "pyasn1-0.4.8-py2.6.egg", hash = "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00"},
+    {file = "pyasn1-0.4.8-py2.7.egg", hash = "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8"},
+    {file = "pyasn1-0.4.8-py2.py3-none-any.whl", hash = "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d"},
+    {file = "pyasn1-0.4.8-py3.1.egg", hash = "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86"},
+    {file = "pyasn1-0.4.8-py3.2.egg", hash = "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7"},
+    {file = "pyasn1-0.4.8-py3.3.egg", hash = "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576"},
+    {file = "pyasn1-0.4.8-py3.4.egg", hash = "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12"},
+    {file = "pyasn1-0.4.8-py3.5.egg", hash = "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2"},
+    {file = "pyasn1-0.4.8-py3.6.egg", hash = "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359"},
+    {file = "pyasn1-0.4.8-py3.7.egg", hash = "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776"},
+    {file = "pyasn1-0.4.8.tar.gz", hash = "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba"},
+]
+pyasn1-modules = [
+    {file = "pyasn1-modules-0.2.8.tar.gz", hash = "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e"},
+    {file = "pyasn1_modules-0.2.8-py2.4.egg", hash = "sha256:0fe1b68d1e486a1ed5473f1302bd991c1611d319bba158e98b106ff86e1d7199"},
+    {file = "pyasn1_modules-0.2.8-py2.5.egg", hash = "sha256:fe0644d9ab041506b62782e92b06b8c68cca799e1a9636ec398675459e031405"},
+    {file = "pyasn1_modules-0.2.8-py2.6.egg", hash = "sha256:a99324196732f53093a84c4369c996713eb8c89d360a496b599fb1a9c47fc3eb"},
+    {file = "pyasn1_modules-0.2.8-py2.7.egg", hash = "sha256:0845a5582f6a02bb3e1bde9ecfc4bfcae6ec3210dd270522fee602365430c3f8"},
+    {file = "pyasn1_modules-0.2.8-py2.py3-none-any.whl", hash = "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74"},
+    {file = "pyasn1_modules-0.2.8-py3.1.egg", hash = "sha256:f39edd8c4ecaa4556e989147ebf219227e2cd2e8a43c7e7fcb1f1c18c5fd6a3d"},
+    {file = "pyasn1_modules-0.2.8-py3.2.egg", hash = "sha256:b80486a6c77252ea3a3e9b1e360bc9cf28eaac41263d173c032581ad2f20fe45"},
+    {file = "pyasn1_modules-0.2.8-py3.3.egg", hash = "sha256:65cebbaffc913f4fe9e4808735c95ea22d7a7775646ab690518c056784bc21b4"},
+    {file = "pyasn1_modules-0.2.8-py3.4.egg", hash = "sha256:15b7c67fabc7fc240d87fb9aabf999cf82311a6d6fb2c70d00d3d0604878c811"},
+    {file = "pyasn1_modules-0.2.8-py3.5.egg", hash = "sha256:426edb7a5e8879f1ec54a1864f16b882c2837bfd06eee62f2c982315ee2473ed"},
+    {file = "pyasn1_modules-0.2.8-py3.6.egg", hash = "sha256:cbac4bc38d117f2a49aeedec4407d23e8866ea4ac27ff2cf7fb3e5b570df19e0"},
+    {file = "pyasn1_modules-0.2.8-py3.7.egg", hash = "sha256:c29a5e5cc7a3f05926aff34e097e84f8589cd790ce0ed41b67aed6857b26aafd"},
 ]
 pycodestyle = [
     {file = "pycodestyle-2.7.0-py2.py3-none-any.whl", hash = "sha256:514f76d918fcc0b55c6680472f0a37970994e07bbb80725808c17089be302068"},
     {file = "pycodestyle-2.7.0.tar.gz", hash = "sha256:c389c1d06bf7904078ca03399a4816f974a1d590090fecea0c63ec26ebaf1cef"},
 ]
+pycparser = [
+    {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"},
+    {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"},
+]
+pydispatcher = [
+    {file = "PyDispatcher-2.0.5.tar.gz", hash = "sha256:5570069e1b1769af1fe481de6dd1d3a388492acddd2cdad7a3bde145615d5caf"},
+    {file = "PyDispatcher-2.0.5.zip", hash = "sha256:5be4a8be12805ef7d712dd9a93284fb8bc53f309867e573f653a72e5fd10e433"},
+]
 pyflakes = [
     {file = "pyflakes-2.3.1-py2.py3-none-any.whl", hash = "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3"},
     {file = "pyflakes-2.3.1.tar.gz", hash = "sha256:f5bc8ecabc05bb9d291eb5203d6810b49040f6ff446a756326104746cc00c1db"},
 ]
 pylint = [
-    {file = "pylint-2.8.3-py3-none-any.whl", hash = "sha256:792b38ff30903884e4a9eab814ee3523731abd3c463f3ba48d7b627e87013484"},
-    {file = "pylint-2.8.3.tar.gz", hash = "sha256:0a049c5d47b629d9070c3932d13bff482b12119b6a241a93bc460b0be16953c8"},
+    {file = "pylint-2.13.5-py3-none-any.whl", hash = "sha256:c149694cfdeaee1aa2465e6eaab84c87a881a7d55e6e93e09466be7164764d1e"},
+    {file = "pylint-2.13.5.tar.gz", hash = "sha256:dab221658368c7a05242e673c275c488670144123f4bd262b2777249c1c0de9b"},
+]
+pyopenssl = [
+    {file = "pyOpenSSL-22.0.0-py2.py3-none-any.whl", hash = "sha256:ea252b38c87425b64116f808355e8da644ef9b07e429398bfece610f893ee2e0"},
+    {file = "pyOpenSSL-22.0.0.tar.gz", hash = "sha256:660b1b1425aac4a1bea1d94168a85d99f0b3144c869dd4390d27629d0087f1bf"},
 ]
 pyparsing = [
-    {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"},
-    {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"},
+    {file = "pyparsing-3.0.8-py3-none-any.whl", hash = "sha256:ef7b523f6356f763771559412c0d7134753f037822dad1b16945b7b846f7ad06"},
+    {file = "pyparsing-3.0.8.tar.gz", hash = "sha256:7bf433498c016c4314268d95df76c81b842a4cb2b276fa3312cfb1e1d85f6954"},
+]
+pypydispatcher = [
+    {file = "PyPyDispatcher-2.1.2.tar.gz", hash = "sha256:b6bec5dfcff9d2535bca2b23c80eae367b1ac250a645106948d315fcfa9130f2"},
 ]
 pytest = [
     {file = "pytest-5.4.3-py3-none-any.whl", hash = "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1"},
@@ -586,12 +1424,36 @@ pytest-cov = [
     {file = "pytest_cov-2.12.1-py2.py3-none-any.whl", hash = "sha256:261bb9e47e65bd099c89c3edf92972865210c36813f80ede5277dceb77a4a62a"},
 ]
 python-dateutil = [
-    {file = "python-dateutil-2.8.1.tar.gz", hash = "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c"},
-    {file = "python_dateutil-2.8.1-py2.py3-none-any.whl", hash = "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"},
+    {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
+    {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
 ]
 pytz = [
-    {file = "pytz-2021.1-py2.py3-none-any.whl", hash = "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"},
-    {file = "pytz-2021.1.tar.gz", hash = "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da"},
+    {file = "pytz-2022.1-py2.py3-none-any.whl", hash = "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"},
+    {file = "pytz-2022.1.tar.gz", hash = "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7"},
+]
+queuelib = [
+    {file = "queuelib-1.6.2-py2.py3-none-any.whl", hash = "sha256:4b96d48f650a814c6fb2fd11b968f9c46178b683aad96d68f930fe13a8574d19"},
+    {file = "queuelib-1.6.2.tar.gz", hash = "sha256:4b207267f2642a8699a1f806045c56eb7ad1a85a10c0e249884580d139c2fcd2"},
+]
+requests = [
+    {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"},
+    {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"},
+]
+requests-file = [
+    {file = "requests-file-1.5.1.tar.gz", hash = "sha256:07d74208d3389d01c38ab89ef403af0cfec63957d53a0081d8eca738d0247d8e"},
+    {file = "requests_file-1.5.1-py2.py3-none-any.whl", hash = "sha256:dfe5dae75c12481f68ba353183c53a65e6044c923e64c24b2209f6c7570ca953"},
+]
+requests-oauthlib = [
+    {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"},
+    {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"},
+]
+scrapy = [
+    {file = "Scrapy-2.6.1-py2.py3-none-any.whl", hash = "sha256:e977f57d4e828f25d2702e8c2212e8abcd7d6ce45be560f1830a39432a494c09"},
+    {file = "Scrapy-2.6.1.tar.gz", hash = "sha256:56fd55a59d0f329ce752892358abee5a6b50b4fc55a40420ea317dc617553827"},
+]
+service-identity = [
+    {file = "service-identity-21.1.0.tar.gz", hash = "sha256:6e6c6086ca271dc11b033d17c3a8bea9f24ebff920c587da090afc9519419d34"},
+    {file = "service_identity-21.1.0-py2.py3-none-any.whl", hash = "sha256:f0b0caac3d40627c3c04d7a51b6e06721857a0e10a8775f2d1d7e72901b3a7db"},
 ]
 six = [
     {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
@@ -601,14 +1463,172 @@ text-unidecode = [
     {file = "text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93"},
     {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"},
 ]
+tldextract = [
+    {file = "tldextract-3.2.0-py3-none-any.whl", hash = "sha256:427703b65db54644f7b81d3dcb79bf355c1a7c28a12944e5cc6787531ccc828a"},
+    {file = "tldextract-3.2.0.tar.gz", hash = "sha256:3d4b6a2105600b7d0290ea237bf30b6b0dc763e50fcbe40e849a019bd6dbcbff"},
+]
 toml = [
     {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
     {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
 ]
+tomli = [
+    {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
+    {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
+]
+twisted = [
+    {file = "Twisted-22.4.0-py3-none-any.whl", hash = "sha256:f9f7a91f94932477a9fc3b169d57f54f96c6e74a23d78d9ce54039a7f48928a2"},
+    {file = "Twisted-22.4.0.tar.gz", hash = "sha256:a047990f57dfae1e0bd2b7df2526d4f16dcdc843774dc108b78c52f2a5f13680"},
+]
+twisted-iocpsupport = [
+    {file = "twisted-iocpsupport-1.0.2.tar.gz", hash = "sha256:72068b206ee809c9c596b57b5287259ea41ddb4774d86725b19f35bf56aa32a9"},
+    {file = "twisted_iocpsupport-1.0.2-cp310-cp310-win32.whl", hash = "sha256:985c06a33f5c0dae92c71a036d1ea63872ee86a21dd9b01e1f287486f15524b4"},
+    {file = "twisted_iocpsupport-1.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:81b3abe3527b367da0220482820cb12a16c661672b7bcfcde328902890d63323"},
+    {file = "twisted_iocpsupport-1.0.2-cp36-cp36m-win32.whl", hash = "sha256:9dbb8823b49f06d4de52721b47de4d3b3026064ef4788ce62b1a21c57c3fff6f"},
+    {file = "twisted_iocpsupport-1.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:b9fed67cf0f951573f06d560ac2f10f2a4bbdc6697770113a2fc396ea2cb2565"},
+    {file = "twisted_iocpsupport-1.0.2-cp37-cp37m-win32.whl", hash = "sha256:b76b4eed9b27fd63ddb0877efdd2d15835fdcb6baa745cb85b66e5d016ac2878"},
+    {file = "twisted_iocpsupport-1.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:851b3735ca7e8102e661872390e3bce88f8901bece95c25a0c8bb9ecb8a23d32"},
+    {file = "twisted_iocpsupport-1.0.2-cp38-cp38-win32.whl", hash = "sha256:bf4133139d77fc706d8f572e6b7d82871d82ec7ef25d685c2351bdacfb701415"},
+    {file = "twisted_iocpsupport-1.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:306becd6e22ab6e8e4f36b6bdafd9c92e867c98a5ce517b27fdd27760ee7ae41"},
+    {file = "twisted_iocpsupport-1.0.2-cp39-cp39-win32.whl", hash = "sha256:3c61742cb0bc6c1ac117a7e5f422c129832f0c295af49e01d8a6066df8cfc04d"},
+    {file = "twisted_iocpsupport-1.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:b435857b9efcbfc12f8c326ef0383f26416272260455bbca2cd8d8eca470c546"},
+    {file = "twisted_iocpsupport-1.0.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:7d972cfa8439bdcb35a7be78b7ef86d73b34b808c74be56dfa785c8a93b851bf"},
+]
+twython = [
+    {file = "twython-3.9.1-py3-none-any.whl", hash = "sha256:aef7ad4faabee91efcbe82a8618b38a948498fc6b3eca4cd76f642f957353818"},
+    {file = "twython-3.9.1.tar.gz", hash = "sha256:5a3f0ac24d10705257028fb4205bfedf432ff28d358b796e0c2f01a2f9990c84"},
+]
+typing-extensions = [
+    {file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"},
+    {file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"},
+]
+urllib3 = [
+    {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"},
+    {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"},
+]
+w3lib = [
+    {file = "w3lib-1.22.0-py2.py3-none-any.whl", hash = "sha256:0161d55537063e00d95a241663ede3395c4c6d7b777972ba2fd58bbab2001e53"},
+    {file = "w3lib-1.22.0.tar.gz", hash = "sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df"},
+]
 wcwidth = [
     {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"},
     {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"},
 ]
 wrapt = [
-    {file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"},
+    {file = "wrapt-1.14.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:5a9a1889cc01ed2ed5f34574c90745fab1dd06ec2eee663e8ebeefe363e8efd7"},
+    {file = "wrapt-1.14.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:9a3ff5fb015f6feb78340143584d9f8a0b91b6293d6b5cf4295b3e95d179b88c"},
+    {file = "wrapt-1.14.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:4b847029e2d5e11fd536c9ac3136ddc3f54bc9488a75ef7d040a3900406a91eb"},
+    {file = "wrapt-1.14.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:9a5a544861b21e0e7575b6023adebe7a8c6321127bb1d238eb40d99803a0e8bd"},
+    {file = "wrapt-1.14.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:88236b90dda77f0394f878324cfbae05ae6fde8a84d548cfe73a75278d760291"},
+    {file = "wrapt-1.14.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f0408e2dbad9e82b4c960274214af533f856a199c9274bd4aff55d4634dedc33"},
+    {file = "wrapt-1.14.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:9d8c68c4145041b4eeae96239802cfdfd9ef927754a5be3f50505f09f309d8c6"},
+    {file = "wrapt-1.14.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:22626dca56fd7f55a0733e604f1027277eb0f4f3d95ff28f15d27ac25a45f71b"},
+    {file = "wrapt-1.14.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:65bf3eb34721bf18b5a021a1ad7aa05947a1767d1aa272b725728014475ea7d5"},
+    {file = "wrapt-1.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:09d16ae7a13cff43660155383a2372b4aa09109c7127aa3f24c3cf99b891c330"},
+    {file = "wrapt-1.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:debaf04f813ada978d7d16c7dfa16f3c9c2ec9adf4656efdc4defdf841fc2f0c"},
+    {file = "wrapt-1.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:748df39ed634851350efa87690c2237a678ed794fe9ede3f0d79f071ee042561"},
+    {file = "wrapt-1.14.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1807054aa7b61ad8d8103b3b30c9764de2e9d0c0978e9d3fc337e4e74bf25faa"},
+    {file = "wrapt-1.14.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:763a73ab377390e2af26042f685a26787c402390f682443727b847e9496e4a2a"},
+    {file = "wrapt-1.14.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:8529b07b49b2d89d6917cfa157d3ea1dfb4d319d51e23030664a827fe5fd2131"},
+    {file = "wrapt-1.14.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:68aeefac31c1f73949662ba8affaf9950b9938b712fb9d428fa2a07e40ee57f8"},
+    {file = "wrapt-1.14.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59d7d92cee84a547d91267f0fea381c363121d70fe90b12cd88241bd9b0e1763"},
+    {file = "wrapt-1.14.0-cp310-cp310-win32.whl", hash = "sha256:3a88254881e8a8c4784ecc9cb2249ff757fd94b911d5df9a5984961b96113fff"},
+    {file = "wrapt-1.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:9a242871b3d8eecc56d350e5e03ea1854de47b17f040446da0e47dc3e0b9ad4d"},
+    {file = "wrapt-1.14.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:a65bffd24409454b889af33b6c49d0d9bcd1a219b972fba975ac935f17bdf627"},
+    {file = "wrapt-1.14.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9d9fcd06c952efa4b6b95f3d788a819b7f33d11bea377be6b8980c95e7d10775"},
+    {file = "wrapt-1.14.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:db6a0ddc1282ceb9032e41853e659c9b638789be38e5b8ad7498caac00231c23"},
+    {file = "wrapt-1.14.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:14e7e2c5f5fca67e9a6d5f753d21f138398cad2b1159913ec9e9a67745f09ba3"},
+    {file = "wrapt-1.14.0-cp35-cp35m-win32.whl", hash = "sha256:6d9810d4f697d58fd66039ab959e6d37e63ab377008ef1d63904df25956c7db0"},
+    {file = "wrapt-1.14.0-cp35-cp35m-win_amd64.whl", hash = "sha256:d808a5a5411982a09fef6b49aac62986274ab050e9d3e9817ad65b2791ed1425"},
+    {file = "wrapt-1.14.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b77159d9862374da213f741af0c361720200ab7ad21b9f12556e0eb95912cd48"},
+    {file = "wrapt-1.14.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:36a76a7527df8583112b24adc01748cd51a2d14e905b337a6fefa8b96fc708fb"},
+    {file = "wrapt-1.14.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0057b5435a65b933cbf5d859cd4956624df37b8bf0917c71756e4b3d9958b9e"},
+    {file = "wrapt-1.14.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a0a4ca02752ced5f37498827e49c414d694ad7cf451ee850e3ff160f2bee9d3"},
+    {file = "wrapt-1.14.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:8c6be72eac3c14baa473620e04f74186c5d8f45d80f8f2b4eda6e1d18af808e8"},
+    {file = "wrapt-1.14.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:21b1106bff6ece8cb203ef45b4f5778d7226c941c83aaaa1e1f0f4f32cc148cd"},
+    {file = "wrapt-1.14.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:493da1f8b1bb8a623c16552fb4a1e164c0200447eb83d3f68b44315ead3f9036"},
+    {file = "wrapt-1.14.0-cp36-cp36m-win32.whl", hash = "sha256:89ba3d548ee1e6291a20f3c7380c92f71e358ce8b9e48161401e087e0bc740f8"},
+    {file = "wrapt-1.14.0-cp36-cp36m-win_amd64.whl", hash = "sha256:729d5e96566f44fccac6c4447ec2332636b4fe273f03da128fff8d5559782b06"},
+    {file = "wrapt-1.14.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:891c353e95bb11abb548ca95c8b98050f3620a7378332eb90d6acdef35b401d4"},
+    {file = "wrapt-1.14.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23f96134a3aa24cc50614920cc087e22f87439053d886e474638c68c8d15dc80"},
+    {file = "wrapt-1.14.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6807bcee549a8cb2f38f73f469703a1d8d5d990815c3004f21ddb68a567385ce"},
+    {file = "wrapt-1.14.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6915682f9a9bc4cf2908e83caf5895a685da1fbd20b6d485dafb8e218a338279"},
+    {file = "wrapt-1.14.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f2f3bc7cd9c9fcd39143f11342eb5963317bd54ecc98e3650ca22704b69d9653"},
+    {file = "wrapt-1.14.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:3a71dbd792cc7a3d772ef8cd08d3048593f13d6f40a11f3427c000cf0a5b36a0"},
+    {file = "wrapt-1.14.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5a0898a640559dec00f3614ffb11d97a2666ee9a2a6bad1259c9facd01a1d4d9"},
+    {file = "wrapt-1.14.0-cp37-cp37m-win32.whl", hash = "sha256:167e4793dc987f77fd476862d32fa404d42b71f6a85d3b38cbce711dba5e6b68"},
+    {file = "wrapt-1.14.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d066ffc5ed0be00cd0352c95800a519cf9e4b5dd34a028d301bdc7177c72daf3"},
+    {file = "wrapt-1.14.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d9bdfa74d369256e4218000a629978590fd7cb6cf6893251dad13d051090436d"},
+    {file = "wrapt-1.14.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2498762814dd7dd2a1d0248eda2afbc3dd9c11537bc8200a4b21789b6df6cd38"},
+    {file = "wrapt-1.14.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f24ca7953f2643d59a9c87d6e272d8adddd4a53bb62b9208f36db408d7aafc7"},
+    {file = "wrapt-1.14.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b835b86bd5a1bdbe257d610eecab07bf685b1af2a7563093e0e69180c1d4af1"},
+    {file = "wrapt-1.14.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b21650fa6907e523869e0396c5bd591cc326e5c1dd594dcdccac089561cacfb8"},
+    {file = "wrapt-1.14.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:354d9fc6b1e44750e2a67b4b108841f5f5ea08853453ecbf44c81fdc2e0d50bd"},
+    {file = "wrapt-1.14.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1f83e9c21cd5275991076b2ba1cd35418af3504667affb4745b48937e214bafe"},
+    {file = "wrapt-1.14.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:61e1a064906ccba038aa3c4a5a82f6199749efbbb3cef0804ae5c37f550eded0"},
+    {file = "wrapt-1.14.0-cp38-cp38-win32.whl", hash = "sha256:28c659878f684365d53cf59dc9a1929ea2eecd7ac65da762be8b1ba193f7e84f"},
+    {file = "wrapt-1.14.0-cp38-cp38-win_amd64.whl", hash = "sha256:b0ed6ad6c9640671689c2dbe6244680fe8b897c08fd1fab2228429b66c518e5e"},
+    {file = "wrapt-1.14.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b3f7e671fb19734c872566e57ce7fc235fa953d7c181bb4ef138e17d607dc8a1"},
+    {file = "wrapt-1.14.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:87fa943e8bbe40c8c1ba4086971a6fefbf75e9991217c55ed1bcb2f1985bd3d4"},
+    {file = "wrapt-1.14.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4775a574e9d84e0212f5b18886cace049a42e13e12009bb0491562a48bb2b758"},
+    {file = "wrapt-1.14.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9d57677238a0c5411c76097b8b93bdebb02eb845814c90f0b01727527a179e4d"},
+    {file = "wrapt-1.14.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00108411e0f34c52ce16f81f1d308a571df7784932cc7491d1e94be2ee93374b"},
+    {file = "wrapt-1.14.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d332eecf307fca852d02b63f35a7872de32d5ba8b4ec32da82f45df986b39ff6"},
+    {file = "wrapt-1.14.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:01f799def9b96a8ec1ef6b9c1bbaf2bbc859b87545efbecc4a78faea13d0e3a0"},
+    {file = "wrapt-1.14.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47045ed35481e857918ae78b54891fac0c1d197f22c95778e66302668309336c"},
+    {file = "wrapt-1.14.0-cp39-cp39-win32.whl", hash = "sha256:2eca15d6b947cfff51ed76b2d60fd172c6ecd418ddab1c5126032d27f74bc350"},
+    {file = "wrapt-1.14.0-cp39-cp39-win_amd64.whl", hash = "sha256:bb36fbb48b22985d13a6b496ea5fb9bb2a076fea943831643836c9f6febbcfdc"},
+    {file = "wrapt-1.14.0.tar.gz", hash = "sha256:8323a43bd9c91f62bb7d4be74cc9ff10090e7ef820e27bfe8815c57e68261311"},
+]
+"zope.interface" = [
+    {file = "zope.interface-5.4.0-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:7df1e1c05304f26faa49fa752a8c690126cf98b40b91d54e6e9cc3b7d6ffe8b7"},
+    {file = "zope.interface-5.4.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:2c98384b254b37ce50eddd55db8d381a5c53b4c10ee66e1e7fe749824f894021"},
+    {file = "zope.interface-5.4.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:08f9636e99a9d5410181ba0729e0408d3d8748026ea938f3b970a0249daa8192"},
+    {file = "zope.interface-5.4.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:0ea1d73b7c9dcbc5080bb8aaffb776f1c68e807767069b9ccdd06f27a161914a"},
+    {file = "zope.interface-5.4.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:273f158fabc5ea33cbc936da0ab3d4ba80ede5351babc4f577d768e057651531"},
+    {file = "zope.interface-5.4.0-cp27-cp27m-win32.whl", hash = "sha256:a1e6e96217a0f72e2b8629e271e1b280c6fa3fe6e59fa8f6701bec14e3354325"},
+    {file = "zope.interface-5.4.0-cp27-cp27m-win_amd64.whl", hash = "sha256:877473e675fdcc113c138813a5dd440da0769a2d81f4d86614e5d62b69497155"},
+    {file = "zope.interface-5.4.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f7ee479e96f7ee350db1cf24afa5685a5899e2b34992fb99e1f7c1b0b758d263"},
+    {file = "zope.interface-5.4.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:b0297b1e05fd128d26cc2460c810d42e205d16d76799526dfa8c8ccd50e74959"},
+    {file = "zope.interface-5.4.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:af310ec8335016b5e52cae60cda4a4f2a60a788cbb949a4fbea13d441aa5a09e"},
+    {file = "zope.interface-5.4.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:9a9845c4c6bb56e508651f005c4aeb0404e518c6f000d5a1123ab077ab769f5c"},
+    {file = "zope.interface-5.4.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:0b465ae0962d49c68aa9733ba92a001b2a0933c317780435f00be7ecb959c702"},
+    {file = "zope.interface-5.4.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:5dd9ca406499444f4c8299f803d4a14edf7890ecc595c8b1c7115c2342cadc5f"},
+    {file = "zope.interface-5.4.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:469e2407e0fe9880ac690a3666f03eb4c3c444411a5a5fddfdabc5d184a79f05"},
+    {file = "zope.interface-5.4.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:52de7fc6c21b419078008f697fd4103dbc763288b1406b4562554bd47514c004"},
+    {file = "zope.interface-5.4.0-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:3dd4952748521205697bc2802e4afac5ed4b02909bb799ba1fe239f77fd4e117"},
+    {file = "zope.interface-5.4.0-cp35-cp35m-win32.whl", hash = "sha256:dd93ea5c0c7f3e25335ab7d22a507b1dc43976e1345508f845efc573d3d779d8"},
+    {file = "zope.interface-5.4.0-cp35-cp35m-win_amd64.whl", hash = "sha256:3748fac0d0f6a304e674955ab1365d515993b3a0a865e16a11ec9d86fb307f63"},
+    {file = "zope.interface-5.4.0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:66c0061c91b3b9cf542131148ef7ecbecb2690d48d1612ec386de9d36766058f"},
+    {file = "zope.interface-5.4.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:d0c1bc2fa9a7285719e5678584f6b92572a5b639d0e471bb8d4b650a1a910920"},
+    {file = "zope.interface-5.4.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2876246527c91e101184f63ccd1d716ec9c46519cc5f3d5375a3351c46467c46"},
+    {file = "zope.interface-5.4.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:334701327f37c47fa628fc8b8d28c7d7730ce7daaf4bda1efb741679c2b087fc"},
+    {file = "zope.interface-5.4.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:71aace0c42d53abe6fc7f726c5d3b60d90f3c5c055a447950ad6ea9cec2e37d9"},
+    {file = "zope.interface-5.4.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:5bb3489b4558e49ad2c5118137cfeaf59434f9737fa9c5deefc72d22c23822e2"},
+    {file = "zope.interface-5.4.0-cp36-cp36m-win32.whl", hash = "sha256:1c0e316c9add0db48a5b703833881351444398b04111188069a26a61cfb4df78"},
+    {file = "zope.interface-5.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f0c02cbb9691b7c91d5009108f975f8ffeab5dff8f26d62e21c493060eff2a1"},
+    {file = "zope.interface-5.4.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:7d97a4306898b05404a0dcdc32d9709b7d8832c0c542b861d9a826301719794e"},
+    {file = "zope.interface-5.4.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:867a5ad16892bf20e6c4ea2aab1971f45645ff3102ad29bd84c86027fa99997b"},
+    {file = "zope.interface-5.4.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5f931a1c21dfa7a9c573ec1f50a31135ccce84e32507c54e1ea404894c5eb96f"},
+    {file = "zope.interface-5.4.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:194d0bcb1374ac3e1e023961610dc8f2c78a0f5f634d0c737691e215569e640d"},
+    {file = "zope.interface-5.4.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:8270252effc60b9642b423189a2fe90eb6b59e87cbee54549db3f5562ff8d1b8"},
+    {file = "zope.interface-5.4.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:15e7d1f7a6ee16572e21e3576d2012b2778cbacf75eb4b7400be37455f5ca8bf"},
+    {file = "zope.interface-5.4.0-cp37-cp37m-win32.whl", hash = "sha256:8892f89999ffd992208754851e5a052f6b5db70a1e3f7d54b17c5211e37a98c7"},
+    {file = "zope.interface-5.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2e5a26f16503be6c826abca904e45f1a44ff275fdb7e9d1b75c10671c26f8b94"},
+    {file = "zope.interface-5.4.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:0f91b5b948686659a8e28b728ff5e74b1be6bf40cb04704453617e5f1e945ef3"},
+    {file = "zope.interface-5.4.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:4de4bc9b6d35c5af65b454d3e9bc98c50eb3960d5a3762c9438df57427134b8e"},
+    {file = "zope.interface-5.4.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:bf68f4b2b6683e52bec69273562df15af352e5ed25d1b6641e7efddc5951d1a7"},
+    {file = "zope.interface-5.4.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:63b82bb63de7c821428d513607e84c6d97d58afd1fe2eb645030bdc185440120"},
+    {file = "zope.interface-5.4.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:db1fa631737dab9fa0b37f3979d8d2631e348c3b4e8325d6873c2541d0ae5a48"},
+    {file = "zope.interface-5.4.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:f44e517131a98f7a76696a7b21b164bcb85291cee106a23beccce454e1f433a4"},
+    {file = "zope.interface-5.4.0-cp38-cp38-win32.whl", hash = "sha256:a9506a7e80bcf6eacfff7f804c0ad5350c8c95b9010e4356a4b36f5322f09abb"},
+    {file = "zope.interface-5.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:3c02411a3b62668200910090a0dff17c0b25aaa36145082a5a6adf08fa281e54"},
+    {file = "zope.interface-5.4.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:0cee5187b60ed26d56eb2960136288ce91bcf61e2a9405660d271d1f122a69a4"},
+    {file = "zope.interface-5.4.0-cp39-cp39-manylinux1_i686.whl", hash = "sha256:a8156e6a7f5e2a0ff0c5b21d6bcb45145efece1909efcbbbf48c56f8da68221d"},
+    {file = "zope.interface-5.4.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:205e40ccde0f37496904572035deea747390a8b7dc65146d30b96e2dd1359a83"},
+    {file = "zope.interface-5.4.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:3f24df7124c323fceb53ff6168da70dbfbae1442b4f3da439cd441681f54fe25"},
+    {file = "zope.interface-5.4.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:5208ebd5152e040640518a77827bdfcc73773a15a33d6644015b763b9c9febc1"},
+    {file = "zope.interface-5.4.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:17776ecd3a1fdd2b2cd5373e5ef8b307162f581c693575ec62e7c5399d80794c"},
+    {file = "zope.interface-5.4.0-cp39-cp39-win32.whl", hash = "sha256:d4d9d6c1a455d4babd320203b918ccc7fcbefe308615c521062bc2ba1aa4d26e"},
+    {file = "zope.interface-5.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:0cba8477e300d64a11a9789ed40ee8932b59f9ee05f85276dbb4b59acee5dd09"},
+    {file = "zope.interface-5.4.0.tar.gz", hash = "sha256:5dba5f530fec3f0988d83b78cc591b58c0b6eb8431a85edd1569a0539a8a5a0e"},
 ]
diff --git a/data_extractor/pyprocess.js b/data_extractor/pyprocess.js
new file mode 100644
index 0000000..5c04e24
--- /dev/null
+++ b/data_extractor/pyprocess.js
@@ -0,0 +1,66 @@
+var script = document.currentScript.getAttribute('data-script');
+
+const loadingIndicator = () => document.getElementById("loading-indicator");
+const processButton = () => document.getElementById("process");
+const fileInput = () => document.getElementById("fileItem");
+const resultElement = () => document.getElementById("results");
+const summaryElement = () => document.getElementById("summary");
+const htmlElement = () => document.getElementById("html");
+
+const pyWorker = new Worker("../pyworker.js");
+
+pyWorker.onerror = console.log;
+pyWorker.onmessage = (event) => {
+  const { eventType } = event.data;
+  if (eventType === "result") {
+    setControlsDisabled(false);
+    summaryElement().textContent = event.data.result.summary;
+    htmlElement().innerHTML = event.data.result.html;
+    resultElement().style.display = "block";
+    console.log(event.data.result);
+  } else if (eventType === "initialized") {
+    fetch(script)
+    .then(response => response.text())
+    .then((script) => {
+      pyWorker.postMessage({ eventType: "loadScript", script});
+    })
+    .then(() => {
+      loadingIndicator().hidden = true;
+      fileInput().disabled = false;
+    })
+  }
+};  
+
+function toggleProcessButton() {
+  const file = fileInput().files[0];
+  processButton().disabled = file === undefined;
+};
+
+function setControlsDisabled(disabled) {
+  fileInput().disabled = disabled;
+  processButton().disabled = disabled;
+};
+
+function process() {
+  console.log("process: started");
+
+  setControlsDisabled(true);
+  resultElement().style.display = "none";
+
+  const file = fileInput().files[0];
+  const reader = file.stream().getReader();
+  const sendToWorker = ({ done, value }) => {
+    if (done) {
+      console.log("process: runPy");
+      pyWorker.postMessage({ eventType: "processData" });
+      return;
+    }
+    console.log("process: send event: data");
+    pyWorker.postMessage({ eventType: "data", chunk: value });
+    reader.read().then(sendToWorker);
+  };
+  console.log("process: send event: initData");
+  pyWorker.postMessage({ eventType: "initData", size: file.size });
+  reader.read().then(sendToWorker);
+}
+
diff --git a/data_extractor/pyproject.toml b/data_extractor/pyproject.toml
index 77dc1db..0d27622 100644
--- a/data_extractor/pyproject.toml
+++ b/data_extractor/pyproject.toml
@@ -13,6 +13,7 @@ numpy = "^1.21.1"
 geopy = "^2.2.0"
 genson = "^1.2.2"
 faker-schema = "^0.1.4"
+advertools = "^0.13.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "^5.2"
diff --git a/data_extractor/pyworker.js b/data_extractor/pyworker.js
new file mode 100644
index 0000000..15815de
--- /dev/null
+++ b/data_extractor/pyworker.js
@@ -0,0 +1,42 @@
+importScripts("https://cdn.jsdelivr.net/pyodide/v0.19.0/full/pyodide.js");
+
+var data = undefined;
+
+loadPyodide({ indexURL: "https://cdn.jsdelivr.net/pyodide/v0.19.0/full/" }).then((pyodide) => {
+  self.pyodide = pyodide;
+  return self.pyodide.loadPackage(["micropip", "numpy", "pandas"]);
+}).then(() => {
+  self.postMessage({ eventType: "initialized" });
+});
+
+let file = undefined
+
+onmessage = (event) => {
+  const { eventType } = event.data;
+  if (eventType === "loadScript") {
+    self.pyodide.runPython(event.data.script)
+  } else if (eventType === "initData") {
+    file = self.pyodide.FS.open("user-data", "w")
+  } else if (eventType === "data") {
+    self.pyodide.FS.write(file, event.data.chunk, 0, event.data.chunk.length)
+  } else if (eventType === "processData") {
+    const result = self.pyodide.runPython(`
+    def _process_data():
+      import json
+      import html
+      result = process(open("user-data", "rb"))
+      data_output = []
+      html_output = []
+      for data in result:
+        html_output.append(f"""<h1 class="text-title4 font-title4 sm:text-title3 sm:font-title3 mt-12 mb-6 text-grey1">{html.escape(data['title'])}</h1>""")
+        df = data['data_frame']
+        html_output.append(df.to_html(classes=["data-donation-extraction-results"], justify="left"))
+        data_output.append({"id": data["id"], "data_frame": df.to_json()})
+      return {
+        "html": "\\n".join(html_output),
+        "data": json.dumps(data_output),
+      }
+    _process_data()`);
+    self.postMessage({ eventType: "result", result: result.toJs({ create_proxies: false, dict_converter: Object.fromEntries }) });
+  }
+};
diff --git a/data_extractor/tests/.DS_Store b/data_extractor/tests/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..1a71cb0776ca1e4c312bf3287e5b925ef4942c2c
GIT binary patch
literal 6148
zcmeH~F$w}f3`G;&La^D=avBfd4F=H@>;+s9Y(zoSdXDZ-CJ2t!BJu;tpJXO1`-+{7
zi0JyZUy1Z0GJ~7S(n4d3ypz3*a+UEuTu#UH>42KmCvn!+@Lrnz*rt#G36KB@kN^q%
z5COZlVY7KvMiL+a5_l4@??Zx{=Fn2rKOG1@0zf;I-LUpq0-CG<&7q|#Dlm=dL8DcD
z46(YmLsOi~p`~hV7meXV<H>4M3`}dgXhH(h?7~0-B+w9;*1Wg-e+&OK|2Hj6Nq_|Y
zjDU8VVY9|d#ohY$dRE^>)z$?L_2URHKLJSWDqg_du%B!J&7q|#Dlq;CI0gn1_$q-1
DiFXpM

literal 0
HcmV?d00001

diff --git a/data_extractor/tests/data/.DS_Store b/data_extractor/tests/data/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..9bc657364e0feac506d39afa5eda46148096a2a6
GIT binary patch
literal 6148
zcmeHKF-`+95Zr|zMM^<RN`HYLSW!|?^8s`T62(a_5dC#L8)m$Vl+!6vDhilAt;b&P
zoX=C-+GggvkJF>s*38y$qJ6Si8t3T?yU7UA`#ajx*p9>4hDGkB0J(SeX0JBlIp_0}
z?&)7?^UTxxaU5D3v9L@ENC7Dz1*Cu!xL$!Qo$Jl@6lF>QDezAe;Qyh(i8XNw?7t2)
zJic>&MWmtM?-Ib0<C-`HVg}|x1qRjgis3;=ykuTYoC1Sx-dDzZa%ImOiuctKFFoD7
z2C9_;QedvYZ49@p{|ESm`~RGzkra>ue@X#g?VtBMY$<E&Vsosu1^xp6G354iG_ykh
f1}w!7XL*Hv<kiF}u+tImbf69b)I}x*enWw8H2)zt

literal 0
HcmV?d00001

diff --git a/data_extractor/tests/test_data_extractor.py b/data_extractor/tests/test_data_extractor.py
deleted file mode 100644
index 618e833..0000000
--- a/data_extractor/tests/test_data_extractor.py
+++ /dev/null
@@ -1,9 +0,0 @@
-from data_extractor import process
-from pathlib import Path
-
-DATA_PATH = Path(__file__).parent / "data"
-
-def test_hello_world():
-    result = process(DATA_PATH.joinpath("hello.zip").open("rb"))
-    assert result["summary"] == 'The following files where read: hello/, hello/world.txt.'
-    assert len(result["data"]) == 2
diff --git a/data_extractor/tests/test_example.py b/data_extractor/tests/test_example.py
new file mode 100644
index 0000000..aa05945
--- /dev/null
+++ b/data_extractor/tests/test_example.py
@@ -0,0 +1,22 @@
+from example import process
+from pathlib import Path
+
+import pandas as pd
+from pandas.testing import assert_frame_equal
+
+DATA_PATH = Path(__file__).parent / "data"
+
+EXPECTED = [
+    {'filename': 'hello/', 'compressed size': 0, 'size': 0},
+    {'filename': 'hello/world.txt', 'compressed size': 267, 'size': 471}
+]
+
+
+def test_hello_world():
+    df_expected = pd.DataFrame(EXPECTED)
+
+    result = process(DATA_PATH.joinpath("hello.zip").open("rb"))
+    assert len(result) == 1
+    assert result[0]["id"] == 'overview'
+    assert result[0]["title"] == 'The following files where read:'
+    assert_frame_equal(result[0]["data_frame"], df_expected)
diff --git a/data_extractor/tests/test_whatsapp.py b/data_extractor/tests/test_whatsapp.py
new file mode 100644
index 0000000..aa05945
--- /dev/null
+++ b/data_extractor/tests/test_whatsapp.py
@@ -0,0 +1,22 @@
+from example import process
+from pathlib import Path
+
+import pandas as pd
+from pandas.testing import assert_frame_equal
+
+DATA_PATH = Path(__file__).parent / "data"
+
+EXPECTED = [
+    {'filename': 'hello/', 'compressed size': 0, 'size': 0},
+    {'filename': 'hello/world.txt', 'compressed size': 267, 'size': 471}
+]
+
+
+def test_hello_world():
+    df_expected = pd.DataFrame(EXPECTED)
+
+    result = process(DATA_PATH.joinpath("hello.zip").open("rb"))
+    assert len(result) == 1
+    assert result[0]["id"] == 'overview'
+    assert result[0]["title"] == 'The following files where read:'
+    assert_frame_equal(result[0]["data_frame"], df_expected)
diff --git a/data_extractor/whatsapp/__init__.py b/data_extractor/whatsapp/__init__.py
new file mode 100644
index 0000000..63f5f31
--- /dev/null
+++ b/data_extractor/whatsapp/__init__.py
@@ -0,0 +1,20 @@
+__version__ = '0.2.0'
+
+import zipfile
+import pandas as pd
+
+
+def process(file_data):
+    names = []
+    zfile = zipfile.ZipFile(file_data)
+    data = []
+    for name in zfile.namelist():
+        names.append(name)
+        info = zfile.getinfo(name)
+        data.append((name, info.compress_size, info.file_size))
+
+    return [{
+        "id": "overview",
+        "title": "The following files where read:",
+        "data_frame": pd.DataFrame(data, columns=["filename", "compressed size", "size"])
+    }]
diff --git a/data_extractor/whatsapp/index.html b/data_extractor/whatsapp/index.html
new file mode 100644
index 0000000..4f5ec08
--- /dev/null
+++ b/data_extractor/whatsapp/index.html
@@ -0,0 +1,11 @@
+<script src="../pyprocess.js" data-script="__init__.py"></script>
+<div id="controls">
+  <p id="loading-indicator">Loading...</p>
+  <input type="file" id="fileItem" disabled onChange="toggleProcessButton()" />
+  <button onClick="process()" id="process" disabled>Process</button>
+</div>
+
+<div id="results" style="display: none">
+  <p id="summary" />
+  <div id="html" />
+</div>
diff --git a/index.html b/index.html
deleted file mode 100644
index c51bfa4..0000000
--- a/index.html
+++ /dev/null
@@ -1,63 +0,0 @@
-<div id="controls">
-  <p id="loading-indicator">Loading...</p>
-  <input type="file" id="fileItem" disabled onChange="toggleProcessButton()" />
-  <button onClick="process()" id="process" disabled>Process</button>
-</div>
-
-<div id="results" style="display: none">
-  <p id="summary" />
-  <div id="html" />
-</div>
-
-<script type="text/javascript">
-  const loadingIndicator = document.getElementById("loading-indicator");
-  const processButton = document.getElementById("process");
-  const fileInput = document.getElementById("fileItem");
-  const resultElement = document.getElementById("results");
-  const summaryElement = document.getElementById("summary");
-  const htmlElement = document.getElementById("html");
-
-  const pyWorker = new Worker("./pyworker.js");
-  pyWorker.onerror = console.log;
-  pyWorker.onmessage = (event) => {
-    const { eventType } = event.data;
-    if (eventType === "result") {
-      setControlsDisabled(false);
-      summaryElement.textContent = event.data.result.summary;
-      htmlElement.innerHTML = event.data.result.html;
-      resultElement.style.display = "block";
-      console.log(event.data.result);
-    } else if (eventType === "initialized") {
-      loadingIndicator.hidden = true;
-      fileInput.disabled = false;
-    }
-  };
-
-  const toggleProcessButton = () => {
-    const file = fileInput.files[0];
-    processButton.disabled = file === undefined;
-  };
-
-  const setControlsDisabled = (disabled) => {
-    fileInput.disabled = disabled;
-    processButton.disabled = disabled;
-  };
-
-  const process = () => {
-    setControlsDisabled(true);
-    resultElement.style.display = "none";
-
-    const file = fileInput.files[0];
-    const reader = file.stream().getReader();
-    const sendToWorker = ({ done, value }) => {
-      if (done) {
-        pyWorker.postMessage({ eventType: "runPy" });
-        return;
-      }
-      pyWorker.postMessage({ eventType: "data", chunk: value });
-      reader.read().then(sendToWorker);
-    };
-    pyWorker.postMessage({ eventType: "initData", size: file.size });
-    reader.read().then(sendToWorker);
-  };
-</script>
diff --git a/pyworker.js b/pyworker.js
deleted file mode 100644
index aaebdff..0000000
--- a/pyworker.js
+++ /dev/null
@@ -1,105 +0,0 @@
-self.languagePluginUrl = "https://cdn.jsdelivr.net/pyodide/v0.16.1/full/";
-importScripts("https://cdn.jsdelivr.net/pyodide/v0.16.1/full/pyodide.js");
-
-class ChunkedFile {
-  constructor(size) {
-    this.data = [];
-    this.offset = 0;
-    this.size = size;
-    this.buffer = new Uint8Array(new ArrayBuffer(size));
-  }
-  tell() {
-    return this.offset;
-  }
-  read(size) {
-    if (size === undefined) {
-      size = this.size - this.offset;
-    }
-    if (this.offset >= this.size) {
-      return null;
-    }
-    const readOffset = this.offset;
-    this.offset += size;
-    return this.buffer.slice(readOffset, readOffset + size);
-  }
-  seek(offset, whence) {
-    switch (whence) {
-      case 0:
-        this.offset = offset;
-        break;
-      case 1:
-        this.offset += offset;
-        break;
-      case 2:
-        this.offset = this.size + offset;
-        break;
-    }
-  }
-  writeChunk(chunk) {
-    this.buffer.set(chunk, this.offset);
-    this.offset = this.offset + chunk.length;
-  }
-}
-
-var data = undefined;
-
-languagePluginLoader
-  .then(() => {
-    return pyodide.loadPackage(["pandas"]);
-  })
-  .then(() => fetch("/data_extractor/data_extractor/__init__.py"))
-  .then(response => response.text())
-  .then((code) => {
-    self.pyodide.runPython(`
-${code}
-
-class _ChunkedFile:
-  def __init__(self, proxy):
-    self.proxy = proxy
-
-  def seekable(self):
-    return True
-
-  def seek(self, offset, whence=0):
-    self.proxy.seek(offset, whence)
-
-  def tell(self):
-    return self.proxy.tell()
-
-  def read(self, size=None):
-    data = self.proxy.read(size)
-    if data:
-      return data.tobytes()
-
-
-def _process_data(data):
-  file_data = _ChunkedFile(data)
-  result = process(file_data)
-  data = []
-  html = []
-  for df in result.get("data_frames", []):
-    html.append(df.to_html())
-    data.append(df.to_dict())
-  return {
-    "summary": result["summary"],
-    "html": "\\n".join(html),
-    "data": data,
-  }
-  `);
-    self.postMessage({ eventType: "initialized" });
-  });
-
-onmessage = (event) => {
-  //await languagePluginLoader;
-  //await pythonLoading;
-  // Don't bother yet with this line, suppose our API is built in such a way:
-  const { eventType } = event.data;
-  if (eventType === "initData") {
-    data = new ChunkedFile(event.data.size);
-  } else if (eventType === "data") {
-    data.writeChunk(event.data.chunk);
-  } else if (eventType === "runPy") {
-    const result = self.pyodide.globals._process_data(data);
-    self.postMessage({ eventType: "result", result });
-  }
-};

From b986614ba6c63bf437a0da76ea2a739c094c6d6c Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 24 May 2022 13:58:46 +0200
Subject: [PATCH 02/42] rename whatsapp folder

---
 data_extractor/{whatsapp => whatsapp_chat}/__init__.py | 0
 data_extractor/{whatsapp => whatsapp_chat}/index.html  | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename data_extractor/{whatsapp => whatsapp_chat}/__init__.py (100%)
 rename data_extractor/{whatsapp => whatsapp_chat}/index.html (100%)

diff --git a/data_extractor/whatsapp/__init__.py b/data_extractor/whatsapp_chat/__init__.py
similarity index 100%
rename from data_extractor/whatsapp/__init__.py
rename to data_extractor/whatsapp_chat/__init__.py
diff --git a/data_extractor/whatsapp/index.html b/data_extractor/whatsapp_chat/index.html
similarity index 100%
rename from data_extractor/whatsapp/index.html
rename to data_extractor/whatsapp_chat/index.html

From 92d465507cc5f39232e2251a99cf2d8ef5427ace Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 24 May 2022 14:08:59 +0200
Subject: [PATCH 03/42] Initial commit

---
 data_extractor/whatsapp_chat/__init__.py | 643 ++++++++++++++++++++++-
 1 file changed, 629 insertions(+), 14 deletions(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index 63f5f31..4c7f8a9 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -1,20 +1,635 @@
+"""Parser utils.
+The main part is extracted from https://github.com/lucasrodes/whatstk.git
+"""
 __version__ = '0.2.0'
 
-import zipfile
+import os
+import re
+from datetime import datetime
 import pandas as pd
+import hashlib
+import zipfile
 
 
-def process(file_data):
-    names = []
-    zfile = zipfile.ZipFile(file_data)
-    data = []
+URL_PATTERN = r'(https?://\S+)'
+LOCATION_PATTERN = r'(Location: https?://\S+)'
+ATTACH_FILE_PATTERN = r'(<attached: \S+>)'
+FILE_RE = re.compile(r".*chat.*.txt$")
+HIDDEN_FILE_RE = re.compile(r".*__MACOSX*")
+
+hformats = ['%m/%d/%y, %H:%M - %name:', '[%d/%m/%y, %H:%M:%S] %name:', '%d-%m-%y %H:%M - %name:']
+
+
+class ColnamesDf:
+    """Access class constants using variable ``utils.COLNAMES_DF``."""
+
+    DATE = 'date'
+    """Date column"""
+
+    USERNAME = 'username'
+    """Username column"""
+
+    MESSAGE = 'message'
+    """Message column"""
+
+    MESSAGE_LENGTH = 'message_length'
+    """Message length column"""
+
+    FirstMessage = 'first_message_date'
+    """Date of first message column"""
+
+    LastMessage = 'last_message_date'
+    """Date of last message column"""
+
+    MESSAGE_NO = 'message_no'
+    """Number of Message  column"""
+
+    WORDS_NO = 'total_words_no'
+    """Total number of words  column"""
+
+    REPLY_2USER = 'reply_2_user'
+    """Who replies to the user the most column"""
+
+    MAX_REPLY_2 = 'max_reply_2'
+    """User replies to who the most column"""
+
+    USER_REPLY2 = 'user_reply2'
+    """User replies to who the most column"""
+
+    URL_NO = 'url_no'
+    """Number of URLs column"""
+
+    LOCATION_NO = 'location_no'
+    """Number of locations column"""
+
+    FILE_NO = 'file_no'
+    """Number of files column"""
+
+    OUT_DEGREE = 'out_degree'
+    """Total number of sent message column"""
+
+    IN_DEGREE = 'in_degree'
+    """Total number of received message column"""
+
+    EMOJI_NO = 'emoji_no'
+    """Total number of emojies column"""
+
+    EMOJI_Fav = 'emoji_fav'
+    """Favorite emojies column"""
+
+
+COLNAMES_DF = ColnamesDf()
+
+# *** parsing functions ***
+regex_simplifier = {
+    '%Y': r'(?P<year>\d{2,4})',
+    '%y': r'(?P<year>\d{2,4})',
+    '%m': r'(?P<month>\d{1,2})',
+    '%d': r'(?P<day>\d{1,2})',
+    '%H': r'(?P<hour>\d{1,2})',
+    '%I': r'(?P<hour>\d{1,2})',
+    '%M': r'(?P<minutes>\d{2})',
+    '%S': r'(?P<seconds>\d{2})',
+    '%P': r'(?P<ampm>[AaPp].? ?[Mm].?)',
+    '%p': r'(?P<ampm>[AaPp].? ?[Mm].?)',
+    '%name': fr'(?P<{COLNAMES_DF.USERNAME}>[^:]*)'
+}
+
+
+def generate_regex(log_error, hformat):
+    r"""Generate regular expression from hformat.
+    Parameters
+    ----------
+    log_error : list
+        List of error messages
+    hformat :str
+        Simplified syntax for the header, e.g. ``'%y-%m-%d, %H:%M:%S - %name:'``.
+    Returns
+    -------
+    str
+        Regular expression corresponding to the specified syntax
+    """
+    items = re.findall(r'\%\w*', hformat)
+
+    for i in items:
+        try:
+            hformat = hformat.replace(i, regex_simplifier[i])
+        except KeyError:
+            log_error(f"Could find regular expression for : {i}")
+
+    hformat = hformat + ' '
+    hformat_x = hformat.split('(?P<username>[^:]*)')[0]
+    return hformat, hformat_x
+
+
+def add_schema(df):
+    """Add default chat schema to df.
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        Chat DataFrame.
+    Returns
+    -------
+    pandas.DataFrame
+        Chat DataFrame with correct dtypes
+    """
+    df = df.astype({
+        COLNAMES_DF.DATE: pd.StringDtype(),
+        COLNAMES_DF.USERNAME: pd.StringDtype(),
+        COLNAMES_DF.MESSAGE: pd.StringDtype()
+    })
+    return df
+
+
+def parse_line(text, headers, i):
+    """Get date, username and message from the i:th intervention.
+    Parameters
+    ----------
+    text : str
+        Whole log chat text
+    headers : list
+        All headers.
+    i : int
+        Index denoting the message number
+    Returns
+    -------
+    dict
+        ith date, username and message.
+    """
+    result_ = headers[i].groupdict()
+    if 'ampm' in result_:
+        hour = int(result_['hour'])
+        mode = result_.get('ampm').lower()
+        if hour == 12 and mode == 'am':
+            hour = 0
+        elif hour != 12 and mode == 'pm':
+            hour += 12
+    else:
+        hour = int(result_['hour'])
+
+    # Check format of year. If year is 2-digit represented we add 2000
+    if len(result_['year']) == 2:
+        year = int(result_['year']) + 2000
+    else:
+        year = int(result_['year'])
+
+    if 'seconds' not in result_:
+        date = datetime(year, int(result_['month']), int(result_['day']), hour,
+                        int(result_['minutes']))
+    else:
+        date = datetime(year, int(result_['month']), int(result_['day']), hour,
+                        int(result_['minutes']), int(result_['seconds']))
+    username = result_[COLNAMES_DF.USERNAME]
+    message = get_message(text, headers, i)
+    line_dict = {
+        COLNAMES_DF.DATE: date,
+        COLNAMES_DF.USERNAME: username,
+        COLNAMES_DF.MESSAGE: message
+    }
+    return line_dict
+
+
+def remove_alerts_from_df(r_x, df):
+    """Try to get rid of alert/notification messages.
+    Parameters
+    ----------
+    r_x : str
+        Regular expression to detect whatsapp warnings
+    df : pandas.DataFrame
+        pandas.DataFrame with all interventions
+    Returns
+    -------
+    pandas.DataFrame
+        Fixed version of input DataFrame
+    """
+    df_new = df.copy()
+    df_new.loc[:, COLNAMES_DF.MESSAGE] = df_new[COLNAMES_DF.MESSAGE].apply(lambda x: remove_alerts_from_line(r_x, x))
+    return df_new
+
+
+def remove_alerts_from_line(r_x, line_df):
+    """Remove line content that is not desirable (automatic alerts etc.).
+    Parameters
+    ----------
+    r_x : str
+        Regula expression to detect WhatsApp warnings
+    line_df : str
+        Message sent as string
+    Returns
+    -------
+    str
+        Cleaned message string
+    """
+    if re.search(r_x, line_df):
+        return line_df[:re.search(r_x, line_df).start()]
+    else:
+        return line_df
+
+
+def get_message(text, headers, i):
+    """Get i:th message from text.
+    Parameters
+    ----------
+        text : str
+            Whole log chat text
+    headers : list
+        All headers
+    i : int
+        Index denoting the message number
+    Returns
+    -------
+    str
+     ith message.
+    """
+    msg_start = headers[i].end()
+    msg_end = headers[i + 1].start() if i < len(headers) - 1 else headers[i].endpos
+    msg = text[msg_start:msg_end].strip()
+    return msg
+
+
+def parse_text(text, regex):
+    """Parse chat using given regex.
+    Parameters
+    ----------
+    text : str
+        Whole log chat text
+    regex : str
+        Regular expression
+    Returns
+    -------
+    pandas.DataFrame
+        pandas.DataFrame with messages sent by users, index is the date the messages was sent.
+    Raises
+    ------
+    RegexError
+        When provided regex could not match the text
+    """
+    result = []
+    headers = list(re.finditer(regex, text))
+    try:
+        for i in range(len(headers)):
+            line_dict = parse_line(text, headers, i)
+            result.append(line_dict)
+    except KeyError:
+        print("Could not match the provided regex with provided text. No match was found.")
+        return None
+
+    df_chat = pd.DataFrame.from_records(result)
+    df_chat = df_chat[[COLNAMES_DF.DATE,COLNAMES_DF.USERNAME, COLNAMES_DF.MESSAGE]]
+
+    # clean username
+    df_chat[COLNAMES_DF.USERNAME] = df_chat[COLNAMES_DF.USERNAME].apply(lambda u: u.strip('\u202c'))
+
+    return df_chat
+
+
+def make_chat_df(log_error, text, hformat):
+    """Load chat as a DataFrame.
+    Parameters
+    ----------
+    log_error : list
+        List of error messages
+    text : str
+        Text of the chat
+    hformat : str
+        Simplified syntax for the header, e.g. ``'%y-%m-%d, %H:%M:%S - %name:'``
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas.DataFrame with three columns, i.e. 'date', 'username', and 'message'
+    """
+    # Bracket is reserved character in RegEx, add backslash before them.
+    hformat = hformat.replace('[', r'\[').replace(']', r'\]')
+
+    # Generate regex for given hformat
+    r, r_x = generate_regex(log_error,hformat=hformat)
+
+    # Parse chat to DataFrame
+    try:
+        df = parse_text(text, r)
+        df = remove_alerts_from_df(r_x, df)
+        df = add_schema(df)
+
+        return df
+    except:
+        print(f"hformat : {hformat} is not match with the given text")
+        return None
+
+
+def parse_chat(log_error, data):
+    """Parse chat and test it with defined hformats.
+    Parameters
+    ----------
+    log_error : list
+       List of error messages.
+    data : str
+        Data read from the chat file
+    Returns
+    -------
+    pandas.dataframe
+        A pandas.DataFrame with three columns, i.e. 'date', 'username', and 'message'
+    """
+    for hformat in hformats:
+        # Build DataFrame
+        df = make_chat_df(log_error, data, hformat)
+        if df is not None:
+             return df
+    log_error("hformats did not match the provided text. No match was found")
+    return None
+
+
+def decode_chat(log_error, f, filename):
+    """Parse the given zip file.
+    Parameters
+    ----------
+    log_error : list
+        List of error messages.
+    f : bytes
+        bytes of the file name in the zip file
+    filename : str
+        Name of a compressed file in the zip file.
+    Returns
+    -------
+    pandas.DataFrame
+        A pandas.DataFrame which includes the content of the given chat file.
+    """
+    try:
+        data = f.decode("utf-8")
+    except:
+        log_error(f"Could not decode to utf-8: {filename}")
+    else:
+        return parse_chat(log_error, data)
+
+
+def parse_zipfile(log_error, zfile):
+    """Parse the given zip file.
+    Parameters
+    ----------
+    log_error : list
+        List of error messages
+    zfile : ZipFile object
+        Regular expression
+    Returns
+    -------
+    list
+        A list of pandas.DataFrames which include the content of chat files.
+    """
+    results = []
     for name in zfile.namelist():
-        names.append(name)
-        info = zfile.getinfo(name)
-        data.append((name, info.compress_size, info.file_size))
-
-    return [{
-        "id": "overview",
-        "title": "The following files where read:",
-        "data_frame": pd.DataFrame(data, columns=["filename", "compressed size", "size"])
-    }]
+        if HIDDEN_FILE_RE.match(name):
+            continue
+        if not FILE_RE.match(name):
+            continue
+        chats = decode_chat(log_error,zfile.read(name),name)
+        results.append(chats)
+    if len(results)==0:
+        log_error("No valid chat file is available")
+    return results
+
+# *** test related function ***
+
+
+def input_df(data_path):
+    """Create inputs df_chats and df_participants, used for test purposes.
+    Parameters
+    ----------
+    data_path : str
+        File path of zip file
+    Returns
+    -------
+    pandas.DataFrame
+        df_chats and df_participants
+    """
+    errors = []
+    log_error = errors.append
+    fp = os.path.join(data_path, "whatsapp_chat.zip")
+    zfile = zipfile.ZipFile(fp)
+    chats = parse_zipfile(log_error, zfile)
+    participants = extract_participants_features(chats, anonymize=False)
+    return chats[0], participants[0]
+
+# *** analysis functions ***
+
+
+def get_response_matrix(df_chat):
+    """Create a response matrix for the usernames mentioned in the given DataFrame.
+    Parameters
+    ----------
+    df_chat: padas.DataFrame
+        A DataFrame including chat data
+    Returns
+    -------
+    pandas.DataFrame
+        A DataFrame with senders in the rows and receivers in the columns
+    """
+    users = set(df_chat[COLNAMES_DF.USERNAME])
+    users = sorted(users)
+
+    # Get list of username transitions and initialize dicitonary with counts
+    user_transitions = df_chat[COLNAMES_DF.USERNAME].tolist()
+    responses = {user: dict(zip(users, [0] * len(users))) for user in users}
+    # Fill count dictionary
+    for i in range(1, len(user_transitions)):
+        sender = user_transitions[i]
+        receiver = user_transitions[i - 1]
+        if sender != receiver:
+            responses[sender][receiver] += 1
+
+    responses = pd.DataFrame.from_dict(responses, orient='index')
+    return responses
+
+
+def make_salt():
+    """Return an string as salt for anonym_txt function.
+    Returns
+    -------
+    str
+        The salt value is deliberately set to be a fixed value for all the usernames, because then we can generate the
+        same hashed value for the same value in the UERNAME, REPLY_2USER, and USER_REPLY2 columns.
+    """
+    return str.encode('WhatsAppProject@2022')
+
+
+def anonym_txt(txt, salt):
+    """Define an internal function for calling internally  inside the anonymize_participants function.
+    Parameters
+    ----------
+    txt: str
+        The text that needs to be anonymized
+    salt: str
+        A string added to the input of a hash function to guarantee irreversible hashed values
+    Returns
+    -------
+    str
+        anonymized feature
+    """
+    anonymized_txt = hashlib.sha256(salt + txt.encode()).hexdigest()
+    return anonymized_txt
+
+
+def anonymize_participants(df_participants):
+    """Anonymize text data.
+    Anonymize USERNAME, REPLY_2USER, and USER_REPLY2 columns of the given DataFrame.
+    Parameters
+    ----------
+    df_participants : pandas.DataFrame
+        A DataFrame including participants data
+    Returns
+    -------
+    pandas.DataFrame
+        An anonymized DataFrame
+    """
+    salt = make_salt()
+    df_participants[COLNAMES_DF.USERNAME] = df_participants[COLNAMES_DF.USERNAME].apply(lambda u: anonym_txt(u, salt))
+    df_participants[COLNAMES_DF.REPLY_2USER] = df_participants[COLNAMES_DF.REPLY_2USER].apply(lambda u: anonym_txt(u,salt))
+    df_participants[COLNAMES_DF.USER_REPLY2] = df_participants[COLNAMES_DF.USER_REPLY2].apply(lambda u: anonym_txt(u,salt))
+    return df_participants
+
+
+def get_participants_features(df_chat):
+    """Calculate participant features from the given chat.
+    Parameter
+    ----------
+    df_chat : pandas.DataFrame
+        A DataFrame including chat data
+    Returns
+    -------
+    pandas.DataFrame
+        A DataFrame which includes participants and their features
+    """
+    # Calculate the number of words in messages
+    df_chat[COLNAMES_DF.WORDS_NO] = df_chat['message'].apply(lambda x: len(x.split()))
+    # number of ulrs
+    df_chat[COLNAMES_DF.URL_NO] = df_chat["message"].apply(lambda x: len(re.findall(URL_PATTERN, x))).astype(int)
+    # number of locations
+    df_chat[COLNAMES_DF.LOCATION_NO] = df_chat["message"].apply(
+        lambda x: len(re.findall(LOCATION_PATTERN, x))).astype(int)
+    # number of files
+    df_chat[COLNAMES_DF.FILE_NO] = df_chat["message"].apply(lambda x: len(re.findall(ATTACH_FILE_PATTERN, x))).astype(
+        int)
+    # number of messages
+    df_chat[COLNAMES_DF.MESSAGE_NO] = 1
+
+    df_participants = df_chat.groupby(COLNAMES_DF.USERNAME).agg({
+        COLNAMES_DF.WORDS_NO: 'sum',
+        COLNAMES_DF.URL_NO: 'sum',
+        COLNAMES_DF.LOCATION_NO: 'sum',
+        COLNAMES_DF.FILE_NO: 'sum',
+        COLNAMES_DF.MESSAGE_NO: 'sum'
+    }).reset_index()
+
+    response_matrix = get_response_matrix(df_chat)
+    out_degree = response_matrix.sum(axis=1)
+    in_degree = response_matrix.T.sum(axis=1)
+    user_reply2 = response_matrix.idxmax(axis=1)
+    reply2_user = response_matrix.T.idxmax(axis=1)
+
+    response_matrix[COLNAMES_DF.OUT_DEGREE] = out_degree
+    response_matrix[COLNAMES_DF.IN_DEGREE] = in_degree
+    response_matrix[COLNAMES_DF.USER_REPLY2] = user_reply2
+    response_matrix[COLNAMES_DF.REPLY_2USER] = reply2_user
+    response_matrix.index.name = COLNAMES_DF.USERNAME
+    response_matrix = response_matrix.loc[:,
+                      [COLNAMES_DF.OUT_DEGREE, COLNAMES_DF.IN_DEGREE, COLNAMES_DF.USER_REPLY2, COLNAMES_DF.REPLY_2USER]]
+    response_matrix = response_matrix.reset_index()
+
+    df_participants = pd.merge(df_participants, response_matrix, how="left", on=COLNAMES_DF.USERNAME, validate="1:1")
+
+    return df_participants
+
+
+def extract_participants_features(chats, anonymize=True):
+    """Parse the given zip file.
+    Parameters
+    ----------
+    chats : list
+        List of DataFrames including chat data
+    anonymize : bool
+        Indicates if usernames should be anonymized
+    Returns
+    -------
+    list
+        A list of DataFrames which include participant features
+    """
+    results = []
+    for chat in chats:
+        df = get_participants_features(chat)
+        if anonymize:
+            df = anonymize_participants(df)
+        results.append(df)
+    return results
+
+# ***** end of analysis functions *****
+
+
+def format_results(df_list):
+    """Format results to the standard format.
+    Parameters
+    ----------
+    df_list: pandas.dataframe
+    Returns
+    -------
+    pandas.dataframe
+    """
+    results = []
+    for df in df_list:
+        results.append(
+            {
+                "id": "overview",
+                "title": "The following data is extracted from the file:",
+                "data_frame": df
+            }
+        )
+    return results
+
+
+def format_errors(errors):
+    """Return errors in the format of dataframe.
+    Parameters
+    ----------
+    errors: str
+    Returns
+    -------
+    pandas.dataframe
+    """
+    data_frame = pd.DataFrame()
+    data_frame["Messages"] = pd.Series(errors, name="Messages")
+    return {"id": "extraction_log", "title": "Extraction log", "data_frame": data_frame}
+
+
+def process(file_data):
+    """Convert whatsapp chat_file.zip to participants dataframe.
+    This is the main function which extracts the participants
+    information from the row chat_file.zip provided by data-donators.
+    Parameters
+    ----------
+    file_data : str
+        The path of the chat_file.zip
+    Returns
+    -------
+    pandas.dataframe
+        Extracted data from the chat_file
+    """
+    errors = []
+    log_error = errors.append
+    zfile = None
+    chats = []
+    try:
+        zfile = zipfile.ZipFile(file_data)
+    except:
+        if FILE_RE.match(file_data.name):
+            zfile = open(file_data, encoding="utf8")
+            chat = parse_chat(log_error, zfile.read())
+            chats.append(chat)
+        else:
+            log_error("There is not a valid file format.")
+            return [format_errors(errors)]
+    else:
+        chats = parse_zipfile(log_error, zfile)
+    if errors:
+        return [format_errors(errors)]
+
+    participants = extract_participants_features(chats)
+    formatted_results = format_results(participants)
+
+    return formatted_results

From 97930ea3989d7e5940459eb5a73613ecc505e6cf Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 24 May 2022 14:15:25 +0200
Subject: [PATCH 04/42] initial commit

---
 .../whatsapp_account_info/__init__.py         | 96 +++++++++++++++++++
 1 file changed, 96 insertions(+)
 create mode 100644 data_extractor/whatsapp_account_info/__init__.py

diff --git a/data_extractor/whatsapp_account_info/__init__.py b/data_extractor/whatsapp_account_info/__init__.py
new file mode 100644
index 0000000..26cbd7e
--- /dev/null
+++ b/data_extractor/whatsapp_account_info/__init__.py
@@ -0,0 +1,96 @@
+__version__ = '0.2.0'
+
+import zipfile
+import re
+import pandas as pd
+import json
+
+HIDDEN_FILE_RE = re.compile(r".*__MACOSX*")
+FILE_RE = re.compile(r".*.json$")
+
+
+class ColnamesDf:
+    GROUPS = 'groups'
+    """Groups column"""
+
+    CONTACTS = 'contacts'
+    """Contacts column"""
+
+
+COLNAMES_DF = ColnamesDf()
+
+
+def format_results(df):
+    results = []
+    results.append(
+        {
+        "id": "Whatsapp account info",
+        "title": "The account information file is read:",
+        "data_frame": df
+        }
+    )
+    return results
+
+
+def format_errors(errors):
+    data_frame = pd.DataFrame()
+    data_frame["Messages"] = pd.Series(errors, name="Messages")
+    return {"id": "extraction_log", "title": "Extraction log", "data_frame": data_frame}
+
+
+def extract_data(log_error, data):
+    # data = pd.read_csv('whatsapp/df_chat.csv')
+    # return 1,1
+    groups_no = 0
+    contacts_no = 0
+    try:
+        groups_no = len(data[COLNAMES_DF.GROUPS])
+    except (TypeError, KeyError) as e:
+        print("No group is available")
+    try:
+        contacts_no = len(data[COLNAMES_DF.CONTACTS])
+    except (TypeError, KeyError) as e:
+        print("No contact is available")
+
+    if (groups_no == 0) and (contacts_no == 0):
+        log_error("Neither group nor contact is available")
+    return groups_no, contacts_no
+
+
+def parse_records(log_error, f):
+    try:
+        data = json.load(f)
+    except json.JSONDecodeError:
+        log_error(f"Could not parse: {f.name}")
+    else:
+        return data
+
+
+def parse_zipfile(log_error, zfile):
+    for name in zfile.namelist():
+        if HIDDEN_FILE_RE.match(name):
+            continue
+        if not FILE_RE.match(name):
+            continue
+        return parse_records(log_error, zfile.open(name))
+    log_error("No Json file is available")
+
+
+def process(file_data):
+    errors = []
+    log_error = errors.append
+    zfile = zipfile.ZipFile(file_data)
+    data = parse_zipfile(log_error, zfile)
+
+    if data is not None:
+        groups_no, contacts_no = extract_data(log_error, data)
+
+    if errors:
+        return [format_errors(errors)]
+
+    d = {'number_of_groups': [groups_no], 'number_of_contacts': [contacts_no]}
+    df = pd.DataFrame(data=d)
+    formatted_results = format_results(df)
+
+    return formatted_results
+

From 476e515769b0b8e19436fbf5b212b65de3118ad7 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 24 May 2022 14:49:22 +0200
Subject: [PATCH 05/42] Rename test_whatsapp to test_whatsapp_chat

---
 data_extractor/tests/{test_whatsapp.py => test_whatsapp_chat.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename data_extractor/tests/{test_whatsapp.py => test_whatsapp_chat.py} (100%)

diff --git a/data_extractor/tests/test_whatsapp.py b/data_extractor/tests/test_whatsapp_chat.py
similarity index 100%
rename from data_extractor/tests/test_whatsapp.py
rename to data_extractor/tests/test_whatsapp_chat.py

From 0852f27713c1a24c90b0c13ad76363687ca4820a Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 24 May 2022 14:56:46 +0200
Subject: [PATCH 06/42] Initial commit

---
 data_extractor/tests/test_whatsapp_chat.py | 45 +++++++++++++++++-----
 1 file changed, 36 insertions(+), 9 deletions(-)

diff --git a/data_extractor/tests/test_whatsapp_chat.py b/data_extractor/tests/test_whatsapp_chat.py
index aa05945..cc64d22 100644
--- a/data_extractor/tests/test_whatsapp_chat.py
+++ b/data_extractor/tests/test_whatsapp_chat.py
@@ -1,22 +1,49 @@
-from example import process
+from whatsapp_chat import process
+from whatsapp_chat import anonymize_participants
 from pathlib import Path
-
 import pandas as pd
 from pandas.testing import assert_frame_equal
 
+
 DATA_PATH = Path(__file__).parent / "data"
 
 EXPECTED = [
-    {'filename': 'hello/', 'compressed size': 0, 'size': 0},
-    {'filename': 'hello/world.txt', 'compressed size': 267, 'size': 471}
+    {'username': 'user1', 'total_words_no': 20, 'url_no': 1, 'location_no': 1, 'file_no': 0, 'message_no': 3,
+     'out_degree': 2, 'in_degree': 3, 'user_reply2': 'user2', 'reply_2_user': 'user2'},
+
+    {'username': 'user2', 'total_words_no': 18, 'url_no': 2, 'location_no': 0, 'file_no': 0, 'message_no': 3,
+     'out_degree': 3, 'in_degree': 3, 'user_reply2': 'user1', 'reply_2_user': 'user1'},
+
+    {'username': 'user3', 'total_words_no': 1, 'url_no': 0, 'location_no': 0, 'file_no': 0, 'message_no': 1,
+     'out_degree': 1, 'in_degree': 1, 'user_reply2': 'user2', 'reply_2_user': 'user2'},
+
+    {'username': 'user4', 'total_words_no': 21, 'url_no': 0, 'location_no': 0, 'file_no': 0, 'message_no': 2,
+     'out_degree': 2, 'in_degree': 1, 'user_reply2': 'user1', 'reply_2_user': 'user1'}
 ]
 
 
-def test_hello_world():
+def test_process():
+    """ Test process function.
+    compares the expected dataframe with the output of the process function to check if all the columns are match.
+    Raises
+    -------
+    AssertionError: When provided expected dataframe could not match the participants dataframe
+    """
+
     df_expected = pd.DataFrame(EXPECTED)
+    df_expected = anonymize_participants(df_expected)
+    df_expected['message_no'] = df_expected['message_no'].astype('int64')
+    df_expected['url_no'] = df_expected['url_no'].astype('int32')
+    df_expected['location_no'] = df_expected['location_no'].astype('int32')
+    df_expected['file_no'] = df_expected['file_no'].astype('int32')
 
-    result = process(DATA_PATH.joinpath("hello.zip").open("rb"))
+    # result = process(DATA_PATH.joinpath("whatsapp_chat.zip"))
+    result = process(DATA_PATH.joinpath("_chat.txt"))
     assert len(result) == 1
-    assert result[0]["id"] == 'overview'
-    assert result[0]["title"] == 'The following files where read:'
-    assert_frame_equal(result[0]["data_frame"], df_expected)
+    df_result = result[0]["data_frame"]
+    assert_frame_equal(df_result, df_expected)
+
+
+if __name__ == '__main__':
+    test_process()
+

From e62987b75ff51c7f056d67c51ab890b79f902313 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 24 May 2022 14:59:15 +0200
Subject: [PATCH 07/42] Initial commit

---
 .../tests/test_whatsapp_account_info.py       | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 data_extractor/tests/test_whatsapp_account_info.py

diff --git a/data_extractor/tests/test_whatsapp_account_info.py b/data_extractor/tests/test_whatsapp_account_info.py
new file mode 100644
index 0000000..23f4efa
--- /dev/null
+++ b/data_extractor/tests/test_whatsapp_account_info.py
@@ -0,0 +1,31 @@
+import pandas as pd
+from pandas.testing import assert_frame_equal
+
+from whatsapp_account_info import process
+from pathlib import Path
+
+
+DATA_PATH = Path(__file__).parent / "data"
+EXPECTED = [
+    {'number_of_groups': 4,
+     'number_of_contacts': 3
+     }
+]
+
+
+def test_process():
+    """ Test process function.
+        compares the expected dataframe with the output of the process function to check if all the columns are matched.
+        Raises
+        -------
+        AssertionError: When provided expected dataframe could not match the participants dataframe
+        """
+    df_expected = pd.DataFrame(EXPECTED)
+
+    result = process(DATA_PATH.joinpath("account_info.zip"))
+    df_result = result[0]["data_frame"]
+    assert_frame_equal(df_result, df_expected)
+
+
+if __name__ == '__main__':
+    test_process()
\ No newline at end of file

From 598406c85a7a73c37ca6f4fb4066444f25aa5939 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 24 May 2022 15:01:57 +0200
Subject: [PATCH 08/42] Add test data

---
 data_extractor/tests/data/_chat.txt         |  11 +++++++++++
 data_extractor/tests/data/account_info.zip  | Bin 0 -> 275 bytes
 data_extractor/tests/data/whatsapp_chat.zip | Bin 0 -> 617 bytes
 3 files changed, 11 insertions(+)
 create mode 100644 data_extractor/tests/data/_chat.txt
 create mode 100644 data_extractor/tests/data/account_info.zip
 create mode 100644 data_extractor/tests/data/whatsapp_chat.zip

diff --git a/data_extractor/tests/data/_chat.txt b/data_extractor/tests/data/_chat.txt
new file mode 100644
index 0000000..2ffeac2
--- /dev/null
+++ b/data_extractor/tests/data/_chat.txt
@@ -0,0 +1,11 @@
+[16/03/2022, 15:20:25] user1: Hi shiva!
+[16/03/2022, 15:25:38] user2: Hi 👋
+[16/03/2022, 15:26:48] user3: Hoi!
+[16/03/2022, 18:39:29] user2: https://youtu.be/KBmUTY6mK_E
+[16/03/2022, 18:35:51] user1: ‎Location: https://maps.google.com/?q=52.089451,5.108469
+[20/03/2022, 20:08:51] user4‬: I’m about to generate some very random messages so that I can make some screenshots for the explanation to participants
+[24/03/2022, 20:19:38] user1‬: @user3 if you remove your Profile picture for a moment I will redo the screenshots 😁
+[26/03/2022, 18:52:15] user2: Well done Utrecht 😁
+Opkomst provinciesteden rond 20 procent, Utrecht hoogste opkomst van G4
+https://nos.nl/l/2421368#UPDATE-container-60145354
+[14/07/2020, 22:05:54] user4: 👍Bedankt
\ No newline at end of file
diff --git a/data_extractor/tests/data/account_info.zip b/data_extractor/tests/data/account_info.zip
new file mode 100644
index 0000000000000000000000000000000000000000..0489b54089e3607d9e4ccccbd8b03c060437173d
GIT binary patch
literal 275
zcmWIWW@Zs#U|`^2(5~DZV)}gl{9GWf5{LzWxFEl%Brz#7C$pqdFRM5|Pqo{TtJy%n
zb-hqpw)tew<y+Y`O74~^K2l=2*dW!pSmDuK(;ZcR{#QuvFz6^XI_N!hf}qi&#AWSm
zvzvG#jUQf~VLku%rzw+{-Ek<D^X2(mv9-Q)Yqsmz=J;0@m65?8f7S|jUAS>%``g;>
zPFv*zycwD78E`pH1?W5wPymydPUV8iFfvFmEcKII-k5!Zk;6m!E{KNl1H4(;K-w9B
LFbPP{2Ac%{W)fPO

literal 0
HcmV?d00001

diff --git a/data_extractor/tests/data/whatsapp_chat.zip b/data_extractor/tests/data/whatsapp_chat.zip
new file mode 100644
index 0000000000000000000000000000000000000000..b3f6edfd76b4f70b94dd4b0ac39d66bf5fde1947
GIT binary patch
literal 617
zcmWIWW@Zs#U|`^2(3n0YWcuRU=PxocF#KU+VBiFb#wTYamgtpKl;lo2n0LiM#P$2s
zU;O+V=c;WKZtqcL2?+@h?bNe!;!u2|malg0NZAJ4#&>O|Or5f8elgcvt#PzxR9~{v
z?X6zd``zzf-22}6q;S$PlZ7R0ErqiRPT3tgwop|j&bwgin<I-4H&;6yyZmVX1inMF
zU&r5<pYYU1IyG~GWaar)4_?l>{4*uA`o5Li@$j_A<`-3ur612K{Ad@RKHpDnM(Ui2
zo#osA{yaA2koV&EJySAbw-}{eO$d<^wg3HR%0&&$XOhasViT8~_*7Z2?Bv|y4n<A1
z=Q-Q|o_<)e{qJIj7&g5RY+)-da`ivxae8qfu2FQC5toRSdf1CY+&`i!ELwvy0+MXI
zVlxG|uI?@0U}DxG@FDv)hwXyz@78XJo3=^x*6w+_x097WYt|TTu$5<G+u0%@9TBqZ
z@W0E4cA9$MRM@iVi*)y4J4UU97jkd9b7n1AUv;T=!|d1h@BcdcyVCd+=j>fo?TNdL
zq!J(MPTMNK{NA?g#dF%P&GLH{wCHohLbmJ@(I3D6r{1uiee6|$|Ahug_T3Y%uYU94
z!8YgIm?g7!bY{AF&Jynzf36=NGe>5sTeYIIpnIj0(X>r#>?aCbn=EI^nHV^)Mfl_~
zrONtc_I(?Ls^0!*4De=TvS+{*^D4l&2LS~ziIfhwphAoc5)4U7P8+U=++bYge#z<v
bkoGy@2<8WPv$BEoF#=&Ukp2NoPz($Jz#$DG

literal 0
HcmV?d00001


From 2dae4f48d4e51e8f1aee4b20c7b0fef85b77ec46 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 24 May 2022 15:08:01 +0200
Subject: [PATCH 09/42] Remove main

---
 data_extractor/tests/test_whatsapp_account_info.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/data_extractor/tests/test_whatsapp_account_info.py b/data_extractor/tests/test_whatsapp_account_info.py
index 23f4efa..d5ecb2e 100644
--- a/data_extractor/tests/test_whatsapp_account_info.py
+++ b/data_extractor/tests/test_whatsapp_account_info.py
@@ -26,6 +26,3 @@ def test_process():
     df_result = result[0]["data_frame"]
     assert_frame_equal(df_result, df_expected)
 
-
-if __name__ == '__main__':
-    test_process()
\ No newline at end of file

From 99b3442040f28a0daa21b81a09c9873759d27fb3 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 24 May 2022 15:08:35 +0200
Subject: [PATCH 10/42] Remove main

---
 data_extractor/tests/test_whatsapp_chat.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/data_extractor/tests/test_whatsapp_chat.py b/data_extractor/tests/test_whatsapp_chat.py
index cc64d22..eaf86c1 100644
--- a/data_extractor/tests/test_whatsapp_chat.py
+++ b/data_extractor/tests/test_whatsapp_chat.py
@@ -44,6 +44,3 @@ def test_process():
     assert_frame_equal(df_result, df_expected)
 
 
-if __name__ == '__main__':
-    test_process()
-

From d829bda8a15ba92e577e3f07c4529d78cb8c02fb Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 31 May 2022 10:32:55 +0200
Subject: [PATCH 11/42] factorizing the sensetive data instead of hashing

---
 data_extractor/whatsapp_chat/__init__.py | 48 +++++++++---------------
 1 file changed, 18 insertions(+), 30 deletions(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index 4c7f8a9..1ef7160 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -9,6 +9,7 @@
 import pandas as pd
 import hashlib
 import zipfile
+from pathlib import Path
 
 
 URL_PATTERN = r'(https?://\S+)'
@@ -439,32 +440,15 @@ def get_response_matrix(df_chat):
     return responses
 
 
-def make_salt():
-    """Return an string as salt for anonym_txt function.
-    Returns
-    -------
-    str
-        The salt value is deliberately set to be a fixed value for all the usernames, because then we can generate the
-        same hashed value for the same value in the UERNAME, REPLY_2USER, and USER_REPLY2 columns.
-    """
-    return str.encode('WhatsAppProject@2022')
-
-
-def anonym_txt(txt, salt):
-    """Define an internal function for calling internally  inside the anonymize_participants function.
-    Parameters
-    ----------
-    txt: str
-        The text that needs to be anonymized
-    salt: str
-        A string added to the input of a hash function to guarantee irreversible hashed values
-    Returns
-    -------
-    str
-        anonymized feature
-    """
-    anonymized_txt = hashlib.sha256(salt + txt.encode()).hexdigest()
-    return anonymized_txt
+# def make_salt():
+#     """Return an string as salt for anonym_txt function.
+#     Returns
+#     -------
+#     str
+#         The salt value is deliberately set to be a fixed value for all the usernames, because then we can generate the
+#         same hashed value for the same value in the UERNAME, REPLY_2USER, and USER_REPLY2 columns.
+#     """
+#     return str.encode('WhatsAppProject@2022')
 
 
 def anonymize_participants(df_participants):
@@ -479,10 +463,13 @@ def anonymize_participants(df_participants):
     pandas.DataFrame
         An anonymized DataFrame
     """
-    salt = make_salt()
-    df_participants[COLNAMES_DF.USERNAME] = df_participants[COLNAMES_DF.USERNAME].apply(lambda u: anonym_txt(u, salt))
-    df_participants[COLNAMES_DF.REPLY_2USER] = df_participants[COLNAMES_DF.REPLY_2USER].apply(lambda u: anonym_txt(u,salt))
-    df_participants[COLNAMES_DF.USER_REPLY2] = df_participants[COLNAMES_DF.USER_REPLY2].apply(lambda u: anonym_txt(u,salt))
+    # salt = make_salt()
+    # df_participants[COLNAMES_DF.USERNAME] = df_participants[COLNAMES_DF.USERNAME].apply(lambda u: anonym_txt(u, salt))
+    # df_participants[COLNAMES_DF.REPLY_2USER] = df_participants[COLNAMES_DF.REPLY_2USER].apply(lambda u: anonym_txt(u,salt))
+    # df_participants[COLNAMES_DF.USER_REPLY2] = df_participants[COLNAMES_DF.USER_REPLY2].apply(lambda u: anonym_txt(u,salt))
+    # df_participants[['username', 'user_reply2']] = df_participants[['username', 'user_reply2']].stack().rank(method='dense').unstack()
+    stacked = df_participants[['username', 'user_reply2', 'reply_2_user']].stack()
+    df_participants[['username', 'user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0], index=stacked.index).unstack()
     return df_participants
 
 
@@ -633,3 +620,4 @@ def process(file_data):
     formatted_results = format_results(participants)
 
     return formatted_results
+

From 6fdde5b0f6782e027ecc6707cca7356a7636faa5 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 31 May 2022 10:32:55 +0200
Subject: [PATCH 12/42] Factorize the sensetive data instead of hashing

---
 data_extractor/whatsapp_chat/__init__.py | 48 +++++++++---------------
 1 file changed, 18 insertions(+), 30 deletions(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index 4c7f8a9..1ef7160 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -9,6 +9,7 @@
 import pandas as pd
 import hashlib
 import zipfile
+from pathlib import Path
 
 
 URL_PATTERN = r'(https?://\S+)'
@@ -439,32 +440,15 @@ def get_response_matrix(df_chat):
     return responses
 
 
-def make_salt():
-    """Return an string as salt for anonym_txt function.
-    Returns
-    -------
-    str
-        The salt value is deliberately set to be a fixed value for all the usernames, because then we can generate the
-        same hashed value for the same value in the UERNAME, REPLY_2USER, and USER_REPLY2 columns.
-    """
-    return str.encode('WhatsAppProject@2022')
-
-
-def anonym_txt(txt, salt):
-    """Define an internal function for calling internally  inside the anonymize_participants function.
-    Parameters
-    ----------
-    txt: str
-        The text that needs to be anonymized
-    salt: str
-        A string added to the input of a hash function to guarantee irreversible hashed values
-    Returns
-    -------
-    str
-        anonymized feature
-    """
-    anonymized_txt = hashlib.sha256(salt + txt.encode()).hexdigest()
-    return anonymized_txt
+# def make_salt():
+#     """Return an string as salt for anonym_txt function.
+#     Returns
+#     -------
+#     str
+#         The salt value is deliberately set to be a fixed value for all the usernames, because then we can generate the
+#         same hashed value for the same value in the UERNAME, REPLY_2USER, and USER_REPLY2 columns.
+#     """
+#     return str.encode('WhatsAppProject@2022')
 
 
 def anonymize_participants(df_participants):
@@ -479,10 +463,13 @@ def anonymize_participants(df_participants):
     pandas.DataFrame
         An anonymized DataFrame
     """
-    salt = make_salt()
-    df_participants[COLNAMES_DF.USERNAME] = df_participants[COLNAMES_DF.USERNAME].apply(lambda u: anonym_txt(u, salt))
-    df_participants[COLNAMES_DF.REPLY_2USER] = df_participants[COLNAMES_DF.REPLY_2USER].apply(lambda u: anonym_txt(u,salt))
-    df_participants[COLNAMES_DF.USER_REPLY2] = df_participants[COLNAMES_DF.USER_REPLY2].apply(lambda u: anonym_txt(u,salt))
+    # salt = make_salt()
+    # df_participants[COLNAMES_DF.USERNAME] = df_participants[COLNAMES_DF.USERNAME].apply(lambda u: anonym_txt(u, salt))
+    # df_participants[COLNAMES_DF.REPLY_2USER] = df_participants[COLNAMES_DF.REPLY_2USER].apply(lambda u: anonym_txt(u,salt))
+    # df_participants[COLNAMES_DF.USER_REPLY2] = df_participants[COLNAMES_DF.USER_REPLY2].apply(lambda u: anonym_txt(u,salt))
+    # df_participants[['username', 'user_reply2']] = df_participants[['username', 'user_reply2']].stack().rank(method='dense').unstack()
+    stacked = df_participants[['username', 'user_reply2', 'reply_2_user']].stack()
+    df_participants[['username', 'user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0], index=stacked.index).unstack()
     return df_participants
 
 
@@ -633,3 +620,4 @@ def process(file_data):
     formatted_results = format_results(participants)
 
     return formatted_results
+

From cfbc84b5a8ff228f5c03d78f3e153af2688f36db Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 31 May 2022 10:58:54 +0200
Subject: [PATCH 13/42] Factorize username, reply_2, and rely_2_user columns
 instead of hashing

---
 data_extractor/whatsapp_chat/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index 1ef7160..39f09d0 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -468,8 +468,10 @@ def anonymize_participants(df_participants):
     # df_participants[COLNAMES_DF.REPLY_2USER] = df_participants[COLNAMES_DF.REPLY_2USER].apply(lambda u: anonym_txt(u,salt))
     # df_participants[COLNAMES_DF.USER_REPLY2] = df_participants[COLNAMES_DF.USER_REPLY2].apply(lambda u: anonym_txt(u,salt))
     # df_participants[['username', 'user_reply2']] = df_participants[['username', 'user_reply2']].stack().rank(method='dense').unstack()
+
     stacked = df_participants[['username', 'user_reply2', 'reply_2_user']].stack()
-    df_participants[['username', 'user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0], index=stacked.index).unstack()
+    df_participants[['username', 'user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0],
+                                                                             index=stacked.index).unstack()
     return df_participants
 
 

From 254fd8de0d021f8283dc0af69222e7d34d26c1e2 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 31 May 2022 11:33:44 +0200
Subject: [PATCH 14/42] Factorize sensetive columns

---
 data_extractor/whatsapp_chat/__init__.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index 3a215e2..dbc5e27 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -468,16 +468,10 @@ def anonymize_participants(df_participants):
     # df_participants[COLNAMES_DF.REPLY_2USER] = df_participants[COLNAMES_DF.REPLY_2USER].apply(lambda u: anonym_txt(u,salt))
     # df_participants[COLNAMES_DF.USER_REPLY2] = df_participants[COLNAMES_DF.USER_REPLY2].apply(lambda u: anonym_txt(u,salt))
     # df_participants[['username', 'user_reply2']] = df_participants[['username', 'user_reply2']].stack().rank(method='dense').unstack()
-<<<<<<< HEAD
 
     stacked = df_participants[['username', 'user_reply2', 'reply_2_user']].stack()
     df_participants[['username', 'user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0],
                                                                              index=stacked.index).unstack()
-=======
-    stacked = df_participants[['username', 'user_reply2', 'reply_2_user']].stack()
-    df_participants[['username', 'user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0], index=stacked.index).unstack()
->>>>>>> d829bda8a15ba92e577e3f07c4529d78cb8c02fb
-    return df_participants
 
 
 def get_participants_features(df_chat):

From 6a1aaa2d548a5ee5fa82bf787538c8ac7fc529a3 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 31 May 2022 11:50:07 +0200
Subject: [PATCH 15/42] Add missed return to the anonymize_participants()

---
 data_extractor/whatsapp_chat/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index dbc5e27..39f09d0 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -472,6 +472,7 @@ def anonymize_participants(df_participants):
     stacked = df_participants[['username', 'user_reply2', 'reply_2_user']].stack()
     df_participants[['username', 'user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0],
                                                                              index=stacked.index).unstack()
+    return df_participants
 
 
 def get_participants_features(df_chat):

From 43c08081c9ceecf8da8f2d15d7835a84b3f3d2c2 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 31 May 2022 12:18:13 +0200
Subject: [PATCH 16/42] Rename imports

---
 data_extractor/tests/test_whatsapp_chat.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/data_extractor/tests/test_whatsapp_chat.py b/data_extractor/tests/test_whatsapp_chat.py
index eaf86c1..6d5a6a6 100644
--- a/data_extractor/tests/test_whatsapp_chat.py
+++ b/data_extractor/tests/test_whatsapp_chat.py
@@ -24,11 +24,11 @@
 
 def test_process():
     """ Test process function.
-    compares the expected dataframe with the output of the process function to check if all the columns are match.
-    Raises
-    -------
-    AssertionError: When provided expected dataframe could not match the participants dataframe
-    """
+        compares the expected dataframe with the output of the process function to check if all the columns are match.
+        Raises
+        -------
+        AssertionError: When provided expected dataframe could not match the participants dataframe
+        """
 
     df_expected = pd.DataFrame(EXPECTED)
     df_expected = anonymize_participants(df_expected)

From ede47ccc448fbe64c45554d221992b0911a70eb4 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Tue, 31 May 2022 12:19:18 +0200
Subject: [PATCH 17/42] Rename anonymized columns

---
 data_extractor/whatsapp_chat/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index 39f09d0..a7673d3 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -472,6 +472,8 @@ def anonymize_participants(df_participants):
     stacked = df_participants[['username', 'user_reply2', 'reply_2_user']].stack()
     df_participants[['username', 'user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0],
                                                                              index=stacked.index).unstack()
+    df_participants[['username', 'user_reply2', 'reply_2_user']] = 'person' + df_participants[['username', 'user_reply2',
+                                                                                        'reply_2_user']].astype(str)
     return df_participants
 
 

From c3e2bcd11d420cef83cd6f898af0056cef633f5b Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Wed, 15 Jun 2022 22:49:00 +0200
Subject: [PATCH 18/42] Add first/last message date

---
 data_extractor/tests/test_whatsapp_chat.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/data_extractor/tests/test_whatsapp_chat.py b/data_extractor/tests/test_whatsapp_chat.py
index 6d5a6a6..492c977 100644
--- a/data_extractor/tests/test_whatsapp_chat.py
+++ b/data_extractor/tests/test_whatsapp_chat.py
@@ -9,16 +9,20 @@
 
 EXPECTED = [
     {'username': 'user1', 'total_words_no': 20, 'url_no': 1, 'location_no': 1, 'file_no': 0, 'message_no': 3,
-     'out_degree': 2, 'in_degree': 3, 'user_reply2': 'user2', 'reply_2_user': 'user2'},
+     'first_message_date': '2022-03-16 15:20:25', 'last_message_date': '2022-03-24 20:19:38', 'out_degree': 2,
+     'in_degree': 3, 'user_reply2': 'user2', 'reply_2_user': 'user2'},
 
     {'username': 'user2', 'total_words_no': 18, 'url_no': 2, 'location_no': 0, 'file_no': 0, 'message_no': 3,
-     'out_degree': 3, 'in_degree': 3, 'user_reply2': 'user1', 'reply_2_user': 'user1'},
+     'first_message_date': '2022-03-16 15:25:38', 'last_message_date': '2022-03-26 18:52:15', 'out_degree': 3,
+     'in_degree': 3, 'user_reply2': 'user1', 'reply_2_user': 'user1'},
 
     {'username': 'user3', 'total_words_no': 1, 'url_no': 0, 'location_no': 0, 'file_no': 0, 'message_no': 1,
-     'out_degree': 1, 'in_degree': 1, 'user_reply2': 'user2', 'reply_2_user': 'user2'},
+     'first_message_date': '2022-03-16 15:26:48', 'last_message_date': '2022-03-16 15:26:48', 'out_degree': 1,
+     'in_degree': 1, 'user_reply2': 'user2', 'reply_2_user': 'user2'},
 
     {'username': 'user4', 'total_words_no': 21, 'url_no': 0, 'location_no': 0, 'file_no': 0, 'message_no': 2,
-     'out_degree': 2, 'in_degree': 1, 'user_reply2': 'user1', 'reply_2_user': 'user1'}
+     'first_message_date': '2020-07-14 22:05:54', 'last_message_date': '2022-03-20 20:08:51', 'out_degree': 2,
+     'in_degree': 1, 'user_reply2': 'user1', 'reply_2_user': 'user1'}
 ]
 
 

From 4d65e0d0e91820550ebdfeaa126efe2b2fb9d0c3 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Wed, 15 Jun 2022 22:49:53 +0200
Subject: [PATCH 19/42] Add first/last message date

---
 data_extractor/whatsapp_chat/__init__.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index a7673d3..c68c7c9 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -488,6 +488,10 @@ def get_participants_features(df_chat):
     pandas.DataFrame
         A DataFrame which includes participants and their features
     """
+    # Calculate first message date
+    df_chat[COLNAMES_DF.FirstMessage] = df_chat[COLNAMES_DF.DATE].astype('datetime64[ns]')
+    # Calculate last message date
+    df_chat[COLNAMES_DF.LastMessage] = df_chat[COLNAMES_DF.DATE].astype('datetime64[ns]')
     # Calculate the number of words in messages
     df_chat[COLNAMES_DF.WORDS_NO] = df_chat['message'].apply(lambda x: len(x.split()))
     # number of ulrs
@@ -506,7 +510,9 @@ def get_participants_features(df_chat):
         COLNAMES_DF.URL_NO: 'sum',
         COLNAMES_DF.LOCATION_NO: 'sum',
         COLNAMES_DF.FILE_NO: 'sum',
-        COLNAMES_DF.MESSAGE_NO: 'sum'
+        COLNAMES_DF.MESSAGE_NO: 'sum',
+        COLNAMES_DF.FirstMessage: 'min',
+        COLNAMES_DF.LastMessage: 'max'
     }).reset_index()
 
     response_matrix = get_response_matrix(df_chat)
@@ -624,4 +630,3 @@ def process(file_data):
     formatted_results = format_results(participants)
 
     return formatted_results
-

From dbf50aef825cb39453bca15d6afe553159b55f87 Mon Sep 17 00:00:00 2001
From: parisa-zahedi <parisa.zahedi@gmail.com>
Date: Thu, 30 Jun 2022 12:01:34 +0200
Subject: [PATCH 20/42] change output format

---
 data_extractor/whatsapp_chat/__init__.py | 138 ++++++++++++++++-------
 1 file changed, 100 insertions(+), 38 deletions(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index c68c7c9..e3f6577 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -10,15 +10,21 @@
 import hashlib
 import zipfile
 from pathlib import Path
+#from nltk.corpus import stopwords
+#from sklearn.feature_extraction.text import TfidfVectorizer
 
 
 URL_PATTERN = r'(https?://\S+)'
 LOCATION_PATTERN = r'(Location: https?://\S+)'
 ATTACH_FILE_PATTERN = r'(<attached: \S+>)'
-FILE_RE = re.compile(r".*chat.*.txt$")
+FILE_RE = re.compile(r".*.txt$")
 HIDDEN_FILE_RE = re.compile(r".*__MACOSX*")
 
-hformats = ['%m/%d/%y, %H:%M - %name:', '[%d/%m/%y, %H:%M:%S] %name:', '%d-%m-%y %H:%M - %name:']
+SYSTEM_MESSAGES=[
+    'Messages and calls are end-to-end encrypted. No one outside of this chat, not even WhatsApp, can read or listen to them.',
+    'Berichten en gesprekken worden end-to-end versleuteld. Niemand buiten deze chat kan ze lezen of beluisteren, zelfs WhatsApp niet.'
+]
+hformats = ['%m/%d/%y, %H:%M - %name:', '[%d/%m/%y, %H:%M:%S] %name:', '%d-%m-%y %H:%M - %name:', '[%d-%m-%y %H:%M:%S] %name:']
 
 
 class ColnamesDf:
@@ -36,16 +42,16 @@ class ColnamesDf:
     MESSAGE_LENGTH = 'message_length'
     """Message length column"""
 
-    FirstMessage = 'first_message_date'
+    FirstMessage = 'Date first message' #'first_message_date'
     """Date of first message column"""
 
-    LastMessage = 'last_message_date'
+    LastMessage = 'Date last message' #'last_message_date'
     """Date of last message column"""
 
-    MESSAGE_NO = 'message_no'
+    MESSAGE_NO = 'Number of messages' #'message_no'
     """Number of Message  column"""
 
-    WORDS_NO = 'total_words_no'
+    WORDS_NO = 'Total number of words' #'total_words_no'
     """Total number of words  column"""
 
     REPLY_2USER = 'reply_2_user'
@@ -57,13 +63,13 @@ class ColnamesDf:
     USER_REPLY2 = 'user_reply2'
     """User replies to who the most column"""
 
-    URL_NO = 'url_no'
+    URL_NO = 'Number of URLs'#,'url_no'
     """Number of URLs column"""
 
-    LOCATION_NO = 'location_no'
+    LOCATION_NO = 'Number of shared locations'#'location_no'
     """Number of locations column"""
 
-    FILE_NO = 'file_no'
+    FILE_NO = 'Number of shared files'#'file_no'
     """Number of files column"""
 
     OUT_DEGREE = 'out_degree'
@@ -78,6 +84,12 @@ class ColnamesDf:
     EMOJI_Fav = 'emoji_fav'
     """Favorite emojies column"""
 
+    DESCRIPTION = 'Description'
+    """Variable column in melted dataframe"""
+
+    VALUE = 'Value'
+    """Value column in melted dataframe"""
+
 
 COLNAMES_DF = ColnamesDf()
 
@@ -372,20 +384,20 @@ def parse_zipfile(log_error, zfile):
         Regular expression
     Returns
     -------
-    list
-        A list of pandas.DataFrames which include the content of chat files.
+    pandas.DataFrame
+        A pandas.DataFrames which include the content of the chat file.
     """
-    results = []
     for name in zfile.namelist():
         if HIDDEN_FILE_RE.match(name):
             continue
         if not FILE_RE.match(name):
             continue
-        chats = decode_chat(log_error,zfile.read(name),name)
-        results.append(chats)
-    if len(results)==0:
+        chat = decode_chat(log_error,zfile.read(name),name)
+
+    if chat is None:
         log_error("No valid chat file is available")
-    return results
+
+    return chat
 
 # *** test related function ***
 
@@ -405,9 +417,9 @@ def input_df(data_path):
     log_error = errors.append
     fp = os.path.join(data_path, "whatsapp_chat.zip")
     zfile = zipfile.ZipFile(fp)
-    chats = parse_zipfile(log_error, zfile)
-    participants = extract_participants_features(chats, anonymize=False)
-    return chats[0], participants[0]
+    chat = parse_zipfile(log_error, zfile)
+    participants = extract_participants_features(chat, anonymize=False)
+    return chat, participants
 
 # *** analysis functions ***
 
@@ -477,6 +489,39 @@ def anonymize_participants(df_participants):
     return df_participants
 
 
+def get_df_per_participant(df, anonymize):
+    """Generate one dataframe for each participant .
+        Parameter
+        ----------
+        df : pandas.DataFrame
+           A DataFrame which includes participants and their features
+
+        anonymize : bool
+            Indicates if usernames should be anonymized
+        Returns
+        -------
+        list pandas.DataFrame
+            A list of pandas.DataFrame. Each data frame includes the description of features and their values extracted
+            from a specific participant
+        """
+    results = []
+    df_melt = pd.melt(df, id_vars=[COLNAMES_DF.USERNAME], value_vars=[COLNAMES_DF.WORDS_NO, COLNAMES_DF.MESSAGE_NO,
+                                                                      COLNAMES_DF.FirstMessage, COLNAMES_DF.LastMessage,
+                                                                      COLNAMES_DF.URL_NO, COLNAMES_DF.FILE_NO,
+                                                                      COLNAMES_DF.LOCATION_NO],
+                      var_name=COLNAMES_DF.DESCRIPTION, value_name=COLNAMES_DF.VALUE)
+
+    usernames = set(df_melt[COLNAMES_DF.USERNAME])
+    for u in usernames:
+        df_user = df_melt[(df_melt[COLNAMES_DF.USERNAME] == u) &
+                          df_melt[COLNAMES_DF.VALUE] != 0]
+        # if anonymize:
+        #     df_user = anonymize_participants(df_user)
+        results.append(df_user)
+
+    return results
+
+
 def get_participants_features(df_chat):
     """Calculate participant features from the given chat.
     Parameter
@@ -534,13 +579,30 @@ def get_participants_features(df_chat):
 
     return df_participants
 
-
-def extract_participants_features(chats, anonymize=True):
+def remove_system_messages(chat):
+    """Removes system messages from chat
+    Parameters
+    ----------
+    chat : pandas.DataFrame
+        A DataFrame that includes chat data
+    Returns
+    -------
+    pandas.DataFrame
+        A filtered dataframe
+    """
+    print(chat.loc[0,COLNAMES_DF.MESSAGE])
+    print(SYSTEM_MESSAGES[1])
+    for m in SYSTEM_MESSAGES:
+        group_name = chat.loc[chat[COLNAMES_DF.MESSAGE]==m,COLNAMES_DF.USERNAME]
+        print(group_name)
+    return chat
+
+def extract_participants_features(chat, anonymize=True):
     """Parse the given zip file.
     Parameters
     ----------
-    chats : list
-        List of DataFrames including chat data
+    chat : pandas.DataFrame
+        A DataFrame that includes chat data
     anonymize : bool
         Indicates if usernames should be anonymized
     Returns
@@ -548,12 +610,9 @@ def extract_participants_features(chats, anonymize=True):
     list
         A list of DataFrames which include participant features
     """
-    results = []
-    for chat in chats:
-        df = get_participants_features(chat)
-        if anonymize:
-            df = anonymize_participants(df)
-        results.append(df)
+
+    df = get_participants_features(chat)
+    results = get_df_per_participant(df, anonymize)
     return results
 
 # ***** end of analysis functions *****
@@ -570,11 +629,12 @@ def format_results(df_list):
     """
     results = []
     for df in df_list:
+        user_name = pd.unique(df[COLNAMES_DF.USERNAME])[0]
         results.append(
             {
-                "id": "overview",
-                "title": "The following data is extracted from the file:",
-                "data_frame": df
+                "id": user_name,#"overview",
+                "title": user_name,#"The following data is extracted from the file:",
+                "data_frame": df[[COLNAMES_DF.DESCRIPTION,COLNAMES_DF.VALUE]].reset_index(drop=True)
             }
         )
     return results
@@ -610,23 +670,25 @@ def process(file_data):
     errors = []
     log_error = errors.append
     zfile = None
-    chats = []
+    #chats = []
     try:
         zfile = zipfile.ZipFile(file_data)
     except:
         if FILE_RE.match(file_data.name):
-            zfile = open(file_data, encoding="utf8")
-            chat = parse_chat(log_error, zfile.read())
-            chats.append(chat)
+            tfile = open(file_data, encoding="utf8")
+            chat = parse_chat(log_error, tfile.read())
+            #chats.append(chat)
         else:
             log_error("There is not a valid file format.")
             return [format_errors(errors)]
     else:
-        chats = parse_zipfile(log_error, zfile)
+        chat = parse_zipfile(log_error, zfile)
     if errors:
         return [format_errors(errors)]
 
-    participants = extract_participants_features(chats)
+    print(chat)
+    chat = remove_system_messages(chat)
+    participants = extract_participants_features(chat)
     formatted_results = format_results(participants)
 
     return formatted_results

From 9a17cd794c86d8f0b17630bc6b4327421b5c933c Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Sat, 2 Jul 2022 00:37:57 +0200
Subject: [PATCH 21/42] Remove group's name

---
 data_extractor/tests/data/_chat.txt | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/data_extractor/tests/data/_chat.txt b/data_extractor/tests/data/_chat.txt
index 2ffeac2..c43d07a 100644
--- a/data_extractor/tests/data/_chat.txt
+++ b/data_extractor/tests/data/_chat.txt
@@ -1,11 +1,9 @@
-[16/03/2022, 15:20:25] user1: Hi shiva!
-[16/03/2022, 15:25:38] user2: Hi 👋
-[16/03/2022, 15:26:48] user3: Hoi!
-[16/03/2022, 18:39:29] user2: https://youtu.be/KBmUTY6mK_E
-[16/03/2022, 18:35:51] user1: ‎Location: https://maps.google.com/?q=52.089451,5.108469
-[20/03/2022, 20:08:51] user4‬: I’m about to generate some very random messages so that I can make some screenshots for the explanation to participants
-[24/03/2022, 20:19:38] user1‬: @user3 if you remove your Profile picture for a moment I will redo the screenshots 😁
-[26/03/2022, 18:52:15] user2: Well done Utrecht 😁
-Opkomst provinciesteden rond 20 procent, Utrecht hoogste opkomst van G4
-https://nos.nl/l/2421368#UPDATE-container-60145354
-[14/07/2020, 22:05:54] user4: 👍Bedankt
\ No newline at end of file
+[16/03/2022, 15:20:25] person1: Hi shiva!
+[16/03/2022, 15:25:38] person2: Hi 👋
+[16/03/2022, 15:26:48] person3: Hoi!
+[16/03/2022, 18:39:29] person2: https://youtu.be/KBmUTY6mK_E
+[16/03/2022, 18:35:51] person1: ‎Location: https://maps.google.com/?q=52.089451,5.108469
+[20/03/2022, 20:08:51] person4: I’m about to generate some very random messages so that I can make some screenshots for the explanation to participants
+[24/03/2022, 20:19:38] person1: @user3 if you remove your Profile picture for a moment I will redo the screenshots 😁
+[26/03/2022, 18:52:15] person2: Well done Utrecht 😁
+[14/07/2020, 22:05:54] person4: 👍Bedankt
\ No newline at end of file

From 435e111a49202ef3893a8e6ff971fbc3f9924b87 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Sat, 2 Jul 2022 00:45:28 +0200
Subject: [PATCH 22/42] adapt to new output format

---
 data_extractor/tests/test_whatsapp_chat.py | 86 ++++++++++++++++------
 1 file changed, 65 insertions(+), 21 deletions(-)

diff --git a/data_extractor/tests/test_whatsapp_chat.py b/data_extractor/tests/test_whatsapp_chat.py
index 492c977..b10b0b6 100644
--- a/data_extractor/tests/test_whatsapp_chat.py
+++ b/data_extractor/tests/test_whatsapp_chat.py
@@ -1,30 +1,35 @@
-from whatsapp_chat import process
-from whatsapp_chat import anonymize_participants
+from data_extractor.whatsapp_chat import process
+from data_extractor.whatsapp_chat import anonymize_participants
+from data_extractor.whatsapp_chat import get_df_per_participant
 from pathlib import Path
 import pandas as pd
 from pandas.testing import assert_frame_equal
 
 
+
+
 DATA_PATH = Path(__file__).parent / "data"
 
 EXPECTED = [
-    {'username': 'user1', 'total_words_no': 20, 'url_no': 1, 'location_no': 1, 'file_no': 0, 'message_no': 3,
-     'first_message_date': '2022-03-16 15:20:25', 'last_message_date': '2022-03-24 20:19:38', 'out_degree': 2,
-     'in_degree': 3, 'user_reply2': 'user2', 'reply_2_user': 'user2'},
+    {'username': 'person1', 'Total number of words': 20, 'Number of URLs': 1, 'Number of shared locations': 1, 'file_no': 0, 'Number of messages': 3,
+     'Date first message': pd.to_datetime('2022-03-16 15:20:25'), 'Date last message': pd.to_datetime('2022-03-24 20:19:38')},
 
-    {'username': 'user2', 'total_words_no': 18, 'url_no': 2, 'location_no': 0, 'file_no': 0, 'message_no': 3,
-     'first_message_date': '2022-03-16 15:25:38', 'last_message_date': '2022-03-26 18:52:15', 'out_degree': 3,
-     'in_degree': 3, 'user_reply2': 'user1', 'reply_2_user': 'user1'},
+    {'username': 'person2', 'Total number of words': 7, 'Number of URLs': 1, 'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 3,
+     'Date first message': pd.to_datetime('2022-03-16 15:25:38'), 'Date last message': pd.to_datetime('2022-03-26 18:52:15')},
 
-    {'username': 'user3', 'total_words_no': 1, 'url_no': 0, 'location_no': 0, 'file_no': 0, 'message_no': 1,
-     'first_message_date': '2022-03-16 15:26:48', 'last_message_date': '2022-03-16 15:26:48', 'out_degree': 1,
-     'in_degree': 1, 'user_reply2': 'user2', 'reply_2_user': 'user2'},
+    {'username': 'person3', 'Total number of words': 1, 'Number of URLs': 0, 'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 1,
+     'Date first message': pd.to_datetime('2022-03-16 15:26:48'), 'Date last message': pd.to_datetime('2022-03-16 15:26:48')},
 
-    {'username': 'user4', 'total_words_no': 21, 'url_no': 0, 'location_no': 0, 'file_no': 0, 'message_no': 2,
-     'first_message_date': '2020-07-14 22:05:54', 'last_message_date': '2022-03-20 20:08:51', 'out_degree': 2,
-     'in_degree': 1, 'user_reply2': 'user1', 'reply_2_user': 'user1'}
+    {'username': 'person4', 'Total number of words': 21, 'Number of URLs': 0, 'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 2,
+     'Date first message': pd.to_datetime('2020-07-14 22:05:54'), 'Date last message': pd.to_datetime('2022-03-20 20:08:51')}
 ]
 
+# EXPECTED_1 = {'Description': ['Total number of words', 'Number of messages', 'Date first message', 'Date last message',
+#                               'Number of URLs', 'Number of shared locations'],
+#               'Value': [20, 3, pd.to_datetime('2022-03-16 15:20:25'), pd.to_datetime('2022-03-24 20:19:38'), 1, 1]}
+#
+# df_expected_1 = pd.DataFrame(data=EXPECTED_1)
+
 
 def test_process():
     """ Test process function.
@@ -36,15 +41,54 @@ def test_process():
 
     df_expected = pd.DataFrame(EXPECTED)
     df_expected = anonymize_participants(df_expected)
-    df_expected['message_no'] = df_expected['message_no'].astype('int64')
-    df_expected['url_no'] = df_expected['url_no'].astype('int32')
-    df_expected['location_no'] = df_expected['location_no'].astype('int32')
+    df_expected['Number of messages'] = df_expected['Number of messages'].astype('int64')
+    df_expected['Number of URLs'] = df_expected['Number of URLs'].astype('int32')
+    df_expected['Number of shared locations'] = df_expected['Number of shared locations'].astype('int32')
     df_expected['file_no'] = df_expected['file_no'].astype('int32')
 
+    results = []
+    df_melt = pd.melt(df_expected, id_vars=["username"],
+                      value_vars=["Total number of words", "Number of messages", "Date first message", "Date last message",
+                                  "Number of URLs", "file_no", "Number of shared locations"], var_name='Description', value_name='Value')
+
+    usernames = df_melt["username"].unique()
+    for u in usernames:
+        df_user = df_melt[(df_melt["username"] == u) & df_melt["Value"] != 0]
+        results.append(df_user)
+
+    expected_results = []
+    for df in results:
+        user_name = pd.unique(df["username"])[0]
+        expected_results.append(
+            {
+                "id": user_name,  # "overview",
+                "title": user_name,  # "The following data is extracted from the file:",
+                "data_frame": df[["Description", "Value"]].reset_index(drop=True)
+            }
+        )
+    # print(type(expected_results[0]["data_frame"]))
+    # print(expected_results[1]["data_frame"])
+    # print('******')
+
     # result = process(DATA_PATH.joinpath("whatsapp_chat.zip"))
-    result = process(DATA_PATH.joinpath("_chat.txt"))
-    assert len(result) == 1
-    df_result = result[0]["data_frame"]
-    assert_frame_equal(df_result, df_expected)
+    df_result = process(DATA_PATH.joinpath("_chat.txt"))
+    # print(df_result[1]["data_frame"])
+    assert_frame_equal(df_result[0]["data_frame"], expected_results[0]["data_frame"])
+    assert_frame_equal(df_result[1]["data_frame"], expected_results[1]["data_frame"])
+    assert_frame_equal(df_result[2]["data_frame"], expected_results[2]["data_frame"])
+    assert_frame_equal(df_result[3]["data_frame"], expected_results[3]["data_frame"])
+
+    # print(type(df_result[0]["data_frame"]))
+    # print(df_result[0]["data_frame"])
+    # print(df_expected_1)
+    #
+    # # assert len(result_file) == 1
+    #
+    #
+    # assert_frame_equal(df_result[0]["data_frame"], df_expected_1)
+
+
+if __name__ == "__main__":
+    test_process()
 
 

From eb6a5962dd254d4105f8aa8b9f4fe7e7a0296213 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Sat, 2 Jul 2022 00:49:38 +0200
Subject: [PATCH 23/42] Replace set with unique

---
 data_extractor/whatsapp_chat/__init__.py | 27 +++++++++++++++---------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index e3f6577..d679838 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -481,11 +481,17 @@ def anonymize_participants(df_participants):
     # df_participants[COLNAMES_DF.USER_REPLY2] = df_participants[COLNAMES_DF.USER_REPLY2].apply(lambda u: anonym_txt(u,salt))
     # df_participants[['username', 'user_reply2']] = df_participants[['username', 'user_reply2']].stack().rank(method='dense').unstack()
 
-    stacked = df_participants[['username', 'user_reply2', 'reply_2_user']].stack()
-    df_participants[['username', 'user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0],
-                                                                             index=stacked.index).unstack()
-    df_participants[['username', 'user_reply2', 'reply_2_user']] = 'person' + df_participants[['username', 'user_reply2',
-                                                                                        'reply_2_user']].astype(str)
+    # stacked = df_participants[['username', 'user_reply2', 'reply_2_user']].stack()
+    # df_participants[['username', 'user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0],
+    #                                                                          index=stacked.index).unstack()
+    # df_participants[['username', 'user_reply2', 'reply_2_user']] = 'person' + df_participants[['username', 'user_reply2',
+    #                                                                                     'reply_2_user']].astype(str)
+    #
+
+    df_participants['username'] = pd.factorize(df_participants.username)[0] + 1
+    df_participants['username'] = 'person' + df_participants['username'].astype(str)
+
+
     return df_participants
 
 
@@ -511,7 +517,8 @@ def get_df_per_participant(df, anonymize):
                                                                       COLNAMES_DF.LOCATION_NO],
                       var_name=COLNAMES_DF.DESCRIPTION, value_name=COLNAMES_DF.VALUE)
 
-    usernames = set(df_melt[COLNAMES_DF.USERNAME])
+    # usernames = set(df_melt[COLNAMES_DF.USERNAME])
+    usernames = df_melt[COLNAMES_DF.USERNAME].unique()
     for u in usernames:
         df_user = df_melt[(df_melt[COLNAMES_DF.USERNAME] == u) &
                           df_melt[COLNAMES_DF.VALUE] != 0]
@@ -590,11 +597,11 @@ def remove_system_messages(chat):
     pandas.DataFrame
         A filtered dataframe
     """
-    print(chat.loc[0,COLNAMES_DF.MESSAGE])
-    print(SYSTEM_MESSAGES[1])
+    # print(chat.loc[0,COLNAMES_DF.MESSAGE])
+    # print(SYSTEM_MESSAGES[1])
     for m in SYSTEM_MESSAGES:
         group_name = chat.loc[chat[COLNAMES_DF.MESSAGE]==m,COLNAMES_DF.USERNAME]
-        print(group_name)
+        # print(group_name)
     return chat
 
 def extract_participants_features(chat, anonymize=True):
@@ -686,7 +693,7 @@ def process(file_data):
     if errors:
         return [format_errors(errors)]
 
-    print(chat)
+    # print(chat)
     chat = remove_system_messages(chat)
     participants = extract_participants_features(chat)
     formatted_results = format_results(participants)

From 5014770e7bdffcdbb43a08928d7c42730f488be9 Mon Sep 17 00:00:00 2001
From: parisa-zahedi <parisa.zahedi@gmail.com>
Date: Mon, 4 Jul 2022 00:53:37 +0200
Subject: [PATCH 24/42] remove system messages

---
 data_extractor/whatsapp_chat/__init__.py | 71 ++++++++----------------
 1 file changed, 24 insertions(+), 47 deletions(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index e3f6577..82023a5 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -10,9 +10,6 @@
 import hashlib
 import zipfile
 from pathlib import Path
-#from nltk.corpus import stopwords
-#from sklearn.feature_extraction.text import TfidfVectorizer
-
 
 URL_PATTERN = r'(https?://\S+)'
 LOCATION_PATTERN = r'(Location: https?://\S+)'
@@ -20,10 +17,7 @@
 FILE_RE = re.compile(r".*.txt$")
 HIDDEN_FILE_RE = re.compile(r".*__MACOSX*")
 
-SYSTEM_MESSAGES=[
-    'Messages and calls are end-to-end encrypted. No one outside of this chat, not even WhatsApp, can read or listen to them.',
-    'Berichten en gesprekken worden end-to-end versleuteld. Niemand buiten deze chat kan ze lezen of beluisteren, zelfs WhatsApp niet.'
-]
+SYSTEM_MESSAGES = ['end-to-end','WhatsApp']
 hformats = ['%m/%d/%y, %H:%M - %name:', '[%d/%m/%y, %H:%M:%S] %name:', '%d-%m-%y %H:%M - %name:', '[%d-%m-%y %H:%M:%S] %name:']
 
 
@@ -481,15 +475,13 @@ def anonymize_participants(df_participants):
     # df_participants[COLNAMES_DF.USER_REPLY2] = df_participants[COLNAMES_DF.USER_REPLY2].apply(lambda u: anonym_txt(u,salt))
     # df_participants[['username', 'user_reply2']] = df_participants[['username', 'user_reply2']].stack().rank(method='dense').unstack()
 
-    stacked = df_participants[['username', 'user_reply2', 'reply_2_user']].stack()
-    df_participants[['username', 'user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0],
-                                                                             index=stacked.index).unstack()
-    df_participants[['username', 'user_reply2', 'reply_2_user']] = 'person' + df_participants[['username', 'user_reply2',
-                                                                                        'reply_2_user']].astype(str)
+    stacked = df_participants[['username']].stack() #, 'user_reply2', 'reply_2_user'
+    df_participants[['username']] = pd.Series(stacked.factorize()[0], index=stacked.index).unstack() #, 'user_reply2', 'reply_2_user'
+    df_participants[['username']] = 'person' + df_participants[['username']].astype(str)
     return df_participants
 
 
-def get_df_per_participant(df, anonymize):
+def get_df_per_participant(df):
     """Generate one dataframe for each participant .
         Parameter
         ----------
@@ -511,12 +503,11 @@ def get_df_per_participant(df, anonymize):
                                                                       COLNAMES_DF.LOCATION_NO],
                       var_name=COLNAMES_DF.DESCRIPTION, value_name=COLNAMES_DF.VALUE)
 
-    usernames = set(df_melt[COLNAMES_DF.USERNAME])
+    usernames = sorted(set(df_melt[COLNAMES_DF.USERNAME]))
     for u in usernames:
         df_user = df_melt[(df_melt[COLNAMES_DF.USERNAME] == u) &
                           df_melt[COLNAMES_DF.VALUE] != 0]
-        # if anonymize:
-        #     df_user = anonymize_participants(df_user)
+
         results.append(df_user)
 
     return results
@@ -560,23 +551,6 @@ def get_participants_features(df_chat):
         COLNAMES_DF.LastMessage: 'max'
     }).reset_index()
 
-    response_matrix = get_response_matrix(df_chat)
-    out_degree = response_matrix.sum(axis=1)
-    in_degree = response_matrix.T.sum(axis=1)
-    user_reply2 = response_matrix.idxmax(axis=1)
-    reply2_user = response_matrix.T.idxmax(axis=1)
-
-    response_matrix[COLNAMES_DF.OUT_DEGREE] = out_degree
-    response_matrix[COLNAMES_DF.IN_DEGREE] = in_degree
-    response_matrix[COLNAMES_DF.USER_REPLY2] = user_reply2
-    response_matrix[COLNAMES_DF.REPLY_2USER] = reply2_user
-    response_matrix.index.name = COLNAMES_DF.USERNAME
-    response_matrix = response_matrix.loc[:,
-                      [COLNAMES_DF.OUT_DEGREE, COLNAMES_DF.IN_DEGREE, COLNAMES_DF.USER_REPLY2, COLNAMES_DF.REPLY_2USER]]
-    response_matrix = response_matrix.reset_index()
-
-    df_participants = pd.merge(df_participants, response_matrix, how="left", on=COLNAMES_DF.USERNAME, validate="1:1")
-
     return df_participants
 
 def remove_system_messages(chat):
@@ -590,11 +564,13 @@ def remove_system_messages(chat):
     pandas.DataFrame
         A filtered dataframe
     """
-    print(chat.loc[0,COLNAMES_DF.MESSAGE])
-    print(SYSTEM_MESSAGES[1])
-    for m in SYSTEM_MESSAGES:
-        group_name = chat.loc[chat[COLNAMES_DF.MESSAGE]==m,COLNAMES_DF.USERNAME]
-        print(group_name)
+
+    message0 = chat.loc[0, COLNAMES_DF.MESSAGE]
+    is_system_message = True if all(s in message0 for s in SYSTEM_MESSAGES) else False
+    if is_system_message:
+        group_name = chat.loc[0, COLNAMES_DF.USERNAME]
+        chat = chat.loc[chat[COLNAMES_DF.USERNAME] != group_name,]
+
     return chat
 
 def extract_participants_features(chat, anonymize=True):
@@ -612,7 +588,10 @@ def extract_participants_features(chat, anonymize=True):
     """
 
     df = get_participants_features(chat)
-    results = get_df_per_participant(df, anonymize)
+    if anonymize:
+        df= anonymize_participants(df)
+
+    results = get_df_per_participant(df)
     return results
 
 # ***** end of analysis functions *****
@@ -655,29 +634,28 @@ def format_errors(errors):
 
 
 def process(file_data):
-    """Convert whatsapp chat_file.zip to participants dataframe.
+    """Convert whatsapp chat file to participant dataframes.
     This is the main function which extracts the participants
-    information from the row chat_file.zip provided by data-donators.
+    information from the row chat file provided by data-donators.
     Parameters
     ----------
     file_data : str
-        The path of the chat_file.zip
+        The path of the chat file. It can be in zip or txt format.
     Returns
     -------
     pandas.dataframe
-        Extracted data from the chat_file
+        Extracted data from the chat file
     """
     errors = []
     log_error = errors.append
-    zfile = None
-    #chats = []
+
     try:
         zfile = zipfile.ZipFile(file_data)
     except:
         if FILE_RE.match(file_data.name):
             tfile = open(file_data, encoding="utf8")
             chat = parse_chat(log_error, tfile.read())
-            #chats.append(chat)
+
         else:
             log_error("There is not a valid file format.")
             return [format_errors(errors)]
@@ -686,7 +664,6 @@ def process(file_data):
     if errors:
         return [format_errors(errors)]
 
-    print(chat)
     chat = remove_system_messages(chat)
     participants = extract_participants_features(chat)
     formatted_results = format_results(participants)

From 986fc0849e9a88139f5cac21b6418d8137120ed0 Mon Sep 17 00:00:00 2001
From: parisa-zahedi <parisa.zahedi@gmail.com>
Date: Thu, 7 Jul 2022 12:53:10 +0200
Subject: [PATCH 25/42] add reply_2_user and user_reply_2

---
 data_extractor/whatsapp_chat/__init__.py | 36 ++++++++++++++++--------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index 82023a5..52727b5 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -36,16 +36,16 @@ class ColnamesDf:
     MESSAGE_LENGTH = 'message_length'
     """Message length column"""
 
-    FirstMessage = 'Date first message' #'first_message_date'
+    FirstMessage = 'Date first message'
     """Date of first message column"""
 
-    LastMessage = 'Date last message' #'last_message_date'
+    LastMessage = 'Date last message'
     """Date of last message column"""
 
-    MESSAGE_NO = 'Number of messages' #'message_no'
+    MESSAGE_NO = 'Number of messages'
     """Number of Message  column"""
 
-    WORDS_NO = 'Total number of words' #'total_words_no'
+    WORDS_NO = 'Total number of words'
     """Total number of words  column"""
 
     REPLY_2USER = 'reply_2_user'
@@ -57,13 +57,13 @@ class ColnamesDf:
     USER_REPLY2 = 'user_reply2'
     """User replies to who the most column"""
 
-    URL_NO = 'Number of URLs'#,'url_no'
+    URL_NO = 'Number of URLs'
     """Number of URLs column"""
 
-    LOCATION_NO = 'Number of shared locations'#'location_no'
+    LOCATION_NO = 'Number of shared locations'
     """Number of locations column"""
 
-    FILE_NO = 'Number of shared files'#'file_no'
+    FILE_NO = 'Number of shared files'
     """Number of files column"""
 
     OUT_DEGREE = 'out_degree'
@@ -475,9 +475,9 @@ def anonymize_participants(df_participants):
     # df_participants[COLNAMES_DF.USER_REPLY2] = df_participants[COLNAMES_DF.USER_REPLY2].apply(lambda u: anonym_txt(u,salt))
     # df_participants[['username', 'user_reply2']] = df_participants[['username', 'user_reply2']].stack().rank(method='dense').unstack()
 
-    stacked = df_participants[['username']].stack() #, 'user_reply2', 'reply_2_user'
-    df_participants[['username']] = pd.Series(stacked.factorize()[0], index=stacked.index).unstack() #, 'user_reply2', 'reply_2_user'
-    df_participants[['username']] = 'person' + df_participants[['username']].astype(str)
+    stacked = df_participants[['username','user_reply2', 'reply_2_user']].stack()
+    df_participants[['username','user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0], index=stacked.index).unstack()
+    df_participants[['username','user_reply2', 'reply_2_user']] = 'person' + df_participants[['username','user_reply2', 'reply_2_user']].astype(str)
     return df_participants
 
 
@@ -500,7 +500,9 @@ def get_df_per_participant(df):
     df_melt = pd.melt(df, id_vars=[COLNAMES_DF.USERNAME], value_vars=[COLNAMES_DF.WORDS_NO, COLNAMES_DF.MESSAGE_NO,
                                                                       COLNAMES_DF.FirstMessage, COLNAMES_DF.LastMessage,
                                                                       COLNAMES_DF.URL_NO, COLNAMES_DF.FILE_NO,
-                                                                      COLNAMES_DF.LOCATION_NO],
+                                                                      COLNAMES_DF.LOCATION_NO,
+                                                                      COLNAMES_DF.REPLY_2USER,
+                                                                      COLNAMES_DF.USER_REPLY2],
                       var_name=COLNAMES_DF.DESCRIPTION, value_name=COLNAMES_DF.VALUE)
 
     usernames = sorted(set(df_melt[COLNAMES_DF.USERNAME]))
@@ -551,6 +553,18 @@ def get_participants_features(df_chat):
         COLNAMES_DF.LastMessage: 'max'
     }).reset_index()
 
+    response_matrix = get_response_matrix(df_chat)
+    user_reply2 = response_matrix.idxmax(axis=1)
+    reply2_user = response_matrix.T.idxmax(axis=1)
+
+    response_matrix[COLNAMES_DF.USER_REPLY2] = user_reply2
+    response_matrix[COLNAMES_DF.REPLY_2USER] = reply2_user
+    response_matrix.index.name = COLNAMES_DF.USERNAME
+    response_matrix = response_matrix.loc[:,[COLNAMES_DF.USER_REPLY2, COLNAMES_DF.REPLY_2USER]]
+    response_matrix = response_matrix.reset_index()
+
+    df_participants = pd.merge(df_participants, response_matrix, how="left", on=COLNAMES_DF.USERNAME, validate="1:1")
+
     return df_participants
 
 def remove_system_messages(chat):

From faedf246b5562d668948659ea76c1b689475587c Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Thu, 7 Jul 2022 13:47:03 +0200
Subject: [PATCH 26/42] Add reply_2

---
 data_extractor/whatsapp_chat/__init__.py | 86 ++++++++++--------------
 1 file changed, 35 insertions(+), 51 deletions(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index d679838..52727b5 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -10,9 +10,6 @@
 import hashlib
 import zipfile
 from pathlib import Path
-#from nltk.corpus import stopwords
-#from sklearn.feature_extraction.text import TfidfVectorizer
-
 
 URL_PATTERN = r'(https?://\S+)'
 LOCATION_PATTERN = r'(Location: https?://\S+)'
@@ -20,10 +17,7 @@
 FILE_RE = re.compile(r".*.txt$")
 HIDDEN_FILE_RE = re.compile(r".*__MACOSX*")
 
-SYSTEM_MESSAGES=[
-    'Messages and calls are end-to-end encrypted. No one outside of this chat, not even WhatsApp, can read or listen to them.',
-    'Berichten en gesprekken worden end-to-end versleuteld. Niemand buiten deze chat kan ze lezen of beluisteren, zelfs WhatsApp niet.'
-]
+SYSTEM_MESSAGES = ['end-to-end','WhatsApp']
 hformats = ['%m/%d/%y, %H:%M - %name:', '[%d/%m/%y, %H:%M:%S] %name:', '%d-%m-%y %H:%M - %name:', '[%d-%m-%y %H:%M:%S] %name:']
 
 
@@ -42,16 +36,16 @@ class ColnamesDf:
     MESSAGE_LENGTH = 'message_length'
     """Message length column"""
 
-    FirstMessage = 'Date first message' #'first_message_date'
+    FirstMessage = 'Date first message'
     """Date of first message column"""
 
-    LastMessage = 'Date last message' #'last_message_date'
+    LastMessage = 'Date last message'
     """Date of last message column"""
 
-    MESSAGE_NO = 'Number of messages' #'message_no'
+    MESSAGE_NO = 'Number of messages'
     """Number of Message  column"""
 
-    WORDS_NO = 'Total number of words' #'total_words_no'
+    WORDS_NO = 'Total number of words'
     """Total number of words  column"""
 
     REPLY_2USER = 'reply_2_user'
@@ -63,13 +57,13 @@ class ColnamesDf:
     USER_REPLY2 = 'user_reply2'
     """User replies to who the most column"""
 
-    URL_NO = 'Number of URLs'#,'url_no'
+    URL_NO = 'Number of URLs'
     """Number of URLs column"""
 
-    LOCATION_NO = 'Number of shared locations'#'location_no'
+    LOCATION_NO = 'Number of shared locations'
     """Number of locations column"""
 
-    FILE_NO = 'Number of shared files'#'file_no'
+    FILE_NO = 'Number of shared files'
     """Number of files column"""
 
     OUT_DEGREE = 'out_degree'
@@ -481,21 +475,13 @@ def anonymize_participants(df_participants):
     # df_participants[COLNAMES_DF.USER_REPLY2] = df_participants[COLNAMES_DF.USER_REPLY2].apply(lambda u: anonym_txt(u,salt))
     # df_participants[['username', 'user_reply2']] = df_participants[['username', 'user_reply2']].stack().rank(method='dense').unstack()
 
-    # stacked = df_participants[['username', 'user_reply2', 'reply_2_user']].stack()
-    # df_participants[['username', 'user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0],
-    #                                                                          index=stacked.index).unstack()
-    # df_participants[['username', 'user_reply2', 'reply_2_user']] = 'person' + df_participants[['username', 'user_reply2',
-    #                                                                                     'reply_2_user']].astype(str)
-    #
-
-    df_participants['username'] = pd.factorize(df_participants.username)[0] + 1
-    df_participants['username'] = 'person' + df_participants['username'].astype(str)
-
-
+    stacked = df_participants[['username','user_reply2', 'reply_2_user']].stack()
+    df_participants[['username','user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0], index=stacked.index).unstack()
+    df_participants[['username','user_reply2', 'reply_2_user']] = 'person' + df_participants[['username','user_reply2', 'reply_2_user']].astype(str)
     return df_participants
 
 
-def get_df_per_participant(df, anonymize):
+def get_df_per_participant(df):
     """Generate one dataframe for each participant .
         Parameter
         ----------
@@ -514,16 +500,16 @@ def get_df_per_participant(df, anonymize):
     df_melt = pd.melt(df, id_vars=[COLNAMES_DF.USERNAME], value_vars=[COLNAMES_DF.WORDS_NO, COLNAMES_DF.MESSAGE_NO,
                                                                       COLNAMES_DF.FirstMessage, COLNAMES_DF.LastMessage,
                                                                       COLNAMES_DF.URL_NO, COLNAMES_DF.FILE_NO,
-                                                                      COLNAMES_DF.LOCATION_NO],
+                                                                      COLNAMES_DF.LOCATION_NO,
+                                                                      COLNAMES_DF.REPLY_2USER,
+                                                                      COLNAMES_DF.USER_REPLY2],
                       var_name=COLNAMES_DF.DESCRIPTION, value_name=COLNAMES_DF.VALUE)
 
-    # usernames = set(df_melt[COLNAMES_DF.USERNAME])
-    usernames = df_melt[COLNAMES_DF.USERNAME].unique()
+    usernames = sorted(set(df_melt[COLNAMES_DF.USERNAME]))
     for u in usernames:
         df_user = df_melt[(df_melt[COLNAMES_DF.USERNAME] == u) &
                           df_melt[COLNAMES_DF.VALUE] != 0]
-        # if anonymize:
-        #     df_user = anonymize_participants(df_user)
+
         results.append(df_user)
 
     return results
@@ -568,18 +554,13 @@ def get_participants_features(df_chat):
     }).reset_index()
 
     response_matrix = get_response_matrix(df_chat)
-    out_degree = response_matrix.sum(axis=1)
-    in_degree = response_matrix.T.sum(axis=1)
     user_reply2 = response_matrix.idxmax(axis=1)
     reply2_user = response_matrix.T.idxmax(axis=1)
 
-    response_matrix[COLNAMES_DF.OUT_DEGREE] = out_degree
-    response_matrix[COLNAMES_DF.IN_DEGREE] = in_degree
     response_matrix[COLNAMES_DF.USER_REPLY2] = user_reply2
     response_matrix[COLNAMES_DF.REPLY_2USER] = reply2_user
     response_matrix.index.name = COLNAMES_DF.USERNAME
-    response_matrix = response_matrix.loc[:,
-                      [COLNAMES_DF.OUT_DEGREE, COLNAMES_DF.IN_DEGREE, COLNAMES_DF.USER_REPLY2, COLNAMES_DF.REPLY_2USER]]
+    response_matrix = response_matrix.loc[:,[COLNAMES_DF.USER_REPLY2, COLNAMES_DF.REPLY_2USER]]
     response_matrix = response_matrix.reset_index()
 
     df_participants = pd.merge(df_participants, response_matrix, how="left", on=COLNAMES_DF.USERNAME, validate="1:1")
@@ -597,11 +578,13 @@ def remove_system_messages(chat):
     pandas.DataFrame
         A filtered dataframe
     """
-    # print(chat.loc[0,COLNAMES_DF.MESSAGE])
-    # print(SYSTEM_MESSAGES[1])
-    for m in SYSTEM_MESSAGES:
-        group_name = chat.loc[chat[COLNAMES_DF.MESSAGE]==m,COLNAMES_DF.USERNAME]
-        # print(group_name)
+
+    message0 = chat.loc[0, COLNAMES_DF.MESSAGE]
+    is_system_message = True if all(s in message0 for s in SYSTEM_MESSAGES) else False
+    if is_system_message:
+        group_name = chat.loc[0, COLNAMES_DF.USERNAME]
+        chat = chat.loc[chat[COLNAMES_DF.USERNAME] != group_name,]
+
     return chat
 
 def extract_participants_features(chat, anonymize=True):
@@ -619,7 +602,10 @@ def extract_participants_features(chat, anonymize=True):
     """
 
     df = get_participants_features(chat)
-    results = get_df_per_participant(df, anonymize)
+    if anonymize:
+        df= anonymize_participants(df)
+
+    results = get_df_per_participant(df)
     return results
 
 # ***** end of analysis functions *****
@@ -662,29 +648,28 @@ def format_errors(errors):
 
 
 def process(file_data):
-    """Convert whatsapp chat_file.zip to participants dataframe.
+    """Convert whatsapp chat file to participant dataframes.
     This is the main function which extracts the participants
-    information from the row chat_file.zip provided by data-donators.
+    information from the row chat file provided by data-donators.
     Parameters
     ----------
     file_data : str
-        The path of the chat_file.zip
+        The path of the chat file. It can be in zip or txt format.
     Returns
     -------
     pandas.dataframe
-        Extracted data from the chat_file
+        Extracted data from the chat file
     """
     errors = []
     log_error = errors.append
-    zfile = None
-    #chats = []
+
     try:
         zfile = zipfile.ZipFile(file_data)
     except:
         if FILE_RE.match(file_data.name):
             tfile = open(file_data, encoding="utf8")
             chat = parse_chat(log_error, tfile.read())
-            #chats.append(chat)
+
         else:
             log_error("There is not a valid file format.")
             return [format_errors(errors)]
@@ -693,7 +678,6 @@ def process(file_data):
     if errors:
         return [format_errors(errors)]
 
-    # print(chat)
     chat = remove_system_messages(chat)
     participants = extract_participants_features(chat)
     formatted_results = format_results(participants)

From ffe7e53afa5a2921abeb7347fe1e501fd8f656a8 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Thu, 7 Jul 2022 13:48:46 +0200
Subject: [PATCH 27/42] Add test for new output format, reply2, and remove
 group_name

---
 data_extractor/tests/test_whatsapp_chat.py | 39 ++++++----------------
 1 file changed, 11 insertions(+), 28 deletions(-)

diff --git a/data_extractor/tests/test_whatsapp_chat.py b/data_extractor/tests/test_whatsapp_chat.py
index b10b0b6..0938b68 100644
--- a/data_extractor/tests/test_whatsapp_chat.py
+++ b/data_extractor/tests/test_whatsapp_chat.py
@@ -1,35 +1,30 @@
 from data_extractor.whatsapp_chat import process
 from data_extractor.whatsapp_chat import anonymize_participants
-from data_extractor.whatsapp_chat import get_df_per_participant
 from pathlib import Path
 import pandas as pd
 from pandas.testing import assert_frame_equal
 
 
-
-
 DATA_PATH = Path(__file__).parent / "data"
 
 EXPECTED = [
     {'username': 'person1', 'Total number of words': 20, 'Number of URLs': 1, 'Number of shared locations': 1, 'file_no': 0, 'Number of messages': 3,
-     'Date first message': pd.to_datetime('2022-03-16 15:20:25'), 'Date last message': pd.to_datetime('2022-03-24 20:19:38')},
+     'Date first message': pd.to_datetime('2022-03-16 15:20:25'), 'Date last message': pd.to_datetime('2022-03-24 20:19:38'),
+     'user_reply2': 'person2', 'reply_2_user': 'person2'},
 
     {'username': 'person2', 'Total number of words': 7, 'Number of URLs': 1, 'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 3,
-     'Date first message': pd.to_datetime('2022-03-16 15:25:38'), 'Date last message': pd.to_datetime('2022-03-26 18:52:15')},
+     'Date first message': pd.to_datetime('2022-03-16 15:25:38'), 'Date last message': pd.to_datetime('2022-03-26 18:52:15'),
+     'user_reply2': 'person1', 'reply_2_user': 'person1'},
 
     {'username': 'person3', 'Total number of words': 1, 'Number of URLs': 0, 'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 1,
-     'Date first message': pd.to_datetime('2022-03-16 15:26:48'), 'Date last message': pd.to_datetime('2022-03-16 15:26:48')},
+     'Date first message': pd.to_datetime('2022-03-16 15:26:48'), 'Date last message': pd.to_datetime('2022-03-16 15:26:48'),
+     'user_reply2': 'person2', 'reply_2_user': 'person2'},
 
     {'username': 'person4', 'Total number of words': 21, 'Number of URLs': 0, 'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 2,
-     'Date first message': pd.to_datetime('2020-07-14 22:05:54'), 'Date last message': pd.to_datetime('2022-03-20 20:08:51')}
+     'Date first message': pd.to_datetime('2020-07-14 22:05:54'), 'Date last message': pd.to_datetime('2022-03-20 20:08:51'),
+     'user_reply2': 'person1', 'reply_2_user': 'person1'}
 ]
 
-# EXPECTED_1 = {'Description': ['Total number of words', 'Number of messages', 'Date first message', 'Date last message',
-#                               'Number of URLs', 'Number of shared locations'],
-#               'Value': [20, 3, pd.to_datetime('2022-03-16 15:20:25'), pd.to_datetime('2022-03-24 20:19:38'), 1, 1]}
-#
-# df_expected_1 = pd.DataFrame(data=EXPECTED_1)
-
 
 def test_process():
     """ Test process function.
@@ -49,7 +44,8 @@ def test_process():
     results = []
     df_melt = pd.melt(df_expected, id_vars=["username"],
                       value_vars=["Total number of words", "Number of messages", "Date first message", "Date last message",
-                                  "Number of URLs", "file_no", "Number of shared locations"], var_name='Description', value_name='Value')
+                                  "Number of URLs", "file_no", "Number of shared locations", "reply_2_user", "user_reply2"],
+                      var_name='Description', value_name='Value')
 
     usernames = df_melt["username"].unique()
     for u in usernames:
@@ -66,27 +62,14 @@ def test_process():
                 "data_frame": df[["Description", "Value"]].reset_index(drop=True)
             }
         )
-    # print(type(expected_results[0]["data_frame"]))
-    # print(expected_results[1]["data_frame"])
-    # print('******')
 
-    # result = process(DATA_PATH.joinpath("whatsapp_chat.zip"))
     df_result = process(DATA_PATH.joinpath("_chat.txt"))
-    # print(df_result[1]["data_frame"])
+
     assert_frame_equal(df_result[0]["data_frame"], expected_results[0]["data_frame"])
     assert_frame_equal(df_result[1]["data_frame"], expected_results[1]["data_frame"])
     assert_frame_equal(df_result[2]["data_frame"], expected_results[2]["data_frame"])
     assert_frame_equal(df_result[3]["data_frame"], expected_results[3]["data_frame"])
 
-    # print(type(df_result[0]["data_frame"]))
-    # print(df_result[0]["data_frame"])
-    # print(df_expected_1)
-    #
-    # # assert len(result_file) == 1
-    #
-    #
-    # assert_frame_equal(df_result[0]["data_frame"], df_expected_1)
-
 
 if __name__ == "__main__":
     test_process()

From 2ea702b5b14caef7bb0323cff2e90bcdbb651d4d Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Thu, 7 Jul 2022 13:50:17 +0200
Subject: [PATCH 28/42] Add groupname

---
 data_extractor/tests/data/_chat.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/data_extractor/tests/data/_chat.txt b/data_extractor/tests/data/_chat.txt
index c43d07a..062e1f0 100644
--- a/data_extractor/tests/data/_chat.txt
+++ b/data_extractor/tests/data/_chat.txt
@@ -1,3 +1,4 @@
+1/21/22, 19:56 - Messages and calls are end-to-end encrypted. No one outside of this chat, not even WhatsApp, can read or listen to them. Tap to learn more.
 [16/03/2022, 15:20:25] person1: Hi shiva!
 [16/03/2022, 15:25:38] person2: Hi 👋
 [16/03/2022, 15:26:48] person3: Hoi!

From 3f934fa95d7184dad8f9ba120998754b324838a2 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Thu, 7 Jul 2022 13:51:54 +0200
Subject: [PATCH 29/42] Add groupname

---
 data_extractor/tests/data/whatsapp_chat.zip | Bin 617 -> 1150 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/data_extractor/tests/data/whatsapp_chat.zip b/data_extractor/tests/data/whatsapp_chat.zip
index b3f6edfd76b4f70b94dd4b0ac39d66bf5fde1947..55bf9ed07df806479a7fb0374a0aca129d91fd48 100644
GIT binary patch
literal 1150
zcmWIWW@Zs#-~hs~+~*+-P@urfz`)6%z!0CDkyxTvQc)5b!pp$E<=C;Ltw0Q-ODnh;
z7+JnCGB5zOGccqEozA;rAaeJ+w%+fVHghaebQBi|357Cl%m`h`rD(p>c%zj423zLM
z+Sx2Z))8Ot|6={ilDBBuNdxWMpLQNU{(O#2YUQaLJfB)0h@N?<+oljM;V?@~>gM+?
zpE)vaRc<Mc%DDYwZTAhe59gXpIm%?4vod3y-h?zPn_S3t%<TZL)Q$zo6)*4f{NSmW
zq8q`#musTl;-!*TIOlkp%T!Id*RV})^5WBbyf!PP-qe`DJ&p62eUr2Nnv<-1!UU|g
zPW0!S7$fpkc~b9*-?!uM^A}Fl`2UdWP~=zpNZqd1gl^7A@x|OqzT3aNFD+_)`EsM3
z{qKO6m;KlCweNiJ)8+Hsv**pXO$?0OoP6n0Yf9?dRmYcXw5t90Q!?E%Oy6tX`HD&D
zrIEUQdXd`}tLbm5a*E}$*tz@XV#m0NYYv_imHJ|&#Ijn@?8Wv6njxzl?bBq~KBu-!
zHk8y|FMnX+#}2g;Gq#AJuqgr$!moQ9bIISB`z6ryl&3JWruO$Kfiug#U0xo)hWQuM
z6`>QxSq<ByWwy!PtKJ=A+Z5~@6MVkyy2{b@7kM`ATd_~Q#(G+c|Fyqw-{*gMx9{eh
z&N=>*b~#Apw|C8R@Anm#KUOs9cZtMl4?SVY)DXX(C9D2e91ENE`iFRcH#^7XR$0;8
zz_<pbivX-CrU57`0!lIQzK+iR!4dj;@z@g&!*Ptnlbj%$oSu-B@WIz7>;r#TM?eG9
z1PNw!0ck@6gI|o93iCHMHflVO5E5cZY%DP-VvrQg@S3sk(H|b3M<;*otm<TCwOm=H
zxKdPgUFFH2nlme33jBTK;MCCQ$hA;SedCTLTP#dfRX3`HS}I6n^;=EoNJ%hoUvb)3
z*OO(H=ay67N<Vt~c%AKB@1d=0pn1kb^OWy7Bi2oyCY|}Db;(DoSEzB8xA&zI-8H<u
zzCAo=b#;BhWUZ&M7**t@bZB+_dm^eY{?$hNz=Q1DWxAE0JBt>o*D~=e4ZA*jqjb*4
zJ7)|cKmT|$D@r{2Bv1ae>#l3o#~j_SZ?NLm-FU&X<-qhP^7Htu%sbJ~Htw-m89(LQ
z_j~*GV<g$L@6Ww`FNF&j*z6ot54Sc+1A`Y7!~x!nOd`yP42v8wpbQHGOBz8e>@k2P
o`=T3*Y$YgGVPHw45fji@GU7PEo0Sb@3KJ0S1k&q)DT#pr0LvM_U;qFB

literal 617
zcmWIWW@Zs#U|`^2(3n0YWcuRU=PxocF#KU+VBiFb#wTYamgtpKl;lo2n0LiM#P$2s
zU;O+V=c;WKZtqcL2?+@h?bNe!;!u2|malg0NZAJ4#&>O|Or5f8elgcvt#PzxR9~{v
z?X6zd``zzf-22}6q;S$PlZ7R0ErqiRPT3tgwop|j&bwgin<I-4H&;6yyZmVX1inMF
zU&r5<pYYU1IyG~GWaar)4_?l>{4*uA`o5Li@$j_A<`-3ur612K{Ad@RKHpDnM(Ui2
zo#osA{yaA2koV&EJySAbw-}{eO$d<^wg3HR%0&&$XOhasViT8~_*7Z2?Bv|y4n<A1
z=Q-Q|o_<)e{qJIj7&g5RY+)-da`ivxae8qfu2FQC5toRSdf1CY+&`i!ELwvy0+MXI
zVlxG|uI?@0U}DxG@FDv)hwXyz@78XJo3=^x*6w+_x097WYt|TTu$5<G+u0%@9TBqZ
z@W0E4cA9$MRM@iVi*)y4J4UU97jkd9b7n1AUv;T=!|d1h@BcdcyVCd+=j>fo?TNdL
zq!J(MPTMNK{NA?g#dF%P&GLH{wCHohLbmJ@(I3D6r{1uiee6|$|Ahug_T3Y%uYU94
z!8YgIm?g7!bY{AF&Jynzf36=NGe>5sTeYIIpnIj0(X>r#>?aCbn=EI^nHV^)Mfl_~
zrONtc_I(?Ls^0!*4De=TvS+{*^D4l&2LS~ziIfhwphAoc5)4U7P8+U=++bYge#z<v
bkoGy@2<8WPv$BEoF#=&Ukp2NoPz($Jz#$DG


From a9234d2bc5ff62e7a69e53b2d0411db2b459b824 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Thu, 7 Jul 2022 14:15:10 +0200
Subject: [PATCH 30/42] change format

---
 data_extractor/tests/data/_chat.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/data_extractor/tests/data/_chat.txt b/data_extractor/tests/data/_chat.txt
index 062e1f0..845305d 100644
--- a/data_extractor/tests/data/_chat.txt
+++ b/data_extractor/tests/data/_chat.txt
@@ -1,4 +1,4 @@
-1/21/22, 19:56 - Messages and calls are end-to-end encrypted. No one outside of this chat, not even WhatsApp, can read or listen to them. Tap to learn more.
+[16/03/2022, 15:10:17] Messages and calls are end-to-end encrypted. No one outside of this chat, not even WhatsApp, can read or listen to them. Tap to learn more.
 [16/03/2022, 15:20:25] person1: Hi shiva!
 [16/03/2022, 15:25:38] person2: Hi 👋
 [16/03/2022, 15:26:48] person3: Hoi!

From f169f1693ea7e579d7ae4986667ec9042c3fbd81 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Thu, 7 Jul 2022 14:15:36 +0200
Subject: [PATCH 31/42] change format

---
 data_extractor/tests/data/whatsapp_chat.zip | Bin 1150 -> 1149 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/data_extractor/tests/data/whatsapp_chat.zip b/data_extractor/tests/data/whatsapp_chat.zip
index 55bf9ed07df806479a7fb0374a0aca129d91fd48..c40699d7fc035e55b114fc2959ff7b97aaad7b0d 100644
GIT binary patch
delta 717
zcmeyz@t31Mz?+#xgn<JH`SYKLFhGGCGXn!Bg91Z*az<i_UP(nsXb3L@`}>o}l0E@3
zh%T+*W?*Fb#>l_`)Xu<=8szJL$3S53XYFO*RXgXVc<O}aPSg^;7-riLu+?d*XPUX#
zNB7+K+p3$3uX^o0*!lmM{sZ-rO-=QaQvKyl=ijs0Z*FYnyV>Tc<a6PKr3o_=*q?vk
zbe_P?qu|Tk`=)x84b!YL`&+eED{}taa?KI`k#kUrWp|I`){=9IWm*oGPg*c0D?i}k
zopaH2PjQ}84V%p=u{G^+tseasgZN*u_$`svv-Qe%i0(gm?@g=AJkGdz&Cc>`5}EAk
zH8|!*oebZ7BIwCg;Ys-u_{;0--#%Kl=tuu)&h)F^o4lTEeektoP4`9CiOaHoeK*>9
z;LDGI`SWV#zIvShzI?WK;HHVsey_Hd&sAQ$>Df$AZ}Ca3Tf5~ZAA9xX?Dy-c{+m|m
zd6{VnPYXS|<=*U%Z)VQgb7GZXsp5kd|6YYP?@@?suAj_p@*>rRBTRYCh4lx6SA;d!
zowDG0cACR|cFLALH7xDNlxJPIDUsk88R*ntAFF@X?Z<(4OZry@`puH?h}^ie=c@AW
zmzVRWGuHBD@ja2uZP=zQ<HaApXYOjbgDchaGS9bVPI`1bgDuiN<i7B}yE-SQt!dvq
zJA2md+_ICb+U}l?a}~F4KioCXt>0H%{<g$PTZvmnlTN$3Ns1~fXNK0vCuyHe{lgyM
z&CX$K@^ITNV043$26CdAY{#U{1xp#<&u%Ve@?hizr6stA$tReV-C!!Pq@4h7MkWzv
rL?WzuxV1?dXj20ai!dm_(j>!@M#IU5EHX%zO%7*KWLpPx5Ca1MMOG}>

delta 718
zcmey%@sFcEz?+#xgn<JH!*ZX8FhGF<GXn!Bg91Z*az<i_UP(nsXb3L@`<7$JlC}ad
zh%T+*W?*Fb#>l_`)Xu<=8gx4Eih;=8@7j95XWGoMNYPPTAS4vZxG^JiA(x{0PUDSI
z_8V-OH*06J2w6vbz5k2#FH7E{X(#Iqv~PdfdHneEIX0=4r*80kYJDJj=AmwzLb!y(
zEHSB@-?x0`$hcLxr8p|%_K&sQH`G3yYcl01lWoq*jCFbw(y(lDA=@#x1H4i@79>}^
zywmf8r(%k31pi*HiF%8dN?zfd<7qBaHRWEzHo3`*Pw(;Ctdx3FV*>Xy&SUmX&hl$c
zvexej6R_Gk(VuT(jL28zNxdh2-;Te}UpQ6c|3j`rkzeg2b-P*<x;Z1o7jrB5ZvXPW
zw5avv%Z+ySzXM)g_FvD}zVpFPm(O?4o;Tk%F)(s-@}*0yDXDK)9bdB1s`lSc$#l;!
zeXn`vD<-9vM(XzIMQ&THroXAmDVED(=kA}29pfghsXus9RO*Y763c2qvlrVRXojqI
zv`>>^`<&V`*-%n<z5IcNA3M}a%-AA=!lnp32*2)a%q4$g?w3H*Q=Y=in%du|1kNn`
zc6oXH8s=Y2SA<R&XEkh-mf0qEuX=ZgZBwvsOz`=(>ncarU*y@eZ^b_K8tZ8({@4D#
zeV_m3-M*W1I_LO1P1@xkmEYbq&%NJQT>e<mq~9eHr#<w9B~wHEdX}vEV{t5O*6SbQ
z0p9E!n_Fc?Zv*2Sls1r4&1739Z7x{SU^u?Hh{=PI7nGRb8YUlSR(6A_z>;_ZycwB9
sm=US4>fzQVX`oFFKrF(b085k%OB#(P8?eYASvEP0MUibi&_N6g0K@DjzW@LL


From da3c616ab0befae90399d99e4d2633ad0335a497 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Thu, 7 Jul 2022 14:19:30 +0200
Subject: [PATCH 32/42] Remove main()

---
 data_extractor/tests/test_whatsapp_chat.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/data_extractor/tests/test_whatsapp_chat.py b/data_extractor/tests/test_whatsapp_chat.py
index 0938b68..cc580af 100644
--- a/data_extractor/tests/test_whatsapp_chat.py
+++ b/data_extractor/tests/test_whatsapp_chat.py
@@ -1,5 +1,5 @@
-from data_extractor.whatsapp_chat import process
-from data_extractor.whatsapp_chat import anonymize_participants
+from whatsapp_chat import process
+from whatsapp_chat import anonymize_participants
 from pathlib import Path
 import pandas as pd
 from pandas.testing import assert_frame_equal
@@ -71,7 +71,5 @@ def test_process():
     assert_frame_equal(df_result[3]["data_frame"], expected_results[3]["data_frame"])
 
 
-if __name__ == "__main__":
-    test_process()
 
 

From 1c7549768ce9d18502dbe28129bf845b54f73c25 Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Thu, 7 Jul 2022 15:48:45 +0200
Subject: [PATCH 33/42] pylint check

---
 data_extractor/tests/test_whatsapp_chat.py | 58 ++++++++++++++--------
 1 file changed, 37 insertions(+), 21 deletions(-)

diff --git a/data_extractor/tests/test_whatsapp_chat.py b/data_extractor/tests/test_whatsapp_chat.py
index cc580af..f6459ff 100644
--- a/data_extractor/tests/test_whatsapp_chat.py
+++ b/data_extractor/tests/test_whatsapp_chat.py
@@ -1,55 +1,75 @@
-from whatsapp_chat import process
-from whatsapp_chat import anonymize_participants
 from pathlib import Path
 import pandas as pd
+
+from whatsapp_chat import process
+from whatsapp_chat import anonymize_participants
+
 from pandas.testing import assert_frame_equal
 
 
 DATA_PATH = Path(__file__).parent / "data"
 
 EXPECTED = [
-    {'username': 'person1', 'Total number of words': 20, 'Number of URLs': 1, 'Number of shared locations': 1, 'file_no': 0, 'Number of messages': 3,
-     'Date first message': pd.to_datetime('2022-03-16 15:20:25'), 'Date last message': pd.to_datetime('2022-03-24 20:19:38'),
+    {'username': 'person1', 'Total number of words': 20, 'Number of URLs': 1,
+     'Number of shared locations': 1,'file_no': 0, 'Number of messages': 3,
+     'Date first message': pd.to_datetime('2022-03-16 15:20:25'),
+     'Date last message': pd.to_datetime('2022-03-24 20:19:38'),
      'user_reply2': 'person2', 'reply_2_user': 'person2'},
 
-    {'username': 'person2', 'Total number of words': 7, 'Number of URLs': 1, 'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 3,
-     'Date first message': pd.to_datetime('2022-03-16 15:25:38'), 'Date last message': pd.to_datetime('2022-03-26 18:52:15'),
+    {'username': 'person2', 'Total number of words': 7, 'Number of URLs': 1,
+     'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 3,
+     'Date first message': pd.to_datetime('2022-03-16 15:25:38'),
+     'Date last message': pd.to_datetime('2022-03-26 18:52:15'),
      'user_reply2': 'person1', 'reply_2_user': 'person1'},
 
-    {'username': 'person3', 'Total number of words': 1, 'Number of URLs': 0, 'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 1,
-     'Date first message': pd.to_datetime('2022-03-16 15:26:48'), 'Date last message': pd.to_datetime('2022-03-16 15:26:48'),
+    {'username': 'person3', 'Total number of words': 1, 'Number of URLs': 0,
+     'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 1,
+     'Date first message': pd.to_datetime('2022-03-16 15:26:48'),
+     'Date last message': pd.to_datetime('2022-03-16 15:26:48'),
      'user_reply2': 'person2', 'reply_2_user': 'person2'},
 
-    {'username': 'person4', 'Total number of words': 21, 'Number of URLs': 0, 'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 2,
-     'Date first message': pd.to_datetime('2020-07-14 22:05:54'), 'Date last message': pd.to_datetime('2022-03-20 20:08:51'),
+    {'username': 'person4', 'Total number of words': 21, 'Number of URLs': 0,
+     'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 2,
+     'Date first message': pd.to_datetime('2020-07-14 22:05:54'),
+     'Date last message': pd.to_datetime('2022-03-20 20:08:51'),
      'user_reply2': 'person1', 'reply_2_user': 'person1'}
 ]
 
 
 def test_process():
     """ Test process function.
-        compares the expected dataframe with the output of the process function to check if all the columns are match.
+        compares the expected dataframe with the output of the process function
+         to check if all the columns are match.
         Raises
         -------
-        AssertionError: When provided expected dataframe could not match the participants dataframe
+        AssertionError: When provided expected dataframe could not match the
+         participants dataframe
         """
 
     df_expected = pd.DataFrame(EXPECTED)
     df_expected = anonymize_participants(df_expected)
     df_expected['Number of messages'] = df_expected['Number of messages'].astype('int64')
     df_expected['Number of URLs'] = df_expected['Number of URLs'].astype('int32')
-    df_expected['Number of shared locations'] = df_expected['Number of shared locations'].astype('int32')
+    df_expected['Number of shared locations'] = \
+        df_expected['Number of shared locations'].astype('int32')
     df_expected['file_no'] = df_expected['file_no'].astype('int32')
 
     results = []
     df_melt = pd.melt(df_expected, id_vars=["username"],
-                      value_vars=["Total number of words", "Number of messages", "Date first message", "Date last message",
-                                  "Number of URLs", "file_no", "Number of shared locations", "reply_2_user", "user_reply2"],
+                      value_vars=["Total number of words",
+                                  "Number of messages",
+                                  "Date first message",
+                                  "Date last message",
+                                  "Number of URLs",
+                                  "file_no",
+                                  "Number of shared locations",
+                                  "reply_2_user",
+                                  "user_reply2"],
                       var_name='Description', value_name='Value')
 
     usernames = df_melt["username"].unique()
-    for u in usernames:
-        df_user = df_melt[(df_melt["username"] == u) & df_melt["Value"] != 0]
+    for user in usernames:
+        df_user = df_melt[(df_melt["username"] == user) & df_melt["Value"] != 0]
         results.append(df_user)
 
     expected_results = []
@@ -69,7 +89,3 @@ def test_process():
     assert_frame_equal(df_result[1]["data_frame"], expected_results[1]["data_frame"])
     assert_frame_equal(df_result[2]["data_frame"], expected_results[2]["data_frame"])
     assert_frame_equal(df_result[3]["data_frame"], expected_results[3]["data_frame"])
-
-
-
-

From 3e0b9cebb40dea798a82778c2e78e7e221a8190c Mon Sep 17 00:00:00 2001
From: Shiva Nadi <s.nadi@uu.nl>
Date: Thu, 7 Jul 2022 16:16:34 +0200
Subject: [PATCH 34/42] run poetry

---
 data_extractor/tests/test_whatsapp_chat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/data_extractor/tests/test_whatsapp_chat.py b/data_extractor/tests/test_whatsapp_chat.py
index f6459ff..87f3768 100644
--- a/data_extractor/tests/test_whatsapp_chat.py
+++ b/data_extractor/tests/test_whatsapp_chat.py
@@ -11,7 +11,7 @@
 
 EXPECTED = [
     {'username': 'person1', 'Total number of words': 20, 'Number of URLs': 1,
-     'Number of shared locations': 1,'file_no': 0, 'Number of messages': 3,
+     'Number of shared locations': 1, 'file_no': 0, 'Number of messages': 3,
      'Date first message': pd.to_datetime('2022-03-16 15:20:25'),
      'Date last message': pd.to_datetime('2022-03-24 20:19:38'),
      'user_reply2': 'person2', 'reply_2_user': 'person2'},

From 43b8c1299a9838d4f7046900f5f3aac967997e62 Mon Sep 17 00:00:00 2001
From: parisa-zahedi <parisa.zahedi@gmail.com>
Date: Fri, 8 Jul 2022 09:37:33 +0200
Subject: [PATCH 35/42] rename reply_2user and user_reply2 fields

---
 data_extractor/whatsapp_chat/__init__.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index 52727b5..2331e8d 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -48,13 +48,13 @@ class ColnamesDf:
     WORDS_NO = 'Total number of words'
     """Total number of words  column"""
 
-    REPLY_2USER = 'reply_2_user'
+    REPLY_2USER = 'Who replies to you the most often?'
     """Who replies to the user the most column"""
 
     MAX_REPLY_2 = 'max_reply_2'
     """User replies to who the most column"""
 
-    USER_REPLY2 = 'user_reply2'
+    USER_REPLY2 = 'Who do you most often reply to?'
     """User replies to who the most column"""
 
     URL_NO = 'Number of URLs'
@@ -475,9 +475,11 @@ def anonymize_participants(df_participants):
     # df_participants[COLNAMES_DF.USER_REPLY2] = df_participants[COLNAMES_DF.USER_REPLY2].apply(lambda u: anonym_txt(u,salt))
     # df_participants[['username', 'user_reply2']] = df_participants[['username', 'user_reply2']].stack().rank(method='dense').unstack()
 
-    stacked = df_participants[['username','user_reply2', 'reply_2_user']].stack()
-    df_participants[['username','user_reply2', 'reply_2_user']] = pd.Series(stacked.factorize()[0], index=stacked.index).unstack()
-    df_participants[['username','user_reply2', 'reply_2_user']] = 'person' + df_participants[['username','user_reply2', 'reply_2_user']].astype(str)
+    stacked = df_participants[[COLNAMES_DF.USERNAME,COLNAMES_DF.USER_REPLY2, COLNAMES_DF.REPLY_2USER]].stack()
+    df_participants[[COLNAMES_DF.USERNAME,COLNAMES_DF.USER_REPLY2, COLNAMES_DF.REPLY_2USER]] = \
+        pd.Series(stacked.factorize()[0], index=stacked.index).unstack()
+    df_participants[[COLNAMES_DF.USERNAME,COLNAMES_DF.USER_REPLY2, COLNAMES_DF.REPLY_2USER]] = \
+        'person' + df_participants[[COLNAMES_DF.USERNAME,COLNAMES_DF.USER_REPLY2, COLNAMES_DF.REPLY_2USER]].astype(str)
     return df_participants
 
 

From 0bdc58b11506b00e5648c0e872259f6c030e315f Mon Sep 17 00:00:00 2001
From: parisa-zahedi <parisa.zahedi@gmail.com>
Date: Fri, 8 Jul 2022 18:11:47 +0200
Subject: [PATCH 36/42] add group_name and system message logs

---
 data_extractor/whatsapp_chat/__init__.py | 76 ++++++++++++++++--------
 1 file changed, 50 insertions(+), 26 deletions(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index 2331e8d..f6b2045 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -7,9 +7,8 @@
 import re
 from datetime import datetime
 import pandas as pd
-import hashlib
 import zipfile
-from pathlib import Path
+
 
 URL_PATTERN = r'(https?://\S+)'
 LOCATION_PATTERN = r'(Location: https?://\S+)'
@@ -51,9 +50,6 @@ class ColnamesDf:
     REPLY_2USER = 'Who replies to you the most often?'
     """Who replies to the user the most column"""
 
-    MAX_REPLY_2 = 'max_reply_2'
-    """User replies to who the most column"""
-
     USER_REPLY2 = 'Who do you most often reply to?'
     """User replies to who the most column"""
 
@@ -66,12 +62,6 @@ class ColnamesDf:
     FILE_NO = 'Number of shared files'
     """Number of files column"""
 
-    OUT_DEGREE = 'out_degree'
-    """Total number of sent message column"""
-
-    IN_DEGREE = 'in_degree'
-    """Total number of received message column"""
-
     EMOJI_NO = 'emoji_no'
     """Total number of emojies column"""
 
@@ -209,9 +199,11 @@ def remove_alerts_from_df(r_x, df):
     pandas.DataFrame
         Fixed version of input DataFrame
     """
+
+    alerts_no = count_alerts(r_x, df)
     df_new = df.copy()
     df_new.loc[:, COLNAMES_DF.MESSAGE] = df_new[COLNAMES_DF.MESSAGE].apply(lambda x: remove_alerts_from_line(r_x, x))
-    return df_new
+    return df_new,alerts_no
 
 
 def remove_alerts_from_line(r_x, line_df):
@@ -228,11 +220,32 @@ def remove_alerts_from_line(r_x, line_df):
         Cleaned message string
     """
     if re.search(r_x, line_df):
+        print(line_df[:re.search(r_x, line_df).start()])
         return line_df[:re.search(r_x, line_df).start()]
     else:
         return line_df
 
 
+def count_alerts(r_x, df):
+    """Count line content that is not desirable (automatic alerts etc.).
+    Parameters
+    ----------
+    r_x : str
+        Regula expression to detect WhatsApp warnings
+    df : pandas.DataFrame
+        pandas.DataFrame with all interventions
+
+    Returns
+    -------
+    int
+        Number of line contents that is not desirable
+    """
+
+    # alerts_count = df[COLNAMES_DF.MESSAGE].apply(lambda x: (re.search(r_x, x) is not None))
+    alerts_count = df[COLNAMES_DF.MESSAGE].apply(lambda x: re.findall(r_x, x))
+    return alerts_count.str.len().sum()
+
+
 def get_message(text, headers, i):
     """Get i:th message from text.
     Parameters
@@ -314,9 +327,12 @@ def make_chat_df(log_error, text, hformat):
     # Parse chat to DataFrame
     try:
         df = parse_text(text, r)
-        df = remove_alerts_from_df(r_x, df)
+        df, alerts_no = remove_alerts_from_df(r_x, df)
         df = add_schema(df)
 
+        if alerts_no>0:
+            log_error("Number of unprocessed system messages: "+str(alerts_no))
+
         return df
     except:
         print(f"hformat : {hformat} is not match with the given text")
@@ -483,7 +499,7 @@ def anonymize_participants(df_participants):
     return df_participants
 
 
-def get_df_per_participant(df):
+def get_wide_to_long_participant(df):
     """Generate one dataframe for each participant .
         Parameter
         ----------
@@ -569,7 +585,7 @@ def get_participants_features(df_chat):
 
     return df_participants
 
-def remove_system_messages(chat):
+def remove_system_messages(log_error, chat):
     """Removes system messages from chat
     Parameters
     ----------
@@ -585,6 +601,7 @@ def remove_system_messages(chat):
     is_system_message = True if all(s in message0 for s in SYSTEM_MESSAGES) else False
     if is_system_message:
         group_name = chat.loc[0, COLNAMES_DF.USERNAME]
+        log_error("Identified group name:"+group_name)
         chat = chat.loc[chat[COLNAMES_DF.USERNAME] != group_name,]
 
     return chat
@@ -607,13 +624,13 @@ def extract_participants_features(chat, anonymize=True):
     if anonymize:
         df= anonymize_participants(df)
 
-    results = get_df_per_participant(df)
+    results = get_wide_to_long_participant(df)
     return results
 
 # ***** end of analysis functions *****
 
 
-def format_results(df_list):
+def format_results(df_list, error):
     """Format results to the standard format.
     Parameters
     ----------
@@ -627,11 +644,13 @@ def format_results(df_list):
         user_name = pd.unique(df[COLNAMES_DF.USERNAME])[0]
         results.append(
             {
-                "id": user_name,#"overview",
-                "title": user_name,#"The following data is extracted from the file:",
+                "id": user_name,
+                "title": user_name,
                 "data_frame": df[[COLNAMES_DF.DESCRIPTION,COLNAMES_DF.VALUE]].reset_index(drop=True)
             }
         )
+    if len(error)>0:
+        results = results+error
     return results
 
 
@@ -644,9 +663,11 @@ def format_errors(errors):
     -------
     pandas.dataframe
     """
+    if len(errors) == 0:
+        return []
     data_frame = pd.DataFrame()
     data_frame["Messages"] = pd.Series(errors, name="Messages")
-    return {"id": "extraction_log", "title": "Extraction log", "data_frame": data_frame}
+    return [{"id": "extraction_log", "title": "Extraction log", "data_frame": data_frame}]
 
 
 def process(file_data):
@@ -674,14 +695,17 @@ def process(file_data):
 
         else:
             log_error("There is not a valid file format.")
-            return [format_errors(errors)]
+            return format_errors(errors)
     else:
         chat = parse_zipfile(log_error, zfile)
-    if errors:
-        return [format_errors(errors)]
 
-    chat = remove_system_messages(chat)
-    participants = extract_participants_features(chat)
-    formatted_results = format_results(participants)
+    if chat is not None:
+        chat = remove_system_messages(log_error,chat)
+        participants = extract_participants_features(chat)
+
+        formatted_results = format_results(participants, format_errors(errors))
+
+    else:
+        return format_errors(errors)
 
     return formatted_results

From ccc6320c8c3972c455b1150f9d63d3cead77ca93 Mon Sep 17 00:00:00 2001
From: parisa-zahedi <parisa.zahedi@gmail.com>
Date: Mon, 11 Jul 2022 14:15:40 +0200
Subject: [PATCH 37/42] extend media, location and url regex

---
 data_extractor/whatsapp_chat/__init__.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index f6b2045..46c3611 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -10,9 +10,10 @@
 import zipfile
 
 
-URL_PATTERN = r'(https?://\S+)'
-LOCATION_PATTERN = r'(Location: https?://\S+)'
-ATTACH_FILE_PATTERN = r'(<attached: \S+>)'
+URL_PATTERN = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))"
+LOCATION_PATTERN = r'((L|l)ocation: https?://\S+)|((l|L)ocatie: https?://\S+)|(.*(l|L)ive locatie gedeeld.*)|(.*(l|L)ive location shared.*)'
+ATTACH_FILE_PATTERN = r'(<attached: \S+>)|(<Media (weggelaten|omitted)>)|((afbeelding|GIF|video|image|audio|(s|S)ticker|.*document.*) (weggelaten|omitted))'
+
 FILE_RE = re.compile(r".*.txt$")
 HIDDEN_FILE_RE = re.compile(r".*__MACOSX*")
 
@@ -220,7 +221,6 @@ def remove_alerts_from_line(r_x, line_df):
         Cleaned message string
     """
     if re.search(r_x, line_df):
-        print(line_df[:re.search(r_x, line_df).start()])
         return line_df[:re.search(r_x, line_df).start()]
     else:
         return line_df

From 41c48c826187ec0372da45f731d8621cf234957b Mon Sep 17 00:00:00 2001
From: parisa-zahedi <parisa.zahedi@gmail.com>
Date: Mon, 11 Jul 2022 14:24:17 +0200
Subject: [PATCH 38/42] fix pylint error in regex

---
 data_extractor/whatsapp_chat/__init__.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index 46c3611..b193d5b 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -10,9 +10,13 @@
 import zipfile
 
 
-URL_PATTERN = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))"
-LOCATION_PATTERN = r'((L|l)ocation: https?://\S+)|((l|L)ocatie: https?://\S+)|(.*(l|L)ive locatie gedeeld.*)|(.*(l|L)ive location shared.*)'
-ATTACH_FILE_PATTERN = r'(<attached: \S+>)|(<Media (weggelaten|omitted)>)|((afbeelding|GIF|video|image|audio|(s|S)ticker|.*document.*) (weggelaten|omitted))'
+URL_PATTERN = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)" \
+              r"(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|" \
+              r"(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))"
+LOCATION_PATTERN = r'((L|l)ocation: https?://\S+)|((l|L)ocatie: https?://\S+)|' \
+                   r'(.*(l|L)ive locatie gedeeld.*)|(.*(l|L)ive location shared.*)'
+ATTACH_FILE_PATTERN = r'(<attached: \S+>)|(<Media (weggelaten|omitted)>)|' \
+                      r'((afbeelding|GIF|video|image|audio|(s|S)ticker|.*document.*) (weggelaten|omitted))'
 
 FILE_RE = re.compile(r".*.txt$")
 HIDDEN_FILE_RE = re.compile(r".*__MACOSX*")

From 0a1bac56c1c023644c4500cf0f8d1bf7c8d53ad5 Mon Sep 17 00:00:00 2001
From: emielvdveen <emiel@applike.nl>
Date: Mon, 11 Jul 2022 15:25:11 +0200
Subject: [PATCH 39/42] Fix pylint errors

---
 data_extractor/google_search_history/simulation_gsh.py | 10 +++++-----
 .../simulation_gslh.py                                 |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/data_extractor/google_search_history/simulation_gsh.py b/data_extractor/google_search_history/simulation_gsh.py
index b1e2f78..d84c562 100644
--- a/data_extractor/google_search_history/simulation_gsh.py
+++ b/data_extractor/google_search_history/simulation_gsh.py
@@ -157,16 +157,16 @@ def browserhistory(num: int, site_diff: float, time_diff: bool,
     parts = _create_bins(num)
     # create browserhistory data
     results = []
-    for moment in PERIODS:
+    for moment, period in PERIODS.items():
         # simulate dates
         if moment == 'during':
             perc = 0.15+site_diff
-            dates = _create_date(num=parts[moment], start=PERIODS[moment][0],
-                                 end=PERIODS[moment][1], time_perc=time_diff)
+            dates = _create_date(num=parts[moment], start=period[0],
+                                 end=period[1], time_perc=time_diff)
         else:
             perc = 0.15
-            dates = _create_date(num=parts[moment], start=PERIODS[moment][0],
-                                 end=PERIODS[moment][1], time_perc=0)
+            dates = _create_date(num=parts[moment], start=period[0],
+                                 end=period[1], time_perc=0)
         # simulate website URLs
         url = _create_website(num=parts[moment], perc=perc, fake=fake)
         for i in range(parts[moment]):
diff --git a/data_extractor/google_semantic_location_history/simulation_gslh.py b/data_extractor/google_semantic_location_history/simulation_gslh.py
index 0f1256e..d5feec8 100644
--- a/data_extractor/google_semantic_location_history/simulation_gslh.py
+++ b/data_extractor/google_semantic_location_history/simulation_gslh.py
@@ -180,7 +180,7 @@ def fake_data(json_file, seed=0):
     places = _create_places(total=max(NPLACES.values()))
 
     # Get json schema from json file
-    with open(json_file) as file_object:
+    with open(json_file, encoding="utf-8") as file_object:
         json_data = json.load(file_object)
         json_schema = get_json_schema(json_data)
 

From 9587167f5695873afffde21def6f9f687c1044c8 Mon Sep 17 00:00:00 2001
From: ShNadi <44059592+ShNadi@users.noreply.github.com>
Date: Sun, 28 Aug 2022 11:14:07 +0200
Subject: [PATCH 40/42] Update pilot version (#21)

* Support new account_info format

* Add support for the old version of the data-package

* Rename reply_2user and user_reply2

* add new header formats and a general regex

* Fix some pylint issues

* Fix pylint issues

* Fix pylint issues

* additional unittests for whatsapp chats

* added some documentation

* fix

* add log if general regular expression is used

Co-authored-by: parisa-zahedi <parisa.zahedi@gmail.com>
Co-authored-by: trbKnl <ncdeschipper@gmail.com>
---
 data_extractor/tests/data/_chat_file1.txt     | 10 ++
 data_extractor/tests/data/_chat_file2.txt     | 12 +++
 data_extractor/tests/data/_chat_file3.txt     | 12 +++
 data_extractor/tests/data/_chat_file4.txt     | 12 +++
 data_extractor/tests/data/_chat_file5.txt     | 12 +++
 data_extractor/tests/data/_chat_file6.txt     | 12 +++
 .../tests/test_whatsapp_account_info.py       |  8 +-
 data_extractor/tests/test_whatsapp_chat.py    | 76 +++++++++-----
 .../whatsapp_account_info/__init__.py         | 99 ++++++++++++++++---
 data_extractor/whatsapp_chat/__init__.py      | 85 +++++++++++++++-
 10 files changed, 292 insertions(+), 46 deletions(-)
 create mode 100644 data_extractor/tests/data/_chat_file1.txt
 create mode 100644 data_extractor/tests/data/_chat_file2.txt
 create mode 100644 data_extractor/tests/data/_chat_file3.txt
 create mode 100644 data_extractor/tests/data/_chat_file4.txt
 create mode 100644 data_extractor/tests/data/_chat_file5.txt
 create mode 100644 data_extractor/tests/data/_chat_file6.txt

diff --git a/data_extractor/tests/data/_chat_file1.txt b/data_extractor/tests/data/_chat_file1.txt
new file mode 100644
index 0000000..404d256
--- /dev/null
+++ b/data_extractor/tests/data/_chat_file1.txt
@@ -0,0 +1,10 @@
+[16/03/2022, 15:10:17] Messages and calls are end-to-end encrypted. No one outside of this chat, not even WhatsApp, can read or listen to them. Tap to learn more.
+[16/03/2022, 15:20:25] person1: Hi shiva!
+[16/03/2022, 15:25:38] person2: Hi 👋
+[16/03/2022, 15:26:48] person3: Hoi!
+[16/03/2022, 18:39:29] person2: https://youtu.be/KBmUTY6mK_E
+[16/03/2022, 18:35:51] person1: ‎Location: https://maps.google.com/?q=52.089451,5.108469
+[20/03/2022, 20:08:51] person4: I’m about to generate some very random messages so that I can make some screenshots for the explanation to participants
+[24/03/2022, 20:19:38] person1: @user3 if you remove your Profile picture for a moment I will redo the screenshots 😁
+[26/03/2022, 18:52:15] person2: Well done Utrecht 😁
+[14/07/2020, 22:05:54] person4: 👍Bedankt
diff --git a/data_extractor/tests/data/_chat_file2.txt b/data_extractor/tests/data/_chat_file2.txt
new file mode 100644
index 0000000..67e2ff3
--- /dev/null
+++ b/data_extractor/tests/data/_chat_file2.txt
@@ -0,0 +1,12 @@
+[03/16/2022, 15:10:17] Messages and calls are end-to-end encrypted. No one outside of this chat, not even WhatsApp, can read or listen to them. Tap to learn more.
+[03/16/2022, 15:20:25] person1: Hi 
+
+shiva!
+[03/16/2022, 15:25:38] person2: Hi 👋
+[03/16/2022, 15:26:48] person3: Hoi!
+[03/16/2022, 18:39:29] person2: https://youtu.be/KBmUTY6mK_E
+[03/16/2022, 18:35:51] person1: ‎Location: https://maps.google.com/?q=52.089451,5.108469
+‎[03/20/2022, 20:08:51] person4: I’m about to generate some very random messages so that I can make some screenshots for the explanation to participants
+‎[03/24/2022, 20:19:38] person1: @user3 if you remove your Profile picture for a moment I will redo the screenshots 😁
+[03/26/2022, 18:52:15] person2: Well done Utrecht 😁
+[07/14/2020, 22:05:54] person4: 👍Bedankt
diff --git a/data_extractor/tests/data/_chat_file3.txt b/data_extractor/tests/data/_chat_file3.txt
new file mode 100644
index 0000000..3e5f1fb
--- /dev/null
+++ b/data_extractor/tests/data/_chat_file3.txt
@@ -0,0 +1,12 @@
+16/03/2022, 15:10 - Messages and calls are end-to-end encrypted. No one outside of this chat, not even WhatsApp, can read or listen to them. Tap to learn more.
+16/03/2022, 15:20 - person1: Hi 
+
+shiva!
+16/03/2022, 15:25 - person2: Hi 👋
+16/03/2022, 15:26 - person3: Hoi!
+16/03/2022, 18:39 - person2: https://youtu.be/KBmUTY6mK_E
+16/03/2022, 18:35 - person1: ‎Location: https://maps.google.com/?q=52.089451,5.108469
+‎20/03/2022, 20:08 - person4: I’m about to generate some very random messages so that I can make some screenshots for the explanation to participants
+‎24/03/2022, 20:19 - person1: @user3 if you remove your Profile picture for a moment I will redo the screenshots 😁
+26/03/2022, 18:52 - person2: Well done Utrecht 😁
+14/07/2020, 22:05 - person4: 👍Bedankt
diff --git a/data_extractor/tests/data/_chat_file4.txt b/data_extractor/tests/data/_chat_file4.txt
new file mode 100644
index 0000000..8231647
--- /dev/null
+++ b/data_extractor/tests/data/_chat_file4.txt
@@ -0,0 +1,12 @@
+16.03.2022, 15:10 - Messages and calls are end-to-end encrypted. No one outside of this chat, not even WhatsApp, can read or listen to them. Tap to learn more.
+16.03.2022, 15:20 - person1: Hi 
+
+shiva!
+16.03.2022, 15:25 - person2: Hi 👋
+16.03.2022, 15:26 - person3: Hoi!
+16.03.2022, 18:39 - person2: https://youtu.be/KBmUTY6mK_E
+16.03.2022, 18:35 - person1: ‎Location: https://maps.google.com/?q=52.089451,5.108469
+‎20.03.2022, 20:08 - person4: I’m about to generate some very random messages so that I can make some screenshots for the explanation to participants
+‎24.03.2022, 20:19 - person1: @user3 if you remove your Profile picture for a moment I will redo the screenshots 😁
+26.03.2022, 18:52 - person2: Well done Utrecht 😁
+14.07.2020, 22:05 - person4: 👍Bedankt
diff --git a/data_extractor/tests/data/_chat_file5.txt b/data_extractor/tests/data/_chat_file5.txt
new file mode 100644
index 0000000..ab29079
--- /dev/null
+++ b/data_extractor/tests/data/_chat_file5.txt
@@ -0,0 +1,12 @@
+[16/03/2022, 03:10:17 PM] Messages and calls are end-to-end encrypted. No one outside of this chat, not even WhatsApp, can read or listen to them. Tap to learn more.
+[16/03/2022, 03:20:25 PM] person1: Hi 
+
+shiva!
+[16/03/2022, 03:25:38 PM] person2: Hi 👋
+[16/03/2022, 03:26:48 PM] person3: Hoi!
+[16/03/2022, 06:39:29 PM] person2: https://youtu.be/KBmUTY6mK_E
+[16/03/2022, 06:35:51 PM] person1: ‎Location: https://maps.google.com/?q=52.089451,5.108469
+‎[20/03/2022, 08:08:51 PM] person4: I’m about to generate some very random messages so that I can make some screenshots for the explanation to participants
+‎[24/03/2022, 08:19:38 PM] person1: @user3 if you remove your Profile picture for a moment I will redo the screenshots 😁
+[26/03/2022, 06:52:15 PM] person2: Well done Utrecht 😁
+[14/07/2020, 10:05:54 PM] person4: 👍Bedankt
diff --git a/data_extractor/tests/data/_chat_file6.txt b/data_extractor/tests/data/_chat_file6.txt
new file mode 100644
index 0000000..b6466a4
--- /dev/null
+++ b/data_extractor/tests/data/_chat_file6.txt
@@ -0,0 +1,12 @@
+[03/16/2022, 03:10:17 PM] Messages and calls are end-to-end encrypted. No one outside of this chat, not even WhatsApp, can read or listen to them. Tap to learn more.
+[03/16/2022, 03:20:25 PM] person1: Hi 
+
+shiva!
+[03/16/2022, 03:25:38 PM] person2: Hi 👋
+[03/16/2022, 03:26:48 PM] person3: Hoi!
+[03/16/2022, 06:39:29 PM] person2: https://youtu.be/KBmUTY6mK_E
+[03/16/2022, 06:35:51 PM] person1: ‎Location: https://maps.google.com/?q=52.089451,5.108469
+‎[03/20/2022, 08:08:51 PM] person4: I’m about to generate some very random messages so that I can make some screenshots for the explanation to participants
+‎[03/24/2022, 08:19:38 PM] person1: @user3 if you remove your Profile picture for a moment I will redo the screenshots 😁
+[03/26/2022, 06:52:15 PM] person2: Well done Utrecht 😁
+[07/14/2020, 10:05:54 PM] person4: 👍Bedankt
diff --git a/data_extractor/tests/test_whatsapp_account_info.py b/data_extractor/tests/test_whatsapp_account_info.py
index d5ecb2e..fb8b945 100644
--- a/data_extractor/tests/test_whatsapp_account_info.py
+++ b/data_extractor/tests/test_whatsapp_account_info.py
@@ -1,8 +1,10 @@
+""" Test script for the whatsapp_account_info script"""
+
+from pathlib import Path
 import pandas as pd
 from pandas.testing import assert_frame_equal
 
 from whatsapp_account_info import process
-from pathlib import Path
 
 
 DATA_PATH = Path(__file__).parent / "data"
@@ -15,7 +17,8 @@
 
 def test_process():
     """ Test process function.
-        compares the expected dataframe with the output of the process function to check if all the columns are matched.
+        compares the expected dataframe with the output of the process function
+        to check if all the columns are matched.
         Raises
         -------
         AssertionError: When provided expected dataframe could not match the participants dataframe
@@ -25,4 +28,3 @@ def test_process():
     result = process(DATA_PATH.joinpath("account_info.zip"))
     df_result = result[0]["data_frame"]
     assert_frame_equal(df_result, df_expected)
-
diff --git a/data_extractor/tests/test_whatsapp_chat.py b/data_extractor/tests/test_whatsapp_chat.py
index 87f3768..85c1880 100644
--- a/data_extractor/tests/test_whatsapp_chat.py
+++ b/data_extractor/tests/test_whatsapp_chat.py
@@ -1,4 +1,8 @@
+""" Test script for the whatsapp_account_info script"""
 from pathlib import Path
+import pytest
+import itertools
+from typing import Tuple
 import pandas as pd
 
 from whatsapp_chat import process
@@ -8,43 +12,45 @@
 
 
 DATA_PATH = Path(__file__).parent / "data"
+FILES_TO_TEST = [ p.name for p in DATA_PATH.glob("*_chat*.txt")]
 
 EXPECTED = [
     {'username': 'person1', 'Total number of words': 20, 'Number of URLs': 1,
      'Number of shared locations': 1, 'file_no': 0, 'Number of messages': 3,
      'Date first message': pd.to_datetime('2022-03-16 15:20:25'),
      'Date last message': pd.to_datetime('2022-03-24 20:19:38'),
-     'user_reply2': 'person2', 'reply_2_user': 'person2'},
+     'Who do you most often reply to?': 'person2',
+     'Who replies to you the most often?': 'person2'},
 
     {'username': 'person2', 'Total number of words': 7, 'Number of URLs': 1,
      'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 3,
      'Date first message': pd.to_datetime('2022-03-16 15:25:38'),
      'Date last message': pd.to_datetime('2022-03-26 18:52:15'),
-     'user_reply2': 'person1', 'reply_2_user': 'person1'},
+     'Who do you most often reply to?': 'person1',
+     'Who replies to you the most often?': 'person1'},
 
     {'username': 'person3', 'Total number of words': 1, 'Number of URLs': 0,
      'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 1,
      'Date first message': pd.to_datetime('2022-03-16 15:26:48'),
      'Date last message': pd.to_datetime('2022-03-16 15:26:48'),
-     'user_reply2': 'person2', 'reply_2_user': 'person2'},
+     'Who do you most often reply to?': 'person2',
+     'Who replies to you the most often?': 'person2'},
 
     {'username': 'person4', 'Total number of words': 21, 'Number of URLs': 0,
      'Number of shared locations': 0, 'file_no': 0, 'Number of messages': 2,
      'Date first message': pd.to_datetime('2020-07-14 22:05:54'),
      'Date last message': pd.to_datetime('2022-03-20 20:08:51'),
-     'user_reply2': 'person1', 'reply_2_user': 'person1'}
+     'Who do you most often reply to?': 'person1',
+     'Who replies to you the most often?': 'person1'}
 ]
 
 
-def test_process():
-    """ Test process function.
-        compares the expected dataframe with the output of the process function
-         to check if all the columns are match.
-        Raises
-        -------
-        AssertionError: When provided expected dataframe could not match the
-         participants dataframe
-        """
+
+def process_data(filename: str, person_index: int) -> Tuple[pd.DataFrame, pd.DataFrame]:
+    """ 
+    Returns a tuple contaning the excepted output dataframe, 
+    and the dataframe from the process function
+    """
 
     df_expected = pd.DataFrame(EXPECTED)
     df_expected = anonymize_participants(df_expected)
@@ -63,8 +69,8 @@ def test_process():
                                   "Number of URLs",
                                   "file_no",
                                   "Number of shared locations",
-                                  "reply_2_user",
-                                  "user_reply2"],
+                                  "Who replies to you the most often?",
+                                  "Who do you most often reply to?"],
                       var_name='Description', value_name='Value')
 
     usernames = df_melt["username"].unique()
@@ -73,19 +79,43 @@ def test_process():
         results.append(df_user)
 
     expected_results = []
-    for df in results:
-        user_name = pd.unique(df["username"])[0]
+    for df_chat in results:
+        user_name = pd.unique(df_chat["username"])[0]
         expected_results.append(
             {
                 "id": user_name,  # "overview",
                 "title": user_name,  # "The following data is extracted from the file:",
-                "data_frame": df[["Description", "Value"]].reset_index(drop=True)
+                "data_frame": df_chat[["Description", "Value"]].reset_index(drop=True)
             }
         )
 
-    df_result = process(DATA_PATH.joinpath("_chat.txt"))
+    file_to_test = DATA_PATH.joinpath(filename)
+    df_result = process(file_to_test)
+
+    return df_result[person_index], expected_results[person_index]
+
+
+# Generate test conditions
+conditions = list(itertools.product(FILES_TO_TEST, range(4), range(7)))
+
+@pytest.mark.parametrize("filename,person_index,condition_index", conditions)
+def test_process(filename: str, person_index: int, condition_index: int):
+    """ 
+    Compares the expected dataframe with the output of the process function
+    """
+
+    df_result, expected_results =  process_data(filename, person_index)
+    df_expected_results = expected_results["data_frame"]
+    df_result = expected_results["data_frame"]
+
+    # check whether the condition can be tested
+    try:
+        description, expected_value = tuple(df_expected_results[["Description", "Value"]].iloc[condition_index])
+        description_result, value = tuple(df_result[["Description", "Value"]].iloc[condition_index])
+    except:
+
+        return
+    assert value == expected_value, f"In {filename} for person {person_index}, test: {description} FAILED, {value} != {expected_value}"
+
+
 
-    assert_frame_equal(df_result[0]["data_frame"], expected_results[0]["data_frame"])
-    assert_frame_equal(df_result[1]["data_frame"], expected_results[1]["data_frame"])
-    assert_frame_equal(df_result[2]["data_frame"], expected_results[2]["data_frame"])
-    assert_frame_equal(df_result[3]["data_frame"], expected_results[3]["data_frame"])
diff --git a/data_extractor/whatsapp_account_info/__init__.py b/data_extractor/whatsapp_account_info/__init__.py
index 26cbd7e..fe06543 100644
--- a/data_extractor/whatsapp_account_info/__init__.py
+++ b/data_extractor/whatsapp_account_info/__init__.py
@@ -2,18 +2,18 @@
 
 import zipfile
 import re
-import pandas as pd
 import json
+import pandas as pd
 
 HIDDEN_FILE_RE = re.compile(r".*__MACOSX*")
 FILE_RE = re.compile(r".*.json$")
 
 
 class ColnamesDf:
-    GROUPS = 'groups'
+    GROUPS = 'wa_groups'
     """Groups column"""
 
-    CONTACTS = 'contacts'
+    CONTACTS = 'wa_contacts'
     """Contacts column"""
 
 
@@ -21,26 +21,57 @@ class ColnamesDf:
 
 
 def format_results(df):
+    """Function for formatting results to the Eyra's standard format"""
     results = []
     results.append(
         {
-        "id": "Whatsapp account info",
-        "title": "The account information file is read:",
-        "data_frame": df
+            "id": "Whatsapp account info",
+            "title": "The account information file is read:",
+            "data_frame": df
         }
     )
     return results
 
 
 def format_errors(errors):
+    """ Function for formatting logging messages as a dataframe """
     data_frame = pd.DataFrame()
     data_frame["Messages"] = pd.Series(errors, name="Messages")
     return {"id": "extraction_log", "title": "Extraction log", "data_frame": data_frame}
 
 
+def extract_groups(log_error, data):
+    """Function for extracting the number of groups"""
+
+    groups_no = 0
+    try:
+        groups_no = len(data[COLNAMES_DF.GROUPS])
+    except (TypeError, KeyError) as e:
+        print("No group is available")
+
+    if groups_no == 0:
+        log_error("No group is available")
+
+    return groups_no
+
+
+def extract_contacts(log_error, data):
+    """Function for extracting the number of contacts"""
+
+    contacts_no = 0
+
+    try:
+        contacts_no = len(data[COLNAMES_DF.CONTACTS])
+    except (TypeError, KeyError) as e:
+        print("No contact is available")
+
+    if contacts_no == 0:
+        log_error("No contact is available")
+    return contacts_no
+
+
 def extract_data(log_error, data):
-    # data = pd.read_csv('whatsapp/df_chat.csv')
-    # return 1,1
+    """Function to extract group_no and contact_no fields - Support for the old format"""
     groups_no = 0
     contacts_no = 0
     try:
@@ -58,6 +89,7 @@ def extract_data(log_error, data):
 
 
 def parse_records(log_error, f):
+    """Function for loading json files content"""
     try:
         data = json.load(f)
     except json.JSONDecodeError:
@@ -67,6 +99,26 @@ def parse_records(log_error, f):
 
 
 def parse_zipfile(log_error, zfile):
+    """Function for extracting input zipfile"""
+    for name in zfile.namelist():
+        if name == 'whatsapp_connections/groups.json':
+            if HIDDEN_FILE_RE.match(name):
+                continue
+            if not FILE_RE.match(name):
+                continue
+            data_groups = parse_records(log_error, zfile.open(name))
+        elif name == 'whatsapp_connections/contacts.json':
+            if HIDDEN_FILE_RE.match(name):
+                continue
+            if not FILE_RE.match(name):
+                continue
+            data_contacts = parse_records(log_error, zfile.open(name))
+    # log_error("No Json file is available")
+    return data_groups, data_contacts
+
+
+def parse_zipfile_old_format(log_error, zfile):
+    """Function for extracting input zipfile"""
     for name in zfile.namelist():
         if HIDDEN_FILE_RE.match(name):
             continue
@@ -77,20 +129,35 @@ def parse_zipfile(log_error, zfile):
 
 
 def process(file_data):
+    """Main function for extracting account information"""
     errors = []
     log_error = errors.append
     zfile = zipfile.ZipFile(file_data)
-    data = parse_zipfile(log_error, zfile)
+    try:
+        data_groups, data_contacts = parse_zipfile(log_error, zfile)
+
+        if data_groups is not None:
+            groups_no = extract_groups(log_error, data_groups)
+
+        if data_contacts is not None:
+            contacts_no = extract_contacts(log_error, data_contacts)
+
+        if errors:
+            return [format_errors(errors)]
 
-    if data is not None:
-        groups_no, contacts_no = extract_data(log_error, data)
+    except:  # Support old format of the account_info data package
+        COLNAMES_DF.GROUPS = 'groups'
+        COLNAMES_DF.CONTACTS = 'contacts'
+        data = parse_zipfile_old_format(log_error, zfile)
+        if data is not None:
+            groups_no, contacts_no = extract_data(log_error, data)
 
-    if errors:
-        return [format_errors(errors)]
+        if errors:
+            return [format_errors(errors)]
 
-    d = {'number_of_groups': [groups_no], 'number_of_contacts': [contacts_no]}
-    df = pd.DataFrame(data=d)
-    formatted_results = format_results(df)
+    data_info = {'number_of_groups': [groups_no], 'number_of_contacts': [contacts_no]}
+    df_info = pd.DataFrame(data=data_info)
+    formatted_results = format_results(df_info)
 
     return formatted_results
 
diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index b193d5b..279d62f 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -10,6 +10,7 @@
 import zipfile
 
 
+
 URL_PATTERN = r"(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)" \
               r"(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|" \
               r"(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'\".,<>?«»“”‘’]))"
@@ -21,8 +22,16 @@
 FILE_RE = re.compile(r".*.txt$")
 HIDDEN_FILE_RE = re.compile(r".*__MACOSX*")
 
+
 SYSTEM_MESSAGES = ['end-to-end','WhatsApp']
-hformats = ['%m/%d/%y, %H:%M - %name:', '[%d/%m/%y, %H:%M:%S] %name:', '%d-%m-%y %H:%M - %name:', '[%d-%m-%y %H:%M:%S] %name:']
+hformats = ['%m/%d/%y, %H:%M - %name:', '[%d/%m/%y, %H:%M:%S] %name:', '%d-%m-%y %H:%M - %name:',
+            '[%d-%m-%y %H:%M:%S] %name:', '[%m/%d/%y, %H:%M:%S] %name:', '%d/%m/%y, %H:%M – %name:',
+            '%d.%m.%y, %H:%M – %name:','[%d/%m/%y, %H:%M:%S %P] %name:','[%m/%d/%y, %H:%M:%S %P] %name:',
+            '[%d.%m.%y, %H:%M:%S] %name:', '[%m/%d/%y %H:%M:%S] %name:', '[%m-%d-%y, %H:%M:%S] %name:',
+            '[%m-%d-%y %H:%M:%S] %name:','%m-%d-%y %H:%M - %name:','%m-%d-%y, %H:%M - %name:',
+            '%m-%d-%y, %H:%M , %name:', '%m/%d/%y, %H:%M , %name:','%d-%m-%y, %H:%M , %name:','%d/%m/%y, %H:%M , %name:',
+            '%d.%m.%y %H:%M – %name:', '%m.%d.%y, %H:%M – %name:', '%m.%d.%y %H:%M – %name:',
+            '[%d.%m.%y %H:%M:%S] %name:','[%m.%d.%y, %H:%M:%S] %name:', '[%m.%d.%y %H:%M:%S] %name:']
 
 
 class ColnamesDf:
@@ -282,7 +291,8 @@ def parse_text(text, regex):
     Returns
     -------
     pandas.DataFrame
-        pandas.DataFrame with messages sent by users, index is the date the messages was sent.
+        pandas.DataFrame with messages sent by users
+
     Raises
     ------
     RegexError
@@ -306,6 +316,67 @@ def parse_text(text, regex):
 
     return df_chat
 
+def make_df_general_regx(log_error,text):
+    """Use a general regex to load chat as a DataFrame.
+        Parameters
+        ----------
+        log_error : list
+            List of error messages
+        text : str
+            Text of the chat
+
+        Returns
+        -------
+        pandas.DataFrame
+            pandas.DataFrame with messages sent by users
+
+        """
+
+    expr_to_test = re.compile(r"^(.*?)(?:\] | - )(.*?): (.*?$)", flags=re.DOTALL)
+    lines = text.split("\n")
+    result = []
+    line_counts = len(lines)
+    for l in lines:
+        try:
+
+            res = expr_to_test.match(l)
+
+            timestamp = res.group(1)
+            username = res.group(2)
+            message = res.group(3)
+
+            # clean timestamp
+            timestamp = timestamp.replace('[','').replace(',','')
+            timestamp = timestamp.strip('\u202c')
+            timestamp = timestamp.strip('\u200e')
+
+            timestamp = pd.to_datetime(timestamp)
+
+
+            line_dict = {
+                COLNAMES_DF.DATE: timestamp,
+                COLNAMES_DF.USERNAME: username,
+                COLNAMES_DF.MESSAGE: message
+            }
+
+            result.append(line_dict)
+
+        except:
+            pass
+
+    df_chat = pd.DataFrame.from_records(result)
+    df_chat = df_chat[[COLNAMES_DF.DATE, COLNAMES_DF.USERNAME, COLNAMES_DF.MESSAGE]]
+
+    # clean username
+    df_chat[COLNAMES_DF.USERNAME] = df_chat[COLNAMES_DF.USERNAME].apply(lambda u: u.strip('\u202c'))
+
+    # log unprocessed_line_no= the number of lines in multiline messages + the number of system messages
+    unprocessed_line_no = line_counts - df_chat.shape[0]
+
+    if unprocessed_line_no > 0:
+        log_error("Number of unprocessed lines: " + str(unprocessed_line_no))
+
+    return df_chat
 
 def make_chat_df(log_error, text, hformat):
     """Load chat as a DataFrame.
@@ -317,6 +388,7 @@ def make_chat_df(log_error, text, hformat):
         Text of the chat
     hformat : str
         Simplified syntax for the header, e.g. ``'%y-%m-%d, %H:%M:%S - %name:'``
+
     Returns
     -------
     pandas.DataFrame
@@ -361,7 +433,12 @@ def parse_chat(log_error, data):
         df = make_chat_df(log_error, data, hformat)
         if df is not None:
              return df
-    log_error("hformats did not match the provided text. No match was found")
+    log_error("hformats did not match the provided text. We try to use a general regex to read the chat file. ")
+    # If header format is unknown to our script we use a loose regular expression to detect
+    df = make_df_general_regx(log_error,data)
+    if df.shape[0] > 0:
+        return df
+    log_error("Failed to read the Chat file.")
     return None
 
 
@@ -605,7 +682,7 @@ def remove_system_messages(log_error, chat):
     is_system_message = True if all(s in message0 for s in SYSTEM_MESSAGES) else False
     if is_system_message:
         group_name = chat.loc[0, COLNAMES_DF.USERNAME]
-        log_error("Identified group name:"+group_name)
+        #log_error("Identified group name:"+group_name)
         chat = chat.loc[chat[COLNAMES_DF.USERNAME] != group_name,]
 
     return chat

From 598cf6637211baaebde5970e7267193456f1b2cd Mon Sep 17 00:00:00 2001
From: emielvdveen <emiel@applike.nl>
Date: Sun, 28 Aug 2022 11:52:41 +0200
Subject: [PATCH 41/42] Fixed support for filename matching (#20)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Filename was a constant: ‘user-data’
- Filename is now copied from the original file
- Parameter in the process method is no longer a BufferedReader but a string (file path).
---
 data_extractor/pyprocess.js              |  3 ++-
 data_extractor/pyworker.js               | 15 +++++++++++++--
 data_extractor/whatsapp_chat/__init__.py |  2 +-
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/data_extractor/pyprocess.js b/data_extractor/pyprocess.js
index 5c04e24..833ef0f 100644
--- a/data_extractor/pyprocess.js
+++ b/data_extractor/pyprocess.js
@@ -48,6 +48,7 @@ function process() {
   resultElement().style.display = "none";
 
   const file = fileInput().files[0];
+  const filename = file.name
   const reader = file.stream().getReader();
   const sendToWorker = ({ done, value }) => {
     if (done) {
@@ -60,7 +61,7 @@ function process() {
     reader.read().then(sendToWorker);
   };
   console.log("process: send event: initData");
-  pyWorker.postMessage({ eventType: "initData", size: file.size });
+  pyWorker.postMessage({ eventType: "initData", filename: filename, size: file.size });
   reader.read().then(sendToWorker);
 }
 
diff --git a/data_extractor/pyworker.js b/data_extractor/pyworker.js
index 15815de..5fb65f5 100644
--- a/data_extractor/pyworker.js
+++ b/data_extractor/pyworker.js
@@ -10,13 +10,15 @@ loadPyodide({ indexURL: "https://cdn.jsdelivr.net/pyodide/v0.19.0/full/" }).then
 });
 
 let file = undefined
+var filename = undefined
 
 onmessage = (event) => {
   const { eventType } = event.data;
   if (eventType === "loadScript") {
     self.pyodide.runPython(event.data.script)
   } else if (eventType === "initData") {
-    file = self.pyodide.FS.open("user-data", "w")
+    filename = event.data.filename
+    file = self.pyodide.FS.open(filename, "w")
   } else if (eventType === "data") {
     self.pyodide.FS.write(file, event.data.chunk, 0, event.data.chunk.length)
   } else if (eventType === "processData") {
@@ -24,7 +26,15 @@ onmessage = (event) => {
     def _process_data():
       import json
       import html
-      result = process(open("user-data", "rb"))
+      import pandas as pd
+
+      result = process("${filename}")
+
+      if not result:
+        data_frame = pd.DataFrame()
+        data_frame["Messages"] = pd.Series(["Unfortunately, no data could be extracted from the selected file."], name="Messages")
+        result = [{"id": "important_feedback", "title": "Important feedback", "data_frame": data_frame}]
+
       data_output = []
       html_output = []
       for data in result:
@@ -32,6 +42,7 @@ onmessage = (event) => {
         df = data['data_frame']
         html_output.append(df.to_html(classes=["data-donation-extraction-results"], justify="left"))
         data_output.append({"id": data["id"], "data_frame": df.to_json()})
+
       return {
         "html": "\\n".join(html_output),
         "data": json.dumps(data_output),
diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index 279d62f..d7160d0 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -770,7 +770,7 @@ def process(file_data):
     try:
         zfile = zipfile.ZipFile(file_data)
     except:
-        if FILE_RE.match(file_data.name):
+        if FILE_RE.match(file_data):
             tfile = open(file_data, encoding="utf8")
             chat = parse_chat(log_error, tfile.read())
 

From 3cb1ab2212a28c8b2ebab288c89857823bb28374 Mon Sep 17 00:00:00 2001
From: parisa-zahedi <parisa.zahedi@gmail.com>
Date: Mon, 19 Sep 2022 11:05:36 +0200
Subject: [PATCH 42/42] Pilot- fix dash (#22)

* Support new account_info format

* Add support for the old version of the data-package

* Rename reply_2user and user_reply2

* add new header formats and a general regex

* Fix some pylint issues

* Fix pylint issues

* Fix pylint issues

* additional unittests for whatsapp chats

* added some documentation

* fix

* add log if general regular expression is used

* change checking file_data based on the changes required from Eyra team

* Added new timestamp with correct character '-'. Note this is a small dash instead of a large dash

* add hformats with short dash

* Fix test error after removing .name

Co-authored-by: Shiva Nadi <s.nadi@uu.nl>
Co-authored-by: trbKnl <ncdeschipper@gmail.com>
Co-authored-by: ShNadi <44059592+ShNadi@users.noreply.github.com>
---
 data_extractor/tests/test_whatsapp_chat.py |  2 +-
 data_extractor/whatsapp_chat/__init__.py   | 12 +++++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/data_extractor/tests/test_whatsapp_chat.py b/data_extractor/tests/test_whatsapp_chat.py
index 85c1880..85f2c6a 100644
--- a/data_extractor/tests/test_whatsapp_chat.py
+++ b/data_extractor/tests/test_whatsapp_chat.py
@@ -90,7 +90,7 @@ def process_data(filename: str, person_index: int) -> Tuple[pd.DataFrame, pd.Dat
         )
 
     file_to_test = DATA_PATH.joinpath(filename)
-    df_result = process(file_to_test)
+    df_result = process(str(file_to_test))
 
     return df_result[person_index], expected_results[person_index]
 
diff --git a/data_extractor/whatsapp_chat/__init__.py b/data_extractor/whatsapp_chat/__init__.py
index d7160d0..348adf5 100644
--- a/data_extractor/whatsapp_chat/__init__.py
+++ b/data_extractor/whatsapp_chat/__init__.py
@@ -26,12 +26,14 @@
 SYSTEM_MESSAGES = ['end-to-end','WhatsApp']
 hformats = ['%m/%d/%y, %H:%M - %name:', '[%d/%m/%y, %H:%M:%S] %name:', '%d-%m-%y %H:%M - %name:',
             '[%d-%m-%y %H:%M:%S] %name:', '[%m/%d/%y, %H:%M:%S] %name:', '%d/%m/%y, %H:%M – %name:',
-            '%d.%m.%y, %H:%M – %name:','[%d/%m/%y, %H:%M:%S %P] %name:','[%m/%d/%y, %H:%M:%S %P] %name:',
+            '%d/%m/%y, %H:%M - %name:', '%d.%m.%y, %H:%M – %name:', '%d.%m.%y, %H:%M - %name:',
+            '[%d/%m/%y, %H:%M:%S %P] %name:', '[%m/%d/%y, %H:%M:%S %P] %name:',
             '[%d.%m.%y, %H:%M:%S] %name:', '[%m/%d/%y %H:%M:%S] %name:', '[%m-%d-%y, %H:%M:%S] %name:',
-            '[%m-%d-%y %H:%M:%S] %name:','%m-%d-%y %H:%M - %name:','%m-%d-%y, %H:%M - %name:',
-            '%m-%d-%y, %H:%M , %name:', '%m/%d/%y, %H:%M , %name:','%d-%m-%y, %H:%M , %name:','%d/%m/%y, %H:%M , %name:',
-            '%d.%m.%y %H:%M – %name:', '%m.%d.%y, %H:%M – %name:', '%m.%d.%y %H:%M – %name:',
-            '[%d.%m.%y %H:%M:%S] %name:','[%m.%d.%y, %H:%M:%S] %name:', '[%m.%d.%y %H:%M:%S] %name:']
+            '[%m-%d-%y %H:%M:%S] %name:', '%m-%d-%y %H:%M - %name:', '%m-%d-%y, %H:%M - %name:',
+            '%m-%d-%y, %H:%M , %name:', '%m/%d/%y, %H:%M , %name:', '%d-%m-%y, %H:%M , %name:',
+            '%d/%m/%y, %H:%M , %name:', '%d.%m.%y %H:%M – %name:', '%m.%d.%y, %H:%M – %name:',
+            '%m.%d.%y %H:%M – %name:', '[%d.%m.%y %H:%M:%S] %name:', '[%m.%d.%y, %H:%M:%S] %name:',
+            '[%m.%d.%y %H:%M:%S] %name:']
 
 
 class ColnamesDf: