culturehack.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "culturehack.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "CMNXF-gWGzIa",
        "S7XciTle5CEe",
        "pHcXoJmK_idI",
        "BtOCv3hwH5ot",
        "O85lx7ScCSoY",
        "_NlrNoIVErP6",
        "in690Yn-NiYG",
        "E8a5akNxLEbF",
        "FCo7KpH9snkr",
        "jA3ekeehWHej",
        "OWAu3IFu_RfP",
        "YkrlGbws_4mT",
        "ghIU5tXy_4mZ",
        "J1iNXIGq_4mb",
        "auNI8y3i_4mf",
        "EicqpMSz_4mg",
        "66900lC0_4mi",
        "6urTYmcS_4mk",
        "o-84c_H-_4ml",
        "B-Byg0LP_4mn",
        "KtV5Oee-HKb1"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# [CultureHackLabs](https://github.com/culturehacklabs)\n",
        "\n",
        "> Mayner WGP, Marshall W, Albantakis L, Findlay G, Marchman R, Tononi G. [PyPhi: A toolbox for integrated information theory](https://doi.org/10.1371/journal.pcbi.1006343). PLOS Computational Biology 14(7): e1006343. 2018."
      ],
      "metadata": {
        "id": "dzGOMk6MApFm"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load requirements"
      ],
      "metadata": {
        "id": "CMNXF-gWGzIa"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Libraries"
      ],
      "metadata": {
        "id": "S7XciTle5CEe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q graphviz kaleido langdetect matplotlib networkx nltk numpy pandas plotly pydot pyphi scikit-learn scipy # pygraphviz\n",
        "!wget -q https://gist.githubusercontent.com/nelsonaloysio/2eb360eeffacdcb3f8ad305ab85dc398/raw/eeab6d6050b53142810aacde4f4bb1296c620f7b/iso639-1.json\n",
        "!wget -q https://gist.githubusercontent.com/nelsonaloysio/302dbbf3963fababde6e9f97669587df/raw/0f0523749a30ded1422a69103547bae7dddc8933/stopwords.py"
      ],
      "metadata": {
        "id": "dmVtl-MUuX-e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "import logging as log\n",
        "import math\n",
        "import os\n",
        "import re\n",
        "import string\n",
        "from abc import ABCMeta, abstractmethod\n",
        "from functools import reduce\n",
        "from inspect import signature\n",
        "from itertools import combinations\n",
        "from math import atan2\n",
        "from math import log as log10\n",
        "from math import pi\n",
        "from re import findall\n",
        "from shutil import move\n",
        "from subprocess import call\n",
        "from time import time\n",
        "from typing import Callable, Union\n",
        "from urllib.request import urlopen\n",
        "os.environ[\"PYPHI_WELCOME_OFF\"] = \"yes\"\n",
        "\n",
        "import graphviz as gv\n",
        "import matplotlib.pyplot as plt\n",
        "import networkx as nx\n",
        "import nltk\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import plotly.graph_objs as go\n",
        "import plotly.io as pio\n",
        "import plotly.offline as py\n",
        "import pyphi\n",
        "from langdetect import detect as lang_detect\n",
        "from langdetect.detector import LangDetectException\n",
        "from nltk.stem.snowball import SnowballStemmer\n",
        "from plotly.graph_objs import Figure\n",
        "from plotly.subplots import make_subplots\n",
        "from sklearn.base import TransformerMixin\n",
        "from sklearn.cluster import KMeans, MiniBatchKMeans\n",
        "from sklearn.decomposition import TruncatedSVD\n",
        "from sklearn.exceptions import NotFittedError\n",
        "from sklearn.feature_extraction.text import (HashingVectorizer,\n",
        "                                             TfidfTransformer,\n",
        "                                             TfidfVectorizer)\n",
        "from sklearn.metrics import silhouette_score\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.preprocessing import Normalizer\n",
        "\n",
        "from stopwords import CUSTOM_STOPWORDS\n",
        "\n",
        "try:\n",
        "    import pygraphviz as pgv\n",
        "except ModuleNotFoundError as e:\n",
        "    log.warning(e)\n",
        "\n",
        "nltk.download(\"punkt\")\n",
        "nltk.download(\"stopwords\")\n",
        "pio.templates.default = \"none\""
      ],
      "metadata": {
        "id": "f8rV9x-_E1C5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ACCENT_REPLACEMENTS = {\n",
        "    ord(\"á\"): \"a\", ord(\"ã\"): \"a\", ord(\"â\"): \"a\",\n",
        "    ord(\"à\"): \"a\", ord(\"è\"): \"e\", ord(\"ê\"): \"e\",\n",
        "    ord(\"é\"): \"e\", ord(\"í\"): \"i\", ord(\"ì\"): \"i\",\n",
        "    ord(\"ñ\"): \"n\", ord(\"ò\"): \"o\", ord(\"ó\"): \"o\",\n",
        "    ord(\"ô\"): \"o\", ord(\"õ\"): \"o\", ord(\"ù\"): \"u\",\n",
        "    ord(\"ú\"): \"u\", ord(\"ü\"): \"u\", ord(\"ç\"): \"c\"}\n",
        "\n",
        "IGNORE_STARTS_WITH = [\"http\", \"www\", \"kkk\"]\n",
        "INVALID_CHARACTERS = \"\\\\\\\"'’…|–—“”‘„•¿¡\"\n",
        "VALID_CHARACTERS = \"@#\"\n",
        "\n",
        "CHARACTER_REPLACEMENTS = str.maketrans(\"\", \"\", \"\".join(\n",
        "    set(string.punctuation + INVALID_CHARACTERS) - set(VALID_CHARACTERS)))\n",
        "\n",
        "LABEL_MAP_FUNC = lambda x: \" \".join(\n",
        "    [x.capitalize() for x in (x if isinstance(x, tuple) else [x])]\n",
        ")\n",
        "\n",
        "REINDEX_MAP_FUNC = lambda x, y: {\n",
        "    k: {k_: y[v_].drop_duplicates().tolist() for k_, v_ in v.items()}\n",
        "    if isinstance(v, dict) else y[v].drop_duplicates().tolist() for k, v in x.items()\n",
        "}\n",
        "\n",
        "RENAME_TITLE_MAP_FUNC = lambda x, y: {\n",
        "    k: {k_: (\"%s %s\" % (y, (v_.lower() if y is not None else v_))) if k_ == \"title\" and k == \"layout_opts\" else v_ for k_, v_ in v.items()}\n",
        "    if type(v) is dict else v for k, v in x.items()\n",
        "}\n",
        "\n",
        "AUTORANGE = True\n",
        "CONNECT_GAPS = False\n",
        "FONT_COLOR = \"grey\"\n",
        "FONT_FAMILY = \"Raleway, Arial, sans-serif\"\n",
        "FONT_SIZE = 16\n",
        "LEGEND_Y = 0.5\n",
        "LEGEND_YREF = \"paper\"\n",
        "MARKER_SIZE = 6\n",
        "TEXT_POSITION = \"top center\"\n",
        "\n",
        "with open(\"iso639-1.json\", \"r\") as j:\n",
        "    ISO639 = json.loads(j.read())"
      ],
      "metadata": {
        "id": "iFes1hI-E2s8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Cluster pipeline"
      ],
      "metadata": {
        "id": "pHcXoJmK_idI"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b3J1Itgx_4ls"
      },
      "source": [
        "class Transformer(metaclass=ABCMeta):\n",
        "    \"\"\"\n",
        "    Asbtract base transformer class.\n",
        "    \"\"\"\n",
        "    @abstractmethod\n",
        "    def __init__(self, **kwargs):\n",
        "        \"\"\" Abstract initializer class. \"\"\"\n",
        "\n",
        "    def fit(self, X, y=None):\n",
        "        \"\"\" Just returns class, nothing to fit. \"\"\"\n",
        "        return self\n",
        "\n",
        "    @abstractmethod\n",
        "    def transform(self, X):\n",
        "        \"\"\" Abstract method for \"DIY\" transformations. \"\"\"\n",
        "\n",
        "\n",
        "class PandasTransformer(Transformer):\n",
        "    \"\"\"\n",
        "    Transform path or data frame into series object.\n",
        "    \"\"\"\n",
        "    def __init__(self,\n",
        "            applymap=lambda x:x,\n",
        "            column=None,\n",
        "            drop_duplicates=False,\n",
        "            dropna=False,\n",
        "            json_lines=False,\n",
        "            low_memory=False,\n",
        "            sep=None,\n",
        "            sort=[],\n",
        "            **kwargs):\n",
        "\n",
        "        self.applymap = applymap\n",
        "        self.column = column\n",
        "        self.drop_duplicates = drop_duplicates\n",
        "        self.dropna = dropna\n",
        "        self.json_lines = json_lines\n",
        "        self.low_memory = low_memory\n",
        "        self.sep = sep\n",
        "        self.sort = sort\n",
        "\n",
        "    def transform(self, path_or_df: Union[str, pd.Series, pd.DataFrame]):\n",
        "        \"\"\" Returns Pandas Series from data frame or file path. \"\"\"\n",
        "        usecols = [self.column] + self.sort if self.column else None\n",
        "        # Load data from all or specified columns\n",
        "        series = path_or_df\\\n",
        "                 if isinstance(path_or_df, pd.Series)\\\n",
        "                 else pd.Series()\\\n",
        "                 if isinstance(path_or_df, pd.DataFrame)\\\n",
        "                 and path_or_df.empty\\\n",
        "                 else self.filter_column(\n",
        "                        self.sort_values(\n",
        "                          self.read_table(\n",
        "                            self.read_json(\n",
        "                              path_or_df\n",
        "                            ),\n",
        "                            sep=self.sep if self.column else \"\\n\",\n",
        "                            usecols=usecols,\n",
        "                            low_memory=self.low_memory\n",
        "                            ),\n",
        "                          columns=self.sort\n",
        "                          ),\n",
        "                        column=self.column)\n",
        "        # Check if type is correct\n",
        "        if isinstance(series, pd.DataFrame):\n",
        "            raise TypeError(f\"Expected a Pandas Series or 1-dimensional DataFrame (column='{self.column}').\")\n",
        "        # Apply map and drop duplicates/nulls\n",
        "        series = series.copy()\n",
        "        if self.applymap:\n",
        "            series = series.apply(self.applymap)\n",
        "        if self.drop_duplicates:\n",
        "            series.drop_duplicates(inplace=True)\n",
        "        if self.dropna:\n",
        "            series.dropna(inplace=True)\n",
        "        # Store difference between indices\n",
        "        self.index_ = series.index\n",
        "        self.skiprows_ = self.index_.difference(series.index)\n",
        "        return series.copy()\n",
        "\n",
        "    def read_table(self, path_or_df: Union[str, list, pd.Series, pd.DataFrame],\n",
        "                   sep=None, skiprows=None, usecols=None, low_memory=False):\n",
        "        \"\"\" Returns or loads Pandas data frame from file path. \"\"\"\n",
        "        return pd.read_table(\n",
        "                  path_or_df,\n",
        "                  usecols=usecols if usecols else None,\n",
        "                  sep=sep if sep else self._get_file_delimiter(path_or_df),\n",
        "                  skiprows=skiprows,\n",
        "                  low_memory=low_memory)\\\n",
        "               if isinstance(path_or_df, str)\\\n",
        "               else self.concat(\n",
        "                  path_or_df,\n",
        "                  usecols)\\\n",
        "               if isinstance(path_or_df, list)\\\n",
        "               else path_or_df[usecols]\\\n",
        "                  if usecols\\\n",
        "                  and any(x for x in path_or_df.shape)\\\n",
        "                  else path_or_df\n",
        "\n",
        "    def read_json(self, X):\n",
        "        \"\"\" Return Pandas data frame from JSON file. \"\"\"\n",
        "        return pd.read_json(X)\\\n",
        "               if isinstance(X, str)\\\n",
        "               and X.endswith(\".json\")\\\n",
        "               else X\n",
        "\n",
        "    def filter_column(self, df: Union[pd.Series, pd.DataFrame], column=None):\n",
        "        \"\"\" Returns Pandas series from loaded data frame. \"\"\"\n",
        "        return df[df.columns[column]\n",
        "                  if isinstance(column, int)\n",
        "                  else column]\\\n",
        "               if column\\\n",
        "               and any(x for x in df.shape)\\\n",
        "               else df\n",
        "\n",
        "    def sort_values(self, df: Union[pd.Series, pd.DataFrame], columns=None):\n",
        "        \"\"\" Returns Pandas series from loaded data frame. \"\"\"\n",
        "        return df.sort_values([\n",
        "                    df.columns[column]\n",
        "                    if isinstance(column, int)\n",
        "                    else column\n",
        "                  for column in columns],\n",
        "                  ascending=False)\\\n",
        "               if columns\\\n",
        "               and any(x for x in df.shape)\\\n",
        "               else df\n",
        "\n",
        "    def concat(self, dfs: list, columns=None):\n",
        "        \"\"\" Return a single data frame with concatenated rows. \"\"\"\n",
        "        if not isinstance(dfs, list):\n",
        "            return dfs\n",
        "\n",
        "        if columns:\n",
        "            dfs = [df.filter(columns) for df in dfs]\n",
        "\n",
        "        df = pd.concat(dfs)\n",
        "        df.index = range(df.shape[0])\n",
        "        return df\n",
        "\n",
        "    @staticmethod\n",
        "    def _get_file_delimiter(path):\n",
        "        \"\"\" Returns character delimiter from file. \"\"\"\n",
        "        delimiters = [\"|\", \"\\t\", \";\", \",\"]\n",
        "        with open(path, \"rt\") as f:\n",
        "            header = f.readline()\n",
        "        for i in delimiters:\n",
        "            if i in header:\n",
        "                return i\n",
        "        return \"\\n\"\n",
        "\n",
        "\n",
        "class LSA(Pipeline):\n",
        "    \"\"\"\n",
        "    Truncated single value decomposition pipeline used for\n",
        "    dimensionality reduction by latent semantic analysis.\n",
        "    \"\"\"\n",
        "    def __init__(self, copy=False, n_components=100, random_state=None, **kwargs):\n",
        "        self.copy = copy\n",
        "        self.n_components = n_components\n",
        "        self.random_state = random_state\n",
        "\n",
        "        steps = (\n",
        "            (\"svd\",\n",
        "                TruncatedSVD(self.n_components,\n",
        "                             random_state=self.random_state)),\n",
        "            (\"normalizer\",\n",
        "                Normalizer(copy=self.copy)))\n",
        "\n",
        "        super().__init__(steps=steps)\n",
        "\n",
        "    def evar(self):\n",
        "        if \"explained_variance_ratio_\" in self.named_steps.svd.__dict__.keys():\n",
        "            return f\"{int(self.named_steps.svd.explained_variance_ratio_.sum()*100)}%\"\n",
        "\n",
        "        raise NotFittedError(\n",
        "            \"This instance is not fitted yet. Call 'fit' with appropriate arguments before using this estimator.\")\n",
        "\n",
        "\n",
        "class TextVectorizer(Pipeline):\n",
        "    \"\"\"\n",
        "    Hashing, vectorizing and tf-idf pipelines.\n",
        "    \"\"\"\n",
        "    def __init__(self,\n",
        "                 alternate_sign=False,\n",
        "                 analyzer=\"word\",\n",
        "                 max_df=1.0,\n",
        "                 min_df=1,\n",
        "                 n_features=None,\n",
        "                 norm=\"l2\",\n",
        "                 stop_words=None,\n",
        "                 use_hashing=False,\n",
        "                 use_idf=True,\n",
        "                 **kwargs):\n",
        "\n",
        "        self.alternate_sign = alternate_sign\n",
        "        self.analyzer = analyzer\n",
        "        self.max_df = max_df\n",
        "        self.min_df = min_df\n",
        "        self.n_features = n_features\n",
        "        self.norm = norm\n",
        "        self.stop_words = stop_words\n",
        "        self.use_hashing = use_hashing\n",
        "        self.use_idf = use_idf\n",
        "\n",
        "        steps = []\n",
        "        if use_hashing:\n",
        "            steps.append(\n",
        "                (\"hasher\",\n",
        "                 HashingVectorizer(\n",
        "                     stop_words=self.stop_words,\n",
        "                     n_features=self.n_features,\n",
        "                     alternate_sign=self.alternate_sign,\n",
        "                     norm=self.norm)\n",
        "                )\n",
        "            )\n",
        "            if use_idf:\n",
        "                steps.append(\n",
        "                    (\"tfidf\",\n",
        "                     TfidfTransformer(\n",
        "                       norm=self.norm\n",
        "                       )\n",
        "                    )\n",
        "                )\n",
        "        else:\n",
        "            steps.append(\n",
        "                (\"tfidf\",\n",
        "                 TfidfVectorizer(\n",
        "                     analyzer=self.analyzer,\n",
        "                     max_df=self.max_df,\n",
        "                     min_df=self.min_df,\n",
        "                     stop_words=self.stop_words,\n",
        "                     max_features=self.n_features,\n",
        "                     use_idf=self.use_idf)\n",
        "                )\n",
        "            )\n",
        "\n",
        "        super().__init__(steps=steps)\n",
        "\n",
        "\n",
        "class Tokenizer(TransformerMixin):\n",
        "\n",
        "    def __init__(self, max_paragraphs=None, stop_words=[], **kwargs):\n",
        "        self.max_paragraphs = max_paragraphs\n",
        "        self.stop_words = stop_words\n",
        "\n",
        "    def fit(self, X, y=None):\n",
        "        \"\"\" Just returns class, nothing to fit. \"\"\"\n",
        "        return self\n",
        "\n",
        "    def transform(self, X):\n",
        "        \"\"\" Abstract method for \"DIY\" transformations. \"\"\"\n",
        "        return [\n",
        "            \" \".join(\n",
        "                \" \".join(\n",
        "                    self.tokenize(sent)\n",
        "                )\\\n",
        "                for sent in (\n",
        "                    x.split(\"\\n\")[:self.max_paragraphs]\n",
        "                    if isinstance(x, str) else \"\"\n",
        "                )\n",
        "            )\n",
        "            for x in X\n",
        "        ]\n",
        "\n",
        "    def tokenize(self, sentence: str):\n",
        "        \"\"\"\n",
        "        Returns word token, cleared from emojis, accents and punctuation.\n",
        "        \"\"\"\n",
        "        return [\n",
        "            x\n",
        "            .replace(\"](\", \" \")\n",
        "            .translate(ACCENT_REPLACEMENTS)\n",
        "            .translate(CHARACTER_REPLACEMENTS)\n",
        "            for x in\n",
        "                self.clear_emojis(sentence)\n",
        "                .lower()\n",
        "                .split()\n",
        "            if\n",
        "                len(x) > 2\n",
        "            and\n",
        "                x.strip(VALID_CHARACTERS) not in self.stop_words\n",
        "            and\n",
        "                not self.is_number(x)\n",
        "            and\n",
        "                not any(x.startswith(_) for _ in IGNORE_STARTS_WITH)\n",
        "        ]\n",
        "\n",
        "    @staticmethod\n",
        "    def is_number(str_word):\n",
        "        \"\"\"\n",
        "        Check string as an integer or float.\n",
        "        \"\"\"\n",
        "        try:\n",
        "            int(str_word)\n",
        "        except:\n",
        "            try:\n",
        "                float(str_word)\n",
        "            except:\n",
        "                return False\n",
        "        return True\n",
        "\n",
        "    @staticmethod\n",
        "    def clear_emojis(str_text, replace_with=r' '):\n",
        "        \"\"\"\n",
        "        Returns string after clearing from emojis.\n",
        "        \"\"\"\n",
        "        return re\\\n",
        "            .compile(\"[\"\n",
        "                u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "                u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "                u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "                u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
        "                u\"\\U00002702-\\U000027B0\"  # extra (1)\n",
        "                u\"\\U000024C2-\\U0001F251\"  # extra (2)\n",
        "                u\"\\U0000200B-\\U0000200D\"  # zero width\n",
        "                \"]+\", flags=re.UNICODE)\\\n",
        "            .sub(replace_with, str_text)\n",
        "\n",
        "    @staticmethod\n",
        "    def ngrams(tokens: list, n=2):\n",
        "        \"\"\"\n",
        "        Returns n-grams from list of tokens.\n",
        "        \"\"\"\n",
        "        return [\n",
        "            g\n",
        "            for g in\n",
        "                list(nltk.ngrams(tokens, n) if tokens else [])\n",
        "            if\n",
        "                len(set(g)) == n\n",
        "        ]\n",
        "\n",
        "\n",
        "class Stemmer(TransformerMixin):\n",
        "    \"\"\"\n",
        "    Returns Pandas series with stemmed words.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, detect_lang=False, ignore_stopwords=True, **kwargs):\n",
        "        self.detect_lang = detect_lang\n",
        "        self.ignore_stopwords = ignore_stopwords\n",
        "\n",
        "    def fit(self, X, y=None):\n",
        "        \"\"\" Just returns class, nothing to fit. \"\"\"\n",
        "        return self\n",
        "\n",
        "    def transform(self, X, y):\n",
        "        \"\"\" Returns word stems only from available languages. \"\"\"\n",
        "        return [\n",
        "            \"\\n\".join(\n",
        "                [self._stem(_, lang) for _ in sent.split('\\n')]\n",
        "            )\n",
        "            for sent, lang in zip(X, y)\n",
        "        ]\n",
        "\n",
        "    def _stem(self, sentence, lang=None):\n",
        "        \"\"\"\n",
        "        Matches ISO 639-1 language code and returns stemmed words.\n",
        "        Optionally tries to detect language if `lang` is set as \"detect\".\n",
        "        \"\"\"\n",
        "        if (lang is None) and self.detect_lang:\n",
        "            try:\n",
        "                lang = lang_detect(sentence)\n",
        "            except LangDetectException as e:  # No detected language\n",
        "                log.debug(f\"LangDetectException: {e}.\")\n",
        "\n",
        "        lang = ISO639.get(lang, lang)\n",
        "\n",
        "        if lang in SnowballStemmer.languages:\n",
        "            stemmer = SnowballStemmer(language=lang, ignore_stopwords=self.ignore_stopwords)\n",
        "            return \" \".join([\n",
        "                stemmer.stem(w)\n",
        "                for w in sentence.split()\n",
        "                if not any(w.startswith(char)\n",
        "                for char in VALID_CHARACTERS.split())\n",
        "            ])\n",
        "\n",
        "        log.debug(f\"SnowballStemmer '{lang}' not found. Skipping...\")\n",
        "        return sentence\n",
        "\n",
        "\n",
        "class KMeansCluster():\n",
        "    \"\"\"\n",
        "    K-Means algorithm with optimal k-value validation\n",
        "    through WCSS, silhouette scores and gap statistics.\n",
        "    \"\"\"\n",
        "    def __init__(\n",
        "        self,\n",
        "        batch_size=1000,\n",
        "        copy=False,\n",
        "        init=\"k-means++\",\n",
        "        init_size=1000,\n",
        "        knee_angle=60,\n",
        "        max_iter=100,\n",
        "        max_k=13,\n",
        "        metric=\"euclidean\",\n",
        "        min_k=1,\n",
        "        mini_batch=False,\n",
        "        n_clusters=None,\n",
        "        n_init=1,\n",
        "        nrefs=3,\n",
        "        random_state=None,\n",
        "        sample_size=1000,\n",
        "        validator=\"wcss\",\n",
        "        verbose=False,\n",
        "        **kwargs\n",
        "    ) -> None:\n",
        "        self.batch_size = batch_size\n",
        "        self.copy = copy\n",
        "        self.init = init\n",
        "        self.init_size = init_size\n",
        "        self.knee_angle = knee_angle\n",
        "        self.max_iter = max_iter\n",
        "        self.max_k = max_k\n",
        "        self.metric = metric\n",
        "        self.min_k = min_k\n",
        "        self.mini_batch = mini_batch\n",
        "        self.n_clusters = n_clusters\n",
        "        self.n_init = n_init\n",
        "        self.nrefs = nrefs\n",
        "        self.random_state = random_state\n",
        "        self.sample_size = sample_size\n",
        "        self.validator = validator\n",
        "        self.verbose = verbose\n",
        "\n",
        "    def KMeans(self, n_clusters: int):\n",
        "        \"\"\"\n",
        "        Returns classical or Mini-Batch K-Means implementation:\n",
        "        https://www.eecs.tufts.edu/~dsculley/papers/fastkmeans.pdf\n",
        "        \"\"\"\n",
        "        if self.mini_batch:\n",
        "            return MiniBatchKMeans(n_clusters=n_clusters,\n",
        "                                   batch_size=self.batch_size,\n",
        "                                   init=self.init,\n",
        "                                   init_size=self.init_size,\n",
        "                                   n_init=self.n_init,\n",
        "                                   random_state=self.random_state,\n",
        "                                   verbose=self.verbose)\n",
        "\n",
        "        return KMeans(n_clusters=n_clusters,\n",
        "                      init=self.init,\n",
        "                      max_iter=self.max_iter,\n",
        "                      n_init=self.n_init,\n",
        "                      random_state=self.random_state,\n",
        "                      verbose=self.verbose)\n",
        "\n",
        "    def KMeansOptimal(self, X, n_clusters: Union[int, list, range, None], alg=\"wcss\"):\n",
        "        \"\"\"\n",
        "        Returns K-Means function with optimal number of clusters\n",
        "        considering their within-cluster sum of square values (WCSS),\n",
        "        an approximation of the 'elbow' approach for fast clustering.\n",
        "        Allows a function as validator, e.g.: silhouette() and gap().\n",
        "        \"\"\"\n",
        "        kms = {}\n",
        "        wcss = {}\n",
        "        x_diff = 1\n",
        "\n",
        "        for k in (n_clusters if n_clusters else range(self.min_k, self.max_k)):\n",
        "\n",
        "            if k <= X.shape[0]:\n",
        "                km = self.KMeans(k).fit(X)\n",
        "                wcss[k] = km.inertia_/X.shape[0]\n",
        "                kms[km] = wcss[k] if alg == \"wcss\" else alg(self, X, km)\n",
        "                log.info(f\"k={k}: {'{:.05f}'.format(kms[km])} ({alg if isinstance(alg, str) else alg.__name__})\")\n",
        "\n",
        "                if alg == \"wcss\":\n",
        "                    y_diff = wcss.get(k-1, wcss[k]) - wcss[k]\n",
        "                    radians = atan2(x_diff, y_diff*100)\n",
        "                    angle = (radians * 180) / pi\n",
        "\n",
        "                    if self.knee_angle < angle < 90:\n",
        "                        log.info(f\"Stopped at k={k} (angle: {'{:.02f}'.format(angle)}º).\")\n",
        "                        return km\n",
        "\n",
        "        # Return maximum k-value if validator is WCSS (knee/angle)\n",
        "        km = km if alg == \"wcss\" else max(kms, key=lambda key: kms[key])\n",
        "        log.info(f\"Optimal number of clusters set as k={km.n_clusters}.\")\n",
        "        return km\n",
        "\n",
        "    def fit(self, X, y=None):\n",
        "        \"\"\"\n",
        "        Executes K-Means clustering algorithm on vectorized text.\n",
        "        \"\"\"\n",
        "        self.kmeans_ = (\n",
        "            self.KMeans(self.n_clusters).fit(X)\\\n",
        "            if isinstance(self.n_clusters, int)\\\n",
        "            else self.KMeansOptimal(X, self.n_clusters, self.validator)\n",
        "        )\n",
        "        return self\n",
        "\n",
        "    def fit_predict(self, X, y=None):\n",
        "        \"\"\" Returns clusters as predicted by K-Means. \"\"\"\n",
        "        return self.fit(X).kmeans_.predict(X)\n",
        "\n",
        "    def fit_transform(self, X, y=None):\n",
        "        \"\"\" Returns data as transformed by K-Means. \"\"\"\n",
        "        return self.fit(X).kmeans_.transform(X)\n",
        "\n",
        "    def predict(self, X, y=None):\n",
        "        \"\"\" Returns clusters as predicted by K-Means. \"\"\"\n",
        "        return self.kmeans_.predict(X)\n",
        "\n",
        "    def transform(self, X, y=None):\n",
        "        \"\"\" Returns data as transformed by K-Means. \"\"\"\n",
        "        return self.kmeans_.transform(X)\n",
        "\n",
        "    @staticmethod\n",
        "    def silhouette(self, X, km):\n",
        "        \"\"\"\n",
        "        Implementation of silhouette coefficients:\n",
        "        https://doi.org/10.1016/0377-0427(87)90125-7\n",
        "        \"\"\"\n",
        "        return silhouette_score(X, km.labels_,\n",
        "                                metric=self.metric,\n",
        "                                sample_size=self.sample_size)\n",
        "\n",
        "    @staticmethod\n",
        "    def gap(self, X, km):\n",
        "        \"\"\"\n",
        "        Implementation of gap statistics:\n",
        "        https://statweb.stanford.edu/~gwalther/gap\n",
        "        \"\"\"\n",
        "        orig_disp = km.inertia_\n",
        "        ref_disp = np.zeros(self.nrefs)\n",
        "\n",
        "        for i in range(self.nrefs):\n",
        "            r = np.random.random_sample(size=X.shape)\n",
        "            ref_disp[i] = self.KMeans(km.n_clusters).fit(r).inertia_\n",
        "\n",
        "        return np.log(np.mean(ref_disp)) - np.log(orig_disp)\n",
        "\n",
        "\n",
        "class Clusterer(Pipeline):\n",
        "    \"\"\"\n",
        "    Clusterer pipeline with support for Pandas data frames\n",
        "    and series, hashing and text vectorizing (Tf-Idf) and\n",
        "    optional truncated single value decomposition (LSA).\n",
        "    \"\"\"\n",
        "    def __init__(\n",
        "        self,\n",
        "        clustering: Callable[[list], list] = KMeansCluster,\n",
        "        use_pandas: bool = True,\n",
        "        use_svd: bool = False,\n",
        "        use_vect: bool = True,\n",
        "        **kwargs,\n",
        "    ):\n",
        "        steps = []\n",
        "\n",
        "        self.clustering = clustering\n",
        "        self.use_pandas = use_pandas\n",
        "        self.use_svd = use_svd\n",
        "        self.use_vect = use_vect\n",
        "\n",
        "        if self.use_pandas:\n",
        "            steps.append(\n",
        "                (\"pandas\",\n",
        "                    PandasTransformer(**kwargs))\n",
        "            )\n",
        "        if self.use_vect:\n",
        "            steps.append(\n",
        "                (\"vect\",\n",
        "                    TextVectorizer(**kwargs))\n",
        "            )\n",
        "        if self.use_svd:\n",
        "            steps.append(\n",
        "                (\"lsa\",\n",
        "                    LSA(**kwargs))\n",
        "            )\n",
        "            if self.use_svd == \"try\":\n",
        "                self.fit = self.__fit\n",
        "\n",
        "        if self.clustering is not None:\n",
        "            steps.append(\n",
        "                (\"cluster\",\n",
        "                    clustering(**kwargs)\n",
        "                    if clustering == KMeansCluster\n",
        "                    else clustering)\n",
        "            )\n",
        "\n",
        "        super().__init__(steps=steps)\n",
        "\n",
        "    def fit_predict(self, X, y=None):\n",
        "        return self.fit(X).predict(X)\n",
        "\n",
        "    def fit_transform(self, X, y=None):\n",
        "        return self.fit(X).transform(X)\n",
        "\n",
        "    def labels(self, name=None):\n",
        "        if \"cluster\" not in self.named_steps.keys():\n",
        "            raise TypeError(\n",
        "                \"This instance does not have an assigned clustering method in its pipeline.\")\n",
        "\n",
        "        cluster = self.named_steps.cluster.kmeans_\\\n",
        "                  if \"kmeans_\" in self.named_steps.cluster.__dict__.keys()\\\n",
        "                  else self.named_steps.cluster\n",
        "\n",
        "        if \"labels_\" in cluster.__dict__.keys():\n",
        "            return pd.Series(\n",
        "                pd.to_numeric(\n",
        "                    cluster.labels_,\n",
        "                    downcast=\"integer\",\n",
        "                ),\n",
        "                index=self.named_steps.pandas.index_ if \"pandas\" in self.named_steps else None,\n",
        "                name=name,\n",
        "            )\n",
        "\n",
        "        raise NotFittedError(\n",
        "            \"This instance is not fitted yet. Call 'fit' with appropriate arguments before using this estimator.\")\n",
        "\n",
        "    def nearest_features(self, max_items=10, unique=False):\n",
        "        if \"cluster\" not in self.named_steps.keys():\n",
        "            raise TypeError(\n",
        "                \"This instance does not have an assigned clustering method in its pipeline.\")\n",
        "\n",
        "        cluster = self.named_steps.cluster.kmeans_\\\n",
        "                  if \"kmeans_\" in self.named_steps.cluster.__dict__.keys()\\\n",
        "                  else self.named_steps.cluster\n",
        "\n",
        "        if \"vect\" not in self.named_steps:\n",
        "            raise TypeError(\n",
        "                \"This instance does not have an assigned vectorizer method in its pipeline.\")\n",
        "\n",
        "        if self.named_steps.vect.analyzer != \"word\": # (\"char\", \"char_wb\")\n",
        "            raise NotImplementedError(\n",
        "                f\"Method not implemented for vectorized n-grams (analyzer='{self.named_steps.vect.analyzer}').\")\n",
        "\n",
        "        feature_names = self.named_steps.vect[\"tfidf\"].get_feature_names()\n",
        "\n",
        "        if \"cluster_centers_\" not in cluster.__dict__.keys():\n",
        "            if \"labels_\" in cluster.__dict__.keys():\n",
        "                raise NotImplementedError(\n",
        "                    f\"Method not implemented for assigned clustering method (cluster='{type(self.named_steps.cluster).__name__}').\")\n",
        "            raise NotFittedError(\n",
        "                \"This instance is not fitted yet. Call 'fit' with appropriate arguments before using this estimator.\")\n",
        "\n",
        "        order_centroids = (\n",
        "            self.named_steps.lsa[\"svd\"].inverse_transform(\n",
        "                cluster.cluster_centers_\n",
        "            )\n",
        "            if \"lsa\" in self.named_steps.keys()\n",
        "            and self.named_steps.lsa != \"passthrough\"\n",
        "            else cluster.cluster_centers_\n",
        "        ).argsort()[:, ::-1]\n",
        "\n",
        "        k_nearest = {\n",
        "            k: [feature_names[x] for x in order_centroids[k][:max_items]]\n",
        "            for k in range(cluster.n_clusters)}\n",
        "\n",
        "        if unique:\n",
        "            counter = pd.Series(\n",
        "                [f for features in k_nearest.values() for f in features])\\\n",
        "                .value_counts()\n",
        "            k_nearest = {\n",
        "                k: [f for f in features if counter[f] == 1]\n",
        "                for k, features in k_nearest.items()}\n",
        "\n",
        "        return k_nearest\n",
        "\n",
        "    def top_features(self):\n",
        "        return {\n",
        "            k: features[0]\n",
        "            for k, features in\n",
        "            self.nearest_features(\n",
        "                max_items=1,\n",
        "                unique=False,\n",
        "            ).items()\n",
        "        }\n",
        "\n",
        "    def __fit(self, X, y=None):\n",
        "        try:\n",
        "            return super().fit(X)\n",
        "        except ValueError as e:\n",
        "            # Recommended >= 100 features for Latent Semantic Analysis (SVD)\n",
        "            if str(e).startswith(\"n_components must be < n_features\"):\n",
        "                log.warning(\n",
        "                    f\"LSA set as 'passthrough': {e}\"\n",
        "                )\n",
        "                self.steps[\n",
        "                    list(self.named_steps.keys()).index(\"lsa\")\n",
        "                ] = (\"lsa\", \"passthrough\")\n",
        "                return super().fit(X)\n",
        "            raise e"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Artifact generation"
      ],
      "metadata": {
        "id": "BtOCv3hwH5ot"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Plot():\n",
        "\n",
        "    def __init__(self):\n",
        "        \"\"\" Initializes class. \"\"\"\n",
        "\n",
        "    @staticmethod\n",
        "    def plot(\n",
        "        data: Union[dict, list, pd.DataFrame, pd.Series],\n",
        "        x: Union[str, int, None] = None,\n",
        "        y: Union[str, int, None] = None,\n",
        "        graph: str = \"scatter\",\n",
        "        layout: str = \"layout\",\n",
        "        layout_opts: dict = {},\n",
        "        name: dict = {},\n",
        "        size: dict = {},\n",
        "        text: Union[dict, list] = None,\n",
        "        resizer: Callable[[float], float] = lambda x: x,\n",
        "        **opts,\n",
        "    )-> go.Figure:\n",
        "        \"\"\"\n",
        "        Returns a Plotly graph object figure from a dictionary,\n",
        "        a list of categories or a Pandas data frame or series.\n",
        "        \"\"\"\n",
        "        layout = getattr(Plot, layout)(**layout_opts)\n",
        "\n",
        "        if not y and isinstance(data, pd.DataFrame):\n",
        "            raise RuntimeError(\n",
        "                \"Missing required 'y' attribute for building Plotly figures from Pandas.DataFrame objects.\")\n",
        "\n",
        "        return go.Figure(\n",
        "            data=[\n",
        "                getattr(Plot, graph)(\n",
        "                    x=list(trace.keys())\n",
        "                    if isinstance(trace, dict)\n",
        "                    else trace[x].values\n",
        "                    if x and isinstance(trace, pd.DataFrame)\n",
        "                    else trace.index,\n",
        "                    y=list(trace.values())\n",
        "                    if isinstance(trace, dict)\n",
        "                    else trace[y].values\n",
        "                    if isinstance(trace, pd.DataFrame)\n",
        "                    else trace.values,\n",
        "                    name=name.get(index, index),\n",
        "                    size=resizer(size.get(index, MARKER_SIZE)),\n",
        "                    text=text.get(index, \"\") if isinstance(text, dict) else text,\n",
        "                    **opts,\n",
        "                )\n",
        "                for index, trace in (\n",
        "                    data.items()\n",
        "                    if len(data)\n",
        "                    and isinstance(data, dict)\n",
        "                    and type(list(data.values())[0])\n",
        "                    in (dict, pd.DataFrame, pd.Series)\n",
        "                    else enumerate(data)\n",
        "                    if isinstance(data, list)\n",
        "                    else [(None, data)]\n",
        "                )\n",
        "            ],\n",
        "            layout=layout,\n",
        "        )\n",
        "\n",
        "    @staticmethod\n",
        "    def subplots(\n",
        "        data: dict,\n",
        "        graph: str = \"scatter\",\n",
        "        height: int = 768,\n",
        "        orient: str = \"ver\",\n",
        "        layout: str = \"layout\",\n",
        "        layout_opts: dict = {},\n",
        "        rows: int = None,\n",
        "        cols: int = None,\n",
        "        title: str = None,\n",
        "        **opts,\n",
        "    ) -> go.Figure:\n",
        "        \"\"\" Returns a Plotly figure with subplots. \"\"\"\n",
        "        cursor = [0, 0]\n",
        "\n",
        "        if orient not in (\"ver\", \"hor\"):\n",
        "            return ValueError(\n",
        "                f\"Received invalid orient parameter: {orient}. Available choices: ('hor', 'ver').\"\n",
        "            )\n",
        "        pointer = (0 if orient == \"ver\" else 1)\n",
        "        cursor[(1 if orient == \"ver\" else 0)] += 1\n",
        "\n",
        "        if cols is None:\n",
        "            cols = (len(data)/(rows or len(data))) or 1\n",
        "            cols = int(cols) + (1 if float(cols) != int(cols) else 0)\n",
        "\n",
        "        if rows is None:\n",
        "            rows = (len(data)/cols) or 1\n",
        "            rows = int(rows) + (1 if float(rows) != int(rows) else 0)\n",
        "        limit = (rows if orient == \"ver\" else cols)\n",
        "\n",
        "        fig = make_subplots(rows=rows, cols=cols)\n",
        "        for key, trace in reversed(data.items()):\n",
        "            cursor[pointer] += 1\n",
        "            fig.append_trace(\n",
        "                getattr(Plot, graph)(\n",
        "                    list(trace.keys()),\n",
        "                    list(trace.values()),\n",
        "                    name=key,\n",
        "                    **opts,\n",
        "                ),\n",
        "                row=cursor[0],\n",
        "                col=cursor[1],\n",
        "            )\n",
        "            if cursor[pointer] == limit:\n",
        "                cursor[pointer] = 0\n",
        "                cursor[pointer-1] += 1\n",
        "\n",
        "        fig.update_layout({\"height\": height, **layout_opts})\n",
        "        return fig\n",
        "\n",
        "    @staticmethod\n",
        "    def bar(x, y, **opts):\n",
        "        \"\"\"\n",
        "        Returns Plotly 2-dimensional scatter.\n",
        "\n",
        "        Input parameters:\n",
        "            * x: list of values for horizontal axis\n",
        "            * y: list of values for vertical axis\n",
        "            * name: to include in point information\n",
        "            * text: to include in trace information\n",
        "        \"\"\"\n",
        "        return go.Bar(\n",
        "            x=x,\n",
        "            y=y,\n",
        "            name=opts.get(\"name\", \"\"),\n",
        "            text=opts.get(\"text\", \"\"),\n",
        "            textfont=dict(\n",
        "                family=FONT_FAMILY\n",
        "            ),\n",
        "        )\n",
        "\n",
        "    @staticmethod\n",
        "    def layout(title=\"\", x_title=\"\", y_title=\"\", **opts):\n",
        "        \"\"\"\n",
        "        Returns Plotly.Figure() layout dictionary.\n",
        "\n",
        "        Input parameters:\n",
        "            * title: Plotly figure title\n",
        "            * x_title: horizontal axis title\n",
        "            * y_title: vertical axis title\n",
        "        \"\"\"\n",
        "        return go.Layout(\n",
        "            xaxis=dict(\n",
        "                autorange=AUTORANGE,\n",
        "                title=x_title,\n",
        "                ),\n",
        "            yaxis=dict(\n",
        "                autorange=AUTORANGE,\n",
        "                title=y_title,\n",
        "                ),\n",
        "            legend=dict(\n",
        "                y=LEGEND_Y,\n",
        "                font=dict(\n",
        "                    family=FONT_FAMILY,\n",
        "                    size=FONT_SIZE,\n",
        "                    color=FONT_COLOR,\n",
        "                    ),\n",
        "                ),\n",
        "            title=title,\n",
        "            **opts,\n",
        "        )\n",
        "\n",
        "    @staticmethod\n",
        "    def scatter(x, y, mode=\"lines+markers\", size=None, name=\"\", text=\"\", **opts):\n",
        "        \"\"\"\n",
        "        Returns Plotly 2-dimensional scatter.\n",
        "\n",
        "        Input parameters:\n",
        "            * x: list of values for horizontal axis\n",
        "            * y: list of values for vertical axis\n",
        "            * mode: \"lines\", \"markers\" or both (default)\n",
        "            * name: to include in point information\n",
        "            * text: to include in trace information\n",
        "        \"\"\"\n",
        "        return go.Scatter(\n",
        "            x=x,\n",
        "            y=y,\n",
        "            name=name,\n",
        "            text=text,\n",
        "            mode=mode,\n",
        "            connectgaps=CONNECT_GAPS,\n",
        "            textposition=TEXT_POSITION,\n",
        "            textfont=dict(\n",
        "                family=FONT_FAMILY,\n",
        "                ),\n",
        "            marker=dict(\n",
        "                size=size if size is not None else MARKER_SIZE,\n",
        "                ),\n",
        "            **opts,\n",
        "            )\n",
        "\n",
        "\n",
        "class Artifacts():\n",
        "\n",
        "    @abstractmethod\n",
        "    def __init__(self, **kwargs):\n",
        "        \"\"\" Abstract initializer. \"\"\"\n",
        "\n",
        "    @staticmethod\n",
        "    def attention_artifact(dates: pd.Series) -> dict:\n",
        "        artifact = dates\\\n",
        "            .value_counts()\\\n",
        "            .sort_index()\\\n",
        "            .to_dict()\n",
        "        return {\n",
        "            \"attention\":\n",
        "                Plot.plot(\n",
        "                    artifact,\n",
        "                    fill=\"tonexty\",\n",
        "                    layout_opts=dict(\n",
        "                        title=\"Attention over time\",\n",
        "                        x_title=\"Date\",\n",
        "                        y_title=\"Hits\",\n",
        "                    ),\n",
        "                ),\n",
        "            \"attention-data\":\n",
        "                artifact,\n",
        "        }\n",
        "\n",
        "    \"\"\"\n",
        "    @staticmethod\n",
        "    def attention_ids_artifact(dates: pd.Series) -> dict:\n",
        "        return {\n",
        "            \"attention-ids\":\n",
        "                dates\\\n",
        "                .groupby(dates)\\\n",
        "                .apply(lambda x: x.index.tolist())\\\n",
        "                .to_dict(),\n",
        "        }\n",
        "    \"\"\"\n",
        "    \n",
        "    @staticmethod\n",
        "    def themes_artifact(\n",
        "        dates: pd.Series,\n",
        "        clusters: pd.Series,\n",
        "        clusters_top_features: dict,\n",
        "    ) -> dict:\n",
        "        artifact = {\n",
        "            clusters_top_features.get(key, f\"Theme #{key+1}\"):\n",
        "                values\n",
        "            for key, values in\n",
        "                clusters\n",
        "                .groupby([dates, clusters])\n",
        "                .size()\n",
        "                .unstack()\n",
        "                .fillna(0)\n",
        "                .to_dict()\n",
        "                .items()\n",
        "        }\n",
        "\n",
        "        return {\n",
        "            \"themes\":\n",
        "                Plot.plot(\n",
        "                    artifact,\n",
        "                    mode=\"lines\",\n",
        "                    stackgroup=\"one\", # \"relative\",\n",
        "                    layout_opts=dict(\n",
        "                        title=\"Theme attention over time\",\n",
        "                        x_title=\"Date\",\n",
        "                        y_title=\"Hits\",\n",
        "                    ),\n",
        "                ),\n",
        "            \"themes-data\":\n",
        "                artifact,\n",
        "        }\n",
        "\n",
        "    @staticmethod\n",
        "    def themes_ids_artifact(\n",
        "        dates: pd.Series,\n",
        "        clusters: pd.Series,\n",
        "        clusters_top_features: dict,\n",
        "    ) -> dict:\n",
        "        return {\n",
        "            \"themes-ids\": {\n",
        "                clusters_top_features.get(key, f\"Theme #{key+1}\"):\n",
        "                    clusters[clusters == key]\n",
        "                    .groupby(dates)\n",
        "                    .apply(lambda x: x.index.tolist())\n",
        "                    .to_dict()\n",
        "                for key in\n",
        "                    sorted(clusters.unique())\n",
        "            },\n",
        "        }\n",
        "\n",
        "    @staticmethod\n",
        "    def top_words_artifact(\n",
        "        clusters_top_words: dict,\n",
        "    ) -> dict:\n",
        "        return {\n",
        "            \"top-words\": {\n",
        "                \" \".join(series.index[:3]):\n",
        "                    series.to_dict()\n",
        "                for cluster, series in\n",
        "                    clusters_top_words.items()\n",
        "            },\n",
        "        }\n",
        "\n",
        "\n",
        "class ArtifactGenerator(Artifacts):\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        func_cluster: Callable[[pd.DataFrame], pd.Series],\n",
        "        **kwargs,\n",
        "    ) -> None:\n",
        "        self.cluster_ = func_cluster\n",
        "        self.stemmer_ = Stemmer(**kwargs).transform\n",
        "        self.tokenizer_ = Tokenizer(**kwargs).transform\n",
        "\n",
        "    def __call__(\n",
        "        self,\n",
        "        df: pd.DataFrame,\n",
        "        artifacts: list = None,\n",
        "        attr_date: str = \"timestamp\",\n",
        "        attr_index: str = \"id\",\n",
        "        attr_lang: str = \"language\",\n",
        "        attr_text: Union[str, list] = \"text\",\n",
        "        datetime_format: str = None,\n",
        "        datetime_unit: str = None,\n",
        "        include_keywords: bool = True,\n",
        "        n_grams: int = 2,\n",
        "        sort_by: Union[list, None] = [\"like_count\", \"repost_count\"],\n",
        "    ) -> dict:\n",
        "        t0 = time()\n",
        "\n",
        "        if (sort_by and any(x in df.columns for x in sort_by)):\n",
        "            df = df\\\n",
        "                .fillna(\n",
        "                    {x: 0 for x in sort_by if x in df.columns}\n",
        "                )\\\n",
        "                .loc[\n",
        "                    df[[x for x in sort_by if x in df.columns]]\n",
        "                    .sum(axis=1)\n",
        "                    .sort_values(ascending=False)\n",
        "                    .index\n",
        "                ]\n",
        "\n",
        "        index = pd.Series(\n",
        "            df.loc[:, attr_index].values\n",
        "            if attr_index and attr_index in df.columns\\\n",
        "            else df.index\n",
        "        )\n",
        "        df.index = range(df.shape[0])\n",
        "\n",
        "        dates = (pd\n",
        "            .to_datetime(\n",
        "                df.loc[:, attr_date],\n",
        "                format=datetime_format,\n",
        "                infer_datetime_format=False if datetime_format or datetime_unit else True,\n",
        "                unit=datetime_unit,\n",
        "            ).apply(\n",
        "                lambda x: x.strftime(\"%Y-%m-%d\")\n",
        "            )\n",
        "        ) if attr_date in df.columns else pd.Series(name=attr_date)\n",
        "\n",
        "        text = (pd\n",
        "            .Series(\n",
        "                self.tokenizer_(\n",
        "                    df.loc[:, attr_text].apply(lambda x: x if x else \"\")\n",
        "                    if isinstance(attr_text, str) else [\n",
        "                        \"\\n\".join(\n",
        "                            x for x in x if isinstance(x, str)\n",
        "                        ) for x in zip(\n",
        "                            *[df[attr] for attr in attr_text if attr in df.columns]\n",
        "                        )\n",
        "                    ]\n",
        "                ),\n",
        "                index=df.index,\n",
        "            )\n",
        "        ) if attr_text in df.columns else pd.Series(name=attr_text)\n",
        "\n",
        "        print(\n",
        "            \"Read %s total or %s unique vectors.\" % (\n",
        "            text.shape[0],\n",
        "            text.drop_duplicates().dropna().shape[0],\n",
        "            )\n",
        "        )\n",
        "\n",
        "        clusters = pd.Series(\n",
        "            self.cluster_(\n",
        "                self.stemmer_(text, df[attr_lang])\n",
        "                if attr_lang in df.columns\n",
        "                else text\n",
        "            ) if text.dropna().shape[0] else [],\n",
        "            index=df.index,\n",
        "        )\n",
        "\n",
        "        clusters_top_words = self.__get_features(\n",
        "            text.loc[clusters.index],\n",
        "            clusters,\n",
        "        )\n",
        "\n",
        "        clusters_top_features = self.__get_features(\n",
        "            text.loc[clusters.index],\n",
        "            clusters,\n",
        "            n_grams=n_grams,\n",
        "            top_features=True,\n",
        "            tfidf=True,\n",
        "        ) if include_keywords else {}\n",
        "\n",
        "        local = locals()\n",
        "        artifacts_dict = {}\n",
        "\n",
        "        for artifact in (artifacts or [x for x in Artifacts.__dict__.keys() if not x.startswith(\"_\")]):\n",
        "            func = getattr(self, artifact)\n",
        "            print(artifact)\n",
        "            artifacts_dict.update(\n",
        "                func(*[local.get(x) for x in signature(func).parameters if x in local])\n",
        "            )\n",
        "        print(f\"Finished in {time()-t0:.3f}s.\")\n",
        "\n",
        "        return {k: REINDEX_MAP_FUNC(v, index)\n",
        "                if k.endswith(\"-ids\") else v\n",
        "                for k, v in artifacts_dict.items()}\n",
        "\n",
        "    @staticmethod\n",
        "    def __get_features(\n",
        "        series: pd.Series,\n",
        "        groupby: pd.Series = None,\n",
        "        map_func: Callable[[str], str] = LABEL_MAP_FUNC,\n",
        "        ignore_startswith: list = [\"@\",\"#\"],\n",
        "        n_grams: int = 1,\n",
        "        normalized: bool = False,\n",
        "        tfidf: bool = False,\n",
        "        top_features: bool = False,\n",
        "    ) -> dict:\n",
        "\n",
        "        if not series.shape[0]:\n",
        "            return dict()\n",
        "\n",
        "        groups = series\\\n",
        "            .groupby(groupby if groupby is not None else [0] * series.shape[0])\\\n",
        "            .apply(lambda x: list(set(x.index.tolist())))\n",
        "\n",
        "        features = {\n",
        "            group:\n",
        "                series\n",
        "                .loc[groups[group]]\n",
        "                .astype(str)\n",
        "                .apply(lambda x: [x for x in x.split() if not any(x.startswith(char) for char in ignore_startswith)])\n",
        "                .apply(lambda x: x if n_grams == 1 else Tokenizer.ngrams(x, n_grams))\n",
        "                .apply(lambda x: list(set(x)) if tfidf else x)\n",
        "                .explode()\n",
        "                .dropna()\n",
        "                .value_counts()\n",
        "            for group in groups.index\n",
        "        }\n",
        "\n",
        "        if tfidf: # tf(t,d)/log(N/df(t))\n",
        "            N = series.shape[0]\n",
        "            df = reduce(lambda x, y: x.add(y, fill_value=0), features.values())\n",
        "            features = {\n",
        "                group:\n",
        "                    (tf/tf.max())\n",
        "                    .divide(\n",
        "                        (df.loc[tf.index]/df.loc[tf.index].max())\n",
        "                        .apply(lambda x: log10(N/(x+1)))\n",
        "                    )\n",
        "                    .sort_values(ascending=False)\n",
        "                for group, tf in features.items()\n",
        "            }\n",
        "\n",
        "        if normalized:\n",
        "            features = {\n",
        "                group:\n",
        "                    series.apply(lambda x: x/x.max(), axis=0)\\\n",
        "                for group, series in features.items()\n",
        "            }\n",
        "\n",
        "        if top_features:\n",
        "            features = pd\\\n",
        "                .Series({\n",
        "                    group:\n",
        "                        series.index[:(len(groups)+1)]\n",
        "                    for group, series in features.items()})\\\n",
        "                .explode()\\\n",
        "                .drop_duplicates(keep=\"first\")\n",
        "            features = features\\\n",
        "                [~features.index.duplicated()]\\\n",
        "                .dropna()\\\n",
        "                .map(map_func)\\\n",
        "                .to_dict()\n",
        "\n",
        "        return features if groupby is not None else features[0]\n",
        "\n",
        "    def __find_all(x, regexp) -> list:\n",
        "        \"\"\"\n",
        "        Mentions are preceeded by an @-sign and may include\n",
        "        letters, numbers and underscores to a max. of 30 chars.\n",
        "        \"\"\"\n",
        "        found = findall(regexp, x.lower()) if isinstance(x, str) else []\n",
        "        return [x for x in found if len(x)>1]\n",
        "\n",
        "\n",
        "def plot_artifacts(artifacts):\n",
        "    \"\"\" Plot all possible artifacts. \"\"\"\n",
        "    for name, artifact in artifacts.items():\n",
        "        try:\n",
        "            py.iplot(artifact)\n",
        "        except:\n",
        "            try: [py.iplot(data) for data in artifact.values()]\n",
        "            except: pass # print(\"Skipping %s...\" % name)\n",
        "\n",
        "\n",
        "def write_artifacts(df, artifacts, output_folder=\"artifacts\", indent=2):\n",
        "    \"\"\" Store artifacts in a readable or presentable format. \"\"\"\n",
        "    if not os.path.isdir(output_folder):\n",
        "        os.makedirs(output_folder) \n",
        "\n",
        "    for name, artifact in artifacts.items():\n",
        "\n",
        "        if isinstance(artifact, Figure):\n",
        "            artifact.write_image(os.path.join(output_folder, f\"{name}.png\"))\n",
        "            artifact.write_html(os.path.join(output_folder, f\"{name}.html\"))\n",
        "            artifact.write_json(file=os.path.join(output_folder, f\"{name}.json\"), pretty=True)\n",
        "\n",
        "        elif name.endswith(\"-ids\"):\n",
        "            for k, v in artifact.items():\n",
        "                subfolder = \"%s/%s\" % (output_folder, name.replace(\"-ids\", \"\"))\n",
        "\n",
        "                if not os.path.isdir(subfolder):\n",
        "                    os.makedirs(subfolder)\n",
        "\n",
        "                if isinstance(v, dict):\n",
        "                    df.loc[list(set([i for i in v.values() for i in i]))]\\\n",
        "                      .to_csv(\"%s/%s.csv\" % (subfolder, k))\n",
        "                else:\n",
        "                    df.loc[list(set([i for i in v]))]\\\n",
        "                      .to_csv(\"%s/%s.csv\" % (subfolder, k))\n",
        "\n",
        "        else:\n",
        "            for key, item in artifact.items():\n",
        "                if isinstance(item, Figure):\n",
        "                    item.write_image(os.path.join(output_folder, f\"{name}-{key}.png\"))\n",
        "                    item.write_html(os.path.join(output_folder, f\"{name}-{key}.html\"))\n",
        "                    item.write_json(file=os.path.join(output_folder, f\"{name}-{key}.json\"), pretty=True)\n",
        "                else:\n",
        "                    pd.Series(item).to_csv(f\"{output_folder}/{name}-{key}.csv\")"
      ],
      "metadata": {
        "id": "dAhaNE6_LNIS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### PyPhi wrapper"
      ],
      "metadata": {
        "id": "O85lx7ScCSoY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class PyPhiNetwork():\n",
        "\n",
        "    def __init__(self):\n",
        "        \"\"\" Initializes class. \"\"\"\n",
        "\n",
        "    def __call__(\n",
        "        self, \n",
        "        matrix,\n",
        "        A=[],\n",
        "        states=[],\n",
        "        current_state=[],\n",
        "        num_states=None,\n",
        "        output_folder=\"artifacts\",\n",
        "    ) -> list:\n",
        "        \"\"\"\n",
        "        Calculates phi and builds PyPhi network.\n",
        "\n",
        "        Input parameters:\n",
        "            * matrix: stochastic matrix containing transition values\n",
        "                as a dictionary of keys or an array of arrays\n",
        "            * A: adjacency matrix for connecting states\n",
        "            * states: list of states (strings) to use as labels\n",
        "            * current_state: observed states in the system\n",
        "            * num_states: maximum number of observed states\n",
        "        \"\"\"\n",
        "        if isinstance(matrix, dict):\n",
        "            if not states:\n",
        "                states = list(matrix.keys())[:num_states]\n",
        "            matrix = list(matrix.values())[:num_states]\n",
        "\n",
        "        if num_states:\n",
        "            matrix = matrix[:num_states]\n",
        "            A = [list(x[:num_states]) for x in A[:num_states]]\n",
        "            states = states[:num_states]\n",
        "\n",
        "        clusters = len(matrix).__str__()\n",
        "        timeslices = len(matrix[0]).__str__()\n",
        "\n",
        "        # state labels\n",
        "        if not states:\n",
        "            states = [str(i) for i in range(len(Q))]\n",
        "\n",
        "        # most recent state\n",
        "        if not current_state:\n",
        "            for x in matrix:\n",
        "                current_state.append(x[-1])\n",
        "\n",
        "        # connectivity matrix\n",
        "        if A == []:\n",
        "            for i in range(len(matrix)):\n",
        "                A.append([])\n",
        "                for j in range(len(matrix)):\n",
        "                    A[i].append(1)\n",
        "\n",
        "        # connected subgraphs\n",
        "        sg = self.get_connected_states(A)\n",
        "\n",
        "        # influence matrix based on state transitions\n",
        "        im = self.generate_influence_matrix(matrix, A)\n",
        "\n",
        "        # transition probability matrix\n",
        "        tpm = self.generate_tpm(matrix, im)\n",
        "\n",
        "        # generate network with PyPhi\n",
        "        network = pyphi.network.Network(tpm, cm=A)\n",
        "\n",
        "        # generate ALL COMPLEXES structures of the network\n",
        "        # may take a lot of time depending on the number of clusters\n",
        "        ac = pyphi.compute.network.all_complexes(network, current_state)\n",
        "\n",
        "        # sort complexes by phi value and select\n",
        "        # the complex with the greatest value\n",
        "        ac.sort(key=self.get_phi, reverse=True)\n",
        "        bc = ac[0] # <-- best complex\n",
        "\n",
        "        # best complex mechanisms and phi values\n",
        "        ces = bc.ces\n",
        "\n",
        "        # store PyPhi results as output string\n",
        "        output = str(\"Clusters: \"+str(clusters)+\n",
        "                    \"\\nTimeslices: \"+str(timeslices)+\n",
        "                    \"\\n\\nStates:\\n\"+str(states)+\n",
        "                    \"\\n\\nPresence Matrix:\\n\"+str(np.array(matrix))+\n",
        "                    \"\\n\\nCurrent state:\\n\"+str(current_state)+\n",
        "                    \"\\n\\nAdjacency Matrix:\\n\"+str(np.array(A))+\n",
        "                    \"\\n\\nConnected states:\\n\"+str(sg)+\n",
        "                    \"\\n\\nInfluence Matrix:\\n\"+str(np.array(im)).replace(\"None\",\"-\")+\n",
        "                    \"\\n\\nTransition Probability Matrix:\\n\"+str(np.array(tpm))+\n",
        "                    \"\\n\\nBest complex mechanisms and phi values:\")\n",
        "\n",
        "        for i in range(len(ces)):\n",
        "            output += str(\"\\n( %s ) Mechanism = %s φ = %s\" % (i, list(ces[i].mechanism), ces[i].phi))\n",
        "\n",
        "        # write to output file\n",
        "        with open(f\"{output_folder}/pyphi.txt\", \"w\") as f:\n",
        "            f.write(output)\n",
        "\n",
        "        # print to standard output\n",
        "        print(output)\n",
        "\n",
        "        # write base view JSON file for PyPhi\n",
        "        # self.create_view_base(bc, states, current_state, A)\n",
        "\n",
        "        return bc, ac\n",
        "\n",
        "    def complex_mechanism(self, c, m=None, output_folder=\"artifacts\"):\n",
        "        \"\"\"\n",
        "        Displays complex (c) cause-effect structure from\n",
        "        mechanism (m) in a slightly optimized plotly view.\n",
        "        \"\"\"\n",
        "        c = c.ces # cause effect-structure\n",
        "        \n",
        "        if m is None:\n",
        "            c = sorted(c, key=lambda x: x.phi, reverse=True)\n",
        "            m = 0\n",
        "\n",
        "        c = c[m]  # complex mechanism\n",
        "        v = [\"cause\",\"effect\"]\n",
        "\n",
        "        for i in v:\n",
        "            # cause and effect\n",
        "            print(\"\\n---\\n\"+i.capitalize()+\":\\n---\\n\")\n",
        "\n",
        "            value = getattr(c,i)\n",
        "            matrix = len(list(value.purview))\n",
        "            x = []\n",
        "            y = []\n",
        "\n",
        "            #print(value)\n",
        "            print(\"\\nMaximally-irreducible\", i, \"φ =\", value.phi)\n",
        "            print(\"\\nMechanism:\", value.mechanism)\n",
        "            print(\"Purview:\", value.purview)\n",
        "            print(\"\\nMIP:\\n\"+ str(value.mip))\n",
        "            print(\"\\nRepertoire:\")\n",
        "\n",
        "            purviewSize = len(list(value.purview))\n",
        "            array = list(value.repertoire.reshape(1,pow(2,purviewSize))[0])\n",
        "\n",
        "            for j in range(len(array)):\n",
        "                binArray = self.create_bin_array(j,purviewSize)\n",
        "                binArrayString = \"\".join(map(str, binArray))\n",
        "                x.append(\"state-\" + binArrayString)\n",
        "                y.append(array[j])\n",
        "\n",
        "            data = [go.Bar(\n",
        "                x=x,\n",
        "                y=y\n",
        "                )]\n",
        "\n",
        "            layout = go.Layout(\n",
        "                autosize=False,\n",
        "                width=800,\n",
        "                height=300,\n",
        "                title=f\"Repertoire: Maximally-irreducible {i} φ = {value.phi} (Mechanism: {value.mechanism})\",\n",
        "                #yaxis=dict(8\n",
        "                #range=[0,1])\n",
        "                )\n",
        "\n",
        "            fig = go.Figure(data=data, layout=layout)\n",
        "            fig.write_html(f\"{output_folder}/pyphi-{i}.html\")\n",
        "            fig.write_image(f\"{output_folder}/pyphi-{i}.png\")\n",
        "            fig.show(renderer=\"colab\")\n",
        "            # py.iplot(fig)\n",
        "    \n",
        "    def create_view_base(self, complex, states, current_state, A):\n",
        "        '''\n",
        "        Writes output JSON file for PyPhi view.\n",
        "\n",
        "        Input parameters:\n",
        "            * complex: to get subsystems\n",
        "            * states: as a list of strings\n",
        "            * current_state: in current tX\n",
        "            * A: adjacency matrix\n",
        "        '''\n",
        "        subsystem = complex.subsystem.__str__()[10:-1].replace(\" \",\"\").split(',')\n",
        "        nodes = self.get_graph_nodes(states, subsystem, current_state)\n",
        "        edges = self.get_graph_edges(states, A)\n",
        "\n",
        "        complexJSON = {\n",
        "\n",
        "            \"states\": states,\n",
        "\n",
        "            \"graph\": {\"nodes\": nodes,\n",
        "                      \"edges\": edges},\n",
        "\n",
        "            'sia': {'bigPhi': complex.phi,\n",
        "                    'subsystem': complex.subsystem.__str__(),\n",
        "                    'ces': []}}\n",
        "\n",
        "        ces = complex.ces\n",
        "\n",
        "        for i in ces:\n",
        "            obj = self.get_CES_JSON_object(i)\n",
        "            complexJSON[\"sia\"][\"ces\"].append(obj)\n",
        "\n",
        "        with open('view.json', 'w') as j:\n",
        "            json.dump(complexJSON, j)\n",
        "\n",
        "    def distance(self, o1, o2):\n",
        "        \"\"\"\n",
        "        Returns centroid distance.\n",
        "        \"\"\"\n",
        "        return 1 - self.semantic_similarity_number(o1, o2)\n",
        "\n",
        "    def filter_connected_states(self, P, A=[], states_to_remove=[]):\n",
        "        \"\"\"\n",
        "        Returns a subset of connected states in matrices.\n",
        "        \"\"\"\n",
        "        if not states_to_remove:\n",
        "            return (P, A)\n",
        "\n",
        "        # get list of states to remove\n",
        "        if isinstance(states_to_remove, str):\n",
        "            states_to_remove = states_to_remove.split(\",\")\n",
        "\n",
        "            try:\n",
        "                states_to_remove = [int(x) for x in states_to_remove]\n",
        "            except ValueError:\n",
        "                if isinstance(P, dict):\n",
        "                    states_to_remove = [list(P.keys()).index(x) for x in states_to_remove]\n",
        "                else:\n",
        "                    raise TypeError(\"expected a dictionary to match list of states as keys\")\n",
        "\n",
        "        # check list of states to remove\n",
        "        if not isinstance(states_to_remove, list):\n",
        "            raise TypeError(\"expected a list of states (indices or dictionary keys) as input\")\n",
        "\n",
        "        keys = list(P.keys())\n",
        "        P = {keys[i]:value for i, value in enumerate(P.values()) if i not in states_to_remove}\n",
        "        return (P, self.filter_matrix(A, states_to_remove) if A else [])\n",
        "\n",
        "    def filter_matrix(self, matrix, states_to_remove=[]):\n",
        "        \"\"\"\n",
        "        Returns matrix without specified items to remove.\n",
        "        \"\"\"\n",
        "        return [[v for k, v in enumerate(matrix[i]) if k not in states_to_remove] for i in range(len(matrix)) if i not in states_to_remove]\n",
        "\n",
        "    def generate_distances_matrix(self, centroids, threshold=\"median\"):\n",
        "        \"\"\"\n",
        "        Returns adjacency matrix based on centroids distance (median/mean):\n",
        "        * if higher than threshold, nodes will not be connected (0);\n",
        "        * if lower or equal, nodes will be connected (1).\n",
        "        Note that a centroid will always be nearest its own cluster.\n",
        "        \"\"\"\n",
        "        n = len(centroids)\n",
        "        matrix = np.zeros(shape=(n,n))\n",
        "        values = []\n",
        "\n",
        "        for i,_ in enumerate(centroids):\n",
        "            for j in range(int(i), len(centroids)):\n",
        "                d = self.distance(centroids[i], centroids[j])\n",
        "                values.append(d)\n",
        "                matrix[i][j] = d\n",
        "                matrix[j][i] = d\n",
        "\n",
        "        if threshold:\n",
        "            adjacency = np.zeros(shape=(n,n), dtype=np.int8)\n",
        "            values = sorted(values)\n",
        "\n",
        "            if threshold == \"median\":\n",
        "                threshold = np.median(values)\n",
        "            elif threshold == \"mean\":\n",
        "                threshold = np.mean(values)\n",
        "            else: # error\n",
        "                log.warning(\"THRESHOLD should be either 'mean' or 'median'.\")\n",
        "                return matrix\n",
        "\n",
        "            for i,_ in enumerate(centroids):\n",
        "                for j,_ in enumerate(centroids):\n",
        "                    if matrix[i][j] <= threshold:\n",
        "                        adjacency[i][j] = 1\n",
        "                    else: adjacency[i][j] = 0\n",
        "\n",
        "            return adjacency.tolist()\n",
        "\n",
        "        return matrix\n",
        "\n",
        "    def generate_influence_matrix(self, M, cm=[], normalized=True):\n",
        "        \"\"\"\n",
        "        Returns matrix based on state transitions from each node to another:\n",
        "        * influence equals the amount of hits a state has in t+1;\n",
        "        * optionally considers only linked states in connectivity matrix (cm);\n",
        "        * if normalized and NxN shaped, the returned matrix will be stochastic.\n",
        "        \"\"\"\n",
        "        matrix = []\n",
        "\n",
        "        for group in range(len(M)):\n",
        "            matrix.append([])\n",
        "            for target in range(len(M)):\n",
        "                hits = 0\n",
        "                if cm == [] or cm[group][target] == 1:\n",
        "                    for i in range(1, len(M[group])):\n",
        "                        if M[group][i] == M[target][i-1]:\n",
        "                            hits += 1\n",
        "                matrix[group].append(hits)\n",
        "\n",
        "        if normalized:\n",
        "            for i, array in enumerate(matrix):\n",
        "                s = sum([(0 if x==None else x) for x in array])\n",
        "                a = [round((0 if x in (0,None) else x/s), 2) for x in array]\n",
        "                # sum of each row must amount 1.0 (unity)\n",
        "                a[a.index(max(a))] = round(max(a) + (1-sum(a)), 100)\n",
        "                matrix[i] = a\n",
        "\n",
        "        matrix = np.array(matrix)\n",
        "        matrix = np.around(matrix, 2)\n",
        "\n",
        "        return matrix.tolist()\n",
        "\n",
        "    def generate_presence_matrix(self, cluster_metrics, threshold=\"median\"):\n",
        "        \"\"\"\n",
        "        Returns matrix for each state based on presence value (median/mean):\n",
        "        * if higher than threshold, state presence will be positive (1);\n",
        "        * if lower or equal, state presence will be negative (0).\n",
        "        \"\"\"\n",
        "        matrix = []\n",
        "        values = []\n",
        "\n",
        "        for i,m in enumerate(cluster_metrics):\n",
        "            matrix.append([])\n",
        "            keys = sorted(list(m.keys())[0:-1])\n",
        "\n",
        "            for k in keys:\n",
        "                metrics = json.loads(m[k])\n",
        "                count = metrics[\"count\"]\n",
        "                matrix[i].append(count)\n",
        "                values.append(count)\n",
        "\n",
        "        if threshold:\n",
        "            values = sorted(values)\n",
        "\n",
        "            if threshold == \"median\":\n",
        "                threshold = np.median(values)\n",
        "            elif threshold == \"mean\":\n",
        "                threshold = np.mean(values)\n",
        "            else: # error\n",
        "                log.warning(\"THRESHOLD should be either 'mean' or 'median'.\")\n",
        "                return matrix\n",
        "\n",
        "            for i,_ in enumerate(matrix):\n",
        "                for j,_ in enumerate(matrix[i]):\n",
        "                    if matrix[i][j] > threshold:\n",
        "                        matrix[i][j] = 1\n",
        "                    else: matrix[i][j] = 0\n",
        "\n",
        "        return matrix\n",
        "    \n",
        "    def generate_presence_matrix_from_themes(self,  themes, by=\"median\", output_folder=\"artifacts\"):\n",
        "        \"\"\" Convert an artifact to a presence matrix composed of binary data. \"\"\"\n",
        "        os.makedirs(output_folder) if not os.path.isdir(output_folder) else None\n",
        "\n",
        "        matrix = {}\n",
        "        threshold = None\n",
        "\n",
        "        dates = sorted(set(x for x in themes.keys() for x in themes[x]))\n",
        "\n",
        "        for label, dct in themes.items():\n",
        "            matrix[label] = [len(dct[date]) if dct.get(date) else 0 for date in dates]\n",
        "\n",
        "            if by == \"median\":\n",
        "                threshold = np.median(matrix[label])\n",
        "            elif by == \"mean\":\n",
        "                threshold = np.mean(matrix[label])\n",
        "            else:\n",
        "                log.warning(f\"Unrecognized threshold method (by='{by}').\")\n",
        "\n",
        "            if threshold:\n",
        "                matrix[label] = [1 if v >= threshold else 0 for v in matrix[label]]\n",
        "\n",
        "        with open(f\"{output_folder}/matrix.json\", \"w\") as j:\n",
        "            json.dump(matrix, j)\n",
        "\n",
        "        return matrix\n",
        "\n",
        "    def generate_tpm(self, matrix, influence_matrix):\n",
        "        \"\"\"\n",
        "        Builds Transition Probability Matrix for PyPhi.\n",
        "\n",
        "        Input parameters:\n",
        "            * matrix: input matching influence_matrix\n",
        "            * influence_matrix: must be stochastic\n",
        "        \"\"\"\n",
        "        tpm = []\n",
        "        times = len(matrix[0])\n",
        "        number_of_groups = len(matrix)\n",
        "        #divisor = number_of_groups*(times-1)\n",
        "\n",
        "        for i in range(pow(2, number_of_groups)):\n",
        "            tpm.append([])\n",
        "            bin_array = self.create_bin_array(i, number_of_groups)\n",
        "\n",
        "            for group in range(len(matrix)):\n",
        "                value = self.get_probability(group, bin_array, influence_matrix, number_of_groups, times-1)\n",
        "                tpm[i].append(value)\n",
        "\n",
        "        return tpm\n",
        "\n",
        "    def generate_users_matrix(self, clusters, obj_array, threshold=\"median\"):\n",
        "        \"\"\"\n",
        "        Returns adjacency matrix based on users in clusters (median/mean):\n",
        "        * if higher than or equal to threshold, nodes will be connected (1);\n",
        "        * if lower, nodes will not be connected (0).\n",
        "        \"\"\"\n",
        "        n = len(clusters)\n",
        "        matrix = np.zeros(shape=(n,n))\n",
        "        users = {}\n",
        "\n",
        "        for c in clusters.keys():\n",
        "            for obj in clusters[c]:\n",
        "                screen_name = obj[\"screen_name\"]\n",
        "                try: users[screen_name].append(c)\n",
        "                except: users[screen_name] = [c]\n",
        "\n",
        "        users = [set(u) for u in users.values()]\n",
        "\n",
        "        for u in users:\n",
        "            for c in u:\n",
        "                matrix[c][c] += 1\n",
        "            for i,j in combinations(u, 2):\n",
        "                matrix[i][j] += 1\n",
        "                matrix[j][i] += 1\n",
        "\n",
        "        if threshold:\n",
        "            adjacency = np.zeros(shape=(n,n), dtype=np.int8)\n",
        "            values = []\n",
        "\n",
        "            for i,x in enumerate(matrix):\n",
        "                adjacency[i][i] = matrix[i][i]\n",
        "                for j,v in enumerate(x):\n",
        "                    values.append(v)\n",
        "\n",
        "            values = sorted(values)\n",
        "\n",
        "            if threshold == \"median\":\n",
        "                threshold = np.median(values)\n",
        "            elif threshold == \"mean\":\n",
        "                threshold = np.mean(values)\n",
        "            else: # error\n",
        "                log.warning(\"THRESHOLD should be either 'mean' or 'median'.\")\n",
        "                return matrix\n",
        "\n",
        "            for i,_ in enumerate(clusters):\n",
        "                for j,_ in enumerate(clusters):\n",
        "                    if i == j or matrix[i][j] >= threshold:\n",
        "                        adjacency[i][j] = 1\n",
        "                    else: adjacency[i][j] = 0\n",
        "\n",
        "            return adjacency.tolist()\n",
        "\n",
        "        return matrix\n",
        "\n",
        "    def get_CES_JSON_object(self, ces):\n",
        "        '''\n",
        "        Returns cause-effect structure as a JSON object.\n",
        "        '''\n",
        "        cesJsonObject = {\n",
        "            \"smallphi\": ces.phi,\n",
        "            \"mechanism\": list(ces.mechanism),\n",
        "            \"cause\":{\n",
        "                \"mip\":ces.cause.mip.__str__(),\n",
        "                \"smallphi\":ces.cause.phi,\n",
        "                \"purview\":list(ces.cause.purview)},\n",
        "            \"effect\":{\n",
        "                \"mip\":ces.effect.mip.__str__(),\n",
        "                \"smallphi\":ces.effect.phi,\n",
        "                \"purview\":list(ces.cause.purview)}}\n",
        "        # cause repertoire array\n",
        "        n = len(list(ces.cause.purview))\n",
        "        cesJsonObject['cause']['repertoire'] = self.get_repertoire_array(ces.cause,n)\n",
        "        # effect repertoire array\n",
        "        n = len(list(ces.effect.purview))\n",
        "        cesJsonObject['effect']['repertoire'] = self.get_repertoire_array(ces.effect,n)\n",
        "        # returns cause effect-structure\n",
        "        return cesJsonObject\n",
        "\n",
        "    def get_connected_states(self, A, states=[]):\n",
        "        \"\"\"\n",
        "        Returns connected states or subgraphs\n",
        "        based on the input adjacency matrix *A*.\n",
        "        \"\"\"\n",
        "        dim = len(A)\n",
        "        subgraphs = []\n",
        "        free_nodes = []\n",
        "\n",
        "        for i in range(dim):\n",
        "            free_nodes.append(i)\n",
        "\n",
        "        while len(free_nodes) > 0:\n",
        "            i = 0\n",
        "            subgraph = []\n",
        "            pivot = free_nodes.pop(0)\n",
        "            subgraph.append(pivot)\n",
        "\n",
        "            while i < len(subgraph):\n",
        "                j = 0\n",
        "                source = subgraph[i]\n",
        "\n",
        "                while j < len(free_nodes):\n",
        "                    target = free_nodes[j]\n",
        "                    hit = A[source][target]\n",
        "\n",
        "                    if hit == 1:\n",
        "                        subgraph.append(free_nodes.pop(j))\n",
        "                    else: j+=1\n",
        "\n",
        "                i+=1\n",
        "\n",
        "            subgraphs.append(subgraph)\n",
        "\n",
        "        if states:\n",
        "            return states if not subgraphs else [[states[x] for x in s] for s in subgraphs]\n",
        "\n",
        "        return subgraphs\n",
        "\n",
        "    def get_repertoire_array(self, ces, n):\n",
        "        '''\n",
        "        Return repertoire list.\n",
        "\n",
        "        Input parameters:\n",
        "            * ces: cause-effect structure\n",
        "            * n: number of digits\n",
        "        '''\n",
        "        repertoire = []\n",
        "        array = list(ces.repertoire.reshape(1,pow(2,n))[0])\n",
        "        # append key and value to list\n",
        "        for i in range(len(array)):\n",
        "            repertoire.append({\n",
        "                \"key\": self.create_bin_array(i,n).__str__().replace(\"[\",\"\").replace(\",\",\"\").replace(\"]\",\"\").replace(\" \",\"\"),\n",
        "                \"value\": array[i]})\n",
        "        # return repertoire list\n",
        "        return repertoire\n",
        "\n",
        "    def get_phi(self, obj):\n",
        "        \"\"\"\n",
        "        Sort all complexes in *obj* by their phi values.\n",
        "        \"\"\"\n",
        "        return obj.phi\n",
        "    \n",
        "    def get_probability(self, group, bin_array, influence_matrix, number_of_groups, max_influence):\n",
        "        \"\"\"\n",
        "        Returns probability of transition for\n",
        "        each group based on influence matrix.\n",
        "\n",
        "        Input parameters:\n",
        "            * group: in influence matrix\n",
        "            * bin_array: binary array\n",
        "            * influence_matrix: must be stochastic\n",
        "            * number_of_groups: to consider for P\n",
        "            * max_influence: to consider for P\n",
        "        \"\"\"\n",
        "        total = 0\n",
        "        none_count = 0\n",
        "\n",
        "        influence_list = influence_matrix[group]\n",
        "\n",
        "        for i in range(len(influence_list)):\n",
        "            if influence_list[i] == None:\n",
        "                none_count += 1\n",
        "            else:\n",
        "                if bin_array[i] == 1:\n",
        "                    total += influence_list[i]\n",
        "                else:\n",
        "                    total += (max_influence - influence_list[i])\n",
        "\n",
        "        try: return total/((number_of_groups-none_count)*(max_influence if max_influence>0 else 1))\n",
        "        except: return 0\n",
        "\n",
        "    def load_json_matrix(self, input_json):\n",
        "        \"\"\"\n",
        "        Loads *input_json* file containing data for PyPhi.\n",
        "        Accepted formats from this file are listed below:\n",
        "\n",
        "        * matrix and connectivity matrix from dictionary:\n",
        "            {\"matrix\": <dictionary of keys or array of arrays>,\n",
        "            \"A\": <array of arrays>}\n",
        "\n",
        "        * matrix as a dictionary of keys:\n",
        "            {\"state1\": [0,1,0,0,1,0],\n",
        "            \"state2\": [1,0,1,1,0,0],\n",
        "            \"state3\": [0,1,0,1,0,1]}\n",
        "\n",
        "        * matrix as an array of arrays:\n",
        "            [[0,1,0,0,1,0],\n",
        "            [1,0,1,1,0,0],\n",
        "            [0,1,0,1,0,1]]\n",
        "\n",
        "        * connectivity matrix as an array of arrays:\n",
        "            [[1,1,1],\n",
        "            [1,1,1],\n",
        "            [1,1,1]]\n",
        "        \"\"\"\n",
        "        with open(input_json, \"r\") as j:\n",
        "            data = json.loads(j.read())\n",
        "\n",
        "        if \"matrix\" in data:\n",
        "            matrix = data[\"matrix\"]\n",
        "            A = data[\"A\"] if \"A\" in data else (data[\"cm\"] if \"cm\" in data else [])\n",
        "        else:\n",
        "            matrix = self.generate_presence_matrix(data)\n",
        "            A = []\n",
        "\n",
        "        return matrix, A\n",
        "\n",
        "    def mc_render(self, matrix, states=[], states_to_remove=[], renderer=\"graphviz\", ext=\"png\", prog=\"dot\", output_folder=\".\"):\n",
        "        \"\"\"\n",
        "        Builds a state-transition diagram or markov chain.\n",
        "        Output files written: \"markov.dot\" and \"markov.png\".\n",
        "\n",
        "        Input parameters:\n",
        "            * matrix: stochastic matrix containing transition values\n",
        "                as a dictionary of keys or an array of arrays\n",
        "            * states: list of states (strings) to use as labels\n",
        "            * renderer: choose \"graphviz\"; \"pygraphviz\"; or \"pydot\"\n",
        "            * ext: image file type format to write as (default: PNG)\n",
        "            * prog: others may be available by graphviz e.g. \"neato\"\n",
        "            * output_folder: output folder name to write files to\n",
        "        \"\"\"\n",
        "        edge_labels = {}\n",
        "\n",
        "        output_dot = f\"{output_folder}/markov.dot\"\n",
        "        output_ext = f\"{output_folder}/markov.\" + ext\n",
        "\n",
        "        # filter matrix states by user list\n",
        "        if states_to_remove:\n",
        "            matrix = self.mc_filter(matrix, states_to_remove, True)\n",
        "\n",
        "        # matrix as a dictionary of keys\n",
        "        if isinstance(matrix, dict):\n",
        "            if not states:\n",
        "                states = list(matrix.keys())\n",
        "            matrix = list(matrix.values())\n",
        "\n",
        "        # prepare graph object\n",
        "        if renderer == \"graphviz\":\n",
        "            G = gv.Digraph(format=ext)\n",
        "            G.attr(\"node\", shape=\"circle\")\n",
        "            G.attr(rankdir=\"LR\", size=\"8\")\n",
        "            G.add_node = G.node\n",
        "            G.add_edge = G.edge\n",
        "\n",
        "        elif renderer == \"pygraphviz\":\n",
        "            G = pgv.AGraph(format=ext, strict=False, directed=True)\n",
        "\n",
        "        elif renderer == \"pydot\":\n",
        "            G = nx.MultiDiGraph()\n",
        "\n",
        "        elif renderer == \"none\":\n",
        "            pass\n",
        "\n",
        "        else: # exit\n",
        "            raise ValueError(f\"Unrecognized renderer: '{renderer}'.\\n\"+\n",
        "                             f\"Available choices: 'graphviz', 'pygraphviz', 'pydot', 'none'.\")\n",
        "\n",
        "        # state labels\n",
        "        if not states:\n",
        "            states = [str(i) for i in range(len(matrix))]\n",
        "\n",
        "        # add state nodes\n",
        "        for s in states:\n",
        "            G.add_node(s)\n",
        "\n",
        "        # build graph network\n",
        "        for i, origin_state in enumerate(states):\n",
        "        # for i in range(len(matrix)):\n",
        "            # origin_state = states[i]\n",
        "            for j, destination_state in enumerate(states):\n",
        "            # for j in range(i+1, len(matrix)):\n",
        "                # destination_state = states[j]\n",
        "                rate = matrix[i][j] # im[i][j]\n",
        "                if (rate != None and rate > 0):\n",
        "                    str_rate = str(\"{:.02f}\".format(rate))\n",
        "                    G.add_edge(origin_state, # states[i],\n",
        "                            destination_state, # states[j],\n",
        "                            weight=rate if renderer != \"graphviz\" else None,\n",
        "                            label=str_rate)\n",
        "                    edge_labels[(origin_state, destination_state)] = label=str_rate\n",
        "\n",
        "        # create output path directory\n",
        "        if not os.path.exists(output_folder):\n",
        "            os.mkdir(output_folder)\n",
        "\n",
        "        if renderer == \"graphviz\":\n",
        "            G.render(f\"{output_folder}/markov\")\n",
        "            move(f\"{output_folder}/markov\", output_dot)\n",
        "            return gv.Source.from_file(output_dot)\n",
        "\n",
        "        elif renderer == \"pygraphviz\":\n",
        "            G.layout(prog=prog)\n",
        "            G.draw(output_ext, format=ext, prog=prog)\n",
        "            return G.draw(format=ext, prog=prog)\n",
        "\n",
        "        elif renderer == \"pydot\":\n",
        "            plt.axis(\"off\")\n",
        "            plt.tight_layout()\n",
        "            pos = nx.nx_pydot.pydot_layout(G, prog=prog)\n",
        "            nx.drawing.nx_pydot.write_dot(G, output_dot)\n",
        "            nx.draw_networkx_nodes(G, pos, node_shape=\"s\", node_color=\"w\", node_size=1000)\n",
        "            nx.draw_networkx_edges(G, pos, width=1.0, alpha=0.5)\n",
        "            nx.draw_networkx_labels(G, pos, font_weight=2)\n",
        "            nx.draw_networkx_edge_labels(G, pos, edge_labels)\n",
        "            plt.savefig(output_ext, format=ext)\n",
        "            return plt.plot()\n",
        "\n",
        "        return G\n",
        "\n",
        "    def mc_filter(self, matrix, states_to_remove=[], normalized=False):\n",
        "        \"\"\"\n",
        "        Returns an optionally normalized matrix with selected rows.\n",
        "        \"\"\"\n",
        "        if not states_to_remove:\n",
        "            return matrix\n",
        "\n",
        "        # get list of states to remove\n",
        "        if isinstance(states_to_remove, str):\n",
        "            states_to_remove = states_to_remove.split(\",\")\n",
        "\n",
        "            try:\n",
        "                states_to_remove = [int(x) for x in states_to_remove]\n",
        "            except ValueError:\n",
        "                if isinstance(matrix, dict):\n",
        "                    states_to_remove = [list(matrix.keys()).index(x) for x in states_to_remove]\n",
        "                else:\n",
        "                    raise TypeError(\"expected a dictionary to match list of states as keys\")\n",
        "\n",
        "        # check list of states to remove\n",
        "        if not isinstance(states_to_remove, list):\n",
        "            raise TypeError(\"expected a list of states (indices or dictionary keys) as input\")\n",
        "\n",
        "        # get matrix values\n",
        "        array = matrix\n",
        "        if isinstance(matrix, dict):\n",
        "            array = list(matrix.values())\n",
        "\n",
        "        # filter matrix values\n",
        "        output = self.normalize_matrix(self.filter_matrix(array, states_to_remove))\\\n",
        "                 if normalized else self.filter_matrix(array, states_to_remove)\n",
        "\n",
        "        if isinstance(matrix, dict):\n",
        "            keys = [list(matrix.keys())[i] for i in range(len(matrix.keys())) if i not in states_to_remove]\n",
        "            return {keys[i]: output[i] for i in range(len(output))}\n",
        "\n",
        "        return output\n",
        "\n",
        "    @staticmethod\n",
        "    def create_bin_array(number, digits):\n",
        "        \"\"\"\n",
        "        Converts a number in a binary array with n digits, as in:\n",
        "            create_bin_array(10,5) -> 01010\n",
        "        \"\"\"\n",
        "        array = []\n",
        "        binString = format(number,\"b\")\n",
        "\n",
        "        for i in range(digits-len(binString)):\n",
        "            array.append(0)\n",
        "\n",
        "        for i in range(len(binString)):\n",
        "            array.append(int(binString[i]))\n",
        "\n",
        "        return array\n",
        "\n",
        "    @staticmethod\n",
        "    def get_graph_nodes(states, subsystem, state):\n",
        "        '''\n",
        "        Returns nodes in graph.\n",
        "\n",
        "        Input parameters:\n",
        "            * states: as a list of strings\n",
        "            * subsystem: from complex\n",
        "            * state: from subsystem\n",
        "        '''\n",
        "        nodes = []\n",
        "\n",
        "        for i in range(len(states)):\n",
        "            status = \"\"\n",
        "\n",
        "            if states[i] in subsystem:\n",
        "                status = \"on\" if (state[i] == 1) else \"off\"\n",
        "            else:\n",
        "                status = \"undefined\"\n",
        "\n",
        "            nodes.append({\n",
        "                \"id\":states[i],\n",
        "                \"status\": status})\n",
        "\n",
        "        return nodes\n",
        "\n",
        "    @staticmethod\n",
        "    def get_graph_edges(states, A):\n",
        "        '''\n",
        "        Returns edges from connected nodes.\n",
        "\n",
        "        Input parameters:\n",
        "            * states: as a list of strings\n",
        "            * A: adjacency matrix\n",
        "        '''\n",
        "        edges = []\n",
        "        for i in range(len(A)):\n",
        "            for j in range(i+1,len(A)):\n",
        "                if A[i][j] == 1:\n",
        "                    edges.append({\n",
        "                        \"source\":states[i],\n",
        "                        \"target\":states[j]})\n",
        "\n",
        "        return edges\n",
        "\n",
        "    @staticmethod\n",
        "    def normalize_matrix(matrix):\n",
        "        \"\"\"\n",
        "        Returns normalized matrix in which the\n",
        "        sum of each row amounts 1.0 (unity).\n",
        "        \"\"\"\n",
        "        for i, array in enumerate(matrix):\n",
        "            s = sum([(0 if x==None else x) for x in array])\n",
        "            a = [round((0 if x in (0,None) else x/s), 2) for x in array]\n",
        "            a[a.index(max(a))] = round(max(a) + (1-sum(a)), 100)\n",
        "            a = np.around(a, 2)\n",
        "            matrix[i] = a\n",
        "        if any(np.around(sum(row)) != 1 for row in matrix):\n",
        "            raise ValueError(\"failed to normalize matrix after filtering (row does not amount unity).\")\n",
        "        return matrix\n",
        "    \n",
        "    @staticmethod\n",
        "    def semantic_similarity_number(o1, o2):\n",
        "        \"\"\"\n",
        "        Returns semantic similarity distance value.\n",
        "        \"\"\"\n",
        "        total = 0\n",
        "        keys  = set(list(o1[\"word_dict\"].keys())+list(o2[\"word_dict\"].keys()))\n",
        "\n",
        "        for key in keys:\n",
        "\n",
        "            try: m1 = o1[\"word_dict\"][key]\n",
        "            except: m1 = 0\n",
        "\n",
        "            try: m2 = o2[\"word_dict\"][key]\n",
        "            except: m2 = 0\n",
        "\n",
        "            total = total + min(m1, m2)\n",
        "\n",
        "        div = max(o1[\"total\"], o2[\"total\"])\n",
        "\n",
        "        return (total/div) if div>0 else 0"
      ],
      "metadata": {
        "id": "2iOX-ixJ5qZh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Analyze"
      ],
      "metadata": {
        "id": "_NlrNoIVErP6"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Load dataset\n",
        "\n",
        "Expects data indexed as columns `author_username`, `id`, `language`, `like_count`, `repost_count`, `text` and `timestamp`.\n",
        "\n"
      ],
      "metadata": {
        "id": "in690Yn-NiYG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "MAX_ROWS = None\n",
        "\n",
        "columns = {\n",
        "    \"\": \"author_username\",\n",
        "    \"\": \"id\",\n",
        "    \"\": \"language\", \n",
        "    \"\": \"like_count\",\n",
        "    \"\": \"repost_count\",\n",
        "    \"\": \"text\",\n",
        "    \"\": \"timestamp\"\n",
        "}\n",
        "df = pd.concat([\n",
        "    pd.read_csv(\"filename.csv\", low_memory=False, usecols=list(columns.keys())),\n",
        "])\n",
        "df.columns = [columns.get(x, x) for x in df.columns]\n",
        "df.index = df[\"id\"]\n",
        "print(f\"Loaded {df.shape} objects.\")"
      ],
      "metadata": {
        "id": "aRQ3sxs5bwqT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Detect clusters i.e. themes"
      ],
      "metadata": {
        "id": "E8a5akNxLEbF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -fr artifacts\n",
        "artifacts = ArtifactGenerator(\n",
        "    func_cluster=Clusterer(\n",
        "        analyzer=\"char\",\n",
        "        max_paragraphs=10,\n",
        "        random_state=0,\n",
        "        use_pandas=False,\n",
        "        use_svd=\"try\",\n",
        "    ).fit_predict,\n",
        "    stop_words=CUSTOM_STOPWORDS,\n",
        ").__call__(\n",
        "    df,\n",
        "    datetime_unit=\"s\",\n",
        "    include_keywords=True,\n",
        "    n_grams=2,\n",
        ")"
      ],
      "metadata": {
        "id": "SdaiR7D3j7xb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Plot artifacts"
      ],
      "metadata": {
        "id": "FCo7KpH9snkr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "plot_artifacts(artifacts)"
      ],
      "metadata": {
        "id": "e5MCJn9mss4E"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Save artifacts"
      ],
      "metadata": {
        "id": "jA3ekeehWHej"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "write_artifacts(df, artifacts)"
      ],
      "metadata": {
        "id": "DXucxNCCWHe-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Execute PyPhi"
      ],
      "metadata": {
        "id": "OWAu3IFu_RfP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pphi = PyPhiNetwork()"
      ],
      "metadata": {
        "id": "fcxt8ZvJjv5L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YkrlGbws_4mT"
      },
      "source": [
        "#### Build matrix from themes\n",
        "\n",
        "Set presence matrix (`P`) for a list of states over time, containing binary elements (`0` for not active; `1` for active)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zQCcgf8GYjcZ"
      },
      "source": [
        "P = pphi.generate_presence_matrix_from_themes(artifacts[\"themes-ids\"]); P"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ghIU5tXy_4mZ"
      },
      "source": [
        "###### Filter connected states (optional)\n",
        "\n",
        "Optionally get a subset of connected states in a matrix, from a list optionally set by the user (comma separated)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vLNnRcfT_4ma"
      },
      "source": [
        "states_to_remove = \"\" # <-- comma separated\n",
        "\n",
        "P, A = pphi.filter_connected_states(P, A=[], states_to_remove=states_to_remove)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J1iNXIGq_4mb"
      },
      "source": [
        "###### Check connected states (optional)\n",
        "\n",
        "Returns a list of connected states based on presence (`P`) and adjacency (`A`) matrices."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DuMCCf5p_4mc"
      },
      "source": [
        "pphi.get_connected_states(A, states=list(P.keys()))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SHT9zA7M_4md"
      },
      "source": [
        "#### Render transition diagram\n",
        "\n",
        "Render a transition diagram from a Markovian chain. Available renderers: `graphviz`, `pygraphviz` and `pydot`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "avovZCfh_4me"
      },
      "source": [
        "states_labels = \"\" # <-- comma separated\n",
        "\n",
        "pphi.mc_render(\n",
        "    pphi.generate_influence_matrix(list(P.values()), A),\n",
        "    states=states_labels.split(\",\") if states_labels else P.keys(),\n",
        "    output_folder=\"artifacts\"\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "auNI8y3i_4mf"
      },
      "source": [
        "#### Cause-effect structure\n",
        "\n",
        "Integrated information theory provides a mathematical framework to fully characterize the cause-effect structure of a physical system. [PyPhi](http://integratedinformationtheory.org) implements a framework for causal analysis and unfolds the full cause-effect structure of discrete dynamical systems of binary elements."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EicqpMSz_4mg"
      },
      "source": [
        "##### Compute network\n",
        "\n",
        "Returns all complexes in the network context of all φ and Φ computation. Here we’ll use the 2-dimensional state-by-node form for the TPM."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": false,
        "id": "7m_uyPTJ_4mh"
      },
      "source": [
        "bc, ac = pphi(P, A)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "66900lC0_4mi"
      },
      "source": [
        "##### Mechanism details\n",
        "\n",
        "See details (cause and effect) of one selected mechanism, based on the list above."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": false,
        "id": "lLwyB4Iw_4mj"
      },
      "source": [
        "m = None # <-- mechanism number from above (optional)\n",
        "\n",
        "pphi.complex_mechanism(bc, m)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6urTYmcS_4mk"
      },
      "source": [
        "##### Best complex data\n",
        "\n",
        "Display best complex with the highest phi value after system irreducibility analysis and its cause-effect structure."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "YgKK0K97_4mk"
      },
      "source": [
        "bc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o-84c_H-_4ml"
      },
      "source": [
        "##### ALL complexes data (!)\n",
        "\n",
        "Display all complexes identified by PyPhi. **Warning:** long text buffer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "ZhWjILqc_4mm"
      },
      "source": [
        "# ac"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B-Byg0LP_4mn"
      },
      "source": [
        "## Compress output →  `output.zip`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8k0lU1AW_4mn"
      },
      "source": [
        "!zip -9r output.zip artifacts"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "_____\n",
        "\n",
        "## References\n",
        "\n",
        "* PyPhi: [website](https://pypi.org/project/pyphi/) | [arxiv](https://arxiv.org/abs/1712.09644) | [documentation](https://pyphi.readthedocs.io/en/latest/) | [GitHub](https://github.com/wmayner/pyphi)"
      ],
      "metadata": {
        "id": "KtV5Oee-HKb1"
      }
    }
  ]
}