Fine_tuning_+_feature_extraction_+_class.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "DW3OQVqUYC0z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8b8ab574-951f-40f3-e97c-5a8cfe088f63"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "llmx 0.0.15a0 requires cohere, which is not installed.\n",
            "llmx 0.0.15a0 requires tiktoken, which is not installed.\u001b[0m\u001b[31m\n",
            "\u001b[0mabsl-py==1.4.0\n",
            "accelerate==0.21.0\n",
            "aiohttp==3.9.1\n",
            "aiosignal==1.3.1\n",
            "alabaster==0.7.13\n",
            "albumentations==1.3.1\n",
            "altair==4.2.2\n",
            "anyio==3.7.1\n",
            "appdirs==1.4.4\n",
            "argon2-cffi==23.1.0\n",
            "argon2-cffi-bindings==21.2.0\n",
            "array-record==0.5.0\n",
            "arviz==0.15.1\n",
            "astropy==5.3.4\n",
            "astunparse==1.6.3\n",
            "async-timeout==4.0.3\n",
            "atpublic==4.0\n",
            "attrs==23.1.0\n",
            "audioread==3.0.1\n",
            "autograd==1.6.2\n",
            "Babel==2.13.1\n",
            "backcall==0.2.0\n",
            "beautifulsoup4==4.11.2\n",
            "bidict==0.22.1\n",
            "bigframes==0.15.0\n",
            "bitsandbytes==0.40.2\n",
            "bleach==6.1.0\n",
            "blinker==1.4\n",
            "blis==0.7.11\n",
            "blosc2==2.0.0\n",
            "bokeh==3.3.1\n",
            "bqplot==0.12.42\n",
            "branca==0.7.0\n",
            "build==1.0.3\n",
            "CacheControl==0.13.1\n",
            "cachetools==5.3.2\n",
            "catalogue==2.0.10\n",
            "certifi==2023.11.17\n",
            "cffi==1.16.0\n",
            "chardet==5.2.0\n",
            "charset-normalizer==3.3.2\n",
            "chex==0.1.7\n",
            "click==8.1.7\n",
            "click-plugins==1.1.1\n",
            "cligj==0.7.2\n",
            "cloudpickle==2.2.1\n",
            "cmake==3.27.7\n",
            "cmdstanpy==1.2.0\n",
            "colorcet==3.0.1\n",
            "colorlover==0.3.0\n",
            "colour==0.1.5\n",
            "community==1.0.0b1\n",
            "confection==0.1.4\n",
            "cons==0.4.6\n",
            "contextlib2==21.6.0\n",
            "contourpy==1.2.0\n",
            "cryptography==41.0.7\n",
            "cufflinks==0.17.3\n",
            "cupy-cuda11x==11.0.0\n",
            "cvxopt==1.3.2\n",
            "cvxpy==1.3.2\n",
            "cycler==0.12.1\n",
            "cymem==2.0.8\n",
            "Cython==3.0.6\n",
            "dask==2023.8.1\n",
            "datascience==0.17.6\n",
            "datasets==2.4.0\n",
            "db-dtypes==1.1.1\n",
            "dbus-python==1.2.18\n",
            "debugpy==1.6.6\n",
            "decorator==4.4.2\n",
            "defusedxml==0.7.1\n",
            "dill==0.3.5.1\n",
            "diskcache==5.6.3\n",
            "distributed==2023.8.1\n",
            "distro==1.7.0\n",
            "dlib==19.24.2\n",
            "dm-tree==0.1.8\n",
            "docker-pycreds==0.4.0\n",
            "docutils==0.18.1\n",
            "dopamine-rl==4.0.6\n",
            "duckdb==0.9.2\n",
            "earthengine-api==0.1.379\n",
            "easydict==1.11\n",
            "ecos==2.0.12\n",
            "editdistance==0.6.2\n",
            "eerepr==0.0.4\n",
            "en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.6.0/en_core_web_sm-3.6.0-py3-none-any.whl#sha256=83276fc78a70045627144786b52e1f2728ad5e29e5e43916ec37ea9c26a11212\n",
            "entrypoints==0.4\n",
            "et-xmlfile==1.1.0\n",
            "etils==1.5.2\n",
            "etuples==0.3.9\n",
            "exceptiongroup==1.2.0\n",
            "fastai==2.7.13\n",
            "fastcore==1.5.29\n",
            "fastdownload==0.0.7\n",
            "fastjsonschema==2.19.0\n",
            "fastprogress==1.0.3\n",
            "fastrlock==0.8.2\n",
            "filelock==3.13.1\n",
            "fiona==1.9.5\n",
            "firebase-admin==5.3.0\n",
            "Flask==2.2.5\n",
            "flatbuffers==23.5.26\n",
            "flax==0.7.5\n",
            "folium==0.14.0\n",
            "fonttools==4.45.1\n",
            "frozendict==2.3.10\n",
            "frozenlist==1.4.0\n",
            "fsspec==2023.6.0\n",
            "future==0.18.3\n",
            "gast==0.5.4\n",
            "gcsfs==2023.6.0\n",
            "GDAL==3.4.3\n",
            "gdown==4.6.6\n",
            "geemap==0.28.2\n",
            "gensim==4.3.2\n",
            "geocoder==1.38.1\n",
            "geographiclib==2.0\n",
            "geopandas==0.13.2\n",
            "geopy==2.3.0\n",
            "gin-config==0.5.0\n",
            "gitdb==4.0.11\n",
            "GitPython==3.1.40\n",
            "glob2==0.7\n",
            "google==2.0.3\n",
            "google-ai-generativelanguage==0.3.3\n",
            "google-api-core==2.11.1\n",
            "google-api-python-client==2.84.0\n",
            "google-auth==2.17.3\n",
            "google-auth-httplib2==0.1.1\n",
            "google-auth-oauthlib==1.0.0\n",
            "google-cloud-bigquery==3.12.0\n",
            "google-cloud-bigquery-connection==1.12.1\n",
            "google-cloud-bigquery-storage==2.23.0\n",
            "google-cloud-core==2.3.3\n",
            "google-cloud-datastore==2.15.2\n",
            "google-cloud-firestore==2.11.1\n",
            "google-cloud-functions==1.13.3\n",
            "google-cloud-iam==2.12.2\n",
            "google-cloud-language==2.9.1\n",
            "google-cloud-resource-manager==1.10.4\n",
            "google-cloud-storage==2.8.0\n",
            "google-cloud-translate==3.11.3\n",
            "google-colab @ file:///colabtools/dist/google-colab-1.0.0.tar.gz#sha256=2c5b30eba68fb0e1fd42f8b67fd92cece08409eb406d1dcde1488e0fed57fc40\n",
            "google-crc32c==1.5.0\n",
            "google-generativeai==0.2.2\n",
            "google-pasta==0.2.0\n",
            "google-resumable-media==2.6.0\n",
            "googleapis-common-protos==1.61.0\n",
            "googledrivedownloader==0.4\n",
            "graphviz==0.20.1\n",
            "greenlet==3.0.1\n",
            "grpc-google-iam-v1==0.12.7\n",
            "grpcio==1.59.3\n",
            "grpcio-status==1.48.2\n",
            "gspread==3.4.2\n",
            "gspread-dataframe==3.3.1\n",
            "gym==0.25.2\n",
            "gym-notices==0.0.8\n",
            "h5netcdf==1.3.0\n",
            "h5py==3.9.0\n",
            "holidays==0.37\n",
            "holoviews==1.17.1\n",
            "html5lib==1.1\n",
            "httpimport==1.3.1\n",
            "httplib2==0.22.0\n",
            "huggingface-hub==0.19.4\n",
            "humanize==4.7.0\n",
            "hyperopt==0.2.7\n",
            "ibis-framework==6.2.0\n",
            "idna==3.6\n",
            "imageio==2.31.6\n",
            "imageio-ffmpeg==0.4.9\n",
            "imagesize==1.4.1\n",
            "imbalanced-learn==0.10.1\n",
            "imgaug==0.4.0\n",
            "importlib-metadata==6.8.0\n",
            "importlib-resources==6.1.1\n",
            "imutils==0.5.4\n",
            "inflect==7.0.0\n",
            "iniconfig==2.0.0\n",
            "install==1.3.5\n",
            "intel-openmp==2023.2.0\n",
            "ipyevents==2.0.2\n",
            "ipyfilechooser==0.6.0\n",
            "ipykernel==5.5.6\n",
            "ipyleaflet==0.18.0\n",
            "ipython==7.34.0\n",
            "ipython-genutils==0.2.0\n",
            "ipython-sql==0.5.0\n",
            "ipytree==0.2.2\n",
            "ipywidgets==7.7.1\n",
            "itsdangerous==2.1.2\n",
            "jax==0.4.20\n",
            "jaxlib @ https://storage.googleapis.com/jax-releases/cuda11/jaxlib-0.4.20+cuda11.cudnn86-cp310-cp310-manylinux2014_x86_64.whl#sha256=01be66238133f884bf5adf15cd7eaaf8445f9d4b056c5c64df28a997a6aff2fe\n",
            "jeepney==0.7.1\n",
            "jieba==0.42.1\n",
            "Jinja2==3.1.2\n",
            "joblib==1.3.2\n",
            "jsonpickle==3.0.2\n",
            "jsonschema==4.19.2\n",
            "jsonschema-specifications==2023.11.1\n",
            "jupyter-client==6.1.12\n",
            "jupyter-console==6.1.0\n",
            "jupyter-server==1.24.0\n",
            "jupyter_core==5.5.0\n",
            "jupyterlab-widgets==3.0.9\n",
            "jupyterlab_pygments==0.3.0\n",
            "kaggle==1.5.16\n",
            "keras==2.14.0\n",
            "keyring==23.5.0\n",
            "kiwisolver==1.4.5\n",
            "langcodes==3.3.0\n",
            "launchpadlib==1.10.16\n",
            "lazr.restfulclient==0.14.4\n",
            "lazr.uri==1.0.6\n",
            "lazy_loader==0.3\n",
            "libclang==16.0.6\n",
            "librosa==0.10.1\n",
            "lida==0.0.10\n",
            "lightgbm==4.1.0\n",
            "linkify-it-py==2.0.2\n",
            "llmx==0.0.15a0\n",
            "llvmlite==0.41.1\n",
            "locket==1.0.0\n",
            "logical-unification==0.4.6\n",
            "lxml==4.9.3\n",
            "malloy==2023.1064\n",
            "Markdown==3.5.1\n",
            "markdown-it-py==3.0.0\n",
            "MarkupSafe==2.1.3\n",
            "matplotlib==3.7.1\n",
            "matplotlib-inline==0.1.6\n",
            "matplotlib-venn==0.11.9\n",
            "mdit-py-plugins==0.4.0\n",
            "mdurl==0.1.2\n",
            "miniKanren==1.0.3\n",
            "missingno==0.5.2\n",
            "mistune==0.8.4\n",
            "mizani==0.9.3\n",
            "mkl==2023.2.0\n",
            "ml-dtypes==0.2.0\n",
            "mlxtend==0.22.0\n",
            "more-itertools==10.1.0\n",
            "moviepy==1.0.3\n",
            "mpmath==1.3.0\n",
            "msgpack==1.0.7\n",
            "multidict==6.0.4\n",
            "multipledispatch==1.0.0\n",
            "multiprocess==0.70.13\n",
            "multitasking==0.0.11\n",
            "murmurhash==1.0.10\n",
            "music21==9.1.0\n",
            "natsort==8.4.0\n",
            "nbclassic==1.0.0\n",
            "nbclient==0.9.0\n",
            "nbconvert==6.5.4\n",
            "nbformat==5.9.2\n",
            "nest-asyncio==1.5.8\n",
            "networkx==3.2.1\n",
            "nibabel==4.0.2\n",
            "nltk==3.8.1\n",
            "notebook==6.5.5\n",
            "notebook_shim==0.2.3\n",
            "numba==0.58.1\n",
            "numexpr==2.8.7\n",
            "numpy==1.23.5\n",
            "oauth2client==4.1.3\n",
            "oauthlib==3.2.2\n",
            "openai==0.22.0\n",
            "opencv-contrib-python==4.8.0.76\n",
            "opencv-python==4.8.0.76\n",
            "opencv-python-headless==4.8.1.78\n",
            "openpyxl==3.1.2\n",
            "opt-einsum==3.3.0\n",
            "optax==0.1.7\n",
            "orbax-checkpoint==0.4.3\n",
            "osqp==0.6.2.post8\n",
            "packaging==23.2\n",
            "pandas==1.5.3\n",
            "pandas-datareader==0.10.0\n",
            "pandas-gbq==0.17.9\n",
            "pandas-stubs==1.5.3.230304\n",
            "pandocfilters==1.5.0\n",
            "panel==1.3.4\n",
            "param==2.0.1\n",
            "parso==0.8.3\n",
            "parsy==2.1\n",
            "partd==1.4.1\n",
            "pathlib==1.0.1\n",
            "pathtools==0.1.2\n",
            "pathy==0.10.3\n",
            "patsy==0.5.3\n",
            "peewee==3.17.0\n",
            "peft==0.4.0\n",
            "pexpect==4.9.0\n",
            "pickleshare==0.7.5\n",
            "Pillow==9.4.0\n",
            "pip-tools==6.13.0\n",
            "platformdirs==4.0.0\n",
            "plotly==5.15.0\n",
            "plotnine==0.12.4\n",
            "pluggy==1.3.0\n",
            "polars==0.17.3\n",
            "pooch==1.8.0\n",
            "portpicker==1.5.2\n",
            "prefetch-generator==1.0.3\n",
            "preshed==3.0.9\n",
            "prettytable==3.9.0\n",
            "proglog==0.1.10\n",
            "progressbar2==4.2.0\n",
            "prometheus-client==0.19.0\n",
            "promise==2.3\n",
            "prompt-toolkit==3.0.41\n",
            "prophet==1.1.5\n",
            "proto-plus==1.22.3\n",
            "protobuf==3.20.3\n",
            "psutil==5.9.5\n",
            "psycopg2==2.9.9\n",
            "ptyprocess==0.7.0\n",
            "py-cpuinfo==9.0.0\n",
            "py4j==0.10.9.7\n",
            "pyarrow==9.0.0\n",
            "pyasn1==0.5.1\n",
            "pyasn1-modules==0.3.0\n",
            "pycocotools==2.0.7\n",
            "pycparser==2.21\n",
            "pyct==0.5.0\n",
            "pydantic==1.10.13\n",
            "pydata-google-auth==1.8.2\n",
            "pydot==1.4.2\n",
            "pydot-ng==2.0.0\n",
            "pydotplus==2.0.2\n",
            "PyDrive==1.3.1\n",
            "PyDrive2==1.6.3\n",
            "pyerfa==2.0.1.1\n",
            "pygame==2.5.2\n",
            "Pygments==2.16.1\n",
            "PyGObject==3.42.1\n",
            "PyJWT==2.3.0\n",
            "pymc==5.7.2\n",
            "pymystem3==0.2.0\n",
            "PyOpenGL==3.1.7\n",
            "pyOpenSSL==23.3.0\n",
            "pyparsing==3.1.1\n",
            "pyperclip==1.8.2\n",
            "pyproj==3.6.1\n",
            "pyproject_hooks==1.0.0\n",
            "pyshp==2.3.1\n",
            "PySocks==1.7.1\n",
            "pytensor==2.14.2\n",
            "pytest==7.4.3\n",
            "python-apt==0.0.0\n",
            "python-box==7.1.1\n",
            "python-dateutil==2.8.2\n",
            "python-louvain==0.16\n",
            "python-slugify==8.0.1\n",
            "python-utils==3.8.1\n",
            "pytz==2023.3.post1\n",
            "pyviz_comms==3.0.0\n",
            "PyWavelets==1.5.0\n",
            "PyYAML==6.0.1\n",
            "pyzmq==23.2.1\n",
            "qdldl==0.1.7.post0\n",
            "qudida==0.0.4\n",
            "ratelim==0.1.6\n",
            "referencing==0.31.1\n",
            "regex==2023.6.3\n",
            "requests==2.31.0\n",
            "requests-oauthlib==1.3.1\n",
            "requirements-parser==0.5.0\n",
            "responses==0.18.0\n",
            "rich==13.7.0\n",
            "rpds-py==0.13.2\n",
            "rpy2==3.4.2\n",
            "rsa==4.9\n",
            "safetensors==0.4.1\n",
            "scikit-image==0.19.3\n",
            "scikit-learn==1.2.2\n",
            "scikit-multilearn==0.2.0\n",
            "scipy==1.11.4\n",
            "scooby==0.9.2\n",
            "scs==3.2.4.post1\n",
            "seaborn==0.12.2\n",
            "SecretStorage==3.3.1\n",
            "Send2Trash==1.8.2\n",
            "sentence-transformers==2.2.2\n",
            "sentencepiece==0.1.99\n",
            "sentry-sdk==1.38.0\n",
            "setproctitle==1.3.3\n",
            "shapely==2.0.2\n",
            "shortuuid==1.0.11\n",
            "six==1.16.0\n",
            "sklearn-pandas==2.2.0\n",
            "smart-open==6.4.0\n",
            "smmap==5.0.1\n",
            "sniffio==1.3.0\n",
            "snowballstemmer==2.2.0\n",
            "sortedcontainers==2.4.0\n",
            "soundfile==0.12.1\n",
            "soupsieve==2.5\n",
            "soxr==0.3.7\n",
            "spacy==3.6.1\n",
            "spacy-legacy==3.0.12\n",
            "spacy-loggers==1.0.5\n",
            "Sphinx==5.0.2\n",
            "sphinxcontrib-applehelp==1.0.7\n",
            "sphinxcontrib-devhelp==1.0.5\n",
            "sphinxcontrib-htmlhelp==2.0.4\n",
            "sphinxcontrib-jsmath==1.0.1\n",
            "sphinxcontrib-qthelp==1.0.6\n",
            "sphinxcontrib-serializinghtml==1.1.9\n",
            "SQLAlchemy==2.0.23\n",
            "sqlglot==17.16.2\n",
            "sqlparse==0.4.4\n",
            "srsly==2.4.8\n",
            "stanio==0.3.0\n",
            "statsmodels==0.14.0\n",
            "sympy==1.12\n",
            "tables==3.8.0\n",
            "tabulate==0.9.0\n",
            "tbb==2021.11.0\n",
            "tblib==3.0.0\n",
            "tenacity==8.2.3\n",
            "tensorboard==2.14.1\n",
            "tensorboard-data-server==0.7.2\n",
            "tensorflow==2.14.0\n",
            "tensorflow-datasets==4.9.3\n",
            "tensorflow-estimator==2.14.0\n",
            "tensorflow-gcs-config==2.14.0\n",
            "tensorflow-hub==0.15.0\n",
            "tensorflow-io-gcs-filesystem==0.34.0\n",
            "tensorflow-metadata==1.14.0\n",
            "tensorflow-probability==0.22.0\n",
            "tensorstore==0.1.45\n",
            "termcolor==2.3.0\n",
            "terminado==0.18.0\n",
            "text-unidecode==1.3\n",
            "textblob==0.17.1\n",
            "tf-slim==1.1.0\n",
            "thinc==8.1.12\n",
            "threadpoolctl==3.2.0\n",
            "tifffile==2023.9.26\n",
            "tinycss2==1.2.1\n",
            "tokenizers==0.13.3\n",
            "toml==0.10.2\n",
            "tomli==2.0.1\n",
            "toolz==0.12.0\n",
            "torch @ https://download.pytorch.org/whl/cu118/torch-2.1.0%2Bcu118-cp310-cp310-linux_x86_64.whl#sha256=a81b554184492005543ddc32e96469f9369d778dedd195d73bda9bed407d6589\n",
            "torchaudio @ https://download.pytorch.org/whl/cu118/torchaudio-2.1.0%2Bcu118-cp310-cp310-linux_x86_64.whl#sha256=cdfd0a129406155eee595f408cafbb92589652da4090d1d2040f5453d4cae71f\n",
            "torchdata==0.7.0\n",
            "torchsummary==1.5.1\n",
            "torchtext==0.16.0\n",
            "torchvision @ https://download.pytorch.org/whl/cu118/torchvision-0.16.0%2Bcu118-cp310-cp310-linux_x86_64.whl#sha256=033712f65d45afe806676c4129dfe601ad1321d9e092df62b15847c02d4061dc\n",
            "tornado==6.3.2\n",
            "tqdm==4.66.1\n",
            "traitlets==5.7.1\n",
            "traittypes==0.2.1\n",
            "transformers==4.31.0\n",
            "triton==2.1.0\n",
            "trl==0.4.7\n",
            "tweepy==4.14.0\n",
            "typer==0.9.0\n",
            "types-pytz==2023.3.1.1\n",
            "types-setuptools==69.0.0.0\n",
            "typing_extensions==4.5.0\n",
            "tzlocal==5.2\n",
            "uc-micro-py==1.0.2\n",
            "uritemplate==4.1.1\n",
            "urllib3==2.0.7\n",
            "vega-datasets==0.9.0\n",
            "wadllib==1.3.6\n",
            "wandb==0.12.19\n",
            "wasabi==1.1.2\n",
            "wcwidth==0.2.12\n",
            "webcolors==1.13\n",
            "webencodings==0.5.1\n",
            "websocket-client==1.6.4\n",
            "Werkzeug==3.0.1\n",
            "widgetsnbextension==3.6.6\n",
            "wordcloud==1.9.2\n",
            "wrapt==1.14.1\n",
            "xarray==2023.7.0\n",
            "xarray-einstats==0.6.0\n",
            "xgboost==2.0.2\n",
            "xlrd==2.0.1\n",
            "xxhash==3.4.1\n",
            "xyzservices==2023.10.1\n",
            "yarl==1.9.3\n",
            "yellowbrick==1.5\n",
            "yfinance==0.2.32\n",
            "zict==3.0.0\n",
            "zipp==3.17.0\n"
          ]
        }
      ],
      "source": [
        "!pip install -q accelerate==0.21.0 peft==0.4.0 bitsandbytes==0.40.2 transformers==4.31.0 trl==0.4.7 datasets==2.4.0 huggingface_hub matplotlib numpy>=1.25 openai==0.22.0 wandb==0.12.19 pandas requests scikit_learn scipy gensim sentence-transformers scikit-multilearn\n",
        "#!pip install sklearn-hierarchical-classification\n",
        "# !pip install -q accelerate peft bitsandbytes transformers trl datasets huggingface_hub matplotlib numpy openai wandb pandas requests scikit_learn scipy gensim\n",
        "!pip freeze > requirements.txt\n",
        "!cat requirements.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GLGv8lVuYC00",
        "outputId": "171fecf5-75e1-4d0e-da68-ca7b41545025"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /gdrive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/gdrive\")\n",
        "import os\n",
        "os.environ[\"COLAB_WORKDIR\"] = \"/gdrive/My Drive/projects/bambas\""
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!echo $COLAB_WORKDIR\n",
        "!git clone https://github.com/lfmatosm/sklearn-hierarchical-classification.git\n",
        "!pip install ./sklearn-hierarchical-classification"
      ],
      "metadata": {
        "id": "Seupsx5902yc",
        "outputId": "c11e9c83-46cf-4dee-8c63-2a647196be50",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/gdrive/My Drive/projects/bambas\n",
            "Cloning into 'sklearn-hierarchical-classification'...\n",
            "remote: Enumerating objects: 760, done.\u001b[K\n",
            "remote: Counting objects: 100% (139/139), done.\u001b[K\n",
            "remote: Compressing objects: 100% (20/20), done.\u001b[K\n",
            "remote: Total 760 (delta 127), reused 121 (delta 119), pack-reused 621\u001b[K\n",
            "Receiving objects: 100% (760/760), 139.46 KiB | 9.30 MiB/s, done.\n",
            "Resolving deltas: 100% (500/500), done.\n",
            "Processing ./sklearn-hierarchical-classification\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: networkx>=2.4 in /usr/local/lib/python3.10/dist-packages (from sklearn-hierarchical-classification==1.3.2) (3.2.1)\n",
            "Requirement already satisfied: numpy>=1.13.1 in /usr/local/lib/python3.10/dist-packages (from sklearn-hierarchical-classification==1.3.2) (1.23.5)\n",
            "Requirement already satisfied: scikit-learn>=0.19.0 in /usr/local/lib/python3.10/dist-packages (from sklearn-hierarchical-classification==1.3.2) (1.2.2)\n",
            "Requirement already satisfied: scipy>=0.19.1 in /usr/local/lib/python3.10/dist-packages (from sklearn-hierarchical-classification==1.3.2) (1.11.4)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.19.0->sklearn-hierarchical-classification==1.3.2) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.19.0->sklearn-hierarchical-classification==1.3.2) (3.2.0)\n",
            "Building wheels for collected packages: sklearn-hierarchical-classification\n",
            "  Building wheel for sklearn-hierarchical-classification (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sklearn-hierarchical-classification: filename=sklearn_hierarchical_classification-1.3.2-py3-none-any.whl size=23601 sha256=3c2549309e069cb3b142fc018da9738a17fd007a3ec0dea0b8c6e1b65f6c5e3e\n",
            "  Stored in directory: /root/.cache/pip/wheels/bd/f7/f6/5636c8150b98522ffa516cf2ebf0a699b64576b9abd58ae843\n",
            "Successfully built sklearn-hierarchical-classification\n",
            "Installing collected packages: sklearn-hierarchical-classification\n",
            "Successfully installed sklearn-hierarchical-classification-1.3.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Class with feature-extraction, last 4 layers"
      ],
      "metadata": {
        "id": "jpgh7sWyBPSt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!PYTHONPATH=\"/gdrive/My Drive/projects/bambas\" python -m src.feature_extraction \\\n",
        "  --model xlm-roberta-base \\\n",
        "  --dataset semeval2024 \\\n",
        "  --extraction_method layers \\\n",
        "  --layers 9 10 11 12 \\\n",
        "  --agg_method \"avg\""
      ],
      "metadata": {
        "id": "rbjiBjEUOJNm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8835cdc5-71ab-4cd3-c264-733af50bd1bc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-12-03 13:53:45.389816: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2023-12-03 13:53:45.389874: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2023-12-03 13:53:45.389909: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2023-12-03 13:53:46.394957: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Arguments: Namespace(dataset='semeval2024', model='xlm-roberta-base', extraction_method='layers', layers=[9, 10, 11, 12], agg_method='avg')\n",
            "Using device: {'': 0}\n",
            "#0:  11% 3/28 [00:00<00:02, 10.48ba/s]\n",
            "\n",
            "#2:   0% 0/28 [00:00<?, ?ba/s]\u001b[A\u001b[A\n",
            "#0:  18% 5/28 [00:00<00:02, 10.90ba/s]\n",
            "\n",
            "#2:   4% 1/28 [00:00<00:03,  7.09ba/s]\u001b[A\u001b[A\n",
            "#0:  25% 7/28 [00:00<00:01, 10.88ba/s]\n",
            "\n",
            "#2:  11% 3/28 [00:00<00:02,  9.92ba/s]\u001b[A\u001b[A\n",
            "#1:  11% 3/28 [00:00<00:02,  9.50ba/s]\u001b[A\n",
            "\n",
            "#0:  32% 9/28 [00:00<00:01, 11.26ba/s]\n",
            "#1:  18% 5/28 [00:00<00:01, 12.92ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#3:   0% 0/28 [00:00<?, ?ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  25% 7/28 [00:00<00:01, 15.01ba/s]\u001b[A\u001b[A\n",
            "#1:  25% 7/28 [00:00<00:01, 14.34ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#0:  39% 11/28 [00:01<00:01, 11.16ba/s]\n",
            "\n",
            "#2:  32% 9/28 [00:00<00:01, 15.32ba/s]\u001b[A\u001b[A\n",
            "#1:  32% 9/28 [00:00<00:01, 13.52ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#0:  46% 13/28 [00:01<00:01, 11.59ba/s]\n",
            "\n",
            "#2:  39% 11/28 [00:00<00:01, 14.36ba/s]\u001b[A\u001b[A\n",
            "#1:  39% 11/28 [00:00<00:01, 13.57ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#3:  18% 5/28 [00:00<00:01, 12.78ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#0:  54% 15/28 [00:01<00:01, 11.72ba/s]\n",
            "\n",
            "\n",
            "#3:  25% 7/28 [00:00<00:01, 14.32ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#0:  64% 18/28 [00:01<00:00, 15.14ba/s]\n",
            "\n",
            "#2:  54% 15/28 [00:01<00:00, 14.16ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  32% 9/28 [00:00<00:01, 15.00ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#0:  71% 20/28 [00:01<00:00, 14.60ba/s]\n",
            "\n",
            "#2:  64% 18/28 [00:01<00:00, 16.62ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  39% 11/28 [00:00<00:01, 13.78ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#0:  79% 22/28 [00:01<00:00, 14.72ba/s]\n",
            "\n",
            "#2:  71% 20/28 [00:01<00:00, 15.23ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  46% 13/28 [00:00<00:01, 14.54ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#0:  89% 25/28 [00:01<00:00, 17.34ba/s]\n",
            "\n",
            "#2:  79% 22/28 [00:01<00:00, 15.12ba/s]\u001b[A\u001b[A\n",
            "#1:  79% 22/28 [00:01<00:00, 17.29ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#0: 100% 28/28 [00:02<00:00, 13.86ba/s]\n",
            "\n",
            "\n",
            "#2:  86% 24/28 [00:01<00:00, 15.36ba/s]\u001b[A\u001b[A\n",
            "#1:  86% 24/28 [00:01<00:00, 17.26ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#3:  61% 17/28 [00:01<00:00, 13.91ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#1:  93% 26/28 [00:01<00:00, 16.61ba/s]\u001b[A\n",
            "\n",
            "#2:  93% 26/28 [00:01<00:00, 14.23ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#1: 100% 28/28 [00:01<00:00, 15.23ba/s]\n",
            "#2: 100% 28/28 [00:01<00:00, 14.71ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3:  75% 21/28 [00:01<00:00, 13.86ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  89% 25/28 [00:01<00:00, 18.14ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3: 100% 28/28 [00:01<00:00, 15.62ba/s]\n",
            "#0: 100% 2/2 [00:00<00:00, 17.48ba/s]\n",
            "\n",
            "#1:   0% 0/2 [00:00<?, ?ba/s]\u001b[A\n",
            "#1: 100% 2/2 [00:00<00:00, 14.20ba/s]\n",
            "\n",
            "\n",
            "#2:   0% 0/2 [00:00<?, ?ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:   0% 0/2 [00:00<?, ?ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2: 100% 2/2 [00:00<00:00, 17.47ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3: 100% 2/2 [00:00<00:00, 13.85ba/s]\n",
            "#0:   0% 0/4 [00:00<?, ?ba/s]\n",
            "#0:  50% 2/4 [00:00<00:00, 16.66ba/s]\n",
            "#0: 100% 4/4 [00:00<00:00, 20.85ba/s]\n",
            "#1: 100% 4/4 [00:00<00:00, 20.78ba/s]\n",
            "\n",
            "\n",
            "#2:   0% 0/4 [00:00<?, ?ba/s]\u001b[A\u001b[A\n",
            "\n",
            "#2:  50% 2/4 [00:00<00:00, 16.50ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#2: 100% 4/4 [00:00<00:00, 18.89ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3: 100% 4/4 [00:00<00:00, 21.01ba/s]\n",
            "You're using a XLMRobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
            "Batch no. 1\n",
            "Batch no. 2\n",
            "Batch no. 3\n",
            "Batch no. 4\n",
            "Batch no. 5\n",
            "Batch no. 6\n",
            "Batch no. 7\n",
            "Batch no. 8\n",
            "Batch no. 9\n",
            "Batch no. 10\n",
            "Batch no. 11\n",
            "Batch no. 12\n",
            "Batch no. 13\n",
            "Batch no. 14\n",
            "Batch no. 15\n",
            "Batch no. 16\n",
            "Batch no. 17\n",
            "Batch no. 18\n",
            "Batch no. 19\n",
            "Batch no. 20\n",
            "Batch no. 21\n",
            "Batch no. 22\n",
            "Batch no. 23\n",
            "Batch no. 24\n",
            "Batch no. 25\n",
            "Batch no. 26\n",
            "Batch no. 27\n",
            "Batch no. 28\n",
            "Batch no. 29\n",
            "Batch no. 30\n",
            "Batch no. 31\n",
            "Batch no. 32\n",
            "Batch no. 33\n",
            "Batch no. 34\n",
            "Batch no. 35\n",
            "Batch no. 36\n",
            "Batch no. 37\n",
            "Batch no. 38\n",
            "Batch no. 39\n",
            "Batch no. 40\n",
            "Batch no. 41\n",
            "Batch no. 42\n",
            "Batch no. 43\n",
            "Batch no. 44\n",
            "Batch no. 45\n",
            "Batch no. 46\n",
            "Batch no. 47\n",
            "Batch no. 48\n",
            "Batch no. 49\n",
            "Batch no. 50\n",
            "Batch no. 51\n",
            "Batch no. 52\n",
            "Batch no. 53\n",
            "Batch no. 54\n",
            "Batch no. 55\n",
            "Batch no. 56\n",
            "Batch no. 57\n",
            "Batch no. 58\n",
            "Batch no. 59\n",
            "Batch no. 60\n",
            "Batch no. 61\n",
            "Batch no. 62\n",
            "Batch no. 63\n",
            "Batch no. 64\n",
            "Batch no. 65\n",
            "Batch no. 66\n",
            "Batch no. 67\n",
            "Batch no. 68\n",
            "Batch no. 69\n",
            "Batch no. 70\n",
            "Batch no. 71\n",
            "Batch no. 72\n",
            "Batch no. 73\n",
            "Batch no. 74\n",
            "Batch no. 75\n",
            "Batch no. 76\n",
            "Batch no. 77\n",
            "Batch no. 78\n",
            "Batch no. 79\n",
            "Batch no. 80\n",
            "Batch no. 81\n",
            "Batch no. 82\n",
            "Batch no. 83\n",
            "Batch no. 84\n",
            "Batch no. 85\n",
            "Batch no. 86\n",
            "Batch no. 87\n",
            "Batch no. 88\n",
            "Batch no. 89\n",
            "Batch no. 90\n",
            "Batch no. 91\n",
            "Batch no. 92\n",
            "Batch no. 93\n",
            "Batch no. 94\n",
            "Batch no. 95\n",
            "Batch no. 96\n",
            "Batch no. 97\n",
            "Batch no. 98\n",
            "Batch no. 99\n",
            "Batch no. 100\n",
            "Batch no. 101\n",
            "Batch no. 102\n",
            "Batch no. 103\n",
            "Batch no. 104\n",
            "Batch no. 105\n",
            "Batch no. 106\n",
            "Batch no. 107\n",
            "Batch no. 108\n",
            "Batch no. 109\n",
            "Batch no. 110\n",
            "Batch no. 1\n",
            "Batch no. 2\n",
            "Batch no. 3\n",
            "Batch no. 4\n",
            "Batch no. 5\n",
            "Batch no. 6\n",
            "Batch no. 7\n",
            "Batch no. 8\n",
            "Batch no. 9\n",
            "Batch no. 10\n",
            "Batch no. 11\n",
            "Batch no. 12\n",
            "Batch no. 13\n",
            "Batch no. 14\n",
            "Batch no. 15\n",
            "Batch no. 16\n",
            "Batch no. 1\n",
            "Batch no. 2\n",
            "Batch no. 3\n",
            "Batch no. 4\n",
            "Batch no. 5\n",
            "Batch no. 6\n",
            "Batch no. 7\n",
            "Batch no. 8\n",
            "Saved feature-extraction file to /gdrive/My Drive/projects/bambas/feature_extraction/1701611901_xlm-roberta-base_train_features.json\n",
            "Saved feature-extraction file to /gdrive/My Drive/projects/bambas/feature_extraction/1701611901_xlm-roberta-base_test_features.json\n",
            "Saved feature-extraction file to /gdrive/My Drive/projects/bambas/feature_extraction/1701611901_xlm-roberta-base_dev_features.json\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!PYTHONPATH=\"/gdrive/My Drive/projects/bambas\" python -m src.classification \\\n",
        "  --classifier \"HiMLP\" \\\n",
        "  --dataset semeval2024 \\\n",
        "  --train_features \"/gdrive/My Drive/projects/bambas/feature_extraction/1701611901_xlm-roberta-base_train_features.json\" \\\n",
        "  --test_features \"/gdrive/My Drive/projects/bambas/feature_extraction/1701611901_xlm-roberta-base_test_features.json\" \\\n",
        "  --dev_features \"/gdrive/My Drive/projects/bambas/feature_extraction/1701611901_xlm-roberta-base_dev_features.json\" \\\n",
        "  --seed 1"
      ],
      "metadata": {
        "id": "yRuPWP-dOSsK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a7503a65-ced9-468d-c412-01572a3ed862"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Arguments: Namespace(classifier='HiMLP', dataset='semeval2024', train_features='/gdrive/My Drive/projects/bambas/feature_extraction/1701611901_xlm-roberta-base_train_features.json', test_features='/gdrive/My Drive/projects/bambas/feature_extraction/1701611901_xlm-roberta-base_test_features.json', dev_features='/gdrive/My Drive/projects/bambas/feature_extraction/1701611901_xlm-roberta-base_dev_features.json', max_iter=400, alpha=0.0001, seed=1)\n",
            "Loading features info files\n",
            "Loading dataset files\n",
            "Labels: [['Causal Oversimplification', 'Doubt', 'Flag-waving', 'Reasoning', 'Pathos', 'Obfuscation, Intentional vagueness, Confusion', 'Black-and-white Fallacy/Dictatorship', 'Simplification', 'Distraction', 'Slogans', 'Smears', 'Glittering generalities (Virtue)', \"Misrepresentation of Someone's Position (Straw Man)\", 'Ad Hominem', 'Name calling/Labeling', 'Reductio ad hitlerum', 'Ethos', 'Repetition', 'Bandwagon', 'Presenting Irrelevant Data (Red Herring)', 'Appeal to fear/prejudice', 'Appeal to authority', 'Loaded Language', 'Whataboutism', 'Appeal to (Strong) Emotions', 'Exaggeration/Minimisation', 'Thought-terminating cliché', 'Logos', 'Transfer', 'Justification']]\n",
            "No. of labels in dataset (includes non-labeled samples): 30\n",
            "Loading features array files\n",
            "Iteration 1, loss = 1.07521634\n",
            "Validation score: 0.499627\n",
            "Iteration 2, loss = 1.02867032\n",
            "Validation score: 0.486950\n",
            "Iteration 3, loss = 1.02701011\n",
            "Validation score: 0.501119\n",
            "Iteration 4, loss = 1.02502699\n",
            "Validation score: 0.488441\n",
            "Iteration 5, loss = 1.02432988\n",
            "Validation score: 0.501119\n",
            "Iteration 6, loss = 1.02726515\n",
            "Validation score: 0.501119\n",
            "Iteration 7, loss = 1.02958255\n",
            "Validation score: 0.501119\n",
            "Iteration 8, loss = 1.02983699\n",
            "Validation score: 0.504847\n",
            "Iteration 9, loss = 1.02759368\n",
            "Validation score: 0.501119\n",
            "Iteration 10, loss = 1.02652886\n",
            "Validation score: 0.515287\n",
            "Iteration 11, loss = 1.02710073\n",
            "Validation score: 0.515287\n",
            "Iteration 12, loss = 1.02866873\n",
            "Validation score: 0.517524\n",
            "Iteration 13, loss = 1.02717517\n",
            "Validation score: 0.504847\n",
            "Iteration 14, loss = 1.02854734\n",
            "Validation score: 0.505593\n",
            "Iteration 15, loss = 1.02831395\n",
            "Validation score: 0.505593\n",
            "Iteration 16, loss = 1.02675111\n",
            "Validation score: 0.519761\n",
            "Iteration 17, loss = 1.02842200\n",
            "Validation score: 0.519761\n",
            "Iteration 18, loss = 1.02271907\n",
            "Validation score: 0.505593\n",
            "Iteration 19, loss = 1.02408275\n",
            "Validation score: 0.505593\n",
            "Iteration 20, loss = 1.02299415\n",
            "Validation score: 0.501119\n",
            "Iteration 21, loss = 1.02386205\n",
            "Validation score: 0.505593\n",
            "Iteration 22, loss = 1.02368336\n",
            "Validation score: 0.515287\n",
            "Iteration 23, loss = 1.02443624\n",
            "Validation score: 0.519761\n",
            "Iteration 24, loss = 1.02246620\n",
            "Validation score: 0.519761\n",
            "Iteration 25, loss = 1.02353160\n",
            "Validation score: 0.515287\n",
            "Iteration 26, loss = 1.02268926\n",
            "Validation score: 0.504847\n",
            "Iteration 27, loss = 1.02220264\n",
            "Validation score: 0.504847\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.14354560\n",
            "Validation score: 0.612971\n",
            "Iteration 2, loss = 0.91531454\n",
            "Validation score: 0.569038\n",
            "Iteration 3, loss = 0.90097920\n",
            "Validation score: 0.573222\n",
            "Iteration 4, loss = 0.89986126\n",
            "Validation score: 0.589958\n",
            "Iteration 5, loss = 0.90165739\n",
            "Validation score: 0.596234\n",
            "Iteration 6, loss = 0.90130939\n",
            "Validation score: 0.569038\n",
            "Iteration 7, loss = 0.90215638\n",
            "Validation score: 0.612971\n",
            "Iteration 8, loss = 0.89748119\n",
            "Validation score: 0.612971\n",
            "Iteration 9, loss = 0.90211159\n",
            "Validation score: 0.569038\n",
            "Iteration 10, loss = 0.89926257\n",
            "Validation score: 0.612971\n",
            "Iteration 11, loss = 0.89930566\n",
            "Validation score: 0.569038\n",
            "Iteration 12, loss = 0.89685012\n",
            "Validation score: 0.600418\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 0.68426842\n",
            "Validation score: 0.803109\n",
            "Iteration 2, loss = 0.52820721\n",
            "Validation score: 0.803109\n",
            "Iteration 3, loss = 0.52055205\n",
            "Validation score: 0.803109\n",
            "Iteration 4, loss = 0.49957309\n",
            "Validation score: 0.803109\n",
            "Iteration 5, loss = 0.49784691\n",
            "Validation score: 0.803109\n",
            "Iteration 6, loss = 0.49376923\n",
            "Validation score: 0.803109\n",
            "Iteration 7, loss = 0.49400477\n",
            "Validation score: 0.803109\n",
            "Iteration 8, loss = 0.49392060\n",
            "Validation score: 0.803109\n",
            "Iteration 9, loss = 0.49330558\n",
            "Validation score: 0.803109\n",
            "Iteration 10, loss = 0.49362627\n",
            "Validation score: 0.803109\n",
            "Iteration 11, loss = 0.49340249\n",
            "Validation score: 0.803109\n",
            "Iteration 12, loss = 0.49387064\n",
            "Validation score: 0.803109\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.46550015\n",
            "Validation score: 0.554839\n",
            "Iteration 2, loss = 0.98943709\n",
            "Validation score: 0.554839\n",
            "Iteration 3, loss = 1.03629298\n",
            "Validation score: 0.554839\n",
            "Iteration 4, loss = 0.98108639\n",
            "Validation score: 0.554839\n",
            "Iteration 5, loss = 0.95011588\n",
            "Validation score: 0.554839\n",
            "Iteration 6, loss = 0.95042299\n",
            "Validation score: 0.554839\n",
            "Iteration 7, loss = 0.94477032\n",
            "Validation score: 0.554839\n",
            "Iteration 8, loss = 0.94502605\n",
            "Validation score: 0.554839\n",
            "Iteration 9, loss = 0.94487139\n",
            "Validation score: 0.554839\n",
            "Iteration 10, loss = 0.94483790\n",
            "Validation score: 0.554839\n",
            "Iteration 11, loss = 0.94309965\n",
            "Validation score: 0.554839\n",
            "Iteration 12, loss = 0.94296878\n",
            "Validation score: 0.554839\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.66948036\n",
            "Validation score: 0.210526\n",
            "Iteration 2, loss = 1.12354360\n",
            "Validation score: 0.578947\n",
            "Iteration 3, loss = 0.88368512\n",
            "Validation score: 0.578947\n",
            "Iteration 4, loss = 0.83745001\n",
            "Validation score: 0.578947\n",
            "Iteration 5, loss = 0.86293363\n",
            "Validation score: 0.578947\n",
            "Iteration 6, loss = 0.88553457\n",
            "Validation score: 0.578947\n",
            "Iteration 7, loss = 0.91071466\n",
            "Validation score: 0.578947\n",
            "Iteration 8, loss = 0.91839008\n",
            "Validation score: 0.578947\n",
            "Iteration 9, loss = 0.91774007\n",
            "Validation score: 0.578947\n",
            "Iteration 10, loss = 0.90751794\n",
            "Validation score: 0.578947\n",
            "Iteration 11, loss = 0.88994494\n",
            "Validation score: 0.578947\n",
            "Iteration 12, loss = 0.86955711\n",
            "Validation score: 0.578947\n",
            "Iteration 13, loss = 0.85122118\n",
            "Validation score: 0.578947\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.44955842\n",
            "Validation score: 0.454545\n",
            "Iteration 2, loss = 1.36617931\n",
            "Validation score: 0.454545\n",
            "Iteration 3, loss = 1.36393564\n",
            "Validation score: 0.454545\n",
            "Iteration 4, loss = 1.35890605\n",
            "Validation score: 0.442688\n",
            "Iteration 5, loss = 1.36028203\n",
            "Validation score: 0.454545\n",
            "Iteration 6, loss = 1.36070351\n",
            "Validation score: 0.454545\n",
            "Iteration 7, loss = 1.35672096\n",
            "Validation score: 0.454545\n",
            "Iteration 8, loss = 1.35650291\n",
            "Validation score: 0.454545\n",
            "Iteration 9, loss = 1.35647879\n",
            "Validation score: 0.454545\n",
            "Iteration 10, loss = 1.36074428\n",
            "Validation score: 0.454545\n",
            "Iteration 11, loss = 1.36637229\n",
            "Validation score: 0.454545\n",
            "Iteration 12, loss = 1.36557686\n",
            "Validation score: 0.454545\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 0.97503250\n",
            "Validation score: 0.729537\n",
            "Iteration 2, loss = 0.73281081\n",
            "Validation score: 0.729537\n",
            "Iteration 3, loss = 0.72823499\n",
            "Validation score: 0.729537\n",
            "Iteration 4, loss = 0.72387607\n",
            "Validation score: 0.729537\n",
            "Iteration 5, loss = 0.72437415\n",
            "Validation score: 0.729537\n",
            "Iteration 6, loss = 0.72483635\n",
            "Validation score: 0.729537\n",
            "Iteration 7, loss = 0.72624552\n",
            "Validation score: 0.729537\n",
            "Iteration 8, loss = 0.72728255\n",
            "Validation score: 0.729537\n",
            "Iteration 9, loss = 0.72518599\n",
            "Validation score: 0.729537\n",
            "Iteration 10, loss = 0.72491918\n",
            "Validation score: 0.729537\n",
            "Iteration 11, loss = 0.72567888\n",
            "Validation score: 0.729537\n",
            "Iteration 12, loss = 0.72508037\n",
            "Validation score: 0.729537\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.18928710\n",
            "Validation score: 0.502392\n",
            "Iteration 2, loss = 1.08715404\n",
            "Validation score: 0.497608\n",
            "Iteration 3, loss = 1.08156938\n",
            "Validation score: 0.502392\n",
            "Iteration 4, loss = 1.07565711\n",
            "Validation score: 0.442584\n",
            "Iteration 5, loss = 1.07773939\n",
            "Validation score: 0.502392\n",
            "Iteration 6, loss = 1.07561252\n",
            "Validation score: 0.502392\n",
            "Iteration 7, loss = 1.07411965\n",
            "Validation score: 0.442584\n",
            "Iteration 8, loss = 1.07347585\n",
            "Validation score: 0.437799\n",
            "Iteration 9, loss = 1.08413139\n",
            "Validation score: 0.497608\n",
            "Iteration 10, loss = 1.07659976\n",
            "Validation score: 0.497608\n",
            "Iteration 11, loss = 1.07573705\n",
            "Validation score: 0.502392\n",
            "Iteration 12, loss = 1.07624782\n",
            "Validation score: 0.442584\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.11967936\n",
            "Validation score: 0.605960\n",
            "Iteration 2, loss = 1.10537379\n",
            "Validation score: 0.605960\n",
            "Iteration 3, loss = 1.10464369\n",
            "Validation score: 0.605960\n",
            "Iteration 4, loss = 1.10661760\n",
            "Validation score: 0.605960\n",
            "Iteration 5, loss = 1.10963339\n",
            "Validation score: 0.605960\n",
            "Iteration 6, loss = 1.10739794\n",
            "Validation score: 0.605960\n",
            "Iteration 7, loss = 1.10843234\n",
            "Validation score: 0.605960\n",
            "Iteration 8, loss = 1.10326613\n",
            "Validation score: 0.605960\n",
            "Iteration 9, loss = 1.10910160\n",
            "Validation score: 0.605960\n",
            "Iteration 10, loss = 1.11109425\n",
            "Validation score: 0.605960\n",
            "Iteration 11, loss = 1.10566297\n",
            "Validation score: 0.605960\n",
            "Iteration 12, loss = 1.10268768\n",
            "Validation score: 0.605960\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "\n",
            "Validation set:\n",
            "\tPrecision: 0.2972027972027972\n",
            "\tRecall: 0.4893711248892825\n",
            "\tF1: 0.3698125836680053\n",
            "\n",
            "Saving validation set results to /gdrive/My Drive/projects/bambas/classification/results.csv\n",
            "3\n",
            "\n",
            "Predicting for test file\n",
            "Finished successfully. dev_unlabeled predictions saved at /gdrive/My Drive/projects/bambas/classification/dev_unlabeled_predictions.json.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Class with CLS token"
      ],
      "metadata": {
        "id": "OUwSYRpDezQX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "!PYTHONPATH=\"/gdrive/My Drive/projects/bambas\" python -m src.feature_extraction \\\n",
        "  --model xlm-roberta-base \\\n",
        "  --dataset semeval2024 \\\n",
        "  --extraction_method layers \\\n",
        "  --layers 4 5 6 7 \\\n",
        "  --agg_method \"avg\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mhWMlJooezkX",
        "outputId": "15b3ba89-29eb-4747-b45a-7bbf25a01f9d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-12-03 02:21:57.401231: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2023-12-03 02:21:57.401284: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2023-12-03 02:21:57.401324: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "Traceback (most recent call last):\n",
            "  File \"<frozen importlib._bootstrap>\", line 1022, in _find_and_load\n",
            "KeyboardInterrupt\n",
            "thread '<unnamed>' panicked at /github/home/.cargo/registry/src/index.crates.io-6f17d22bba15001f/pyo3-0.18.3/src/err/mod.rs:790:5:\n",
            "Python API call failed\n",
            "note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/lib/python3.10/runpy.py\", line 196, in _run_module_as_main\n",
            "    return _run_code(code, main_globals, None,\n",
            "  File \"/usr/lib/python3.10/runpy.py\", line 86, in _run_code\n",
            "    exec(code, run_globals)\n",
            "  File \"/gdrive/My Drive/projects/bambas/src/feature_extraction.py\", line 14, in <module>\n",
            "    from transformers import AutoModel, AutoTokenizer, DataCollatorWithPadding\n",
            "  File \"<frozen importlib._bootstrap>\", line 1075, in _handle_fromlist\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/utils/import_utils.py\", line 1089, in __getattr__\n",
            "    module = self._get_module(self._class_to_module[name])\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/utils/import_utils.py\", line 1099, in _get_module\n",
            "    return importlib.import_module(\".\" + module_name, self.__name__)\n",
            "  File \"/usr/lib/python3.10/importlib/__init__.py\", line 126, in import_module\n",
            "    return _bootstrap._gcd_import(name[level:], package, level)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/data/__init__.py\", line 27, in <module>\n",
            "    from .processors import (\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/data/processors/__init__.py\", line 15, in <module>\n",
            "    from .glue import glue_convert_examples_to_features, glue_output_modes, glue_processors, glue_tasks_num_labels\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/data/processors/glue.py\", line 30, in <module>\n",
            "    import tensorflow as tf\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/__init__.py\", line 38, in <module>\n",
            "    from tensorflow.python.tools import module_util as _module_util\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/__init__.py\", line 43, in <module>\n",
            "    from tensorflow.python.tpu import api\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/tpu/api.py\", line 23, in <module>\n",
            "    from tensorflow.python.tpu import feature_column_v2\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/tpu/feature_column_v2.py\", line 19, in <module>\n",
            "    from tensorflow.python.feature_column import feature_column as fc\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/feature_column/feature_column.py\", line 143, in <module>\n",
            "    from tensorflow.python.layers import base\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/layers/base.py\", line 16, in <module>\n",
            "    from tensorflow.python.keras.legacy_tf_layers import base\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/keras/__init__.py\", line 25, in <module>\n",
            "    from tensorflow.python.keras import models\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/keras/models.py\", line 20, in <module>\n",
            "    from tensorflow.python.keras import metrics as metrics_module\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/keras/metrics.py\", line 35, in <module>\n",
            "    from tensorflow.python.keras import activations\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/keras/activations.py\", line 18, in <module>\n",
            "    from tensorflow.python.keras.layers import advanced_activations\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/keras/layers/__init__.py\", line 22, in <module>\n",
            "    from tensorflow.python.keras.engine.input_layer import Input\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/keras/engine/input_layer.py\", line 24, in <module>\n",
            "    from tensorflow.python.keras.engine import base_layer\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/keras/engine/base_layer.py\", line 54, in <module>\n",
            "    from tensorflow.python.keras.mixed_precision import loss_scale_optimizer\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/keras/mixed_precision/loss_scale_optimizer.py\", line 17, in <module>\n",
            "    from tensorflow.python.distribute import collective_all_reduce_strategy\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/distribute/collective_all_reduce_strategy.py\", line 32, in <module>\n",
            "    from tensorflow.python.distribute import mirrored_strategy\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/distribute/mirrored_strategy.py\", line 34, in <module>\n",
            "    from tensorflow.python.distribute.cluster_resolver import tfconfig_cluster_resolver\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/distribute/cluster_resolver/__init__.py\", line 27, in <module>\n",
            "    from tensorflow.python.distribute.cluster_resolver.gce_cluster_resolver import GCEClusterResolver\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/tensorflow/python/distribute/cluster_resolver/gce_cluster_resolver.py\", line 24, in <module>\n",
            "    from googleapiclient import discovery  # pylint: disable=g-import-not-at-top\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/googleapiclient/discovery.py\", line 45, in <module>\n",
            "    from google.oauth2 import service_account\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/google/oauth2/service_account.py\", line 77, in <module>\n",
            "    from google.auth import _service_account_info\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/google/auth/_service_account_info.py\", line 22, in <module>\n",
            "    from google.auth import crypt\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/google/auth/crypt/__init__.py\", line 43, in <module>\n",
            "    from google.auth.crypt import rsa\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/google/auth/crypt/rsa.py\", line 20, in <module>\n",
            "    from google.auth.crypt import _cryptography_rsa\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/google/auth/crypt/_cryptography_rsa.py\", line 22, in <module>\n",
            "    import cryptography.exceptions\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cryptography/exceptions.py\", line 9, in <module>\n",
            "    from cryptography.hazmat.bindings._rust import exceptions as rust_exceptions\n",
            "pyo3_runtime.PanicException: Python API call failed\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "cf0nn31LeztV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Class with sentence feature-extraction"
      ],
      "metadata": {
        "id": "rEEQUz5o2qcz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!PYTHONPATH=\"/gdrive/My Drive/projects/bambas\" python -m src.feature_extraction \\\n",
        "  --model \"sentence-transformers/stsb-xlm-r-multilingual\" \\\n",
        "  --dataset semeval2024 \\\n",
        "  --extraction_method sentence"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HM8hyMJe2qvJ",
        "outputId": "0adcdca7-2579-456b-f12e-4205c5e09b82"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-12-03 13:35:46.274870: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2023-12-03 13:35:46.274932: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2023-12-03 13:35:46.274972: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2023-12-03 13:35:47.762878: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Arguments: Namespace(dataset='semeval2024', model='sentence-transformers/stsb-xlm-r-multilingual', extraction_method='sentence', layers=[], agg_method=None)\n",
            "Using device: {'': 0}\n",
            "#0:   7% 2/28 [00:00<00:01, 14.78ba/s]\n",
            "#0:  14% 4/28 [00:00<00:01, 17.00ba/s]\n",
            "#0:  25% 7/28 [00:00<00:01, 16.62ba/s]\n",
            "#0:  32% 9/28 [00:00<00:01, 15.90ba/s]\n",
            "#0:  39% 11/28 [00:00<00:01, 16.52ba/s]\n",
            "#0:  50% 14/28 [00:00<00:00, 18.15ba/s]\n",
            "#1:  32% 9/28 [00:00<00:01, 14.11ba/s]\u001b[A\n",
            "\n",
            "#0:  57% 16/28 [00:00<00:00, 16.18ba/s]\n",
            "#1:  39% 11/28 [00:00<00:01, 14.06ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#3:   0% 0/28 [00:00<?, ?ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#0:  64% 18/28 [00:01<00:00, 15.50ba/s]\n",
            "#1:  46% 13/28 [00:00<00:01, 13.40ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#3:   4% 1/28 [00:00<00:03,  7.73ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  11% 3/28 [00:00<00:02, 10.57ba/s]\u001b[A\u001b[A\n",
            "#0:  71% 20/28 [00:01<00:00, 14.03ba/s]\n",
            "\n",
            "\n",
            "#3:  11% 3/28 [00:00<00:02, 11.89ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  18% 5/28 [00:00<00:01, 12.37ba/s]\u001b[A\u001b[A\n",
            "#0:  79% 22/28 [00:01<00:00, 13.88ba/s]\n",
            "\n",
            "\n",
            "#3:  18% 5/28 [00:00<00:01, 14.98ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#1:  68% 19/28 [00:01<00:00, 14.32ba/s]\u001b[A\n",
            "\n",
            "#0:  89% 25/28 [00:01<00:00, 15.79ba/s]\n",
            "\n",
            "\n",
            "#3:  25% 7/28 [00:00<00:01, 15.25ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#1:  75% 21/28 [00:01<00:00, 14.98ba/s]\u001b[A\n",
            "\n",
            "#0: 100% 28/28 [00:01<00:00, 16.10ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3:  32% 9/28 [00:00<00:01, 13.61ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#1:  82% 23/28 [00:01<00:00, 14.83ba/s]\u001b[A\n",
            "\n",
            "#2:  43% 12/28 [00:00<00:01, 14.57ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  39% 11/28 [00:00<00:01, 12.95ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#1:  89% 25/28 [00:01<00:00, 14.70ba/s]\u001b[A\n",
            "\n",
            "#2:  50% 14/28 [00:01<00:00, 14.70ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  46% 13/28 [00:00<00:01, 13.97ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#1:  96% 27/28 [00:01<00:00, 14.97ba/s]\u001b[A\n",
            "\n",
            "#1: 100% 28/28 [00:01<00:00, 14.49ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3:  54% 15/28 [00:01<00:00, 14.28ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  64% 18/28 [00:01<00:00, 15.15ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  61% 17/28 [00:01<00:00, 14.25ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  71% 20/28 [00:01<00:00, 14.27ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  71% 20/28 [00:01<00:00, 15.35ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  79% 22/28 [00:01<00:00, 14.11ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  79% 22/28 [00:01<00:00, 16.16ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  86% 24/28 [00:01<00:00, 13.56ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  86% 24/28 [00:01<00:00, 16.23ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  93% 26/28 [00:01<00:00, 13.09ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#2: 100% 28/28 [00:01<00:00, 14.15ba/s]\n",
            "#3: 100% 28/28 [00:01<00:00, 15.04ba/s]\n",
            "#0:   0% 0/2 [00:00<?, ?ba/s]\n",
            "#0: 100% 2/2 [00:00<00:00, 15.63ba/s]\n",
            "\n",
            "#1: 100% 2/2 [00:00<00:00, 14.96ba/s]\n",
            "\n",
            "\n",
            "#2:   0% 0/2 [00:00<?, ?ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:   0% 0/2 [00:00<?, ?ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2: 100% 2/2 [00:00<00:00, 18.44ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3: 100% 2/2 [00:00<00:00, 15.11ba/s]\n",
            "#0:   0% 0/4 [00:00<?, ?ba/s]\n",
            "#0:  50% 2/4 [00:00<00:00, 18.96ba/s]\n",
            "#0: 100% 4/4 [00:00<00:00, 26.09ba/s]\n",
            "#1: 100% 4/4 [00:00<00:00, 20.32ba/s]\n",
            "\n",
            "\n",
            "#2:   0% 0/4 [00:00<?, ?ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:   0% 0/4 [00:00<?, ?ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  25% 1/4 [00:00<00:00,  8.27ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  50% 2/4 [00:00<00:00, 14.24ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2: 100% 4/4 [00:00<00:00, 17.25ba/s]\n",
            "#3: 100% 4/4 [00:00<00:00, 19.08ba/s]\n",
            "You're using a XLMRobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
            "Batch no. 1\n",
            "Batch no. 2\n",
            "Batch no. 3\n",
            "Batch no. 4\n",
            "Batch no. 5\n",
            "Batch no. 6\n",
            "Batch no. 7\n",
            "Batch no. 8\n",
            "Batch no. 9\n",
            "Batch no. 10\n",
            "Batch no. 11\n",
            "Batch no. 12\n",
            "Batch no. 13\n",
            "Batch no. 14\n",
            "Batch no. 15\n",
            "Batch no. 16\n",
            "Batch no. 17\n",
            "Batch no. 18\n",
            "Batch no. 19\n",
            "Batch no. 20\n",
            "Batch no. 21\n",
            "Batch no. 22\n",
            "Batch no. 23\n",
            "Batch no. 24\n",
            "Batch no. 25\n",
            "Batch no. 26\n",
            "Batch no. 27\n",
            "Batch no. 28\n",
            "Batch no. 29\n",
            "Batch no. 30\n",
            "Batch no. 31\n",
            "Batch no. 32\n",
            "Batch no. 33\n",
            "Batch no. 34\n",
            "Batch no. 35\n",
            "Batch no. 36\n",
            "Batch no. 37\n",
            "Batch no. 38\n",
            "Batch no. 39\n",
            "Batch no. 40\n",
            "Batch no. 41\n",
            "Batch no. 42\n",
            "Batch no. 43\n",
            "Batch no. 44\n",
            "Batch no. 45\n",
            "Batch no. 46\n",
            "Batch no. 47\n",
            "Batch no. 48\n",
            "Batch no. 49\n",
            "Batch no. 50\n",
            "Batch no. 51\n",
            "Batch no. 52\n",
            "Batch no. 53\n",
            "Batch no. 54\n",
            "Batch no. 55\n",
            "Batch no. 56\n",
            "Batch no. 57\n",
            "Batch no. 58\n",
            "Batch no. 59\n",
            "Batch no. 60\n",
            "Batch no. 61\n",
            "Batch no. 62\n",
            "Batch no. 63\n",
            "Batch no. 64\n",
            "Batch no. 65\n",
            "Batch no. 66\n",
            "Batch no. 67\n",
            "Batch no. 68\n",
            "Batch no. 69\n",
            "Batch no. 70\n",
            "Batch no. 71\n",
            "Batch no. 72\n",
            "Batch no. 73\n",
            "Batch no. 74\n",
            "Batch no. 75\n",
            "Batch no. 76\n",
            "Batch no. 77\n",
            "Batch no. 78\n",
            "Batch no. 79\n",
            "Batch no. 80\n",
            "Batch no. 81\n",
            "Batch no. 82\n",
            "Batch no. 83\n",
            "Batch no. 84\n",
            "Batch no. 85\n",
            "Batch no. 86\n",
            "Batch no. 87\n",
            "Batch no. 88\n",
            "Batch no. 89\n",
            "Batch no. 90\n",
            "Batch no. 91\n",
            "Batch no. 92\n",
            "Batch no. 93\n",
            "Batch no. 94\n",
            "Batch no. 95\n",
            "Batch no. 96\n",
            "Batch no. 97\n",
            "Batch no. 98\n",
            "Batch no. 99\n",
            "Batch no. 100\n",
            "Batch no. 101\n",
            "Batch no. 102\n",
            "Batch no. 103\n",
            "Batch no. 104\n",
            "Batch no. 105\n",
            "Batch no. 106\n",
            "Batch no. 107\n",
            "Batch no. 108\n",
            "Batch no. 109\n",
            "Batch no. 110\n",
            "Batch no. 1\n",
            "Batch no. 2\n",
            "Batch no. 3\n",
            "Batch no. 4\n",
            "Batch no. 5\n",
            "Batch no. 6\n",
            "Batch no. 7\n",
            "Batch no. 8\n",
            "Batch no. 9\n",
            "Batch no. 10\n",
            "Batch no. 11\n",
            "Batch no. 12\n",
            "Batch no. 13\n",
            "Batch no. 14\n",
            "Batch no. 15\n",
            "Batch no. 16\n",
            "Batch no. 1\n",
            "Batch no. 2\n",
            "Batch no. 3\n",
            "Batch no. 4\n",
            "Batch no. 5\n",
            "Batch no. 6\n",
            "Batch no. 7\n",
            "Batch no. 8\n",
            "Saved feature-extraction file to /gdrive/My Drive/projects/bambas/feature_extraction/1701610860_sentence-transformers-stsb-xlm-r-multilingual_train_features.json\n",
            "Saved feature-extraction file to /gdrive/My Drive/projects/bambas/feature_extraction/1701610860_sentence-transformers-stsb-xlm-r-multilingual_test_features.json\n",
            "Saved feature-extraction file to /gdrive/My Drive/projects/bambas/feature_extraction/1701610860_sentence-transformers-stsb-xlm-r-multilingual_dev_features.json\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!PYTHONPATH=\"/gdrive/My Drive/projects/bambas\" python -m src.classification \\\n",
        "  --classifier \"HiMLP\" \\\n",
        "  --dataset semeval2024 \\\n",
        "  --train_features \"/gdrive/My Drive/projects/bambas/feature_extraction/1701610860_sentence-transformers-stsb-xlm-r-multilingual_train_features.json\" \\\n",
        "  --test_features \"/gdrive/My Drive/projects/bambas/feature_extraction/1701610860_sentence-transformers-stsb-xlm-r-multilingual_test_features.json\" \\\n",
        "  --dev_features \"/gdrive/My Drive/projects/bambas/feature_extraction/1701610860_sentence-transformers-stsb-xlm-r-multilingual_dev_features.json\" \\\n",
        "  --seed 1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gBvSdlA32q2d",
        "outputId": "70dd964e-e13f-4cc9-8aa1-33e61f80131c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Arguments: Namespace(classifier='HiMLP', dataset='semeval2024', train_features='/gdrive/My Drive/projects/bambas/feature_extraction/1701610860_sentence-transformers-stsb-xlm-r-multilingual_train_features.json', test_features='/gdrive/My Drive/projects/bambas/feature_extraction/1701610860_sentence-transformers-stsb-xlm-r-multilingual_test_features.json', dev_features='/gdrive/My Drive/projects/bambas/feature_extraction/1701610860_sentence-transformers-stsb-xlm-r-multilingual_dev_features.json', max_iter=400, alpha=0.0001, seed=1)\n",
            "Loading features info files\n",
            "Loading dataset files\n",
            "Labels: [['Name calling/Labeling', 'Exaggeration/Minimisation', 'Transfer', 'Logos', 'Smears', 'Bandwagon', 'Distraction', 'Causal Oversimplification', \"Misrepresentation of Someone's Position (Straw Man)\", 'Black-and-white Fallacy/Dictatorship', 'Slogans', 'Flag-waving', 'Obfuscation, Intentional vagueness, Confusion', 'Doubt', 'Whataboutism', 'Ethos', 'Justification', 'Thought-terminating cliché', 'Glittering generalities (Virtue)', 'Presenting Irrelevant Data (Red Herring)', 'Loaded Language', 'Pathos', 'Appeal to authority', 'Appeal to fear/prejudice', 'Reasoning', 'Simplification', 'Ad Hominem', 'Repetition', 'Reductio ad hitlerum', 'Appeal to (Strong) Emotions']]\n",
            "No. of labels in dataset (includes non-labeled samples): 30\n",
            "Loading features array files\n",
            "Iteration 1, loss = 1.05305687\n",
            "Validation score: 0.519016\n",
            "Iteration 2, loss = 1.03952215\n",
            "Validation score: 0.500373\n",
            "Iteration 3, loss = 1.03693829\n",
            "Validation score: 0.499627\n",
            "Iteration 4, loss = 1.03437400\n",
            "Validation score: 0.505593\n",
            "Iteration 5, loss = 1.03535070\n",
            "Validation score: 0.505593\n",
            "Iteration 6, loss = 1.03551778\n",
            "Validation score: 0.501119\n",
            "Iteration 7, loss = 1.02901160\n",
            "Validation score: 0.501119\n",
            "Iteration 8, loss = 1.03286013\n",
            "Validation score: 0.519761\n",
            "Iteration 9, loss = 1.03040400\n",
            "Validation score: 0.503356\n",
            "Iteration 10, loss = 1.02937178\n",
            "Validation score: 0.505593\n",
            "Iteration 11, loss = 1.02520426\n",
            "Validation score: 0.517524\n",
            "Iteration 12, loss = 1.02340775\n",
            "Validation score: 0.519016\n",
            "Iteration 13, loss = 1.02675235\n",
            "Validation score: 0.434004\n",
            "Iteration 14, loss = 1.02572381\n",
            "Validation score: 0.519016\n",
            "Iteration 15, loss = 1.02380495\n",
            "Validation score: 0.504847\n",
            "Iteration 16, loss = 1.02419126\n",
            "Validation score: 0.519016\n",
            "Iteration 17, loss = 1.02465339\n",
            "Validation score: 0.519761\n",
            "Iteration 18, loss = 1.02095074\n",
            "Validation score: 0.519761\n",
            "Iteration 19, loss = 1.02223584\n",
            "Validation score: 0.518270\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 0.99000616\n",
            "Validation score: 0.600418\n",
            "Iteration 2, loss = 0.90640824\n",
            "Validation score: 0.573222\n",
            "Iteration 3, loss = 0.90521466\n",
            "Validation score: 0.569038\n",
            "Iteration 4, loss = 0.90659513\n",
            "Validation score: 0.353556\n",
            "Iteration 5, loss = 0.91489696\n",
            "Validation score: 0.569038\n",
            "Iteration 6, loss = 0.91999102\n",
            "Validation score: 0.596234\n",
            "Iteration 7, loss = 0.91621940\n",
            "Validation score: 0.581590\n",
            "Iteration 8, loss = 0.90094203\n",
            "Validation score: 0.612971\n",
            "Iteration 9, loss = 0.90753126\n",
            "Validation score: 0.569038\n",
            "Iteration 10, loss = 0.90521684\n",
            "Validation score: 0.596234\n",
            "Iteration 11, loss = 0.91109886\n",
            "Validation score: 0.596234\n",
            "Iteration 12, loss = 0.90656990\n",
            "Validation score: 0.596234\n",
            "Iteration 13, loss = 0.91071154\n",
            "Validation score: 0.612971\n",
            "Iteration 14, loss = 0.90246854\n",
            "Validation score: 0.608787\n",
            "Iteration 15, loss = 0.90108227\n",
            "Validation score: 0.608787\n",
            "Iteration 16, loss = 0.90185400\n",
            "Validation score: 0.585774\n",
            "Iteration 17, loss = 0.90003347\n",
            "Validation score: 0.585774\n",
            "Iteration 18, loss = 0.90120977\n",
            "Validation score: 0.581590\n",
            "Iteration 19, loss = 0.90062473\n",
            "Validation score: 0.569038\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 0.61151385\n",
            "Validation score: 0.803109\n",
            "Iteration 2, loss = 0.51600952\n",
            "Validation score: 0.803109\n",
            "Iteration 3, loss = 0.51042903\n",
            "Validation score: 0.803109\n",
            "Iteration 4, loss = 0.50251505\n",
            "Validation score: 0.803109\n",
            "Iteration 5, loss = 0.50602815\n",
            "Validation score: 0.803109\n",
            "Iteration 6, loss = 0.50138741\n",
            "Validation score: 0.803109\n",
            "Iteration 7, loss = 0.50024156\n",
            "Validation score: 0.803109\n",
            "Iteration 8, loss = 0.49396926\n",
            "Validation score: 0.803109\n",
            "Iteration 9, loss = 0.49324374\n",
            "Validation score: 0.803109\n",
            "Iteration 10, loss = 0.49496462\n",
            "Validation score: 0.803109\n",
            "Iteration 11, loss = 0.49433253\n",
            "Validation score: 0.803109\n",
            "Iteration 12, loss = 0.49423700\n",
            "Validation score: 0.803109\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.14119042\n",
            "Validation score: 0.554839\n",
            "Iteration 2, loss = 0.98099666\n",
            "Validation score: 0.554839\n",
            "Iteration 3, loss = 0.96289128\n",
            "Validation score: 0.554839\n",
            "Iteration 4, loss = 0.96114065\n",
            "Validation score: 0.554839\n",
            "Iteration 5, loss = 0.96633550\n",
            "Validation score: 0.554839\n",
            "Iteration 6, loss = 0.95155252\n",
            "Validation score: 0.554839\n",
            "Iteration 7, loss = 0.94926033\n",
            "Validation score: 0.554839\n",
            "Iteration 8, loss = 0.94871260\n",
            "Validation score: 0.554839\n",
            "Iteration 9, loss = 0.94899940\n",
            "Validation score: 0.554839\n",
            "Iteration 10, loss = 0.95242324\n",
            "Validation score: 0.554839\n",
            "Iteration 11, loss = 0.94781583\n",
            "Validation score: 0.554839\n",
            "Iteration 12, loss = 0.95371511\n",
            "Validation score: 0.554839\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.62583204\n",
            "Validation score: 0.578947\n",
            "Iteration 2, loss = 0.92110845\n",
            "Validation score: 0.578947\n",
            "Iteration 3, loss = 1.06544375\n",
            "Validation score: 0.578947\n",
            "Iteration 4, loss = 1.02786243\n",
            "Validation score: 0.578947\n",
            "Iteration 5, loss = 0.95013322\n",
            "Validation score: 0.578947\n",
            "Iteration 6, loss = 0.89375928\n",
            "Validation score: 0.578947\n",
            "Iteration 7, loss = 0.84770673\n",
            "Validation score: 0.578947\n",
            "Iteration 8, loss = 0.85649453\n",
            "Validation score: 0.578947\n",
            "Iteration 9, loss = 0.87589632\n",
            "Validation score: 0.578947\n",
            "Iteration 10, loss = 0.86253821\n",
            "Validation score: 0.578947\n",
            "Iteration 11, loss = 0.85131306\n",
            "Validation score: 0.578947\n",
            "Iteration 12, loss = 0.84475558\n",
            "Validation score: 0.578947\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.43585443\n",
            "Validation score: 0.454545\n",
            "Iteration 2, loss = 1.37049438\n",
            "Validation score: 0.454545\n",
            "Iteration 3, loss = 1.37135534\n",
            "Validation score: 0.454545\n",
            "Iteration 4, loss = 1.36373632\n",
            "Validation score: 0.454545\n",
            "Iteration 5, loss = 1.36077832\n",
            "Validation score: 0.442688\n",
            "Iteration 6, loss = 1.36206322\n",
            "Validation score: 0.454545\n",
            "Iteration 7, loss = 1.36898455\n",
            "Validation score: 0.454545\n",
            "Iteration 8, loss = 1.37107766\n",
            "Validation score: 0.442688\n",
            "Iteration 9, loss = 1.36190790\n",
            "Validation score: 0.454545\n",
            "Iteration 10, loss = 1.36714910\n",
            "Validation score: 0.454545\n",
            "Iteration 11, loss = 1.37719283\n",
            "Validation score: 0.454545\n",
            "Iteration 12, loss = 1.36783712\n",
            "Validation score: 0.442688\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 0.82685887\n",
            "Validation score: 0.729537\n",
            "Iteration 2, loss = 0.72982890\n",
            "Validation score: 0.729537\n",
            "Iteration 3, loss = 0.73196665\n",
            "Validation score: 0.729537\n",
            "Iteration 4, loss = 0.72867170\n",
            "Validation score: 0.729537\n",
            "Iteration 5, loss = 0.72660570\n",
            "Validation score: 0.729537\n",
            "Iteration 6, loss = 0.72772451\n",
            "Validation score: 0.729537\n",
            "Iteration 7, loss = 0.72912620\n",
            "Validation score: 0.729537\n",
            "Iteration 8, loss = 0.73482867\n",
            "Validation score: 0.729537\n",
            "Iteration 9, loss = 0.72902627\n",
            "Validation score: 0.729537\n",
            "Iteration 10, loss = 0.72874913\n",
            "Validation score: 0.729537\n",
            "Iteration 11, loss = 0.72630520\n",
            "Validation score: 0.729537\n",
            "Iteration 12, loss = 0.72684258\n",
            "Validation score: 0.729537\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.14579918\n",
            "Validation score: 0.502392\n",
            "Iteration 2, loss = 1.08602859\n",
            "Validation score: 0.497608\n",
            "Iteration 3, loss = 1.08632069\n",
            "Validation score: 0.502392\n",
            "Iteration 4, loss = 1.08422199\n",
            "Validation score: 0.502392\n",
            "Iteration 5, loss = 1.08396475\n",
            "Validation score: 0.437799\n",
            "Iteration 6, loss = 1.07961845\n",
            "Validation score: 0.442584\n",
            "Iteration 7, loss = 1.07674814\n",
            "Validation score: 0.502392\n",
            "Iteration 8, loss = 1.07373828\n",
            "Validation score: 0.437799\n",
            "Iteration 9, loss = 1.09320841\n",
            "Validation score: 0.437799\n",
            "Iteration 10, loss = 1.08417627\n",
            "Validation score: 0.497608\n",
            "Iteration 11, loss = 1.08444988\n",
            "Validation score: 0.497608\n",
            "Iteration 12, loss = 1.09195801\n",
            "Validation score: 0.502392\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.18023346\n",
            "Validation score: 0.605960\n",
            "Iteration 2, loss = 1.11092580\n",
            "Validation score: 0.605960\n",
            "Iteration 3, loss = 1.12016365\n",
            "Validation score: 0.605960\n",
            "Iteration 4, loss = 1.10715507\n",
            "Validation score: 0.605960\n",
            "Iteration 5, loss = 1.11912825\n",
            "Validation score: 0.605960\n",
            "Iteration 6, loss = 1.11293351\n",
            "Validation score: 0.605960\n",
            "Iteration 7, loss = 1.11107922\n",
            "Validation score: 0.605960\n",
            "Iteration 8, loss = 1.11597791\n",
            "Validation score: 0.605960\n",
            "Iteration 9, loss = 1.11600426\n",
            "Validation score: 0.605960\n",
            "Iteration 10, loss = 1.11355359\n",
            "Validation score: 0.605960\n",
            "Iteration 11, loss = 1.11093950\n",
            "Validation score: 0.605960\n",
            "Iteration 12, loss = 1.10706666\n",
            "Validation score: 0.605960\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "\n",
            "Validation set:\n",
            "\tPrecision: 0.3456521739130435\n",
            "\tRecall: 0.2816651904340124\n",
            "\tF1: 0.31039531478770127\n",
            "\n",
            "Saving validation set results to /gdrive/My Drive/projects/bambas/classification/results.csv\n",
            "2\n",
            "\n",
            "Predicting for test file\n",
            "Finished successfully. dev_unlabeled predictions saved at /gdrive/My Drive/projects/bambas/classification/dev_unlabeled_predictions.json.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Class with fine-tuning on Semeval2024"
      ],
      "metadata": {
        "id": "1855wAVMBUyY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!PYTHONPATH=\"/gdrive/My Drive/projects/bambas\" python -m src.fine_tuning \\\n",
        "  --model xlm-roberta-base \\\n",
        "  --dataset ptc2019 \\\n",
        "  --fine_tuned_name xlm-roberta-base-ptc2019 \\\n",
        "  --save_model"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Uhi7U-8CBfA5",
        "outputId": "c699bd8a-502f-40ea-8813-350cb84d4b91"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-12-03 17:41:52.370297: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2023-12-03 17:41:52.370373: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2023-12-03 17:41:52.370420: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2023-12-03 17:41:53.612575: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Arguments: Namespace(model='xlm-roberta-base', dataset='ptc2019', fine_tuned_name='xlm-roberta-base-ptc2019', lr=0.0002, weight_decay=0.001, mlm_probability=0.15, save_model=True, push_model_to_hf_hub=False)\n",
            "config.json: 100% 615/615 [00:00<00:00, 2.22MB/s]\n",
            "sentencepiece.bpe.model: 100% 5.07M/5.07M [00:00<00:00, 15.5MB/s]\n",
            "tokenizer.json: 100% 9.10M/9.10M [00:00<00:00, 15.1MB/s]\n",
            "Using device: {'': 0}\n",
            "\n",
            "#0:   0% 0/23 [00:00<?, ?ba/s]\n",
            "\n",
            "#2:   0% 0/23 [00:00<?, ?ba/s]\u001b[A\u001b[A\n",
            "#0:  13% 3/23 [00:00<00:00, 25.31ba/s]\n",
            "\n",
            "#2:  13% 3/23 [00:00<00:00, 28.24ba/s]\u001b[A\u001b[A\n",
            "#0:  35% 8/23 [00:00<00:00, 38.10ba/s]\n",
            "\n",
            "#2:  35% 8/23 [00:00<00:00, 36.65ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:   0% 0/23 [00:00<?, ?ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#0:  52% 12/23 [00:00<00:00, 36.18ba/s]\n",
            "\n",
            "#2:  57% 13/23 [00:00<00:00, 39.82ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:   9% 2/23 [00:00<00:01, 18.30ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#1: 100% 23/23 [00:00<00:00, 48.87ba/s]\n",
            "#0:  78% 18/23 [00:00<00:00, 42.10ba/s]\n",
            "\n",
            "#0: 100% 23/23 [00:00<00:00, 41.28ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3:  39% 9/23 [00:00<00:00, 45.52ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2: 100% 23/23 [00:00<00:00, 38.86ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3: 100% 23/23 [00:00<00:00, 61.21ba/s]\n",
            "#0: 100% 4/4 [00:00<00:00, 40.11ba/s]\n",
            "\n",
            "#1:   0% 0/4 [00:00<?, ?ba/s]\u001b[A\n",
            "#1: 100% 4/4 [00:00<00:00, 33.34ba/s]\n",
            "\n",
            "\n",
            "#2:   0% 0/4 [00:00<?, ?ba/s]\u001b[A\u001b[A\n",
            "\n",
            "#2: 100% 4/4 [00:00<00:00, 35.91ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3: 100% 4/4 [00:00<00:00, 54.34ba/s]\n",
            "model.safetensors: 100% 1.12G/1.12G [00:06<00:00, 164MB/s]\n",
            "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaForMaskedLM: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "- This IS expected if you are initializing XLMRobertaForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing XLMRobertaForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of XLMRobertaForMaskedLM were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['lm_head.decoder.weight', 'lm_head.decoder.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n",
            "  0% 0/2130 [00:00<?, ?it/s]You're using a XLMRobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
            "{'loss': 4.7194, 'learning_rate': 0.00015305164319248828, 'epoch': 0.7}\n",
            " 33% 710/2130 [07:02<10:04,  2.35it/s]\n",
            "  0% 0/99 [00:00<?, ?it/s]\u001b[A\n",
            "  3% 3/99 [00:00<00:04, 22.94it/s]\u001b[A\n",
            "  6% 6/99 [00:00<00:05, 15.51it/s]\u001b[A\n",
            "  8% 8/99 [00:00<00:06, 14.37it/s]\u001b[A\n",
            " 10% 10/99 [00:00<00:06, 13.67it/s]\u001b[A\n",
            " 12% 12/99 [00:00<00:06, 13.39it/s]\u001b[A\n",
            " 14% 14/99 [00:01<00:06, 13.13it/s]\u001b[A\n",
            " 16% 16/99 [00:01<00:06, 12.98it/s]\u001b[A\n",
            " 18% 18/99 [00:01<00:06, 13.04it/s]\u001b[A\n",
            " 20% 20/99 [00:01<00:06, 12.94it/s]\u001b[A\n",
            " 22% 22/99 [00:01<00:06, 12.77it/s]\u001b[A\n",
            " 24% 24/99 [00:01<00:05, 12.76it/s]\u001b[A\n",
            " 26% 26/99 [00:01<00:05, 13.11it/s]\u001b[A\n",
            " 28% 28/99 [00:02<00:04, 14.20it/s]\u001b[A\n",
            " 30% 30/99 [00:02<00:04, 14.82it/s]\u001b[A\n",
            " 32% 32/99 [00:02<00:04, 15.48it/s]\u001b[A\n",
            " 34% 34/99 [00:02<00:04, 13.97it/s]\u001b[A\n",
            " 36% 36/99 [00:02<00:05, 11.74it/s]\u001b[A\n",
            " 38% 38/99 [00:02<00:05, 10.52it/s]\u001b[A\n",
            " 40% 40/99 [00:03<00:06,  9.80it/s]\u001b[A\n",
            " 42% 42/99 [00:03<00:06,  9.07it/s]\u001b[A\n",
            " 43% 43/99 [00:03<00:06,  8.65it/s]\u001b[A\n",
            " 44% 44/99 [00:03<00:06,  8.31it/s]\u001b[A\n",
            " 45% 45/99 [00:03<00:06,  8.02it/s]\u001b[A\n",
            " 46% 46/99 [00:03<00:06,  7.78it/s]\u001b[A\n",
            " 47% 47/99 [00:04<00:06,  7.51it/s]\u001b[A\n",
            " 48% 48/99 [00:04<00:06,  7.38it/s]\u001b[A\n",
            " 49% 49/99 [00:04<00:06,  7.40it/s]\u001b[A\n",
            " 51% 50/99 [00:04<00:06,  7.35it/s]\u001b[A\n",
            " 52% 51/99 [00:04<00:06,  7.74it/s]\u001b[A\n",
            " 53% 52/99 [00:04<00:05,  7.92it/s]\u001b[A\n",
            " 54% 53/99 [00:04<00:05,  8.11it/s]\u001b[A\n",
            " 55% 54/99 [00:05<00:05,  8.21it/s]\u001b[A\n",
            " 56% 55/99 [00:05<00:05,  8.27it/s]\u001b[A\n",
            " 57% 56/99 [00:05<00:05,  8.44it/s]\u001b[A\n",
            " 58% 57/99 [00:05<00:04,  8.47it/s]\u001b[A\n",
            " 59% 58/99 [00:05<00:04,  8.40it/s]\u001b[A\n",
            " 60% 59/99 [00:05<00:05,  7.16it/s]\u001b[A\n",
            " 61% 60/99 [00:05<00:05,  6.52it/s]\u001b[A\n",
            " 62% 61/99 [00:06<00:06,  6.13it/s]\u001b[A\n",
            " 63% 62/99 [00:06<00:06,  5.85it/s]\u001b[A\n",
            " 64% 63/99 [00:06<00:06,  5.74it/s]\u001b[A\n",
            " 65% 64/99 [00:06<00:06,  5.53it/s]\u001b[A\n",
            " 66% 65/99 [00:06<00:06,  5.53it/s]\u001b[A\n",
            " 67% 66/99 [00:06<00:05,  5.50it/s]\u001b[A\n",
            " 68% 67/99 [00:07<00:05,  5.48it/s]\u001b[A\n",
            " 69% 68/99 [00:07<00:04,  6.27it/s]\u001b[A\n",
            " 70% 69/99 [00:07<00:04,  7.05it/s]\u001b[A\n",
            " 71% 70/99 [00:07<00:03,  7.71it/s]\u001b[A\n",
            " 72% 71/99 [00:07<00:03,  8.23it/s]\u001b[A\n",
            " 73% 72/99 [00:07<00:03,  8.49it/s]\u001b[A\n",
            " 75% 74/99 [00:07<00:02,  9.13it/s]\u001b[A\n",
            " 76% 75/99 [00:07<00:02,  9.15it/s]\u001b[A\n",
            " 78% 77/99 [00:08<00:02,  9.57it/s]\u001b[A\n",
            " 80% 79/99 [00:08<00:02,  9.78it/s]\u001b[A\n",
            " 82% 81/99 [00:08<00:01,  9.88it/s]\u001b[A\n",
            " 84% 83/99 [00:08<00:01, 10.11it/s]\u001b[A\n",
            " 86% 85/99 [00:08<00:01, 10.71it/s]\u001b[A\n",
            " 88% 87/99 [00:09<00:01, 11.16it/s]\u001b[A\n",
            " 90% 89/99 [00:09<00:00, 11.59it/s]\u001b[A\n",
            " 92% 91/99 [00:09<00:00, 11.49it/s]\u001b[A\n",
            " 94% 93/99 [00:09<00:00, 11.16it/s]\u001b[A\n",
            " 96% 95/99 [00:09<00:00, 10.80it/s]\u001b[A\n",
            " 98% 97/99 [00:10<00:00, 10.58it/s]\u001b[A\n",
            "                                      \n",
            "\u001b[A{'eval_loss': 4.0810041427612305, 'eval_runtime': 10.2617, 'eval_samples_per_second': 76.791, 'eval_steps_per_second': 9.648, 'epoch': 1.0}\n",
            " 33% 710/2130 [07:12<10:04,  2.35it/s]\n",
            "100% 99/99 [00:10<00:00, 10.44it/s]\u001b[A\n",
            "{'loss': 4.1072, 'learning_rate': 0.00010610328638497653, 'epoch': 1.41}\n",
            " 67% 1420/2130 [14:22<05:52,  2.02it/s]\n",
            "  0% 0/99 [00:00<?, ?it/s]\u001b[A\n",
            "  3% 3/99 [00:00<00:04, 19.33it/s]\u001b[A\n",
            "  5% 5/99 [00:00<00:05, 15.81it/s]\u001b[A\n",
            "  7% 7/99 [00:00<00:06, 14.35it/s]\u001b[A\n",
            "  9% 9/99 [00:00<00:06, 13.43it/s]\u001b[A\n",
            " 11% 11/99 [00:00<00:06, 13.04it/s]\u001b[A\n",
            " 13% 13/99 [00:00<00:06, 12.85it/s]\u001b[A\n",
            " 15% 15/99 [00:01<00:06, 12.71it/s]\u001b[A\n",
            " 17% 17/99 [00:01<00:06, 12.70it/s]\u001b[A\n",
            " 19% 19/99 [00:01<00:06, 12.78it/s]\u001b[A\n",
            " 21% 21/99 [00:01<00:06, 12.84it/s]\u001b[A\n",
            " 23% 23/99 [00:01<00:05, 12.68it/s]\u001b[A\n",
            " 25% 25/99 [00:01<00:05, 12.72it/s]\u001b[A\n",
            " 27% 27/99 [00:02<00:05, 13.58it/s]\u001b[A\n",
            " 29% 29/99 [00:02<00:04, 14.33it/s]\u001b[A\n",
            " 31% 31/99 [00:02<00:04, 14.92it/s]\u001b[A\n",
            " 33% 33/99 [00:02<00:04, 15.33it/s]\u001b[A\n",
            " 35% 35/99 [00:02<00:05, 12.37it/s]\u001b[A\n",
            " 37% 37/99 [00:02<00:05, 10.71it/s]\u001b[A\n",
            " 39% 39/99 [00:03<00:06,  9.96it/s]\u001b[A\n",
            " 41% 41/99 [00:03<00:06,  9.44it/s]\u001b[A\n",
            " 43% 43/99 [00:03<00:06,  8.57it/s]\u001b[A\n",
            " 44% 44/99 [00:03<00:06,  8.26it/s]\u001b[A\n",
            " 45% 45/99 [00:03<00:06,  7.92it/s]\u001b[A\n",
            " 46% 46/99 [00:04<00:06,  7.72it/s]\u001b[A\n",
            " 47% 47/99 [00:04<00:06,  7.49it/s]\u001b[A\n",
            " 48% 48/99 [00:04<00:06,  7.43it/s]\u001b[A\n",
            " 49% 49/99 [00:04<00:06,  7.36it/s]\u001b[A\n",
            " 51% 50/99 [00:04<00:06,  7.32it/s]\u001b[A\n",
            " 52% 51/99 [00:04<00:06,  7.61it/s]\u001b[A\n",
            " 53% 52/99 [00:04<00:06,  7.81it/s]\u001b[A\n",
            " 54% 53/99 [00:04<00:05,  7.95it/s]\u001b[A\n",
            " 55% 54/99 [00:05<00:05,  8.11it/s]\u001b[A\n",
            " 56% 55/99 [00:05<00:05,  8.21it/s]\u001b[A\n",
            " 57% 56/99 [00:05<00:05,  8.33it/s]\u001b[A\n",
            " 58% 57/99 [00:05<00:05,  8.40it/s]\u001b[A\n",
            " 59% 58/99 [00:05<00:04,  8.42it/s]\u001b[A\n",
            " 60% 59/99 [00:05<00:05,  7.12it/s]\u001b[A\n",
            " 61% 60/99 [00:05<00:06,  6.37it/s]\u001b[A\n",
            " 62% 61/99 [00:06<00:06,  6.07it/s]\u001b[A\n",
            " 63% 62/99 [00:06<00:06,  5.81it/s]\u001b[A\n",
            " 64% 63/99 [00:06<00:06,  5.67it/s]\u001b[A\n",
            " 65% 64/99 [00:06<00:06,  5.58it/s]\u001b[A\n",
            " 66% 65/99 [00:06<00:06,  5.51it/s]\u001b[A\n",
            " 67% 66/99 [00:07<00:06,  5.45it/s]\u001b[A\n",
            " 68% 67/99 [00:07<00:05,  5.42it/s]\u001b[A\n",
            " 69% 68/99 [00:07<00:04,  6.23it/s]\u001b[A\n",
            " 70% 69/99 [00:07<00:04,  7.01it/s]\u001b[A\n",
            " 71% 70/99 [00:07<00:03,  7.68it/s]\u001b[A\n",
            " 72% 71/99 [00:07<00:03,  8.19it/s]\u001b[A\n",
            " 74% 73/99 [00:07<00:02,  8.86it/s]\u001b[A\n",
            " 75% 74/99 [00:07<00:02,  9.04it/s]\u001b[A\n",
            " 76% 75/99 [00:08<00:02,  9.26it/s]\u001b[A\n",
            " 77% 76/99 [00:08<00:02,  9.39it/s]\u001b[A\n",
            " 79% 78/99 [00:08<00:02,  9.74it/s]\u001b[A\n",
            " 80% 79/99 [00:08<00:02,  9.80it/s]\u001b[A\n",
            " 82% 81/99 [00:08<00:01,  9.91it/s]\u001b[A\n",
            " 84% 83/99 [00:08<00:01, 10.03it/s]\u001b[A\n",
            " 86% 85/99 [00:09<00:01, 10.75it/s]\u001b[A\n",
            " 88% 87/99 [00:09<00:01, 11.33it/s]\u001b[A\n",
            " 90% 89/99 [00:09<00:00, 11.71it/s]\u001b[A\n",
            " 92% 91/99 [00:09<00:00, 11.60it/s]\u001b[A\n",
            " 94% 93/99 [00:09<00:00, 11.18it/s]\u001b[A\n",
            " 96% 95/99 [00:09<00:00, 10.89it/s]\u001b[A\n",
            " 98% 97/99 [00:10<00:00, 10.65it/s]\u001b[A\n",
            "                                       \n",
            "\u001b[A{'eval_loss': 3.412595272064209, 'eval_runtime': 10.3365, 'eval_samples_per_second': 76.235, 'eval_steps_per_second': 9.578, 'epoch': 2.0}\n",
            " 67% 1420/2130 [14:32<05:52,  2.02it/s]\n",
            "100% 99/99 [00:10<00:00, 10.47it/s]\u001b[A\n",
            "{'loss': 3.4813, 'learning_rate': 5.915492957746479e-05, 'epoch': 2.11}\n",
            "{'loss': 2.9806, 'learning_rate': 1.2206572769953052e-05, 'epoch': 2.82}\n",
            "100% 2130/2130 [22:06<00:00,  2.08it/s]\n",
            "  0% 0/99 [00:00<?, ?it/s]\u001b[A\n",
            "  3% 3/99 [00:00<00:04, 20.54it/s]\u001b[A\n",
            "  6% 6/99 [00:00<00:06, 15.05it/s]\u001b[A\n",
            "  8% 8/99 [00:00<00:06, 14.00it/s]\u001b[A\n",
            " 10% 10/99 [00:00<00:06, 13.24it/s]\u001b[A\n",
            " 12% 12/99 [00:00<00:06, 12.99it/s]\u001b[A\n",
            " 14% 14/99 [00:01<00:06, 12.83it/s]\u001b[A\n",
            " 16% 16/99 [00:01<00:06, 12.75it/s]\u001b[A\n",
            " 18% 18/99 [00:01<00:06, 12.77it/s]\u001b[A\n",
            " 20% 20/99 [00:01<00:06, 12.88it/s]\u001b[A\n",
            " 22% 22/99 [00:01<00:06, 12.72it/s]\u001b[A\n",
            " 24% 24/99 [00:01<00:05, 12.75it/s]\u001b[A\n",
            " 26% 26/99 [00:01<00:05, 13.15it/s]\u001b[A\n",
            " 28% 28/99 [00:02<00:05, 14.09it/s]\u001b[A\n",
            " 30% 30/99 [00:02<00:04, 14.78it/s]\u001b[A\n",
            " 32% 32/99 [00:02<00:04, 15.23it/s]\u001b[A\n",
            " 34% 34/99 [00:02<00:04, 13.67it/s]\u001b[A\n",
            " 36% 36/99 [00:02<00:05, 11.56it/s]\u001b[A\n",
            " 38% 38/99 [00:02<00:05, 10.37it/s]\u001b[A\n",
            " 40% 40/99 [00:03<00:06,  9.73it/s]\u001b[A\n",
            " 42% 42/99 [00:03<00:06,  9.02it/s]\u001b[A\n",
            " 43% 43/99 [00:03<00:06,  8.62it/s]\u001b[A\n",
            " 44% 44/99 [00:03<00:06,  8.23it/s]\u001b[A\n",
            " 45% 45/99 [00:03<00:06,  7.95it/s]\u001b[A\n",
            " 46% 46/99 [00:04<00:06,  7.72it/s]\u001b[A\n",
            " 47% 47/99 [00:04<00:06,  7.52it/s]\u001b[A\n",
            " 48% 48/99 [00:04<00:06,  7.39it/s]\u001b[A\n",
            " 49% 49/99 [00:04<00:06,  7.29it/s]\u001b[A\n",
            " 51% 50/99 [00:04<00:06,  7.28it/s]\u001b[A\n",
            " 52% 51/99 [00:04<00:06,  7.62it/s]\u001b[A\n",
            " 53% 52/99 [00:04<00:05,  7.90it/s]\u001b[A\n",
            " 54% 53/99 [00:04<00:05,  8.07it/s]\u001b[A\n",
            " 55% 54/99 [00:05<00:05,  8.24it/s]\u001b[A\n",
            " 56% 55/99 [00:05<00:05,  8.37it/s]\u001b[A\n",
            " 57% 56/99 [00:05<00:05,  8.40it/s]\u001b[A\n",
            " 58% 57/99 [00:05<00:04,  8.48it/s]\u001b[A\n",
            " 59% 58/99 [00:05<00:04,  8.44it/s]\u001b[A\n",
            " 60% 59/99 [00:05<00:05,  7.16it/s]\u001b[A\n",
            " 61% 60/99 [00:05<00:05,  6.52it/s]\u001b[A\n",
            " 62% 61/99 [00:06<00:06,  6.13it/s]\u001b[A\n",
            " 63% 62/99 [00:06<00:06,  5.90it/s]\u001b[A\n",
            " 64% 63/99 [00:06<00:06,  5.74it/s]\u001b[A\n",
            " 65% 64/99 [00:06<00:06,  5.64it/s]\u001b[A\n",
            " 66% 65/99 [00:06<00:06,  5.57it/s]\u001b[A\n",
            " 67% 66/99 [00:07<00:05,  5.50it/s]\u001b[A\n",
            " 68% 67/99 [00:07<00:05,  5.51it/s]\u001b[A\n",
            " 69% 68/99 [00:07<00:04,  6.27it/s]\u001b[A\n",
            " 71% 70/99 [00:07<00:03,  7.60it/s]\u001b[A\n",
            " 72% 71/99 [00:07<00:03,  8.06it/s]\u001b[A\n",
            " 73% 72/99 [00:07<00:03,  8.48it/s]\u001b[A\n",
            " 74% 73/99 [00:07<00:02,  8.78it/s]\u001b[A\n",
            " 75% 74/99 [00:07<00:02,  9.07it/s]\u001b[A\n",
            " 76% 75/99 [00:08<00:02,  9.25it/s]\u001b[A\n",
            " 78% 77/99 [00:08<00:02,  9.63it/s]\u001b[A\n",
            " 80% 79/99 [00:08<00:02,  9.81it/s]\u001b[A\n",
            " 82% 81/99 [00:08<00:01,  9.98it/s]\u001b[A\n",
            " 84% 83/99 [00:08<00:01, 10.03it/s]\u001b[A\n",
            " 86% 85/99 [00:08<00:01, 10.76it/s]\u001b[A\n",
            " 88% 87/99 [00:09<00:01, 11.28it/s]\u001b[A\n",
            " 90% 89/99 [00:09<00:00, 11.68it/s]\u001b[A\n",
            " 92% 91/99 [00:09<00:00, 11.68it/s]\u001b[A\n",
            " 94% 93/99 [00:09<00:00, 11.24it/s]\u001b[A\n",
            " 96% 95/99 [00:09<00:00, 10.90it/s]\u001b[A\n",
            " 98% 97/99 [00:10<00:00, 10.74it/s]\u001b[A\n",
            "                                       \n",
            "\u001b[A{'eval_loss': 3.080418348312378, 'eval_runtime': 10.2575, 'eval_samples_per_second': 76.822, 'eval_steps_per_second': 9.651, 'epoch': 3.0}\n",
            "100% 2130/2130 [22:17<00:00,  2.08it/s]\n",
            "100% 99/99 [00:10<00:00, 10.66it/s]\u001b[A\n",
            "{'train_runtime': 1337.1282, 'train_samples_per_second': 12.73, 'train_steps_per_second': 1.593, 'train_loss': 3.759982872904746, 'epoch': 3.0}\n",
            "100% 2130/2130 [22:17<00:00,  1.59it/s]\n",
            "100% 99/99 [00:10<00:00,  9.65it/s]\n",
            "(dev_set) Perplexity: 19.34\n",
            "(dev_set) Evaluation results: {\n",
            "    \"eval_loss\": 2.9624135494232178,\n",
            "    \"eval_runtime\": 10.2703,\n",
            "    \"eval_samples_per_second\": 76.726,\n",
            "    \"eval_steps_per_second\": 9.639,\n",
            "    \"epoch\": 3.0\n",
            "}\n",
            "Saving fine-tuned model to /gdrive/My Drive/projects/bambas/fine_tuning/xlm-roberta-base-ptc2019\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Class with CLS extraction"
      ],
      "metadata": {
        "id": "qIBfXLPkr6U9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!PYTHONPATH=\"/gdrive/My Drive/projects/bambas\" python -m src.feature_extraction \\\n",
        "  --model \"/gdrive/My Drive/projects/bambas/fine_tuning/xlm-roberta-base-ptc2019/\" \\\n",
        "  --dataset semeval2024 \\\n",
        "  --extraction_method cls"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zELfb0lyBfFW",
        "outputId": "8f946c46-3657-45db-cf49-51ca1cd729a0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-12-03 19:41:24.633854: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2023-12-03 19:41:24.633907: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2023-12-03 19:41:24.633958: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2023-12-03 19:41:25.605522: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Arguments: Namespace(dataset='semeval2024', model='/gdrive/My Drive/projects/bambas/fine_tuning/xlm-roberta-base-ptc2019/', extraction_method='cls', layers=[], agg_method=None)\n",
            "Using device: {'': 0}\n",
            "Some weights of XLMRobertaModel were not initialized from the model checkpoint at /gdrive/My Drive/projects/bambas/fine_tuning/xlm-roberta-base-ptc2019/ and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "#0:   4% 1/28 [00:00<00:04,  5.65ba/s]\n",
            "#0:  11% 3/28 [00:00<00:02,  9.05ba/s]\n",
            "#0:  18% 5/28 [00:00<00:02, 10.54ba/s]\n",
            "#0:  25% 7/28 [00:00<00:01, 11.35ba/s]\n",
            "#1:  18% 5/28 [00:00<00:02, 11.25ba/s]\u001b[A\n",
            "\n",
            "#0:  32% 9/28 [00:00<00:01, 11.84ba/s]\n",
            "#1:  25% 7/28 [00:00<00:01, 12.10ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#3:   0% 0/28 [00:00<?, ?ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#0:  39% 11/28 [00:00<00:01, 12.71ba/s]\n",
            "#1:  32% 9/28 [00:00<00:01, 13.20ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#0:  46% 13/28 [00:01<00:01, 13.62ba/s]\n",
            "\n",
            "#2:  11% 3/28 [00:00<00:02, 12.06ba/s]\u001b[A\u001b[A\n",
            "#1:  39% 11/28 [00:00<00:01, 14.26ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#3:  11% 3/28 [00:00<00:01, 12.53ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#0:  54% 15/28 [00:01<00:00, 14.01ba/s]\n",
            "#1:  46% 13/28 [00:01<00:01, 13.94ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#0:  61% 17/28 [00:01<00:00, 14.15ba/s]\n",
            "\n",
            "#2:  25% 7/28 [00:00<00:01, 13.33ba/s]\u001b[A\u001b[A\n",
            "#1:  54% 15/28 [00:01<00:00, 14.80ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#3:  25% 7/28 [00:00<00:01, 15.98ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#0:  68% 19/28 [00:01<00:00, 13.58ba/s]\n",
            "#1:  61% 17/28 [00:01<00:00, 14.48ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#3:  32% 9/28 [00:00<00:01, 15.92ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#1:  68% 19/28 [00:01<00:00, 15.66ba/s]\u001b[A\n",
            "\n",
            "#0:  75% 21/28 [00:01<00:00, 14.22ba/s]\n",
            "\n",
            "\n",
            "#3:  39% 11/28 [00:00<00:01, 13.62ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#0:  86% 24/28 [00:01<00:00, 17.15ba/s]\n",
            "\n",
            "#2:  50% 14/28 [00:00<00:00, 15.72ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  54% 15/28 [00:00<00:00, 20.26ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#0:  93% 26/28 [00:01<00:00, 17.58ba/s]\n",
            "\n",
            "#2:  57% 16/28 [00:01<00:00, 16.43ba/s]\u001b[A\u001b[A\n",
            "#1: 100% 28/28 [00:01<00:00, 16.56ba/s]\n",
            "#0: 100% 28/28 [00:01<00:00, 14.34ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3:  64% 18/28 [00:01<00:00, 20.08ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  64% 18/28 [00:01<00:00, 17.15ba/s]\u001b[A\u001b[A\n",
            "\n",
            "#2:  71% 20/28 [00:01<00:00, 17.00ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  75% 21/28 [00:01<00:00, 20.59ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  82% 23/28 [00:01<00:00, 20.01ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  86% 24/28 [00:01<00:00, 22.61ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3: 100% 28/28 [00:01<00:00, 19.55ba/s]\n",
            "\n",
            "\n",
            "#2: 100% 28/28 [00:01<00:00, 17.76ba/s]\n",
            "\n",
            "#1:   0% 0/2 [00:00<?, ?ba/s]\u001b[A\n",
            "#1: 100% 2/2 [00:00<00:00, 12.51ba/s]\n",
            "#0: 100% 2/2 [00:00<00:00, 16.36ba/s]\n",
            "\n",
            "\n",
            "#2:   0% 0/2 [00:00<?, ?ba/s]\u001b[A\u001b[A\n",
            "\n",
            "#2: 100% 2/2 [00:00<00:00, 16.81ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3: 100% 2/2 [00:00<00:00, 23.17ba/s]\n",
            "#0:  50% 2/4 [00:00<00:00, 15.90ba/s]\n",
            "#0: 100% 4/4 [00:00<00:00, 20.03ba/s]\n",
            "\n",
            "#1: 100% 4/4 [00:00<00:00, 19.08ba/s]\n",
            "\n",
            "\n",
            "#2:   0% 0/4 [00:00<?, ?ba/s]\u001b[A\u001b[A\n",
            "\n",
            "#2:  50% 2/4 [00:00<00:00, 14.23ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:   0% 0/4 [00:00<?, ?ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2: 100% 4/4 [00:00<00:00, 15.06ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3: 100% 4/4 [00:00<00:00, 32.41ba/s]\n",
            "You're using a XLMRobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
            "Batch no. 1\n",
            "Batch no. 2\n",
            "Batch no. 3\n",
            "Batch no. 4\n",
            "Batch no. 5\n",
            "Batch no. 6\n",
            "Batch no. 7\n",
            "Batch no. 8\n",
            "Batch no. 9\n",
            "Batch no. 10\n",
            "Batch no. 11\n",
            "Batch no. 12\n",
            "Batch no. 13\n",
            "Batch no. 14\n",
            "Batch no. 15\n",
            "Batch no. 16\n",
            "Batch no. 17\n",
            "Batch no. 18\n",
            "Batch no. 19\n",
            "Batch no. 20\n",
            "Batch no. 21\n",
            "Batch no. 22\n",
            "Batch no. 23\n",
            "Batch no. 24\n",
            "Batch no. 25\n",
            "Batch no. 26\n",
            "Batch no. 27\n",
            "Batch no. 28\n",
            "Batch no. 29\n",
            "Batch no. 30\n",
            "Batch no. 31\n",
            "Batch no. 32\n",
            "Batch no. 33\n",
            "Batch no. 34\n",
            "Batch no. 35\n",
            "Batch no. 36\n",
            "Batch no. 37\n",
            "Batch no. 38\n",
            "Batch no. 39\n",
            "Batch no. 40\n",
            "Batch no. 41\n",
            "Batch no. 42\n",
            "Batch no. 43\n",
            "Batch no. 44\n",
            "Batch no. 45\n",
            "Batch no. 46\n",
            "Batch no. 47\n",
            "Batch no. 48\n",
            "Batch no. 49\n",
            "Batch no. 50\n",
            "Batch no. 51\n",
            "Batch no. 52\n",
            "Batch no. 53\n",
            "Batch no. 54\n",
            "Batch no. 55\n",
            "Batch no. 56\n",
            "Batch no. 57\n",
            "Batch no. 58\n",
            "Batch no. 59\n",
            "Batch no. 60\n",
            "Batch no. 61\n",
            "Batch no. 62\n",
            "Batch no. 63\n",
            "Batch no. 64\n",
            "Batch no. 65\n",
            "Batch no. 66\n",
            "Batch no. 67\n",
            "Batch no. 68\n",
            "Batch no. 69\n",
            "Batch no. 70\n",
            "Batch no. 71\n",
            "Batch no. 72\n",
            "Batch no. 73\n",
            "Batch no. 74\n",
            "Batch no. 75\n",
            "Batch no. 76\n",
            "Batch no. 77\n",
            "Batch no. 78\n",
            "Batch no. 79\n",
            "Batch no. 80\n",
            "Batch no. 81\n",
            "Batch no. 82\n",
            "Batch no. 83\n",
            "Batch no. 84\n",
            "Batch no. 85\n",
            "Batch no. 86\n",
            "Batch no. 87\n",
            "Batch no. 88\n",
            "Batch no. 89\n",
            "Batch no. 90\n",
            "Batch no. 91\n",
            "Batch no. 92\n",
            "Batch no. 93\n",
            "Batch no. 94\n",
            "Batch no. 95\n",
            "Batch no. 96\n",
            "Batch no. 97\n",
            "Batch no. 98\n",
            "Batch no. 99\n",
            "Batch no. 100\n",
            "Batch no. 101\n",
            "Batch no. 102\n",
            "Batch no. 103\n",
            "Batch no. 104\n",
            "Batch no. 105\n",
            "Batch no. 106\n",
            "Batch no. 107\n",
            "Batch no. 108\n",
            "Batch no. 109\n",
            "Batch no. 110\n",
            "Batch no. 1\n",
            "Batch no. 2\n",
            "Batch no. 3\n",
            "Batch no. 4\n",
            "Batch no. 5\n",
            "Batch no. 6\n",
            "Batch no. 7\n",
            "Batch no. 8\n",
            "Batch no. 9\n",
            "Batch no. 10\n",
            "Batch no. 11\n",
            "Batch no. 12\n",
            "Batch no. 13\n",
            "Batch no. 14\n",
            "Batch no. 15\n",
            "Batch no. 16\n",
            "Batch no. 1\n",
            "Batch no. 2\n",
            "Batch no. 3\n",
            "Batch no. 4\n",
            "Batch no. 5\n",
            "Batch no. 6\n",
            "Batch no. 7\n",
            "Batch no. 8\n",
            "Saved feature-extraction file to /gdrive/My Drive/projects/bambas/feature_extraction/1701632817_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_train_features.json\n",
            "Saved feature-extraction file to /gdrive/My Drive/projects/bambas/feature_extraction/1701632817_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_test_features.json\n",
            "Saved feature-extraction file to /gdrive/My Drive/projects/bambas/feature_extraction/1701632817_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_dev_features.json\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!PYTHONPATH=\"/gdrive/My Drive/projects/bambas\" python -m src.classification \\\n",
        "  --classifier \"HiMLP\" \\\n",
        "  --dataset semeval2024 \\\n",
        "  --train_features \"/gdrive/My Drive/projects/bambas/feature_extraction/1701632817_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_train_features.json\" \\\n",
        "  --test_features \"/gdrive/My Drive/projects/bambas/feature_extraction/1701632817_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_test_features.json\" \\\n",
        "  --dev_features \"/gdrive/My Drive/projects/bambas/feature_extraction/1701632817_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_dev_features.json\" \\\n",
        "  --seed 1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5gYK2nnHBVJk",
        "outputId": "431a05e8-a883-4fc0-d7c0-cec86d2e852d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Arguments: Namespace(classifier='HiMLP', dataset='semeval2024', train_features='/gdrive/My Drive/projects/bambas/feature_extraction/1701632817_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_train_features.json', test_features='/gdrive/My Drive/projects/bambas/feature_extraction/1701632817_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_test_features.json', dev_features='/gdrive/My Drive/projects/bambas/feature_extraction/1701632817_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_dev_features.json', max_iter=400, alpha=0.0001, seed=1)\n",
            "Loading features info files\n",
            "Loading dataset files\n",
            "Labels: [['Repetition', 'Slogans', 'Distraction', 'Black-and-white Fallacy/Dictatorship', 'Justification', 'Thought-terminating cliché', 'Transfer', 'Name calling/Labeling', 'Loaded Language', 'Appeal to fear/prejudice', 'Bandwagon', 'Exaggeration/Minimisation', 'Flag-waving', 'Simplification', 'Appeal to authority', 'Glittering generalities (Virtue)', 'Reductio ad hitlerum', 'Smears', 'Presenting Irrelevant Data (Red Herring)', 'Appeal to (Strong) Emotions', \"Misrepresentation of Someone's Position (Straw Man)\", 'Logos', 'Causal Oversimplification', 'Doubt', 'Whataboutism', 'Ad Hominem', 'Ethos', 'Pathos', 'Obfuscation, Intentional vagueness, Confusion', 'Reasoning']]\n",
            "No. of labels in DAG: 30\n",
            "Loading features array files\n",
            "Iteration 1, loss = 1.04948098\n",
            "Validation score: 0.513796\n",
            "Iteration 2, loss = 1.02731851\n",
            "Validation score: 0.515287\n",
            "Iteration 3, loss = 1.02647341\n",
            "Validation score: 0.505593\n",
            "Iteration 4, loss = 1.02559288\n",
            "Validation score: 0.488441\n",
            "Iteration 5, loss = 1.02465920\n",
            "Validation score: 0.501119\n",
            "Iteration 6, loss = 1.02461871\n",
            "Validation score: 0.501119\n",
            "Iteration 7, loss = 1.03370384\n",
            "Validation score: 0.515287\n",
            "Iteration 8, loss = 1.02961403\n",
            "Validation score: 0.517524\n",
            "Iteration 9, loss = 1.02812343\n",
            "Validation score: 0.519761\n",
            "Iteration 10, loss = 1.02378931\n",
            "Validation score: 0.501119\n",
            "Iteration 11, loss = 1.02681016\n",
            "Validation score: 0.519761\n",
            "Iteration 12, loss = 1.02558574\n",
            "Validation score: 0.519016\n",
            "Iteration 13, loss = 1.02419995\n",
            "Validation score: 0.519761\n",
            "Iteration 14, loss = 1.02560125\n",
            "Validation score: 0.519761\n",
            "Iteration 15, loss = 1.02610296\n",
            "Validation score: 0.505593\n",
            "Iteration 16, loss = 1.02373490\n",
            "Validation score: 0.519761\n",
            "Iteration 17, loss = 1.02374516\n",
            "Validation score: 0.519761\n",
            "Iteration 18, loss = 1.02065120\n",
            "Validation score: 0.519761\n",
            "Iteration 19, loss = 1.02200395\n",
            "Validation score: 0.519761\n",
            "Iteration 20, loss = 1.02104081\n",
            "Validation score: 0.501119\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.13799304\n",
            "Validation score: 0.608787\n",
            "Iteration 2, loss = 0.91447688\n",
            "Validation score: 0.569038\n",
            "Iteration 3, loss = 0.90189721\n",
            "Validation score: 0.581590\n",
            "Iteration 4, loss = 0.89745205\n",
            "Validation score: 0.608787\n",
            "Iteration 5, loss = 0.89755482\n",
            "Validation score: 0.581590\n",
            "Iteration 6, loss = 0.90147882\n",
            "Validation score: 0.569038\n",
            "Iteration 7, loss = 0.90200789\n",
            "Validation score: 0.608787\n",
            "Iteration 8, loss = 0.89629365\n",
            "Validation score: 0.608787\n",
            "Iteration 9, loss = 0.89936418\n",
            "Validation score: 0.608787\n",
            "Iteration 10, loss = 0.89714385\n",
            "Validation score: 0.581590\n",
            "Iteration 11, loss = 0.90060673\n",
            "Validation score: 0.569038\n",
            "Iteration 12, loss = 0.89744572\n",
            "Validation score: 0.608787\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 0.68801568\n",
            "Validation score: 0.803109\n",
            "Iteration 2, loss = 0.57171171\n",
            "Validation score: 0.803109\n",
            "Iteration 3, loss = 0.50434525\n",
            "Validation score: 0.803109\n",
            "Iteration 4, loss = 0.50058845\n",
            "Validation score: 0.803109\n",
            "Iteration 5, loss = 0.50220430\n",
            "Validation score: 0.803109\n",
            "Iteration 6, loss = 0.49778221\n",
            "Validation score: 0.803109\n",
            "Iteration 7, loss = 0.49553383\n",
            "Validation score: 0.803109\n",
            "Iteration 8, loss = 0.49600961\n",
            "Validation score: 0.803109\n",
            "Iteration 9, loss = 0.49506081\n",
            "Validation score: 0.803109\n",
            "Iteration 10, loss = 0.49501929\n",
            "Validation score: 0.803109\n",
            "Iteration 11, loss = 0.49463825\n",
            "Validation score: 0.803109\n",
            "Iteration 12, loss = 0.49465190\n",
            "Validation score: 0.803109\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.17393562\n",
            "Validation score: 0.554839\n",
            "Iteration 2, loss = 1.07155155\n",
            "Validation score: 0.554839\n",
            "Iteration 3, loss = 0.99188253\n",
            "Validation score: 0.554839\n",
            "Iteration 4, loss = 0.96048349\n",
            "Validation score: 0.554839\n",
            "Iteration 5, loss = 0.96171737\n",
            "Validation score: 0.554839\n",
            "Iteration 6, loss = 0.94823391\n",
            "Validation score: 0.554839\n",
            "Iteration 7, loss = 0.94761429\n",
            "Validation score: 0.554839\n",
            "Iteration 8, loss = 0.94743492\n",
            "Validation score: 0.554839\n",
            "Iteration 9, loss = 0.94745163\n",
            "Validation score: 0.554839\n",
            "Iteration 10, loss = 0.94682292\n",
            "Validation score: 0.554839\n",
            "Iteration 11, loss = 0.94512748\n",
            "Validation score: 0.554839\n",
            "Iteration 12, loss = 0.94386924\n",
            "Validation score: 0.554839\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.18382268\n",
            "Validation score: 0.578947\n",
            "Iteration 2, loss = 0.83068039\n",
            "Validation score: 0.578947\n",
            "Iteration 3, loss = 0.89876721\n",
            "Validation score: 0.578947\n",
            "Iteration 4, loss = 0.93998922\n",
            "Validation score: 0.578947\n",
            "Iteration 5, loss = 0.93924092\n",
            "Validation score: 0.578947\n",
            "Iteration 6, loss = 0.91434531\n",
            "Validation score: 0.578947\n",
            "Iteration 7, loss = 0.86556292\n",
            "Validation score: 0.578947\n",
            "Iteration 8, loss = 0.83204251\n",
            "Validation score: 0.578947\n",
            "Iteration 9, loss = 0.83307377\n",
            "Validation score: 0.578947\n",
            "Iteration 10, loss = 0.85204256\n",
            "Validation score: 0.578947\n",
            "Iteration 11, loss = 0.86108950\n",
            "Validation score: 0.578947\n",
            "Iteration 12, loss = 0.85282205\n",
            "Validation score: 0.578947\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.43235398\n",
            "Validation score: 0.454545\n",
            "Iteration 2, loss = 1.36837630\n",
            "Validation score: 0.454545\n",
            "Iteration 3, loss = 1.36286196\n",
            "Validation score: 0.454545\n",
            "Iteration 4, loss = 1.36115348\n",
            "Validation score: 0.454545\n",
            "Iteration 5, loss = 1.36123030\n",
            "Validation score: 0.454545\n",
            "Iteration 6, loss = 1.36137142\n",
            "Validation score: 0.454545\n",
            "Iteration 7, loss = 1.36277157\n",
            "Validation score: 0.454545\n",
            "Iteration 8, loss = 1.35896344\n",
            "Validation score: 0.454545\n",
            "Iteration 9, loss = 1.35836212\n",
            "Validation score: 0.454545\n",
            "Iteration 10, loss = 1.36258938\n",
            "Validation score: 0.454545\n",
            "Iteration 11, loss = 1.37164640\n",
            "Validation score: 0.454545\n",
            "Iteration 12, loss = 1.36379273\n",
            "Validation score: 0.442688\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 0.95165040\n",
            "Validation score: 0.729537\n",
            "Iteration 2, loss = 0.73369160\n",
            "Validation score: 0.729537\n",
            "Iteration 3, loss = 0.72686701\n",
            "Validation score: 0.729537\n",
            "Iteration 4, loss = 0.72440777\n",
            "Validation score: 0.729537\n",
            "Iteration 5, loss = 0.72475350\n",
            "Validation score: 0.729537\n",
            "Iteration 6, loss = 0.72457298\n",
            "Validation score: 0.729537\n",
            "Iteration 7, loss = 0.72541352\n",
            "Validation score: 0.729537\n",
            "Iteration 8, loss = 0.72854301\n",
            "Validation score: 0.729537\n",
            "Iteration 9, loss = 0.72626775\n",
            "Validation score: 0.729537\n",
            "Iteration 10, loss = 0.72647782\n",
            "Validation score: 0.729537\n",
            "Iteration 11, loss = 0.72505867\n",
            "Validation score: 0.729537\n",
            "Iteration 12, loss = 0.72459557\n",
            "Validation score: 0.729537\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.15551580\n",
            "Validation score: 0.437799\n",
            "Iteration 2, loss = 1.09236274\n",
            "Validation score: 0.497608\n",
            "Iteration 3, loss = 1.08422904\n",
            "Validation score: 0.437799\n",
            "Iteration 4, loss = 1.08471142\n",
            "Validation score: 0.442584\n",
            "Iteration 5, loss = 1.08766072\n",
            "Validation score: 0.502392\n",
            "Iteration 6, loss = 1.07613917\n",
            "Validation score: 0.442584\n",
            "Iteration 7, loss = 1.07488217\n",
            "Validation score: 0.442584\n",
            "Iteration 8, loss = 1.07468348\n",
            "Validation score: 0.437799\n",
            "Iteration 9, loss = 1.08563812\n",
            "Validation score: 0.437799\n",
            "Iteration 10, loss = 1.08173168\n",
            "Validation score: 0.442584\n",
            "Iteration 11, loss = 1.07802836\n",
            "Validation score: 0.502392\n",
            "Iteration 12, loss = 1.07761097\n",
            "Validation score: 0.442584\n",
            "Iteration 13, loss = 1.07786007\n",
            "Validation score: 0.502392\n",
            "Iteration 14, loss = 1.09196174\n",
            "Validation score: 0.442584\n",
            "Iteration 15, loss = 1.07756718\n",
            "Validation score: 0.497608\n",
            "Iteration 16, loss = 1.07617438\n",
            "Validation score: 0.437799\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.12548182\n",
            "Validation score: 0.605960\n",
            "Iteration 2, loss = 1.10721512\n",
            "Validation score: 0.605960\n",
            "Iteration 3, loss = 1.10838339\n",
            "Validation score: 0.605960\n",
            "Iteration 4, loss = 1.10337178\n",
            "Validation score: 0.605960\n",
            "Iteration 5, loss = 1.12090481\n",
            "Validation score: 0.605960\n",
            "Iteration 6, loss = 1.10584674\n",
            "Validation score: 0.605960\n",
            "Iteration 7, loss = 1.10637682\n",
            "Validation score: 0.605960\n",
            "Iteration 8, loss = 1.10888593\n",
            "Validation score: 0.605960\n",
            "Iteration 9, loss = 1.10908351\n",
            "Validation score: 0.605960\n",
            "Iteration 10, loss = 1.11029068\n",
            "Validation score: 0.605960\n",
            "Iteration 11, loss = 1.10756389\n",
            "Validation score: 0.605960\n",
            "Iteration 12, loss = 1.10919936\n",
            "Validation score: 0.605960\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "\n",
            "Validation set:\n",
            "\tPrecision: 0.34333070244672453\n",
            "\tRecall: 0.3852967227635075\n",
            "\tF1: 0.3631051752921536\n",
            "\n",
            "Saving validation set results to /gdrive/My Drive/projects/bambas/classification/results.csv\n",
            "\n",
            "Predicting for test file\n",
            "Finished successfully. dev_unlabeled predictions saved at /gdrive/My Drive/projects/bambas/classification/dev_unlabeled_predictions.json.txt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Class with last layers extraction"
      ],
      "metadata": {
        "id": "4GChX-yZr-5Y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!PYTHONPATH=\"/gdrive/My Drive/projects/bambas\" python -m src.feature_extraction \\\n",
        "  --model \"/gdrive/My Drive/projects/bambas/fine_tuning/xlm-roberta-base-ptc2019/\" \\\n",
        "  --dataset semeval2024 \\\n",
        "  --extraction_method layers \\\n",
        "  --layers 9 10 11 12 \\\n",
        "  --agg_method \"avg\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zgs-Qf50r_HM",
        "outputId": "2d398f9d-ea09-4be6-bb88-bff8c51171d6"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2023-12-03 22:00:09.483721: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2023-12-03 22:00:09.483802: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2023-12-03 22:00:09.483853: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2023-12-03 22:00:10.553128: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Arguments: Namespace(dataset='semeval2024', model='/gdrive/My Drive/projects/bambas/fine_tuning/xlm-roberta-base-ptc2019/', extraction_method='layers', layers=[9, 10, 11, 12], agg_method='avg')\n",
            "Using device: {'': 0}\n",
            "Some weights of XLMRobertaModel were not initialized from the model checkpoint at /gdrive/My Drive/projects/bambas/fine_tuning/xlm-roberta-base-ptc2019/ and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "#0:  18% 5/28 [00:00<00:02, 10.05ba/s]\n",
            "#0:  25% 7/28 [00:00<00:02,  7.35ba/s]\n",
            "#0:  32% 9/28 [00:01<00:02,  8.76ba/s]\n",
            "#0:  39% 11/28 [00:01<00:01,  9.73ba/s]\n",
            "#1:  18% 5/28 [00:00<00:02,  9.28ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#3:   0% 0/28 [00:00<?, ?ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:   0% 0/28 [00:00<?, ?ba/s]\u001b[A\u001b[A\n",
            "\n",
            "#2:   4% 1/28 [00:00<00:03,  7.48ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:   4% 1/28 [00:00<00:03,  7.14ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#0:  46% 13/28 [00:01<00:01, 10.51ba/s]\n",
            "\n",
            "#2:  11% 3/28 [00:00<00:02, 10.89ba/s]\u001b[A\u001b[A\n",
            "#1:  32% 9/28 [00:00<00:01, 11.43ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#0:  54% 15/28 [00:01<00:01, 10.16ba/s]\n",
            "\n",
            "#2:  18% 5/28 [00:00<00:01, 13.01ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  18% 5/28 [00:00<00:01, 13.09ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#0:  61% 17/28 [00:01<00:01, 10.65ba/s]\n",
            "\n",
            "#2:  25% 7/28 [00:00<00:01, 12.43ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  25% 7/28 [00:00<00:01, 12.16ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#0:  68% 19/28 [00:01<00:00, 11.06ba/s]\n",
            "\n",
            "#2:  32% 9/28 [00:00<00:01, 13.51ba/s]\u001b[A\u001b[A\n",
            "#1:  54% 15/28 [00:01<00:01, 12.04ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#0:  75% 21/28 [00:02<00:00, 11.46ba/s]\n",
            "\n",
            "#2:  39% 11/28 [00:00<00:01, 13.49ba/s]\u001b[A\u001b[A\n",
            "#0:  82% 23/28 [00:02<00:00, 12.53ba/s]\n",
            "\n",
            "\n",
            "#3:  39% 11/28 [00:00<00:01, 11.25ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#0:  89% 25/28 [00:02<00:00, 12.90ba/s]\n",
            "\n",
            "\n",
            "#3:  46% 13/28 [00:01<00:01, 11.97ba/s]\u001b[A\u001b[A\u001b[A\n",
            "#1:  68% 19/28 [00:01<00:00, 11.75ba/s]\u001b[A\n",
            "\n",
            "#2:  54% 15/28 [00:01<00:00, 13.10ba/s]\u001b[A\u001b[A\n",
            "#1:  75% 21/28 [00:01<00:00, 12.62ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#3:  54% 15/28 [00:01<00:01, 12.53ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#0: 100% 28/28 [00:02<00:00, 10.95ba/s]\n",
            "\n",
            "#1:  82% 23/28 [00:02<00:00, 13.04ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#3:  61% 17/28 [00:01<00:00, 12.80ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  68% 19/28 [00:01<00:00, 12.19ba/s]\u001b[A\u001b[A\n",
            "#1:  89% 25/28 [00:02<00:00, 12.64ba/s]\u001b[A\n",
            "\n",
            "\n",
            "#3:  68% 19/28 [00:01<00:00, 12.20ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  75% 21/28 [00:01<00:00, 11.72ba/s]\u001b[A\u001b[A\n",
            "#1: 100% 28/28 [00:02<00:00, 11.63ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3:  75% 21/28 [00:01<00:00, 11.51ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  82% 23/28 [00:01<00:00, 10.85ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  82% 23/28 [00:01<00:00, 11.72ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2:  89% 25/28 [00:02<00:00, 10.88ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:  89% 25/28 [00:02<00:00, 11.90ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2: 100% 28/28 [00:02<00:00, 12.33ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3: 100% 28/28 [00:02<00:00, 11.93ba/s]\n",
            "#0:   0% 0/2 [00:00<?, ?ba/s]\n",
            "#0: 100% 2/2 [00:00<00:00, 14.98ba/s]\n",
            "\n",
            "#1: 100% 2/2 [00:00<00:00, 13.97ba/s]\n",
            "\n",
            "\n",
            "#2:   0% 0/2 [00:00<?, ?ba/s]\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3:   0% 0/2 [00:00<?, ?ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "#2: 100% 2/2 [00:00<00:00, 17.29ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3: 100% 2/2 [00:00<00:00, 13.87ba/s]\n",
            "#0: 100% 4/4 [00:00<00:00, 15.47ba/s]\n",
            "\n",
            "#1:   0% 0/4 [00:00<?, ?ba/s]\u001b[A\n",
            "\n",
            "#2:   0% 0/4 [00:00<?, ?ba/s]\u001b[A\u001b[A\n",
            "#1:  25% 1/4 [00:00<00:00,  9.49ba/s]\u001b[A\n",
            "#1:  75% 3/4 [00:00<00:00, 15.33ba/s]\u001b[A\n",
            "\n",
            "#1: 100% 4/4 [00:00<00:00, 15.83ba/s]\n",
            "#2: 100% 4/4 [00:00<00:00, 17.08ba/s]\n",
            "\n",
            "\n",
            "\n",
            "#3:   0% 0/4 [00:00<?, ?ba/s]\u001b[A\u001b[A\u001b[A\n",
            "\n",
            "\n",
            "#3: 100% 4/4 [00:00<00:00, 17.87ba/s]\n",
            "You're using a XLMRobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
            "Batch no. 1\n",
            "Batch no. 2\n",
            "Batch no. 3\n",
            "Batch no. 4\n",
            "Batch no. 5\n",
            "Batch no. 6\n",
            "Batch no. 7\n",
            "Batch no. 8\n",
            "Batch no. 9\n",
            "Batch no. 10\n",
            "Batch no. 11\n",
            "Batch no. 12\n",
            "Batch no. 13\n",
            "Batch no. 14\n",
            "Batch no. 15\n",
            "Batch no. 16\n",
            "Batch no. 17\n",
            "Batch no. 18\n",
            "Batch no. 19\n",
            "Batch no. 20\n",
            "Batch no. 21\n",
            "Batch no. 22\n",
            "Batch no. 23\n",
            "Batch no. 24\n",
            "Batch no. 25\n",
            "Batch no. 26\n",
            "Batch no. 27\n",
            "Batch no. 28\n",
            "Batch no. 29\n",
            "Batch no. 30\n",
            "Batch no. 31\n",
            "Batch no. 32\n",
            "Batch no. 33\n",
            "Batch no. 34\n",
            "Batch no. 35\n",
            "Batch no. 36\n",
            "Batch no. 37\n",
            "Batch no. 38\n",
            "Batch no. 39\n",
            "Batch no. 40\n",
            "Batch no. 41\n",
            "Batch no. 42\n",
            "Batch no. 43\n",
            "Batch no. 44\n",
            "Batch no. 45\n",
            "Batch no. 46\n",
            "Batch no. 47\n",
            "Batch no. 48\n",
            "Batch no. 49\n",
            "Batch no. 50\n",
            "Batch no. 51\n",
            "Batch no. 52\n",
            "Batch no. 53\n",
            "Batch no. 54\n",
            "Batch no. 55\n",
            "Batch no. 56\n",
            "Batch no. 57\n",
            "Batch no. 58\n",
            "Batch no. 59\n",
            "Batch no. 60\n",
            "Batch no. 61\n",
            "Batch no. 62\n",
            "Batch no. 63\n",
            "Batch no. 64\n",
            "Batch no. 65\n",
            "Batch no. 66\n",
            "Batch no. 67\n",
            "Batch no. 68\n",
            "Batch no. 69\n",
            "Batch no. 70\n",
            "Batch no. 71\n",
            "Batch no. 72\n",
            "Batch no. 73\n",
            "Batch no. 74\n",
            "Batch no. 75\n",
            "Batch no. 76\n",
            "Batch no. 77\n",
            "Batch no. 78\n",
            "Batch no. 79\n",
            "Batch no. 80\n",
            "Batch no. 81\n",
            "Batch no. 82\n",
            "Batch no. 83\n",
            "Batch no. 84\n",
            "Batch no. 85\n",
            "Batch no. 86\n",
            "Batch no. 87\n",
            "Batch no. 88\n",
            "Batch no. 89\n",
            "Batch no. 90\n",
            "Batch no. 91\n",
            "Batch no. 92\n",
            "Batch no. 93\n",
            "Batch no. 94\n",
            "Batch no. 95\n",
            "Batch no. 96\n",
            "Batch no. 97\n",
            "Batch no. 98\n",
            "Batch no. 99\n",
            "Batch no. 100\n",
            "Batch no. 101\n",
            "Batch no. 102\n",
            "Batch no. 103\n",
            "Batch no. 104\n",
            "Batch no. 105\n",
            "Batch no. 106\n",
            "Batch no. 107\n",
            "Batch no. 108\n",
            "Batch no. 109\n",
            "Batch no. 110\n",
            "Batch no. 1\n",
            "Batch no. 2\n",
            "Batch no. 3\n",
            "Batch no. 4\n",
            "Batch no. 5\n",
            "Batch no. 6\n",
            "Batch no. 7\n",
            "Batch no. 8\n",
            "Batch no. 9\n",
            "Batch no. 10\n",
            "Batch no. 11\n",
            "Batch no. 12\n",
            "Batch no. 13\n",
            "Batch no. 14\n",
            "Batch no. 15\n",
            "Batch no. 16\n",
            "Batch no. 1\n",
            "Batch no. 2\n",
            "Batch no. 3\n",
            "Batch no. 4\n",
            "Batch no. 5\n",
            "Batch no. 6\n",
            "Batch no. 7\n",
            "Batch no. 8\n",
            "Saved feature-extraction file to /gdrive/My Drive/projects/bambas/feature_extraction/1701641130_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_train_features.json\n",
            "Saved feature-extraction file to /gdrive/My Drive/projects/bambas/feature_extraction/1701641130_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_test_features.json\n",
            "Saved feature-extraction file to /gdrive/My Drive/projects/bambas/feature_extraction/1701641130_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_dev_features.json\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!PYTHONPATH=\"/gdrive/My Drive/projects/bambas\" python -m src.classification \\\n",
        "  --classifier \"HiMLP\" \\\n",
        "  --dataset semeval2024 \\\n",
        "  --train_features \"/gdrive/My Drive/projects/bambas/feature_extraction/1701641130_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_train_features.json\" \\\n",
        "  --test_features \"/gdrive/My Drive/projects/bambas/feature_extraction/1701641130_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_test_features.json\" \\\n",
        "  --dev_features \"/gdrive/My Drive/projects/bambas/feature_extraction/1701641130_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_dev_features.json\" \\\n",
        "  --seed 1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2FWyLYtpr_Lu",
        "outputId": "3d02a4a9-569a-4b6b-b0e8-d778fc4c0211"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Running on Google Colab, workdir: /gdrive/My Drive/projects/bambas\n",
            "Arguments: Namespace(classifier='HiMLP', dataset='semeval2024', train_features='/gdrive/My Drive/projects/bambas/feature_extraction/1701641130_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_train_features.json', test_features='/gdrive/My Drive/projects/bambas/feature_extraction/1701641130_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_test_features.json', dev_features='/gdrive/My Drive/projects/bambas/feature_extraction/1701641130_-gdrive-My Drive-projects-bambas-fine_tuning-xlm-roberta-base-ptc2019-_dev_features.json', max_iter=400, alpha=0.0001, seed=1)\n",
            "Loading features info files\n",
            "Loading dataset files\n",
            "Labels: [['Name calling/Labeling', 'Slogans', 'Appeal to authority', 'Transfer', 'Ad Hominem', 'Smears', 'Reasoning', 'Appeal to fear/prejudice', \"Misrepresentation of Someone's Position (Straw Man)\", 'Ethos', 'Logos', 'Thought-terminating cliché', 'Bandwagon', 'Exaggeration/Minimisation', 'Repetition', 'Glittering generalities (Virtue)', 'Simplification', 'Appeal to (Strong) Emotions', 'Presenting Irrelevant Data (Red Herring)', 'Justification', 'Pathos', 'Reductio ad hitlerum', 'Black-and-white Fallacy/Dictatorship', 'Causal Oversimplification', 'Distraction', 'Doubt', 'Loaded Language', 'Whataboutism', 'Flag-waving', 'Obfuscation, Intentional vagueness, Confusion']]\n",
            "No. of labels in DAG: 30\n",
            "Loading features array files\n",
            "Iteration 1, loss = 1.06256548\n",
            "Validation score: 0.519761\n",
            "Iteration 2, loss = 1.02967191\n",
            "Validation score: 0.519761\n",
            "Iteration 3, loss = 1.04075453\n",
            "Validation score: 0.486950\n",
            "Iteration 4, loss = 1.03530491\n",
            "Validation score: 0.505593\n",
            "Iteration 5, loss = 1.03394206\n",
            "Validation score: 0.519761\n",
            "Iteration 6, loss = 1.03022170\n",
            "Validation score: 0.519761\n",
            "Iteration 7, loss = 1.02811887\n",
            "Validation score: 0.519016\n",
            "Iteration 8, loss = 1.03608085\n",
            "Validation score: 0.419836\n",
            "Iteration 9, loss = 1.03514887\n",
            "Validation score: 0.517524\n",
            "Iteration 10, loss = 1.02905249\n",
            "Validation score: 0.519761\n",
            "Iteration 11, loss = 1.02692108\n",
            "Validation score: 0.517524\n",
            "Iteration 12, loss = 1.02725223\n",
            "Validation score: 0.505593\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.01632040\n",
            "Validation score: 0.361925\n",
            "Iteration 2, loss = 0.90997612\n",
            "Validation score: 0.608787\n",
            "Iteration 3, loss = 0.90335889\n",
            "Validation score: 0.585774\n",
            "Iteration 4, loss = 0.90327887\n",
            "Validation score: 0.366109\n",
            "Iteration 5, loss = 0.91292768\n",
            "Validation score: 0.581590\n",
            "Iteration 6, loss = 0.91952083\n",
            "Validation score: 0.608787\n",
            "Iteration 7, loss = 0.92666827\n",
            "Validation score: 0.370293\n",
            "Iteration 8, loss = 0.90631915\n",
            "Validation score: 0.612971\n",
            "Iteration 9, loss = 0.90854458\n",
            "Validation score: 0.569038\n",
            "Iteration 10, loss = 0.90511522\n",
            "Validation score: 0.612971\n",
            "Iteration 11, loss = 0.91134886\n",
            "Validation score: 0.581590\n",
            "Iteration 12, loss = 0.90237102\n",
            "Validation score: 0.612971\n",
            "Iteration 13, loss = 0.92032249\n",
            "Validation score: 0.370293\n",
            "Iteration 14, loss = 0.90255146\n",
            "Validation score: 0.585774\n",
            "Iteration 15, loss = 0.90477798\n",
            "Validation score: 0.612971\n",
            "Iteration 16, loss = 0.90480763\n",
            "Validation score: 0.612971\n",
            "Iteration 17, loss = 0.90013016\n",
            "Validation score: 0.585774\n",
            "Iteration 18, loss = 0.90894508\n",
            "Validation score: 0.343096\n",
            "Iteration 19, loss = 0.90288508\n",
            "Validation score: 0.585774\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 0.67580996\n",
            "Validation score: 0.803109\n",
            "Iteration 2, loss = 0.54369337\n",
            "Validation score: 0.803109\n",
            "Iteration 3, loss = 0.51876534\n",
            "Validation score: 0.803109\n",
            "Iteration 4, loss = 0.50957999\n",
            "Validation score: 0.803109\n",
            "Iteration 5, loss = 0.50754487\n",
            "Validation score: 0.803109\n",
            "Iteration 6, loss = 0.50113834\n",
            "Validation score: 0.803109\n",
            "Iteration 7, loss = 0.49858544\n",
            "Validation score: 0.803109\n",
            "Iteration 8, loss = 0.49425254\n",
            "Validation score: 0.803109\n",
            "Iteration 9, loss = 0.49326479\n",
            "Validation score: 0.803109\n",
            "Iteration 10, loss = 0.49519095\n",
            "Validation score: 0.803109\n",
            "Iteration 11, loss = 0.49453929\n",
            "Validation score: 0.803109\n",
            "Iteration 12, loss = 0.49406545\n",
            "Validation score: 0.803109\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.07738049\n",
            "Validation score: 0.554839\n",
            "Iteration 2, loss = 0.97193746\n",
            "Validation score: 0.554839\n",
            "Iteration 3, loss = 0.94535715\n",
            "Validation score: 0.554839\n",
            "Iteration 4, loss = 0.97551505\n",
            "Validation score: 0.554839\n",
            "Iteration 5, loss = 0.97659461\n",
            "Validation score: 0.554839\n",
            "Iteration 6, loss = 0.95622536\n",
            "Validation score: 0.554839\n",
            "Iteration 7, loss = 0.94618392\n",
            "Validation score: 0.554839\n",
            "Iteration 8, loss = 0.95559104\n",
            "Validation score: 0.554839\n",
            "Iteration 9, loss = 0.94614041\n",
            "Validation score: 0.554839\n",
            "Iteration 10, loss = 0.95274860\n",
            "Validation score: 0.554839\n",
            "Iteration 11, loss = 0.95031960\n",
            "Validation score: 0.554839\n",
            "Iteration 12, loss = 0.95208975\n",
            "Validation score: 0.554839\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.00532023\n",
            "Validation score: 0.578947\n",
            "Iteration 2, loss = 0.96236930\n",
            "Validation score: 0.578947\n",
            "Iteration 3, loss = 0.86772639\n",
            "Validation score: 0.578947\n",
            "Iteration 4, loss = 0.88065001\n",
            "Validation score: 0.578947\n",
            "Iteration 5, loss = 0.86099141\n",
            "Validation score: 0.578947\n",
            "Iteration 6, loss = 0.88186514\n",
            "Validation score: 0.578947\n",
            "Iteration 7, loss = 0.84679532\n",
            "Validation score: 0.578947\n",
            "Iteration 8, loss = 0.85213262\n",
            "Validation score: 0.578947\n",
            "Iteration 9, loss = 0.83636624\n",
            "Validation score: 0.578947\n",
            "Iteration 10, loss = 0.83532670\n",
            "Validation score: 0.578947\n",
            "Iteration 11, loss = 0.84353726\n",
            "Validation score: 0.578947\n",
            "Iteration 12, loss = 0.83236673\n",
            "Validation score: 0.578947\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.50744423\n",
            "Validation score: 0.454545\n",
            "Iteration 2, loss = 1.38144131\n",
            "Validation score: 0.454545\n",
            "Iteration 3, loss = 1.37558489\n",
            "Validation score: 0.454545\n",
            "Iteration 4, loss = 1.36365290\n",
            "Validation score: 0.454545\n",
            "Iteration 5, loss = 1.36494713\n",
            "Validation score: 0.454545\n",
            "Iteration 6, loss = 1.36908843\n",
            "Validation score: 0.454545\n",
            "Iteration 7, loss = 1.36892197\n",
            "Validation score: 0.454545\n",
            "Iteration 8, loss = 1.36542557\n",
            "Validation score: 0.454545\n",
            "Iteration 9, loss = 1.36243840\n",
            "Validation score: 0.454545\n",
            "Iteration 10, loss = 1.36396097\n",
            "Validation score: 0.454545\n",
            "Iteration 11, loss = 1.37324001\n",
            "Validation score: 0.454545\n",
            "Iteration 12, loss = 1.36461906\n",
            "Validation score: 0.454545\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 0.89361554\n",
            "Validation score: 0.729537\n",
            "Iteration 2, loss = 0.73625306\n",
            "Validation score: 0.729537\n",
            "Iteration 3, loss = 0.73192501\n",
            "Validation score: 0.729537\n",
            "Iteration 4, loss = 0.72876298\n",
            "Validation score: 0.729537\n",
            "Iteration 5, loss = 0.72919899\n",
            "Validation score: 0.729537\n",
            "Iteration 6, loss = 0.72835993\n",
            "Validation score: 0.729537\n",
            "Iteration 7, loss = 0.72845291\n",
            "Validation score: 0.729537\n",
            "Iteration 8, loss = 0.73385789\n",
            "Validation score: 0.729537\n",
            "Iteration 9, loss = 0.72672951\n",
            "Validation score: 0.729537\n",
            "Iteration 10, loss = 0.73038963\n",
            "Validation score: 0.729537\n",
            "Iteration 11, loss = 0.72740503\n",
            "Validation score: 0.729537\n",
            "Iteration 12, loss = 0.72747883\n",
            "Validation score: 0.729537\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.23516326\n",
            "Validation score: 0.442584\n",
            "Iteration 2, loss = 1.09824507\n",
            "Validation score: 0.502392\n",
            "Iteration 3, loss = 1.08374558\n",
            "Validation score: 0.497608\n",
            "Iteration 4, loss = 1.08030524\n",
            "Validation score: 0.502392\n",
            "Iteration 5, loss = 1.08203706\n",
            "Validation score: 0.502392\n",
            "Iteration 6, loss = 1.07609633\n",
            "Validation score: 0.442584\n",
            "Iteration 7, loss = 1.07950004\n",
            "Validation score: 0.502392\n",
            "Iteration 8, loss = 1.07458189\n",
            "Validation score: 0.502392\n",
            "Iteration 9, loss = 1.09389832\n",
            "Validation score: 0.442584\n",
            "Iteration 10, loss = 1.08247180\n",
            "Validation score: 0.442584\n",
            "Iteration 11, loss = 1.08472256\n",
            "Validation score: 0.497608\n",
            "Iteration 12, loss = 1.09190703\n",
            "Validation score: 0.502392\n",
            "Iteration 13, loss = 1.08683203\n",
            "Validation score: 0.502392\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "Iteration 1, loss = 1.19080819\n",
            "Validation score: 0.605960\n",
            "Iteration 2, loss = 1.11382036\n",
            "Validation score: 0.605960\n",
            "Iteration 3, loss = 1.12647721\n",
            "Validation score: 0.605960\n",
            "Iteration 4, loss = 1.10777266\n",
            "Validation score: 0.605960\n",
            "Iteration 5, loss = 1.12684692\n",
            "Validation score: 0.605960\n",
            "Iteration 6, loss = 1.11333958\n",
            "Validation score: 0.605960\n",
            "Iteration 7, loss = 1.11337126\n",
            "Validation score: 0.605960\n",
            "Iteration 8, loss = 1.11374285\n",
            "Validation score: 0.605960\n",
            "Iteration 9, loss = 1.12021777\n",
            "Validation score: 0.605960\n",
            "Iteration 10, loss = 1.11948259\n",
            "Validation score: 0.605960\n",
            "Iteration 11, loss = 1.11389169\n",
            "Validation score: 0.605960\n",
            "Iteration 12, loss = 1.10737172\n",
            "Validation score: 0.605960\n",
            "Validation score did not improve more than tol=0.000100 for 10 consecutive epochs. Stopping.\n",
            "\n",
            "Validation set:\n",
            "\tPrecision: 0.3056179775280899\n",
            "\tRecall: 0.4216120460584588\n",
            "\tF1: 0.35436441466592217\n",
            "\n",
            "Saving validation set results to /gdrive/My Drive/projects/bambas/classification/results.csv\n",
            "\n",
            "Predicting for test file\n",
            "Finished successfully. dev_unlabeled predictions saved at /gdrive/My Drive/projects/bambas/classification/dev_unlabeled_predictions.json.txt\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "OUwSYRpDezQX"
      ]
    },
    "interpreter": {
      "hash": "38cca0c38332a56087b24af0bc80247f4fced29cb4f7f437d91dc159adec9c4e"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.5"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}