requirements.lock

# generated by rye
# use `rye lock` or `rye sync` to update this lockfile
#
# last locked with the following flags:
#   pre: false
#   features: []
#   all-features: false
#   with-sources: true

--extra-index-url https://download.pytorch.org/whl/nightly/cu121
--index-url https://pypi.org/simple/

absl-py==2.1.0
    # via rouge-score
accelerate==0.30.1
aenum==3.1.15
    # via lighteval
aiobotocore==2.13.0
    # via s3fs
aiohttp==3.9.5
    # via aiobotocore
    # via datasets
    # via fsspec
    # via s3fs
aioitertools==0.11.0
    # via aiobotocore
aiosignal==1.3.1
    # via aiohttp
annotated-types==0.6.0
    # via pydantic
antlr4-python3-runtime==4.13.1
    # via moto
attrs==23.2.0
    # via aiohttp
    # via jschema-to-python
    # via jsonlines
    # via jsonschema
    # via referencing
    # via sarif-om
aws-sam-translator==1.89.0
    # via cfn-lint
aws-xray-sdk==2.13.0
    # via moto
babel==2.15.0
    # via courlan
better-exceptions==0.3.3
    # via print-on-steroids
bitsandbytes==0.43.1
blinker==1.8.2
    # via flask
blis==0.7.11
    # via thinc
boto3==1.34.106
    # via aws-sam-translator
    # via moto
botocore==1.34.106
    # via aiobotocore
    # via aws-xray-sdk
    # via boto3
    # via moto
    # via s3transfer
catalogue==2.0.10
    # via spacy
    # via srsly
    # via thinc
certifi==2024.2.2
    # via requests
    # via sentry-sdk
    # via trafilatura
cffi==1.16.0
    # via cryptography
cfn-lint==0.87.2
    # via moto
chardet==5.2.0
    # via mbstrdecoder
charset-normalizer==3.3.2
    # via htmldate
    # via requests
    # via trafilatura
click==8.1.7
    # via flask
    # via nltk
    # via typer
    # via wandb
cloudpathlib==0.16.0
    # via weasel
colorama==0.4.6
    # via lighteval
    # via sacrebleu
coloredlogs==15.0.1
    # via optimum
confection==0.1.4
    # via thinc
    # via weasel
contourpy==1.2.1
    # via matplotlib
courlan==1.1.0
    # via trafilatura
cryptography==42.0.7
    # via joserfc
    # via moto
cycler==0.12.1
    # via matplotlib
cymem==2.0.8
    # via preshed
    # via spacy
    # via thinc
dataproperty==1.0.1
    # via pytablewriter
    # via tabledata
datasets==2.19.0
    # via datatrove
    # via deepfocus
    # via evaluate
    # via lighteval
    # via optimum
    # via wechsel
datatrove==0.2.0
dateparser==1.2.0
    # via htmldate
debugpy==1.8.1
deepfocus==1.0.1
dill==0.3.8
    # via datasets
    # via datatrove
    # via evaluate
    # via multiprocess
docker==7.0.0
    # via moto
docker-pycreds==0.4.0
    # via wandb
docstring-parser==0.16
    # via simple-parsing
entmax==1.3
    # via deepfocus
evaluate==0.4.2
execnet==2.1.1
    # via pytest-xdist
fastdist==1.1.6
    # via deepfocus
fasteners==0.19
    # via datatrove
fasttext @ git+https://github.com/facebookresearch/fastText@1142dc4c4ecbc19cc16eee5cdd28472e689267e6
    # via deepfocus
    # via wechsel
fasttext-wheel==0.9.2
    # via datatrove
faust-cchardet==2.1.19
    # via datatrove
filelock==3.13.1
    # via datasets
    # via huggingface-hub
    # via pytorch-triton
    # via tldextract
    # via torch
    # via transformers
fire==0.6.0
flask==3.0.3
    # via flask-cors
    # via moto
flask-cors==4.0.1
    # via moto
fonttools==4.51.0
    # via matplotlib
frozenlist==1.4.1
    # via aiohttp
    # via aiosignal
fsspec==2024.2.0
    # via datasets
    # via datatrove
    # via evaluate
    # via huggingface-hub
    # via lightning
    # via pytorch-lightning
    # via s3fs
    # via torch
ftfy==6.2.0
    # via datatrove
gensim==4.3.2
    # via wechsel
gitdb==4.0.11
    # via gitpython
gitpython==3.1.43
    # via lighteval
    # via wandb
graphql-core==3.2.3
    # via moto
hf-transfer==0.1.6
htmldate==1.8.1
    # via trafilatura
huggingface-hub==0.22.2
    # via accelerate
    # via datasets
    # via datatrove
    # via evaluate
    # via lighteval
    # via optimum
    # via tokenizers
    # via transformers
humanfriendly==10.0
    # via coloredlogs
humanize==4.9.0
    # via datatrove
idna==3.7
    # via requests
    # via tldextract
    # via yarl
iniconfig==2.0.0
    # via pytest
inscriptis==2.5.0
    # via datatrove
itsdangerous==2.2.0
    # via flask
jinja2==3.1.3
    # via flask
    # via moto
    # via spacy
    # via torch
jmespath==1.0.1
    # via boto3
    # via botocore
joblib==1.4.2
    # via nltk
    # via scikit-learn
joserfc==0.10.0
    # via moto
jschema-to-python==1.2.3
    # via cfn-lint
jsonargparse==4.28.0
jsondiff==2.0.0
    # via moto
jsonlines==4.0.0
jsonpatch==1.33
    # via cfn-lint
jsonpath-ng==1.6.1
    # via moto
jsonpickle==3.0.4
    # via jschema-to-python
jsonpointer==2.4
    # via jsonpatch
jsonschema==4.22.0
    # via aws-sam-translator
    # via cfn-lint
    # via openapi-schema-validator
    # via openapi-spec-validator
jsonschema-path==0.3.2
    # via openapi-spec-validator
jsonschema-specifications==2023.12.1
    # via jsonschema
    # via openapi-schema-validator
junit-xml==1.9
    # via cfn-lint
justext==3.0.1
    # via trafilatura
kiwisolver==1.4.5
    # via matplotlib
langcodes==3.4.0
    # via spacy
language-data==1.2.0
    # via langcodes
lazy-object-proxy==1.10.0
    # via openapi-spec-validator
lighteval==0.3.0
lightning==2.2.4
lightning-utilities==0.11.2
    # via lightning
    # via pytorch-lightning
    # via torchmetrics
llvmlite==0.42.0
    # via numba
loguru==0.7.2
    # via datatrove
lxml==5.2.1
    # via htmldate
    # via inscriptis
    # via justext
    # via lxml-html-clean
    # via sacrebleu
    # via trafilatura
lxml-html-clean==0.1.1
    # via lxml
marisa-trie==1.1.1
    # via language-data
markdown-it-py==3.0.0
    # via rich
markupsafe==2.1.5
    # via jinja2
    # via werkzeug
matplotlib==3.9.0
mbstrdecoder==1.1.3
    # via dataproperty
    # via pytablewriter
    # via typepy
mdurl==0.1.2
    # via markdown-it-py
moto==5.0.7
    # via datatrove
mpmath==1.2.1
    # via sympy
multidict==6.0.5
    # via aiohttp
    # via yarl
multiprocess==0.70.16
    # via datasets
    # via datatrove
    # via evaluate
murmurhash==1.0.10
    # via preshed
    # via spacy
    # via thinc
networkx==3.2.1
    # via cfn-lint
    # via torch
ninja==1.11.1.1
nltk==3.8.1
    # via datatrove
    # via lighteval
    # via rouge-score
    # via wechsel
numba==0.59.1
    # via deepfocus
numpy==1.26.4
    # via accelerate
    # via bitsandbytes
    # via blis
    # via contourpy
    # via datasets
    # via datatrove
    # via deepfocus
    # via evaluate
    # via fasttext
    # via fasttext-wheel
    # via gensim
    # via lightning
    # via matplotlib
    # via numba
    # via optimum
    # via pandas
    # via pyarrow
    # via pytorch-lightning
    # via rouge-score
    # via sacrebleu
    # via scikit-learn
    # via scipy
    # via seqeval
    # via spacy
    # via thinc
    # via torchmetrics
    # via transformers
nvidia-cublas-cu12==12.1.3.1
    # via nvidia-cudnn-cu12
    # via nvidia-cusolver-cu12
    # via torch
nvidia-cuda-cupti-cu12==12.1.105
    # via torch
nvidia-cuda-nvrtc-cu12==12.1.105
    # via torch
nvidia-cuda-runtime-cu12==12.1.105
    # via torch
nvidia-cudnn-cu12==8.9.2.26
    # via torch
nvidia-cufft-cu12==11.0.2.54
    # via torch
nvidia-curand-cu12==10.3.2.106
    # via torch
nvidia-cusolver-cu12==11.4.5.107
    # via torch
nvidia-cusparse-cu12==12.1.0.106
    # via nvidia-cusolver-cu12
    # via torch
nvidia-nccl-cu12==2.20.5
    # via torch
nvidia-nvjitlink-cu12==12.1.105
    # via nvidia-cusolver-cu12
    # via nvidia-cusparse-cu12
nvidia-nvtx-cu12==12.1.105
    # via torch
openapi-schema-validator==0.6.2
    # via openapi-spec-validator
openapi-spec-validator==0.7.1
    # via moto
optimum==1.19.2
packaging==22.0
    # via accelerate
    # via datasets
    # via docker
    # via evaluate
    # via huggingface-hub
    # via lightning
    # via lightning-utilities
    # via matplotlib
    # via optimum
    # via pytest
    # via pytorch-lightning
    # via spacy
    # via thinc
    # via transformers
    # via typepy
    # via weasel
pandas==2.2.2
    # via datasets
    # via evaluate
pathable==0.4.3
    # via jsonschema-path
pathvalidate==3.2.0
    # via pytablewriter
pbr==6.0.0
    # via jschema-to-python
    # via sarif-om
pillow==9.3.0
    # via matplotlib
platformdirs==4.2.2
    # via wandb
pluggy==1.5.0
    # via pytest
ply==3.11
    # via jsonpath-ng
polars==0.20.26
portalocker==2.7.0
    # via sacrebleu
preshed==3.0.9
    # via spacy
    # via thinc
print-on-steroids==1.2.3
    # via deepfocus
protobuf==3.20.3
    # via lighteval
    # via transformers
    # via wandb
psutil==5.9.8
    # via accelerate
    # via wandb
py-partiql-parser==0.5.5
    # via moto
pyarrow==16.0.0
    # via datasets
    # via datatrove
pyarrow-hotfix==0.6
    # via datasets
pybind11==2.12.0
    # via fasttext
    # via fasttext-wheel
pycountry==23.12.11
    # via lighteval
pycparser==2.22
    # via cffi
pydantic==2.7.1
    # via aws-sam-translator
    # via confection
    # via spacy
    # via thinc
    # via weasel
pydantic-core==2.18.2
    # via pydantic
pygments==2.18.0
    # via rich
pyparsing==3.1.2
    # via matplotlib
    # via moto
pytablewriter==1.2.0
    # via lighteval
pytest==8.2.0
    # via datatrove
    # via pytest-timeout
    # via pytest-xdist
pytest-timeout==2.3.1
    # via datatrove
pytest-xdist==3.6.1
    # via datatrove
python-dateutil==2.9.0
    # via botocore
    # via dateparser
    # via htmldate
    # via matplotlib
    # via moto
    # via pandas
    # via typepy
python-magic==0.4.27
    # via datatrove
pytorch-lightning==2.2.4
    # via lightning
pytorch-triton==3.0.0+45fff310c8
    # via torch
pytz==2024.1
    # via dateparser
    # via pandas
    # via typepy
pyyaml==6.0.1
    # via accelerate
    # via cfn-lint
    # via datasets
    # via huggingface-hub
    # via jsonargparse
    # via jsonschema-path
    # via lightning
    # via moto
    # via pytorch-lightning
    # via responses
    # via transformers
    # via wandb
referencing==0.31.1
    # via jsonschema
    # via jsonschema-path
    # via jsonschema-specifications
regex==2024.4.16
    # via cfn-lint
    # via dateparser
    # via nltk
    # via sacrebleu
    # via transformers
requests==2.31.0
    # via datasets
    # via deepfocus
    # via docker
    # via evaluate
    # via huggingface-hub
    # via inscriptis
    # via jsonschema-path
    # via moto
    # via requests-file
    # via responses
    # via spacy
    # via tldextract
    # via transformers
    # via wandb
    # via weasel
    # via wechsel
requests-file==2.0.0
    # via tldextract
responses==0.25.0
    # via moto
rfc3339-validator==0.1.4
    # via openapi-schema-validator
rich==13.7.1
    # via datatrove
    # via print-on-steroids
rouge-score==0.1.2
    # via lighteval
rpds-py==0.18.1
    # via jsonschema
    # via referencing
ruff==0.4.4
    # via datatrove
s3fs==2024.2.0
    # via datatrove
s3transfer==0.10.1
    # via boto3
sacrebleu==2.4.2
    # via lighteval
safetensors==0.4.3
    # via accelerate
    # via transformers
sarif-om==1.0.4
    # via cfn-lint
scikit-learn==1.4.2
    # via lighteval
    # via seqeval
    # via wechsel
scipy==1.13.0
    # via deepfocus
    # via gensim
    # via scikit-learn
    # via wechsel
sentencepiece==0.2.0
    # via lighteval
    # via transformers
sentry-sdk==2.2.0
    # via wandb
seqeval==1.2.2
setproctitle==1.3.3
    # via wandb
setuptools==69.5.1
    # via fasttext
    # via fasttext-wheel
    # via lightning-utilities
    # via marisa-trie
    # via moto
    # via pytablewriter
    # via spacy
    # via thinc
    # via wandb
simple-parsing==0.1.5
six==1.16.0
    # via docker-pycreds
    # via fire
    # via junit-xml
    # via python-dateutil
    # via rfc3339-validator
    # via rouge-score
    # via warcio
smart-open==6.4.0
    # via gensim
    # via spacy
    # via weasel
smmap==5.0.1
    # via gitdb
spacy==3.7.2
    # via lighteval
spacy-legacy==3.0.12
    # via spacy
spacy-loggers==1.0.5
    # via spacy
srsly==2.4.8
    # via confection
    # via spacy
    # via thinc
    # via weasel
sympy==1.12
    # via cfn-lint
    # via optimum
    # via torch
tabledata==1.3.3
    # via pytablewriter
tabulate==0.9.0
    # via sacrebleu
tcolorpy==0.1.6
    # via pytablewriter
termcolor==2.3.0
    # via fire
    # via lighteval
thinc==8.2.3
    # via spacy
threadpoolctl==3.5.0
    # via scikit-learn
tld==0.13
    # via courlan
tldextract==5.1.2
    # via datatrove
tokenizers==0.19.1
    # via datatrove
    # via transformers
torch==2.4.0.dev20240515+cu121
    # via accelerate
    # via bitsandbytes
    # via deepfocus
    # via entmax
    # via lighteval
    # via lightning
    # via optimum
    # via pytorch-lightning
    # via torchmetrics
torchmetrics==1.1.2
    # via lightning
    # via pytorch-lightning
tqdm==4.66.2
    # via datasets
    # via datatrove
    # via deepfocus
    # via evaluate
    # via huggingface-hub
    # via lightning
    # via nltk
    # via print-on-steroids
    # via pytorch-lightning
    # via spacy
    # via transformers
    # via wechsel
trafilatura==1.8.0
    # via datatrove
transformers==4.40.2
    # via deepfocus
    # via lighteval
    # via optimum
typepy==1.3.2
    # via dataproperty
    # via pytablewriter
    # via tabledata
typer==0.9.4
    # via spacy
    # via weasel
typing-extensions==4.8.0
    # via aws-sam-translator
    # via huggingface-hub
    # via lightning
    # via lightning-utilities
    # via pydantic
    # via pydantic-core
    # via pytorch-lightning
    # via simple-parsing
    # via torch
    # via typer
tzdata==2024.1
    # via pandas
tzlocal==5.2
    # via dateparser
urllib3==2.2.1
    # via botocore
    # via courlan
    # via docker
    # via htmldate
    # via requests
    # via responses
    # via sentry-sdk
    # via trafilatura
wandb==0.17.0
warcio==1.7.4
    # via datatrove
wasabi==1.1.2
    # via spacy
    # via thinc
    # via weasel
wcwidth==0.2.13
    # via ftfy
weasel==0.3.4
    # via spacy
wechsel==0.0.4
werkzeug==3.0.3
    # via flask
    # via moto
wheel==0.43.0
wrapt==1.16.0
    # via aiobotocore
    # via aws-xray-sdk
xmltodict==0.13.0
    # via moto
xxhash==3.4.1
    # via datasets
    # via datatrove
    # via evaluate
yarl==1.9.4
    # via aiohttp
zstandard==0.22.0