Skip to content

Commit

Permalink
python312Packages.tokenizers: 0.20.0 -> 0.20.1
Browse files Browse the repository at this point in the history
  • Loading branch information
GaetanLepage committed Oct 10, 2024
1 parent 565db77 commit d4fef60
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 57 deletions.
120 changes: 66 additions & 54 deletions pkgs/development/python-modules/tokenizers/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 4 additions & 3 deletions pkgs/development/python-modules/tokenizers/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ let
};
"big.txt" = fetchurl {
url = "https://norvig.com/big.txt";
sha256 = "sha256-+gZsfUDw8gGsQUTmUqpiQw5YprOAXscGUPZ42lgE6Hs=";
hash = "sha256-+gZsfUDw8gGsQUTmUqpiQw5YprOAXscGUPZ42lgE6Hs=";
};
"bert-wiki.json" = fetchurl {
url = "https://s3.amazonaws.com/models.huggingface.co/bert/anthony/doc-pipeline/tokenizer.json";
Expand All @@ -74,14 +74,14 @@ let
in
buildPythonPackage rec {
pname = "tokenizers";
version = "0.20.0";
version = "0.20.1";
pyproject = true;

src = fetchFromGitHub {
owner = "huggingface";
repo = "tokenizers";
rev = "refs/tags/v${version}";
hash = "sha256-uuSHsdyx77YQjf1aiz7EJ/X+6RaOgfmjGqHSlMaCWDI=";
hash = "sha256-QTe1QdmJHSoosNG9cCJS7uQNdoMwgL+CJHQQUX5VtSY=";
};

cargoDeps = rustPlatform.importCargoLock { lockFile = ./Cargo.lock; };
Expand Down Expand Up @@ -149,6 +149,7 @@ buildPythonPackage rec {
meta = {
description = "Fast State-of-the-Art Tokenizers optimized for Research and Production";
homepage = "https://github.com/huggingface/tokenizers";
changelog = "https://github.com/huggingface/tokenizers/releases/tag/v${version}";
license = lib.licenses.asl20;
maintainers = with lib.maintainers; [ GaetanLepage ];
platforms = lib.platforms.unix;
Expand Down

0 comments on commit d4fef60

Please sign in to comment.