Skip to content

Commit

Permalink
fix(utils): replace insecure_hashlib with hashlib
Browse files Browse the repository at this point in the history
Replaced `insecure_hashlib` with `hashlib` in `file_utils.py` to ensure
compatibility with Python 3.9 and above. Added a partial function for
`sha256` to handle the `usedforsecurity` argument conditionally.
  • Loading branch information
psmyth94 committed Nov 18, 2024
1 parent 77aad5e commit 18db02c
Showing 1 changed file with 7 additions and 4 deletions.
11 changes: 7 additions & 4 deletions src/biofit/utils/file_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
Copyright by the AllenNLP authors.
"""

import functools
import io
import os
import posixpath
Expand All @@ -17,8 +18,7 @@
from pathlib import Path
from typing import Optional, TypeVar, Union
from urllib.parse import unquote, urlparse

from huggingface_hub.utils import insecure_hashlib
import hashlib

from . import logging

Expand All @@ -29,6 +29,9 @@
PathLike = Union[str, Path]
T = TypeVar("T", str, Path)

_kwargs = {"usedforsecurity": False} if sys.version_info >= (3, 9) else {}
sha256 = functools.partial(hashlib.sha256, **_kwargs)


def is_remote_url(url_or_filename: Union[str, Path]) -> bool:
return urlparse(str(url_or_filename)).scheme != "" and not os.path.ismount(
Expand Down Expand Up @@ -143,12 +146,12 @@ def hash_url_to_filename(url, etag=None):
(see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380)
"""
url_bytes = url.encode("utf-8")
url_hash = insecure_hashlib.sha256(url_bytes)
url_hash = sha256(url_bytes)
filename = url_hash.hexdigest()

if etag:
etag_bytes = etag.encode("utf-8")
etag_hash = insecure_hashlib.sha256(etag_bytes)
etag_hash = sha256(etag_bytes)
filename += "." + etag_hash.hexdigest()

if url.endswith(".py"):
Expand Down

0 comments on commit 18db02c

Please sign in to comment.