Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

progress: remove unused logging/tqdm lock #817

Merged
merged 1 commit into from
Jan 14, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 0 additions & 13 deletions src/datachain/progress.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,5 @@
"""Manages progress bars."""

import logging
from threading import RLock

from fsspec import Callback
from fsspec.callbacks import TqdmCallback
from tqdm.auto import tqdm

logger = logging.getLogger(__name__)
tqdm.set_lock(RLock())


class CombinedDownloadCallback(Callback):
Expand All @@ -24,10 +15,6 @@ def increment_file_count(self, n: int = 1) -> None:
class TqdmCombinedDownloadCallback(CombinedDownloadCallback, TqdmCallback):
def __init__(self, tqdm_kwargs=None, *args, **kwargs):
self.files_count = 0
tqdm_kwargs = tqdm_kwargs or {}
tqdm_kwargs.setdefault("postfix", {}).setdefault("files", self.files_count)
kwargs = kwargs or {}
kwargs["tqdm_cls"] = tqdm
super().__init__(tqdm_kwargs, *args, **kwargs)

def increment_file_count(self, n: int = 1) -> None:
Expand Down
3 changes: 2 additions & 1 deletion src/datachain/query/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -355,14 +355,15 @@ def process_udf_outputs(

def get_download_callback(suffix: str = "", **kwargs) -> CombinedDownloadCallback:
return TqdmCombinedDownloadCallback(
{
tqdm_kwargs={
"desc": "Download" + suffix,
"unit": "B",
"unit_scale": True,
"unit_divisor": 1024,
"leave": False,
**kwargs,
},
tqdm_cls=tqdm,
)


Expand Down
Loading