diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 86af2acb..fb10e00a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -86,7 +86,7 @@ repos: - anyio - fh-llm-client>=0.0.9 # Match pyproject.toml - fhaviary[llm]>=0.14 # Match pyproject.toml - - ldp>=0.17 # Match pyproject.toml + - ldp>=0.20 # Match pyproject.toml - html2text - httpx - pybtex diff --git a/paperqa/_ldp_shims.py b/paperqa/_ldp_shims.py index 02dfb13a..0e690a82 100644 --- a/paperqa/_ldp_shims.py +++ b/paperqa/_ldp_shims.py @@ -14,8 +14,8 @@ "SimpleAgentState", "UIndexMemoryModel", "_Memories", + "bulk_evaluate_consensus", "discounted_returns", - "evaluate_consensus", "set_training_mode", ] @@ -34,7 +34,7 @@ Callback, ComputeTrajectoryMetricsMixin, RolloutManager, - evaluate_consensus, + bulk_evaluate_consensus, ) from ldp.graph.memory import Memory, UIndexMemoryModel from ldp.graph.op_utils import set_training_mode @@ -63,5 +63,5 @@ class Callback: # type: ignore[no-redef] SimpleAgentState = None # type: ignore[assignment,misc] UIndexMemoryModel = None # type: ignore[assignment,misc] discounted_returns = None # type: ignore[assignment] - evaluate_consensus = None # type: ignore[assignment] + bulk_evaluate_consensus = None # type: ignore[assignment] set_training_mode = None # type: ignore[assignment] diff --git a/paperqa/agents/task.py b/paperqa/agents/task.py index bbc496a9..4784b853 100644 --- a/paperqa/agents/task.py +++ b/paperqa/agents/task.py @@ -36,7 +36,7 @@ from paperqa._ldp_shims import ( Callback, ComputeTrajectoryMetricsMixin, - evaluate_consensus, + bulk_evaluate_consensus, ) from paperqa.docs import Docs from paperqa.litqa import ( @@ -254,7 +254,7 @@ def extract_ideal(x: GradablePaperQAEnvironment | Frame) -> str: return query["ideal_answer"] try: - consensus, accuracy = await evaluate_consensus( + consensus, accuracy = await bulk_evaluate_consensus( data=data, grouping_fn=extract_question, extract_answer_fn=extract_answer, diff --git a/pyproject.toml b/pyproject.toml index e681d17f..4f316c1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ dev = [ "vcrpy>=6", # Pin for https://github.com/kevin1024/vcrpy/issues/884 ] ldp = [ - "ldp>=0.17", # For evaluate_consensus + "ldp>=0.20", # For bulk_evaluate_consensus ] local = [ "sentence-transformers", diff --git a/uv.lock b/uv.lock index 25dbf0aa..5012d636 100644 --- a/uv.lock +++ b/uv.lock @@ -240,7 +240,7 @@ name = "click" version = "8.1.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "platform_system == 'Windows'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593 } wheels = [ @@ -1056,7 +1056,7 @@ wheels = [ [[package]] name = "ldp" -version = "0.18.0" +version = "0.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiofiles" }, @@ -1073,9 +1073,9 @@ dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.12'" }, { name = "usearch" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/89/e9/366ae99956c88a7cd02dddbada029c4497297d3403e64d3b42190de8a166/ldp-0.18.0.tar.gz", hash = "sha256:f7adea5442abf62c6b3998caab7159327abe3dbe37c85de73211801a93c29416", size = 383364 } +sdist = { url = "https://files.pythonhosted.org/packages/93/5b/06afc06b7a53a7222190f98d080001b4d02f9034cec8eea5e7bcf617413f/ldp-0.20.0.tar.gz", hash = "sha256:8cb07cfd0e542c8e2ac41996142e440d73b5aa1e622e61b329ffe0bea630ded0", size = 432918 } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/83/09b614bdfb4208313c1b0654db4aa4df1814149c3fc648c5ee799799ca58/ldp-0.18.0-py3-none-any.whl", hash = "sha256:bb64368c7eec3db05552e501db1eef8cc38d08ddab8d4bb80c97878cef016631", size = 98846 }, + { url = "https://files.pythonhosted.org/packages/f7/6d/09823e4b3334ba2ed045f30dda35f7fc1dd425e1388e123a6e61818c916d/ldp-0.20.0-py3-none-any.whl", hash = "sha256:16cda839fc071e46b0ecfb0516e52b82a1a3daccda876b2f167f7c15d49e1cff", size = 133682 }, ] [[package]] @@ -1644,7 +1644,7 @@ wheels = [ [[package]] name = "paper-qa" -version = "5.10.2.dev2+gbdb1a3b" +version = "5.10.3.dev6+ge87dba1.d20250122" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -1716,7 +1716,31 @@ zotero = [ [package.dev-dependencies] dev = [ - { name = "paper-qa", extra = ["dev"] }, + { name = "datasets" }, + { name = "ipython" }, + { name = "ldp" }, + { name = "mypy" }, + { name = "pandas-stubs" }, + { name = "pre-commit" }, + { name = "pydantic" }, + { name = "pylint-pydantic" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-recording" }, + { name = "pytest-rerunfailures" }, + { name = "pytest-subtests" }, + { name = "pytest-sugar" }, + { name = "pytest-timer", extra = ["colorama"] }, + { name = "pytest-xdist" }, + { name = "python-dotenv" }, + { name = "pyzotero" }, + { name = "qdrant-client" }, + { name = "refurb" }, + { name = "sentence-transformers" }, + { name = "typeguard" }, + { name = "types-pyyaml" }, + { name = "types-setuptools" }, + { name = "vcrpy" }, ] [package.metadata] @@ -1729,7 +1753,7 @@ requires-dist = [ { name = "html2text" }, { name = "httpx" }, { name = "ipython", marker = "extra == 'dev'", specifier = ">=8" }, - { name = "ldp", marker = "extra == 'ldp'", specifier = ">=0.17" }, + { name = "ldp", marker = "extra == 'ldp'", specifier = ">=0.20" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.8" }, { name = "numpy" }, { name = "pandas-stubs", marker = "extra == 'typing'" }, @@ -1861,7 +1885,7 @@ name = "portalocker" version = "2.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "pywin32", marker = "platform_system == 'Windows'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ed/d3/c6c64067759e87af98cc668c1cc75171347d0f1577fab7ca3749134e3cd4/portalocker-2.10.1.tar.gz", hash = "sha256:ef1bf844e878ab08aee7e40184156e1151f228f103aa5c6bd0724cc330960f8f", size = 40891 } wheels = [ @@ -2171,7 +2195,7 @@ name = "pympler" version = "1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "pywin32", marker = "platform_system == 'Windows'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/dd/37/c384631908029676d8e7213dd956bb686af303a80db7afbc9be36bc49495/pympler-1.1.tar.gz", hash = "sha256:1eaa867cb8992c218430f1708fdaccda53df064144d1c5656b1e6f1ee6000424", size = 179954 } wheels = [ @@ -2943,21 +2967,21 @@ dependencies = [ { name = "fsspec" }, { name = "jinja2" }, { name = "networkx" }, - { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, + { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and platform_system == 'Linux'" }, { name = "setuptools", marker = "python_full_version >= '3.12'" }, { name = "sympy" }, - { name = "triton", marker = "python_full_version < '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "triton", marker = "python_full_version < '3.13' and platform_machine == 'x86_64' and platform_system == 'Linux'" }, { name = "typing-extensions" }, ] wheels = [ @@ -2977,7 +3001,7 @@ name = "tqdm" version = "4.67.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "platform_system == 'Windows'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737 } wheels = [