From 3af21b5b23e19e1b940533abebab1b31e80f5d84 Mon Sep 17 00:00:00 2001 From: Raphael Mitsch Date: Mon, 6 Nov 2023 16:38:03 +0100 Subject: [PATCH] Remove comment on doc attribute handling during sharding, as this is done by spaCy's slicing directly. --- spacy_llm/tasks/util/sharding.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/spacy_llm/tasks/util/sharding.py b/spacy_llm/tasks/util/sharding.py index 04ffac45..d1bedc41 100644 --- a/spacy_llm/tasks/util/sharding.py +++ b/spacy_llm/tasks/util/sharding.py @@ -68,12 +68,6 @@ def map_doc_to_shards( ) fraction /= 2 - # todo doc properties, such as .ents, have to be included for some tasks (e. g. REL, EL) to work. how - # should this be done in cases where the properties transcend shard limits? - # - should sharding never cut across entities/other properties? - # - should entities or all other properties be dropped if they transcend shard properties? this seems - # like the most pragmatic solution for now. - # - which properties should be copied to shards other than .ents? assert shard is not None shards.append(shard) fraction = 1