From ba654b56479de0bb76f001b0179c1743a307d303 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Thu, 26 Sep 2024 14:19:18 +0000 Subject: [PATCH 1/4] Fix RSFetcher starting --- counterparty-core/counterpartycore/lib/backend/rsfetcher.py | 2 ++ counterparty-core/counterpartycore/lib/blocks.py | 2 +- counterparty-core/counterpartycore/test/indexer_test.py | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/counterparty-core/counterpartycore/lib/backend/rsfetcher.py b/counterparty-core/counterpartycore/lib/backend/rsfetcher.py index 7fd4937a4..6b253b862 100644 --- a/counterparty-core/counterpartycore/lib/backend/rsfetcher.py +++ b/counterparty-core/counterpartycore/lib/backend/rsfetcher.py @@ -36,6 +36,8 @@ def __init__(self, start_height=0, indexer_config=None): self.next_height = start_height self.fetcher = None self.prefetch_task = None + + def start_fetcher(self): self.start() # prefetching self.stopped = False diff --git a/counterparty-core/counterpartycore/lib/blocks.py b/counterparty-core/counterpartycore/lib/blocks.py index c6c2c15b6..3a2b507f2 100644 --- a/counterparty-core/counterpartycore/lib/blocks.py +++ b/counterparty-core/counterpartycore/lib/blocks.py @@ -1322,7 +1322,7 @@ def catch_up(db, check_asset_conservation=True): fetch_time_start = time.time() if fetcher is None: fetcher = rsfetcher.RSFetcher(util.CURRENT_BLOCK_INDEX + 1) - + fetcher.start_fetcher() decoded_block = fetcher.get_block() block_height = decoded_block.get("height") fetch_time_end = time.time() diff --git a/counterparty-core/counterpartycore/test/indexer_test.py b/counterparty-core/counterpartycore/test/indexer_test.py index 1def3b90e..b6040d33d 100644 --- a/counterparty-core/counterpartycore/test/indexer_test.py +++ b/counterparty-core/counterpartycore/test/indexer_test.py @@ -23,9 +23,11 @@ @pytest.mark.skip() def test_fetcher_singleton(): fetcher = rsfetcher.RSFetcher(config=TEST_CONFIG) + fetcher.start_fetcher() fetcher.is_me = "Coucou" fetcher2 = rsfetcher.RSFetcher(config=TEST_CONFIG) + fetcher2.start_fetcher() assert fetcher2.is_me == "Coucou" assert fetcher.instance() is fetcher2.instance() @@ -36,6 +38,7 @@ def test_fetcher_singleton(): @pytest.mark.skip() def test_fetcher_interrupt(): fetcher = rsfetcher.RSFetcher(config=TEST_CONFIG) + fetcher.start_fetcher() interrupted = False try: for _ in range(500): From a3e39d73f8bf68845c25f5b0c0c60546e2ad6982 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Thu, 26 Sep 2024 14:22:16 +0000 Subject: [PATCH 2/4] release notes --- release-notes/release-notes-v10.4.2.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 release-notes/release-notes-v10.4.2.md diff --git a/release-notes/release-notes-v10.4.2.md b/release-notes/release-notes-v10.4.2.md new file mode 100644 index 000000000..bc7b7f51b --- /dev/null +++ b/release-notes/release-notes-v10.4.2.md @@ -0,0 +1,25 @@ +# Release Notes - Counterparty Core v10.4.2 (2024-??-??) + + +# Upgrading + + +# ChangeLog + +## Protocol Changes + +## Bugfixes + +* Fix RSFetcher startup + +## Codebase + +## API + +## CLI + +# Credits + +* Ouziel Slama +* Warren Puffett +* Adam Krellenstein From a01a59606a90e3a8be1e29d24d6bbd8962aabb82 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Thu, 26 Sep 2024 14:31:15 +0000 Subject: [PATCH 3/4] tweaks --- .../counterpartycore/lib/backend/rsfetcher.py | 17 +++++++---------- .../counterpartycore/lib/blocks.py | 2 +- .../counterpartycore/test/indexer_test.py | 6 +++--- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/backend/rsfetcher.py b/counterparty-core/counterpartycore/lib/backend/rsfetcher.py index 6b253b862..5f8501525 100644 --- a/counterparty-core/counterpartycore/lib/backend/rsfetcher.py +++ b/counterparty-core/counterpartycore/lib/backend/rsfetcher.py @@ -37,16 +37,6 @@ def __init__(self, start_height=0, indexer_config=None): self.fetcher = None self.prefetch_task = None - def start_fetcher(self): - self.start() - # prefetching - self.stopped = False - self.prefetch_queue = {} - self.prefetch_queue_size = 0 - self.executor = ThreadPoolExecutor(max_workers=WORKER_THREADS) - self.prefetch_task = self.executor.submit(self.prefetch_blocks) - self.prefetch_queue_initialized = False - def start(self): logger.debug("Starting Prefetcher...") try: @@ -67,6 +57,13 @@ def start(self): except Exception as e: logger.error(f"Failed to initialize fetcher: {e}. Retrying in 5 seconds...") raise e + # prefetching + self.stopped = False + self.prefetch_queue = {} + self.prefetch_queue_size = 0 + self.executor = ThreadPoolExecutor(max_workers=WORKER_THREADS) + self.prefetch_task = self.executor.submit(self.prefetch_blocks) + self.prefetch_queue_initialized = False def get_block(self): logger.trace("Fetching block with Rust backend.") diff --git a/counterparty-core/counterpartycore/lib/blocks.py b/counterparty-core/counterpartycore/lib/blocks.py index 3a2b507f2..1ab3a0fe7 100644 --- a/counterparty-core/counterpartycore/lib/blocks.py +++ b/counterparty-core/counterpartycore/lib/blocks.py @@ -1322,7 +1322,7 @@ def catch_up(db, check_asset_conservation=True): fetch_time_start = time.time() if fetcher is None: fetcher = rsfetcher.RSFetcher(util.CURRENT_BLOCK_INDEX + 1) - fetcher.start_fetcher() + fetcher.start() decoded_block = fetcher.get_block() block_height = decoded_block.get("height") fetch_time_end = time.time() diff --git a/counterparty-core/counterpartycore/test/indexer_test.py b/counterparty-core/counterpartycore/test/indexer_test.py index b6040d33d..7141d8cd7 100644 --- a/counterparty-core/counterpartycore/test/indexer_test.py +++ b/counterparty-core/counterpartycore/test/indexer_test.py @@ -23,11 +23,11 @@ @pytest.mark.skip() def test_fetcher_singleton(): fetcher = rsfetcher.RSFetcher(config=TEST_CONFIG) - fetcher.start_fetcher() + fetcher.start() fetcher.is_me = "Coucou" fetcher2 = rsfetcher.RSFetcher(config=TEST_CONFIG) - fetcher2.start_fetcher() + fetcher2.start() assert fetcher2.is_me == "Coucou" assert fetcher.instance() is fetcher2.instance() @@ -38,7 +38,7 @@ def test_fetcher_singleton(): @pytest.mark.skip() def test_fetcher_interrupt(): fetcher = rsfetcher.RSFetcher(config=TEST_CONFIG) - fetcher.start_fetcher() + fetcher.start() interrupted = False try: for _ in range(500): From 42dc86d1de65e1237604391da00b4c6e031577b6 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Thu, 26 Sep 2024 14:38:10 +0000 Subject: [PATCH 4/4] fix the fix --- .../counterpartycore/lib/backend/rsfetcher.py | 11 +++++------ counterparty-core/counterpartycore/lib/blocks.py | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/backend/rsfetcher.py b/counterparty-core/counterpartycore/lib/backend/rsfetcher.py index 5f8501525..8c24e24c2 100644 --- a/counterparty-core/counterpartycore/lib/backend/rsfetcher.py +++ b/counterparty-core/counterpartycore/lib/backend/rsfetcher.py @@ -16,7 +16,7 @@ class RSFetcher(metaclass=util.SingletonMeta): thread_index_counter = 0 # Add a thread index counter - def __init__(self, start_height=0, indexer_config=None): + def __init__(self, indexer_config=None): RSFetcher.thread_index_counter += 1 if indexer_config is None: self.config = { @@ -27,19 +27,18 @@ def __init__(self, start_height=0, indexer_config=None): "log_file": config.FETCHER_LOG, "log_level": config.LOG_LEVEL_STRING, "json_format": config.JSON_LOGS, - "start_height": start_height, } else: - self.config = indexer_config | {"start_height": start_height} + self.config = indexer_config self.config["network"] = "testnet" if config.TESTNET else "mainnet" - self.start_height = start_height - self.next_height = start_height self.fetcher = None self.prefetch_task = None - def start(self): + def start(self, start_height=0): logger.debug("Starting Prefetcher...") try: + self.config["start_height"] = start_height + self.next_height = start_height self.fetcher = indexer.Indexer(self.config) # check fetcher version fetcher_version = self.fetcher.get_version() diff --git a/counterparty-core/counterpartycore/lib/blocks.py b/counterparty-core/counterpartycore/lib/blocks.py index 1ab3a0fe7..804c7e9d7 100644 --- a/counterparty-core/counterpartycore/lib/blocks.py +++ b/counterparty-core/counterpartycore/lib/blocks.py @@ -1321,8 +1321,8 @@ def catch_up(db, check_asset_conservation=True): # Get block information and transactions fetch_time_start = time.time() if fetcher is None: - fetcher = rsfetcher.RSFetcher(util.CURRENT_BLOCK_INDEX + 1) - fetcher.start() + fetcher = rsfetcher.RSFetcher() + fetcher.start(util.CURRENT_BLOCK_INDEX + 1) decoded_block = fetcher.get_block() block_height = decoded_block.get("height") fetch_time_end = time.time()