diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index d88cb006d4..6c3a355ada 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -40,7 +40,7 @@ jobs: # Report only issues of a given confidence level or higher. Can be LOW, MEDIUM or HIGH. Default is UNDEFINED (everything) # confidence: # optional, default is UNDEFINED # comma-separated list of paths (glob patterns supported) to exclude from scan (note that these are in addition to the excluded paths provided in the config file) (default: .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg) - excluded_paths: "*/test/*,*/counterparty-lib/tools/*" # optional, default is DEFAULT + excluded_paths: "*/test/*,*/counterparty-core/tools/*" # optional, default is DEFAULT # comma-separated list of test IDs to skip skips: "B101" # optional, default is DEFAULT # path to a .bandit file that supplies command line arguments diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1c46217787..07338d6d5d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -52,10 +52,6 @@ jobs: with: args: --release --out dist --sdist -m ${{ env.COUNTERPARTY_RS_DIR }}/Cargo.toml - - name: Build counterparty-lib - run: | - cd counterparty-lib && hatch build ../dist - - name: Build counterparty-core run: | cd counterparty-core && hatch build ../dist @@ -66,11 +62,11 @@ jobs: run: | pip install dist/*.whl --force-reinstall - # Run counterparty-lib tests + # Run counterparty-core tests - - name: Run counterparty-lib tests + - name: Run counterparty-core tests run: | - cd counterparty-lib + cd counterparty-core pytest # Upload wheels @@ -116,10 +112,6 @@ jobs: with: args: --release --out dist --sdist -m ${{ env.COUNTERPARTY_RS_DIR }}/Cargo.toml - - name: Build counterparty-lib - run: | - cd counterparty-lib && hatch build ../dist - - name: Build counterparty-core run: | cd counterparty-core && hatch build ../dist @@ -132,11 +124,11 @@ jobs: export CPATH="$CPATH:$(brew --prefix)/include" pip3.10 install dist/*.whl --force-reinstall - # Run counterparty-lib tests + # Run counterparty-core tests - - name: Run counterparty-lib tests + - name: Run counterparty-core tests run: | - cd counterparty-lib + cd counterparty-core pytest # Upload wheels @@ -182,10 +174,6 @@ jobs: with: args: --release --out dist --sdist -m ${{ env.COUNTERPARTY_RS_DIR }}/Cargo.toml - - name: Build counterparty-lib - run: | - cd counterparty-lib && hatch build ../dist - - name: Build counterparty-core run: | cd counterparty-core && hatch build ../dist @@ -196,11 +184,11 @@ jobs: run: | pip install dist/*.whl --force-reinstall - # Run counterparty-lib tests + # Run counterparty-core tests - - name: Run counterparty-lib tests + - name: Run counterparty-core tests run: | - cd counterparty-lib + cd counterparty-core pytest # Upload wheels diff --git a/.github/workflows/license_scanner.yml b/.github/workflows/license_scanner.yml index e6963b1953..02c395fc0a 100644 --- a/.github/workflows/license_scanner.yml +++ b/.github/workflows/license_scanner.yml @@ -29,12 +29,11 @@ jobs: python -m pip install --upgrade pip pip install license_scanner maturin sh cd counterparty-rs && pip install -e . && cd .. - cd counterparty-lib && pip install -e . && cd .. cd counterparty-core && pip install -e . && cd .. pip install evdev - name: Analysing dependencies with licence_scanner run: | - python counterparty-lib/tools/checklicences.py + python counterparty-core/tools/checklicences.py - name: Upload SARIF uses: github/codeql-action/upload-sarif/@v2 with: diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml index f884f45675..3fcbf35c8f 100644 --- a/.github/workflows/publish_docker_image.yml +++ b/.github/workflows/publish_docker_image.yml @@ -20,7 +20,7 @@ jobs: sudo sh get-docker.sh - name: Build, tag, login and push image run: | - export VERSION=v$(cat counterparty-lib/counterpartylib/lib/config.py | grep '__version__ =' | awk -F '"' '{print $2}') + export VERSION=v$(cat counterparty-core/counterpartycore/lib/config.py | grep '__version__ =' | awk -F '"' '{print $2}') docker build -t $DOCKER_REPO:$VERSION . docker tag $DOCKER_REPO:$VERSION $DOCKER_REPO:latest docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index dcbe4b46ed..f3bcdf6d0a 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -30,12 +30,11 @@ jobs: pip install pylint maturin pip install pylint-sarif-unofficial cd counterparty-rs && pip install -e . && cd .. - cd counterparty-lib && pip install -e . && cd .. cd counterparty-core && pip install -e . && cd .. pip install evdev - name: Analysing the code with pylint run: | - pylint2sarif $(git ls-files '*.py' | grep -v counterparty-rs/tests/ | grep -v counterparty-lib/counterpartylib/test/ | grep -v counterparty-lib/tools/) || true + pylint2sarif $(git ls-files '*.py' | grep -v counterparty-rs/tests/ | grep -v counterparty-core/counterpartycore/test/ | grep -v counterparty-core/tools/) || true - name: Upload SARIF uses: github/codeql-action/upload-sarif/@v2 with: diff --git a/.github/workflows/test_book.yml b/.github/workflows/test_book.yml index 3779ea1c30..531ab78b52 100644 --- a/.github/workflows/test_book.yml +++ b/.github/workflows/test_book.yml @@ -29,12 +29,11 @@ jobs: python -m pip install --upgrade pip pip install pylint maturin pytest cd counterparty-rs && pip install -e . && cd .. - cd counterparty-lib && pip install -e . && cd .. cd counterparty-core && pip install -e . && cd .. - name: Bootstrap testnet database run: | counterparty-server --testnet bootstrap --no-confirm - name: Run tests run: | - cd counterparty-lib - pytest counterpartylib/test/book_test.py --testbook -s + cd counterparty-core + pytest counterpartycore/test/book_test.py --testbook -s diff --git a/.gitignore b/.gitignore index 6dc06b0cb4..7c110e02c7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ profile.txt *.pyc # files generated by py.test -/counterpartylib/test/fixtures/scenarios/*.new.* +/counterpartycore/test/fixtures/scenarios/*.new.* .coverage # Compiled python modules. diff --git a/Dockerfile b/Dockerfile index 5f48e59b33..382d544316 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,17 +17,12 @@ RUN pip3 install maturin # copy repository COPY README.md /README.md COPY ./counterparty-rs /counterparty-rs -COPY ./counterparty-lib /counterparty-lib COPY ./counterparty-core /counterparty-core -# install counterparty-lib +# install counterparty-rs WORKDIR /counterparty-rs RUN pip3 install . -# install counterparty-lib -WORKDIR /counterparty-lib -RUN pip3 install . - # install counterparty-core WORKDIR /counterparty-core RUN pip3 install . diff --git a/RELEASE.md b/RELEASE.md index 3f32d08715..5e10cec22d 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -5,6 +5,7 @@ - [ ] Update Counterparty Docker images versions in the `docker-compose.yml` files - [ ] Review all open pull requests - [ ] Write release notes +- [ ] Add a checkpoint verified on all supported versions - [ ] Create pull request against `master` - [ ] Ensure all tests pass in CI - [ ] Merge PR into `master` diff --git a/counterparty-core/counterpartycore/cli.py b/counterparty-core/counterpartycore/cli.py new file mode 100755 index 0000000000..327c36d549 --- /dev/null +++ b/counterparty-core/counterpartycore/cli.py @@ -0,0 +1,456 @@ +#! /usr/bin/env python3 + +import argparse +import logging +from urllib.parse import quote_plus as urlencode + +from termcolor import cprint + +from counterpartycore import server +from counterpartycore.lib import config, log, setup + +logger = logging.getLogger(config.LOGGER_NAME) + +APP_NAME = "counterparty-server" +APP_VERSION = config.VERSION_STRING + +CONFIG_ARGS = [ + [ + ("-v", "--verbose"), + { + "dest": "verbose", + "action": "store_true", + "default": False, + "help": "sets log level to DEBUG", + }, + ], + [ + ("--quiet",), + { + "dest": "quiet", + "action": "store_true", + "default": False, + "help": "sets log level to ERROR", + }, + ], + [ + ("--mainnet",), + { + "action": "store_true", + "default": True, + "help": f"use {config.BTC_NAME} mainet addresses and block numbers", + }, + ], + [ + ("--testnet",), + { + "action": "store_true", + "default": False, + "help": f"use {config.BTC_NAME} testnet addresses and block numbers", + }, + ], + [ + ("--testcoin",), + { + "action": "store_true", + "default": False, + "help": f"use the test {config.XCP_NAME} network on every blockchain", + }, + ], + [ + ("--regtest",), + { + "action": "store_true", + "default": False, + "help": f"use {config.BTC_NAME} regtest addresses and block numbers", + }, + ], + [ + ("--customnet",), + { + "default": "", + "help": "use a custom network (specify as UNSPENDABLE_ADDRESS|ADDRESSVERSION|P2SH_ADDRESSVERSION with version bytes in HH hex format)", + }, + ], + [ + ("--api-limit-rows",), + { + "type": int, + "default": 1000, + "help": "limit api calls to the set results (defaults to 1000). Setting to 0 removes the limit.", + }, + ], + [("--backend-name",), {"default": "addrindex", "help": "the backend name to connect to"}], + [ + ("--backend-connect",), + {"default": "localhost", "help": "the hostname or IP of the backend server"}, + ], + [("--backend-port",), {"type": int, "help": "the backend port to connect to"}], + [ + ("--backend-user",), + {"default": "rpc", "help": "the username used to communicate with backend"}, + ], + [ + ("--backend-password",), + {"default": "rpc", "help": "the password used to communicate with backend"}, + ], + [ + ("--backend-ssl",), + { + "action": "store_true", + "default": False, + "help": "use SSL to connect to backend (default: false)", + }, + ], + [ + ("--backend-ssl-no-verify",), + { + "action": "store_true", + "default": False, + "help": "verify SSL certificate of backend; disallow use of self‐signed certificates (default: true)", + }, + ], + [ + ("--backend-poll-interval",), + {"type": float, "default": 0.5, "help": "poll interval, in seconds (default: 0.5)"}, + ], + [ + ("--check-asset-conservation",), + { + "action": "store_true", + "default": False, + "help": "Skip asset conservation checking (default: false)", + }, + ], + [ + ("--p2sh-dust-return-pubkey",), + { + "help": "pubkey to receive dust when multisig encoding is used for P2SH source (default: none)" + }, + ], + [ + ("--indexd-connect",), + {"default": "localhost", "help": "the hostname or IP of the indexd server"}, + ], + [("--indexd-port",), {"type": int, "help": "the indexd server port to connect to"}], + [ + ("--rpc-host",), + { + "default": "localhost", + "help": "the IP of the interface to bind to for providing JSON-RPC API access (0.0.0.0 for all interfaces)", + }, + ], + [ + ("--rpc-port",), + {"type": int, "help": f"port on which to provide the {config.APP_NAME} JSON-RPC API"}, + ], + [ + ("--rpc-user",), + { + "default": "rpc", + "help": f"required username to use the {config.APP_NAME} JSON-RPC API (via HTTP basic auth)", + }, + ], + [ + ("--rpc-password",), + { + "default": "rpc", + "help": f"required password (for rpc-user) to use the {config.APP_NAME} JSON-RPC API (via HTTP basic auth)", + }, + ], + [ + ("--rpc-no-allow-cors",), + {"action": "store_true", "default": False, "help": "allow ajax cross domain request"}, + ], + [ + ("--rpc-batch-size",), + { + "type": int, + "default": config.DEFAULT_RPC_BATCH_SIZE, + "help": f"number of RPC queries by batch (default: {config.DEFAULT_RPC_BATCH_SIZE})", + }, + ], + [ + ("--requests-timeout",), + { + "type": int, + "default": config.DEFAULT_REQUESTS_TIMEOUT, + "help": "timeout value (in seconds) used for all HTTP requests (default: 5)", + }, + ], + [ + ("--force",), + { + "action": "store_true", + "default": False, + "help": "skip backend check, version check, process lock (NOT FOR USE ON PRODUCTION SYSTEMS)", + }, + ], + [ + ("--no-confirm",), + {"action": "store_true", "default": False, "help": "don't ask for confirmation"}, + ], + [("--database-file",), {"default": None, "help": "the path to the SQLite3 database file"}], + [ + ("--log-file",), + {"nargs": "?", "const": None, "default": False, "help": "log to the specified file"}, + ], + [ + ("--api-log-file",), + { + "nargs": "?", + "const": None, + "default": False, + "help": "log API requests to the specified file", + }, + ], + [ + ("--no-log-files",), + {"action": "store_true", "default": False, "help": "Don't write log files"}, + ], + [ + ("--json-log",), + {"action": "store_true", "default": False, "help": "Log events in JSON format"}, + ], + [ + ("--utxo-locks-max-addresses",), + { + "type": int, + "default": config.DEFAULT_UTXO_LOCKS_MAX_ADDRESSES, + "help": "max number of addresses for which to track UTXO locks", + }, + ], + [ + ("--utxo-locks-max-age",), + { + "type": int, + "default": config.DEFAULT_UTXO_LOCKS_MAX_AGE, + "help": "how long to keep a lock on a UTXO being tracked", + }, + ], + [ + ("--no-mempool",), + {"action": "store_true", "default": False, "help": "Disable mempool parsing"}, + ], +] + + +def welcome_message(action, server_configfile): + cprint(f"Running v{config.__version__} of {config.FULL_APP_NAME}.", "magenta") + + # print some info + cprint(f"Configuration file: {server_configfile}", "light_grey") + cprint(f"Counterparty database: {config.DATABASE}", "light_grey") + if config.LOG: + cprint(f"Writing log to file: `{config.LOG}`", "light_grey") + else: + cprint("Warning: log disabled", "yellow") + if config.API_LOG: + cprint(f"Writing API accesses log to file: `{config.API_LOG}`", "light_grey") + else: + cprint("Warning: API log disabled", "yellow") + + if config.VERBOSE: + if config.TESTNET: + cprint("NETWORK: Testnet", "light_grey") + elif config.REGTEST: + cprint("NETWORK: Regtest", "light_grey") + else: + cprint("NETWORK: Mainnet", "light_grey") + + pass_str = f":{urlencode(config.BACKEND_PASSWORD)}@" + cleaned_backend_url = config.BACKEND_URL.replace(pass_str, ":*****@") + cprint(f"BACKEND_URL: {cleaned_backend_url}", "light_grey") + cprint(f"INDEXD_URL: {config.INDEXD_URL}", "light_grey") + pass_str = f":{urlencode(config.RPC_PASSWORD)}@" + cleaned_rpc_url = config.RPC.replace(pass_str, ":*****@") + cprint(f"RPC: {cleaned_rpc_url}", "light_grey") + + cprint(f"{'-' * 30} {action} {'-' * 30}\n", "green") + + +class VersionError(Exception): + pass + + +def main(): + # Post installation tasks + server_configfile = setup.generate_server_config_file(CONFIG_ARGS) + + # Parse command-line arguments. + parser = argparse.ArgumentParser( + prog=APP_NAME, + description=f"Server for the {config.XCP_NAME} protocol", + add_help=False, + exit_on_error=False, + ) + parser.add_argument( + "-h", "--help", dest="help", action="store_true", help="show this help message and exit" + ) + parser.add_argument( + "-V", + "--version", + action="version", + version=f"{APP_NAME} v{APP_VERSION}; counterparty-core v{config.VERSION_STRING}", + ) + parser.add_argument("--config-file", help="the path to the configuration file") + + cmd_args = parser.parse_known_args()[0] + config_file_path = getattr(cmd_args, "config_file", None) + configfile = setup.read_config_file("server.conf", config_file_path) + + setup.add_config_arguments(parser, CONFIG_ARGS, configfile, add_default=True) + + subparsers = parser.add_subparsers(dest="action", help="the action to be taken") + + parser_server = subparsers.add_parser("start", help="run the server") + parser_server.add_argument("--config-file", help="the path to the configuration file") + parser_server.add_argument( + "--catch-up", + choices=["normal", "bootstrap"], + default="normal", + help="Catch up mode (default: normal)", + ) + setup.add_config_arguments(parser_server, CONFIG_ARGS, configfile) + + parser_reparse = subparsers.add_parser( + "reparse", help="reparse all transactions in the database" + ) + parser_reparse.add_argument( + "block_index", type=int, help="the index of the last known good block" + ) + setup.add_config_arguments(parser_reparse, CONFIG_ARGS, configfile) + + parser_vacuum = subparsers.add_parser( + "vacuum", help="VACUUM the database (to improve performance)" + ) + setup.add_config_arguments(parser_vacuum, CONFIG_ARGS, configfile) + + parser_rollback = subparsers.add_parser("rollback", help="rollback database") + parser_rollback.add_argument( + "block_index", type=int, help="the index of the last known good block" + ) + setup.add_config_arguments(parser_rollback, CONFIG_ARGS, configfile) + + parser_kickstart = subparsers.add_parser( + "kickstart", help="rapidly build database by reading from Bitcoin Core blockchain" + ) + parser_kickstart.add_argument("--bitcoind-dir", help="Bitcoin Core data directory") + parser_kickstart.add_argument( + "--max-queue-size", type=int, help="Size of the multiprocessing.Queue for parsing blocks" + ) + parser_kickstart.add_argument( + "--debug-block", type=int, help="Rollback and run kickstart for a single block;" + ) + setup.add_config_arguments(parser_kickstart, CONFIG_ARGS, configfile) + + parser_bootstrap = subparsers.add_parser( + "bootstrap", help="bootstrap database with hosted snapshot" + ) + setup.add_config_arguments(parser_bootstrap, CONFIG_ARGS, configfile) + + parser_checkdb = subparsers.add_parser("check-db", help="do an integrity check on the database") + setup.add_config_arguments(parser_checkdb, CONFIG_ARGS, configfile) + + parser_show_config = subparsers.add_parser( + "show-params", help="Show counterparty-server configuration" + ) + setup.add_config_arguments(parser_show_config, CONFIG_ARGS, configfile) + + args = parser.parse_args() + + # Help message + if args.help: + parser.print_help() + exit(0) + + # Configuration + init_args = dict( + database_file=args.database_file, + testnet=args.testnet, + testcoin=args.testcoin, + regtest=args.regtest, + customnet=args.customnet, + api_limit_rows=args.api_limit_rows, + backend_connect=args.backend_connect, + backend_port=args.backend_port, + backend_user=args.backend_user, + backend_password=args.backend_password, + backend_ssl=args.backend_ssl, + backend_ssl_no_verify=args.backend_ssl_no_verify, + backend_poll_interval=args.backend_poll_interval, + indexd_connect=args.indexd_connect, + indexd_port=args.indexd_port, + rpc_host=args.rpc_host, + rpc_port=args.rpc_port, + rpc_user=args.rpc_user, + rpc_password=args.rpc_password, + rpc_no_allow_cors=args.rpc_no_allow_cors, + requests_timeout=args.requests_timeout, + rpc_batch_size=args.rpc_batch_size, + check_asset_conservation=args.check_asset_conservation, + force=args.force, + p2sh_dust_return_pubkey=args.p2sh_dust_return_pubkey, + utxo_locks_max_addresses=args.utxo_locks_max_addresses, + utxo_locks_max_age=args.utxo_locks_max_age, + no_mempool=args.no_mempool, + ) + + server.initialise_log_config( + verbose=args.verbose, + quiet=args.quiet, + log_file=args.log_file, + api_log_file=args.api_log_file, + no_log_files=args.no_log_files, + testnet=args.testnet, + testcoin=args.testcoin, + regtest=args.regtest, + json_log=args.json_log, + ) + + # set up logging + log.set_up( + verbose=config.VERBOSE, + quiet=config.QUIET, + log_file=config.LOG, + log_in_console=args.action == "start", + ) + + server.initialise_config(**init_args) + + logger.info(f"Running v{APP_VERSION} of {APP_NAME}.") + + welcome_message(args.action, server_configfile) + + # Bootstrapping + if args.action == "bootstrap": + server.bootstrap(no_confirm=args.no_confirm) + + # PARSING + elif args.action == "reparse": + server.reparse(block_index=args.block_index) + + elif args.action == "rollback": + server.rollback(block_index=args.block_index) + + elif args.action == "kickstart": + server.kickstart( + bitcoind_dir=args.bitcoind_dir, + force=args.force, + max_queue_size=args.max_queue_size, + debug_block=args.debug_block, + ) + + elif args.action == "start": + server.start_all(catch_up=args.catch_up) + + elif args.action == "show-params": + server.show_params() + + elif args.action == "vacuum": + server.vacuum() + + elif args.action == "check-db": + server.check_database() + else: + parser.print_help() diff --git a/counterparty-lib/counterpartylib/__init__.py b/counterparty-core/counterpartycore/lib/__init__.py similarity index 100% rename from counterparty-lib/counterpartylib/__init__.py rename to counterparty-core/counterpartycore/lib/__init__.py diff --git a/counterparty-lib/counterpartylib/lib/address.py b/counterparty-core/counterpartycore/lib/address.py similarity index 97% rename from counterparty-lib/counterpartylib/lib/address.py rename to counterparty-core/counterpartycore/lib/address.py index 3f8cd8a686..ab63b18631 100644 --- a/counterparty-lib/counterpartylib/lib/address.py +++ b/counterparty-core/counterpartycore/lib/address.py @@ -3,7 +3,7 @@ import bitcoin -from counterpartylib.lib import config, exceptions, ledger, script # noqa: F401 +from counterpartycore.lib import config, exceptions, ledger, script # noqa: F401 logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/api.py b/counterparty-core/counterpartycore/lib/api.py similarity index 95% rename from counterparty-lib/counterpartylib/lib/api.py rename to counterparty-core/counterpartycore/lib/api.py index 3f974d100b..f4a6e61cfa 100644 --- a/counterparty-lib/counterpartylib/lib/api.py +++ b/counterparty-core/counterpartycore/lib/api.py @@ -19,6 +19,8 @@ import requests # noqa: F401 +import counterpartycore.lib.sentry # noqa: F401 + D = decimal.Decimal import binascii # noqa: E402 import inspect # noqa: E402 @@ -32,9 +34,10 @@ from flask_httpauth import HTTPBasicAuth # noqa: E402 from jsonrpc import dispatcher # noqa: E402 from jsonrpc.exceptions import JSONRPCDispatchException # noqa: E402 +from werkzeug.serving import make_server # noqa: E402 from xmltodict import unparse as serialize_to_xml # noqa: E402 -from counterpartylib.lib import ( # noqa: E402 +from counterpartycore.lib import ( # noqa: E402 backend, blocks, # noqa: F401 config, @@ -47,8 +50,8 @@ transaction, util, ) -from counterpartylib.lib.kickstart.blocks_parser import BlockchainParser # noqa: E402 -from counterpartylib.lib.messages import ( # noqa: E402 +from counterpartycore.lib.kickstart.blocks_parser import BlockchainParser # noqa: E402 +from counterpartycore.lib.messages import ( # noqa: E402 bet, # noqa: F401 broadcast, # noqa: F401 btcpay, # noqa: F401 @@ -64,26 +67,16 @@ send, sweep, # noqa: F401 ) -from counterpartylib.lib.messages.versions import enhanced_send # noqa: E402 +from counterpartycore.lib.messages.versions import enhanced_send # noqa: E402 +from counterpartycore.lib.telemetry.util import ( # noqa: E402 + get_addrindexrs_version, + get_uptime, + is_docker, + is_force_enabled, +) logger = logging.getLogger(config.LOGGER_NAME) -if os.environ.get("SENTRY_DSN"): - import sentry_sdk - - environment = os.environ.get("SENTRY_ENVIRONMENT", "development") - - release = os.environ.get("SENTRY_RELEASE", config.__version__) - - logger.info("Sentry DSN found, initializing Sentry") - - sentry_sdk.init( - dsn=os.environ["SENTRY_DSN"], - environment=environment, - release=release, - traces_sample_rate=1.0, - ) - API_TABLES = [ "assets", @@ -243,7 +236,9 @@ class DatabaseError(Exception): def check_database_state(db, blockcount): f"""Checks {config.XCP_NAME} database to see if is caught up with backend.""" # noqa: B021 if ledger.CURRENT_BLOCK_INDEX + 1 < blockcount: - raise DatabaseError(f"{config.XCP_NAME} database is behind backend.") + raise DatabaseError( + f"{config.XCP_NAME} database is behind backend. [{ledger.CURRENT_BLOCK_INDEX }/{blockcount} blocks parsed]" + ) # logger.debug("Database state check passed.") return @@ -540,6 +535,15 @@ def adjust_get_transactions_results(query_result): return filtered_results +def get_default_args(func): + signature = inspect.signature(func) + return { + k: v.default + for k, v in signature.parameters.items() + if v.default is not inspect.Parameter.empty + } + + def compose_transaction( db, name, @@ -598,7 +602,7 @@ def compose_transaction( if not script.is_fully_valid(binascii.unhexlify(pubkey)): raise script.AddressError(f"invalid public key: {pubkey}") - compose_method = sys.modules[f"counterpartylib.lib.messages.{name}"].compose + compose_method = sys.modules[f"counterpartycore.lib.messages.{name}"].compose compose_params = inspect.getfullargspec(compose_method)[0] missing_params = [p for p in compose_params if p not in params and p != "db"] for param in missing_params: @@ -688,16 +692,23 @@ def __init__(self): self.last_database_check = 0 threading.Thread.__init__(self) self.stop_event = threading.Event() + self.stopping = False + self.stopped = False + self.db = None def stop(self): - self.stop_event.set() + logger.info("Stopping API Status Poller...") + self.stopping = True + self.db.close() + while not self.stopped: + time.sleep(0.1) def run(self): - logger.debug("Starting API Status Poller.") + logger.debug("Starting API Status Poller...") global CURRENT_API_STATUS_CODE, CURRENT_API_STATUS_RESPONSE_JSON # noqa: PLW0603 - db = database.get_connection(read_only=True) + self.db = database.get_connection(read_only=True) - while self.stop_event.is_set() != True: # noqa: E712 + while not self.stopping: # noqa: E712 try: # Check that backend is running, communicable, and caught up with the blockchain. # Check that the database has caught up with bitcoind. @@ -710,7 +721,7 @@ def run(self): check_backend_state() code = 12 logger.debug("Checking database state.") - check_database_state(db, backend.getblockcount()) + check_database_state(self.db, backend.getblockcount()) self.last_database_check = time.time() except (BackendError, DatabaseError) as e: exception_name = e.__class__.__name__ @@ -724,7 +735,9 @@ def run(self): else: CURRENT_API_STATUS_CODE = None CURRENT_API_STATUS_RESPONSE_JSON = None - time.sleep(config.BACKEND_POLL_INTERVAL) + time.sleep(0.5) # sleep for 0.5 seconds + if self.stopping: + self.stopped = True class APIServer(threading.Thread): @@ -733,15 +746,17 @@ class APIServer(threading.Thread): def __init__(self, db=None): self.db = db self.is_ready = False + self.server = None + self.ctx = None threading.Thread.__init__(self) - self.stop_event = threading.Event() def stop(self): - self.join() - self.stop_event.set() + logger.info("Stopping API Server...") + self.server.shutdown() + self.db.close() def run(self): - logger.info("Starting API Server.") + logger.info("Starting API Server...") self.db = self.db or database.get_connection(read_only=True) app = flask.Flask(__name__) auth = HTTPBasicAuth() @@ -1028,6 +1043,8 @@ def get_running_info(): server_ready = caught_up and indexd_caught_up + addrindexrs_version = get_addrindexrs_version().split(".") + return { "server_ready": server_ready, "db_caught_up": caught_up, @@ -1043,6 +1060,12 @@ def get_running_info(): "version_major": config.VERSION_MAJOR, "version_minor": config.VERSION_MINOR, "version_revision": config.VERSION_REVISION, + "addrindexrs_version_major": int(addrindexrs_version[0]), + "addrindexrs_version_minor": int(addrindexrs_version[1]), + "addrindexrs_version_revision": int(addrindexrs_version[2]), + "uptime": int(get_uptime()), + "dockerized": is_docker(), + "force_enabled": is_force_enabled(), } @dispatcher.add_method @@ -1513,14 +1536,10 @@ def handle_rest(path_args, flask_request): return response # Init the HTTP Server. + self.is_ready = True + self.server = make_server(config.RPC_HOST, config.RPC_PORT, app, threaded=True) init_api_access_log(app) - + self.ctx = app.app_context() + self.ctx.push() # Run app server (blocking) - self.is_ready = True - app.run(host=config.RPC_HOST, port=config.RPC_PORT, threaded=True) - - self.db.close() - return - - -# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 + self.server.serve_forever() diff --git a/counterparty-lib/counterpartylib/lib/arc4.py b/counterparty-core/counterpartycore/lib/arc4.py similarity index 100% rename from counterparty-lib/counterpartylib/lib/arc4.py rename to counterparty-core/counterpartycore/lib/arc4.py diff --git a/counterparty-lib/counterpartylib/lib/backend/__init__.py b/counterparty-core/counterpartycore/lib/backend/__init__.py similarity index 97% rename from counterparty-lib/counterpartylib/lib/backend/__init__.py rename to counterparty-core/counterpartycore/lib/backend/__init__.py index 14ce8478b7..bf476b658e 100644 --- a/counterparty-lib/counterpartylib/lib/backend/__init__.py +++ b/counterparty-core/counterpartycore/lib/backend/__init__.py @@ -8,8 +8,8 @@ import bitcoin.rpc as bitcoinlib_rpc # noqa: F401 from bitcoin.core import CBlock -from counterpartylib.lib import config, exceptions, ledger, prefetcher, script, util # noqa: F401 -from counterpartylib.lib.backend import addrindexrs # noqa: F401 +from counterpartycore.lib import config, exceptions, ledger, prefetcher, script, util # noqa: F401 +from counterpartycore.lib.backend import addrindexrs # noqa: F401 logger = logging.getLogger(config.LOGGER_NAME) @@ -38,7 +38,7 @@ def getit(adict): def backend(initialize=True): - mdl = sys.modules[f"counterpartylib.lib.backend.{config.BACKEND_NAME}"] + mdl = sys.modules[f"counterpartycore.lib.backend.{config.BACKEND_NAME}"] global INITIALIZED # noqa: PLW0603 if not INITIALIZED and initialize: mdl.init() @@ -47,6 +47,7 @@ def backend(initialize=True): def stop(): + logger.info("Stopping backend...") backend(initialize=False).stop() diff --git a/counterparty-lib/counterpartylib/lib/backend/addrindexrs.py b/counterparty-core/counterpartycore/lib/backend/addrindexrs.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/backend/addrindexrs.py rename to counterparty-core/counterpartycore/lib/backend/addrindexrs.py index 10c8dbe5d6..2ea4fd4d9c 100644 --- a/counterparty-lib/counterpartylib/lib/backend/addrindexrs.py +++ b/counterparty-core/counterpartycore/lib/backend/addrindexrs.py @@ -18,7 +18,7 @@ from pkg_resources import parse_version # noqa: F401 from requests.exceptions import ConnectionError, ReadTimeout, Timeout -from counterpartylib.lib import config, exceptions, ledger, util +from counterpartycore.lib import config, exceptions, ledger, util logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/backend/indexd.py b/counterparty-core/counterpartycore/lib/backend/indexd.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/backend/indexd.py rename to counterparty-core/counterpartycore/lib/backend/indexd.py index 5f6b42394f..e9b97d0cb4 100644 --- a/counterparty-lib/counterpartylib/lib/backend/indexd.py +++ b/counterparty-core/counterpartycore/lib/backend/indexd.py @@ -12,7 +12,7 @@ import requests from requests.exceptions import ConnectionError, ReadTimeout, Timeout -from counterpartylib.lib import config, util +from counterpartycore.lib import config, util logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/blocks.py b/counterparty-core/counterpartycore/lib/blocks.py similarity index 95% rename from counterparty-lib/counterpartylib/lib/blocks.py rename to counterparty-core/counterpartycore/lib/blocks.py index ffd4edd71d..639fe9e3eb 100644 --- a/counterparty-lib/counterpartylib/lib/blocks.py +++ b/counterparty-core/counterpartycore/lib/blocks.py @@ -25,7 +25,7 @@ from halo import Halo # noqa: E402 from termcolor import colored # noqa: E402 -from counterpartylib.lib import ( # noqa: E402 +from counterpartycore.lib import ( # noqa: E402 arc4, # noqa: F401 backend, check, @@ -39,9 +39,9 @@ script, # noqa: F401 util, # noqa: F401 ) -from counterpartylib.lib.gettxinfo import get_tx_info # noqa: E402 -from counterpartylib.lib.kickstart.blocks_parser import BlockchainParser # noqa: E402 -from counterpartylib.lib.transaction_helper import p2sh_encoding # noqa: E402, F401 +from counterpartycore.lib.gettxinfo import get_tx_info # noqa: E402 +from counterpartycore.lib.kickstart.blocks_parser import BlockchainParser # noqa: E402 +from counterpartycore.lib.transaction_helper import p2sh_encoding # noqa: E402, F401 from .exceptions import BTCOnlyError, DecodeError # noqa: E402, F401 from .kickstart.utils import ib2h # noqa: E402, F401 @@ -246,6 +246,7 @@ def parse_block( previous_txlist_hash=None, txlist_hash=None, previous_messages_hash=None, + reparsing=False, ): """Parse the block, return hash of new ledger, txlist and messages. @@ -275,6 +276,28 @@ def parse_block( txlist = [] for tx in list(cursor): try: + # Add manual event to journal because transaction already exists + if reparsing: + transaction_bindings = { + "tx_index": tx["tx_index"], + "tx_hash": tx["tx_hash"], + "block_index": tx["block_index"], + "block_hash": tx["block_hash"], + "block_time": tx["block_time"], + "source": tx["source"], + "destination": tx["destination"], + "btc_amount": tx["btc_amount"], + "fee": tx["fee"], + "data": tx["data"], + } + ledger.add_to_journal( + db, + block_index, + "insert", + "transactions", + "NEW_TRANSACTION", + transaction_bindings, + ) parse_tx(db, tx) data = binascii.hexlify(tx["data"]).decode("UTF-8") if tx["data"] else "" txlist.append( @@ -806,7 +829,22 @@ def reparse(db, block_index=0): for block in cursor.fetchall(): start_time_block_parse = time.time() ledger.CURRENT_BLOCK_INDEX = block["block_index"] - parse_block(db, block["block_index"], block["block_time"]) + # Add event manually to journal because block already exists + ledger.add_to_journal( + db, + block["block_index"], + "insert", + "blocks", + "NEW_BLOCK", + { + "block_index": block["block_index"], + "block_hash": block["block_hash"], + "block_time": block["block_time"], + "previous_block_hash": block["previous_block_hash"], + "difficulty": block["difficulty"], + }, + ) + parse_block(db, block["block_index"], block["block_time"], reparsing=True) block_parsed_count += 1 message = generate_progression_message( block, @@ -860,9 +898,6 @@ def follow(db): check.software_version() last_software_check = time.time() - # Initialise. - initialise(db) - # Get index of last block. if ledger.CURRENT_BLOCK_INDEX == 0: logger.warning("New database.") @@ -979,7 +1014,7 @@ def follow(db): # Rollback the DB. rollback(db, block_index=current_index - 1) - block_index = current_index + block_index = current_index - 1 tx_index = get_next_tx_index(db) continue @@ -1042,6 +1077,8 @@ def follow(db): if block_index == block_count: if config.CHECK_ASSET_CONSERVATION: check.asset_conservation(db) + else: + logger.debug("Skip asset conservation check.") # Remove any non‐supported transactions older than ten blocks. while len(not_supported_sorted) and not_supported_sorted[0][0] <= block_index - 10: diff --git a/counterparty-lib/counterpartylib/lib/check.py b/counterparty-core/counterpartycore/lib/check.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/check.py rename to counterparty-core/counterpartycore/lib/check.py index 2fd2e31d1c..f1bc2d3f44 100644 --- a/counterparty-lib/counterpartylib/lib/check.py +++ b/counterparty-core/counterpartycore/lib/check.py @@ -6,7 +6,7 @@ import requests -from counterpartylib.lib import config, database, exceptions, ledger, util # noqa: F401 +from counterpartycore.lib import config, database, exceptions, ledger, util # noqa: F401 logger = logging.getLogger(config.LOGGER_NAME) @@ -628,6 +628,10 @@ "ledger_hash": "af0a56fcb6fd0da3c007c311f063e256060997a0658ea1128ea08763eaa95383", "txlist_hash": "7735e7b52d2e2bca4ef76de0a5a543a910a2bfe41f2e189236f04aa09a455340", }, + 839910: { + "ledger_hash": "e41bae2ddd431d1ddfeb9403b1b2f04e2ea75baeb991636e1a7fc5b4302605f4", + "txlist_hash": "76ff941f15588a41124839acc80cc655407242154ba452285a348f17146807b7", + }, } CONSENSUS_HASH_VERSION_TESTNET = 7 diff --git a/counterparty-lib/counterpartylib/lib/config.py b/counterparty-core/counterpartycore/lib/config.py similarity index 97% rename from counterparty-lib/counterpartylib/lib/config.py rename to counterparty-core/counterpartycore/lib/config.py index 55e887ef95..5c2667f6c0 100644 --- a/counterparty-lib/counterpartylib/lib/config.py +++ b/counterparty-core/counterpartycore/lib/config.py @@ -7,7 +7,7 @@ # Semantic Version -__version__ = "10.1.0" # for hatch +__version__ = "10.1.1" # for hatch VERSION_STRING = __version__ version = VERSION_STRING.split("-")[0].split(".") VERSION_MAJOR = int(version[0]) @@ -133,9 +133,6 @@ # Custom exit codes EXITCODE_UPDATE_REQUIRED = 5 - -DEFAULT_CHECK_ASSET_CONSERVATION = False # TODO: Re-enable after optimization - BACKEND_RAW_TRANSACTIONS_CACHE_SIZE = 20000 BACKEND_RPC_BATCH_NUM_WORKERS = 6 diff --git a/counterparty-lib/counterpartylib/lib/database.py b/counterparty-core/counterpartycore/lib/database.py similarity index 94% rename from counterparty-lib/counterpartylib/lib/database.py rename to counterparty-core/counterpartycore/lib/database.py index 6ab866e7f8..7841822c90 100644 --- a/counterparty-lib/counterpartylib/lib/database.py +++ b/counterparty-core/counterpartycore/lib/database.py @@ -7,7 +7,7 @@ import logging # noqa: E402 import time # noqa: E402, F401 -from counterpartylib.lib import config, exceptions, ledger, log # noqa: E402, F401 +from counterpartycore.lib import config, exceptions, ledger, log # noqa: E402, F401 logger = logging.getLogger(config.LOGGER_NAME) @@ -108,6 +108,13 @@ def vacuum(db): logger.info("Database VACUUM completed.") +def optimize(db): + logger.info("Running PRAGMA optimize...") + cursor = db.cursor() + cursor.execute("PRAGMA optimize") + logger.info("PRAGMA optimize done.") + + def field_is_pk(cursor, table, field): cursor.execute(f"PRAGMA table_info({table})") for row in cursor: diff --git a/counterparty-lib/counterpartylib/lib/exceptions.py b/counterparty-core/counterpartycore/lib/exceptions.py similarity index 95% rename from counterparty-lib/counterpartylib/lib/exceptions.py rename to counterparty-core/counterpartycore/lib/exceptions.py index 085f2d0005..07f846c577 100644 --- a/counterparty-lib/counterpartylib/lib/exceptions.py +++ b/counterparty-core/counterpartycore/lib/exceptions.py @@ -79,4 +79,8 @@ class InvalidVersion(Exception): pass +class ComposeTransactionError(Exception): + pass + + # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 diff --git a/counterparty-lib/counterpartylib/lib/gettxinfo.py b/counterparty-core/counterpartycore/lib/gettxinfo.py similarity index 97% rename from counterparty-lib/counterpartylib/lib/gettxinfo.py rename to counterparty-core/counterpartycore/lib/gettxinfo.py index c1077553e8..d44134db7a 100644 --- a/counterparty-lib/counterpartylib/lib/gettxinfo.py +++ b/counterparty-core/counterpartycore/lib/gettxinfo.py @@ -5,13 +5,13 @@ import bitcoin as bitcoinlib # noqa: F401 from bitcoin.core.script import CScriptInvalidError # noqa: F401 -from counterpartylib.lib import arc4, backend, config, ledger, script -from counterpartylib.lib.exceptions import BTCOnlyError, DecodeError -from counterpartylib.lib.kickstart.blocks_parser import BlockchainParser -from counterpartylib.lib.kickstart.utils import ib2h -from counterpartylib.lib.messages import dispenser -from counterpartylib.lib.opcodes import * # noqa: F403 -from counterpartylib.lib.transaction_helper import p2sh_encoding +from counterpartycore.lib import arc4, backend, config, ledger, script +from counterpartycore.lib.exceptions import BTCOnlyError, DecodeError +from counterpartycore.lib.kickstart.blocks_parser import BlockchainParser +from counterpartycore.lib.kickstart.utils import ib2h +from counterpartycore.lib.messages import dispenser +from counterpartycore.lib.opcodes import * # noqa: F403 +from counterpartycore.lib.transaction_helper import p2sh_encoding logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/kickstart/__init__.py b/counterparty-core/counterpartycore/lib/kickstart/__init__.py similarity index 95% rename from counterparty-lib/counterpartylib/lib/kickstart/__init__.py rename to counterparty-core/counterpartycore/lib/kickstart/__init__.py index b44c95cb74..4ca89a68c4 100644 --- a/counterparty-lib/counterpartylib/lib/kickstart/__init__.py +++ b/counterparty-core/counterpartycore/lib/kickstart/__init__.py @@ -10,11 +10,11 @@ from halo import Halo from termcolor import colored -from counterpartylib import server -from counterpartylib.lib import backend, blocks, config, database, ledger, log # noqa: F401 -from counterpartylib.lib.backend.addrindexrs import AddrindexrsSocket # noqa: F401 -from counterpartylib.lib.kickstart.blocks_parser import BlockchainParser, ChainstateParser -from counterpartylib.lib.kickstart.utils import remove_shm_from_resource_tracker +from counterpartycore import server +from counterpartycore.lib import backend, blocks, config, database, ledger, log # noqa: F401 +from counterpartycore.lib.backend.addrindexrs import AddrindexrsSocket # noqa: F401 +from counterpartycore.lib.kickstart.blocks_parser import BlockchainParser, ChainstateParser +from counterpartycore.lib.kickstart.utils import remove_shm_from_resource_tracker logger = logging.getLogger(config.LOGGER_NAME) @@ -300,20 +300,15 @@ def parse_block(kickstart_db, cursor, block, block_parser, tx_index): with kickstart_db: # ensure all the block or nothing # insert block - cursor.execute( - """ - INSERT INTO blocks - (block_index, block_hash, block_time, previous_block_hash, difficulty) - VALUES (?, ?, ?, ?, ?) - """, - ( - block["block_index"], - block["block_hash"], - block["block_time"], - block["hash_prev"], - block["bits"], - ), - ) + block_bindings = { + "block_index": block["block_index"], + "block_hash": block["block_hash"], + "block_time": block["block_time"], + "previous_block_hash": block["hash_prev"], + "difficulty": block["bits"], + } + ledger.insert_record(kickstart_db, "blocks", block_bindings, "NEW_BLOCK") + # save transactions for transaction in block["transactions"]: # Cache transaction. We do that here because the block is fetched by another process. diff --git a/counterparty-lib/counterpartylib/lib/kickstart/bc_data_stream.py b/counterparty-core/counterpartycore/lib/kickstart/bc_data_stream.py similarity index 100% rename from counterparty-lib/counterpartylib/lib/kickstart/bc_data_stream.py rename to counterparty-core/counterpartycore/lib/kickstart/bc_data_stream.py diff --git a/counterparty-lib/counterpartylib/lib/kickstart/blocks_parser.py b/counterparty-core/counterpartycore/lib/kickstart/blocks_parser.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/kickstart/blocks_parser.py rename to counterparty-core/counterpartycore/lib/kickstart/blocks_parser.py index ee0f516a60..116a7f6e7f 100644 --- a/counterparty-lib/counterpartylib/lib/kickstart/blocks_parser.py +++ b/counterparty-core/counterpartycore/lib/kickstart/blocks_parser.py @@ -12,8 +12,8 @@ import apsw -from counterpartylib.lib import config, gettxinfo, ledger -from counterpartylib.lib.exceptions import DecodeError +from counterpartycore.lib import config, gettxinfo, ledger +from counterpartycore.lib.exceptions import DecodeError from .bc_data_stream import BCDataStream from .utils import ( diff --git a/counterparty-lib/counterpartylib/lib/kickstart/exceptions.py b/counterparty-core/counterpartycore/lib/kickstart/exceptions.py similarity index 100% rename from counterparty-lib/counterpartylib/lib/kickstart/exceptions.py rename to counterparty-core/counterpartycore/lib/kickstart/exceptions.py diff --git a/counterparty-lib/counterpartylib/lib/kickstart/utils.py b/counterparty-core/counterpartycore/lib/kickstart/utils.py similarity index 100% rename from counterparty-lib/counterpartylib/lib/kickstart/utils.py rename to counterparty-core/counterpartycore/lib/kickstart/utils.py diff --git a/counterparty-lib/counterpartylib/lib/ledger.py b/counterparty-core/counterpartycore/lib/ledger.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/ledger.py rename to counterparty-core/counterpartycore/lib/ledger.py index 43bcaf22f6..4870cbadc5 100644 --- a/counterparty-lib/counterpartylib/lib/ledger.py +++ b/counterparty-core/counterpartycore/lib/ledger.py @@ -6,7 +6,7 @@ import time from decimal import Decimal as D -from counterpartylib.lib import config, exceptions, log, util +from counterpartycore.lib import config, exceptions, log, util logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/log.py b/counterparty-core/counterpartycore/lib/log.py similarity index 92% rename from counterparty-lib/counterpartylib/lib/log.py rename to counterparty-core/counterpartycore/lib/log.py index b4577f4b8a..1df6153e37 100644 --- a/counterparty-lib/counterpartylib/lib/log.py +++ b/counterparty-core/counterpartycore/lib/log.py @@ -1,6 +1,5 @@ import decimal import logging -import os import sys import traceback from datetime import datetime @@ -9,7 +8,7 @@ from dateutil.tz import tzlocal from termcolor import cprint -from counterpartylib.lib import config +from counterpartycore.lib import config logger = logging.getLogger(config.LOGGER_NAME) D = decimal.Decimal @@ -35,18 +34,7 @@ def set_up(verbose=False, quiet=True, log_file=None, log_in_console=False): # File Logging if log_file: max_log_size = 20 * 1024 * 1024 # 20 MB - if os.name == "nt": - from counterpartylib.lib import util_windows - - fileh = util_windows.SanitizedRotatingFileHandler( - log_file, - maxBytes=max_log_size, - backupCount=5, # noqa: F821 - ) - else: - fileh = logging.handlers.RotatingFileHandler( - log_file, maxBytes=max_log_size, backupCount=5 - ) + fileh = logging.handlers.RotatingFileHandler(log_file, maxBytes=max_log_size, backupCount=5) fileh.setLevel(log_level) log_format = "%(asctime)s [%(levelname)s] %(message)s" formatter = logging.Formatter(log_format, "%Y-%m-%d-T%H:%M:%S%z") diff --git a/counterparty-lib/counterpartylib/lib/message_type.py b/counterparty-core/counterpartycore/lib/message_type.py similarity index 96% rename from counterparty-lib/counterpartylib/lib/message_type.py rename to counterparty-core/counterpartycore/lib/message_type.py index 7ddbf8f854..d7ac11ecd5 100644 --- a/counterparty-lib/counterpartylib/lib/message_type.py +++ b/counterparty-core/counterpartycore/lib/message_type.py @@ -1,7 +1,7 @@ import logging import struct -from counterpartylib.lib import config, ledger +from counterpartycore.lib import config, ledger logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/messages/__init__.py b/counterparty-core/counterpartycore/lib/messages/__init__.py similarity index 100% rename from counterparty-lib/counterpartylib/lib/messages/__init__.py rename to counterparty-core/counterpartycore/lib/messages/__init__.py diff --git a/counterparty-lib/counterpartylib/lib/messages/bet.py b/counterparty-core/counterpartycore/lib/messages/bet.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/messages/bet.py rename to counterparty-core/counterpartycore/lib/messages/bet.py index 352afa63e0..6a56955827 100644 --- a/counterparty-lib/counterpartylib/lib/messages/bet.py +++ b/counterparty-core/counterpartycore/lib/messages/bet.py @@ -19,7 +19,7 @@ D = decimal.Decimal import logging # noqa: E402 -from counterpartylib.lib import ( # noqa: E402 +from counterpartycore.lib import ( # noqa: E402 config, database, exceptions, diff --git a/counterparty-lib/counterpartylib/lib/messages/broadcast.py b/counterparty-core/counterpartycore/lib/messages/broadcast.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/messages/broadcast.py rename to counterparty-core/counterpartycore/lib/messages/broadcast.py index c6d5df638f..6a7dff6049 100644 --- a/counterparty-lib/counterpartylib/lib/messages/broadcast.py +++ b/counterparty-core/counterpartycore/lib/messages/broadcast.py @@ -31,7 +31,7 @@ from bitcoin.core import VarIntSerializer # noqa: E402 -from counterpartylib.lib import ( # noqa: E402 +from counterpartycore.lib import ( # noqa: E402 config, database, exceptions, diff --git a/counterparty-lib/counterpartylib/lib/messages/btcpay.py b/counterparty-core/counterpartycore/lib/messages/btcpay.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/messages/btcpay.py rename to counterparty-core/counterpartycore/lib/messages/btcpay.py index ef02ee5b0b..f4dec2b7bb 100644 --- a/counterparty-lib/counterpartylib/lib/messages/btcpay.py +++ b/counterparty-core/counterpartycore/lib/messages/btcpay.py @@ -6,7 +6,7 @@ import pprint # noqa: F401 import struct -from counterpartylib.lib import ( # noqa: F401 +from counterpartycore.lib import ( # noqa: F401 config, database, exceptions, diff --git a/counterparty-lib/counterpartylib/lib/messages/burn.py b/counterparty-core/counterpartycore/lib/messages/burn.py similarity index 98% rename from counterparty-lib/counterpartylib/lib/messages/burn.py rename to counterparty-core/counterpartycore/lib/messages/burn.py index 235a706264..568cb51d6e 100644 --- a/counterparty-lib/counterpartylib/lib/messages/burn.py +++ b/counterparty-core/counterpartycore/lib/messages/burn.py @@ -7,7 +7,7 @@ D = decimal.Decimal from fractions import Fraction # noqa: E402 -from counterpartylib.lib import config, database, exceptions, ledger # noqa: E402 +from counterpartycore.lib import config, database, exceptions, ledger # noqa: E402 logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/messages/cancel.py b/counterparty-core/counterpartycore/lib/messages/cancel.py similarity index 98% rename from counterparty-lib/counterpartylib/lib/messages/cancel.py rename to counterparty-core/counterpartycore/lib/messages/cancel.py index 5b3bbde4ea..22c57e59bd 100644 --- a/counterparty-lib/counterpartylib/lib/messages/cancel.py +++ b/counterparty-core/counterpartycore/lib/messages/cancel.py @@ -9,7 +9,7 @@ import logging import struct -from counterpartylib.lib import config, database, exceptions, ledger, message_type +from counterpartycore.lib import config, database, exceptions, ledger, message_type from . import bet, order, rps diff --git a/counterparty-lib/counterpartylib/lib/messages/destroy.py b/counterparty-core/counterpartycore/lib/messages/destroy.py similarity index 95% rename from counterparty-lib/counterpartylib/lib/messages/destroy.py rename to counterparty-core/counterpartycore/lib/messages/destroy.py index bec442735c..5793a0c15b 100644 --- a/counterparty-lib/counterpartylib/lib/messages/destroy.py +++ b/counterparty-core/counterpartycore/lib/messages/destroy.py @@ -6,9 +6,9 @@ import logging import struct -from counterpartylib.lib import config, database, ledger, message_type, script -from counterpartylib.lib.exceptions import * # noqa: F403 -from counterpartylib.lib.script import AddressError +from counterpartycore.lib import config, database, ledger, message_type, script +from counterpartycore.lib.exceptions import * # noqa: F403 +from counterpartycore.lib.script import AddressError logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/messages/dispenser.py b/counterparty-core/counterpartycore/lib/messages/dispenser.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/messages/dispenser.py rename to counterparty-core/counterpartycore/lib/messages/dispenser.py index 4f2ea6780c..2206bab853 100644 --- a/counterparty-lib/counterpartylib/lib/messages/dispenser.py +++ b/counterparty-core/counterpartycore/lib/messages/dispenser.py @@ -11,7 +11,7 @@ import struct from math import floor -from counterpartylib.lib import ( +from counterpartycore.lib import ( address, backend, config, diff --git a/counterparty-lib/counterpartylib/lib/messages/dividend.py b/counterparty-core/counterpartycore/lib/messages/dividend.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/messages/dividend.py rename to counterparty-core/counterpartycore/lib/messages/dividend.py index ec6a868aa2..eec5dd54c8 100644 --- a/counterparty-lib/counterpartylib/lib/messages/dividend.py +++ b/counterparty-core/counterpartycore/lib/messages/dividend.py @@ -9,7 +9,7 @@ D = decimal.Decimal import logging # noqa: E402 -from counterpartylib.lib import config, database, exceptions, ledger, message_type # noqa: E402 +from counterpartycore.lib import config, database, exceptions, ledger, message_type # noqa: E402 logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/messages/issuance.py b/counterparty-core/counterpartycore/lib/messages/issuance.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/messages/issuance.py rename to counterparty-core/counterpartycore/lib/messages/issuance.py index c0469aa0d2..91db18cc2f 100644 --- a/counterparty-lib/counterpartylib/lib/messages/issuance.py +++ b/counterparty-core/counterpartycore/lib/messages/issuance.py @@ -9,7 +9,7 @@ import logging import struct -from counterpartylib.lib import config, database, exceptions, ledger, message_type, util +from counterpartycore.lib import config, database, exceptions, ledger, message_type, util logger = logging.getLogger(config.LOGGER_NAME) D = decimal.Decimal @@ -345,7 +345,17 @@ def validate( ) -def compose(db, source, transfer_destination, asset, quantity, divisible, lock, reset, description): +def compose( + db, + source, + asset, + quantity, + transfer_destination=None, + divisible=None, + lock=None, + reset=None, + description=None, +): # Callability is deprecated, so for re‐issuances set relevant parameters # to old values; for first issuances, make uncallable. issuances = ledger.get_issuances(db, asset=asset, status="valid", first=True) diff --git a/counterparty-lib/counterpartylib/lib/messages/order.py b/counterparty-core/counterpartycore/lib/messages/order.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/messages/order.py rename to counterparty-core/counterpartycore/lib/messages/order.py index afafb3a804..6339b44438 100644 --- a/counterparty-lib/counterpartylib/lib/messages/order.py +++ b/counterparty-core/counterpartycore/lib/messages/order.py @@ -7,7 +7,7 @@ import logging import struct -from counterpartylib.lib import ( # noqa: F401 +from counterpartycore.lib import ( # noqa: F401 config, database, exceptions, diff --git a/counterparty-lib/counterpartylib/lib/messages/rps.py b/counterparty-core/counterpartycore/lib/messages/rps.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/messages/rps.py rename to counterparty-core/counterpartycore/lib/messages/rps.py index 0aba236df1..daf1f10b64 100644 --- a/counterparty-lib/counterpartylib/lib/messages/rps.py +++ b/counterparty-core/counterpartycore/lib/messages/rps.py @@ -25,7 +25,7 @@ import string import struct -from counterpartylib.lib import ( # noqa: F401 +from counterpartycore.lib import ( # noqa: F401 config, database, exceptions, diff --git a/counterparty-lib/counterpartylib/lib/messages/rpsresolve.py b/counterparty-core/counterpartycore/lib/messages/rpsresolve.py similarity index 98% rename from counterparty-lib/counterpartylib/lib/messages/rpsresolve.py rename to counterparty-core/counterpartycore/lib/messages/rpsresolve.py index a10fcc061f..9e4634ae2f 100644 --- a/counterparty-lib/counterpartylib/lib/messages/rpsresolve.py +++ b/counterparty-core/counterpartycore/lib/messages/rpsresolve.py @@ -5,7 +5,7 @@ import string import struct -from counterpartylib.lib import config, database, exceptions, ledger, message_type, util +from counterpartycore.lib import config, database, exceptions, ledger, message_type, util from . import rps diff --git a/counterparty-lib/counterpartylib/lib/messages/send.py b/counterparty-core/counterpartycore/lib/messages/send.py similarity index 98% rename from counterparty-lib/counterpartylib/lib/messages/send.py rename to counterparty-core/counterpartycore/lib/messages/send.py index 102485ad00..fbe0a3a96e 100644 --- a/counterparty-lib/counterpartylib/lib/messages/send.py +++ b/counterparty-core/counterpartycore/lib/messages/send.py @@ -1,7 +1,7 @@ #! /usr/bin/python3 -from counterpartylib.lib import config, database, exceptions, ledger, util -from counterpartylib.lib.messages.versions import enhanced_send, mpma, send1 +from counterpartycore.lib import config, database, exceptions, ledger, util +from counterpartycore.lib.messages.versions import enhanced_send, mpma, send1 ID = send1.ID diff --git a/counterparty-lib/counterpartylib/lib/messages/sweep.py b/counterparty-core/counterpartycore/lib/messages/sweep.py similarity index 98% rename from counterparty-lib/counterpartylib/lib/messages/sweep.py rename to counterparty-core/counterpartycore/lib/messages/sweep.py index 51fad57cdf..9ed81c7ae2 100644 --- a/counterparty-lib/counterpartylib/lib/messages/sweep.py +++ b/counterparty-core/counterpartycore/lib/messages/sweep.py @@ -3,8 +3,8 @@ import logging import struct -from counterpartylib.lib import address, config, database, exceptions, ledger, message_type -from counterpartylib.lib.exceptions import * # noqa: F403 +from counterpartycore.lib import address, config, database, exceptions, ledger, message_type +from counterpartycore.lib.exceptions import * # noqa: F403 logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/__init__.py b/counterparty-core/counterpartycore/lib/messages/versions/__init__.py similarity index 100% rename from counterparty-lib/counterpartylib/lib/__init__.py rename to counterparty-core/counterpartycore/lib/messages/versions/__init__.py diff --git a/counterparty-lib/counterpartylib/lib/messages/versions/enhanced_send.py b/counterparty-core/counterpartycore/lib/messages/versions/enhanced_send.py similarity index 98% rename from counterparty-lib/counterpartylib/lib/messages/versions/enhanced_send.py rename to counterparty-core/counterpartycore/lib/messages/versions/enhanced_send.py index d46e49d835..a7bbca7945 100644 --- a/counterparty-lib/counterpartylib/lib/messages/versions/enhanced_send.py +++ b/counterparty-core/counterpartycore/lib/messages/versions/enhanced_send.py @@ -4,7 +4,7 @@ import logging import struct -from counterpartylib.lib import address, config, exceptions, ledger, message_type, util +from counterpartycore.lib import address, config, exceptions, ledger, message_type, util logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/messages/versions/mpma.py b/counterparty-core/counterpartycore/lib/messages/versions/mpma.py similarity index 98% rename from counterparty-lib/counterpartylib/lib/messages/versions/mpma.py rename to counterparty-core/counterpartycore/lib/messages/versions/mpma.py index c6a145ccf4..95010bafd2 100644 --- a/counterparty-lib/counterpartylib/lib/messages/versions/mpma.py +++ b/counterparty-core/counterpartycore/lib/messages/versions/mpma.py @@ -11,7 +11,7 @@ from bitcoin.core import key # noqa: F401 from bitstring import ReadError -from counterpartylib.lib import config, exceptions, ledger, message_type, util +from counterpartycore.lib import config, exceptions, ledger, message_type, util from .mpma_util.internals import _decode_mpma_send_decode, _encode_mpma_send diff --git a/counterparty-lib/counterpartylib/lib/messages/versions/__init__.py b/counterparty-core/counterpartycore/lib/messages/versions/mpma_util/__init__.py similarity index 100% rename from counterparty-lib/counterpartylib/lib/messages/versions/__init__.py rename to counterparty-core/counterpartycore/lib/messages/versions/mpma_util/__init__.py diff --git a/counterparty-lib/counterpartylib/lib/messages/versions/mpma_util/internals.py b/counterparty-core/counterpartycore/lib/messages/versions/mpma_util/internals.py similarity index 98% rename from counterparty-lib/counterpartylib/lib/messages/versions/mpma_util/internals.py rename to counterparty-core/counterpartycore/lib/messages/versions/mpma_util/internals.py index 00e5569267..44133b498c 100644 --- a/counterparty-lib/counterpartylib/lib/messages/versions/mpma_util/internals.py +++ b/counterparty-core/counterpartycore/lib/messages/versions/mpma_util/internals.py @@ -9,7 +9,7 @@ from bitcoin.core import key # noqa: F401 from bitstring import BitArray, BitStream, ConstBitStream, ReadError # noqa: F401 -from counterpartylib.lib import address, config, exceptions, ledger, util # noqa: F401 +from counterpartycore.lib import address, config, exceptions, ledger, util # noqa: F401 logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/messages/versions/send1.py b/counterparty-core/counterpartycore/lib/messages/versions/send1.py similarity index 100% rename from counterparty-lib/counterpartylib/lib/messages/versions/send1.py rename to counterparty-core/counterpartycore/lib/messages/versions/send1.py diff --git a/counterparty-lib/counterpartylib/lib/opcodes.py b/counterparty-core/counterpartycore/lib/opcodes.py similarity index 100% rename from counterparty-lib/counterpartylib/lib/opcodes.py rename to counterparty-core/counterpartycore/lib/opcodes.py diff --git a/counterparty-lib/counterpartylib/lib/prefetcher.py b/counterparty-core/counterpartycore/lib/prefetcher.py similarity index 97% rename from counterparty-lib/counterpartylib/lib/prefetcher.py rename to counterparty-core/counterpartycore/lib/prefetcher.py index 8ddbbaa6e5..90bcc72632 100644 --- a/counterparty-lib/counterpartylib/lib/prefetcher.py +++ b/counterparty-core/counterpartycore/lib/prefetcher.py @@ -5,7 +5,7 @@ import bitcoin as bitcoinlib -from counterpartylib.lib import backend, config, ledger +from counterpartycore.lib import backend, config, ledger logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/script.py b/counterparty-core/counterpartycore/lib/script.py similarity index 98% rename from counterparty-lib/counterpartylib/lib/script.py rename to counterparty-core/counterpartycore/lib/script.py index bbd2f6306d..f8ae1364fe 100644 --- a/counterparty-lib/counterpartylib/lib/script.py +++ b/counterparty-core/counterpartycore/lib/script.py @@ -16,8 +16,8 @@ # from Crypto.Hash import RIPEMD160 from ripemd import ripemd160 as RIPEMD160 # nosec B413 -from counterpartylib.lib import config, exceptions, ledger, opcodes, util -from counterpartylib.lib.opcodes import * # noqa: F403 +from counterpartycore.lib import config, exceptions, ledger, opcodes, util +from counterpartycore.lib.opcodes import * # noqa: F403 B58_DIGITS = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" @@ -176,7 +176,7 @@ def base58_check_decode_py(s, version): def base58_check_decode(s, version): try: decoded = b58.b58_decode(s) - except BaseException: # TODO: update pycoin_rs to raise a specific exception + except ValueError: raise Base58Error("invalid base58 string") # noqa: B904 if decoded[0] != ord(version): diff --git a/counterparty-core/counterpartycore/lib/sentry.py b/counterparty-core/counterpartycore/lib/sentry.py new file mode 100644 index 0000000000..aecccb2671 --- /dev/null +++ b/counterparty-core/counterpartycore/lib/sentry.py @@ -0,0 +1,43 @@ +import os + +import sentry_sdk + +from counterpartycore.lib import config, database +from counterpartycore.lib.telemetry.collector import TelemetryCollectorLive + + +def send_exception(exception): + sentry_sdk.capture_exception(exception) + + +environment = os.environ.get("SENTRY_ENVIRONMENT", "development") + +release = os.environ.get("SENTRY_RELEASE", config.__version__) + + +def before_send(event, _hint): + db = database.get_connection(read_only=True) + + data = TelemetryCollectorLive(db).collect() + + event["tags"] = event.get("tags", {}) + + event["tags"]["core_version"] = data["version"] + event["tags"]["addrindexrs_version"] = data["addrindexrs_version"] + event["tags"]["docker"] = data["docker"] + event["tags"]["network"] = data["network"] + event["tags"]["force_enabled"] = data["force_enabled"] + + event["extra"] = event.get("extra", {}) + event["extra"]["last_block"] = data["last_block"] + + return event + + +sentry_sdk.init( + dsn=os.environ.get("SENTRY_DSN"), + environment=environment, + release=release, + traces_sample_rate=1.0, + before_send=before_send, +) diff --git a/counterparty-lib/counterpartylib/lib/setup.py b/counterparty-core/counterpartycore/lib/setup.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/setup.py rename to counterparty-core/counterpartycore/lib/setup.py index 8bed7e2290..e0d8be0a2a 100644 --- a/counterparty-lib/counterpartylib/lib/setup.py +++ b/counterparty-core/counterpartycore/lib/setup.py @@ -9,7 +9,7 @@ import appdirs -from counterpartylib.lib import config +from counterpartycore.lib import config logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/messages/versions/mpma_util/__init__.py b/counterparty-core/counterpartycore/lib/telemetry/__init__.py similarity index 100% rename from counterparty-lib/counterpartylib/lib/messages/versions/mpma_util/__init__.py rename to counterparty-core/counterpartycore/lib/telemetry/__init__.py diff --git a/counterparty-core/counterpartycore/lib/telemetry/client.py b/counterparty-core/counterpartycore/lib/telemetry/client.py new file mode 100644 index 0000000000..4a0ac3d385 --- /dev/null +++ b/counterparty-core/counterpartycore/lib/telemetry/client.py @@ -0,0 +1,16 @@ +import logging + + +# INTERFACE +class TelemetryClientI: + def send(self, data): + raise NotImplementedError() + + +# IMPLEMENTATIONS +class TelemetryClientLocal(TelemetryClientI): + def __init__(self): + self.logger = logging.getLogger(__name__) + + def send(self, data): + self.logger.info(data) diff --git a/counterparty-core/counterpartycore/lib/telemetry/collector.py b/counterparty-core/counterpartycore/lib/telemetry/collector.py new file mode 100644 index 0000000000..2d53b88eae --- /dev/null +++ b/counterparty-core/counterpartycore/lib/telemetry/collector.py @@ -0,0 +1,65 @@ +# INTERFACE +from counterpartycore.lib import config, blocks, ledger # noqa: I001, F401 +import os + +import counterpartycore.lib.telemetry.util as util + + +class TelemetryCollectorI: + def collect(self): + raise NotImplementedError() + + +# DEFAULT IMPLEMENTATION +class TelemetryCollectorBase(TelemetryCollectorI): + def __init__(self, **kwargs): + self.static_attrs = kwargs + + def collect(self): + return self.static_attrs + + +class TelemetryCollectorLive(TelemetryCollectorBase): + def __init__(self, db, **kwargs): + super().__init__(**kwargs) + self.db = db + + def collect(self): + version = util.get_version() + addrindexrs_version = util.get_addrindexrs_version() + uptime = util.get_uptime() + is_docker = util.is_docker() + network = util.get_network() + force_enabled = util.is_force_enabled() + + block_index = ledger.last_message(self.db)["block_index"] + cursor = self.db.cursor() + last_block = cursor.execute( + "SELECT * FROM blocks where block_index = ?", [block_index] + ).fetchone() + + return { + "version": version, + "addrindexrs_version": addrindexrs_version, + "uptime": int(uptime), + "dockerized": is_docker, + "network": network, + "force_enabled": force_enabled, + "last_block": last_block, + **self.static_attrs, + } + + def is_running_in_docker(self): + """ + Checks if the current process is running inside a Docker container. + Returns: + bool: True if running inside a Docker container, False otherwise. + """ + return ( + os.path.exists("/.dockerenv") + or "DOCKER_HOST" in os.environ + or "KUBERNETES_SERVICE_HOST" in os.environ + ) + + def __read_config_with_default(self, key, default): + return getattr(config, key, default) diff --git a/counterparty-core/counterpartycore/lib/telemetry/daemon.py b/counterparty-core/counterpartycore/lib/telemetry/daemon.py new file mode 100644 index 0000000000..3ca3077865 --- /dev/null +++ b/counterparty-core/counterpartycore/lib/telemetry/daemon.py @@ -0,0 +1,36 @@ +import threading # noqa: I001 +import time + +from .collector import TelemetryCollectorI +from .client import TelemetryClientI + +DEFAULT_INTERVAL = 60 + + +class TelemetryDaemon: + def __init__( + self, + collector: TelemetryCollectorI, + client: TelemetryClientI, + interval=DEFAULT_INTERVAL, + ): + self.thread = threading.Thread(target=self._run) + self.thread.daemon = True + self.client = client + self.collector = collector + self.interval = interval + self.is_running = False + + def start(self): + self.is_running = True + self.thread.start() + + def _run(self): + while self.is_running: + data = self.collector.collect() + self.client.send(data) + time.sleep(self.interval) + + def stop(self): + self.is_running = False + self.thread.join() diff --git a/counterparty-core/counterpartycore/lib/telemetry/util.py b/counterparty-core/counterpartycore/lib/telemetry/util.py new file mode 100644 index 0000000000..9d3619a07d --- /dev/null +++ b/counterparty-core/counterpartycore/lib/telemetry/util.py @@ -0,0 +1,43 @@ +import os +import time + +from counterpartycore.lib import config + +start_time = time.time() + + +def get_version(): + return config.__version__ + + +def get_addrindexrs_version(): + return config.ADDRINDEXRS_VERSION + + +def get_uptime(): + return time.time() - start_time + + +def is_docker(): + """ + Checks if the current process is running inside a Docker container. + Returns: + bool: True if running inside a Docker container, False otherwise. + """ + return ( + os.path.exists("/.dockerenv") + or "DOCKER_HOST" in os.environ + or "KUBERNETES_SERVICE_HOST" in os.environ + ) + + +def get_network(): + return "TESTNET" if __read_config_with_default("TESTNET", False) else "MAINNET" + + +def is_force_enabled(): + return __read_config_with_default("FORCE", False) + + +def __read_config_with_default(key, default): + return getattr(config, key, default) diff --git a/counterparty-lib/counterpartylib/lib/transaction.py b/counterparty-core/counterpartycore/lib/transaction.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/transaction.py rename to counterparty-core/counterpartycore/lib/transaction.py index 66595800fd..13819c7106 100644 --- a/counterparty-lib/counterpartylib/lib/transaction.py +++ b/counterparty-core/counterpartycore/lib/transaction.py @@ -23,7 +23,7 @@ from bitcoin.core import CTransaction, b2lx, x # noqa: F401 from bitcoin.core.script import CScript # noqa: F401 -from counterpartylib.lib import ( +from counterpartycore.lib import ( arc4, # noqa: F401 backend, blocks, # noqa: F401 @@ -34,8 +34,8 @@ script, util, ) -from counterpartylib.lib.kickstart.blocks_parser import BlockchainParser -from counterpartylib.lib.transaction_helper import p2sh_encoding, serializer +from counterpartycore.lib.kickstart.blocks_parser import BlockchainParser +from counterpartycore.lib.transaction_helper import p2sh_encoding, serializer # Constants OP_RETURN = b"\x6a" diff --git a/counterparty-lib/counterpartylib/lib/transaction_helper/__init__.py b/counterparty-core/counterpartycore/lib/transaction_helper/__init__.py similarity index 100% rename from counterparty-lib/counterpartylib/lib/transaction_helper/__init__.py rename to counterparty-core/counterpartycore/lib/transaction_helper/__init__.py diff --git a/counterparty-lib/counterpartylib/lib/transaction_helper/p2sh_encoding.py b/counterparty-core/counterpartycore/lib/transaction_helper/p2sh_encoding.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/transaction_helper/p2sh_encoding.py rename to counterparty-core/counterpartycore/lib/transaction_helper/p2sh_encoding.py index 546bb394c1..4e2ec40e1b 100644 --- a/counterparty-lib/counterpartylib/lib/transaction_helper/p2sh_encoding.py +++ b/counterparty-core/counterpartycore/lib/transaction_helper/p2sh_encoding.py @@ -11,7 +11,7 @@ import bitcoin as bitcoinlib from bitcoin.core.script import CScript -from counterpartylib.lib import config, exceptions, script +from counterpartycore.lib import config, exceptions, script logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/transaction_helper/serializer.py b/counterparty-core/counterpartycore/lib/transaction_helper/serializer.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/transaction_helper/serializer.py rename to counterparty-core/counterpartycore/lib/transaction_helper/serializer.py index f24f34d458..90009f49a0 100644 --- a/counterparty-lib/counterpartylib/lib/transaction_helper/serializer.py +++ b/counterparty-core/counterpartycore/lib/transaction_helper/serializer.py @@ -22,8 +22,8 @@ from bitcoin.core.script import CScript # noqa: F401 from bitcoin.wallet import P2PKHBitcoinAddress, P2SHBitcoinAddress # noqa: F401 -from counterpartylib.lib import arc4, backend, config, exceptions, script, util # noqa: F401 -from counterpartylib.lib.transaction_helper import p2sh_encoding +from counterpartycore.lib import arc4, backend, config, exceptions, script, util # noqa: F401 +from counterpartycore.lib.transaction_helper import p2sh_encoding logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/lib/util.py b/counterparty-core/counterpartycore/lib/util.py similarity index 99% rename from counterparty-lib/counterpartylib/lib/util.py rename to counterparty-core/counterpartycore/lib/util.py index 1700f03693..30c65e9681 100644 --- a/counterparty-lib/counterpartylib/lib/util.py +++ b/counterparty-core/counterpartycore/lib/util.py @@ -16,7 +16,7 @@ import requests -from counterpartylib.lib import config, exceptions +from counterpartycore.lib import config, exceptions logger = logging.getLogger(config.LOGGER_NAME) diff --git a/counterparty-lib/counterpartylib/mainnet_burns.csv b/counterparty-core/counterpartycore/mainnet_burns.csv similarity index 100% rename from counterparty-lib/counterpartylib/mainnet_burns.csv rename to counterparty-core/counterpartycore/mainnet_burns.csv diff --git a/counterparty-lib/counterpartylib/protocol_changes.json b/counterparty-core/counterpartycore/protocol_changes.json similarity index 100% rename from counterparty-lib/counterpartylib/protocol_changes.json rename to counterparty-core/counterpartycore/protocol_changes.json diff --git a/counterparty-core/counterpartycore/server.py b/counterparty-core/counterpartycore/server.py index e6a616a0f0..39ba41f79d 100755 --- a/counterparty-core/counterpartycore/server.py +++ b/counterparty-core/counterpartycore/server.py @@ -1,463 +1,788 @@ #! /usr/bin/env python3 -import argparse +import binascii +import decimal import logging +import os +import platform +import socket +import tarfile +import tempfile +import time +import urllib from urllib.parse import quote_plus as urlencode -from counterpartylib import server -from counterpartylib.lib import config, log, setup -from termcolor import cprint +import appdirs +import apsw +import bitcoin as bitcoinlib +from halo import Halo +from termcolor import colored, cprint + +from counterpartycore.lib import ( + api, + backend, + blocks, + check, + config, + database, + ledger, + transaction, + util, +) +from counterpartycore.lib import kickstart as kickstarter +from counterpartycore.lib.telemetry.client import TelemetryClientLocal +from counterpartycore.lib.telemetry.collector import TelemetryCollectorLive +from counterpartycore.lib.telemetry.daemon import TelemetryDaemon logger = logging.getLogger(config.LOGGER_NAME) +D = decimal.Decimal -APP_NAME = "counterparty-server" -APP_VERSION = config.VERSION_STRING - -CONFIG_ARGS = [ - [ - ("-v", "--verbose"), - { - "dest": "verbose", - "action": "store_true", - "default": False, - "help": "sets log level to DEBUG", - }, - ], - [ - ("--quiet",), - { - "dest": "quiet", - "action": "store_true", - "default": False, - "help": "sets log level to ERROR", - }, - ], - [ - ("--mainnet",), - { - "action": "store_true", - "default": True, - "help": f"use {config.BTC_NAME} mainet addresses and block numbers", - }, - ], - [ - ("--testnet",), - { - "action": "store_true", - "default": False, - "help": f"use {config.BTC_NAME} testnet addresses and block numbers", - }, - ], - [ - ("--testcoin",), - { - "action": "store_true", - "default": False, - "help": f"use the test {config.XCP_NAME} network on every blockchain", - }, - ], - [ - ("--regtest",), - { - "action": "store_true", - "default": False, - "help": f"use {config.BTC_NAME} regtest addresses and block numbers", - }, - ], - [ - ("--customnet",), - { - "default": "", - "help": "use a custom network (specify as UNSPENDABLE_ADDRESS|ADDRESSVERSION|P2SH_ADDRESSVERSION with version bytes in HH hex format)", - }, - ], - [ - ("--api-limit-rows",), - { - "type": int, - "default": 1000, - "help": "limit api calls to the set results (defaults to 1000). Setting to 0 removes the limit.", - }, - ], - [("--backend-name",), {"default": "addrindex", "help": "the backend name to connect to"}], - [ - ("--backend-connect",), - {"default": "localhost", "help": "the hostname or IP of the backend server"}, - ], - [("--backend-port",), {"type": int, "help": "the backend port to connect to"}], - [ - ("--backend-user",), - {"default": "rpc", "help": "the username used to communicate with backend"}, - ], - [ - ("--backend-password",), - {"default": "rpc", "help": "the password used to communicate with backend"}, - ], - [ - ("--backend-ssl",), - { - "action": "store_true", - "default": False, - "help": "use SSL to connect to backend (default: false)", - }, - ], - [ - ("--backend-ssl-no-verify",), - { - "action": "store_true", - "default": False, - "help": "verify SSL certificate of backend; disallow use of self‐signed certificates (default: true)", - }, - ], - [ - ("--backend-poll-interval",), - {"type": float, "default": 0.5, "help": "poll interval, in seconds (default: 0.5)"}, - ], - [ - ("--no-check-asset-conservation",), - { - "action": "store_true", - "default": False, - "help": "Skip asset conservation checking (default: false)", - }, - ], - [ - ("--p2sh-dust-return-pubkey",), - { - "help": "pubkey to receive dust when multisig encoding is used for P2SH source (default: none)" - }, - ], - [ - ("--indexd-connect",), - {"default": "localhost", "help": "the hostname or IP of the indexd server"}, - ], - [("--indexd-port",), {"type": int, "help": "the indexd server port to connect to"}], - [ - ("--rpc-host",), - { - "default": "localhost", - "help": "the IP of the interface to bind to for providing JSON-RPC API access (0.0.0.0 for all interfaces)", - }, - ], - [ - ("--rpc-port",), - {"type": int, "help": f"port on which to provide the {config.APP_NAME} JSON-RPC API"}, - ], - [ - ("--rpc-user",), - { - "default": "rpc", - "help": f"required username to use the {config.APP_NAME} JSON-RPC API (via HTTP basic auth)", - }, - ], - [ - ("--rpc-password",), - { - "default": "rpc", - "help": f"required password (for rpc-user) to use the {config.APP_NAME} JSON-RPC API (via HTTP basic auth)", - }, - ], - [ - ("--rpc-no-allow-cors",), - {"action": "store_true", "default": False, "help": "allow ajax cross domain request"}, - ], - [ - ("--rpc-batch-size",), - { - "type": int, - "default": config.DEFAULT_RPC_BATCH_SIZE, - "help": f"number of RPC queries by batch (default: {config.DEFAULT_RPC_BATCH_SIZE})", - }, - ], - [ - ("--requests-timeout",), - { - "type": int, - "default": config.DEFAULT_REQUESTS_TIMEOUT, - "help": "timeout value (in seconds) used for all HTTP requests (default: 5)", - }, - ], - [ - ("--force",), - { - "action": "store_true", - "default": False, - "help": "skip backend check, version check, process lock (NOT FOR USE ON PRODUCTION SYSTEMS)", - }, - ], - [ - ("--no-confirm",), - {"action": "store_true", "default": False, "help": "don't ask for confirmation"}, - ], - [("--database-file",), {"default": None, "help": "the path to the SQLite3 database file"}], - [ - ("--log-file",), - {"nargs": "?", "const": None, "default": False, "help": "log to the specified file"}, - ], - [ - ("--api-log-file",), - { - "nargs": "?", - "const": None, - "default": False, - "help": "log API requests to the specified file", - }, - ], - [ - ("--no-log-files",), - {"action": "store_true", "default": False, "help": "Don't write log files"}, - ], - [ - ("--json-log",), - {"action": "store_true", "default": False, "help": "Log events in JSON format"}, - ], - [ - ("--utxo-locks-max-addresses",), - { - "type": int, - "default": config.DEFAULT_UTXO_LOCKS_MAX_ADDRESSES, - "help": "max number of addresses for which to track UTXO locks", - }, - ], - [ - ("--utxo-locks-max-age",), - { - "type": int, - "default": config.DEFAULT_UTXO_LOCKS_MAX_AGE, - "help": "how long to keep a lock on a UTXO being tracked", - }, - ], - [ - ("--no-mempool",), - {"action": "store_true", "default": False, "help": "Disable mempool parsing"}, - ], - [ - ("--skip-db-check",), - {"action": "store_true", "default": False, "help": "Skip integrity check on the database"}, - ], -] - - -def welcome_message(action, server_configfile): - cprint(f"Running v{config.__version__} of {config.FULL_APP_NAME}.", "magenta") - - # print some info - cprint(f"Configuration file: {server_configfile}", "light_grey") - cprint(f"Counterparty database: {config.DATABASE}", "light_grey") - if config.LOG: - cprint(f"Writing log to file: `{config.LOG}`", "light_grey") +OK_GREEN = colored("[OK]", "green") +SPINNER_STYLE = "bouncingBar" + + +class ConfigurationError(Exception): + pass + + +# Lock database access by opening a socket. +class LockingError(Exception): + pass + + +def get_lock(): + logger.info("Acquiring lock.") + + # Cross‐platform. + if platform.system() == "Darwin": # Windows or OS X + # Not database‐specific. + socket_family = socket.AF_INET + socket_address = ("localhost", 8999) + error = "Another copy of server is currently running." + else: + socket_family = socket.AF_UNIX + socket_address = "\0" + config.DATABASE + error = f"Another copy of server is currently writing to database {config.DATABASE}" + + lock_socket = socket.socket(socket_family, socket.SOCK_DGRAM) + try: + lock_socket.bind(socket_address) + except socket.error: + raise LockingError(error) # noqa: B904 + logger.info("Lock acquired.") + + +def initialise(*args, **kwargs): + initialise_log_config( + verbose=kwargs.pop("verbose", False), + quiet=kwargs.pop("quiet", False), + log_file=kwargs.pop("log_file", None), + api_log_file=kwargs.pop("api_log_file", None), + no_log_files=kwargs.pop("no_log_files", False), + testnet=kwargs.get("testnet", False), + testcoin=kwargs.get("testcoin", False), + regtest=kwargs.get("regtest", False), + ) + initialise_config(*args, **kwargs) + return initialise_db() + + +def initialise_log_config( + verbose=False, + quiet=False, + log_file=None, + api_log_file=None, + no_log_files=False, + testnet=False, + testcoin=False, + regtest=False, + json_log=False, +): + # Log directory + log_dir = appdirs.user_log_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME) + if not os.path.isdir(log_dir): + os.makedirs(log_dir, mode=0o755) + + # Set up logging. + config.VERBOSE = verbose + config.QUIET = quiet + + network = "" + if testnet: + network += ".testnet" + if regtest: + network += ".regtest" + if testcoin: + network += ".testcoin" + + # Log + if no_log_files: + config.LOG = None + elif not log_file: # default location + filename = f"server{network}.log" + config.LOG = os.path.join(log_dir, filename) + else: # user-specified location + config.LOG = log_file + + if no_log_files: # no file logging + config.API_LOG = None + elif not api_log_file: # default location + filename = f"server{network}.access.log" + config.API_LOG = os.path.join(log_dir, filename) + else: # user-specified location + config.API_LOG = api_log_file + + config.JSON_LOG = json_log + + +def initialise_config( + database_file=None, + testnet=False, + testcoin=False, + regtest=False, + api_limit_rows=1000, + backend_connect=None, + backend_port=None, + backend_user=None, + backend_password=None, + indexd_connect=None, + indexd_port=None, + backend_ssl=False, + backend_ssl_no_verify=False, + backend_poll_interval=None, + rpc_host=None, + rpc_port=None, + rpc_user=None, + rpc_password=None, + rpc_no_allow_cors=False, + force=False, + requests_timeout=config.DEFAULT_REQUESTS_TIMEOUT, + rpc_batch_size=config.DEFAULT_RPC_BATCH_SIZE, + check_asset_conservation=False, + backend_ssl_verify=None, + rpc_allow_cors=None, + p2sh_dust_return_pubkey=None, + utxo_locks_max_addresses=config.DEFAULT_UTXO_LOCKS_MAX_ADDRESSES, + utxo_locks_max_age=config.DEFAULT_UTXO_LOCKS_MAX_AGE, + estimate_fee_per_kb=None, + customnet=None, + no_mempool=False, +): + # log config alreasdy initialized + logger.debug("VERBOSE: %s", config.VERBOSE) + logger.debug("QUIET: %s", config.QUIET) + logger.debug("LOG: %s", config.LOG) + logger.debug("API_LOG: %s", config.API_LOG) + + # Data directory + data_dir = appdirs.user_data_dir( + appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True + ) + if not os.path.isdir(data_dir): + os.makedirs(data_dir, mode=0o755) + + # testnet + if testnet: + config.TESTNET = testnet else: - cprint("Warning: log disabled", "yellow") - if config.API_LOG: - cprint(f"Writing API accesses log to file: `{config.API_LOG}`", "light_grey") + config.TESTNET = False + + # testcoin + if testcoin: + config.TESTCOIN = testcoin + else: + config.TESTCOIN = False + + # regtest + if regtest: + config.REGTEST = regtest + else: + config.REGTEST = False + + if customnet != None and len(customnet) > 0: # noqa: E711 + config.CUSTOMNET = True + config.REGTEST = True # Custom nets are regtests with different parameters + else: + config.CUSTOMNET = False + + if config.TESTNET: + bitcoinlib.SelectParams("testnet") + logger.debug("NETWORK: testnet") + elif config.REGTEST: + bitcoinlib.SelectParams("regtest") + logger.debug("NETWORK: regtest") + else: + bitcoinlib.SelectParams("mainnet") + logger.debug("NETWORK: mainnet") + + network = "" + if config.TESTNET: + network += ".testnet" + if config.REGTEST: + network += ".regtest" + if config.TESTCOIN: + network += ".testcoin" + + # Database + if database_file: + config.DATABASE = database_file + else: + filename = f"{config.APP_NAME}{network}.db" + config.DATABASE = os.path.join(data_dir, filename) + + logger.debug("DATABASE: %s", config.DATABASE) + + config.API_LIMIT_ROWS = api_limit_rows + + ############## + # THINGS WE CONNECT TO + + # Backend name + config.BACKEND_NAME = "addrindexrs" + + # Backend RPC host (Bitcoin Core) + if backend_connect: + config.BACKEND_CONNECT = backend_connect else: - cprint("Warning: API log disabled", "yellow") + config.BACKEND_CONNECT = "localhost" - if config.VERBOSE: + # Backend Core RPC port (Bitcoin Core) + if backend_port: + config.BACKEND_PORT = backend_port + else: if config.TESTNET: - cprint("NETWORK: Testnet", "light_grey") + config.BACKEND_PORT = config.DEFAULT_BACKEND_PORT_TESTNET elif config.REGTEST: - cprint("NETWORK: Regtest", "light_grey") + config.BACKEND_PORT = config.DEFAULT_BACKEND_PORT_REGTEST else: - cprint("NETWORK: Mainnet", "light_grey") + config.BACKEND_PORT = config.DEFAULT_BACKEND_PORT + + try: + config.BACKEND_PORT = int(config.BACKEND_PORT) + if not (int(config.BACKEND_PORT) > 1 and int(config.BACKEND_PORT) < 65535): + raise ConfigurationError("invalid backend API port number") + except: # noqa: E722 + raise ConfigurationError( # noqa: B904 + "Please specific a valid port number backend-port configuration parameter" + ) - pass_str = f":{urlencode(config.BACKEND_PASSWORD)}@" - cleaned_backend_url = config.BACKEND_URL.replace(pass_str, ":*****@") - cprint(f"BACKEND_URL: {cleaned_backend_url}", "light_grey") - cprint(f"INDEXD_URL: {config.INDEXD_URL}", "light_grey") - pass_str = f":{urlencode(config.RPC_PASSWORD)}@" - cleaned_rpc_url = config.RPC.replace(pass_str, ":*****@") - cprint(f"RPC: {cleaned_rpc_url}", "light_grey") + # Backend Core RPC user (Bitcoin Core) + if backend_user: + config.BACKEND_USER = backend_user + else: + config.BACKEND_USER = "rpc" - cprint(f"{'-' * 30} {action} {'-' * 30}\n", "green") + # Backend Core RPC password (Bitcoin Core) + if backend_password: + config.BACKEND_PASSWORD = backend_password + else: + raise ConfigurationError( + "Please specific a valid password backend-password configuration parameter" + ) + # Backend Core RPC SSL + if backend_ssl: + config.BACKEND_SSL = backend_ssl + else: + config.BACKEND_SSL = False # Default to off. -class VersionError(Exception): - pass + # Backend Core RPC SSL Verify + if backend_ssl_verify is not None: + cprint( + "The server parameter `backend_ssl_verify` is deprecated. Use `backend_ssl_no_verify` instead.", + "yellow", + ) + config.BACKEND_SSL_NO_VERIFY = not backend_ssl_verify + else: + if backend_ssl_no_verify: + config.BACKEND_SSL_NO_VERIFY = backend_ssl_no_verify + else: + config.BACKEND_SSL_NO_VERIFY = ( + False # Default to on (don't support self‐signed certificates) + ) + # Backend Poll Interval + if backend_poll_interval: + config.BACKEND_POLL_INTERVAL = backend_poll_interval + else: + config.BACKEND_POLL_INTERVAL = 3.0 + + # Construct backend URL. + config.BACKEND_URL = ( + config.BACKEND_USER + + ":" + + config.BACKEND_PASSWORD + + "@" + + config.BACKEND_CONNECT + + ":" + + str(config.BACKEND_PORT) + ) + if config.BACKEND_SSL: + config.BACKEND_URL = "https://" + config.BACKEND_URL + else: + config.BACKEND_URL = "http://" + config.BACKEND_URL -def main(): - # Post installation tasks - server_configfile = setup.generate_server_config_file(CONFIG_ARGS) + cleaned_backend_url = config.BACKEND_URL.replace(f":{config.BACKEND_PASSWORD}@", ":*****@") + logger.debug("BACKEND_URL: %s", cleaned_backend_url) - # Parse command-line arguments. - parser = argparse.ArgumentParser( - prog=APP_NAME, - description=f"Server for the {config.XCP_NAME} protocol", - add_help=False, - exit_on_error=False, - ) - parser.add_argument( - "-h", "--help", dest="help", action="store_true", help="show this help message and exit" - ) - parser.add_argument( - "-V", - "--version", - action="version", - version=f"{APP_NAME} v{APP_VERSION}; counterparty-lib v{config.VERSION_STRING}", - ) - parser.add_argument("--config-file", help="the path to the configuration file") + # Indexd RPC host + if indexd_connect: + config.INDEXD_CONNECT = indexd_connect + else: + config.INDEXD_CONNECT = "localhost" - cmd_args = parser.parse_known_args()[0] - config_file_path = getattr(cmd_args, "config_file", None) - configfile = setup.read_config_file("server.conf", config_file_path) + # Indexd RPC port + if indexd_port: + config.INDEXD_PORT = indexd_port + else: + if config.TESTNET: + config.INDEXD_PORT = config.DEFAULT_INDEXD_PORT_TESTNET + elif config.REGTEST: + config.INDEXD_PORT = config.DEFAULT_INDEXD_PORT_REGTEST + else: + config.INDEXD_PORT = config.DEFAULT_INDEXD_PORT + + try: + config.INDEXD_PORT = int(config.INDEXD_PORT) + if not (int(config.INDEXD_PORT) > 1 and int(config.INDEXD_PORT) < 65535): + raise ConfigurationError("invalid Indexd API port number") + except: # noqa: E722 + raise ConfigurationError( # noqa: B904 + "Please specific a valid port number indexd-port configuration parameter" + ) - setup.add_config_arguments(parser, CONFIG_ARGS, configfile, add_default=True) + # Construct Indexd URL. + config.INDEXD_URL = "http://" + config.INDEXD_CONNECT + ":" + str(config.INDEXD_PORT) - subparsers = parser.add_subparsers(dest="action", help="the action to be taken") + logger.debug("INDEXD_URL: %s", config.INDEXD_URL) - parser_server = subparsers.add_parser("start", help="run the server") - parser_server.add_argument("--config-file", help="the path to the configuration file") - parser_server.add_argument( - "--catch-up", - choices=["normal", "bootstrap"], - default="normal", - help="Catch up mode (default: normal)", - ) - setup.add_config_arguments(parser_server, CONFIG_ARGS, configfile) + ############## + # THINGS WE SERVE - parser_reparse = subparsers.add_parser( - "reparse", help="reparse all transactions in the database" - ) - parser_reparse.add_argument( - "block_index", type=int, help="the index of the last known good block" - ) - setup.add_config_arguments(parser_reparse, CONFIG_ARGS, configfile) + # Server API RPC host + if rpc_host: + config.RPC_HOST = rpc_host + else: + config.RPC_HOST = "localhost" - parser_vacuum = subparsers.add_parser( - "vacuum", help="VACUUM the database (to improve performance)" - ) - setup.add_config_arguments(parser_vacuum, CONFIG_ARGS, configfile) + # The web root directory for API calls, eg. localhost:14000/rpc/ + config.RPC_WEBROOT = "/rpc/" - parser_rollback = subparsers.add_parser("rollback", help="rollback database") - parser_rollback.add_argument( - "block_index", type=int, help="the index of the last known good block" - ) - setup.add_config_arguments(parser_rollback, CONFIG_ARGS, configfile) + # Server API RPC port + if rpc_port: + config.RPC_PORT = rpc_port + else: + if config.TESTNET: + if config.TESTCOIN: + config.RPC_PORT = config.DEFAULT_RPC_PORT_TESTNET + 1 + else: + config.RPC_PORT = config.DEFAULT_RPC_PORT_TESTNET + elif config.REGTEST: + if config.TESTCOIN: + config.RPC_PORT = config.DEFAULT_RPC_PORT_REGTEST + 1 + else: + config.RPC_PORT = config.DEFAULT_RPC_PORT_REGTEST + else: + if config.TESTCOIN: + config.RPC_PORT = config.DEFAULT_RPC_PORT + 1 + else: + config.RPC_PORT = config.DEFAULT_RPC_PORT + try: + config.RPC_PORT = int(config.RPC_PORT) + if not (int(config.RPC_PORT) > 1 and int(config.RPC_PORT) < 65535): + raise ConfigurationError("invalid server API port number") + except: # noqa: E722 + raise ConfigurationError( # noqa: B904 + "Please specific a valid port number rpc-port configuration parameter" + ) - parser_kickstart = subparsers.add_parser( - "kickstart", help="rapidly build database by reading from Bitcoin Core blockchain" - ) - parser_kickstart.add_argument("--bitcoind-dir", help="Bitcoin Core data directory") - parser_kickstart.add_argument( - "--max-queue-size", type=int, help="Size of the multiprocessing.Queue for parsing blocks" - ) - parser_kickstart.add_argument( - "--debug-block", type=int, help="Rollback and run kickstart for a single block;" - ) - setup.add_config_arguments(parser_kickstart, CONFIG_ARGS, configfile) + # Server API RPC user + if rpc_user: + config.RPC_USER = rpc_user + else: + config.RPC_USER = "rpc" - parser_bootstrap = subparsers.add_parser( - "bootstrap", help="bootstrap database with hosted snapshot" - ) - setup.add_config_arguments(parser_bootstrap, CONFIG_ARGS, configfile) + configure_rpc(rpc_password) - parser_checkdb = subparsers.add_parser("check-db", help="do an integrity check on the database") - setup.add_config_arguments(parser_checkdb, CONFIG_ARGS, configfile) + # RPC CORS + if rpc_allow_cors is not None: + cprint( + "The server parameter `rpc_allow_cors` is deprecated. Use `rpc_no_allow_cors` instead.", + "yellow", + ) + config.RPC_NO_ALLOW_CORS = not rpc_allow_cors + else: + if rpc_no_allow_cors: + config.RPC_NO_ALLOW_CORS = rpc_no_allow_cors + else: + config.RPC_NO_ALLOW_CORS = False - parser_show_config = subparsers.add_parser( - "show-params", help="Show counterparty-server configuration" - ) - setup.add_config_arguments(parser_show_config, CONFIG_ARGS, configfile) - - args = parser.parse_args() - - # Help message - if args.help: - parser.print_help() - exit(0) - - # Configuration - init_args = dict( - database_file=args.database_file, - testnet=args.testnet, - testcoin=args.testcoin, - regtest=args.regtest, - customnet=args.customnet, - api_limit_rows=args.api_limit_rows, - backend_connect=args.backend_connect, - backend_port=args.backend_port, - backend_user=args.backend_user, - backend_password=args.backend_password, - backend_ssl=args.backend_ssl, - backend_ssl_no_verify=args.backend_ssl_no_verify, - backend_poll_interval=args.backend_poll_interval, - indexd_connect=args.indexd_connect, - indexd_port=args.indexd_port, - rpc_host=args.rpc_host, - rpc_port=args.rpc_port, - rpc_user=args.rpc_user, - rpc_password=args.rpc_password, - rpc_no_allow_cors=args.rpc_no_allow_cors, - requests_timeout=args.requests_timeout, - rpc_batch_size=args.rpc_batch_size, - check_asset_conservation=not args.no_check_asset_conservation, - force=args.force, - p2sh_dust_return_pubkey=args.p2sh_dust_return_pubkey, - utxo_locks_max_addresses=args.utxo_locks_max_addresses, - utxo_locks_max_age=args.utxo_locks_max_age, - no_mempool=args.no_mempool, - skip_db_check=args.skip_db_check, - ) + config.RPC_BATCH_SIZE = rpc_batch_size - server.initialise_log_config( - verbose=args.verbose, - quiet=args.quiet, - log_file=args.log_file, - api_log_file=args.api_log_file, - no_log_files=args.no_log_files, - testnet=args.testnet, - testcoin=args.testcoin, - regtest=args.regtest, - json_log=args.json_log, - ) + ############## + # OTHER SETTINGS - # set up logging - log.set_up( - verbose=config.VERBOSE, - quiet=config.QUIET, - log_file=config.LOG, - log_in_console=args.action == "start", - ) + # skip checks + if force: + config.FORCE = force + else: + config.FORCE = False + + # Encoding + if config.TESTCOIN: + config.PREFIX = b"XX" # 2 bytes (possibly accidentally created) + else: + config.PREFIX = b"CNTRPRTY" # 8 bytes + + # (more) Testnet + if config.TESTNET: + config.MAGIC_BYTES = config.MAGIC_BYTES_TESTNET + if config.TESTCOIN: + config.ADDRESSVERSION = config.ADDRESSVERSION_TESTNET + config.P2SH_ADDRESSVERSION = config.P2SH_ADDRESSVERSION_TESTNET + config.BLOCK_FIRST = config.BLOCK_FIRST_TESTNET_TESTCOIN + config.BURN_START = config.BURN_START_TESTNET_TESTCOIN + config.BURN_END = config.BURN_END_TESTNET_TESTCOIN + config.UNSPENDABLE = config.UNSPENDABLE_TESTNET + config.P2SH_DUST_RETURN_PUBKEY = p2sh_dust_return_pubkey + else: + config.ADDRESSVERSION = config.ADDRESSVERSION_TESTNET + config.P2SH_ADDRESSVERSION = config.P2SH_ADDRESSVERSION_TESTNET + config.BLOCK_FIRST = config.BLOCK_FIRST_TESTNET + config.BURN_START = config.BURN_START_TESTNET + config.BURN_END = config.BURN_END_TESTNET + config.UNSPENDABLE = config.UNSPENDABLE_TESTNET + config.P2SH_DUST_RETURN_PUBKEY = p2sh_dust_return_pubkey + elif config.CUSTOMNET: + custom_args = customnet.split("|") + + if len(custom_args) == 3: + config.MAGIC_BYTES = config.MAGIC_BYTES_REGTEST + config.ADDRESSVERSION = binascii.unhexlify(custom_args[1]) + config.P2SH_ADDRESSVERSION = binascii.unhexlify(custom_args[2]) + config.BLOCK_FIRST = config.BLOCK_FIRST_REGTEST + config.BURN_START = config.BURN_START_REGTEST + config.BURN_END = config.BURN_END_REGTEST + config.UNSPENDABLE = custom_args[0] + config.P2SH_DUST_RETURN_PUBKEY = p2sh_dust_return_pubkey + else: + raise "Custom net parameter needs to be like UNSPENDABLE_ADDRESS|ADDRESSVERSION|P2SH_ADDRESSVERSION (version bytes in HH format)" # noqa: B016 + elif config.REGTEST: + config.MAGIC_BYTES = config.MAGIC_BYTES_REGTEST + if config.TESTCOIN: + config.ADDRESSVERSION = config.ADDRESSVERSION_REGTEST + config.P2SH_ADDRESSVERSION = config.P2SH_ADDRESSVERSION_REGTEST + config.BLOCK_FIRST = config.BLOCK_FIRST_REGTEST_TESTCOIN + config.BURN_START = config.BURN_START_REGTEST_TESTCOIN + config.BURN_END = config.BURN_END_REGTEST_TESTCOIN + config.UNSPENDABLE = config.UNSPENDABLE_REGTEST + config.P2SH_DUST_RETURN_PUBKEY = p2sh_dust_return_pubkey + else: + config.ADDRESSVERSION = config.ADDRESSVERSION_REGTEST + config.P2SH_ADDRESSVERSION = config.P2SH_ADDRESSVERSION_REGTEST + config.BLOCK_FIRST = config.BLOCK_FIRST_REGTEST + config.BURN_START = config.BURN_START_REGTEST + config.BURN_END = config.BURN_END_REGTEST + config.UNSPENDABLE = config.UNSPENDABLE_REGTEST + config.P2SH_DUST_RETURN_PUBKEY = p2sh_dust_return_pubkey + else: + config.MAGIC_BYTES = config.MAGIC_BYTES_MAINNET + if config.TESTCOIN: + config.ADDRESSVERSION = config.ADDRESSVERSION_MAINNET + config.P2SH_ADDRESSVERSION = config.P2SH_ADDRESSVERSION_MAINNET + config.BLOCK_FIRST = config.BLOCK_FIRST_MAINNET_TESTCOIN + config.BURN_START = config.BURN_START_MAINNET_TESTCOIN + config.BURN_END = config.BURN_END_MAINNET_TESTCOIN + config.UNSPENDABLE = config.UNSPENDABLE_MAINNET + config.P2SH_DUST_RETURN_PUBKEY = p2sh_dust_return_pubkey + else: + config.ADDRESSVERSION = config.ADDRESSVERSION_MAINNET + config.P2SH_ADDRESSVERSION = config.P2SH_ADDRESSVERSION_MAINNET + config.BLOCK_FIRST = config.BLOCK_FIRST_MAINNET + config.BURN_START = config.BURN_START_MAINNET + config.BURN_END = config.BURN_END_MAINNET + config.UNSPENDABLE = config.UNSPENDABLE_MAINNET + config.P2SH_DUST_RETURN_PUBKEY = p2sh_dust_return_pubkey - server.initialise_config(**init_args) + # Misc + config.REQUESTS_TIMEOUT = requests_timeout + config.CHECK_ASSET_CONSERVATION = check_asset_conservation + config.UTXO_LOCKS_MAX_ADDRESSES = utxo_locks_max_addresses + config.UTXO_LOCKS_MAX_AGE = utxo_locks_max_age - logger.info(f"Running v{APP_VERSION} of {APP_NAME}.") + if estimate_fee_per_kb is not None: + config.ESTIMATE_FEE_PER_KB = estimate_fee_per_kb - welcome_message(args.action, server_configfile) + config.NO_MEMPOOL = no_mempool - # Bootstrapping - if args.action == "bootstrap": - server.bootstrap(no_confirm=args.no_confirm) + logger.info(f"Running v{config.VERSION_STRING} of counterparty-core.") - # PARSING - elif args.action == "reparse": - server.reparse(block_index=args.block_index) - elif args.action == "rollback": - server.rollback(block_index=args.block_index) +def initialise_db(): + if config.FORCE: + cprint("THE OPTION `--force` IS NOT FOR USE ON PRODUCTION SYSTEMS.", "yellow") + + # Lock + if not config.FORCE: + get_lock() + + # Database + logger.info(f"Connecting to database (SQLite {apsw.apswversion()}).") + db = database.get_connection(read_only=False) + + ledger.CURRENT_BLOCK_INDEX = blocks.last_db_index(db) + + return db + + +def connect_to_backend(): + if not config.FORCE: + backend.getblockcount() + + +def connect_to_addrindexrs(): + step = "Connecting to `addrindexrs`..." + with Halo(text=step, spinner=SPINNER_STYLE): + ledger.CURRENT_BLOCK_INDEX = 0 + backend.backend() + check_addrindexrs = {} + while check_addrindexrs == {}: + check_address = ( + "mrHFGUKSiNMeErqByjX97qPKfumdZxe6mC" + if config.TESTNET + else "1GsjsKKT4nH4GPmDnaxaZEDWgoBpmexwMA" + ) + check_addrindexrs = backend.get_oldest_tx(check_address, 99999999999) + if check_addrindexrs == {}: + logger.info("`addrindexrs` is not ready. Waiting one second.") + time.sleep(1) + print(f"{OK_GREEN} {step}") - elif args.action == "kickstart": - server.kickstart( - bitcoind_dir=args.bitcoind_dir, - force=args.force, - max_queue_size=args.max_queue_size, - debug_block=args.debug_block, - ) - elif args.action == "start": - server.start_all(catch_up=args.catch_up) +def start_all(catch_up="normal"): + api_status_poller, api_server, db = None, None, None - elif args.action == "show-params": - server.show_params() + try: + # Backend. + connect_to_backend() - elif args.action == "vacuum": - server.vacuum() + if not os.path.exists(config.DATABASE) and catch_up == "bootstrap": + bootstrap(no_confirm=True) - elif args.action == "check-db": - server.check_database() + db = initialise_db() + blocks.initialise(db) + + telemetry_daemon = TelemetryDaemon( + interval=60, + collector=TelemetryCollectorLive(db=database.get_connection(read_only=True)), + client=TelemetryClientLocal(), + ) + + telemetry_daemon.start() + + # Reset UTXO_LOCKS. This previously was done in + # initilise_config + transaction.initialise() + + # API Status Poller. + api_status_poller = api.APIStatusPoller() + api_status_poller.daemon = True + api_status_poller.start() + + # API Server. + api_server = api.APIServer() + api_server.daemon = True + api_server.start() + + # Server + blocks.follow(db) + except KeyboardInterrupt: + pass + finally: + if api_status_poller: + api_status_poller.stop() + if api_server: + api_server.stop() + backend.stop() + if db: + database.optimize(db) + logger.info("Closing database...") + db.close() + logger.info("Shutting down logging...") + logging.shutdown() + + +def reparse(block_index): + connect_to_addrindexrs() + db = initialise_db() + try: + blocks.reparse(db, block_index=block_index) + finally: + backend.stop() + database.optimize(db) + db.close() + + +def rollback(block_index=None): + db = initialise_db() + try: + blocks.rollback(db, block_index=block_index) + finally: + database.optimize(db) + db.close() + + +def kickstart(bitcoind_dir, force=False, max_queue_size=None, debug_block=None): + kickstarter.run( + bitcoind_dir=bitcoind_dir, + force=force, + max_queue_size=max_queue_size, + debug_block=debug_block, + ) + + +def vacuum(): + db = initialise_db() + step = "Vacuuming database..." + with Halo(text=step, spinner=SPINNER_STYLE): + database.vacuum(db) + print(f"{OK_GREEN} {step}") + + +def check_database(): + db = initialise_db() + + start_all_time = time.time() + + start_time = time.time() + step = "Checking asset conservation..." + with Halo(text=step, spinner=SPINNER_STYLE): + check.asset_conservation(db) + print(f"{OK_GREEN} {step} (in {time.time() - start_time:.2f}s)") + + start_time = time.time() + step = "Checking database foreign keys...." + with Halo(text=step, spinner=SPINNER_STYLE): + database.check_foreign_keys(db) + print(f"{OK_GREEN} {step} (in {time.time() - start_time:.2f}s)") + + start_time = time.time() + step = "Checking database integrity..." + with Halo(text=step, spinner=SPINNER_STYLE): + database.intergrity_check(db) + print(f"{OK_GREEN} {step} (in {time.time() - start_time:.2f}s)") + + cprint(f"Database checks complete in {time.time() - start_all_time:.2f}s.", "green") + + +def show_params(): + output = vars(config) + for k in list(output.keys()): + if k.isupper(): + print(f"{k}: {output[k]}") + + +def generate_move_random_hash(move): + move = int(move).to_bytes(2, byteorder="big") + random_bin = os.urandom(16) + move_random_hash_bin = util.dhash(random_bin + move) + return binascii.hexlify(random_bin).decode("utf8"), binascii.hexlify( + move_random_hash_bin + ).decode("utf8") + + +def configure_rpc(rpc_password=None): + # Server API RPC password + if rpc_password: + config.RPC_PASSWORD = rpc_password + config.API_ROOT = ( + "http://" + + urlencode(config.RPC_USER) + + ":" + + urlencode(config.RPC_PASSWORD) + + "@" + + config.RPC_HOST + + ":" + + str(config.RPC_PORT) + ) else: - parser.print_help() + config.API_ROOT = "http://" + config.RPC_HOST + ":" + str(config.RPC_PORT) + config.RPC = config.API_ROOT + config.RPC_WEBROOT + + cleaned_rpc_url = config.RPC.replace(f":{urlencode(config.RPC_PASSWORD)}@", ":*****@") + logger.debug("RPC: %s", cleaned_rpc_url) + +def bootstrap(no_confirm=False): + warning_message = """WARNING: `counterparty-server bootstrap` downloads a recent snapshot of a Counterparty database +from a centralized server maintained by the Counterparty Core development team. +Because this method does not involve verifying the history of Counterparty transactions yourself, +the `bootstrap` command should not be used for mission-critical, commercial or public-facing nodes. + """ + cprint(warning_message, "yellow") + + if not no_confirm: + confirmation_message = colored("Continue? (y/N): ", "magenta") + if input(confirmation_message).lower() != "y": + exit() + + data_dir = appdirs.user_data_dir( + appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True + ) -# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 + # Set Constants. + bootstrap_url = config.BOOTSTRAP_URL_TESTNET if config.TESTNET else config.BOOTSTRAP_URL_MAINNET + tar_filename = os.path.basename(bootstrap_url) + tarball_path = os.path.join(tempfile.gettempdir(), tar_filename) + database_path = os.path.join(data_dir, config.APP_NAME) + if config.TESTNET: + database_path += ".testnet" + database_path += ".db" + + # Prepare Directory. + if not os.path.exists(data_dir): + os.makedirs(data_dir, mode=0o755) + if os.path.exists(database_path): + os.remove(database_path) + # Delete SQLite Write-Ahead-Log + wal_path = database_path + "-wal" + shm_path = database_path + "-shm" + if os.path.exists(wal_path): + os.remove(wal_path) + if os.path.exists(shm_path): + os.remove(shm_path) + + # Define Progress Bar. + step = f"Downloading database from {bootstrap_url}..." + spinner = Halo(text=step, spinner=SPINNER_STYLE) + + def bootstrap_progress(blocknum, blocksize, totalsize): + readsofar = blocknum * blocksize + if totalsize > 0: + percent = readsofar * 1e2 / totalsize + message = f"Downloading database: {percent:5.1f}% {readsofar} / {totalsize}" + spinner.text = message + + # Downloading + spinner.start() + urllib.request.urlretrieve(bootstrap_url, tarball_path, bootstrap_progress) # nosec B310 # noqa: S310 + spinner.stop() + print(f"{OK_GREEN} {step}") + + # TODO: check checksum, filenames, etc. + step = f"Extracting database to {data_dir}..." + with Halo(text=step, spinner=SPINNER_STYLE): + with tarfile.open(tarball_path, "r:gz") as tar_file: + tar_file.extractall(path=data_dir) # nosec B202 # noqa: S202 + print(f"{OK_GREEN} {step}") + + assert os.path.exists(database_path) + # user and group have "rw" access + os.chmod(database_path, 0o660) # nosec B103 + + step = "Cleaning up..." + with Halo(text=step, spinner=SPINNER_STYLE): + os.remove(tarball_path) + print(f"{OK_GREEN} {step}") + + cprint(f"Database has been successfully bootstrapped to {database_path}.", "green") diff --git a/counterparty-lib/counterpartylib/test/__init__.py b/counterparty-core/counterpartycore/test/__init__.py similarity index 100% rename from counterparty-lib/counterpartylib/test/__init__.py rename to counterparty-core/counterpartycore/test/__init__.py diff --git a/counterparty-lib/counterpartylib/test/arc4_test.py b/counterparty-core/counterpartycore/test/arc4_test.py similarity index 94% rename from counterparty-lib/counterpartylib/test/arc4_test.py rename to counterparty-core/counterpartycore/test/arc4_test.py index 2926fada16..ba0d7e09c7 100644 --- a/counterparty-lib/counterpartylib/test/arc4_test.py +++ b/counterparty-core/counterpartycore/test/arc4_test.py @@ -3,16 +3,16 @@ import pytest -from counterpartylib.lib import arc4, backend, transaction, util # noqa: F401 -from counterpartylib.lib.messages import send +from counterpartycore.lib import arc4, backend, transaction, util # noqa: F401 +from counterpartycore.lib.messages import send # this is require near the top to do setup of the test suite -from counterpartylib.test import ( +from counterpartycore.test import ( conftest, # noqa: F401 util_test, ) -from counterpartylib.test.fixtures.params import ADDR, DP # noqa: F401 -from counterpartylib.test.util_test import CURR_DIR +from counterpartycore.test.fixtures.params import ADDR, DP # noqa: F401 +from counterpartycore.test.util_test import CURR_DIR FIXTURE_SQL_FILE = CURR_DIR + "/fixtures/scenarios/unittest_fixture.sql" FIXTURE_DB = tempfile.gettempdir() + "/fixtures.unittest_fixture.db" diff --git a/counterparty-lib/counterpartylib/test/book_test.py b/counterparty-core/counterpartycore/test/book_test.py similarity index 88% rename from counterparty-lib/counterpartylib/test/book_test.py rename to counterparty-core/counterpartycore/test/book_test.py index d6771540ee..297c57b799 100644 --- a/counterparty-lib/counterpartylib/test/book_test.py +++ b/counterparty-core/counterpartycore/test/book_test.py @@ -3,7 +3,7 @@ import pytest # this is require near the top to do setup of the test suite -from counterpartylib.test import ( +from counterpartycore.test import ( conftest, # noqa: F401 util_test, ) diff --git a/counterparty-lib/counterpartylib/test/bytespersigop_test.py b/counterparty-core/counterpartycore/test/bytespersigop_test.py similarity index 92% rename from counterparty-lib/counterpartylib/test/bytespersigop_test.py rename to counterparty-core/counterpartycore/test/bytespersigop_test.py index 19fd451747..25e336538c 100644 --- a/counterparty-lib/counterpartylib/test/bytespersigop_test.py +++ b/counterparty-core/counterpartycore/test/bytespersigop_test.py @@ -5,16 +5,16 @@ import bitcoin as bitcoinlib import pytest # noqa: F401 -from counterpartylib.lib import api, blocks, exceptions, ledger, transaction, util # noqa: F401 -from counterpartylib.test import ( +from counterpartycore.lib import api, blocks, exceptions, ledger, transaction, util # noqa: F401 +from counterpartycore.test import ( conftest, # noqa: F401 util_test, ) -from counterpartylib.test.fixtures.params import ADDR +from counterpartycore.test.fixtures.params import ADDR # this is require near the top to do setup of the test suite -from counterpartylib.test.fixtures.params import DEFAULT_PARAMS as DP # noqa: F401 -from counterpartylib.test.util_test import CURR_DIR +from counterpartycore.test.fixtures.params import DEFAULT_PARAMS as DP # noqa: F401 +from counterpartycore.test.util_test import CURR_DIR FIXTURE_SQL_FILE = CURR_DIR + "/fixtures/scenarios/unittest_fixture.sql" FIXTURE_DB = tempfile.gettempdir() + "/fixtures.unittest_fixture.db" diff --git a/counterparty-lib/counterpartylib/test/complex_unit_test.py b/counterparty-core/counterpartycore/test/complex_unit_test.py similarity index 98% rename from counterparty-lib/counterpartylib/test/complex_unit_test.py rename to counterparty-core/counterpartycore/test/complex_unit_test.py index 3f8603dfe6..62087416a3 100644 --- a/counterparty-lib/counterpartylib/test/complex_unit_test.py +++ b/counterparty-core/counterpartycore/test/complex_unit_test.py @@ -6,15 +6,15 @@ import requests from apsw import ConstraintError -from counterpartylib.lib import api, blocks, config, ledger, util +from counterpartycore.lib import api, blocks, config, ledger, util # this is require near the top to do setup of the test suite -from counterpartylib.test import ( +from counterpartycore.test import ( conftest, # noqa: F401 util_test, ) -from counterpartylib.test.fixtures.params import ADDR, DP # noqa: F401 -from counterpartylib.test.util_test import CURR_DIR +from counterpartycore.test.fixtures.params import ADDR, DP # noqa: F401 +from counterpartycore.test.util_test import CURR_DIR FIXTURE_SQL_FILE = CURR_DIR + "/fixtures/scenarios/unittest_fixture.sql" FIXTURE_DB = tempfile.gettempdir() + "/fixtures.unittest_fixture.db" diff --git a/counterparty-lib/counterpartylib/test/config_context_test.py b/counterparty-core/counterpartycore/test/config_context_test.py similarity index 84% rename from counterparty-lib/counterpartylib/test/config_context_test.py rename to counterparty-core/counterpartycore/test/config_context_test.py index af3d2b9bc8..3033168208 100644 --- a/counterparty-lib/counterpartylib/test/config_context_test.py +++ b/counterparty-core/counterpartycore/test/config_context_test.py @@ -2,15 +2,15 @@ import pprint # noqa: F401 import tempfile -from counterpartylib.lib import blocks, config, ledger # noqa: F401 -from counterpartylib.test import ( +from counterpartycore.lib import blocks, config, ledger # noqa: F401 +from counterpartycore.test import ( conftest, # noqa: F401 util_test, ) # this is require near the top to do setup of the test suite -from counterpartylib.test.fixtures.params import DEFAULT_PARAMS as DP # noqa: F401 -from counterpartylib.test.util_test import CURR_DIR +from counterpartycore.test.fixtures.params import DEFAULT_PARAMS as DP # noqa: F401 +from counterpartycore.test.util_test import CURR_DIR FIXTURE_SQL_FILE = CURR_DIR + "/fixtures/scenarios/parseblock_unittest_fixture.sql" FIXTURE_DB = tempfile.gettempdir() + "/fixtures.parseblock_unittest_fixture.db" diff --git a/counterparty-lib/counterpartylib/test/conftest.py b/counterparty-core/counterpartycore/test/conftest.py similarity index 92% rename from counterparty-lib/counterpartylib/test/conftest.py rename to counterparty-core/counterpartycore/test/conftest.py index a0ac913b92..1073c4e90c 100644 --- a/counterparty-lib/counterpartylib/test/conftest.py +++ b/counterparty-core/counterpartycore/test/conftest.py @@ -17,12 +17,12 @@ from Crypto.Cipher import ARC4 from pycoin.coins.bitcoin import Tx # noqa: F401 -from counterpartylib import server -from counterpartylib.lib import api, arc4, config, database, ledger, log, script, util -from counterpartylib.test import util_test -from counterpartylib.test.fixtures.params import DEFAULT_PARAMS -from counterpartylib.test.fixtures.scenarios import INTEGRATION_SCENARIOS -from counterpartylib.test.fixtures.vectors import UNITTEST_VECTOR +from counterpartycore import server +from counterpartycore.lib import api, arc4, config, database, ledger, log, script, util +from counterpartycore.test import util_test +from counterpartycore.test.fixtures.params import DEFAULT_PARAMS +from counterpartycore.test.fixtures.scenarios import INTEGRATION_SCENARIOS +from counterpartycore.test.fixtures.vectors import UNITTEST_VECTOR logger = logging.getLogger(config.LOGGER_NAME) @@ -449,23 +449,23 @@ def init_arc4(seed): else: return ARC4.new(binascii.unhexlify("00" * 32)) - monkeypatch.setattr("counterpartylib.lib.transaction.arc4.init_arc4", init_arc4) - monkeypatch.setattr("counterpartylib.lib.backend.get_unspent_txouts", get_unspent_txouts) - monkeypatch.setattr("counterpartylib.lib.log.isodt", isodt) - monkeypatch.setattr("counterpartylib.lib.ledger.curr_time", curr_time) - monkeypatch.setattr("counterpartylib.lib.util.date_passed", date_passed) - monkeypatch.setattr("counterpartylib.lib.api.init_api_access_log", init_api_access_log) + monkeypatch.setattr("counterpartycore.lib.transaction.arc4.init_arc4", init_arc4) + monkeypatch.setattr("counterpartycore.lib.backend.get_unspent_txouts", get_unspent_txouts) + monkeypatch.setattr("counterpartycore.lib.log.isodt", isodt) + monkeypatch.setattr("counterpartycore.lib.ledger.curr_time", curr_time) + monkeypatch.setattr("counterpartycore.lib.util.date_passed", date_passed) + monkeypatch.setattr("counterpartycore.lib.api.init_api_access_log", init_api_access_log) if hasattr(config, "PREFIX"): - monkeypatch.setattr("counterpartylib.lib.config.PREFIX", b"TESTXXXX") - monkeypatch.setattr("counterpartylib.lib.backend.getrawtransaction", mocked_getrawtransaction) + monkeypatch.setattr("counterpartycore.lib.config.PREFIX", b"TESTXXXX") + monkeypatch.setattr("counterpartycore.lib.backend.getrawtransaction", mocked_getrawtransaction) monkeypatch.setattr( - "counterpartylib.lib.backend.getrawtransaction_batch", mocked_getrawtransaction_batch + "counterpartycore.lib.backend.getrawtransaction_batch", mocked_getrawtransaction_batch ) monkeypatch.setattr( - "counterpartylib.lib.backend.search_raw_transactions", mocked_search_raw_transactions + "counterpartycore.lib.backend.search_raw_transactions", mocked_search_raw_transactions ) - monkeypatch.setattr("counterpartylib.lib.backend.pubkeyhash_to_pubkey", pubkeyhash_to_pubkey) + monkeypatch.setattr("counterpartycore.lib.backend.pubkeyhash_to_pubkey", pubkeyhash_to_pubkey) monkeypatch.setattr( - "counterpartylib.lib.backend.multisig_pubkeyhashes_to_pubkeys", + "counterpartycore.lib.backend.multisig_pubkeyhashes_to_pubkeys", multisig_pubkeyhashes_to_pubkeys, ) diff --git a/counterparty-lib/counterpartylib/test/database_version_test.py b/counterparty-core/counterpartycore/test/database_version_test.py similarity index 84% rename from counterparty-lib/counterpartylib/test/database_version_test.py rename to counterparty-core/counterpartycore/test/database_version_test.py index 4bf12c5b34..2662c9bc93 100644 --- a/counterparty-lib/counterpartylib/test/database_version_test.py +++ b/counterparty-core/counterpartycore/test/database_version_test.py @@ -3,15 +3,15 @@ import pytest -from counterpartylib import server -from counterpartylib.lib import check, config, database +from counterpartycore import server +from counterpartycore.lib import check, config, database # this is require near the top to do setup of the test suite -from counterpartylib.test import ( +from counterpartycore.test import ( conftest, # noqa: F401 util_test, ) -from counterpartylib.test.util_test import CURR_DIR +from counterpartycore.test.util_test import CURR_DIR def test_check_database_version(): diff --git a/counterparty-lib/counterpartylib/test/deserialize_test.py b/counterparty-core/counterpartycore/test/deserialize_test.py similarity index 98% rename from counterparty-lib/counterpartylib/test/deserialize_test.py rename to counterparty-core/counterpartycore/test/deserialize_test.py index a1c5f946ac..050d8c7a6e 100644 --- a/counterparty-lib/counterpartylib/test/deserialize_test.py +++ b/counterparty-core/counterpartycore/test/deserialize_test.py @@ -1,7 +1,7 @@ import time -from counterpartylib.lib import backend -from counterpartylib.lib.kickstart import blocks_parser, utils +from counterpartycore.lib import backend +from counterpartycore.lib.kickstart import blocks_parser, utils def test_deserialize(): diff --git a/counterparty-lib/counterpartylib/test/estimate_fee_per_kb_test.py b/counterparty-core/counterpartycore/test/estimate_fee_per_kb_test.py similarity index 82% rename from counterparty-lib/counterpartylib/test/estimate_fee_per_kb_test.py rename to counterparty-core/counterpartycore/test/estimate_fee_per_kb_test.py index 12d8a1036b..5a05f50b7d 100644 --- a/counterparty-lib/counterpartylib/test/estimate_fee_per_kb_test.py +++ b/counterparty-core/counterpartycore/test/estimate_fee_per_kb_test.py @@ -4,15 +4,15 @@ import bitcoin as bitcoinlib -from counterpartylib.lib import api, backend, transaction -from counterpartylib.test import ( +from counterpartycore.lib import api, backend, transaction +from counterpartycore.test import ( util_test, ) -from counterpartylib.test.fixtures.params import ADDR +from counterpartycore.test.fixtures.params import ADDR # this is require near the top to do setup of the test suite -from counterpartylib.test.fixtures.params import DEFAULT_PARAMS as DP # noqa: F401 -from counterpartylib.test.util_test import CURR_DIR +from counterpartycore.test.fixtures.params import DEFAULT_PARAMS as DP # noqa: F401 +from counterpartycore.test.util_test import CURR_DIR FIXTURE_SQL_FILE = CURR_DIR + "/fixtures/scenarios/unittest_fixture.sql" FIXTURE_DB = tempfile.gettempdir() + "/fixtures.unittest_fixture.db" @@ -32,7 +32,7 @@ def test_estimate_fee_per_kb(fee_per_kb, fee_per_kb_used, server_db, monkeypatch def _fee_per_kb(conf_target, mode): return fee_per_kb - monkeypatch.setattr("counterpartylib.lib.backend.fee_per_kb", _fee_per_kb) + monkeypatch.setattr("counterpartycore.lib.backend.fee_per_kb", _fee_per_kb) utxos = dict( ((utxo["txid"], utxo["vout"]), utxo) for utxo in backend.get_unspent_txouts(ADDR[0]) diff --git a/counterparty-lib/counterpartylib/test/fixtures/__init__.py b/counterparty-core/counterpartycore/test/fixtures/__init__.py similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/__init__.py rename to counterparty-core/counterpartycore/test/fixtures/__init__.py diff --git a/counterparty-lib/counterpartylib/test/fixtures/params.py b/counterparty-core/counterpartycore/test/fixtures/params.py similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/params.py rename to counterparty-core/counterpartycore/test/fixtures/params.py diff --git a/counterparty-lib/counterpartylib/test/fixtures/rawtransactions.db b/counterparty-core/counterpartycore/test/fixtures/rawtransactions.db similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/rawtransactions.db rename to counterparty-core/counterpartycore/test/fixtures/rawtransactions.db diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios.py b/counterparty-core/counterpartycore/test/fixtures/scenarios.py similarity index 92% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios.py rename to counterparty-core/counterpartycore/test/fixtures/scenarios.py index 8c32a7ac02..d04f4ed995 100644 --- a/counterparty-lib/counterpartylib/test/fixtures/scenarios.py +++ b/counterparty-core/counterpartycore/test/fixtures/scenarios.py @@ -10,9 +10,9 @@ You need to do this every time you update UNITTEST_FIXTURE. ``` -mv counterpartylib/test/fixtures/scenarios/unittest_fixture.new.json counterpartylib/test/fixtures/scenarios/unittest_fixture.json -mv counterpartylib/test/fixtures/scenarios/unittest_fixture.new.sql counterpartylib/test/fixtures/scenarios/unittest_fixture.sql -mv counterpartylib/test/fixtures/scenarios/unittest_fixture.new.log counterpartylib/test/fixtures/scenarios/unittest_fixture.log +mv counterpartycore/test/fixtures/scenarios/unittest_fixture.new.json counterpartycore/test/fixtures/scenarios/unittest_fixture.json +mv counterpartycore/test/fixtures/scenarios/unittest_fixture.new.sql counterpartycore/test/fixtures/scenarios/unittest_fixture.sql +mv counterpartycore/test/fixtures/scenarios/unittest_fixture.new.log counterpartycore/test/fixtures/scenarios/unittest_fixture.log ``` Before every entry in UNITTEST_FIXTURE is executed a block is inserted first, so each of them has a +1 block_index. @@ -34,25 +34,25 @@ ["burn", (ADDR[0], DP["burn_quantity"]), {"encoding": "multisig"}], # 310000 [ "issuance", - (ADDR[0], None, "DIVISIBLE", DP["quantity"] * 1000, True, None, None, "Divisible asset"), + (ADDR[0], "DIVISIBLE", DP["quantity"] * 1000, None, True, None, None, "Divisible asset"), {"encoding": "multisig"}, ], [ "issuance", - (ADDR[0], None, "NODIVISIBLE", 1000, False, None, None, "No divisible asset"), + (ADDR[0], "NODIVISIBLE", 1000, None, False, None, None, "No divisible asset"), {"encoding": "multisig"}, ], [ "issuance", - (ADDR[0], None, "CALLABLE", 1000, True, None, None, "Callable asset"), + (ADDR[0], "CALLABLE", 1000, None, True, None, None, "Callable asset"), {"encoding": "multisig"}, ], [ "issuance", - (ADDR[0], None, "LOCKED", 1000, True, None, None, "Locked asset"), + (ADDR[0], "LOCKED", 1000, None, True, None, None, "Locked asset"), {"encoding": "multisig"}, ], - ["issuance", (ADDR[0], None, "LOCKED", 0, True, None, None, "LOCK"), {"encoding": "multisig"}], + ["issuance", (ADDR[0], "LOCKED", 0, None, True, None, None, "LOCK"), {"encoding": "multisig"}], [ "order", (ADDR[0], "XCP", DP["quantity"], "DIVISIBLE", DP["quantity"], 2000, 0), @@ -94,7 +94,7 @@ ["send", (ADDR[0], MULTISIGADDR[0], "NODIVISIBLE", 10), {"encoding": "multisig"}, None], [ "issuance", - (ADDR[0], None, "MAXI", 2**63 - 1, True, None, None, "Maximum quantity"), + (ADDR[0], "MAXI", 2**63 - 1, None, True, None, None, "Maximum quantity"), {"encoding": "multisig"}, ], [ @@ -120,7 +120,7 @@ ["burn", (P2SH_ADDR[0], int(DP["burn_quantity"] / 2)), {"encoding": "opreturn"}], [ "issuance", - (P2SH_ADDR[0], None, "PAYTOSCRIPT", 1000, False, None, None, "PSH issued asset"), + (P2SH_ADDR[0], "PAYTOSCRIPT", 1000, None, False, None, None, "PSH issued asset"), {"encoding": "multisig", "dust_return_pubkey": False}, ], ["send", (ADDR[0], P2SH_ADDR[0], "DIVISIBLE", DP["quantity"]), {"encoding": "multisig"}, None], @@ -137,17 +137,17 @@ # locked with an issuance after lock [ "issuance", - (ADDR[6], None, "LOCKEDPREV", 1000, True, None, None, "Locked asset"), + (ADDR[6], "LOCKEDPREV", 1000, None, True, None, None, "Locked asset"), {"encoding": "multisig"}, ], [ "issuance", - (ADDR[6], None, "LOCKEDPREV", 0, True, None, None, "LOCK"), + (ADDR[6], "LOCKEDPREV", 0, None, True, None, None, "LOCK"), {"encoding": "multisig"}, ], [ "issuance", - (ADDR[6], None, "LOCKEDPREV", 0, True, None, None, "changed"), + (ADDR[6], "LOCKEDPREV", 0, None, True, None, None, "changed"), {"encoding": "multisig"}, ], ["burn", (P2WPKH_ADDR[0], DP["burn_quantity"]), {"encoding": "opreturn"}], @@ -209,23 +209,23 @@ ["burn", (ADDR[2], DP["burn_quantity"]), {"encoding": "multisig"}], [ "issuance", - (ADDR[2], None, "DIVIDEND", 100, True, "Test dividend", None, None), + (ADDR[2], "DIVIDEND", 100, None, True, "Test dividend", None, None), {"encoding": "multisig"}, ], ["send", (ADDR[2], ADDR[3], "DIVIDEND", 10), {"encoding": "multisig"}, None], ["send", (ADDR[2], ADDR[3], "XCP", 92945878046), {"encoding": "multisig"}, None], [ "issuance", - (ADDR[0], None, "PARENT", DP["quantity"] * 1, True, None, None, "Parent asset"), + (ADDR[0], "PARENT", DP["quantity"] * 1, None, True, None, None, "Parent asset"), {"encoding": "opreturn"}, ], [ "issuance", ( ADDR[0], - None, "PARENT.already.issued", DP["quantity"] * 1, + None, True, None, None, @@ -267,12 +267,12 @@ def generate_standard_scenario(address1, address2, order_matches): ["btcpay", (address1, order_matches[0]), {"encoding": "multisig"}], [ "issuance", - (address1, None, "BBBB", DP["quantity"] * 10, True, None, None, ""), + (address1, "BBBB", DP["quantity"] * 10, None, True, None, None, ""), {"encoding": "multisig"}, ], [ "issuance", - (address1, None, "BBBC", round(DP["quantity"] / 1000), False, None, None, "foobar"), + (address1, "BBBC", round(DP["quantity"] / 1000), None, False, None, None, "foobar"), {"encoding": "multisig"}, ], [ diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_1_of_2.json b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_2.json similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_1_of_2.json rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_2.json diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_1_of_2.log b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_2.log similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_1_of_2.log rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_2.log diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_1_of_2.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_2.sql similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_1_of_2.sql rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_2.sql diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_1_of_3.json b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_3.json similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_1_of_3.json rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_3.json diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_1_of_3.log b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_3.log similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_1_of_3.log rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_3.log diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_1_of_3.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_3.sql similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_1_of_3.sql rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_1_of_3.sql diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_2_of_2.json b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_2.json similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_2_of_2.json rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_2.json diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_2_of_2.log b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_2.log similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_2_of_2.log rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_2.log diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_2_of_2.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_2.sql similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_2_of_2.sql rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_2.sql diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_2_of_3.json b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_3.json similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_2_of_3.json rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_3.json diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_2_of_3.log b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_3.log similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_2_of_3.log rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_3.log diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_2_of_3.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_3.sql similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_2_of_3.sql rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_2_of_3.sql diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_3_of_3.json b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_3_of_3.json similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_3_of_3.json rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_3_of_3.json diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_3_of_3.log b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_3_of_3.log similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_3_of_3.log rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_3_of_3.log diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_3_of_3.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_3_of_3.sql similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/multisig_3_of_3.sql rename to counterparty-core/counterpartycore/test/fixtures/scenarios/multisig_3_of_3.sql diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/parseblock_unittest_fixture.json b/counterparty-core/counterpartycore/test/fixtures/scenarios/parseblock_unittest_fixture.json similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/parseblock_unittest_fixture.json rename to counterparty-core/counterpartycore/test/fixtures/scenarios/parseblock_unittest_fixture.json diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/parseblock_unittest_fixture.log b/counterparty-core/counterpartycore/test/fixtures/scenarios/parseblock_unittest_fixture.log similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/parseblock_unittest_fixture.log rename to counterparty-core/counterpartycore/test/fixtures/scenarios/parseblock_unittest_fixture.log diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/parseblock_unittest_fixture.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/parseblock_unittest_fixture.sql similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/parseblock_unittest_fixture.sql rename to counterparty-core/counterpartycore/test/fixtures/scenarios/parseblock_unittest_fixture.sql diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/simplesig.json b/counterparty-core/counterpartycore/test/fixtures/scenarios/simplesig.json similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/simplesig.json rename to counterparty-core/counterpartycore/test/fixtures/scenarios/simplesig.json diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/simplesig.log b/counterparty-core/counterpartycore/test/fixtures/scenarios/simplesig.log similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/simplesig.log rename to counterparty-core/counterpartycore/test/fixtures/scenarios/simplesig.log diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/simplesig.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/simplesig.sql similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/simplesig.sql rename to counterparty-core/counterpartycore/test/fixtures/scenarios/simplesig.sql diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/unittest_fixture.json b/counterparty-core/counterpartycore/test/fixtures/scenarios/unittest_fixture.json similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/unittest_fixture.json rename to counterparty-core/counterpartycore/test/fixtures/scenarios/unittest_fixture.json diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/unittest_fixture.log b/counterparty-core/counterpartycore/test/fixtures/scenarios/unittest_fixture.log similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/unittest_fixture.log rename to counterparty-core/counterpartycore/test/fixtures/scenarios/unittest_fixture.log diff --git a/counterparty-lib/counterpartylib/test/fixtures/scenarios/unittest_fixture.sql b/counterparty-core/counterpartycore/test/fixtures/scenarios/unittest_fixture.sql similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/scenarios/unittest_fixture.sql rename to counterparty-core/counterpartycore/test/fixtures/scenarios/unittest_fixture.sql diff --git a/counterparty-lib/counterpartylib/test/fixtures/unspent_outputs.json b/counterparty-core/counterpartycore/test/fixtures/unspent_outputs.json similarity index 100% rename from counterparty-lib/counterpartylib/test/fixtures/unspent_outputs.json rename to counterparty-core/counterpartycore/test/fixtures/unspent_outputs.json diff --git a/counterparty-lib/counterpartylib/test/fixtures/vectors.py b/counterparty-core/counterpartycore/test/fixtures/vectors.py similarity index 99% rename from counterparty-lib/counterpartylib/test/fixtures/vectors.py rename to counterparty-core/counterpartycore/test/fixtures/vectors.py index 3fbca51dc9..15a554dc60 100644 --- a/counterparty-lib/counterpartylib/test/fixtures/vectors.py +++ b/counterparty-core/counterpartycore/test/fixtures/vectors.py @@ -14,12 +14,12 @@ import bitcoin as bitcoinlib -from counterpartylib.lib import address, config, exceptions, script # noqa: F401 -from counterpartylib.lib.api import APIError -from counterpartylib.lib.kickstart.blocks_parser import BlockchainParser -from counterpartylib.lib.ledger import CreditError, DebitError -from counterpartylib.lib.messages import issuance -from counterpartylib.lib.util import QuantityError, RPCError +from counterpartycore.lib import address, config, exceptions, script # noqa: F401 +from counterpartycore.lib.api import APIError +from counterpartycore.lib.kickstart.blocks_parser import BlockchainParser +from counterpartycore.lib.ledger import CreditError, DebitError +from counterpartycore.lib.messages import issuance +from counterpartycore.lib.util import QuantityError, RPCError from .params import ( ADDR, @@ -3874,19 +3874,19 @@ # compose (db, source, transfer_destination, asset, quantity, divisible, lock, reset=None, description=None) "compose": [ { - "in": (ADDR[0], None, "ASSET", 1000, True, False, None, ""), + "in": (ADDR[0], "ASSET", 1000, None, True, False, None, ""), "error": (exceptions.AssetNameError, "non‐numeric asset name starts with ‘A’"), }, { - "in": (ADDR[0], None, "BSSET1", 1000, True, False, None, ""), + "in": (ADDR[0], "BSSET1", 1000, None, True, False, None, ""), "error": (exceptions.AssetNameError, "invalid character:"), }, { - "in": (ADDR[0], None, "SET", 1000, True, False, None, ""), + "in": (ADDR[0], "SET", 1000, None, True, False, None, ""), "error": (exceptions.AssetNameError, "too short"), }, { - "in": (ADDR[0], None, "BSSET", 1000, True, False, None, ""), + "in": (ADDR[0], "BSSET", 1000, None, True, False, None, ""), "out": ( "mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc", [], @@ -3894,7 +3894,7 @@ ), }, { - "in": (ADDR[0], None, "BASSET", 1000, True, False, None, ""), + "in": (ADDR[0], "BASSET", 1000, None, True, False, None, ""), "out": ( "mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc", [], @@ -3902,7 +3902,7 @@ ), }, { - "in": (P2SH_ADDR[0], None, "BSSET", 1000, True, False, None, ""), + "in": (P2SH_ADDR[0], "BSSET", 1000, None, True, False, None, ""), "out": ( P2SH_ADDR[0], [], @@ -3912,9 +3912,9 @@ { "in": ( ADDR[0], - None, "BSSET", 1000, + None, True, False, None, @@ -3927,7 +3927,7 @@ ), }, { - "in": (ADDR[0], ADDR[1], "DIVISIBLE", 0, True, False, None, ""), + "in": (ADDR[0], "DIVISIBLE", 0, ADDR[1], True, False, None, ""), "out": ( "mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc", [("mtQheFaSfWELRB2MyMBaiWjdDm6ux9Ezns", None)], @@ -3935,7 +3935,7 @@ ), }, { - "in": (MULTISIGADDR[0], None, "BSSET", 1000, True, False, None, ""), + "in": (MULTISIGADDR[0], "BSSET", 1000, None, True, False, None, ""), "out": ( "1_mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc_mtQheFaSfWELRB2MyMBaiWjdDm6ux9Ezns_2", [], @@ -3943,7 +3943,7 @@ ), }, { - "in": (ADDR[0], MULTISIGADDR[0], "DIVISIBLE", 0, True, False, None, ""), + "in": (ADDR[0], "DIVISIBLE", 0, MULTISIGADDR[0], True, False, None, ""), "out": ( "mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc", [ @@ -3956,7 +3956,7 @@ ), }, { - "in": (ADDR[0], None, "MAXIMUM", 2**63 - 1, True, False, None, "Maximum quantity"), + "in": (ADDR[0], "MAXIMUM", 2**63 - 1, None, True, False, None, "Maximum quantity"), "out": ( "mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc", [], @@ -3964,7 +3964,7 @@ ), }, { - "in": (ADDR[0], None, f"A{2 ** 64 - 1}", 1000, None, False, None, None), + "in": (ADDR[0], f"A{2 ** 64 - 1}", 1000, None, None, False, None, None), "out": ( "mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc", [], @@ -3972,16 +3972,16 @@ ), }, { - "in": (ADDR[0], None, f"A{2 ** 64}", 1000, True, False, None, ""), + "in": (ADDR[0], f"A{2 ** 64}", 1000, None, True, False, None, ""), "error": (exceptions.AssetNameError, "numeric asset name not in range"), }, { - "in": (ADDR[0], None, f"A{26 ** 12}", 1000, True, False, None, ""), + "in": (ADDR[0], f"A{26 ** 12}", 1000, None, True, False, None, ""), "error": (exceptions.AssetNameError, "numeric asset name not in range"), }, { "comment": "basic child asset", - "in": (ADDR[0], None, "PARENT.child1", 100000000, True, False, None, ""), + "in": (ADDR[0], "PARENT.child1", 100000000, None, True, False, None, ""), "out": ( "mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc", [], @@ -3994,7 +3994,7 @@ }, { "comment": "basic child asset with description", - "in": (ADDR[0], None, "PARENT.child1", 100000000, True, False, None, "hello world"), + "in": (ADDR[0], "PARENT.child1", 100000000, None, True, False, None, "hello world"), "out": ( "mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc", [], @@ -4014,7 +4014,7 @@ # └────────────────── Type ID (4 bytes) - type 21/subasset }, { - "in": (ADDR[0], None, "PARENT.a.b.c", 1000, True, False, None, ""), + "in": (ADDR[0], "PARENT.a.b.c", 1000, None, True, False, None, ""), "out": ( "mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc", [], @@ -4026,7 +4026,7 @@ # 00000015|01530821671b1001|00000000000003e8|01|0a|014a74856171ca3c559f }, { - "in": (ADDR[0], None, "PARENT.a-zA-Z0-9.-_@!", 1000, True, False, None, ""), + "in": (ADDR[0], "PARENT.a-zA-Z0-9.-_@!", 1000, None, True, False, None, ""), "out": ( "mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc", [], @@ -4038,7 +4038,7 @@ }, { "comment": "make sure compose catches asset name syntax errors", - "in": (ADDR[0], None, "BADASSETx.child1", 1000, True, False, None, ""), + "in": (ADDR[0], "BADASSETx.child1", 1000, None, True, False, None, ""), "error": ( exceptions.AssetNameError, "parent asset name contains invalid character:", @@ -4046,12 +4046,12 @@ }, { "comment": "make sure compose catches validation errors", - "in": (ADDR[1], None, "PARENT.child1", 1000, True, False, None, ""), + "in": (ADDR[1], "PARENT.child1", 1000, None, True, False, None, ""), "error": (exceptions.ComposeError, ["parent asset owned by another address"]), }, { "comment": "referencing parent asset by name composes a reissuance", - "in": (ADDR[0], None, "PARENT.already.issued", 1000, True, False, None, ""), + "in": (ADDR[0], "PARENT.already.issued", 1000, None, True, False, None, ""), "out": ( "mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc", [], @@ -4061,7 +4061,7 @@ { "comment": "basic child asset with compact message type id", "mock_protocol_changes": {"short_tx_type_id": True}, - "in": (ADDR[0], None, "PARENT.child1", 100000000, True, False, None, ""), + "in": (ADDR[0], "PARENT.child1", 100000000, None, True, False, None, ""), "out": ( "mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc", [], @@ -4074,9 +4074,9 @@ { "in": ( ADDR[0], - None, f"A{26 ** 12 + 101}", 200000000, + None, True, None, None, @@ -4091,9 +4091,9 @@ { "in": ( ADDR[0], - ADDR[1], "DIVISIBLEB", 0, + ADDR[1], True, False, None, @@ -4106,7 +4106,7 @@ ), }, { - "in": (ADDR[0], None, "DIVISIBLEC", 0, True, True, None, "third divisible asset"), + "in": (ADDR[0], "DIVISIBLEC", 0, None, True, True, None, "third divisible asset"), "out": ( "mn6q3dS2EnDUx3bmyWc6D4szJNVGtaR7zc", [], diff --git a/counterparty-lib/counterpartylib/test/integration_test.py b/counterparty-core/counterpartycore/test/integration_test.py similarity index 97% rename from counterparty-lib/counterpartylib/test/integration_test.py rename to counterparty-core/counterpartycore/test/integration_test.py index 3932f951d4..9b5a5342d1 100644 --- a/counterparty-lib/counterpartylib/test/integration_test.py +++ b/counterparty-core/counterpartycore/test/integration_test.py @@ -2,7 +2,7 @@ import pytest # noqa: F401 # this is require near the top to do setup of the test suite -from counterpartylib.test import ( +from counterpartycore.test import ( conftest, util_test, ) diff --git a/counterparty-lib/counterpartylib/test/mocked_utxoset_test.py b/counterparty-core/counterpartycore/test/mocked_utxoset_test.py similarity index 95% rename from counterparty-lib/counterpartylib/test/mocked_utxoset_test.py rename to counterparty-core/counterpartycore/test/mocked_utxoset_test.py index 9c20dce516..f97092e5de 100644 --- a/counterparty-lib/counterpartylib/test/mocked_utxoset_test.py +++ b/counterparty-core/counterpartycore/test/mocked_utxoset_test.py @@ -3,15 +3,15 @@ import pytest -from counterpartylib.lib import backend, util +from counterpartycore.lib import backend, util # this is require near the top to do setup of the test suite -from counterpartylib.test import ( +from counterpartycore.test import ( conftest, # noqa: F401 util_test, ) -from counterpartylib.test.fixtures.params import ADDR, DP # noqa: F401 -from counterpartylib.test.util_test import CURR_DIR +from counterpartycore.test.fixtures.params import ADDR, DP # noqa: F401 +from counterpartycore.test.util_test import CURR_DIR FIXTURE_SQL_FILE = CURR_DIR + "/fixtures/scenarios/unittest_fixture.sql" FIXTURE_DB = tempfile.gettempdir() + "/fixtures.unittest_fixture.db" diff --git a/counterparty-lib/counterpartylib/test/p2sh_encoding_test.py b/counterparty-core/counterpartycore/test/p2sh_encoding_test.py similarity index 98% rename from counterparty-lib/counterpartylib/test/p2sh_encoding_test.py rename to counterparty-core/counterpartycore/test/p2sh_encoding_test.py index 35ded5283b..7cf1dc9fdd 100644 --- a/counterparty-lib/counterpartylib/test/p2sh_encoding_test.py +++ b/counterparty-core/counterpartycore/test/p2sh_encoding_test.py @@ -10,16 +10,16 @@ import pytest # this is require near the top to do setup of the test suite -from counterpartylib.test import ( +from counterpartycore.test import ( conftest, # noqa: F401 util_test, ) -from counterpartylib.test.fixtures.params import ADDR, DP, P2SH_ADDR -from counterpartylib.test.util_test import CURR_DIR +from counterpartycore.test.fixtures.params import ADDR, DP, P2SH_ADDR +from counterpartycore.test.util_test import CURR_DIR logger = logging.getLogger(__name__) -from counterpartylib.lib import ( # noqa: E402 +from counterpartycore.lib import ( # noqa: E402 api, backend, config, @@ -29,8 +29,8 @@ script, util, # noqa: F401 ) -from counterpartylib.lib.kickstart.blocks_parser import BlockchainParser # noqa: E402 -from counterpartylib.lib.transaction_helper import p2sh_encoding, serializer # noqa: E402, F401 +from counterpartycore.lib.kickstart.blocks_parser import BlockchainParser # noqa: E402 +from counterpartycore.lib.transaction_helper import p2sh_encoding, serializer # noqa: E402, F401 FIXTURE_SQL_FILE = CURR_DIR + "/fixtures/scenarios/unittest_fixture.sql" FIXTURE_DB = tempfile.gettempdir() + "/fixtures.unittest_fixture.db" diff --git a/counterparty-lib/counterpartylib/test/parse_block_test.py b/counterparty-core/counterpartycore/test/parse_block_test.py similarity index 83% rename from counterparty-lib/counterpartylib/test/parse_block_test.py rename to counterparty-core/counterpartycore/test/parse_block_test.py index f2678113c1..68ea61722c 100644 --- a/counterparty-lib/counterpartylib/test/parse_block_test.py +++ b/counterparty-core/counterpartycore/test/parse_block_test.py @@ -2,14 +2,14 @@ import pprint import tempfile -from counterpartylib.lib import blocks -from counterpartylib.test import ( +from counterpartycore.lib import blocks +from counterpartycore.test import ( conftest, # noqa: F401 ) # this is require near the top to do setup of the test suite -from counterpartylib.test.fixtures.params import DEFAULT_PARAMS as DP -from counterpartylib.test.util_test import CURR_DIR +from counterpartycore.test.fixtures.params import DEFAULT_PARAMS as DP +from counterpartycore.test.util_test import CURR_DIR FIXTURE_SQL_FILE = CURR_DIR + "/fixtures/scenarios/parseblock_unittest_fixture.sql" FIXTURE_DB = tempfile.gettempdir() + "/fixtures.parseblock_unittest_fixture.db" diff --git a/counterparty-lib/counterpartylib/test/pycoin_rs_test.py b/counterparty-core/counterpartycore/test/pycoin_rs_test.py similarity index 98% rename from counterparty-lib/counterpartylib/test/pycoin_rs_test.py rename to counterparty-core/counterpartycore/test/pycoin_rs_test.py index 12a49a1936..ee32f5b802 100644 --- a/counterparty-lib/counterpartylib/test/pycoin_rs_test.py +++ b/counterparty-core/counterpartycore/test/pycoin_rs_test.py @@ -6,9 +6,9 @@ from bitcoin.core.script import CScript from counterparty_rs import utils -from counterpartylib.lib import config -from counterpartylib.lib.opcodes import * # noqa: F403 -from counterpartylib.lib.script import ( +from counterpartycore.lib import config +from counterpartycore.lib.opcodes import * # noqa: F403 +from counterpartycore.lib.script import ( base58_check_decode, base58_check_decode_py, base58_check_encode, diff --git a/counterparty-lib/counterpartylib/test/pytest.ini b/counterparty-core/counterpartycore/test/pytest.ini similarity index 100% rename from counterparty-lib/counterpartylib/test/pytest.ini rename to counterparty-core/counterpartycore/test/pytest.ini diff --git a/counterparty-core/counterpartycore/test/telemetry_test.py b/counterparty-core/counterpartycore/test/telemetry_test.py new file mode 100644 index 0000000000..563bb4e526 --- /dev/null +++ b/counterparty-core/counterpartycore/test/telemetry_test.py @@ -0,0 +1,75 @@ +import time +from unittest.mock import MagicMock, patch + +from counterpartycore.lib.telemetry.collector import TelemetryCollectorLive +from counterpartycore.lib.telemetry.daemon import TelemetryDaemon + + +class TestTelemetryDaemon: + def test_init(self): + collector = MagicMock() + client = MagicMock() + daemon = TelemetryDaemon(collector, client) + assert daemon.client == client + assert daemon.collector == collector + assert daemon.interval == 60 + + def test_init_with_custom_interval(self): + collector = MagicMock() + client = MagicMock() + daemon = TelemetryDaemon(collector, client, interval=10) + assert daemon.client == client + assert daemon.collector == collector + assert daemon.interval == 10 + + def test_send_at_intervals(self): + collector = MagicMock() + client = MagicMock() + daemon = TelemetryDaemon(collector, client, interval=0.1) + daemon.start() + assert daemon.is_running == True # noqa: E712 + time.sleep(0.5) + daemon.stop() + assert daemon.is_running == False # noqa: E712 + assert client.send.call_count > 1 + assert collector.collect.call_count > 1 + + +class TestTelemetryCollectorLive: + @patch("counterpartycore.lib.telemetry.util.config") + @patch("counterpartycore.lib.telemetry.collector.ledger") + def test_collect(self, mock_ledger, mock_config): + mock_db = MagicMock() + mock_ledger.last_message.return_value = {"block_index": 12345} + mock_config.__version__ = "1.2.3" + mock_config.ADDRINDEXRS_VERSION = "4.5.6" + mock_config.TESTNET = False + mock_config.FORCE = False + + collector = TelemetryCollectorLive(mock_db) + time.sleep(0.1) + data = collector.collect() + + print("\n\n\n", data) + + mock_ledger.last_message.assert_called_with(mock_db) + mock_db.cursor().execute.assert_called_with( + "SELECT * FROM blocks where block_index = ?", [12345] + ) + + assert data["version"] == "1.2.3" + assert data["addrindexrs_version"] == "4.5.6" + assert data["uptime"] > 0 + assert data["network"] == "MAINNET" + assert data["dockerized"] == False # noqa: E712 + assert data["force_enabled"] == False # noqa: E712 + + @patch("counterpartycore.lib.telemetry.collector.ledger") + @patch("counterpartycore.lib.telemetry.collector.os.path.exists") + def test_collect_with_docker(self, mock_exists, mock_ledger): + mock_db = MagicMock() + mock_exists.return_value = True + mock_ledger.last_message.return_value = {"block_index": 12345} + collector = TelemetryCollectorLive(mock_db) + data = collector.collect() + assert data["dockerized"] == True # noqa: E712 diff --git a/counterparty-lib/counterpartylib/test/thread_safe_ttl_cache.py b/counterparty-core/counterpartycore/test/thread_safe_ttl_cache.py similarity index 96% rename from counterparty-lib/counterpartylib/test/thread_safe_ttl_cache.py rename to counterparty-core/counterpartycore/test/thread_safe_ttl_cache.py index ff9b347b12..55ce109d26 100644 --- a/counterparty-lib/counterpartylib/test/thread_safe_ttl_cache.py +++ b/counterparty-core/counterpartycore/test/thread_safe_ttl_cache.py @@ -1,6 +1,6 @@ import threading -from counterpartylib.lib import transaction +from counterpartycore.lib import transaction # This is a pretty contrived case as we never call diff --git a/counterparty-lib/counterpartylib/test/unit_test.py b/counterparty-core/counterpartycore/test/unit_test.py similarity index 90% rename from counterparty-lib/counterpartylib/test/unit_test.py rename to counterparty-core/counterpartycore/test/unit_test.py index 8de62fd661..1493487940 100644 --- a/counterparty-lib/counterpartylib/test/unit_test.py +++ b/counterparty-core/counterpartycore/test/unit_test.py @@ -3,14 +3,14 @@ import pytest -from counterpartylib.lib import check, exceptions # noqa: F401 +from counterpartycore.lib import check, exceptions # noqa: F401 # this is require near the top to do setup of the test suite -from counterpartylib.test import ( +from counterpartycore.test import ( conftest, util_test, ) -from counterpartylib.test.util_test import CURR_DIR +from counterpartycore.test.util_test import CURR_DIR FIXTURE_SQL_FILE = CURR_DIR + "/fixtures/scenarios/unittest_fixture.sql" FIXTURE_DB = tempfile.gettempdir() + "/fixtures.unittest_fixture.db" @@ -38,7 +38,7 @@ def test_vector( with util_test.ConfigContext(**config_context): # force unit tests to always run against latest protocol changes - from counterpartylib.test import conftest + from counterpartycore.test import conftest conftest.ALWAYS_LATEST_PROTOCOL_CHANGES = True conftest.ENABLE_MOCK_PROTOCOL_CHANGES_AT_BLOCK = True diff --git a/counterparty-lib/counterpartylib/test/util_test.py b/counterparty-core/counterpartycore/test/util_test.py similarity index 97% rename from counterparty-lib/counterpartylib/test/util_test.py rename to counterparty-core/counterpartycore/test/util_test.py index d0ad1b3fcc..80bbb4493f 100644 --- a/counterparty-lib/counterpartylib/test/util_test.py +++ b/counterparty-core/counterpartycore/test/util_test.py @@ -29,8 +29,8 @@ ) sys.path.append(os.path.normpath(os.path.join(CURR_DIR, ".."))) -from counterpartylib import server # noqa: E402 -from counterpartylib.lib import ( # noqa: E402 +from counterpartycore import server # noqa: E402 +from counterpartycore.lib import ( # noqa: E402 backend, blocks, check, @@ -42,13 +42,13 @@ transaction, util, ) -from counterpartylib.lib.backend.indexd import ( # noqa: E402 +from counterpartycore.lib.backend.indexd import ( # noqa: E402 extract_addresses, # noqa: F401 extract_addresses_from_txlist, ) -from counterpartylib.lib.kickstart.blocks_parser import BlockchainParser # noqa: E402 -from counterpartylib.test.fixtures.params import DEFAULT_PARAMS as DP # noqa: E402 -from counterpartylib.test.fixtures.scenarios import ( # noqa: E402 +from counterpartycore.lib.kickstart.blocks_parser import BlockchainParser # noqa: E402 +from counterpartycore.test.fixtures.params import DEFAULT_PARAMS as DP # noqa: E402 +from counterpartycore.test.fixtures.scenarios import ( # noqa: E402 INTEGRATION_SCENARIOS, UNITTEST_FIXTURE, # noqa: F401 standard_scenarios_params, @@ -599,7 +599,7 @@ def run_scenario(scenario): if tx[0] != "create_next_block": mock_protocol_changes = tx[3] if len(tx) == 4 else {} with MockProtocolChangesContext(**(mock_protocol_changes or {})): - module = sys.modules[f"counterpartylib.lib.messages.{tx[0]}"] + module = sys.modules[f"counterpartycore.lib.messages.{tx[0]}"] compose = module.compose unsigned_tx_hex = transaction.construct( db=db, tx_info=compose(db, *tx[1]), regular_dust_size=5430, **tx[2] @@ -801,9 +801,9 @@ def check_outputs( """Check actual and expected outputs of a particular function.""" try: - tested_module = sys.modules[f"counterpartylib.lib.{tx_name}"] + tested_module = sys.modules[f"counterpartycore.lib.{tx_name}"] except KeyError: # TODO: hack - tested_module = sys.modules[f"counterpartylib.lib.messages.{tx_name}"] + tested_module = sys.modules[f"counterpartycore.lib.messages.{tx_name}"] tested_method = getattr(tested_module, method) with MockProtocolChangesContext(**(mock_protocol_changes or {})): @@ -916,7 +916,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): class MockProtocolChangesContext(object): def __init__(self, **kwargs): - from counterpartylib.test.conftest import MOCK_PROTOCOL_CHANGES + from counterpartycore.test.conftest import MOCK_PROTOCOL_CHANGES self.mock_protocol_changes = MOCK_PROTOCOL_CHANGES self._extend = kwargs diff --git a/counterparty-lib/counterpartylib/test/utxolocks_test.py b/counterparty-core/counterpartycore/test/utxolocks_test.py similarity index 96% rename from counterparty-lib/counterpartylib/test/utxolocks_test.py rename to counterparty-core/counterpartycore/test/utxolocks_test.py index 91e544b693..733d1502c9 100644 --- a/counterparty-lib/counterpartylib/test/utxolocks_test.py +++ b/counterparty-core/counterpartycore/test/utxolocks_test.py @@ -6,14 +6,14 @@ import bitcoin import pytest # noqa: F401 -from counterpartylib.lib import transaction -from counterpartylib.lib.messages import send -from counterpartylib.test import ( +from counterpartycore.lib import transaction +from counterpartycore.lib.messages import send +from counterpartycore.test import ( conftest, # noqa: F401 ) # this is require near the top to do setup of the test suite -from counterpartylib.test.util_test import CURR_DIR +from counterpartycore.test.util_test import CURR_DIR FIXTURE_SQL_FILE = CURR_DIR + "/fixtures/scenarios/parseblock_unittest_fixture.sql" FIXTURE_DB = tempfile.gettempdir() + "/fixtures.parseblock_unittest_fixture.db" diff --git a/counterparty-core/pyproject.toml b/counterparty-core/pyproject.toml index cc395941ac..c8a6d2f58a 100644 --- a/counterparty-core/pyproject.toml +++ b/counterparty-core/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" name = "counterparty-core" requires-python = ">= 3.10" dynamic = ["version", "dependencies"] -description = "Counterparty Protocol Command-Line Interface" +description = "Counterparty Protocol Reference Implementation" readme = "../README.md" license = "MIT" authors = [ @@ -27,18 +27,12 @@ classifiers = [ "Topic :: System :: Distributed Computing" ] +[tool.hatch.metadata] +allow-direct-references = true + [tool.hatch.metadata.hooks.requirements_txt] files = ["requirements.txt"] -[tool.hatch.envs.default] -pre-install-commands = [ - "pip install -e ../counterparty-rs", - "pip install -e ../counterparty-lib", -] - -[project.scripts] -counterparty-server = "counterpartycore:server.main" - [project.urls] "Latest release" = "https://github.com/CounterpartyXCP/counterparty-core/releases/latest" "Documentation" = "https://docs.counterparty.io/" @@ -46,4 +40,38 @@ counterparty-server = "counterpartycore:server.main" "Home Page" = "https://counterparty.io/" [tool.hatch.version] -path = "../counterparty-lib/counterpartylib/lib/config.py" +path = "counterpartycore/lib/config.py" + +[tool.hatch.build.targets.wheel] +include = ["counterpartycore"] + +[tool.hatch.envs.default] +pre-install-commands = [ + "pip install -e ../counterparty-rs", +] + +[project.scripts] +counterparty-server = "counterpartycore:cli.main" + +[tool.license_scanner] +allowed-licences = [ + 'Apache license', + 'Apache license 2.0', + 'BSD 2-clause license', + 'BSD 3-clause license', + 'BSD license', + 'GNU general public license v2 (gplv2)', + 'GNU lesser general public license', + 'GNU lesser general public license v2 (lgplv2)', + 'GNU lesser general public license v3 (lgplv3)', + 'ISC license (iscl)', 'MIT license', + 'Mozilla public license 2.0 (mpl 2.0)', + 'Python software foundation license', + 'The Unlicense (Unlicense)', + 'Public domain', + 'Creative Commons Zero, CC-0', +] +allowed-packages = [ + 'counterparty-core', 'counterparty-rs', + 'maturin', 'apsw', +] diff --git a/counterparty-core/requirements.txt b/counterparty-core/requirements.txt index 631857d5e9..83822330be 100644 --- a/counterparty-core/requirements.txt +++ b/counterparty-core/requirements.txt @@ -1,8 +1,29 @@ +apsw==3.45.1.0 appdirs==1.4.4 setuptools-markdown==0.4.1 -prettytable==3.9.0 -colorlog==6.8.0 python-dateutil==2.8.2 +Flask-HTTPAuth==4.8.0 +Flask==3.0.0 +colorlog==6.8.0 +prettytable==3.9.0 +json-rpc==1.15.0 +pycoin==0.92.20230326 +pycryptodome==3.20.0 +ripemd-hash==1.0.1 +safe-pysha3==1.0.4 +pytest==7.4.4 +pytest-cov==4.1.0 +python-bitcoinlib==0.12.2 requests==2.31.0 +tendo==0.3.0 +xmltodict==0.13.0 +cachetools==5.3.2 +bitstring==4.1.4 +Werkzeug==3.0.1 +itsdangerous==2.1.2 +plyvel==1.5.1 +arc4==0.4.0 +halo==0.0.31 termcolor==2.4.0 -counterparty-lib==10.1.0 +sentry-sdk==1.45.0 +counterparty-rs==10.1.1 diff --git a/counterparty-lib/tools/benchmark.py b/counterparty-core/tools/benchmark.py similarity index 100% rename from counterparty-lib/tools/benchmark.py rename to counterparty-core/tools/benchmark.py diff --git a/counterparty-lib/tools/checklicences.py b/counterparty-core/tools/checklicences.py similarity index 98% rename from counterparty-lib/tools/checklicences.py rename to counterparty-core/tools/checklicences.py index c9e3d5cbca..a33fe18492 100644 --- a/counterparty-lib/tools/checklicences.py +++ b/counterparty-core/tools/checklicences.py @@ -8,7 +8,7 @@ # pylint: disable=no-name-in-module from sh import license_scanner -PACKAGE_NAME = "counterparty-lib" +PACKAGE_NAME = "counterparty-core" def scan_licenses(): diff --git a/counterparty-lib/tools/compareledger.py b/counterparty-core/tools/compareledger.py similarity index 100% rename from counterparty-lib/tools/compareledger.py rename to counterparty-core/tools/compareledger.py diff --git a/counterparty-lib/tools/copyscenarios.py b/counterparty-core/tools/copyscenarios.py similarity index 82% rename from counterparty-lib/tools/copyscenarios.py rename to counterparty-core/tools/copyscenarios.py index d450167fc0..b52302449c 100755 --- a/counterparty-lib/tools/copyscenarios.py +++ b/counterparty-core/tools/copyscenarios.py @@ -5,7 +5,7 @@ import sys CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) -SCENARIOS_DIR = os.path.join(CURRENT_DIR, "..", "counterpartylib/test/fixtures/scenarios") +SCENARIOS_DIR = os.path.join(CURRENT_DIR, "..", "counterpartycore/test/fixtures/scenarios") dryrun = "--dry-run" in sys.argv or "--dryrun" in sys.argv diff --git a/counterparty-lib/tools/updatetxids.py b/counterparty-core/tools/updatetxids.py similarity index 92% rename from counterparty-lib/tools/updatetxids.py rename to counterparty-core/tools/updatetxids.py index 6e57c00e20..ead8c76cdd 100755 --- a/counterparty-lib/tools/updatetxids.py +++ b/counterparty-core/tools/updatetxids.py @@ -8,8 +8,8 @@ import sys COMMIT = "8906a8188ba841599f66627157e29a270ca838cf" -UNITTEST_FIXTURE_SQL = "counterpartylib/test/fixtures/scenarios/unittest_fixture.sql" -UNITTEST_VECTORS_PY = "counterpartylib/test/fixtures/vectors.py" +UNITTEST_FIXTURE_SQL = "counterpartycore/test/fixtures/scenarios/unittest_fixture.sql" +UNITTEST_VECTORS_PY = "counterpartycore/test/fixtures/vectors.py" REGEX = r"^(?P[+-])INSERT INTO transactions VALUES\((?P\d+),'(?P.+?)'," diff --git a/counterparty-lib/tools/upgradesqlitepagesize.py b/counterparty-core/tools/upgradesqlitepagesize.py similarity index 100% rename from counterparty-lib/tools/upgradesqlitepagesize.py rename to counterparty-core/tools/upgradesqlitepagesize.py diff --git a/counterparty-lib/counterpartylib/lib/util_windows.py b/counterparty-lib/counterpartylib/lib/util_windows.py deleted file mode 100644 index 8026a021aa..0000000000 --- a/counterparty-lib/counterpartylib/lib/util_windows.py +++ /dev/null @@ -1,235 +0,0 @@ -import codecs -import copy -import logging -import logging.handlers -import sys -import unicodedata -from ctypes import POINTER, WINFUNCTYPE, byref, c_int, windll -from ctypes.wintypes import BOOL, DWORD, HANDLE, LPCWSTR, LPVOID, LPWSTR - -from counterpartylib.lib import config - -logger = logging.getLogger(config.LOGGER_NAME) - - -class SanitizedRotatingFileHandler(logging.handlers.RotatingFileHandler): - def emit(self, record): - # If the message doesn't need to be rendered we take a shortcut. - if record.levelno < self.level: - return - # Make sure the message is a string. - message = record.msg - # Sanitize and clean up the message - message = unicodedata.normalize("NFKD", message).encode("ascii", "ignore").decode() - # Copy the original record so we don't break other handlers. - record = copy.copy(record) - record.msg = message - # Use the built-in stream handler to handle output. - logging.handlers.RotatingFileHandler.emit(self, record) - - -def fix_win32_unicode(): - """Thanks to http://stackoverflow.com/a/3259271 ! (converted to python3)""" - if sys.platform != "win32": - return - - original_stderr = sys.stderr - - # If any exception occurs in this code, we'll probably try to print it on stderr, - # which makes for frustrating debugging if stderr is directed to our wrapper. - # So be paranoid about catching errors and reporting them to original_stderr, - # so that we can at least see them. - def _complain(message): - print(message if isinstance(message, str) else repr(message), file=original_stderr) - - # Work around . - codecs.register(lambda name: codecs.lookup("utf-8") if name == "cp65001" else None) - - # Make Unicode console output work independently of the current code page. - # This also fixes . - # Credit to Michael Kaplan - # and TZOmegaTZIOY - # . - try: - # - # HANDLE WINAPI GetStdHandle(DWORD nStdHandle); - # returns invalid_handle_value, NULL, or a valid handle - # - # - # DWORD WINAPI GetFileType(DWORD hFile); - # - # - # BOOL WINAPI GetConsoleMode(HANDLE hConsole, LPDWORD lpMode); - - get_std_handle = WINFUNCTYPE(HANDLE, DWORD)(("GetStdHandle", windll.kernel32)) - std_output_handle = DWORD(-11) - std_error_handle = DWORD(-12) - get_file_type = WINFUNCTYPE(DWORD, DWORD)(("GetFileType", windll.kernel32)) - file_type_char = 0x0002 - file_type_remote = 0x8000 - get_console_mode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))( - ("GetConsoleMode", windll.kernel32) - ) - invalid_handle_value = DWORD(-1).value - - def not_a_console(handle): - if handle == invalid_handle_value or handle is None: - return True - return ( - get_file_type(handle) & ~file_type_remote - ) != file_type_char or get_console_mode(handle, byref(DWORD())) == 0 - - old_stdout_fileno = None - old_stderr_fileno = None - if hasattr(sys.stdout, "fileno"): - old_stdout_fileno = sys.stdout.fileno() - if hasattr(sys.stderr, "fileno"): - old_stderr_fileno = sys.stderr.fileno() - - stdout_fileno = 1 - stderr_fileno = 2 - real_stdout = old_stdout_fileno == stdout_fileno - real_stderr = old_stderr_fileno == stderr_fileno - - if real_stdout: - h_std_out = get_std_handle(std_output_handle) - if not_a_console(h_std_out): - real_stdout = False - - if real_stderr: - h_std_err = get_std_handle(std_error_handle) - if not_a_console(h_std_err): - real_stderr = False - - if real_stdout or real_stderr: - # BOOL WINAPI WriteConsoleW(HANDLE hOutput, LPWSTR lpBuffer, DWORD nChars, - # LPDWORD lpCharsWritten, LPVOID lpReserved); - - write_console_w = WINFUNCTYPE(BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID)( - ("WriteConsoleW", windll.kernel32) - ) - - class UnicodeOutput: - def __init__(self, h_console, stream, fileno, name): - self._h_console = h_console - self._stream = stream - self._fileno = fileno - self.closed = False - self.softspace = False - self.mode = "w" - self.encoding = "utf-8" - self.name = name - self.errors = "" - self.flush() - - def isatty(self): - return False - - def close(self): - # don't really close the handle, that would only cause problems - self.closed = True - - def fileno(self): - return self._fileno - - def flush(self): - if self._h_console is None: - try: - self._stream.flush() - except Exception as e: - _complain(f"{self.name}.flush: {e!r} from {self._stream!r}") - raise - - def write(self, text): - try: - if self._h_console is None: - if isinstance(text, str): - text = text.encode("utf-8") - self._stream.write(text) - else: - if not isinstance(text, str): - text = str(text).decode("utf-8") - remaining = len(text) - while remaining: - n = DWORD(0) - # There is a shorter-than-documented limitation on the - # length of the string passed to WriteConsoleW (see - # . - retval = write_console_w( - self._h_console, text, min(remaining, 10000), byref(n), None - ) - if retval == 0 or n.value == 0: - raise IOError( - f"write_console_w returned {retval!r}, n.value = {n.value!r}" - ) - remaining -= n.value - if not remaining: - break - text = text[n.value :] - except Exception as e: - _complain(f"{self.name}.write: {e!r}") - raise - - def writelines(self, lines): - try: - for line in lines: - self.write(line) - except Exception as e: - _complain(f"{self.name}.writelines: {e!r}") - raise - - if real_stdout: - sys.stdout = UnicodeOutput( - h_std_out, None, stdout_fileno, "" - ) - else: - sys.stdout = UnicodeOutput( - None, sys.stdout, old_stdout_fileno, "" - ) - - if real_stderr: - sys.stderr = UnicodeOutput( - h_std_err, None, stderr_fileno, "" - ) - else: - sys.stderr = UnicodeOutput( - None, sys.stderr, old_stderr_fileno, "" - ) - except Exception as e: - _complain(f"exception {e!r} while fixing up sys.stdout and sys.stderr") - - # While we're at it, let's unmangle the command-line arguments: - - # This works around . - get_command_line_w = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) - command_line_to_argv_w = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( - ("CommandLineToArgvW", windll.shell32) - ) - - argc = c_int(0) - argv_unicode = command_line_to_argv_w(get_command_line_w(), byref(argc)) - - argv = [argv_unicode[i].encode("utf-8").decode("utf-8") for i in range(0, argc.value)] - - if not hasattr(sys, "frozen"): - # If this is an executable produced by py2exe or bbfreeze, then it will - # have been invoked directly. Otherwise, unicode_argv[0] is the Python - # interpreter, so skip that. - argv = argv[1:] - - # Also skip option arguments to the Python interpreter. - while len(argv) > 0: - arg = argv[0] - if not arg.startswith("-") or arg == "-": - break - argv = argv[1:] - if arg == "-m": - # sys.argv[0] should really be the absolute path of the module source, - # but never mind - break - if arg == "-c": - argv[0] = "-c" - break - - # if you like: - sys.argv = argv diff --git a/counterparty-lib/counterpartylib/server.py b/counterparty-lib/counterpartylib/server.py deleted file mode 100755 index f788b66f0c..0000000000 --- a/counterparty-lib/counterpartylib/server.py +++ /dev/null @@ -1,806 +0,0 @@ -#! /usr/bin/env python3 - -import binascii -import decimal -import logging -import os -import platform -import signal -import socket -import sys -import tarfile -import tempfile -import time -import urllib -from urllib.parse import quote_plus as urlencode - -import appdirs -import apsw -import bitcoin as bitcoinlib -from halo import Halo -from termcolor import colored, cprint - -from counterpartylib.lib import ( - api, - backend, - blocks, - check, - config, - database, - ledger, - log, # noqa: F401 - transaction, - util, -) -from counterpartylib.lib import kickstart as kickstarter - -logger = logging.getLogger(config.LOGGER_NAME) -D = decimal.Decimal - -OK_GREEN = colored("[OK]", "green") -SPINNER_STYLE = "bouncingBar" - - -class ConfigurationError(Exception): - pass - - -SIGTERM_CALLBACKS = [] - - -def sigterm_handler(_signo, _stack_frame): - if _signo == 15: - signal_name = "SIGTERM" - elif _signo == 2: - signal_name = "SIGINT" - else: - assert False # noqa: B011 - logger.info(f"Received {signal_name}.") - - if "api_server" in globals(): - logger.info("Stopping API server.") - api_server.stop() # noqa: F821 - api_status_poller.stop() # noqa: F821 - logger.info("Stopping backend.") - backend.stop() - logger.info("Shutting down.") - logging.shutdown() - for callback in SIGTERM_CALLBACKS: - callback() - sys.exit(0) - - -signal.signal(signal.SIGTERM, sigterm_handler) -signal.signal(signal.SIGINT, sigterm_handler) - - -# Lock database access by opening a socket. -class LockingError(Exception): - pass - - -def get_lock(): - logger.info("Acquiring lock.") - - # Cross‐platform. - if os.name == "nt" or platform.system() == "Darwin": # Windows or OS X - # Not database‐specific. - socket_family = socket.AF_INET - socket_address = ("localhost", 8999) - error = "Another copy of server is currently running." - else: - socket_family = socket.AF_UNIX - socket_address = "\0" + config.DATABASE - error = f"Another copy of server is currently writing to database {config.DATABASE}" - - lock_socket = socket.socket(socket_family, socket.SOCK_DGRAM) - try: - lock_socket.bind(socket_address) - except socket.error: - raise LockingError(error) # noqa: B904 - logger.debug("Lock acquired.") - - -def initialise(*args, **kwargs): - initialise_log_config( - verbose=kwargs.pop("verbose", False), - quiet=kwargs.pop("quiet", False), - log_file=kwargs.pop("log_file", None), - api_log_file=kwargs.pop("api_log_file", None), - no_log_files=kwargs.pop("no_log_files", False), - testnet=kwargs.get("testnet", False), - testcoin=kwargs.get("testcoin", False), - regtest=kwargs.get("regtest", False), - ) - initialise_config(*args, **kwargs) - return initialise_db() - - -def initialise_log_config( - verbose=False, - quiet=False, - log_file=None, - api_log_file=None, - no_log_files=False, - testnet=False, - testcoin=False, - regtest=False, - json_log=False, -): - # Log directory - log_dir = appdirs.user_log_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME) - if not os.path.isdir(log_dir): - os.makedirs(log_dir, mode=0o755) - - # Set up logging. - config.VERBOSE = verbose - config.QUIET = quiet - - network = "" - if testnet: - network += ".testnet" - if regtest: - network += ".regtest" - if testcoin: - network += ".testcoin" - - # Log - if no_log_files: - config.LOG = None - elif not log_file: # default location - filename = f"server{network}.log" - config.LOG = os.path.join(log_dir, filename) - else: # user-specified location - config.LOG = log_file - - if no_log_files: # no file logging - config.API_LOG = None - elif not api_log_file: # default location - filename = f"server{network}.access.log" - config.API_LOG = os.path.join(log_dir, filename) - else: # user-specified location - config.API_LOG = api_log_file - - config.JSON_LOG = json_log - - -def initialise_config( - database_file=None, - testnet=False, - testcoin=False, - regtest=False, - api_limit_rows=1000, - backend_connect=None, - backend_port=None, - backend_user=None, - backend_password=None, - indexd_connect=None, - indexd_port=None, - backend_ssl=False, - backend_ssl_no_verify=False, - backend_poll_interval=None, - rpc_host=None, - rpc_port=None, - rpc_user=None, - rpc_password=None, - rpc_no_allow_cors=False, - force=False, - requests_timeout=config.DEFAULT_REQUESTS_TIMEOUT, - rpc_batch_size=config.DEFAULT_RPC_BATCH_SIZE, - check_asset_conservation=config.DEFAULT_CHECK_ASSET_CONSERVATION, - backend_ssl_verify=None, - rpc_allow_cors=None, - p2sh_dust_return_pubkey=None, - utxo_locks_max_addresses=config.DEFAULT_UTXO_LOCKS_MAX_ADDRESSES, - utxo_locks_max_age=config.DEFAULT_UTXO_LOCKS_MAX_AGE, - estimate_fee_per_kb=None, - customnet=None, - no_mempool=False, - skip_db_check=False, -): - # log config alreasdy initialized - logger.debug("VERBOSE: %s", config.VERBOSE) - logger.debug("QUIET: %s", config.QUIET) - logger.debug("LOG: %s", config.LOG) - logger.debug("API_LOG: %s", config.API_LOG) - - # Data directory - data_dir = appdirs.user_data_dir( - appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True - ) - if not os.path.isdir(data_dir): - os.makedirs(data_dir, mode=0o755) - - # testnet - if testnet: - config.TESTNET = testnet - else: - config.TESTNET = False - - # testcoin - if testcoin: - config.TESTCOIN = testcoin - else: - config.TESTCOIN = False - - # regtest - if regtest: - config.REGTEST = regtest - else: - config.REGTEST = False - - if customnet != None and len(customnet) > 0: # noqa: E711 - config.CUSTOMNET = True - config.REGTEST = True # Custom nets are regtests with different parameters - else: - config.CUSTOMNET = False - - if config.TESTNET: - bitcoinlib.SelectParams("testnet") - logger.debug("NETWORK: testnet") - elif config.REGTEST: - bitcoinlib.SelectParams("regtest") - logger.debug("NETWORK: regtest") - else: - bitcoinlib.SelectParams("mainnet") - logger.debug("NETWORK: mainnet") - - network = "" - if config.TESTNET: - network += ".testnet" - if config.REGTEST: - network += ".regtest" - if config.TESTCOIN: - network += ".testcoin" - - # Database - if database_file: - config.DATABASE = database_file - else: - filename = f"{config.APP_NAME}{network}.db" - config.DATABASE = os.path.join(data_dir, filename) - - logger.debug("DATABASE: %s", config.DATABASE) - - config.API_LIMIT_ROWS = api_limit_rows - - ############## - # THINGS WE CONNECT TO - - # Backend name - config.BACKEND_NAME = "addrindexrs" - - # Backend RPC host (Bitcoin Core) - if backend_connect: - config.BACKEND_CONNECT = backend_connect - else: - config.BACKEND_CONNECT = "localhost" - - # Backend Core RPC port (Bitcoin Core) - if backend_port: - config.BACKEND_PORT = backend_port - else: - if config.TESTNET: - config.BACKEND_PORT = config.DEFAULT_BACKEND_PORT_TESTNET - elif config.REGTEST: - config.BACKEND_PORT = config.DEFAULT_BACKEND_PORT_REGTEST - else: - config.BACKEND_PORT = config.DEFAULT_BACKEND_PORT - - try: - config.BACKEND_PORT = int(config.BACKEND_PORT) - if not (int(config.BACKEND_PORT) > 1 and int(config.BACKEND_PORT) < 65535): - raise ConfigurationError("invalid backend API port number") - except: # noqa: E722 - raise ConfigurationError( # noqa: B904 - "Please specific a valid port number backend-port configuration parameter" - ) - - # Backend Core RPC user (Bitcoin Core) - if backend_user: - config.BACKEND_USER = backend_user - else: - config.BACKEND_USER = "rpc" - - # Backend Core RPC password (Bitcoin Core) - if backend_password: - config.BACKEND_PASSWORD = backend_password - else: - raise ConfigurationError( - "Please specific a valid password backend-password configuration parameter" - ) - - # Backend Core RPC SSL - if backend_ssl: - config.BACKEND_SSL = backend_ssl - else: - config.BACKEND_SSL = False # Default to off. - - # Backend Core RPC SSL Verify - if backend_ssl_verify is not None: - cprint( - "The server parameter `backend_ssl_verify` is deprecated. Use `backend_ssl_no_verify` instead.", - "yellow", - ) - config.BACKEND_SSL_NO_VERIFY = not backend_ssl_verify - else: - if backend_ssl_no_verify: - config.BACKEND_SSL_NO_VERIFY = backend_ssl_no_verify - else: - config.BACKEND_SSL_NO_VERIFY = ( - False # Default to on (don't support self‐signed certificates) - ) - - # Backend Poll Interval - if backend_poll_interval: - config.BACKEND_POLL_INTERVAL = backend_poll_interval - else: - config.BACKEND_POLL_INTERVAL = 3.0 - - # Construct backend URL. - config.BACKEND_URL = ( - config.BACKEND_USER - + ":" - + config.BACKEND_PASSWORD - + "@" - + config.BACKEND_CONNECT - + ":" - + str(config.BACKEND_PORT) - ) - if config.BACKEND_SSL: - config.BACKEND_URL = "https://" + config.BACKEND_URL - else: - config.BACKEND_URL = "http://" + config.BACKEND_URL - - cleaned_backend_url = config.BACKEND_URL.replace(f":{config.BACKEND_PASSWORD}@", ":*****@") - logger.debug("BACKEND_URL: %s", cleaned_backend_url) - - # Indexd RPC host - if indexd_connect: - config.INDEXD_CONNECT = indexd_connect - else: - config.INDEXD_CONNECT = "localhost" - - # Indexd RPC port - if indexd_port: - config.INDEXD_PORT = indexd_port - else: - if config.TESTNET: - config.INDEXD_PORT = config.DEFAULT_INDEXD_PORT_TESTNET - elif config.REGTEST: - config.INDEXD_PORT = config.DEFAULT_INDEXD_PORT_REGTEST - else: - config.INDEXD_PORT = config.DEFAULT_INDEXD_PORT - - try: - config.INDEXD_PORT = int(config.INDEXD_PORT) - if not (int(config.INDEXD_PORT) > 1 and int(config.INDEXD_PORT) < 65535): - raise ConfigurationError("invalid Indexd API port number") - except: # noqa: E722 - raise ConfigurationError( # noqa: B904 - "Please specific a valid port number indexd-port configuration parameter" - ) - - # Construct Indexd URL. - config.INDEXD_URL = "http://" + config.INDEXD_CONNECT + ":" + str(config.INDEXD_PORT) - - logger.debug("INDEXD_URL: %s", config.INDEXD_URL) - - ############## - # THINGS WE SERVE - - # Server API RPC host - if rpc_host: - config.RPC_HOST = rpc_host - else: - config.RPC_HOST = "localhost" - - # The web root directory for API calls, eg. localhost:14000/rpc/ - config.RPC_WEBROOT = "/rpc/" - - # Server API RPC port - if rpc_port: - config.RPC_PORT = rpc_port - else: - if config.TESTNET: - if config.TESTCOIN: - config.RPC_PORT = config.DEFAULT_RPC_PORT_TESTNET + 1 - else: - config.RPC_PORT = config.DEFAULT_RPC_PORT_TESTNET - elif config.REGTEST: - if config.TESTCOIN: - config.RPC_PORT = config.DEFAULT_RPC_PORT_REGTEST + 1 - else: - config.RPC_PORT = config.DEFAULT_RPC_PORT_REGTEST - else: - if config.TESTCOIN: - config.RPC_PORT = config.DEFAULT_RPC_PORT + 1 - else: - config.RPC_PORT = config.DEFAULT_RPC_PORT - try: - config.RPC_PORT = int(config.RPC_PORT) - if not (int(config.RPC_PORT) > 1 and int(config.RPC_PORT) < 65535): - raise ConfigurationError("invalid server API port number") - except: # noqa: E722 - raise ConfigurationError( # noqa: B904 - "Please specific a valid port number rpc-port configuration parameter" - ) - - # Server API RPC user - if rpc_user: - config.RPC_USER = rpc_user - else: - config.RPC_USER = "rpc" - - configure_rpc(rpc_password) - - # RPC CORS - if rpc_allow_cors is not None: - cprint( - "The server parameter `rpc_allow_cors` is deprecated. Use `rpc_no_allow_cors` instead.", - "yellow", - ) - config.RPC_NO_ALLOW_CORS = not rpc_allow_cors - else: - if rpc_no_allow_cors: - config.RPC_NO_ALLOW_CORS = rpc_no_allow_cors - else: - config.RPC_NO_ALLOW_CORS = False - - config.RPC_BATCH_SIZE = rpc_batch_size - - ############## - # OTHER SETTINGS - - # skip checks - if force: - config.FORCE = force - else: - config.FORCE = False - - # Encoding - if config.TESTCOIN: - config.PREFIX = b"XX" # 2 bytes (possibly accidentally created) - else: - config.PREFIX = b"CNTRPRTY" # 8 bytes - - # (more) Testnet - if config.TESTNET: - config.MAGIC_BYTES = config.MAGIC_BYTES_TESTNET - if config.TESTCOIN: - config.ADDRESSVERSION = config.ADDRESSVERSION_TESTNET - config.P2SH_ADDRESSVERSION = config.P2SH_ADDRESSVERSION_TESTNET - config.BLOCK_FIRST = config.BLOCK_FIRST_TESTNET_TESTCOIN - config.BURN_START = config.BURN_START_TESTNET_TESTCOIN - config.BURN_END = config.BURN_END_TESTNET_TESTCOIN - config.UNSPENDABLE = config.UNSPENDABLE_TESTNET - config.P2SH_DUST_RETURN_PUBKEY = p2sh_dust_return_pubkey - else: - config.ADDRESSVERSION = config.ADDRESSVERSION_TESTNET - config.P2SH_ADDRESSVERSION = config.P2SH_ADDRESSVERSION_TESTNET - config.BLOCK_FIRST = config.BLOCK_FIRST_TESTNET - config.BURN_START = config.BURN_START_TESTNET - config.BURN_END = config.BURN_END_TESTNET - config.UNSPENDABLE = config.UNSPENDABLE_TESTNET - config.P2SH_DUST_RETURN_PUBKEY = p2sh_dust_return_pubkey - elif config.CUSTOMNET: - custom_args = customnet.split("|") - - if len(custom_args) == 3: - config.MAGIC_BYTES = config.MAGIC_BYTES_REGTEST - config.ADDRESSVERSION = binascii.unhexlify(custom_args[1]) - config.P2SH_ADDRESSVERSION = binascii.unhexlify(custom_args[2]) - config.BLOCK_FIRST = config.BLOCK_FIRST_REGTEST - config.BURN_START = config.BURN_START_REGTEST - config.BURN_END = config.BURN_END_REGTEST - config.UNSPENDABLE = custom_args[0] - config.P2SH_DUST_RETURN_PUBKEY = p2sh_dust_return_pubkey - else: - raise "Custom net parameter needs to be like UNSPENDABLE_ADDRESS|ADDRESSVERSION|P2SH_ADDRESSVERSION (version bytes in HH format)" # noqa: B016 - elif config.REGTEST: - config.MAGIC_BYTES = config.MAGIC_BYTES_REGTEST - if config.TESTCOIN: - config.ADDRESSVERSION = config.ADDRESSVERSION_REGTEST - config.P2SH_ADDRESSVERSION = config.P2SH_ADDRESSVERSION_REGTEST - config.BLOCK_FIRST = config.BLOCK_FIRST_REGTEST_TESTCOIN - config.BURN_START = config.BURN_START_REGTEST_TESTCOIN - config.BURN_END = config.BURN_END_REGTEST_TESTCOIN - config.UNSPENDABLE = config.UNSPENDABLE_REGTEST - config.P2SH_DUST_RETURN_PUBKEY = p2sh_dust_return_pubkey - else: - config.ADDRESSVERSION = config.ADDRESSVERSION_REGTEST - config.P2SH_ADDRESSVERSION = config.P2SH_ADDRESSVERSION_REGTEST - config.BLOCK_FIRST = config.BLOCK_FIRST_REGTEST - config.BURN_START = config.BURN_START_REGTEST - config.BURN_END = config.BURN_END_REGTEST - config.UNSPENDABLE = config.UNSPENDABLE_REGTEST - config.P2SH_DUST_RETURN_PUBKEY = p2sh_dust_return_pubkey - else: - config.MAGIC_BYTES = config.MAGIC_BYTES_MAINNET - if config.TESTCOIN: - config.ADDRESSVERSION = config.ADDRESSVERSION_MAINNET - config.P2SH_ADDRESSVERSION = config.P2SH_ADDRESSVERSION_MAINNET - config.BLOCK_FIRST = config.BLOCK_FIRST_MAINNET_TESTCOIN - config.BURN_START = config.BURN_START_MAINNET_TESTCOIN - config.BURN_END = config.BURN_END_MAINNET_TESTCOIN - config.UNSPENDABLE = config.UNSPENDABLE_MAINNET - config.P2SH_DUST_RETURN_PUBKEY = p2sh_dust_return_pubkey - else: - config.ADDRESSVERSION = config.ADDRESSVERSION_MAINNET - config.P2SH_ADDRESSVERSION = config.P2SH_ADDRESSVERSION_MAINNET - config.BLOCK_FIRST = config.BLOCK_FIRST_MAINNET - config.BURN_START = config.BURN_START_MAINNET - config.BURN_END = config.BURN_END_MAINNET - config.UNSPENDABLE = config.UNSPENDABLE_MAINNET - config.P2SH_DUST_RETURN_PUBKEY = p2sh_dust_return_pubkey - - # Misc - config.REQUESTS_TIMEOUT = requests_timeout - config.CHECK_ASSET_CONSERVATION = check_asset_conservation - config.UTXO_LOCKS_MAX_ADDRESSES = utxo_locks_max_addresses - config.UTXO_LOCKS_MAX_AGE = utxo_locks_max_age - - if estimate_fee_per_kb is not None: - config.ESTIMATE_FEE_PER_KB = estimate_fee_per_kb - - config.NO_MEMPOOL = no_mempool - config.SKIP_DB_CHECK = skip_db_check - - logger.info(f"Running v{config.VERSION_STRING} of counterparty-lib.") - - -def initialise_db(): - if config.FORCE: - cprint("THE OPTION `--force` IS NOT FOR USE ON PRODUCTION SYSTEMS.", "yellow") - - # Lock - if not config.FORCE: - get_lock() - - # Database - logger.info(f"Connecting to database (SQLite {apsw.apswversion()}).") - db = database.get_connection(read_only=False) - - # perform quick integrity check - if not config.SKIP_DB_CHECK: - logger.info("Running PRAGMA quick_check...") - db.execute("PRAGMA quick_check") - logger.info("PRAGMA quick_check done.") - else: - logger.warning("Skipping PRAGMA quick_check.") - - ledger.CURRENT_BLOCK_INDEX = blocks.last_db_index(db) - - def shutdown_handler(): - """run PRAGMA optimize before sys.exit()""" - logger.info("Running PRAGMA optimize...") - cursor = db.cursor() - cursor.execute("PRAGMA optimize") - logger.info("PRAGMA optimize done.") - # NOTE: it's not necessary to explicitly close the connection with db.close() - - SIGTERM_CALLBACKS.append(shutdown_handler) - - return db - - -def connect_to_backend(): - if not config.FORCE: - backend.getblockcount() - - -def connect_to_addrindexrs(): - step = "Connecting to `addrindexrs`..." - with Halo(text=step, spinner=SPINNER_STYLE): - ledger.CURRENT_BLOCK_INDEX = 0 - backend.backend() - check_addrindexrs = {} - while check_addrindexrs == {}: - check_address = ( - "mrHFGUKSiNMeErqByjX97qPKfumdZxe6mC" - if config.TESTNET - else "1GsjsKKT4nH4GPmDnaxaZEDWgoBpmexwMA" - ) - check_addrindexrs = backend.get_oldest_tx(check_address, 99999999999) - if check_addrindexrs == {}: - logger.info("`addrindexrs` is not ready. Waiting one second.") - time.sleep(1) - print(f"{OK_GREEN} {step}") - - -def start_all(catch_up="normal"): - # Backend. - connect_to_backend() - - if not os.path.exists(config.DATABASE) and catch_up == "bootstrap": - bootstrap(no_confirm=True) - - db = initialise_db() - - # Reset UTXO_LOCKS. This previously was done in - # initilise_config - transaction.initialise() - - # API Status Poller. - api_status_poller = api.APIStatusPoller() - api_status_poller.daemon = True - api_status_poller.start() - - # API Server. - api_server = api.APIServer() - api_server.daemon = True - api_server.start() - - # Server - blocks.follow(db) - - -def reparse(block_index): - connect_to_addrindexrs() - try: - db = initialise_db() - blocks.reparse(db, block_index=block_index) - finally: - backend.stop() - db.close() - - -def rollback(block_index=None): - db = initialise_db() - blocks.rollback(db, block_index=block_index) - - -def kickstart(bitcoind_dir, force=False, max_queue_size=None, debug_block=None): - kickstarter.run( - bitcoind_dir=bitcoind_dir, - force=force, - max_queue_size=max_queue_size, - debug_block=debug_block, - ) - - -def vacuum(): - db = initialise_db() - step = "Vacuuming database..." - with Halo(text=step, spinner=SPINNER_STYLE): - database.vacuum(db) - print(f"{OK_GREEN} {step}") - - -def check_database(): - db = initialise_db() - - start_all_time = time.time() - - start_time = time.time() - step = "Checking asset conservation..." - with Halo(text=step, spinner=SPINNER_STYLE): - check.asset_conservation(db) - print(f"{OK_GREEN} {step} (in {time.time() - start_time:.2f}s)") - - start_time = time.time() - step = "Checking database foreign keys...." - with Halo(text=step, spinner=SPINNER_STYLE): - database.check_foreign_keys(db) - print(f"{OK_GREEN} {step} (in {time.time() - start_time:.2f}s)") - - start_time = time.time() - step = "Checking database integrity..." - with Halo(text=step, spinner=SPINNER_STYLE): - database.intergrity_check(db) - print(f"{OK_GREEN} {step} (in {time.time() - start_time:.2f}s)") - - cprint(f"Database checks complete in {time.time() - start_all_time:.2f}s.", "green") - - -def show_params(): - output = vars(config) - for k in list(output.keys()): - if k.isupper(): - print(f"{k}: {output[k]}") - - -def generate_move_random_hash(move): - move = int(move).to_bytes(2, byteorder="big") - random_bin = os.urandom(16) - move_random_hash_bin = util.dhash(random_bin + move) - return binascii.hexlify(random_bin).decode("utf8"), binascii.hexlify( - move_random_hash_bin - ).decode("utf8") - - -def configure_rpc(rpc_password=None): - # Server API RPC password - if rpc_password: - config.RPC_PASSWORD = rpc_password - config.API_ROOT = ( - "http://" - + urlencode(config.RPC_USER) - + ":" - + urlencode(config.RPC_PASSWORD) - + "@" - + config.RPC_HOST - + ":" - + str(config.RPC_PORT) - ) - else: - config.API_ROOT = "http://" + config.RPC_HOST + ":" + str(config.RPC_PORT) - config.RPC = config.API_ROOT + config.RPC_WEBROOT - - cleaned_rpc_url = config.RPC.replace(f":{urlencode(config.RPC_PASSWORD)}@", ":*****@") - logger.debug("RPC: %s", cleaned_rpc_url) - - -def bootstrap(no_confirm=False): - warning_message = """WARNING: `counterparty-server bootstrap` downloads a recent snapshot of a Counterparty database -from a centralized server maintained by the Counterparty Core development team. -Because this method does not involve verifying the history of Counterparty transactions yourself, -the `bootstrap` command should not be used for mission-critical, commercial or public-facing nodes. - """ - cprint(warning_message, "yellow") - - if not no_confirm: - confirmation_message = colored("Continue? (y/N): ", "magenta") - if input(confirmation_message).lower() != "y": - exit() - - data_dir = appdirs.user_data_dir( - appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True - ) - - # Set Constants. - bootstrap_url = config.BOOTSTRAP_URL_TESTNET if config.TESTNET else config.BOOTSTRAP_URL_MAINNET - tar_filename = os.path.basename(bootstrap_url) - tarball_path = os.path.join(tempfile.gettempdir(), tar_filename) - database_path = os.path.join(data_dir, config.APP_NAME) - if config.TESTNET: - database_path += ".testnet" - database_path += ".db" - - # Prepare Directory. - if not os.path.exists(data_dir): - os.makedirs(data_dir, mode=0o755) - if os.path.exists(database_path): - os.remove(database_path) - # Delete SQLite Write-Ahead-Log - wal_path = database_path + "-wal" - shm_path = database_path + "-shm" - if os.path.exists(wal_path): - os.remove(wal_path) - if os.path.exists(shm_path): - os.remove(shm_path) - - # Define Progress Bar. - step = f"Downloading database from {bootstrap_url}..." - spinner = Halo(text=step, spinner=SPINNER_STYLE) - - def bootstrap_progress(blocknum, blocksize, totalsize): - readsofar = blocknum * blocksize - if totalsize > 0: - percent = readsofar * 1e2 / totalsize - message = f"Downloading database: {percent:5.1f}% {readsofar} / {totalsize}" - spinner.text = message - - # Downloading - spinner.start() - urllib.request.urlretrieve(bootstrap_url, tarball_path, bootstrap_progress) # nosec B310 # noqa: S310 - spinner.stop() - print(f"{OK_GREEN} {step}") - - # TODO: check checksum, filenames, etc. - step = f"Extracting database to {data_dir}..." - with Halo(text=step, spinner=SPINNER_STYLE): - with tarfile.open(tarball_path, "r:gz") as tar_file: - tar_file.extractall(path=data_dir) # nosec B202 # noqa: S202 - print(f"{OK_GREEN} {step}") - - assert os.path.exists(database_path) - # user and group have "rw" access - os.chmod(database_path, 0o660) # nosec B103 - - step = "Cleaning up..." - with Halo(text=step, spinner=SPINNER_STYLE): - os.remove(tarball_path) - print(f"{OK_GREEN} {step}") - - cprint(f"Database has been successfully bootstrapped to {database_path}.", "green") diff --git a/counterparty-lib/pyproject.toml b/counterparty-lib/pyproject.toml deleted file mode 100644 index a0722c8eb6..0000000000 --- a/counterparty-lib/pyproject.toml +++ /dev/null @@ -1,74 +0,0 @@ -[build-system] -requires = ["hatchling==1.14.0", "hatch-requirements-txt==0.4.0"] -build-backend = "hatchling.build" - -[project] -name = "counterparty-lib" -requires-python = ">= 3.10" -dynamic = ["version", "dependencies"] -description = "Counterparty Protocol Reference Implementation" -readme = "../README.md" -license = "MIT" -authors = [ - { name = "Counterparty Developers", email = "dev@counterparty.io" }, -] -keywords = ['counterparty', 'bitcoin', 'blockchain', 'crypto', 'cryptocurrency', 'wallet', 'exchange', 'trading', 'finance'] -classifiers = [ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: Financial and Insurance Industry", - "License :: OSI Approved :: MIT License", - "Natural Language :: English", - "Operating System :: Microsoft :: Windows", - "Operating System :: POSIX", - "Programming Language :: Python :: 3 :: Only", - "Topic :: Office/Business :: Financial", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: System :: Distributed Computing" -] - -[tool.hatch.metadata] -allow-direct-references = true - -[tool.hatch.metadata.hooks.requirements_txt] -files = ["requirements.txt"] - -[project.urls] -"Latest release" = "https://github.com/CounterpartyXCP/counterparty-core/releases/latest" -"Documentation" = "https://docs.counterparty.io/" -"Source code" = "https://github.com/CounterpartyXCP/" -"Home Page" = "https://counterparty.io/" - -[tool.hatch.version] -path = "counterpartylib/lib/config.py" - -[tool.hatch.build.targets.wheel] -include = ["counterpartylib"] - -[tool.hatch.envs.default] -pre-install-commands = [ - "pip install -e ../counterparty-rs", -] - -[tool.license_scanner] -allowed-licences = [ - 'Apache license', - 'Apache license 2.0', - 'BSD 2-clause license', - 'BSD 3-clause license', - 'BSD license', - 'GNU general public license v2 (gplv2)', - 'GNU lesser general public license', - 'GNU lesser general public license v2 (lgplv2)', - 'GNU lesser general public license v3 (lgplv3)', - 'ISC license (iscl)', 'MIT license', - 'Mozilla public license 2.0 (mpl 2.0)', - 'Python software foundation license', - 'The Unlicense (Unlicense)', - 'Public domain', - 'Creative Commons Zero, CC-0', -] -allowed-packages = [ - 'counterparty-core', 'counterparty-lib', 'counterparty-rs', - 'maturin', 'apsw', -] diff --git a/counterparty-lib/requirements.txt b/counterparty-lib/requirements.txt deleted file mode 100644 index 1490115860..0000000000 --- a/counterparty-lib/requirements.txt +++ /dev/null @@ -1,29 +0,0 @@ -apsw==3.45.1.0 -appdirs==1.4.4 -setuptools-markdown==0.4.1 -python-dateutil==2.8.2 -Flask-HTTPAuth==4.8.0 -Flask==3.0.0 -colorlog==6.8.0 -json-rpc==1.15.0 -pycoin==0.92.20230326 -pycryptodome==3.20.0 -ripemd-hash==1.0.1 -safe-pysha3==1.0.4 -pytest==7.4.4 -pytest-cov==4.1.0 -python-bitcoinlib==0.12.2 -requests==2.31.0 -tendo==0.3.0 -xmltodict==0.13.0 -cachetools==5.3.2 -bitstring==4.1.4 -Werkzeug==3.0.1 -itsdangerous==2.1.2 -plyvel==1.5.1 -arc4==0.4.0 -halo==0.0.31 -termcolor==2.4.0 -counterparty-rs==10.1.0 -sentry-sdk==1.45.0 - diff --git a/counterparty-rs/Cargo.lock b/counterparty-rs/Cargo.lock index ef49ad32a7..4d546a6246 100644 --- a/counterparty-rs/Cargo.lock +++ b/counterparty-rs/Cargo.lock @@ -198,7 +198,7 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "counterparty-rs" -version = "10.1.0" +version = "10.1.1" dependencies = [ "bip32", "bitcoin", diff --git a/counterparty-rs/Cargo.toml b/counterparty-rs/Cargo.toml index 89b9d4bd69..161da89745 100644 --- a/counterparty-rs/Cargo.toml +++ b/counterparty-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "counterparty-rs" -version = "10.1.0" +version = "10.1.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/counterparty-rs/README.md b/counterparty-rs/README.md index d61f32ec9e..3c18fe1a85 100644 --- a/counterparty-rs/README.md +++ b/counterparty-rs/README.md @@ -1,6 +1,6 @@ # counterparty_rs -Rust and pyo3-based speed-ups for `counterparty-lib`. +Rust and pyo3-based speed-ups for `counterparty-core`. This is a rust-based python wheel that wraps [rust-bitcoin](https://docs.rs/bitcoin/latest/bitcoin/). diff --git a/counterparty-rs/pyproject.toml b/counterparty-rs/pyproject.toml index f6a8013c14..24547c4fde 100644 --- a/counterparty-rs/pyproject.toml +++ b/counterparty-rs/pyproject.toml @@ -22,4 +22,4 @@ dependencies = [ strip = true [tool.hatch.version] -path = "../counterparty-lib/counterpartylib/lib/config.py" +path = "../counterparty-core/counterpartycore/lib/config.py" diff --git a/counterparty-rs/src/b58.rs b/counterparty-rs/src/b58.rs index 2c1dce84da..00525223dd 100644 --- a/counterparty-rs/src/b58.rs +++ b/counterparty-rs/src/b58.rs @@ -1,5 +1,6 @@ use pyo3::prelude::*; use pyo3::types::PyBytes; +use pyo3::exceptions::PyValueError; #[pyfunction] fn b58_encode(decoded: &[u8]) -> String { @@ -12,17 +13,25 @@ fn b58_encode_list(decoded_list: Vec<&[u8]>) -> Vec { } #[pyfunction] -fn b58_decode<'p>(py: Python<'p>, encoded: &str) -> &'p PyBytes { - let s = bs58::decode(encoded) +fn b58_decode<'p>(py: Python<'p>, encoded: &str) -> PyResult<&'p PyBytes> { + let decoded = bs58::decode(encoded) .with_check(None) - .into_vec() - .expect("bad"); - PyBytes::new(py, &s) + .into_vec(); + + match decoded { + Ok(s) => Ok(PyBytes::new(py, &s)), + Err(_) => Err(PyValueError::new_err("Bad input")), + } } #[pyfunction] -fn b58_decode_list<'p>(py: Python<'p>, encoded_list: Vec<&str>) -> Vec<&'p PyBytes> { - encoded_list.iter().map(|x| b58_decode(py, x)).collect() +fn b58_decode_list<'p>(py: Python<'p>, encoded_list: Vec<&str>) -> PyResult> { + let mut decoded_list = Vec::new(); + for encoded in encoded_list { + let decoded = b58_decode(py, encoded)?; + decoded_list.push(decoded); + } + Ok(decoded_list) } /// A Python module implemented in Rust. diff --git a/counterparty-rs/tests/test_b58.py b/counterparty-rs/tests/test_b58.py index 02d50ce623..59589c958e 100644 --- a/counterparty-rs/tests/test_b58.py +++ b/counterparty-rs/tests/test_b58.py @@ -1,6 +1,12 @@ +import pytest from counterparty_rs import b58 # noqa: F401 # TODO -def test_b58_check_decode(): - pass +def test_b58(): + assert b58.b58_encode(b"hello world") == "3vQB7B6MrGQZaxCuFg4oh" + assert b58.b58_decode("3vQB7B6MrGQZaxCuFg4oh") == b"hello world" + + with pytest.raises(ValueError) as excinfo: + b58.b58_decode("hello world") + assert str(excinfo.value) == "Bad input" diff --git a/counterparty-wallet/counterpartywallet/__init__.py b/counterparty-wallet/counterpartywallet/__init__.py index 3a560c9cd1..aca767f317 100644 --- a/counterparty-wallet/counterpartywallet/__init__.py +++ b/counterparty-wallet/counterpartywallet/__init__.py @@ -1,7 +1,7 @@ import os import sys -from counterpartylib.lib import config +from counterpartycore.lib import config APP_VERSION = config.VERSION_STRING diff --git a/counterparty-wallet/counterpartywallet/client.py b/counterparty-wallet/counterpartywallet/client.py index 2e2b48717e..34f148c168 100755 --- a/counterparty-wallet/counterpartywallet/client.py +++ b/counterparty-wallet/counterpartywallet/client.py @@ -4,9 +4,9 @@ import sys from decimal import Decimal as D -from counterpartylib.lib import config, log, script, setup -from counterpartylib.lib.exceptions import TransactionError -from counterpartylib.lib.util import BET_TYPE_NAME +from counterpartycore.lib import config, log, script, setup +from counterpartycore.lib.exceptions import TransactionError +from counterpartycore.lib.util import BET_TYPE_NAME from counterpartywallet import APP_VERSION, clientapi, console, messages, util, wallet @@ -197,7 +197,7 @@ def main(): "-V", "--version", action="version", - version=f"{APP_NAME} v{APP_VERSION}; counterparty-lib v{config.VERSION_STRING}", + version=f"{APP_NAME} v{APP_VERSION}; counterparty-core v{config.VERSION_STRING}", ) parser.add_argument("--config-file", help="the location of the configuration file") diff --git a/counterparty-wallet/counterpartywallet/clientapi.py b/counterparty-wallet/counterpartywallet/clientapi.py index c45064fdcf..671a941630 100755 --- a/counterparty-wallet/counterpartywallet/clientapi.py +++ b/counterparty-wallet/counterpartywallet/clientapi.py @@ -3,7 +3,7 @@ import sys from urllib.parse import quote_plus as urlencode -from counterpartylib.lib import config, script +from counterpartycore.lib import config, script from counterpartywallet import messages, util, wallet from counterpartywallet.messages import get_pubkeys diff --git a/counterparty-wallet/counterpartywallet/messages.py b/counterparty-wallet/counterpartywallet/messages.py index f71fe864ed..f044a41771 100755 --- a/counterparty-wallet/counterpartywallet/messages.py +++ b/counterparty-wallet/counterpartywallet/messages.py @@ -7,9 +7,9 @@ import bitcoin as bitcoinlib import dateutil.parser -from counterpartylib.lib import config, exceptions, script, transaction -from counterpartylib.lib.kickstart.utils import ib2h -from counterpartylib.lib.util import BET_TYPE_ID, dhash +from counterpartycore.lib import config, exceptions, script, transaction +from counterpartycore.lib.kickstart.utils import ib2h +from counterpartycore.lib.util import BET_TYPE_ID, dhash from counterpartywallet import util, wallet diff --git a/counterparty-wallet/counterpartywallet/util.py b/counterparty-wallet/counterpartywallet/util.py index 8741807095..020a46a1a7 100644 --- a/counterparty-wallet/counterpartywallet/util.py +++ b/counterparty-wallet/counterpartywallet/util.py @@ -4,8 +4,8 @@ import time import requests -from counterpartylib.lib import config -from counterpartylib.lib.util import value_input, value_output +from counterpartycore.lib import config +from counterpartycore.lib.util import value_input, value_output logger = logging.getLogger(config.LOGGER_NAME) D = decimal.Decimal diff --git a/counterparty-wallet/counterpartywallet/wallet/__init__.py b/counterparty-wallet/counterpartywallet/wallet/__init__.py index b1d0438e77..ae2abc3c1f 100644 --- a/counterparty-wallet/counterpartywallet/wallet/__init__.py +++ b/counterparty-wallet/counterpartywallet/wallet/__init__.py @@ -2,7 +2,7 @@ import sys # noqa: E402 from decimal import Decimal as D -from counterpartylib.lib import config, exceptions, script, util # noqa: E402, F401 +from counterpartycore.lib import config, exceptions, script, util # noqa: E402, F401 from pycoin.coins.bitcoin.Tx import Tx # noqa: E402 from pycoin.ecdsa.secp256k1 import secp256k1_generator as generator_secp256k1 # noqa: E402 from pycoin.encoding.sec import public_pair_to_hash160_sec # noqa: E402 diff --git a/counterparty-wallet/pyproject.toml b/counterparty-wallet/pyproject.toml index 22ba4f7fb9..fe43622bfb 100644 --- a/counterparty-wallet/pyproject.toml +++ b/counterparty-wallet/pyproject.toml @@ -33,7 +33,7 @@ files = ["requirements.txt"] [tool.hatch.envs.default] pre-install-commands = [ "pip install -e ../counterparty-rs", - "pip install -e ../counterparty-lib", + "pip install -e ../counterparty-core", ] [project.scripts] @@ -46,4 +46,4 @@ counterparty-wallet = "counterpartywallet:client.main" "Home Page" = "https://counterparty.io/" [tool.hatch.version] -path = "../counterparty-lib/counterpartylib/lib/config.py" +path = "../counterparty-core/counterpartycore/lib/config.py" diff --git a/counterparty-wallet/requirements.txt b/counterparty-wallet/requirements.txt index 631857d5e9..95ef2ba0db 100644 --- a/counterparty-wallet/requirements.txt +++ b/counterparty-wallet/requirements.txt @@ -5,4 +5,4 @@ colorlog==6.8.0 python-dateutil==2.8.2 requests==2.31.0 termcolor==2.4.0 -counterparty-lib==10.1.0 +counterparty-core==10.1.1 diff --git a/docker-compose.yml b/docker-compose.yml index ac1de9aa0b..84cf14f6d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,7 +47,7 @@ services: counterparty-core: - image: counterparty/counterparty:v10.1.0 + image: counterparty/counterparty:v10.1.1 links: - bitcoind - addrindexrs diff --git a/release-notes/release-notes-v10.1.0.md b/release-notes/release-notes-v10.1.0.md index 12d7a920a8..0757f28b2f 100644 --- a/release-notes/release-notes-v10.1.0.md +++ b/release-notes/release-notes-v10.1.0.md @@ -20,7 +20,7 @@ pip3 uninstall counterparty-rs counterparty-lib counterparty-cli * Validate non-empty `block_indexes` in call to `api.get_blocks` (fix for #1621) * Reproduce order expiration bug in v9.61.x (fix for #1631) * Fix `get_blocks` call when several block indexes are provided (fix for #1629) -* Fix `create_send` when one of the output is a dispenser (fix for #1119) +* Fix `create_send` when one of the outputs is a dispenser (fix for #1119) * Fix `get_dispenser_info` RPC call ## Codebase diff --git a/release-notes/release-notes-v10.1.1.md b/release-notes/release-notes-v10.1.1.md new file mode 100644 index 0000000000..8ffceb8afa --- /dev/null +++ b/release-notes/release-notes-v10.1.1.md @@ -0,0 +1,37 @@ +# Release Notes - Counterparty Core v10.1.1 (2024-04-19) + +This is a relatively small release with a number of bugfixes, one of which is critical---in v10.0.x and v10.1.0 there is a bug which can cause nodes to crash upon a blockchain reorganization. + + +# Upgrading + +To upgrade from v10.1.0 manually, you must first uninstall the following Counterparty Core Python packages: + +```bash +pip3 uninstall counterparty-rs counterparty-lib counterparty-cli +``` + +This release contains no protocol changes, and the API has not been modified. + + +# ChangeLog + +## Bugfixes +* Fix missing events (`NEW_BLOCK` and `NEW_TRANSACTION`) when kickstarting and reparsing. To correct the values in the `messages` table, a full reparse is required. +* Fix the current block index after a blockchain reorganisation. +* Fix database shutdown, which caused a recovery of the WAL file on each startup. +* Eliminate some extraneous error messages + +## Codebase +* Merge `counterparty-lib` and `counterparty-core` package into `counterparty-core` +* Integrate telemetry with optional Sentry service + +## Command-Line Interface +* Replace `--no-check-asset-conservation` with `--check-asset-conservation` +* Disable automatic DB integrity check on startup + +# Credits +* Ouziel Slama +* Adam Krellenstein +* Warren Puffett +* Matt Marcello