Skip to content

Commit

Permalink
Merge pull request #1320 from ethereum/testgen-reorg
Browse files Browse the repository at this point in the history
Implement proposed test organization and structure improvements
  • Loading branch information
djrtwo authored Aug 3, 2019
2 parents ab156f3 + 63e2915 commit 73a9e86
Show file tree
Hide file tree
Showing 70 changed files with 1,563 additions and 948 deletions.
4 changes: 2 additions & 2 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,13 @@ commands:
description: "Restore the cache with pyspec keys"
steps:
- restore_cached_venv:
venv_name: v3-pyspec-bump2
venv_name: v4-pyspec
reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}
save_pyspec_cached_venv:
description: Save a venv into a cache with pyspec keys"
steps:
- save_cached_venv:
venv_name: v3-pyspec-bump2
venv_name: v4-pyspec
reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}
venv_path: ./test_libs/pyspec/venv
restore_deposit_contract_cached_venv:
Expand Down
46 changes: 25 additions & 21 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,20 @@ SPEC_DIR = ./specs
SCRIPT_DIR = ./scripts
TEST_LIBS_DIR = ./test_libs
PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
YAML_TEST_DIR = ./eth2.0-spec-tests/tests
TEST_VECTOR_DIR = ./eth2.0-spec-tests/tests
GENERATOR_DIR = ./test_generators
DEPOSIT_CONTRACT_DIR = ./deposit_contract
CONFIGS_DIR = ./configs

# Collect a list of generator names
GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/)))
# Map this list of generator paths to a list of test output paths
YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERATORS))
GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.)))
# Map this list of generator paths to "gen_{generator name}" entries
GENERATOR_TARGETS = $(patsubst $(GENERATOR_DIR)/%/, gen_%, $(GENERATORS))
GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS))

# To check generator matching:
#$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}])

PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py
PY_SPEC_PHASE_0_DEPS = $(SPEC_DIR)/core/0_*.md

Expand All @@ -24,14 +27,14 @@ PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS)
COV_HTML_OUT=.htmlcov
COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html

.PHONY: clean all test citest lint gen_yaml_tests pyspec phase0 phase1 install_test open_cov \
.PHONY: clean partial_clean all test citest lint generate_tests pyspec phase0 phase1 install_test open_cov \
install_deposit_contract_test test_deposit_contract compile_deposit_contract

all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
all: $(PY_SPEC_ALL_TARGETS)

# deletes everything except the venvs
partial_clean:
rm -rf $(YAML_TEST_DIR)
rm -rf $(TEST_VECTOR_DIR)
rm -rf $(GENERATOR_VENVS)
rm -rf $(PY_SPEC_DIR)/.pytest_cache
rm -rf $(PY_SPEC_ALL_TARGETS)
Expand All @@ -44,8 +47,8 @@ clean: partial_clean
rm -rf $(PY_SPEC_DIR)/venv
rm -rf $(DEPOSIT_CONTRACT_DIR)/venv

# "make gen_yaml_tests" to run generators
gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS)
# "make generate_tests" to run all generators
generate_tests: $(PY_SPEC_ALL_TARGETS) $(GENERATOR_TARGETS)

# installs the packages to run pyspec tests
install_test:
Expand Down Expand Up @@ -90,8 +93,8 @@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS)

CURRENT_DIR = ${CURDIR}

# The function that builds a set of suite files, by calling a generator for the given type (param 1)
define build_yaml_tests
# Runs a generator, identified by param 1
define run_generator
# Started!
# Create output directory
# Navigate to the generator
Expand All @@ -100,24 +103,25 @@ define build_yaml_tests
# Install all the necessary requirements
# Run the generator. The generator is assumed to have an "main.py" file.
# We output to the tests dir (generator program should accept a "-o <filepath>" argument.
# `-l minimal general` can be added to the generator call to filter to smaller configs, when testing.
echo "generator $(1) started"; \
mkdir -p $(YAML_TEST_DIR)$(1); \
cd $(GENERATOR_DIR)$(1); \
mkdir -p $(TEST_VECTOR_DIR); \
cd $(GENERATOR_DIR)/$(1); \
if ! test -d venv; then python3 -m venv venv; fi; \
. venv/bin/activate; \
pip3 install -r requirements.txt; \
python3 main.py -o $(CURRENT_DIR)/$(YAML_TEST_DIR)$(1) -c $(CURRENT_DIR)/$(CONFIGS_DIR); \
python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) -c $(CURRENT_DIR)/$(CONFIGS_DIR); \
echo "generator $(1) finished"
endef

# The tests dir itself is simply build by creating the directory (recursively creating deeper directories if necessary)
$(YAML_TEST_DIR):
$(info creating directory, to output yaml targets to: ${YAML_TEST_TARGETS})
$(TEST_VECTOR_DIR):
$(info creating test output directory, for generators: ${GENERATOR_TARGETS})
mkdir -p $@
$(YAML_TEST_DIR)/:
$(info ignoring duplicate yaml tests dir)
$(TEST_VECTOR_DIR)/:
$(info ignoring duplicate tests dir)

# For any target within the tests dir, build it using the build_yaml_tests function.
# For any generator, build it using the run_generator function.
# (creation of output dir is a dependency)
$(YAML_TEST_DIR)%: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR)
$(call build_yaml_tests,$*)
gen_%: $(PY_SPEC_ALL_TARGETS) $(TEST_VECTOR_DIR)
$(call run_generator,$*)
36 changes: 36 additions & 0 deletions configs/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Configs

This directory contains a set of constants presets used for testing, testnets, and mainnet.

A preset file contains all the constants known for its target.
Later-fork constants can be ignored, e.g. ignore phase1 constants as a client that only supports phase 0 currently.


## Forking

Configs are not replaced, but extended with forks. This is to support syncing from one state to the other over a fork boundary, without hot-swapping a config.
Instead, for forks that introduce changes in a constant, the constant name is prefixed with a short abbreviation of the fork.

Over time, the need to sync an older state may be deprecated.
In this case, the prefix on the new constant may be removed, and the old constant will keep a special name before completely being removed.

A previous iteration of forking made use of "timelines", but this collides with the definitions used in the spec (constants for special forking slots etc.),
and was not integrated sufficiently in any of the spec tools or implementations.
Instead, the config essentially doubles as fork definition now, changing the value for e.g. `PHASE_1_GENESIS_SLOT` changes the fork.

Another reason to prefer forking through constants is the ability to program a forking moment based on context, instead of being limited to a static slot number.


## Format

Each preset is a key-value mapping.

**Key**: an `UPPER_SNAKE_CASE` (a.k.a. "macro case") formatted string, name of the constant.

**Value** can be either:
- an unsigned integer number, can be up to 64 bits (incl.)
- a hexadecimal string, prefixed with `0x`

Presets may contain comments to describe the values.

See [`mainnet.yaml`](./mainnet.yaml) for a complete example.
20 changes: 0 additions & 20 deletions configs/constant_presets/README.md

This file was deleted.

19 changes: 0 additions & 19 deletions configs/fork_timelines/README.md

This file was deleted.

12 changes: 0 additions & 12 deletions configs/fork_timelines/mainnet.yaml

This file was deleted.

6 changes: 0 additions & 6 deletions configs/fork_timelines/testing.yaml

This file was deleted.

File renamed without changes.
File renamed without changes.
Loading

0 comments on commit 73a9e86

Please sign in to comment.