From a6fc615bc476f489ee125212808cb70754332f9c Mon Sep 17 00:00:00 2001 From: "Matthew D. Scholefield" Date: Mon, 27 Apr 2020 21:43:54 -0500 Subject: [PATCH] Make tests more concise/consistent Fixtures are used consistently A new TempFolder class has been created for reusability A session-wide trained_model fixture has been added to speed up testing --- test/scripts/conftest.py | 38 +++++++++-- test/scripts/test_add_noise.py | 22 ++---- test/scripts/test_combined.py | 14 ++-- test/scripts/test_convert.py | 10 +-- test/scripts/test_engine.py | 9 +-- test/scripts/test_train.py | 27 ++------ test/scripts/test_utils/__init__.py | 13 ++++ test/scripts/test_utils/dummy_noise_folder.py | 35 ++++++++++ test/scripts/test_utils/dummy_train_folder.py | 68 +++++++++++++++++++ .../temp_folder.py} | 26 ++----- 10 files changed, 182 insertions(+), 80 deletions(-) create mode 100644 test/scripts/test_utils/__init__.py create mode 100644 test/scripts/test_utils/dummy_noise_folder.py create mode 100644 test/scripts/test_utils/dummy_train_folder.py rename test/scripts/{dummy_audio_folder.py => test_utils/temp_folder.py} (61%) diff --git a/test/scripts/conftest.py b/test/scripts/conftest.py index c3b763d5..8e73cbf7 100644 --- a/test/scripts/conftest.py +++ b/test/scripts/conftest.py @@ -12,15 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import shutil + import pytest from precise.scripts.train import TrainScript -from test.scripts.test_train import DummyTrainFolder +from test.scripts.test_utils.temp_folder import TempFolder +from test.scripts.test_utils.dummy_train_folder import DummyTrainFolder @pytest.fixture() def train_folder(): - folder = DummyTrainFolder(10) + folder = DummyTrainFolder() + folder.generate_default() try: yield folder finally: @@ -28,5 +32,31 @@ def train_folder(): @pytest.fixture() -def train_script(train_folder): - return TrainScript.create(model=train_folder.model, folder=train_folder.root, epochs=1) +def temp_folder(): + folder = TempFolder() + try: + yield folder + finally: + folder.cleanup() + + +@pytest.fixture(scope='session') +def _trained_model(): + """Session wide model that gets trained once""" + folder = DummyTrainFolder() + folder.generate_default() + script = TrainScript.create(model=folder.model, folder=folder.root, epochs=100) + script.run() + try: + yield folder.model + finally: + folder.cleanup() + + +@pytest.fixture() +def trained_model(_trained_model, temp_folder): + """Copy of session wide model""" + model = temp_folder.path('trained_model.net') + shutil.copy(_trained_model, model) + shutil.copy(_trained_model + '.params', model + '.params') + return model diff --git a/test/scripts/test_add_noise.py b/test/scripts/test_add_noise.py index 2aea0ee2..b25c22a6 100644 --- a/test/scripts/test_add_noise.py +++ b/test/scripts/test_add_noise.py @@ -13,25 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. from precise.scripts.add_noise import AddNoiseScript - -from test.scripts.dummy_audio_folder import DummyAudioFolder - - -class DummyNoiseFolder(DummyAudioFolder): - def __init__(self, count=10): - super().__init__(count) - self.source = self.subdir('source') - self.noise = self.subdir('noise') - self.output = self.subdir('output') - - self.generate_samples(self.subdir('source', 'wake-word'), 'ww-{}.wav', 1.0, self.rand(0, 2)) - self.generate_samples(self.subdir('source', 'not-wake-word'), 'nww-{}.wav', 0.0, self.rand(0, 2)) - self.generate_samples(self.noise, 'noise-{}.wav', 0.5, self.rand(10, 20)) +from test.scripts.test_utils.dummy_noise_folder import DummyNoiseFolder class TestAddNoise: def get_base_data(self, count): - folders = DummyNoiseFolder(count) + folders = DummyNoiseFolder() + folders.generate_default(count) base_args = dict( folder=folders.source, noise_folder=folders.noise, output_folder=folders.output @@ -42,10 +30,10 @@ def test_run_basic(self): folders, base_args = self.get_base_data(10) script = AddNoiseScript.create(inflation_factor=1, **base_args) script.run() - assert folders.count_files(folders.output) == 20 + assert folders.count_files(folders.output) == 40 def test_run_basic_2(self): folders, base_args = self.get_base_data(10) script = AddNoiseScript.create(inflation_factor=2, **base_args) script.run() - assert folders.count_files(folders.output) == 40 + assert folders.count_files(folders.output) == 80 diff --git a/test/scripts/test_combined.py b/test/scripts/test_combined.py index 175a8cc8..2148c62d 100644 --- a/test/scripts/test_combined.py +++ b/test/scripts/test_combined.py @@ -18,6 +18,7 @@ from precise.scripts.calc_threshold import CalcThresholdScript from precise.scripts.eval import EvalScript from precise.scripts.graph import GraphScript +from test.scripts.test_utils.dummy_train_folder import DummyTrainFolder def read_content(filename): @@ -25,21 +26,20 @@ def read_content(filename): return f.read() -def test_combined(train_folder, train_script): +def test_combined(train_folder: DummyTrainFolder, trained_model: str): """Test a "normal" development cycle, train, evaluate and calc threshold. """ - train_script.run() - params_file = train_folder.model + '.params' - assert isfile(train_folder.model) + params_file = trained_model + '.params' + assert isfile(trained_model) assert isfile(params_file) EvalScript.create(folder=train_folder.root, - models=[train_folder.model]).run() + models=[trained_model]).run() # Ensure that the graph script generates a numpy savez file out_file = train_folder.path('outputs.npz') graph_script = GraphScript.create(folder=train_folder.root, - models=[train_folder.model], + models=[trained_model], output_file=out_file) graph_script.run() assert isfile(out_file) @@ -47,6 +47,6 @@ def test_combined(train_folder, train_script): # Esure the params are updated after threshold is calculated params_before = read_content(params_file) CalcThresholdScript.create(folder=train_folder.root, - model=train_folder.model, + model=trained_model, input_file=out_file).run() assert params_before != read_content(params_file) diff --git a/test/scripts/test_convert.py b/test/scripts/test_convert.py index 53dec5cc..45b3b12d 100755 --- a/test/scripts/test_convert.py +++ b/test/scripts/test_convert.py @@ -15,10 +15,10 @@ from os.path import isfile from precise.scripts.convert import ConvertScript +from test.scripts.test_utils.temp_folder import TempFolder -def test_convert(train_folder, train_script): - train_script.run() - - ConvertScript.create(model=train_folder.model, out=train_folder.model + '.pb').run() - assert isfile(train_folder.model + '.pb') +def test_convert(temp_folder: TempFolder, trained_model: str): + pb_model = temp_folder.path('model.pb') + ConvertScript.create(model=trained_model, out=pb_model).run() + assert isfile(pb_model) diff --git a/test/scripts/test_engine.py b/test/scripts/test_engine.py index 09129cd3..620b78fe 100755 --- a/test/scripts/test_engine.py +++ b/test/scripts/test_engine.py @@ -21,6 +21,8 @@ from precise.scripts.engine import EngineScript from runner.precise_runner import ReadWriteStream +from test.scripts.test_utils.dummy_train_folder import DummyTrainFolder + class FakeStdin: def __init__(self, data: bytes): @@ -35,18 +37,17 @@ def __init__(self): self.buffer = ReadWriteStream() -def test_engine(train_folder, train_script): +def test_engine(train_folder: DummyTrainFolder, trained_model: str): """ - Test t hat the output format of the engina matches a decimal form in the + Test t hat the output format of the engine matches a decimal form in the range 0.0 - 1.0. """ - train_script.run() with open(glob.glob(join(train_folder.root, 'wake-word', '*.wav'))[0], 'rb') as f: data = f.read() try: sys.stdin = FakeStdin(data) sys.stdout = FakeStdout() - EngineScript.create(model_name=train_folder.model).run() + EngineScript.create(model_name=trained_model).run() assert re.match(rb'[01]\.[0-9]+', sys.stdout.buffer.buffer) finally: sys.stdin = sys.__stdin__ diff --git a/test/scripts/test_train.py b/test/scripts/test_train.py index 20e1ccb7..3e208fcb 100644 --- a/test/scripts/test_train.py +++ b/test/scripts/test_train.py @@ -14,29 +14,14 @@ # limitations under the License. from os.path import isfile -from precise.params import pr from precise.scripts.train import TrainScript -from test.scripts.dummy_audio_folder import DummyAudioFolder - - -class DummyTrainFolder(DummyAudioFolder): - def __init__(self, count=10): - super().__init__(count) - self.generate_samples(self.subdir('wake-word'), 'ww-{}.wav', 1.0, - self.rand(0, 2 * pr.buffer_t)) - self.generate_samples(self.subdir('not-wake-word'), 'nww-{}.wav', 0.0, - self.rand(0, 2 * pr.buffer_t)) - self.generate_samples(self.subdir('test', 'wake-word'), 'ww-{}.wav', - 1.0, self.rand(0, 2 * pr.buffer_t)) - self.generate_samples(self.subdir('test', 'not-wake-word'), - 'nww-{}.wav', 0.0, self.rand(0, 2 * pr.buffer_t)) - self.model = self.path('model.net') +from test.scripts.test_utils.dummy_train_folder import DummyTrainFolder class TestTrain: - def test_run_basic(self): + def test_run_basic(self, train_folder: DummyTrainFolder): """Run a training and check that a model is generated.""" - folders = DummyTrainFolder(10) - script = TrainScript.create(model=folders.model, folder=folders.root) - script.run() - assert isfile(folders.model) + train_script = TrainScript.create(model=train_folder.model, folder=train_folder.root, epochs=10) + train_script.run() + assert isfile(train_folder.model) + assert isfile(train_folder.model + '.params') diff --git a/test/scripts/test_utils/__init__.py b/test/scripts/test_utils/__init__.py new file mode 100644 index 00000000..57014e8e --- /dev/null +++ b/test/scripts/test_utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 Mycroft AI Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/scripts/test_utils/dummy_noise_folder.py b/test/scripts/test_utils/dummy_noise_folder.py new file mode 100644 index 00000000..be433eff --- /dev/null +++ b/test/scripts/test_utils/dummy_noise_folder.py @@ -0,0 +1,35 @@ +# Copyright 2020 Mycroft AI Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np + +from test.scripts.test_utils.temp_folder import TempFolder +from test.scripts.test_utils.dummy_train_folder import DummyTrainFolder + + +class DummyNoiseFolder(TempFolder): + def __init__(self): + super().__init__() + self.source = self.subdir('source') + self.noise = self.subdir('noise') + self.output = self.subdir('output') + + self.source_folder = DummyTrainFolder(root=self.source) + self.noise_folder = DummyTrainFolder(root=self.noise) + + def generate_default(self, count=10): + self.source_folder.generate_default(count) + self.noise_folder.generate_samples( + count, [], 'noise-{}.wav', + lambda: np.ones(self.noise_folder.get_duration(), dtype=float) + ) diff --git a/test/scripts/test_utils/dummy_train_folder.py b/test/scripts/test_utils/dummy_train_folder.py new file mode 100644 index 00000000..22517bc7 --- /dev/null +++ b/test/scripts/test_utils/dummy_train_folder.py @@ -0,0 +1,68 @@ +# Copyright 2020 Mycroft AI Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import random +from os.path import join + +import numpy as np + +from precise.params import pr +from precise.util import save_audio +from test.scripts.test_utils.temp_folder import TempFolder + + +class DummyTrainFolder(TempFolder): + def __init__(self, root=None): + super().__init__(root) + self.model = self.path('model.net') + + def generate_samples(self, count, subfolder, name, generator): + """ + Generate sample audio files in a folder + + The file is generated in the specified folder, with the specified + name and generated value. + + Args: + count: Number of samples to generate + subfolder: String or list of subfolder path + name: Format string used to generate each sample + generator: Function called to get the data for each sample + """ + if isinstance(subfolder, str): + subfolder = [subfolder] + for i in range(count): + save_audio(join(self.subdir(*subfolder), name.format(i)), generator()) + + def get_duration(self): + """Generate a random sample duration""" + return int(random.random() * 2 * pr.buffer_samples) + + def generate_default(self, count=10): + self.generate_samples( + count, 'wake-word', 'ww-{}.wav', + lambda: np.ones(self.get_duration(), dtype=float) + ) + self.generate_samples( + count, 'not-wake-word', 'nww-{}.wav', + lambda: np.random.random(self.get_duration()) * 2 - 1 + ) + self.generate_samples( + count, ('test', 'wake-word'), 'ww-{}.wav', + lambda: np.ones(self.get_duration(), dtype=float) + ) + self.generate_samples( + count, ('test', 'not-wake-word'), 'nww-{}.wav', + lambda: np.random.random(self.get_duration()) * 2 - 1 + ) + self.model = self.path('model.net') \ No newline at end of file diff --git a/test/scripts/dummy_audio_folder.py b/test/scripts/test_utils/temp_folder.py similarity index 61% rename from test/scripts/dummy_audio_folder.py rename to test/scripts/test_utils/temp_folder.py index ae30b764..ecc5ab6b 100644 --- a/test/scripts/dummy_audio_folder.py +++ b/test/scripts/test_utils/temp_folder.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2019 Mycroft AI Inc. +# Copyright 2020 Mycroft AI Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,36 +14,18 @@ # limitations under the License. import atexit -import numpy as np import os from os import makedirs from os.path import isdir, join from shutil import rmtree from tempfile import mkdtemp -from precise.params import pr -from precise.util import save_audio - -class DummyAudioFolder: - def __init__(self, count=10): - self.count = count - self.root = mkdtemp() +class TempFolder: + def __init__(self, root=None): + self.root = mkdtemp() if root is None else root atexit.register(self.cleanup) - def rand(self, min, max): - return min + (max - min) * np.random.random() * pr.buffer_t - - def generate_samples(self, folder, name, value, duration): - """Generate sample file. - - The file is generated in the specified folder, with the specified name, - dummy value and duration. - """ - for i in range(self.count): - save_audio(join(folder, name.format(i)), - np.array([value] * int(duration * pr.sample_rate))) - def subdir(self, *parts): folder = self.path(*parts) if not isdir(folder):