Adding the testing folder and some basic tests

The tests folder contains files with the prefix "test_" which contain unit tests.  The remaining folders include config files utilized by the tests and sequestered database files to be used by the tests.

data_files/.gitignore

@@ -0,0 +1,6 @@
+# locally ignore all .dat files and newly added .sqlite db's
+*.sqlite
+*.dat
+*.xlsx
+*.log
+

tests/conftest.py

@@ -0,0 +1,21 @@
+import logging
+import os
+
+from definitions import PROJECT_ROOT
+
+# set up logger in conftest.py so that it is properly anchored in the test folder.
+
+# set the target folder for output from testing
+output_path = os.path.join(PROJECT_ROOT, "tests", "test_log")
+if not os.path.exists(output_path):
+    os.mkdir(output_path)
+
+logging.getLogger("pyomo").setLevel(logging.INFO)
+filename = "testing.log"
+logging.basicConfig(
+    filename=os.path.join(output_path, filename),
+    filemode="w",
+    format="%(asctime)s | %(module)s | %(levelname)s | %(message)s",
+    datefmt="%d-%b-%y %H:%M:%S",
+    level=logging.DEBUG,  # <-- global change for testing activities is here
+)

tests/legacy_test_values.py

@@ -0,0 +1,26 @@
+"""
+a container for test values from legacy code (Python 3.7 / Pyomo 5.5) captured for continuity/development testing
+"""
+from enum import Enum
+
+# Written by:  J. F. Hyink
+# [email protected]
+# https://westernspark.us
+# Created on:  6/27/23
+
+
+class TestVals(Enum):
+    OBJ_VALUE = 'obj_value'
+    EFF_DOMAIN_SIZE = 'eff_domain_size'
+    EFF_INDEX_SIZE = 'eff_index_size'
+
+
+# these values were captured on base level runs of the .dat files in the tests/testing_data folder
+test_vals = {'config_test_system': {TestVals.OBJ_VALUE: 491977.7000753,
+                                    TestVals.EFF_DOMAIN_SIZE: 30720,
+                                    TestVals.EFF_INDEX_SIZE: 74},
+             'config_utopia': {TestVals.OBJ_VALUE: 36535.631200,
+                               TestVals.EFF_DOMAIN_SIZE: 12312,
+                               TestVals.EFF_INDEX_SIZE: 64},
+             }
+

tests/myopic_utopia_results/README.txt

@@ -0,0 +1,7 @@
+The outputs csv file in this folder is a capture of myopic run on Utopia dataset for 2 time periods.
+
+myopic runs don't (easily) produce an objective value that can be accessed and it seems that the emissions
+values should be an OK proxy for that.  This particular commodity has emissions values in all the years of interest
+across the myopic periods and should serve as an OK functional check.
+
+To be done:  After re-working the temoa_myopic runs, use this as a basis of comparison

tests/myopic_utopia_results/emissions_outputs.csv

@@ -0,0 +1,2 @@
+Region,Technology,EmissionCommodity,Sector,1990,2000,2010
+utopia,IMPDSL1,co2,supply,2.89480516875,2.45492238525,5.453948355

tests/temoa_model/README.txt

@@ -0,0 +1,8 @@
+This file location is only to support the current setup for temoa_myopic runs which use this file location (relative to
+the test) to store the run statements needed.
+
+Running the myopic tests is currently hard-coded to use this folder name, so it (currently) should not be changed and
+the config file herein is dynamically generated/updated by running the test suite.
+
+This folder includes the necessary sqlite databases to run the tests, which are currently (unfortunately) part of the
+git repo as a convenience.  Eventually, they should be removed from VCS to prevent bloat.

tests/temoa_model/config_sample

@@ -0,0 +1,57 @@
+#-----------------------------------------------------
+# This is a sample configuration file for Temoa
+# It allows you to specify (and document) all run-time model options 
+# Legal chars in path: a-z A-Z 0-9 - _ \ / . : 
+# Comment out non-mandatory options to omit them
+#-----------------------------------------------------
+
+# Input File (Mandatory) 
+# Input can be a .sqlite or .dat file
+# Both relative path and absolute path are accepted
+--input=testing_data/temoa_utopia.dat
+
+# Output File (Mandatory)
+# The output file must be a existing .sqlite file
+--output=testing_data/temoa_utopia.sqlite
+
+# Scenario Name (Mandatory) 
+# This scenario name is used to store results within the output .sqlite file
+--scenario=test_run
+
+# Path to folder containing input dataset (Mandatory)
+# This is the location where database files reside
+--path_to_data=testing_data
+
+# Solve Myopically (Optional)
+# Allows user to solve one model time period at a time, sequentially
+# Default operation is "perfect foresight"
+--myopic
+--myopic_periods=2
+#--keep_myopic_databases
+
+# Report Duals (Optional) 
+# Store Duals results in the output .sqlite file
+#--saveDUALS
+
+# Spreadsheet Output (Optional) 
+# Direct model output to a spreadsheet
+# Scenario name specified above is used to name the spreadsheet
+--saveEXCEL
+
+# Save the log file output (Optional)
+# This is the same output provided to the shell
+# --saveTEXTFILE
+
+# Solver-related arguments (Optional)
+#--neos                            # Optional, specify if you want to use NEOS server to solve
+#--solver=cplex                    # Optional, indicate the solver
+#--keep_pyomo_lp_file             # Optional, generate Pyomo-compatible LP file
+--solver=cbc
+
+# Modeling-to-Generate Alternatives (Optional)
+# Run name will be automatically generated by appending '_mga_' and iteration number to scenario name
+#--mga {
+#	slack=0.1                     # Objective function slack value in MGA runs
+#	iteration=4                   # Number of MGA iterations
+#	weight=integer                # MGA objective function weighting method, currently "integer" or "normalized"
+#}

tests/test_full_runs.py

@@ -0,0 +1,104 @@
+"""
+Test a couple full-runs to match objective function value and some internals
+"""
+import logging
+import os
+import pathlib
+import shutil
+import sqlite3
+
+import pyomo.environ as pyo
+import pytest
+
+from definitions import PROJECT_ROOT
+# from src.temoa_model.temoa_model import temoa_create_model
+from temoa.temoa_model.temoa_model import temoa_create_model
+from temoa.temoa_model.temoa_run import TemoaSolver
+from tests.legacy_test_values import TestVals, test_vals
+
+# Written by:  J. F. Hyink
+# [email protected]
+# https://westernspark.us
+# Created on:  6/27/23
+
+
+logger = logging.getLogger(__name__)
+# list of test scenarios for which we have captured results in legacy_test_values.py
+legacy_config_files = ['config_utopia', 'config_test_system', ]
+
+
+@pytest.fixture(params=legacy_config_files)
+def system_test_run(request):
+    """
+    spin up the model, solve it, and hand over the model and result for inspection
+    """
+    filename = request.param
+    config_file = pathlib.Path(PROJECT_ROOT, 'tests', 'testing_configs', filename)
+    # make a TemoaSolver and pass it a model instance and the config file
+    model = temoa_create_model()  # TemoaModel()  <-- for after conversion
+    temoa_solver = TemoaSolver(model, config_filename=config_file)
+    for _ in temoa_solver.createAndSolve():
+        pass
+
+    instance_object = temoa_solver.instance_hook
+    res = instance_object.result
+    mdl = instance_object.instance
+    return filename, res, mdl
+
+
+def test_against_legacy_outputs(system_test_run):
+    """
+    This test compares tests of legacy models to captured test results
+    """
+    filename, res, mdl = system_test_run
+    logger.info("Starting output test on scenario: %s", filename)
+    expected_vals = test_vals.get(filename)  # a dictionary of expected results
+
+    # inspect some summary results
+    assert pyo.value(res['Solution'][0]['Status'].key) == 'optimal'
+    assert pyo.value(res['Solution'][0]['Objective']['TotalCost']['Value']) == pytest.approx(
+        expected_vals[TestVals.OBJ_VALUE], 0.00001)
+
+    # inspect a couple set sizes
+    efficiency_param: pyo.Param = mdl.Efficiency
+    assert len(tuple(efficiency_param.sparse_iterkeys())) == expected_vals[
+        TestVals.EFF_INDEX_SIZE], 'should match legacy numbers'
+    assert len(efficiency_param._index) == expected_vals[TestVals.EFF_DOMAIN_SIZE], 'should match legacy numbers'
+
+
+@pytest.mark.skip('not ready yet...')
+def test_myopic_utopia():
+    """
+    test the myopic functionality on Utopia.  We need to copy the source db to make the output and then erase
+    it because re-runs with the same output db are not possible....get "UNIQUE" errors in db on 2nd run
+
+    We will use the output target in the config file for this test as a shortcut to make/remove the database
+
+    This test will change after conversion of temoa_myopic.py.  RN, it is a good placeholder
+
+    """
+    eps = 1e-3
+    config_file = pathlib.Path(PROJECT_ROOT, 'tests', 'testing_configs', 'config_utopia_myopic')
+    # config_file = pathlib.Path(PROJECT_ROOT, 'tests', 'testing_configs', 'config_utopia_myopic')
+    input_db = pathlib.Path(PROJECT_ROOT, 'tests', 'testing_data', 'temoa_utopia.sqlite')
+    output_db = pathlib.Path(PROJECT_ROOT, 'tests', 'testing_data', 'temoa_utopia_output_catcher.sqlite')
+    if os.path.isfile(output_db):
+        os.remove(output_db)
+    shutil.copy(input_db, output_db)  # put a new copy in place, ones that are used before fail.
+    model = temoa_create_model()  # TODO: TemoaModel()
+    temoa_solver = TemoaSolver(model, config_filename=config_file)
+    for _ in temoa_solver.createAndSolve():
+        pass
+    # inspect the output db for results
+    con = sqlite3.connect(output_db)
+    cur = con.cursor()
+    query = "SELECT t_periods, emissions FROM Output_Emissions WHERE tech is 'IMPDSL1'"
+    emission = cur.execute(query).fetchall()
+
+    # The emissions for diesel are present in each year and should be a good proxy for comparing
+    # results
+    diesel_emissions_by_year = {y: e for (y, e) in emission}
+    assert abs(diesel_emissions_by_year[1990] - 2.8948) < eps
+    assert abs(diesel_emissions_by_year[2000] - 2.4549) < eps
+    assert abs(diesel_emissions_by_year[2010] - 5.4539) < eps
+    os.remove(output_db)

tests/test_set_consistency.py

@@ -0,0 +1,70 @@
+"""
+These tests are designed to check the construction of the numerous sets in the 2 exemplar models:
+Utopia and Test System.
+
+They construct all the pyomo Sets associated with the model and compare them with cached results that are stored
+in json files
+"""
+
+# Written by:  J. F. Hyink
+# [email protected]
+# https://westernspark.us
+# Created on:  9/26/23
+
+import json
+import pathlib
+
+from pyomo import environ as pyo
+
+from definitions import PROJECT_ROOT
+from temoa.temoa_model.temoa_model import TemoaModel, temoa_create_model
+from temoa.temoa_model.temoa_run import TemoaSolver
+
+
+def test_upoptia_set_consistency():
+    """
+    test the set membership of the utopia model against cached values to ensure consistency
+    """
+    config_file = pathlib.Path(PROJECT_ROOT, 'tests', 'testing_configs', 'config_utopia')
+    model = temoa_create_model()  # TODO: TemoaModel()
+    temoa_solver = TemoaSolver(model=model, config_filename=config_file)
+    for _ in temoa_solver.createAndSolve():
+        pass
+
+    # capture the sets within the model
+    model_sets = temoa_solver.instance_hook.instance.component_map(ctype=pyo.Set)
+    model_sets = {k: set(v) for k, v in model_sets.items()}
+
+    # retrieve the cache and convert the set values from list -> set (json can't store sets)
+    cache_file = pathlib.Path(PROJECT_ROOT, 'tests', 'testing_data', 'utopia_sets.json')
+    with open(cache_file, 'r') as src:
+        cached_sets = json.load(src)
+    cached_sets = {k: set(tuple(t) if isinstance(t, list) else t for t in v) for (k, v) in cached_sets.items()}
+
+    sets_match = model_sets == cached_sets
+    # TODO:  The matching above is abstracted from the assert statement because if it fails, the output appears
+    #        to be difficult to process.  If it becomes useful, a better assert would be for matching the keys
+    #        then contents separately and sequentially (for set contents) so that error ouptut is "small"
+    #        same for test below for test_system
+    assert sets_match, 'The Test System run-produced sets did not match cached values'
+
+
+def test_test_system_set_consistency():
+    """
+    Test the set membership of the Test System model against cache.
+    """
+    # this could be combined with the similar test for utopia to use the fixture at some time...
+    config_file = pathlib.Path(PROJECT_ROOT, 'tests', 'testing_configs', 'config_test_system')
+    model = temoa_create_model()  # TemoaModel()
+    temoa_solver = TemoaSolver(model=model, config_filename=config_file)
+    for _ in temoa_solver.createAndSolve():
+        pass
+    model_sets = temoa_solver.instance_hook.instance.component_map(ctype=pyo.Set)
+    model_sets = {k: set(v) for k, v in model_sets.items()}
+
+    cache_file = pathlib.Path(PROJECT_ROOT, 'tests', 'testing_data', 'test_system_sets.json')
+    with open(cache_file, 'r') as src:
+        cached_sets = json.load(src)
+    cached_sets = {k: set(tuple(t) if isinstance(t, list) else t for t in v) for (k, v) in cached_sets.items()}
+    sets_match = model_sets == cached_sets
+    assert sets_match, 'The Test System run-produced sets did not match cached values'

tests/testing_configs/README.txt

@@ -0,0 +1,4 @@
+These config files are used when running the tests in this folder.  They should not need modification.
+
+The "for utility" configs are constructed so that they may be accessed from the tests/utilities folders for
+the purpose of running things from there, currently to gather set components for testing.

tests/testing_configs/config_test_system

@@ -0,0 +1,57 @@
+#-----------------------------------------------------
+# This is a sample configuration file for Temoa
+# It allows you to specify (and document) all run-time model options 
+# Legal chars in path: a-z A-Z 0-9 - _ \ / . : 
+# Comment out non-mandatory options to omit them
+#-----------------------------------------------------
+
+# Input File (Mandatory) 
+# Input can be a .sqlite or .dat file
+# Both relative path and absolute path are accepted
+--input=testing_data/temoa_test_system.dat
+
+# Output File (Mandatory)
+# The output file must be a existing .sqlite file
+--output=data_files/temoa_test_system.sqlite
+
+# Scenario Name (Mandatory) 
+# This scenario name is used to store results within the output .sqlite file
+--scenario=test_run
+
+# Path to folder containing input dataset (Mandatory)
+# This is the location where database files reside
+--path_to_data=testing_data
+
+# Solve Myopically (Optional)
+# Allows user to solve one model time period at a time, sequentially
+# Default operation is "perfect foresight"
+#--myopic
+#--myopic_periods=2
+#--keep_myopic_databases
+
+# Report Duals (Optional) 
+# Store Duals results in the output .sqlite file
+#--saveDUALS
+
+# Spreadsheet Output (Optional) 
+# Direct model output to a spreadsheet
+# Scenario name specified above is used to name the spreadsheet
+--saveEXCEL
+
+# Save the log file output (Optional)
+# This is the same output provided to the shell
+# --saveTEXTFILE
+
+# Solver-related arguments (Optional)
+#--neos                            # Optional, specify if you want to use NEOS server to solve
+#--solver=cplex                    # Optional, indicate the solver
+#--keep_pyomo_lp_file             # Optional, generate Pyomo-compatible LP file
+--solver=cbc
+
+# Modeling-to-Generate Alternatives (Optional)
+# Run name will be automatically generated by appending '_mga_' and iteration number to scenario name
+#--mga {
+#	slack=0.1                     # Objective function slack value in MGA runs
+#	iteration=4                   # Number of MGA iterations
+#	weight=integer                # MGA objective function weighting method, currently "integer" or "normalized"
+#}