diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 474b5d2..517a4a9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,77 +2,68 @@ ci: autoupdate_commit_msg: "chore: update pre-commit hooks" autofix_commit_msg: "style: pre-commit fixes" -repos: - - repo: https://github.com/psf/black-pre-commit-mirror - rev: "23.7.0" - hooks: - - id: black-jupyter - - repo: https://github.com/adamchainz/blacken-docs - rev: "1.16.0" - hooks: - - id: blacken-docs - additional_dependencies: [black==23.7.0] +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + - id: check-symlinks + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + exclude_types: [svg] + - id: mixed-line-ending + - id: requirements-txt-fixer + - id: trailing-whitespace + - id: name-tests-test + args: ["--pytest-test-first"] - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: "v4.4.0" - hooks: - - id: check-added-large-files - - id: check-case-conflict - - id: check-merge-conflict - - id: check-symlinks - - id: check-yaml - - id: debug-statements - - id: end-of-file-fixer - - id: mixed-line-ending - - id: name-tests-test - args: ["--pytest-test-first"] - - id: requirements-txt-fixer - - id: trailing-whitespace +- repo: https://github.com/asottile/setup-cfg-fmt + rev: v2.4.0 + hooks: + - id: setup-cfg-fmt + args: [--include-version-classifiers, --max-py-version=3.11] - - repo: https://github.com/pre-commit/pygrep-hooks - rev: "v1.10.0" - hooks: - - id: rst-backticks - - id: rst-directive-colons - - id: rst-inline-touching-normal +- repo: https://github.com/psf/black-pre-commit-mirror + rev: 23.9.1 + hooks: + - id: black - - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v3.0.3" - hooks: - - id: prettier - types_or: [yaml, markdown, html, css, scss, javascript, json] - args: [--prose-wrap=always] +- repo: https://github.com/cheshirekow/cmake-format-precommit + rev: v0.6.13 + hooks: + - id: cmake-format + additional_dependencies: [pyyaml] - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.0.287" - hooks: - - id: ruff - args: ["--fix", "--show-fixes"] +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.291 + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] - - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.5.1" - hooks: - - id: mypy - files: src|tests - args: [] - additional_dependencies: - - pytest +- repo: https://github.com/codespell-project/codespell + rev: v2.2.5 + hooks: + - id: codespell + args: ["-L", "ue,subjet,parms,fo,numer,thre,gaus"] - - repo: https://github.com/codespell-project/codespell - rev: "v2.2.5" - hooks: - - id: codespell +- repo: local + hooks: + - id: disallow-caps + name: disallow improper capitalization + language: pygrep + entry: PyBind|Cmake|CCache|Github|PyTest + exclude: .pre-commit-config.yaml - - repo: https://github.com/shellcheck-py/shellcheck-py - rev: "v0.9.0.5" - hooks: - - id: shellcheck +- repo: https://github.com/shellcheck-py/shellcheck-py + rev: "v0.9.0.5" + hooks: + - id: shellcheck - - repo: local - hooks: - - id: disallow-caps - name: Disallow improper capitalization - language: pygrep - entry: PyBind|Numpy|Cmake|CCache|Github|PyTest - exclude: .pre-commit-config.yaml +- repo: https://github.com/asottile/pyupgrade + rev: v3.13.0 + hooks: + - id: pyupgrade diff --git a/README.md b/README.md index 7d7f3b2..427a85f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Odapt +# odapt [![Actions Status][actions-badge]][actions-link] [![Documentation Status][rtd-badge]][rtd-link] diff --git a/docs/conf.py b/docs/conf.py index 6316841..6e7644e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,7 +2,7 @@ import importlib.metadata -project = "Odapt" +project = "odapt" copyright = "2023, Zoë Bilodeau" author = "Zoë Bilodeau" version = release = importlib.metadata.version("odapt") diff --git a/docs/index.md b/docs/index.md index f926b6f..18cdf68 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,4 +1,4 @@ -# Odapt +# odapt ```{toctree} :maxdepth: 2 diff --git a/pyproject.toml b/pyproject.toml index 5edf88b..dcaa5e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ "Typing :: Typed", ] dynamic = ["version"] -dependencies = [] +dependencies = ["numpy>=1.18.0", "uproot>=5.0.0"] [project.optional-dependencies] test = [ @@ -49,10 +49,10 @@ docs = [ ] [project.urls] -Homepage = "https://github.com/zbilodea/Odapt" -"Bug Tracker" = "https://github.com/zbilodea/Odapt/issues" -Discussions = "https://github.com/zbilodea/Odapt/discussions" -Changelog = "https://github.com/zbilodea/Odapt/releases" +Homepage = "https://github.com/zbilodea/odapt" +"Bug Tracker" = "https://github.com/zbilodea/odapt/issues" +Discussions = "https://github.com/zbilodea/odapt/discussions" +Changelog = "https://github.com/zbilodea/odapt/releases" [tool.hatch] @@ -160,3 +160,6 @@ messages_control.disable = [ "missing-module-docstring", "wrong-import-position", ] + +[project.scripts] +odapt = "odapt.src.operations.hadd:main" diff --git a/src/odapt/__init__.py b/src/odapt/__init__.py index 872d7dc..5e1880d 100644 --- a/src/odapt/__init__.py +++ b/src/odapt/__init__.py @@ -1,10 +1,11 @@ """ Copyright (c) 2023 Zoë Bilodeau. All rights reserved. -Odapt: File conversion package. +odapt: File conversion package. """ from __future__ import annotations from odapt._version import version as __version__ +from odapt.operations import hadd # noqa: F401 __all__ = ["__version__"] diff --git a/src/odapt/operations/__init__.py b/src/odapt/operations/__init__.py new file mode 100644 index 0000000..d477324 --- /dev/null +++ b/src/odapt/operations/__init__.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +from odapt.operations.hadd import hadd, main # noqa: F401 diff --git a/src/odapt/operations/hadd.py b/src/odapt/operations/hadd.py new file mode 100644 index 0000000..f2bb792 --- /dev/null +++ b/src/odapt/operations/hadd.py @@ -0,0 +1,478 @@ +from __future__ import annotations + +import argparse +from pathlib import Path + +import numpy as np +import uproot + + +def hadd_1d(destination, file, key, first): + """ + Args: + destination (path-like): Name of the output file or file path. + file (ReadOnlyDirectory): ROOT file to read histogram from. + key (str): key to reference histogram to be added. + first (bool): if True, special case for first of a certain histogram + to be added to the new file. + """ + outfile = uproot.open(destination) + try: + hist = file[key] + except ValueError: + msg = "Key missing from {file}" + raise ValueError(msg) from None + if first: + member_data = np.array( + [ + hist.member("fEntries"), + hist.member("fTsumw"), + hist.member("fTsumw2"), + hist.member("fTsumwx"), + hist.member("fTsumwx2"), + ] + ) + return uproot.writing.identify.to_TH1x( + hist.member("fName"), + hist.member("fTitle"), + hist.values(flow=True), + *member_data, + hist.variances(flow=True), + hist.member("fXaxis"), + ) + if hist.member("fN") == outfile[key].member("fN"): + member_data = np.array( + [ + hist.member("fEntries"), + hist.member("fTsumw"), + hist.member("fTsumw2"), + hist.member("fTsumwx"), + hist.member("fTsumwx2"), + ] + ) + h_sum = uproot.writing.identify.to_TH1x( + hist.member("fName"), + hist.member("fTitle"), + outfile[key].values(flow=True) + hist.values(flow=True), + *np.add( + np.array( + [ + outfile[key].member("fEntries"), + outfile[key].member("fTsumw"), + outfile[key].member("fTsumw2"), + outfile[key].member("fTsumwx"), + outfile[key].member("fTsumwx2"), + ] + ), + member_data, + ), + outfile[key].variances(flow=True) + hist.variances(flow=True), + hist.member("fXaxis"), + ) + outfile.close() + return h_sum + + msg = "Bins must be the same for histograms to be added, not " + raise ValueError( + msg, + hist.member("fN"), + " and ", + outfile[key].member("fN"), + ) from None + + +def hadd_2d(destination, file, key, first): + """ + Args: + destination (path-like): Name of the output file or file path. + file (ReadOnlyDirectory): ROOT file to read histogram from. + key (str): key to reference histogram to be added. + first (bool): if True, special case for first of a certain histogram + to be added to the new file. + """ + outfile = uproot.open(destination) + try: + hist = file[key] + except ValueError: + msg = "Key missing from {file}" + raise ValueError(msg) from None + if first: + member_data = np.array( + [ + hist.member("fEntries"), + hist.member("fTsumw"), + hist.member("fTsumw2"), + hist.member("fTsumwx"), + hist.member("fTsumwx2"), + hist.member("fTsumwy"), + hist.member("fTsumwy2"), + hist.member("fTsumwxy"), + ] + ) + return uproot.writing.identify.to_TH2x( + hist.member("fName"), + hist.member("fTitle"), + np.ravel(hist.values(flow=True), order="C"), + *member_data, + np.ravel(hist.variances(flow=True), order="C"), + hist.member("fXaxis"), + hist.member("fYaxis"), + ) + if hist.member("fN") == outfile[key].member("fN"): + member_data = np.array( + [ + hist.member("fEntries"), + hist.member("fTsumw"), + hist.member("fTsumw2"), + hist.member("fTsumwx"), + hist.member("fTsumwx2"), + hist.member("fTsumwy"), + hist.member("fTsumwy2"), + hist.member("fTsumwxy"), + ] + ) + + h_sum = uproot.writing.identify.to_TH2x( + hist.member("fName"), + hist.member("fTitle"), + np.ravel( + outfile[key].values(flow=True) + hist.values(flow=True), order="C" + ), + *np.add( + np.array( + [ + outfile[key].member("fEntries"), + outfile[key].member("fTsumw"), + outfile[key].member("fTsumw2"), + outfile[key].member("fTsumwx"), + outfile[key].member("fTsumwx2"), + outfile[key].member("fTsumwy"), + outfile[key].member("fTsumwy2"), + outfile[key].member("fTsumwxy"), + ] + ), + member_data, + ), + np.ravel( + outfile[key].variances(flow=True) + hist.variances(flow=True), order="C" + ), + hist.member("fXaxis"), + hist.member("fYaxis"), + ) + outfile.close() + return h_sum + + msg = "Bins must be the same for histograms to be added, not " + raise ValueError( + msg, + hist.member("fN"), + " and ", + outfile[key].member("fN"), + ) from None + + +def hadd_3d(destination, file, key, first): + """ + Args: + destination (path-like): Name of the output file or file path. + file (ReadOnlyDirectory): ROOT file to read histogram from. + key (str): key to reference histogram to be added. + first (bool): if True, special case for first of a certain histogram + to be added to the new file. + """ + outfile = uproot.open(destination) + try: + hist = file[key] + except ValueError: + msg = "Key missing from {file}" + raise ValueError(msg) from None + if first: + member_data = np.array( + [ + hist.member("fEntries"), + hist.member("fTsumw"), + hist.member("fTsumw2"), + hist.member("fTsumwx"), + hist.member("fTsumwx2"), + hist.member("fTsumwy"), + hist.member("fTsumwy2"), + hist.member("fTsumwxy"), + hist.member("fTsumwz"), + hist.member("fTsumwz2"), + hist.member("fTsumwxz"), + hist.member("fTsumwyz"), + ] + ) + return uproot.writing.identify.to_TH3x( + hist.member("fName"), + hist.member("fTitle"), + np.ravel(hist.values(flow=True), order="C"), + *member_data, + np.ravel(hist.variances(flow=True), order="C"), + hist.member("fXaxis"), + hist.member("fYaxis"), + hist.member("fZaxis"), + ) + if hist.member("fN") == outfile[key].member("fN"): + member_data = np.add( + np.array( + [ + hist.member("fEntries"), + hist.member("fTsumw"), + hist.member("fTsumw2"), + hist.member("fTsumwx"), + hist.member("fTsumwx2"), + hist.member("fTsumwy"), + hist.member("fTsumwy2"), + hist.member("fTsumwxy"), + hist.member("fTsumwz"), + hist.member("fTsumwz2"), + hist.member("fTsumwxz"), + hist.member("fTsumwyz"), + ] + ), + np.array( + [ + hist.member("fEntries"), + outfile[key].member("fTsumw"), + outfile[key].member("fTsumw2"), + outfile[key].member("fTsumwx"), + outfile[key].member("fTsumwx2"), + outfile[key].member("fTsumwy"), + outfile[key].member("fTsumwy2"), + outfile[key].member("fTsumwxy"), + outfile[key].member("fTsumwz"), + outfile[key].member("fTsumwz2"), + outfile[key].member("fTsumwxz"), + outfile[key].member("fTsumwyz"), + ] + ), + ) + h_sum = uproot.writing.identify.to_TH3x( + hist.member("fName"), + hist.member("fTitle"), + np.ravel( + outfile[key].values(flow=True) + hist.values(flow=True), order="C" + ), + *member_data, + np.ravel( + (outfile[key].variances(flow=True) + hist.variances(flow=True)), + order="C", + ), + hist.member("fXaxis"), + hist.member("fYaxis"), + hist.member("fZaxis"), + ) + outfile.close() + return h_sum + + msg = "Bins must be the same for histograms to be added, not " + raise ValueError( + msg, + hist.member("fN"), + " and ", + outfile[key].member("fN"), + ) from None + + +def hadd( + destination, + files, + *, + force=True, + append=False, + compression="lz4", + compression_level=1, + skip_bad_files=False, + union=True, +): + """ + Args: + destination (path-like): Name of the output file or file path. + files (Str or list of str): List of local ROOT files to read histograms from. + May contain glob patterns. + force (bool): If True, overwrites destination file if it exists. Force and append + cannot both be True. + append (bool): If True, appends histograms to an existing file. Force and append + cannot both be True. + compression (str): Sets compression level for root file to write to. Can be one of + "ZLIB", "LZMA", "LZ4", or "ZSTD". By default the compression algorithm is "LZ4". + compression_level (int): Use a compression level particular to the chosen compressor. + By default the compression level is 1. + skip_bad_files (bool): If True, skips corrupt or non-existent files without exiting. + max_opened_files (int): Limits the number of files to be open at the same time. If 0, + this gets set to system limit. + union (bool): If True, adds the histograms that have the same name and copies all others + to the new file. + + Adds together histograms from local ROOT files of a collection of ROOT files, and writes them to + a new or existing ROOT file. + + >>> odapt.add_histograms("destination.root", ["file1_to_hadd.root", "file2_to_hadd.root"]) + + """ + if compression in ("ZLIB", "zlib"): + compression_code = uproot.const.kZLIB + elif compression in ("LZMA", "lzma"): + compression_code = uproot.const.kLZMA + elif compression in ("LZ4", "lz4"): + compression_code = uproot.const.kLZ4 + elif compression in ("ZSTD", "zstd"): + compression_code = uproot.const.kZSTD + else: + msg = f"unrecognized compression algorithm: {compression}. Only ZLIB, LZMA, LZ4, and ZSTD are accepted." + raise ValueError(msg) + p = Path(destination) + if Path.is_file(p): + if not force and not append: + raise FileExistsError + if force and append: + msg = "Cannot append to a new file. Either force or append can be true." + raise ValueError(msg) + else: + if append: + raise FileNotFoundError( + "File %s" + destination + " not found. File must exist to append." + ) + file_out = uproot.recreate( + destination, + compression=uproot.compression.Compression.from_code_pair( + compression_code, compression_level + ), + ) + if not isinstance(files, list): + path = Path(files) + files = sorted(path.glob("**/*.root")) + + if len(files) <= 1: + msg = "Cannot hadd one file. Use root_to_root to copy a ROOT file." + raise ValueError(msg) from None + + with uproot.open(files[0]) as file: + keys = file.keys(filter_classname="TH[1|2|3][I|S|F|D|C]", cycle=False) + if union: + for i, _value in enumerate(files[1:]): + with uproot.open(files[i]) as file: + keys = np.union1d( + keys, + file.keys(filter_classname="TH[1|2|3][I|S|F|D|C]", cycle=False), + ) + # if files[i + 1] == files[-1]: + # keys_axes = dict(zip(keys, (len(file[j].axes) for j in keys))) + else: + for i, _value in enumerate(files[1:]): + with uproot.open(files[i]) as file: + keys = np.intersect1d( + keys, + file.keys(filter_classname="TH[1|2|3][I|S|F|D|C]", cycle=False), + ) + # if files[i + 1] == files[-1]: + # keys_axes = dict(zip(keys, (len(file[j].axes) for j in keys))) + + first = True + for input_file in files: + p = Path(input_file) + if Path.is_file(p): + file_out = uproot.update(destination) + else: + file_out = uproot.recreate( + destination, + compression=uproot.compression.Compression.from_code_pair( + compression_code, compression_level + ), + ) + + try: + file = uproot.open(input_file) + except FileNotFoundError: + if skip_bad_files: + continue + msg = "File: {input_file} does not exist or is corrupt." + raise FileNotFoundError(msg) from None + + for key in keys: + try: + file[key] + except ValueError: + if not union: + continue + msg = "Union key filter error." + raise ValueError(msg) from None + if len(file[key].axes) == 1: + h_sum = hadd_1d(destination, file, key, first) + + elif len(file[key].axes) == 2: + h_sum = hadd_2d(destination, file, key, first) + + else: + h_sum = hadd_3d(destination, file, key, first) + + if h_sum is not None: + file_out[key] = h_sum + + first = False + file.close() + + +def main(): + """ + Implementation of cmd-line executables. + """ + argparser = argparse.ArgumentParser(description="Hadd ROOT histograms with Uproot") + argparser.add_argument("destination", type=str, help="path of output file") + argparser.add_argument( + "input_files", + type=str, + nargs="+", + help="list or directory (glob syntax accepted) of input files", + ) + argparser.add_argument( + "-f", + "--force", + action="store_true", + default=True, + help="force overwrite of output file", + ) + argparser.add_argument( + "-a", "--append", action="store", default=False, help="append to existing file" + ) + argparser.add_argument( + "-c", + "--compression", + action="store", + default="lz4", + help="set compression level between 1-9", + ) + argparser.add_argument( + "-c[0-9]", + "--compression_level", + action="store", + default=1, + help="set compression level between 1-9", + ) + argparser.add_argument( + "-k", + "--skip_bad_files", + action="store", + default=False, + help="corrupt or non-existent input files are ignored", + ) + argparser.add_argument( + "-u", + action="union", + default=True, + help="all histograms get copied to new file, only those with same name get added", + ) + + args = argparser.parse_args() + + hadd( + args.destination, + args.input_file, + force=args.force, + append=args.append, + compression=args.compression, + compression_level=args.compression_level, + skip_bad_files=args.skip_bad_files, + union=args.union, + ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/samples/__init__.py b/tests/samples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/samples/file1.root b/tests/samples/file1.root new file mode 100644 index 0000000..249d632 Binary files /dev/null and b/tests/samples/file1.root differ diff --git a/tests/samples/file2.root b/tests/samples/file2.root new file mode 100644 index 0000000..2d48731 Binary files /dev/null and b/tests/samples/file2.root differ diff --git a/tests/samples/file3.root b/tests/samples/file3.root new file mode 100644 index 0000000..7458c53 Binary files /dev/null and b/tests/samples/file3.root differ diff --git a/tests/test_hadd.py b/tests/test_hadd.py new file mode 100644 index 0000000..87ddf25 --- /dev/null +++ b/tests/test_hadd.py @@ -0,0 +1,516 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest +import uproot + +import odapt + +ROOT = pytest.importorskip("pyarrow") + + +def write_root_file(hist, path): + outHistFile = ROOT.TFile.Open(path, "RECREATE") + outHistFile.cd() + hist.Write() + outHistFile.Close() + + +def generate_1D_gaussian(file_paths): + gauss_1 = ROOT.TH1I("name", "title", 5, -4, 4) + gauss_1.FillRandom("gaus") + gauss_1.Sumw2() + gauss_1.SetDirectory(0) + outHistFile = ROOT.TFile.Open(file_paths[0], "RECREATE") + outHistFile.cd() + gauss_1.Write() + outHistFile.Close() + gauss_1 = uproot.from_pyroot(gauss_1) + + gauss_2 = ROOT.TH1I("name", "title", 5, -4, 4) + gauss_2.FillRandom("gaus") + gauss_2.Sumw2() + gauss_2.SetDirectory(0) + outHistFile = ROOT.TFile.Open(file_paths[1], "RECREATE") + outHistFile.cd() + gauss_2.Write() + outHistFile.Close() + gauss_2 = uproot.from_pyroot(gauss_2) + + gauss_3 = ROOT.TH1I("name", "title", 5, -4, 4) + gauss_3.FillRandom("gaus") + gauss_3.Sumw2() + gauss_3.SetDirectory(0) + outHistFile = ROOT.TFile.Open(file_paths[2], "RECREATE") + outHistFile.cd() + gauss_3.Write() + outHistFile.Close() + gauss_3 = uproot.from_pyroot(gauss_3) + + return gauss_1, gauss_2, gauss_3 + + +def generate_1D_simple(): + h1 = ROOT.TH1F("name", "", 10, 0.0, 10.0) + data1 = [11.5, 12.0, 9.0, 8.1, 6.4, 6.32, 5.3, 3.0, 2.0, 1.0] + for i in range(len(data1)): + h1.Fill(i, data1[i]) + + outHistFile = ROOT.TFile.Open("tests/file1dim1.root", "RECREATE") + outHistFile.cd() + h1.Write() + outHistFile.Close() + h1 = uproot.from_pyroot(h1) + + h2 = ROOT.TH1F("name", "", 10, 0.0, 10.0) + data2 = [21.5, 10.0, 9.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.0] + + for i in range(len(data2)): + h2.Fill(i, data2[i]) + + outHistFile = ROOT.TFile.Open("tests/file2dim1.root", "RECREATE") + outHistFile.cd() + h2.Write() + outHistFile.Close() + h2 = uproot.from_pyroot(h2) + return h1, h2 + + +def test_simple(tmp_path, file_paths): + gauss_1 = ROOT.TH1I("name", "title", 5, -4, 4) + gauss_1.FillRandom("gaus") + gauss_1.Sumw2() + gauss_1.SetDirectory(0) + outHistFile = ROOT.TFile.Open(file_paths[0], "RECREATE") + outHistFile.cd() + gauss_1.Write() + outHistFile.Close() + h1 = uproot.from_pyroot(gauss_1) + + gauss_2 = ROOT.TH1I("name", "title", 5, -4, 4) + gauss_2.FillRandom("gaus") + gauss_2.Sumw2() + gauss_2.SetDirectory(0) + outHistFile = ROOT.TFile.Open(file_paths[1], "RECREATE") + outHistFile.cd() + gauss_2.Write() + outHistFile.Close() + h2 = uproot.from_pyroot(gauss_2) + + gauss_3 = ROOT.TH1I("name", "title", 5, -4, 4) + gauss_3.FillRandom("gaus") + gauss_3.Sumw2() + gauss_3.SetDirectory(0) + outHistFile = ROOT.TFile.Open(file_paths[2], "RECREATE") + outHistFile.cd() + gauss_3.Write() + outHistFile.Close() + h3 = uproot.from_pyroot(gauss_3) + + path = Path(tmp_path) + destination = path / "destination.root" + odapt.operations.hadd(destination, file_paths, force=True) + + with uproot.open(destination) as file: + added = uproot.from_pyroot( + gauss_1 + gauss_2 + gauss_3 + ) # test odapt vs Pyroot histogram adding + assert file["name"].member("fN") == added.member("fN") + assert file["name"].member("fTsumw") == added.member("fTsumw") + assert np.equal(file["name"].values(flow=True), added.values(flow=True)).all + assert file["name"].member("fTsumw") == h1.member("fTsumw") + h2.member( + "fTsumw" + ) + h3.member("fTsumw") + assert np.equal( + file["name"].values(flow=True), + np.array(h1.values(flow=True) + h2.values(flow=True)), + ).all + + +test_simple( + "tests", + [ + "tests/samples/file1.root", + "tests/samples/file2.root", + "tests/samples/file3.root", + ], +) + + +def test_3_glob(file_paths): + h1, h2, h3 = generate_1D_gaussian(file_paths) + + odapt.operations.hadd("tests/place.root", "tests/samples", force=True) + + with uproot.open("tests/place.root") as file: + assert file["name"].member("fN") == h1.member("fN") + assert file["name"].member("fTsumw") == h1.member("fTsumw") + h2.member( + "fTsumw" + ) + h3.member("fTsumw") + assert np.equal( + file["name"].values(flow=True), + np.array( + h1.values(flow=True) + h2.values(flow=True) + h3.values(flow=True) + ), + ).all + + +def simple_1dim_F(): + h1, h2 = generate_1D_simple() + odapt.operations.hadd( + "tests/place2.root", + ["tests/file1dim1.root", "tests/file2dim1.root"], + force=True, + ) + + with uproot.open("tests/place2.root") as file: + assert file["name"].member("fN") == h1.member("fN") + assert file["name"].member("fTsumw") == h1.member("fTsumw") + h2.member( + "fTsumw" + ) + assert np.equal( + file["name"].member("fXaxis").edges(flow=True), + h1.member("fXaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fXaxis").edges(flow=True), + h2.member("fXaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fYaxis").edges(flow=True), + h1.member("fYaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fYaxis").edges(flow=True), + h2.member("fYaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].values(flow=True), + np.array(h1.values(flow=True) + h2.values(flow=True)), + ).all + + +def mult_2D_hists(): + h1 = ROOT.TH2F("name", "", 10, 0.0, 10.0, 8, 0.0, 8.0) + data1 = [ + [13.5, 11.0, 10.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.0], + [11.5, 10.0, 9.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.0], + [10.5, 10.0, 8.0, 7.2, 6.8, 5.32, 5.3, 2.0, 1.6, 1.0], + [9.5, 9.0, 8.0, 7.2, 6.8, 5.32, 5.3, 2.0, 1.6, 1.0], + [8.5, 8.0, 9.0, 7.2, 6.8, 5.32, 5.3, 2.0, 1.0, 0.5], + [4.5, 7.0, 7.0, 7.2, 6.8, 5.32, 5.3, 2.0, 0.54, 0.25], + [3.5, 4.0, 4.0, 4.2, 6.8, 5.32, 5.3, 2.0, 0.2, 0.1], + [1.5, 1.01, 0.21, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01], + ] + for i in range(len(data1)): + for j in range(len(data1[0])): + h1.Fill(i, j, data1[i][j]) + + outHistFile = ROOT.TFile.Open("tests/file3dim2.root", "RECREATE") + outHistFile.cd() + h1.Write() + outHistFile.Close() + h1 = uproot.from_pyroot(h1) + + h2 = ROOT.TH2F("second", "", 10, 0.0, 10.0, 8, 0.0, 8.0) + data2 = [ + [21.5, 10.0, 9.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.0], + [15.5, 13.0, 9.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.5], + [12.5, 10.0, 9.5, 8.2, 6.8, 6.32, 5.2, 3.0, 2.0, 1.25], + [9.5, 9.0, 8.0, 7.2, 5.8, 5.32, 5.3, 2.0, 1.6, 0.5], + [8.5, 8.0, 6.0, 7.2, 5.8, 5.32, 5.3, 2.0, 1.0, 0.4], + [4.5, 4.0, 4.0, 7.2, 5.8, 5.32, 5.3, 2.0, 0.54, 0.3], + [3.5, 4.0, 4.0, 4.2, 5.8, 5.32, 5.3, 2.0, 0.2, 0.1], + [1.5, 1.01, 0.21, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.02], + ] + + for i in range(len(data2)): + for j in range(len(data2[0])): + h2.Fill(i, j, data2[i][j]) + + outHistFile = ROOT.TFile.Open("tests/file3dim2.root", "UPDATE") + outHistFile.cd() + h2.Write() + outHistFile.Close() + h2 = uproot.from_pyroot(h2) + + h3 = ROOT.TH2F("name", "", 10, 0.0, 10.0, 8, 0.0, 8.0) + data3 = [ + [13.5, 11.0, 10.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.0], + [11.5, 10.0, 9.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.0], + [10.5, 10.0, 8.0, 7.2, 6.8, 5.32, 5.3, 2.5, 1.6, 1.0], + [9.5, 9.0, 8.0, 7.2, 6.8, 5.32, 5.3, 3.0, 3.6, 1.0], + [8.5, 8.0, 9.0, 7.2, 6.8, 5.32, 5.3, 2.0, 2.0, 0.25], + [4.5, 7.0, 7.0, 7.2, 6.8, 5.32, 5.3, 2.0, 0.54, 0.25], + [3.5, 4.0, 4.0, 4.2, 6.8, 5.32, 5.3, 2.0, 1.2, 0.1], + [1.5, 1.01, 0.21, 0.01, 0.51, 0.01, 0.41, 0.01, 0.01, 0.01], + ] + for i in range(len(data3)): + for j in range(len(data3[0])): + h3.Fill(i, j, data3[i][j]) + + outHistFile = ROOT.TFile.Open("tests/file4dim2.root", "RECREATE") + outHistFile.cd() + h3.Write() + outHistFile.Close() + h3 = uproot.from_pyroot(h3) + + h4 = ROOT.TH2F("second", "", 10, 0.0, 10.0, 8, 0.0, 8.0) + data4 = [ + [21.5, 10.0, 9.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.0], + [15.5, 13.0, 9.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.5], + [12.5, 10.0, 9.5, 8.2, 6.8, 6.32, 5.2, 3.0, 2.0, 1.25], + [9.5, 9.0, 8.0, 7.2, 5.8, 5.32, 5.3, 2.0, 1.6, 0.5], + [8.5, 8.0, 6.0, 7.2, 5.8, 5.32, 5.3, 2.0, 1.0, 0.4], + [4.5, 4.0, 4.0, 7.2, 5.8, 5.32, 5.3, 2.0, 0.54, 0.3], + [3.5, 4.0, 4.0, 4.2, 5.8, 5.32, 5.3, 2.0, 0.2, 0.1], + [1.5, 1.01, 0.21, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.02], + ] + + for i in range(len(data4)): + for j in range(len(data4[0])): + h4.Fill(i, j, data4[i][j]) + + outHistFile = ROOT.TFile.Open("tests/file4dim2.root", "UPDATE") + outHistFile.cd() + h4.Write() + outHistFile.Close() + h4 = uproot.from_pyroot(h4) + + odapt.operations.hadd( + "tests/place2.root", + ["tests/file3dim2.root", "tests/file4dim2.root"], + force=True, + ) + + with uproot.open("tests/place2.root") as file: + assert file["name"].member("fN") == h1.member("fN") + assert file["name"].member("fTsumw") == h1.member("fTsumw") + h3.member( + "fTsumw" + ) + assert np.equal( + file["name"].member("fXaxis").edges(flow=True), + h1.member("fXaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fXaxis").edges(flow=True), + h2.member("fXaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fYaxis").edges(flow=True), + h1.member("fYaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fYaxis").edges(flow=True), + h2.member("fYaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fXaxis").edges(flow=True), + h3.member("fXaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fXaxis").edges(flow=True), + h4.member("fXaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fYaxis").edges(flow=True), + h3.member("fYaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fYaxis").edges(flow=True), + h4.member("fYaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].values(flow=True), + np.array(h1.values(flow=True) + h3.values(flow=True)), + ).all + assert np.equal( + file["second"].values(flow=True), + np.array(h2.values(flow=True) + h4.values(flow=True)), + ).all + + +def simple_2dim_F(): + fName = "name" + h1 = ROOT.TH2F(fName, "", 10, 0.0, 10.0, 8, 0.0, 8.0) + data1 = [ + [13.5, 11.0, 10.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.0], + [11.5, 10.0, 9.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.0], + [10.5, 10.0, 8.0, 7.2, 6.8, 5.32, 5.3, 2.0, 1.6, 1.0], + [9.5, 9.0, 8.0, 7.2, 6.8, 5.32, 5.3, 2.0, 1.6, 1.0], + [8.5, 8.0, 9.0, 7.2, 6.8, 5.32, 5.3, 2.0, 1.0, 0.5], + [4.5, 7.0, 7.0, 7.2, 6.8, 5.32, 5.3, 2.0, 0.54, 0.25], + [3.5, 4.0, 4.0, 4.2, 6.8, 5.32, 5.3, 2.0, 0.2, 0.1], + [1.5, 1.01, 0.21, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01], + ] + for i in range(len(data1)): + for j in range(len(data1[0])): + h1.Fill(i, j, data1[i][j]) + + h1 = uproot.from_pyroot(h1) + + h2 = ROOT.TH2F(fName, "", 10, 0.0, 10.0, 8, 0.0, 8.0) + data2 = [ + [21.5, 10.0, 9.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.0], + [15.5, 13.0, 9.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.5], + [12.5, 10.0, 9.5, 8.2, 6.8, 6.32, 5.2, 3.0, 2.0, 1.25], + [9.5, 9.0, 8.0, 7.2, 5.8, 5.32, 5.3, 2.0, 1.6, 0.5], + [8.5, 8.0, 6.0, 7.2, 5.8, 5.32, 5.3, 2.0, 1.0, 0.4], + [4.5, 4.0, 4.0, 7.2, 5.8, 5.32, 5.3, 2.0, 0.54, 0.3], + [3.5, 4.0, 4.0, 4.2, 5.8, 5.32, 5.3, 2.0, 0.2, 0.1], + [1.5, 1.01, 0.21, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.02], + ] + + for i in range(len(data2)): + for j in range(len(data2[0])): + h2.Fill(i, j, data2[i][j]) + + h2 = uproot.from_pyroot(h2) + + odapt.operations.hadd( + "tests/place2.root", + ["tests/file1dim2.root", "tests/file2dim2.root"], + force=True, + ) + + with uproot.open("tests/place2.root") as file: + assert file["name"].member("fN") == h1.member("fN") + assert file["name"].member("fTsumw") == h1.member("fTsumw") + h2.member( + "fTsumw" + ) + assert np.equal( + file["name"].member("fXaxis").edges(flow=True), + h1.member("fXaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fXaxis").edges(flow=True), + h2.member("fXaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fYaxis").edges(flow=True), + h1.member("fYaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fYaxis").edges(flow=True), + h2.member("fYaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].values(flow=True), + np.array(h1.values(flow=True) + h2.values(flow=True)), + ).all + + +def simple_2D(): + h2 = ROOT.TH2F("name", "", 10, 0.0, 10.0, 8, 0.0, 8.0) + data2 = [ + [21.5, 10.0, 9.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.0], + [15.5, 13.0, 9.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.5], + [12.5, 10.0, 9.5, 8.2, 6.8, 6.32, 5.2, 3.0, 2.0, 1.25], + [9.5, 9.0, 8.0, 7.2, 5.8, 5.32, 5.3, 2.0, 1.6, 0.5], + [8.5, 8.0, 6.0, 7.2, 5.8, 5.32, 5.3, 2.0, 1.0, 0.4], + [4.5, 4.0, 4.0, 7.2, 5.8, 5.32, 5.3, 2.0, 0.54, 0.3], + [3.5, 4.0, 4.0, 4.2, 5.8, 5.32, 5.3, 2.0, 0.2, 0.1], + [1.5, 1.01, 0.21, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.02], + ] + + for i in range(len(data2)): + for j in range(len(data2[0])): + h2.Fill(i, j, data2[i][j]) + outHistFile = ROOT.TFile.Open("tests/file2dim2.root", "UPDATE") + outHistFile.cd() + h2.Write() + outHistFile.Close() + h2 = uproot.from_pyroot(h2) + + h1 = ROOT.TH2F("name", "", 10, 0.0, 10.0, 8, 0.0, 8.0) + data1 = [ + [13.5, 11.0, 10.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.0], + [11.5, 10.0, 9.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.0], + [10.5, 10.0, 8.0, 7.2, 6.8, 5.32, 5.3, 2.0, 1.6, 1.0], + [9.5, 9.0, 8.0, 7.2, 6.8, 5.32, 5.3, 2.0, 1.6, 1.0], + [8.5, 8.0, 9.0, 7.2, 6.8, 5.32, 5.3, 2.0, 1.0, 0.5], + [4.5, 7.0, 7.0, 7.2, 6.8, 5.32, 5.3, 2.0, 0.54, 0.25], + [3.5, 4.0, 4.0, 4.2, 6.8, 5.32, 5.3, 2.0, 0.2, 0.1], + [1.5, 1.01, 0.21, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01], + ] + for i in range(len(data1)): + for j in range(len(data1[0])): + h1.Fill(i, j, data1[i][j]) + + outHistFile = ROOT.TFile.Open("tests/file1dim2.root", "RECREATE") + outHistFile.cd() + h1.Write() + outHistFile.Close() + h1 = uproot.from_pyroot(h1) + + odapt.operations.hadd( + "tests/place2.root", + ["tests/file1dim2.root", "tests/file2dim2.root"], + force=True, + ) + + with uproot.open("tests/place2.root") as file: + assert file["name"].member("fN") == h1.member("fN") + assert file["name"].member("fTsumw") == h1.member("fTsumw") + h2.member( + "fTsumw" + ) + assert np.equal( + file["name"].member("fXaxis").edges(flow=True), + h1.member("fXaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fXaxis").edges(flow=True), + h2.member("fXaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fYaxis").edges(flow=True), + h1.member("fYaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].member("fYaxis").edges(flow=True), + h2.member("fYaxis").edges(flow=True), + ).all + assert np.equal( + file["name"].values(flow=True), + np.array(h1.values(flow=True) + h2.values(flow=True)), + ).all + + +mult_2D_hists() + +simple_1dim_F() + +simple_2dim_F() + + +def break_bins(): + h1 = ROOT.TH1F("name", "", 8, 0.0, 10.0) + data1 = [11.5, 12.0, 9.0, 8.1, 6.4, 6.32, 5.3, 3.0] + for i in range(len(data1)): + h1.Fill(i, data1[i]) + + outHistFile = ROOT.TFile.Open("tests/file1dim1break.root", "RECREATE") + outHistFile.cd() + h1.Write() + outHistFile.Close() + h1 = uproot.from_pyroot(h1) + + h2 = ROOT.TH1F("name", "", 10, 0.0, 10.0) + data2 = [21.5, 10.0, 9.0, 8.2, 6.8, 6.32, 5.3, 3.0, 2.0, 1.0] + + for i in range(len(data2)): + h2.Fill(i, data2[i]) + + outHistFile = ROOT.TFile.Open("tests/file2dim1break.root", "RECREATE") + outHistFile.cd() + h2.Write() + outHistFile.Close() + h2 = uproot.from_pyroot(h2) + + odapt.operations.hadd( + "tests/place2break.root", + ["tests/file1dim1break.root", "tests/file2dim1break.root"], + force=True, + )