From 42bd463357662c497c83cd1054a5f2f1d49e5c34 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 12:34:58 +0000 Subject: [PATCH 01/34] remove macos up the versions --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bd9bd70..e513e61 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,16 +6,16 @@ on: push jobs: tester: name: Test the code - runs-on: [ubuntu-latest, macos-latest, windows-latest] + runs-on: [ubuntu-latest, windows-latest] strategy: matrix: python-version: [3.8, '3.10'] steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Python setup - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} architecture: 'x64' From 2a8f22a77835ea30921577133ade954232011b27 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 12:40:13 +0000 Subject: [PATCH 02/34] try test without windows --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e513e61..ce8942e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,7 @@ on: push jobs: tester: name: Test the code - runs-on: [ubuntu-latest, windows-latest] + runs-on: ubuntu-latest strategy: matrix: python-version: [3.8, '3.10'] From edc06b609b4cda6578e477cfe81799957367ad0f Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 14:05:05 +0000 Subject: [PATCH 03/34] add windows back --- .github/workflows/test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce8942e..1605252 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,12 +5,12 @@ on: push jobs: tester: - name: Test the code - runs-on: ubuntu-latest + name: 'Test the code' strategy: matrix: - python-version: [3.8, '3.10'] - + python-version: ['3.8', '3.10'] + os: ['ubuntu-latest', 'windows-latest'] + runs-on: ${{ matrix.os }} steps: - name: Checkout uses: actions/checkout@v3 From 88794091d82aaeaa9daedd0d5bab112976ef6915 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 14:08:52 +0000 Subject: [PATCH 04/34] fix --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1605252..3b2abc7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: python-version: ['3.8', '3.10'] - os: ['ubuntu-latest', 'windows-latest'] + os: ['ubuntu-latest', windows-latest'] runs-on: ${{ matrix.os }} steps: - name: Checkout From de72e86d77d5315ce0a8454875f1dd070da57128 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 14:29:15 +0000 Subject: [PATCH 05/34] fix windows stuff --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3b2abc7..a8ba52a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: python-version: ['3.8', '3.10'] - os: ['ubuntu-latest', windows-latest'] + os: ['ubuntu-latest', 'windows-latest'] runs-on: ${{ matrix.os }} steps: - name: Checkout From d645a9cd19ec9e363978cbb8bc25c33ecfc73a98 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 14:33:42 +0000 Subject: [PATCH 06/34] make things windows friendly --- py/minimint/mist_interpolator.py | 2 +- setup.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 301157c..1337508 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -61,7 +61,7 @@ def getheader(f): def read_grid(eep_prefix, outp_prefix): - fs = glob.glob('%s/*EEPS/*eep' % (eep_prefix, )) + fs = glob.glob(os.path.join([eep_prefix, '*EEPS', '*eep'])) assert (len(fs) > 0) tmpfile = utils.tail_head(fs[0], 11, 10) tab0 = atpy.Table().read(tmpfile, format='ascii.fast_commented_header') diff --git a/setup.py b/setup.py index fcee637..ae0c1dd 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def read(fname): VERSIONPIP = read('version.txt').rstrip() VERSION = VERSIONPIP + get_revision() -with open('py/minimint/_version.py', 'w') as fp: +with open(os.path.join(['py', 'minimint', '_version.py']), 'w') as fp: print('version="%s"' % (VERSION), file=fp) setup( @@ -55,8 +55,8 @@ def read(fname): url="http://github.com/segasai/minimint", packages=['minimint'], scripts=[fname for fname in glob.glob(os.path.join('bin', '*'))], - package_dir={'': 'py/'}, - package_data={'minimint': ['tests/']}, + package_dir={'': 'py'}, + package_data={'minimint': ['tests']}, long_description=read('README.md'), long_description_content_type='text/markdown', classifiers=[ From 70875e6629ba13941ea9f4a7c6536244c83a9d59 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 14:36:23 +0000 Subject: [PATCH 07/34] fix prev commit --- py/minimint/mist_interpolator.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 1337508..f2d890a 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -61,7 +61,7 @@ def getheader(f): def read_grid(eep_prefix, outp_prefix): - fs = glob.glob(os.path.join([eep_prefix, '*EEPS', '*eep'])) + fs = glob.glob(os.path.join(eep_prefix, '*EEPS', '*eep')) assert (len(fs) > 0) tmpfile = utils.tail_head(fs[0], 11, 10) tab0 = atpy.Table().read(tmpfile, format='ascii.fast_commented_header') diff --git a/setup.py b/setup.py index ae0c1dd..cf475df 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def read(fname): VERSIONPIP = read('version.txt').rstrip() VERSION = VERSIONPIP + get_revision() -with open(os.path.join(['py', 'minimint', '_version.py']), 'w') as fp: +with open(os.path.join('py', 'minimint', '_version.py'), 'w') as fp: print('version="%s"' % (VERSION), file=fp) setup( From 543a84b9ac5cde8a6c3dd8ab3dfc5ab2f62eb30b Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 14:48:25 +0000 Subject: [PATCH 08/34] fixes --- .github/workflows/test.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a8ba52a..1b9a7b1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,12 +28,13 @@ jobs: pip install pytest-cov pip install coveralls pip install . + - name: Pylint + if: ${{ ( matrix.os != 'windows-latest' ) }} + run: pylint -E --disable=E1101 py/minimint/*py - name: Test - run: | - pytest --cov=minimint -s - pylint -E --disable=E1101 py/minimint/*py + run: pytest --cov=minimint -s - name: Coveralls - if: ${{ success() }} + if: ${{ success() && (matrix.os != 'windows-latest') }} run: coveralls --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From e33f84e20a3620b2272d3b0d59beedaacb5895a5 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 14:54:23 +0000 Subject: [PATCH 09/34] final fixes for windows --- py/minimint/mist_interpolator.py | 16 ++++++++-------- py/minimint/utils.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index f2d890a..b0fb9af 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -233,9 +233,9 @@ def prepare(eep_prefix, if k == 'logage': grid[:, :, :] = np.cumsum(grid, axis=2) - np.save(outp_prefix + '/' + get_file(k), grid) + np.save(os.path.join(outp_prefix, get_file(k)), grid) - with open(outp_prefix + '/' + INTERP_PKL, 'wb') as fp: + with open(os.path.join(outp_prefix, INTERP_PKL), 'wb') as fp: pickle.dump(dict(umass=umass, ufeh=ufeh, neep=neep), fp) print('Reading/processing bolometric corrections') bolom.prepare(bolom_prefix, outp_prefix, filters) @@ -353,13 +353,13 @@ def __init__(self, prefix=None): """ if prefix is None: prefix = utils.get_data_path() - self.logg_grid = np.load(prefix + '/' + get_file('logg')) - self.logl_grid = np.load(prefix + '/' + get_file('logl')) - self.logteff_grid = np.load(prefix + '/' + get_file('logteff')) - self.logage_grid = np.load(prefix + '/' + get_file('logage')) - self.phase_grid = np.load(prefix + '/' + get_file('phase')) + (self.logg_grid, self.logl_grid, self.logteff_grid, self.logage_grid, + self.phase_grid) = [ + np.load(os.path.join(prefix, get_file(curt))) + for curt in ['logg', 'logl', 'logteff', 'logage', 'phase'] + ] - with open(prefix + '/' + INTERP_PKL, 'rb') as fp: + with open(os.path.join(prefix, INTERP_PKL), 'rb') as fp: D = pickle.load(fp) self.umass = np.array(D['umass']) self.ufeh = np.array(D['ufeh']) diff --git a/py/minimint/utils.py b/py/minimint/utils.py index 1334f4e..a8a662e 100644 --- a/py/minimint/utils.py +++ b/py/minimint/utils.py @@ -7,7 +7,7 @@ def get_data_path(): path = os.environ.get('MINIMINT_DATA_PATH') if path is not None: return path - path = str(pathlib.Path(__file__).parent.absolute()) + '/data/' + path = os.path.join(str(pathlib.Path(__file__).parent.absolute()), 'data') os.makedirs(path, exist_ok=True) return path From 5e2e0576fd53cd23c5e25467bfcc933a26996122 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 15:00:14 +0000 Subject: [PATCH 10/34] further win fixes --- py/minimint/mist_interpolator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index b0fb9af..1068db7 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -93,7 +93,7 @@ def read_grid(eep_prefix, outp_prefix): tabs.remove_column(k) os.makedirs(outp_prefix, exist_ok=True) - tabs.write(outp_prefix + '/' + TRACKS_FILE, overwrite=True) + tabs.write(os.path.join(outp_prefix, TRACKS_FILE), overwrite=True) def grid3d_filler(ima): @@ -159,7 +159,7 @@ def writer(url, pref): print('Downloading', url) fd = urllib.request.urlopen(url) fname = url.split('/')[-1] - fdout = open(pref + '/' + fname, 'wb') + fdout = open(os.path.join(pref, fname), 'wb') fdout.write(fd.read()) fdout.close() fd.close() @@ -198,8 +198,8 @@ def prepare(eep_prefix, and bolometric corrections') read_grid(eep_prefix, outp_prefix) print('Processing EEPs') - tab = atpy.Table().read(outp_prefix + '/' + TRACKS_FILE) - os.unlink(outp_prefix + '/' + TRACKS_FILE) # remove after reading + tab = atpy.Table().read(os.path.join(outp_prefix, TRACKS_FILE)) + os.unlink(os.path.join(outp_prefix, TRACKS_FILE)) # remove after reading umass, mass_id = np.unique(np.array(tab['initial_mass']), return_inverse=True) From 94e3ee6d7acc8a7977f04e6b953bf33ebf84cfd6 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 15:09:13 +0000 Subject: [PATCH 11/34] deal with windows specifics --- py/minimint/mist_interpolator.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 1068db7..96c7f05 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -164,7 +164,11 @@ def writer(url, pref): fdout.close() fd.close() cmd = 'cd %s; tar xfJ %s' % (pref, fname) - os.system(cmd) + if os.name == 'nt': + cmd = 'cd %s ; tar.exe xFJ %s' % (pref, fname) + status = os.system(cmd) + if status != 0: + raise RuntimeError('Failed to untar the files') with tempfile.TemporaryDirectory(dir=tmp_prefix) as T: for curfilt in filters: From 83f9501d4843bad19868336dc00d52325a254b68 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 15:21:06 +0000 Subject: [PATCH 12/34] use subprocess to untar --- py/minimint/mist_interpolator.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 96c7f05..33e6202 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -3,10 +3,12 @@ import glob import os import gc +import subprocess import pickle import urllib.request import astropy.table as atpy import scipy.interpolate + import numpy as np from minimint import bolom, utils """ @@ -166,9 +168,11 @@ def writer(url, pref): cmd = 'cd %s; tar xfJ %s' % (pref, fname) if os.name == 'nt': cmd = 'cd %s ; tar.exe xFJ %s' % (pref, fname) - status = os.system(cmd) - if status != 0: - raise RuntimeError('Failed to untar the files') + + ret = subprocess.run(cmd, capture_output=True, shell=True) + if ret.returncode != 0: + raise RuntimeError('Failed to untar the files' + + ret.stdout.decode() + ret.stderr.decode()) with tempfile.TemporaryDirectory(dir=tmp_prefix) as T: for curfilt in filters: From 6cd4ccb8f4f0ab63eea78353770fb92f0e5976cf Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 15:28:27 +0000 Subject: [PATCH 13/34] oh windows --- py/minimint/mist_interpolator.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 33e6202..76d9fd8 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -161,13 +161,14 @@ def writer(url, pref): print('Downloading', url) fd = urllib.request.urlopen(url) fname = url.split('/')[-1] - fdout = open(os.path.join(pref, fname), 'wb') + fname_out = os.path.join(pref, fname) + fdout = open(fname_out, 'wb') fdout.write(fd.read()) fdout.close() fd.close() - cmd = 'cd %s; tar xfJ %s' % (pref, fname) + cmd = f'cd {pref}; tar xfJ {fname_out}' if os.name == 'nt': - cmd = 'cd %s ; tar.exe xFJ %s' % (pref, fname) + cmd = f'cd {pref} && tar.exe xFJ {fname_out}' ret = subprocess.run(cmd, capture_output=True, shell=True) if ret.returncode != 0: From 3f6df336fc6fb4333d4b9d5ae4a7ce32036d28c1 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 15:31:05 +0000 Subject: [PATCH 14/34] tar.exe fixed --- py/minimint/mist_interpolator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 76d9fd8..a1c7236 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -168,7 +168,7 @@ def writer(url, pref): fd.close() cmd = f'cd {pref}; tar xfJ {fname_out}' if os.name == 'nt': - cmd = f'cd {pref} && tar.exe xFJ {fname_out}' + cmd = f'cd {pref} && tar.exe -xfJ {fname_out}' ret = subprocess.run(cmd, capture_output=True, shell=True) if ret.returncode != 0: From 2b6911a65900cddd290a753b5c2daafb7d0c1499 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 20:50:58 +0000 Subject: [PATCH 15/34] try to fix untar.xz --- py/minimint/mist_interpolator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index a1c7236..8226df0 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -168,7 +168,8 @@ def writer(url, pref): fd.close() cmd = f'cd {pref}; tar xfJ {fname_out}' if os.name == 'nt': - cmd = f'cd {pref} && tar.exe -xfJ {fname_out}' + fname_out1 = fname_out.replace('.xz', '') + cmd = f'cd {pref} && 7z x {fname_out} && tar -xf {fname_out1}' ret = subprocess.run(cmd, capture_output=True, shell=True) if ret.returncode != 0: From 52984a24b03b10cf41ca1645edbb1b05e98602db Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Tue, 26 Dec 2023 22:49:28 +0000 Subject: [PATCH 16/34] add timeout --- py/minimint/mist_interpolator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 8226df0..9fcc027 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -171,7 +171,7 @@ def writer(url, pref): fname_out1 = fname_out.replace('.xz', '') cmd = f'cd {pref} && 7z x {fname_out} && tar -xf {fname_out1}' - ret = subprocess.run(cmd, capture_output=True, shell=True) + ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) if ret.returncode != 0: raise RuntimeError('Failed to untar the files' + ret.stdout.decode() + ret.stderr.decode()) From 86725fefbb8c73377e52976e2f5e3f886cbfd79c Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 01:23:37 +0000 Subject: [PATCH 17/34] testing change --- .github/workflows/test.yml | 3 ++- py/minimint/mist_interpolator.py | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1b9a7b1..28ac8ef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,8 @@ jobs: strategy: matrix: python-version: ['3.8', '3.10'] - os: ['ubuntu-latest', 'windows-latest'] + #os: ['ubuntu-latest', 'windows-latest'] + os: ['windows-latest'] runs-on: ${{ matrix.os }} steps: - name: Checkout diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 9fcc027..db03281 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -166,12 +166,14 @@ def writer(url, pref): fdout.write(fd.read()) fdout.close() fd.close() - cmd = f'cd {pref}; tar xfJ {fname_out}' if os.name == 'nt': fname_out1 = fname_out.replace('.xz', '') - cmd = f'cd {pref} && 7z x {fname_out} && tar -xf {fname_out1}' - - ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) + cmd = f'cd {pref} && 7z x {fname_out} && tar -xf {fname_out1}' + else: + cmd = f'cd {pref}; tar xfJ {fname_out}' + + #ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) + ret = subprocess.run(cmd, shell=True, timeout=60) if ret.returncode != 0: raise RuntimeError('Failed to untar the files' + ret.stdout.decode() + ret.stderr.decode()) From 95a6b526a04f3cbfeb716522ca2840a03c5a8943 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 01:31:18 +0000 Subject: [PATCH 18/34] try another fix --- py/minimint/mist_interpolator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index db03281..e515b75 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -167,8 +167,8 @@ def writer(url, pref): fdout.close() fd.close() if os.name == 'nt': - fname_out1 = fname_out.replace('.xz', '') - cmd = f'cd {pref} && 7z x {fname_out} && tar -xf {fname_out1}' + fname_out1 = fname_out.replace('.txz', '.tar') + cmd = f'cd {pref} && 7z x {fname_out} -so > {fname_out1} && tar -xf {fname_out1}' else: cmd = f'cd {pref}; tar xfJ {fname_out}' From cbb180bfc51afc51a77c3136842311e7f1c4d4fa Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 01:44:03 +0000 Subject: [PATCH 19/34] further debugging --- py/minimint/mist_interpolator.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index e515b75..8679520 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -63,8 +63,10 @@ def getheader(f): def read_grid(eep_prefix, outp_prefix): - fs = glob.glob(os.path.join(eep_prefix, '*EEPS', '*eep')) - assert (len(fs) > 0) + mask = os.path.join(eep_prefix, '*EEPS', '*eep') + fs = glob.glob(mask) + if len(fs) == 0: + raise RuntimeError(f'Failed to find eep files {mask}') tmpfile = utils.tail_head(fs[0], 11, 10) tab0 = atpy.Table().read(tmpfile, format='ascii.fast_commented_header') os.unlink(tmpfile) @@ -168,12 +170,14 @@ def writer(url, pref): fd.close() if os.name == 'nt': fname_out1 = fname_out.replace('.txz', '.tar') - cmd = f'cd {pref} && 7z x {fname_out} -so > {fname_out1} && tar -xf {fname_out1}' + cmd = (f'cd {pref} && ' + f'7z x {fname_out} -so > {fname_out1} && ' + f'tar -xf {fname_out1}') else: cmd = f'cd {pref}; tar xfJ {fname_out}' - - #ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) - ret = subprocess.run(cmd, shell=True, timeout=60) + print(fname_out, fname_out1) + ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) + # ret = subprocess.run(cmd, shell=True, timeout=60) if ret.returncode != 0: raise RuntimeError('Failed to untar the files' + ret.stdout.decode() + ret.stderr.decode()) From 1f4ece84b7a9edf8f94f45f2a7c96960ee1461ad Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 01:51:45 +0000 Subject: [PATCH 20/34] another debug --- py/minimint/mist_interpolator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 8679520..f087832 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -66,7 +66,9 @@ def read_grid(eep_prefix, outp_prefix): mask = os.path.join(eep_prefix, '*EEPS', '*eep') fs = glob.glob(mask) if len(fs) == 0: - raise RuntimeError(f'Failed to find eep files {mask}') + raise RuntimeError(f'Failed to find eep files {mask}' + + str(os.path.join(eep_prefix, '*')) + '\n' + + str(os.path.join(eep_prefix, '*', '*'))) tmpfile = utils.tail_head(fs[0], 11, 10) tab0 = atpy.Table().read(tmpfile, format='ascii.fast_commented_header') os.unlink(tmpfile) From 8728726b74e8942ef283936c54cf2af5e9232b65 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 01:57:40 +0000 Subject: [PATCH 21/34] debug --- py/minimint/mist_interpolator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index f087832..247ed6b 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -67,8 +67,9 @@ def read_grid(eep_prefix, outp_prefix): fs = glob.glob(mask) if len(fs) == 0: raise RuntimeError(f'Failed to find eep files {mask}' + - str(os.path.join(eep_prefix, '*')) + '\n' + - str(os.path.join(eep_prefix, '*', '*'))) + str(glob.glob(os.path.join(eep_prefix, '*'))) + + '\r\n' + + str(glob.glob(os.path.join(eep_prefix, '*', '*')))) tmpfile = utils.tail_head(fs[0], 11, 10) tab0 = atpy.Table().read(tmpfile, format='ascii.fast_commented_header') os.unlink(tmpfile) From 621b88ab50d33f22063869925cb10029e6f1644e Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 02:05:42 +0000 Subject: [PATCH 22/34] debug --- py/minimint/mist_interpolator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 247ed6b..06aff84 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -178,8 +178,8 @@ def writer(url, pref): f'tar -xf {fname_out1}') else: cmd = f'cd {pref}; tar xfJ {fname_out}' - print(fname_out, fname_out1) ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) + print(fname_out, fname_out1, ret.stdout.decode() + ret.stderr.decode()) # ret = subprocess.run(cmd, shell=True, timeout=60) if ret.returncode != 0: raise RuntimeError('Failed to untar the files' + From 678d75cf69062d6e3e0af8b8eff881b4dab4d160 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 02:18:48 +0000 Subject: [PATCH 23/34] update --- py/minimint/mist_interpolator.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 06aff84..37f6388 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -173,9 +173,11 @@ def writer(url, pref): fd.close() if os.name == 'nt': fname_out1 = fname_out.replace('.txz', '.tar') - cmd = (f'cd {pref} && ' - f'7z x {fname_out} -so > {fname_out1} && ' - f'tar -xf {fname_out1}') + cmd = ( + f'cd {pref} && ' + f'7z x {fname_out} -so > {fname_out1} && ' + # f'tar -xvf {fname_out1}' + f'7z x {fname_out1}') else: cmd = f'cd {pref}; tar xfJ {fname_out}' ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) From ad63634e93c8660b655644e1625c73d14c97ad92 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 02:35:29 +0000 Subject: [PATCH 24/34] udate --- py/minimint/mist_interpolator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 37f6388..0d572af 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -177,11 +177,12 @@ def writer(url, pref): f'cd {pref} && ' f'7z x {fname_out} -so > {fname_out1} && ' # f'tar -xvf {fname_out1}' - f'7z x {fname_out1}') + f'7z -bb3 x {fname_out1}') else: cmd = f'cd {pref}; tar xfJ {fname_out}' ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) - print(fname_out, fname_out1, ret.stdout.decode() + ret.stderr.decode()) + print(cmd, fname_out, fname_out1, + ret.stdout.decode() + ret.stderr.decode()) # ret = subprocess.run(cmd, shell=True, timeout=60) if ret.returncode != 0: raise RuntimeError('Failed to untar the files' + From cb255d1a457a339cc4bce52b7a37b0b6e42a0327 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 02:45:49 +0000 Subject: [PATCH 25/34] another test --- py/minimint/mist_interpolator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 0d572af..1ff33ee 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -181,6 +181,10 @@ def writer(url, pref): else: cmd = f'cd {pref}; tar xfJ {fname_out}' ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) + fname_out2 = fname_out1.replace('.tar', '') + aa = glob.glob(os.path.join(pref, fname_out2, '*')) + print((aa)) + assert len(aa) > 0 print(cmd, fname_out, fname_out1, ret.stdout.decode() + ret.stderr.decode()) # ret = subprocess.run(cmd, shell=True, timeout=60) From 11c7372497ace30ed12481575f2af19148e21e43 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 02:57:03 +0000 Subject: [PATCH 26/34] update --- py/minimint/mist_interpolator.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 1ff33ee..c5e075c 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -173,20 +173,18 @@ def writer(url, pref): fd.close() if os.name == 'nt': fname_out1 = fname_out.replace('.txz', '.tar') - cmd = ( - f'cd {pref} && ' - f'7z x {fname_out} -so > {fname_out1} && ' - # f'tar -xvf {fname_out1}' - f'7z -bb3 x {fname_out1}') + cmd = (f'cd {pref} && ' + f'7z x {fname_out} && ' + f'7z -bb3 x {fname_out1}') else: cmd = f'cd {pref}; tar xfJ {fname_out}' ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) fname_out2 = fname_out1.replace('.tar', '') aa = glob.glob(os.path.join(pref, fname_out2, '*')) - print((aa)) - assert len(aa) > 0 print(cmd, fname_out, fname_out1, ret.stdout.decode() + ret.stderr.decode()) + print((aa)) + assert len(aa) > 0 # ret = subprocess.run(cmd, shell=True, timeout=60) if ret.returncode != 0: raise RuntimeError('Failed to untar the files' + From 2caef7130704364c8d5414496cedb0aa1b92213e Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 03:01:49 +0000 Subject: [PATCH 27/34] tmp --- py/minimint/mist_interpolator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index c5e075c..0c95c24 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -171,10 +171,11 @@ def writer(url, pref): fdout.write(fd.read()) fdout.close() fd.close() + fname_out = os.path.basename(fname_out) if os.name == 'nt': fname_out1 = fname_out.replace('.txz', '.tar') cmd = (f'cd {pref} && ' - f'7z x {fname_out} && ' + f'7z -bb3 x {fname_out} && ' f'7z -bb3 x {fname_out1}') else: cmd = f'cd {pref}; tar xfJ {fname_out}' From fa9276f912a4126380e0443a63cd898a2d0a3c73 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 03:08:49 +0000 Subject: [PATCH 28/34] try --- py/minimint/mist_interpolator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 0c95c24..82c6a9b 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -174,7 +174,7 @@ def writer(url, pref): fname_out = os.path.basename(fname_out) if os.name == 'nt': fname_out1 = fname_out.replace('.txz', '.tar') - cmd = (f'cd {pref} && ' + cmd = (f'cd /d {pref} && ' f'7z -bb3 x {fname_out} && ' f'7z -bb3 x {fname_out1}') else: From c655e761754c84ab458521f0d3eaa305738c3145 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 03:11:49 +0000 Subject: [PATCH 29/34] update --- py/minimint/mist_interpolator.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 82c6a9b..30e3b10 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -171,7 +171,6 @@ def writer(url, pref): fdout.write(fd.read()) fdout.close() fd.close() - fname_out = os.path.basename(fname_out) if os.name == 'nt': fname_out1 = fname_out.replace('.txz', '.tar') cmd = (f'cd /d {pref} && ' @@ -180,12 +179,8 @@ def writer(url, pref): else: cmd = f'cd {pref}; tar xfJ {fname_out}' ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) - fname_out2 = fname_out1.replace('.tar', '') - aa = glob.glob(os.path.join(pref, fname_out2, '*')) print(cmd, fname_out, fname_out1, ret.stdout.decode() + ret.stderr.decode()) - print((aa)) - assert len(aa) > 0 # ret = subprocess.run(cmd, shell=True, timeout=60) if ret.returncode != 0: raise RuntimeError('Failed to untar the files' + From f12b7990e4e4e3bd9e5757b7f7a63e3bb989f1eb Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 03:19:44 +0000 Subject: [PATCH 30/34] cleanup debug code --- .github/workflows/test.yml | 3 +-- py/minimint/mist_interpolator.py | 10 +++------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 28ac8ef..1b9a7b1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,8 +9,7 @@ jobs: strategy: matrix: python-version: ['3.8', '3.10'] - #os: ['ubuntu-latest', 'windows-latest'] - os: ['windows-latest'] + os: ['ubuntu-latest', 'windows-latest'] runs-on: ${{ matrix.os }} steps: - name: Checkout diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 30e3b10..4fc5984 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -66,10 +66,7 @@ def read_grid(eep_prefix, outp_prefix): mask = os.path.join(eep_prefix, '*EEPS', '*eep') fs = glob.glob(mask) if len(fs) == 0: - raise RuntimeError(f'Failed to find eep files {mask}' + - str(glob.glob(os.path.join(eep_prefix, '*'))) + - '\r\n' + - str(glob.glob(os.path.join(eep_prefix, '*', '*')))) + raise RuntimeError(f'Failed to find eep files {mask}') tmpfile = utils.tail_head(fs[0], 11, 10) tab0 = atpy.Table().read(tmpfile, format='ascii.fast_commented_header') os.unlink(tmpfile) @@ -174,14 +171,13 @@ def writer(url, pref): if os.name == 'nt': fname_out1 = fname_out.replace('.txz', '.tar') cmd = (f'cd /d {pref} && ' - f'7z -bb3 x {fname_out} && ' - f'7z -bb3 x {fname_out1}') + f'7z x {fname_out} && ' + f'7z x {fname_out1}') else: cmd = f'cd {pref}; tar xfJ {fname_out}' ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) print(cmd, fname_out, fname_out1, ret.stdout.decode() + ret.stderr.decode()) - # ret = subprocess.run(cmd, shell=True, timeout=60) if ret.returncode != 0: raise RuntimeError('Failed to untar the files' + ret.stdout.decode() + ret.stderr.decode()) From f25cc10030a180610eed32490e4a29e45ab762dc Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 03:21:45 +0000 Subject: [PATCH 31/34] fix the failure --- py/minimint/mist_interpolator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 4fc5984..141e17d 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -176,8 +176,6 @@ def writer(url, pref): else: cmd = f'cd {pref}; tar xfJ {fname_out}' ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) - print(cmd, fname_out, fname_out1, - ret.stdout.decode() + ret.stderr.decode()) if ret.returncode != 0: raise RuntimeError('Failed to untar the files' + ret.stdout.decode() + ret.stderr.decode()) From b4a3fb469f94e45e13ee09582beea14f9c42d801 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 09:34:48 +0000 Subject: [PATCH 32/34] fix hanging --- py/minimint/mist_interpolator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 141e17d..52a5ea6 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -175,7 +175,8 @@ def writer(url, pref): f'7z x {fname_out1}') else: cmd = f'cd {pref}; tar xfJ {fname_out}' - ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) + #ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) + ret = subprocess.run(cmd, shell=True, timeout=60) if ret.returncode != 0: raise RuntimeError('Failed to untar the files' + ret.stdout.decode() + ret.stderr.decode()) From 10392dcc1f206ee444669ae0ed843c135e3af002 Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 15:15:21 +0000 Subject: [PATCH 33/34] speed up reading --- py/minimint/mist_interpolator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/py/minimint/mist_interpolator.py b/py/minimint/mist_interpolator.py index 52a5ea6..1b1b690 100644 --- a/py/minimint/mist_interpolator.py +++ b/py/minimint/mist_interpolator.py @@ -75,7 +75,7 @@ def read_grid(eep_prefix, outp_prefix): for i, f in enumerate(fs): if i % (N // 100) == 0: print('%d/%d' % (i, N)) - curt = atpy.Table().read(f, format='ascii') + curt = atpy.Table().read(f, format='ascii.fast_no_header') for i, k in enumerate(list(curt.columns)): curt.rename_column(k, list(tab0.columns)[i]) D = getheader(f) @@ -175,7 +175,6 @@ def writer(url, pref): f'7z x {fname_out1}') else: cmd = f'cd {pref}; tar xfJ {fname_out}' - #ret = subprocess.run(cmd, capture_output=True, shell=True, timeout=60) ret = subprocess.run(cmd, shell=True, timeout=60) if ret.returncode != 0: raise RuntimeError('Failed to untar the files' + From fe5440d1656500be0373d5426d8cf5ada787fa8f Mon Sep 17 00:00:00 2001 From: "Sergey E. Koposov" Date: Wed, 27 Dec 2023 15:59:14 +0000 Subject: [PATCH 34/34] final windows fixes --- py/minimint/bolom.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/py/minimint/bolom.py b/py/minimint/bolom.py index 13fa265..d72fdcd 100644 --- a/py/minimint/bolom.py +++ b/py/minimint/bolom.py @@ -84,8 +84,8 @@ def __call__(self, p): bad = np.zeros(p.shape[0], dtype=bool) for i in range(self.ndim): pos1[:, i] = np.searchsorted(self.uvecs[i], p[:, i], 'right') - 1 - bad = bad | (pos1[:, i] < 0) | (pos1[:, i] >= - (len(self.uvecs[i]) - 1)) + bad = bad | (pos1[:, i] < 0) | (pos1[:, i] + >= (len(self.uvecs[i]) - 1)) pos1[:, i][bad] = 0 xs[:, i] = (p[:, i] - self.uvecs[i][pos1[:, i]]) / ( self.uvecs[i][pos1[:, i] + 1] - self.uvecs[i][pos1[:, i]] @@ -119,10 +119,12 @@ def list_filters(path=None): if path is None: path = get_data_path() - fs = glob.glob(path + '/' + FILT_NPY % '*') + fs = glob.glob(os.path.join(path, FILT_NPY % '*')) filts = [] for f in fs: - filts.append(re.match(FILT_NPY % '(.*)', f.split('/')[-1]).group(1)) + filts.append( + re.match(FILT_NPY % '(.*)', + f.split(os.path.sep)[-1]).group(1)) return filts @@ -140,7 +142,7 @@ def prepare(iprefix, raise Exception("shouldn't happen") last_vec = vec.copy() if i == 0: - np.save(oprefix + '/' + POINTS_NPY, vec) + np.save(os.path.join(oprefix, POINTS_NPY), vec) for k in tabs.columns: if k not in cols_ex: - np.save(oprefix + '/' + FILT_NPY % (k), tabs[k]) + np.save(os.path.join(oprefix, FILT_NPY % k), tabs[k])