diff --git a/training-tools/README.md b/training-tools/README.md new file mode 100644 index 00000000..1475d253 --- /dev/null +++ b/training-tools/README.md @@ -0,0 +1,20 @@ +# lczero-training-tools +Some tools to assist you in getting data for Leela Chess Zero + +# HOW TO USE +## Prepare your data +Prepare your data by going to [download site](https://storage.lczero.org/files/training_data/test80/) \ +Copy as many lines as you want. When your done, put them in linksdirty.txt (see linksdirty.txt for an example) \ +Run `python3 cleanlinks.py` to generate links.txt (see links.txt for an example)\ +Create a folder called lctd (Leela Chess Training Data) + +## Download the data +Download your data by running `python3 getdata.py` \ +A progress bar will show your downloads. 10 files will be downloaded at a time. \ +After your files finish being downloaded, run `python3 untar.py ./lctd/` \ +Again, you have a progress bar. One shows the overall progress for the extraction, and the other shows the progress for each file. + +## That's all! +Hopefully you find these tools helpful in your chess endevours. Good luck and happy training! + +### Note: Windows users may need to change the paths as this code is for linux. It should work on Windows but I am unable to test. diff --git a/training-tools/cleanlinks.py b/training-tools/cleanlinks.py new file mode 100644 index 00000000..76d3fe82 --- /dev/null +++ b/training-tools/cleanlinks.py @@ -0,0 +1,10 @@ +input_file = 'linksdirty.txt' +output_file = 'links.txt' + +with open(input_file, 'r') as f_in, open(output_file, 'w') as f_out: + for line in f_in: + index = line.find('.tar') + if index != -1: + tar_name = line[:index + 4] # Keep '.tar' in the output + f_out.write(tar_name + '\n') + diff --git a/training-tools/getdata.py b/training-tools/getdata.py new file mode 100644 index 00000000..03af5da8 --- /dev/null +++ b/training-tools/getdata.py @@ -0,0 +1,94 @@ +import requests +import threading +import os +import select +import sys +from tqdm import tqdm + +FILE_NAME = 'links.txt' +BASE_URL = 'https://storage.lczero.org/files/training_data/test80/' +DOWNLOAD_FOLDER = '~/Downloads/' +NUM_DOWNLOADS = 10 + + +def download_file(url, filename): + try: + response = requests.get(url, stream=True) + response.raise_for_status() + + total_size = int(response.headers.get('content-length', 0)) + progress_bar = tqdm(total=total_size, unit='iB', unit_scale=True, desc=filename) + + with open(os.path.join(DOWNLOAD_FOLDER, filename+'.tmp'), 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + progress_bar.update(len(chunk)) + + progress_bar.close() + print(f'Downloaded: {filename}') + os.system(f"mv {str(os.path.join(DOWNLOAD_FOLDER, filename+'.tmp'))} {str(os.path.join(DOWNLOAD_FOLDER+'lctd/', filename))}") + + except requests.exceptions.RequestException as e: + print(f'Error downloading {filename}: {e}') + + +def download_batch(links): + threads = [] + + for link in links: + url = BASE_URL + link + t = threading.Thread(target=download_file, args=(url, link)) + t.start() + threads.append(t) + + for t in threads: + t.join() + + +def remove_links_from_file(file, links): + with open(file, 'r') as f: + lines = f.readlines() + + with open(file, 'w') as f: + for line in lines: + if line.strip() not in links: + f.write(line) + + +def get_user_input(timeout): + print(f'Press ENTER to continue downloading or type "stop" to stop (waiting for {timeout} seconds): ') + i, _, _ = select.select([sys.stdin], [], [], timeout) + if i: + user_input = sys.stdin.readline().strip() + else: + user_input = "" + return user_input + +def main(): + if not os.path.exists(DOWNLOAD_FOLDER): + os.makedirs(DOWNLOAD_FOLDER) + + while True: + with open(FILE_NAME, 'r') as f: + links = [line.strip() for line in f.readlines() if line.strip()][:NUM_DOWNLOADS] + + if not links: + print('No more links to download.') + break + + download_batch(links) + + with open(FILE_NAME, 'r') as f: + lines = f.readlines() + + with open(FILE_NAME, 'w') as f: + for line in lines: + if line.strip() not in links: + f.write(line) + + user_input = get_user_input(timeout=60) + if user_input.lower() == 'stop': + break + +if __name__ == '__main__': + main() diff --git a/training-tools/links.txt b/training-tools/links.txt new file mode 100644 index 00000000..f9473701 --- /dev/null +++ b/training-tools/links.txt @@ -0,0 +1,303 @@ +training-run1-test80-20220407-0117.tar +training-run1-test80-20220407-0217.tar +training-run1-test80-20220407-0317.tar +training-run1-test80-20220407-0417.tar +training-run1-test80-20220407-0517.tar +training-run1-test80-20220407-0617.tar +training-run1-test80-20220407-0717.tar +training-run1-test80-20220407-0817.tar +training-run1-test80-20220407-0917.tar +training-run1-test80-20220407-1017.tar +training-run1-test80-20220407-1117.tar +training-run1-test80-20220407-1217.tar +training-run1-test80-20220407-1317.tar +training-run1-test80-20220407-1417.tar +training-run1-test80-20220407-1517.tar +training-run1-test80-20220407-1617.tar +training-run1-test80-20220407-1717.tar +training-run1-test80-20220407-1817.tar +training-run1-test80-20220407-1917.tar +training-run1-test80-20220407-2017.tar +training-run1-test80-20220407-2117.tar +training-run1-test80-20220407-2217.tar +training-run1-test80-20220407-2317.tar +training-run1-test80-20220408-0017.tar +training-run1-test80-20220408-0117.tar +training-run1-test80-20220408-0217.tar +training-run1-test80-20220408-0317.tar +training-run1-test80-20220408-0417.tar +training-run1-test80-20220408-0517.tar +training-run1-test80-20220408-0617.tar +training-run1-test80-20220408-0717.tar +training-run1-test80-20220408-0817.tar +training-run1-test80-20220408-0917.tar +training-run1-test80-20220408-1017.tar +training-run1-test80-20220408-1117.tar +training-run1-test80-20220408-1217.tar +training-run1-test80-20220408-1317.tar +training-run1-test80-20220408-1417.tar +training-run1-test80-20220408-1517.tar +training-run1-test80-20220408-1617.tar +training-run1-test80-20220408-1717.tar +training-run1-test80-20220408-1817.tar +training-run1-test80-20220408-1917.tar +training-run1-test80-20220408-2017.tar +training-run1-test80-20220408-2117.tar +training-run1-test80-20220408-2217.tar +training-run1-test80-20220408-2317.tar +training-run1-test80-20220409-0017.tar +training-run1-test80-20220409-0117.tar +training-run1-test80-20220409-0217.tar +training-run1-test80-20220409-0317.tar +training-run1-test80-20220409-0417.tar +training-run1-test80-20220409-0517.tar +training-run1-test80-20220409-0617.tar +training-run1-test80-20220409-0717.tar +training-run1-test80-20220409-0817.tar +training-run1-test80-20220409-0917.tar +training-run1-test80-20220409-1017.tar +training-run1-test80-20220409-1117.tar +training-run1-test80-20220409-1217.tar +training-run1-test80-20220409-1317.tar +training-run1-test80-20220409-1417.tar +training-run1-test80-20220409-1517.tar +training-run1-test80-20220409-1617.tar +training-run1-test80-20220409-1717.tar +training-run1-test80-20220409-1817.tar +training-run1-test80-20220409-1917.tar +training-run1-test80-20220409-2017.tar +training-run1-test80-20220409-2117.tar +training-run1-test80-20220409-2217.tar +training-run1-test80-20220409-2317.tar +training-run1-test80-20220410-0017.tar +training-run1-test80-20220410-0117.tar +training-run1-test80-20220410-0217.tar +training-run1-test80-20220410-0317.tar +training-run1-test80-20220410-0417.tar +training-run1-test80-20220410-0517.tar +training-run1-test80-20220410-0617.tar +training-run1-test80-20220410-0717.tar +training-run1-test80-20220410-0817.tar +training-run1-test80-20220410-0917.tar +training-run1-test80-20220410-1017.tar +training-run1-test80-20220410-1117.tar +training-run1-test80-20220410-1217.tar +training-run1-test80-20220410-1317.tar +training-run1-test80-20220410-1417.tar +training-run1-test80-20220410-1517.tar +training-run1-test80-20220410-1617.tar +training-run1-test80-20220410-1717.tar +training-run1-test80-20220410-1817.tar +training-run1-test80-20220410-1917.tar +training-run1-test80-20220410-2017.tar +training-run1-test80-20220410-2117.tar +training-run1-test80-20220410-2217.tar +training-run1-test80-20220410-2317.tar +training-run1-test80-20220411-0017.tar +training-run1-test80-20220411-0117.tar +training-run1-test80-20220411-0217.tar +training-run1-test80-20220411-0317.tar +training-run1-test80-20220411-0417.tar +training-run1-test80-20220411-0517.tar +training-run1-test80-20220411-0617.tar +training-run1-test80-20220411-0717.tar +training-run1-test80-20220411-0817.tar +training-run1-test80-20220411-0917.tar +training-run1-test80-20220411-1117.tar +training-run1-test80-20220411-1217.tar +training-run1-test80-20220411-1317.tar +training-run1-test80-20220411-1417.tar +training-run1-test80-20220411-1517.tar +training-run1-test80-20220411-1617.tar +training-run1-test80-20220411-1717.tar +training-run1-test80-20220411-1817.tar +training-run1-test80-20220411-1917.tar +training-run1-test80-20220411-2017.tar +training-run1-test80-20220411-2117.tar +training-run1-test80-20220411-2217.tar +training-run1-test80-20220411-2317.tar +training-run1-test80-20220412-0017.tar +training-run1-test80-20220412-0117.tar +training-run1-test80-20220412-0217.tar +training-run1-test80-20220412-0317.tar +training-run1-test80-20220412-0417.tar +training-run1-test80-20220412-0517.tar +training-run1-test80-20220412-0617.tar +training-run1-test80-20220412-0717.tar +training-run1-test80-20220412-0817.tar +training-run1-test80-20220412-0917.tar +training-run1-test80-20220412-1017.tar +training-run1-test80-20220412-1117.tar +training-run1-test80-20220412-1217.tar +training-run1-test80-20220412-1317.tar +training-run1-test80-20220412-1417.tar +training-run1-test80-20220412-1517.tar +training-run1-test80-20220412-1617.tar +training-run1-test80-20220412-1717.tar +training-run1-test80-20220412-1817.tar +training-run1-test80-20220412-1917.tar +training-run1-test80-20220412-2017.tar +training-run1-test80-20220412-2117.tar +training-run1-test80-20220412-2217.tar +training-run1-test80-20220412-2317.tar +training-run1-test80-20220413-0017.tar +training-run1-test80-20220413-0117.tar +training-run1-test80-20220413-0217.tar +training-run1-test80-20220413-0317.tar +training-run1-test80-20220413-0417.tar +training-run1-test80-20220413-0517.tar +training-run1-test80-20220413-0617.tar +training-run1-test80-20220413-0717.tar +training-run1-test80-20220413-0817.tar +training-run1-test80-20220413-0917.tar +training-run1-test80-20220413-1017.tar +training-run1-test80-20220413-1117.tar +training-run1-test80-20220413-1217.tar +training-run1-test80-20220413-1317.tar +training-run1-test80-20220413-1417.tar +training-run1-test80-20220413-1517.tar +training-run1-test80-20220413-1617.tar +training-run1-test80-20220413-1717.tar +training-run1-test80-20220413-1817.tar +training-run1-test80-20220413-1917.tar +training-run1-test80-20220413-2017.tar +training-run1-test80-20220413-2117.tar +training-run1-test80-20220413-2217.tar +training-run1-test80-20220413-2317.tar +training-run1-test80-20220414-0017.tar +training-run1-test80-20220414-0117.tar +training-run1-test80-20220414-0217.tar +training-run1-test80-20220414-1717.tar +training-run1-test80-20220414-1817.tar +training-run1-test80-20220414-1917.tar +training-run1-test80-20220414-2017.tar +training-run1-test80-20220414-2117.tar +training-run1-test80-20220414-2217.tar +training-run1-test80-20220414-2317.tar +training-run1-test80-20220415-0017.tar +training-run1-test80-20220415-0117.tar +training-run1-test80-20220415-0217.tar +training-run1-test80-20220415-0317.tar +training-run1-test80-20220415-0417.tar +training-run1-test80-20220415-0517.tar +training-run1-test80-20220415-0617.tar +training-run1-test80-20220415-0717.tar +training-run1-test80-20220415-0817.tar +training-run1-test80-20220415-0917.tar +training-run1-test80-20220415-1017.tar +training-run1-test80-20220415-1117.tar +training-run1-test80-20220415-1217.tar +training-run1-test80-20220415-1317.tar +training-run1-test80-20220415-1417.tar +training-run1-test80-20220415-1517.tar +training-run1-test80-20220415-1617.tar +training-run1-test80-20220415-1717.tar +training-run1-test80-20220415-1817.tar +training-run1-test80-20220415-1917.tar +training-run1-test80-20220415-2017.tar +training-run1-test80-20220415-2117.tar +training-run1-test80-20220415-2217.tar +training-run1-test80-20220415-2317.tar +training-run1-test80-20220416-0017.tar +training-run1-test80-20220416-0117.tar +training-run1-test80-20220416-0217.tar +training-run1-test80-20220416-0317.tar +training-run1-test80-20220416-0417.tar +training-run1-test80-20220416-0517.tar +training-run1-test80-20220416-0617.tar +training-run1-test80-20220416-0717.tar +training-run1-test80-20220416-0817.tar +training-run1-test80-20220416-0917.tar +training-run1-test80-20220416-1017.tar +training-run1-test80-20220416-1117.tar +training-run1-test80-20220416-1217.tar +training-run1-test80-20220416-1317.tar +training-run1-test80-20220416-1417.tar +training-run1-test80-20220416-1517.tar +training-run1-test80-20220416-1617.tar +training-run1-test80-20220416-1717.tar +training-run1-test80-20220416-1817.tar +training-run1-test80-20220416-1917.tar +training-run1-test80-20220416-2017.tar +training-run1-test80-20220416-2117.tar +training-run1-test80-20220416-2217.tar +training-run1-test80-20220416-2317.tar +training-run1-test80-20220417-0017.tar +training-run1-test80-20220417-0117.tar +training-run1-test80-20220417-0217.tar +training-run1-test80-20220417-0317.tar +training-run1-test80-20220417-0417.tar +training-run1-test80-20220417-0517.tar +training-run1-test80-20220417-0617.tar +training-run1-test80-20220417-0717.tar +training-run1-test80-20220417-0817.tar +training-run1-test80-20220417-0917.tar +training-run1-test80-20220417-1017.tar +training-run1-test80-20220417-1117.tar +training-run1-test80-20220417-1217.tar +training-run1-test80-20220417-1317.tar +training-run1-test80-20220417-1417.tar +training-run1-test80-20220417-1517.tar +training-run1-test80-20220417-1617.tar +training-run1-test80-20220417-1717.tar +training-run1-test80-20220417-1817.tar +training-run1-test80-20220417-1917.tar +training-run1-test80-20220417-2017.tar +training-run1-test80-20220417-2117.tar +training-run1-test80-20220417-2217.tar +training-run1-test80-20220417-2317.tar +training-run1-test80-20220418-0017.tar +training-run1-test80-20220418-0117.tar +training-run1-test80-20220418-0217.tar +training-run1-test80-20220418-0317.tar +training-run1-test80-20220418-0417.tar +training-run1-test80-20220418-0517.tar +training-run1-test80-20220418-0617.tar +training-run1-test80-20220418-0717.tar +training-run1-test80-20220418-0817.tar +training-run1-test80-20220418-0917.tar +training-run1-test80-20220418-1017.tar +training-run1-test80-20220418-1117.tar +training-run1-test80-20220418-1217.tar +training-run1-test80-20220418-1317.tar +training-run1-test80-20220418-1417.tar +training-run1-test80-20220418-1517.tar +training-run1-test80-20220418-1617.tar +training-run1-test80-20220418-1717.tar +training-run1-test80-20220418-1817.tar +training-run1-test80-20220418-1917.tar +training-run1-test80-20220418-2017.tar +training-run1-test80-20220418-2117.tar +training-run1-test80-20220418-2217.tar +training-run1-test80-20220418-2317.tar +training-run1-test80-20220419-0017.tar +training-run1-test80-20220419-0117.tar +training-run1-test80-20220419-0217.tar +training-run1-test80-20220419-0317.tar +training-run1-test80-20220419-0417.tar +training-run1-test80-20220419-0517.tar +training-run1-test80-20220419-0617.tar +training-run1-test80-20220419-0717.tar +training-run1-test80-20220419-0817.tar +training-run1-test80-20220419-0917.tar +training-run1-test80-20220419-1017.tar +training-run1-test80-20220419-1117.tar +training-run1-test80-20220419-1217.tar +training-run1-test80-20220419-1317.tar +training-run1-test80-20220419-1417.tar +training-run1-test80-20220419-1517.tar +training-run1-test80-20220419-1617.tar +training-run1-test80-20220419-1717.tar +training-run1-test80-20220419-1817.tar +training-run1-test80-20220419-1917.tar +training-run1-test80-20220419-2017.tar +training-run1-test80-20220419-2117.tar +training-run1-test80-20220419-2217.tar +training-run1-test80-20220419-2317.tar +training-run1-test80-20220420-0017.tar +training-run1-test80-20220420-0117.tar +training-run1-test80-20220420-0217.tar +training-run1-test80-20220420-0317.tar +training-run1-test80-20220420-0417.tar +training-run1-test80-20220420-0517.tar +training-run1-test80-20220420-0617.tar diff --git a/training-tools/linksdirty.txt b/training-tools/linksdirty.txt new file mode 100644 index 00000000..8c1b901f --- /dev/null +++ b/training-tools/linksdirty.txt @@ -0,0 +1,303 @@ +training-run1-test80-20220407-0117.tar 07-Apr-2022 01:20 1080289280 +training-run1-test80-20220407-0217.tar 07-Apr-2022 02:20 1123194880 +training-run1-test80-20220407-0317.tar 07-Apr-2022 03:20 1178480640 +training-run1-test80-20220407-0417.tar 07-Apr-2022 04:20 1172019200 +training-run1-test80-20220407-0517.tar 07-Apr-2022 05:21 1279549440 +training-run1-test80-20220407-0617.tar 07-Apr-2022 06:20 1181757440 +training-run1-test80-20220407-0717.tar 07-Apr-2022 07:20 1184512000 +training-run1-test80-20220407-0817.tar 07-Apr-2022 08:20 1152327680 +training-run1-test80-20220407-0917.tar 07-Apr-2022 09:20 1141176320 +training-run1-test80-20220407-1017.tar 07-Apr-2022 10:20 1230336000 +training-run1-test80-20220407-1117.tar 07-Apr-2022 11:19 972871680 +training-run1-test80-20220407-1217.tar 07-Apr-2022 12:20 1091389440 +training-run1-test80-20220407-1317.tar 07-Apr-2022 13:20 1070428160 +training-run1-test80-20220407-1417.tar 07-Apr-2022 14:20 1192796160 +training-run1-test80-20220407-1517.tar 07-Apr-2022 15:20 1091901440 +training-run1-test80-20220407-1617.tar 07-Apr-2022 16:20 1072650240 +training-run1-test80-20220407-1717.tar 07-Apr-2022 17:19 1063864320 +training-run1-test80-20220407-1817.tar 07-Apr-2022 18:20 1174661120 +training-run1-test80-20220407-1917.tar 07-Apr-2022 19:20 1100615680 +training-run1-test80-20220407-2017.tar 07-Apr-2022 20:20 1097379840 +training-run1-test80-20220407-2117.tar 07-Apr-2022 21:20 1079326720 +training-run1-test80-20220407-2217.tar 07-Apr-2022 22:21 1255004160 +training-run1-test80-20220407-2317.tar 07-Apr-2022 23:20 1157642240 +training-run1-test80-20220408-0017.tar 08-Apr-2022 00:20 1164083200 +training-run1-test80-20220408-0117.tar 08-Apr-2022 01:20 1130414080 +training-run1-test80-20220408-0217.tar 08-Apr-2022 02:20 1125130240 +training-run1-test80-20220408-0317.tar 08-Apr-2022 03:20 1234862080 +training-run1-test80-20220408-0417.tar 08-Apr-2022 04:20 1141903360 +training-run1-test80-20220408-0517.tar 08-Apr-2022 05:20 1122959360 +training-run1-test80-20220408-0617.tar 08-Apr-2022 06:19 960901120 +training-run1-test80-20220408-0717.tar 08-Apr-2022 07:20 1222430720 +training-run1-test80-20220408-0817.tar 08-Apr-2022 08:20 1134336000 +training-run1-test80-20220408-0917.tar 08-Apr-2022 09:20 1149163520 +training-run1-test80-20220408-1017.tar 08-Apr-2022 10:19 1141288960 +training-run1-test80-20220408-1117.tar 08-Apr-2022 11:20 1246607360 +training-run1-test80-20220408-1217.tar 08-Apr-2022 12:20 1160611840 +training-run1-test80-20220408-1317.tar 08-Apr-2022 13:20 1128099840 +training-run1-test80-20220408-1417.tar 08-Apr-2022 14:20 1278658560 +training-run1-test80-20220408-1517.tar 08-Apr-2022 15:20 1191127040 +training-run1-test80-20220408-1617.tar 08-Apr-2022 16:20 1229578240 +training-run1-test80-20220408-1717.tar 08-Apr-2022 17:20 1201838080 +training-run1-test80-20220408-1817.tar 08-Apr-2022 18:20 1199687680 +training-run1-test80-20220408-1917.tar 08-Apr-2022 19:20 1313730560 +training-run1-test80-20220408-2017.tar 08-Apr-2022 20:20 1362565120 +training-run1-test80-20220408-2117.tar 08-Apr-2022 21:21 1387653120 +training-run1-test80-20220408-2217.tar 08-Apr-2022 22:20 1234964480 +training-run1-test80-20220408-2317.tar 08-Apr-2022 23:20 1265786880 +training-run1-test80-20220409-0017.tar 09-Apr-2022 00:20 1252608000 +training-run1-test80-20220409-0117.tar 09-Apr-2022 01:19 1094420480 +training-run1-test80-20220409-0217.tar 09-Apr-2022 02:20 1270568960 +training-run1-test80-20220409-0317.tar 09-Apr-2022 03:20 1273845760 +training-run1-test80-20220409-0417.tar 09-Apr-2022 04:20 1347543040 +training-run1-test80-20220409-0517.tar 09-Apr-2022 05:20 1265233920 +training-run1-test80-20220409-0617.tar 09-Apr-2022 06:20 1283348480 +training-run1-test80-20220409-0717.tar 09-Apr-2022 07:20 1301678080 +training-run1-test80-20220409-0817.tar 09-Apr-2022 08:20 1264957440 +training-run1-test80-20220409-0917.tar 09-Apr-2022 09:20 1230336000 +training-run1-test80-20220409-1017.tar 09-Apr-2022 10:20 1218048000 +training-run1-test80-20220409-1117.tar 09-Apr-2022 11:20 1299148800 +training-run1-test80-20220409-1217.tar 09-Apr-2022 12:20 1272053760 +training-run1-test80-20220409-1317.tar 09-Apr-2022 13:20 1249607680 +training-run1-test80-20220409-1417.tar 09-Apr-2022 14:20 1270046720 +training-run1-test80-20220409-1517.tar 09-Apr-2022 15:20 1261916160 +training-run1-test80-20220409-1617.tar 09-Apr-2022 16:20 1278392320 +training-run1-test80-20220409-1717.tar 09-Apr-2022 17:20 1269391360 +training-run1-test80-20220409-1817.tar 09-Apr-2022 18:20 1199298560 +training-run1-test80-20220409-1917.tar 09-Apr-2022 19:20 1339084800 +training-run1-test80-20220409-2017.tar 09-Apr-2022 20:20 1280665600 +training-run1-test80-20220409-2117.tar 09-Apr-2022 21:20 1265305600 +training-run1-test80-20220409-2217.tar 09-Apr-2022 22:20 1276262400 +training-run1-test80-20220409-2317.tar 09-Apr-2022 23:20 1243166720 +training-run1-test80-20220410-0017.tar 10-Apr-2022 00:20 1248020480 +training-run1-test80-20220410-0117.tar 10-Apr-2022 01:20 1266380800 +training-run1-test80-20220410-0217.tar 10-Apr-2022 02:20 1324595200 +training-run1-test80-20220410-0317.tar 10-Apr-2022 03:21 1336473600 +training-run1-test80-20220410-0417.tar 10-Apr-2022 04:20 1259008000 +training-run1-test80-20220410-0517.tar 10-Apr-2022 05:20 1271818240 +training-run1-test80-20220410-0617.tar 10-Apr-2022 06:20 1265192960 +training-run1-test80-20220410-0717.tar 10-Apr-2022 07:20 1244098560 +training-run1-test80-20220410-0817.tar 10-Apr-2022 08:20 1264773120 +training-run1-test80-20220410-0917.tar 10-Apr-2022 09:20 1230960640 +training-run1-test80-20220410-1017.tar 10-Apr-2022 10:20 1357824000 +training-run1-test80-20220410-1117.tar 10-Apr-2022 11:20 1280727040 +training-run1-test80-20220410-1217.tar 10-Apr-2022 12:20 1159321600 +training-run1-test80-20220410-1317.tar 10-Apr-2022 13:20 1260912640 +training-run1-test80-20220410-1417.tar 10-Apr-2022 14:20 1237667840 +training-run1-test80-20220410-1517.tar 10-Apr-2022 15:21 1282856960 +training-run1-test80-20220410-1617.tar 10-Apr-2022 16:20 1294059520 +training-run1-test80-20220410-1717.tar 10-Apr-2022 17:21 1382635520 +training-run1-test80-20220410-1817.tar 10-Apr-2022 18:20 1294325760 +training-run1-test80-20220410-1917.tar 10-Apr-2022 19:20 1300008960 +training-run1-test80-20220410-2017.tar 10-Apr-2022 20:20 1280788480 +training-run1-test80-20220410-2117.tar 10-Apr-2022 21:20 1273712640 +training-run1-test80-20220410-2217.tar 10-Apr-2022 22:20 1282682880 +training-run1-test80-20220410-2317.tar 10-Apr-2022 23:20 1133445120 +training-run1-test80-20220411-0017.tar 11-Apr-2022 00:20 1302026240 +training-run1-test80-20220411-0117.tar 11-Apr-2022 01:20 1304289280 +training-run1-test80-20220411-0217.tar 11-Apr-2022 02:20 1305763840 +training-run1-test80-20220411-0317.tar 11-Apr-2022 03:21 1430087680 +training-run1-test80-20220411-0417.tar 11-Apr-2022 04:20 1353328640 +training-run1-test80-20220411-0517.tar 11-Apr-2022 05:20 1235701760 +training-run1-test80-20220411-0617.tar 11-Apr-2022 06:21 1333493760 +training-run1-test80-20220411-0717.tar 11-Apr-2022 07:21 1345771520 +training-run1-test80-20220411-0817.tar 11-Apr-2022 08:21 1357516800 +training-run1-test80-20220411-0917.tar 11-Apr-2022 09:20 1344952320 +training-run1-test80-20220411-1117.tar 11-Apr-2022 11:21 1301370880 +training-run1-test80-20220411-1217.tar 11-Apr-2022 12:20 1296250880 +training-run1-test80-20220411-1317.tar 11-Apr-2022 13:20 1300664320 +training-run1-test80-20220411-1417.tar 11-Apr-2022 14:21 1386762240 +training-run1-test80-20220411-1517.tar 11-Apr-2022 15:20 1284689920 +training-run1-test80-20220411-1617.tar 11-Apr-2022 16:20 1286451200 +training-run1-test80-20220411-1717.tar 11-Apr-2022 17:20 1285171200 +training-run1-test80-20220411-1817.tar 11-Apr-2022 18:21 1298585600 +training-run1-test80-20220411-1917.tar 11-Apr-2022 19:20 1271541760 +training-run1-test80-20220411-2017.tar 11-Apr-2022 20:20 1265192960 +training-run1-test80-20220411-2117.tar 11-Apr-2022 21:20 1300377600 +training-run1-test80-20220411-2217.tar 11-Apr-2022 22:20 1267916800 +training-run1-test80-20220411-2317.tar 11-Apr-2022 23:20 1297684480 +training-run1-test80-20220412-0017.tar 12-Apr-2022 00:20 1292093440 +training-run1-test80-20220412-0117.tar 12-Apr-2022 01:21 1367644160 +training-run1-test80-20220412-0217.tar 12-Apr-2022 02:21 1367449600 +training-run1-test80-20220412-0317.tar 12-Apr-2022 03:21 1328015360 +training-run1-test80-20220412-0417.tar 12-Apr-2022 04:20 1323059200 +training-run1-test80-20220412-0517.tar 12-Apr-2022 05:20 1183651840 +training-run1-test80-20220412-0617.tar 12-Apr-2022 06:20 1333483520 +training-run1-test80-20220412-0717.tar 12-Apr-2022 07:20 1307105280 +training-run1-test80-20220412-0817.tar 12-Apr-2022 08:20 1221468160 +training-run1-test80-20220412-0917.tar 12-Apr-2022 09:20 1240842240 +training-run1-test80-20220412-1017.tar 12-Apr-2022 10:20 1342617600 +training-run1-test80-20220412-1117.tar 12-Apr-2022 11:20 1122488320 +training-run1-test80-20220412-1217.tar 12-Apr-2022 12:21 1337364480 +training-run1-test80-20220412-1317.tar 12-Apr-2022 13:21 1346140160 +training-run1-test80-20220412-1417.tar 12-Apr-2022 14:21 1355407360 +training-run1-test80-20220412-1517.tar 12-Apr-2022 15:21 1374914560 +training-run1-test80-20220412-1617.tar 12-Apr-2022 16:21 1373624320 +training-run1-test80-20220412-1717.tar 12-Apr-2022 17:21 1370859520 +training-run1-test80-20220412-1817.tar 12-Apr-2022 18:20 1351434240 +training-run1-test80-20220412-1917.tar 12-Apr-2022 19:21 1353390080 +training-run1-test80-20220412-2017.tar 12-Apr-2022 20:20 1332090880 +training-run1-test80-20220412-2117.tar 12-Apr-2022 21:20 1330094080 +training-run1-test80-20220412-2217.tar 12-Apr-2022 22:20 1246515200 +training-run1-test80-20220412-2317.tar 12-Apr-2022 23:20 1349806080 +training-run1-test80-20220413-0017.tar 13-Apr-2022 00:20 1290874880 +training-run1-test80-20220413-0117.tar 13-Apr-2022 01:20 1277286400 +training-run1-test80-20220413-0217.tar 13-Apr-2022 02:20 1320785920 +training-run1-test80-20220413-0317.tar 13-Apr-2022 03:20 1301565440 +training-run1-test80-20220413-0417.tar 13-Apr-2022 04:20 1252044800 +training-run1-test80-20220413-0517.tar 13-Apr-2022 05:20 1285335040 +training-run1-test80-20220413-0617.tar 13-Apr-2022 06:21 1290280960 +training-run1-test80-20220413-0717.tar 13-Apr-2022 07:20 1336033280 +training-run1-test80-20220413-0817.tar 13-Apr-2022 08:21 1383905280 +training-run1-test80-20220413-0917.tar 13-Apr-2022 09:20 1193861120 +training-run1-test80-20220413-1017.tar 13-Apr-2022 10:20 1343580160 +training-run1-test80-20220413-1117.tar 13-Apr-2022 11:20 1347153920 +training-run1-test80-20220413-1217.tar 13-Apr-2022 12:21 1352888320 +training-run1-test80-20220413-1317.tar 13-Apr-2022 13:21 1346846720 +training-run1-test80-20220413-1417.tar 13-Apr-2022 14:21 1362503680 +training-run1-test80-20220413-1517.tar 13-Apr-2022 15:21 1359708160 +training-run1-test80-20220413-1617.tar 13-Apr-2022 16:20 1369804800 +training-run1-test80-20220413-1717.tar 13-Apr-2022 17:21 1336197120 +training-run1-test80-20220413-1817.tar 13-Apr-2022 18:21 1352878080 +training-run1-test80-20220413-1917.tar 13-Apr-2022 19:21 1381324800 +training-run1-test80-20220413-2017.tar 13-Apr-2022 20:21 1398261760 +training-run1-test80-20220413-2117.tar 13-Apr-2022 21:20 1319168000 +training-run1-test80-20220413-2217.tar 13-Apr-2022 22:20 1324574720 +training-run1-test80-20220413-2317.tar 13-Apr-2022 23:20 1302947840 +training-run1-test80-20220414-0017.tar 14-Apr-2022 00:20 1304954880 +training-run1-test80-20220414-0117.tar 14-Apr-2022 01:19 954798080 +training-run1-test80-20220414-0217.tar 14-Apr-2022 02:20 1229762560 +training-run1-test80-20220414-1717.tar 14-Apr-2022 17:17 14489600 +training-run1-test80-20220414-1817.tar 14-Apr-2022 18:18 498442240 +training-run1-test80-20220414-1917.tar 14-Apr-2022 19:18 426506240 +training-run1-test80-20220414-2017.tar 14-Apr-2022 20:20 1134090240 +training-run1-test80-20220414-2117.tar 14-Apr-2022 21:20 1156935680 +training-run1-test80-20220414-2217.tar 14-Apr-2022 22:21 1340651520 +training-run1-test80-20220414-2317.tar 14-Apr-2022 23:20 1222287360 +training-run1-test80-20220415-0017.tar 15-Apr-2022 00:20 1265838080 +training-run1-test80-20220415-0117.tar 15-Apr-2022 01:20 1281669120 +training-run1-test80-20220415-0217.tar 15-Apr-2022 02:20 1288888320 +training-run1-test80-20220415-0317.tar 15-Apr-2022 03:21 1324974080 +training-run1-test80-20220415-0417.tar 15-Apr-2022 04:20 1312491520 +training-run1-test80-20220415-0517.tar 15-Apr-2022 05:20 1302087680 +training-run1-test80-20220415-0617.tar 15-Apr-2022 06:21 1334272000 +training-run1-test80-20220415-0717.tar 15-Apr-2022 07:21 1437788160 +training-run1-test80-20220415-0817.tar 15-Apr-2022 08:21 1345935360 +training-run1-test80-20220415-0917.tar 15-Apr-2022 09:21 1323458560 +training-run1-test80-20220415-1017.tar 15-Apr-2022 10:21 1338798080 +training-run1-test80-20220415-1117.tar 15-Apr-2022 11:20 1194506240 +training-run1-test80-20220415-1217.tar 15-Apr-2022 12:21 1319567360 +training-run1-test80-20220415-1317.tar 15-Apr-2022 13:21 1318471680 +training-run1-test80-20220415-1417.tar 15-Apr-2022 14:21 1310504960 +training-run1-test80-20220415-1517.tar 15-Apr-2022 15:20 1287997440 +training-run1-test80-20220415-1617.tar 15-Apr-2022 16:21 1363527680 +training-run1-test80-20220415-1717.tar 15-Apr-2022 17:21 1357629440 +training-run1-test80-20220415-1817.tar 15-Apr-2022 18:20 1294202880 +training-run1-test80-20220415-1917.tar 15-Apr-2022 19:21 1306705920 +training-run1-test80-20220415-2017.tar 15-Apr-2022 20:20 1309603840 +training-run1-test80-20220415-2117.tar 15-Apr-2022 21:21 1368576000 +training-run1-test80-20220415-2217.tar 15-Apr-2022 22:20 1202237440 +training-run1-test80-20220415-2317.tar 15-Apr-2022 23:21 1415106560 +training-run1-test80-20220416-0017.tar 16-Apr-2022 00:21 1357649920 +training-run1-test80-20220416-0117.tar 16-Apr-2022 01:21 1333944320 +training-run1-test80-20220416-0217.tar 16-Apr-2022 02:21 1324810240 +training-run1-test80-20220416-0317.tar 16-Apr-2022 03:21 1378785280 +training-run1-test80-20220416-0417.tar 16-Apr-2022 04:20 1295155200 +training-run1-test80-20220416-0517.tar 16-Apr-2022 05:21 1353728000 +training-run1-test80-20220416-0617.tar 16-Apr-2022 06:21 1391206400 +training-run1-test80-20220416-0717.tar 16-Apr-2022 07:21 1404692480 +training-run1-test80-20220416-0817.tar 16-Apr-2022 08:21 1395486720 +training-run1-test80-20220416-0917.tar 16-Apr-2022 09:21 1379788800 +training-run1-test80-20220416-1017.tar 16-Apr-2022 10:21 1404272640 +training-run1-test80-20220416-1117.tar 16-Apr-2022 11:21 1421967360 +training-run1-test80-20220416-1217.tar 16-Apr-2022 12:21 1387509760 +training-run1-test80-20220416-1317.tar 16-Apr-2022 13:21 1373808640 +training-run1-test80-20220416-1417.tar 16-Apr-2022 14:21 1386557440 +training-run1-test80-20220416-1517.tar 16-Apr-2022 15:21 1340262400 +training-run1-test80-20220416-1617.tar 16-Apr-2022 16:21 1366814720 +training-run1-test80-20220416-1717.tar 16-Apr-2022 17:21 1359083520 +training-run1-test80-20220416-1817.tar 16-Apr-2022 18:21 1361858560 +training-run1-test80-20220416-1917.tar 16-Apr-2022 19:21 1385205760 +training-run1-test80-20220416-2017.tar 16-Apr-2022 20:21 1308405760 +training-run1-test80-20220416-2117.tar 16-Apr-2022 21:20 1317447680 +training-run1-test80-20220416-2217.tar 16-Apr-2022 22:20 1335480320 +training-run1-test80-20220416-2317.tar 16-Apr-2022 23:21 1332049920 +training-run1-test80-20220417-0017.tar 17-Apr-2022 00:20 1274787840 +training-run1-test80-20220417-0117.tar 17-Apr-2022 01:20 1020917760 +training-run1-test80-20220417-0217.tar 17-Apr-2022 02:21 1297500160 +training-run1-test80-20220417-0317.tar 17-Apr-2022 03:20 1236183040 +training-run1-test80-20220417-0417.tar 17-Apr-2022 04:20 1203978240 +training-run1-test80-20220417-0517.tar 17-Apr-2022 05:20 1210091520 +training-run1-test80-20220417-0617.tar 17-Apr-2022 06:20 1204940800 +training-run1-test80-20220417-0717.tar 17-Apr-2022 07:20 1223424000 +training-run1-test80-20220417-0817.tar 17-Apr-2022 08:21 1315215360 +training-run1-test80-20220417-0917.tar 17-Apr-2022 09:20 1231728640 +training-run1-test80-20220417-1017.tar 17-Apr-2022 10:20 1226496000 +training-run1-test80-20220417-1117.tar 17-Apr-2022 11:20 1202380800 +training-run1-test80-20220417-1217.tar 17-Apr-2022 12:20 1295226880 +training-run1-test80-20220417-1317.tar 17-Apr-2022 13:20 1138698240 +training-run1-test80-20220417-1417.tar 17-Apr-2022 14:21 1335193600 +training-run1-test80-20220417-1517.tar 17-Apr-2022 15:21 1345443840 +training-run1-test80-20220417-1617.tar 17-Apr-2022 16:20 1277358080 +training-run1-test80-20220417-1717.tar 17-Apr-2022 17:21 1276815360 +training-run1-test80-20220417-1817.tar 17-Apr-2022 18:21 1273262080 +training-run1-test80-20220417-1917.tar 17-Apr-2022 19:20 1267107840 +training-run1-test80-20220417-2017.tar 17-Apr-2022 20:20 1249730560 +training-run1-test80-20220417-2117.tar 17-Apr-2022 21:20 1256079360 +training-run1-test80-20220417-2217.tar 17-Apr-2022 22:21 1280122880 +training-run1-test80-20220417-2317.tar 17-Apr-2022 23:21 1357639680 +training-run1-test80-20220418-0017.tar 18-Apr-2022 00:20 1291714560 +training-run1-test80-20220418-0117.tar 18-Apr-2022 01:20 1313812480 +training-run1-test80-20220418-0217.tar 18-Apr-2022 02:21 1306460160 +training-run1-test80-20220418-0317.tar 18-Apr-2022 03:21 1319669760 +training-run1-test80-20220418-0417.tar 18-Apr-2022 04:20 1296271360 +training-run1-test80-20220418-0517.tar 18-Apr-2022 05:21 1322004480 +training-run1-test80-20220418-0617.tar 18-Apr-2022 06:20 1064949760 +training-run1-test80-20220418-0717.tar 18-Apr-2022 07:21 1429841920 +training-run1-test80-20220418-0817.tar 18-Apr-2022 08:21 1360230400 +training-run1-test80-20220418-0917.tar 18-Apr-2022 09:21 1369999360 +training-run1-test80-20220418-1017.tar 18-Apr-2022 10:21 1369845760 +training-run1-test80-20220418-1117.tar 18-Apr-2022 11:21 1348526080 +training-run1-test80-20220418-1217.tar 18-Apr-2022 12:20 1323335680 +training-run1-test80-20220418-1317.tar 18-Apr-2022 13:20 1271111680 +training-run1-test80-20220418-1417.tar 18-Apr-2022 14:21 1301504000 +training-run1-test80-20220418-1517.tar 18-Apr-2022 15:20 1314283520 +training-run1-test80-20220418-1617.tar 18-Apr-2022 16:21 1279856640 +training-run1-test80-20220418-1717.tar 18-Apr-2022 17:21 1329469440 +training-run1-test80-20220418-1817.tar 18-Apr-2022 18:21 1375354880 +training-run1-test80-20220418-1917.tar 18-Apr-2022 19:21 1403207680 +training-run1-test80-20220418-2017.tar 18-Apr-2022 20:21 1423820800 +training-run1-test80-20220418-2117.tar 18-Apr-2022 21:21 1375549440 +training-run1-test80-20220418-2217.tar 18-Apr-2022 22:21 1333073920 +training-run1-test80-20220418-2317.tar 18-Apr-2022 23:20 1173002240 +training-run1-test80-20220419-0017.tar 19-Apr-2022 00:21 1342310400 +training-run1-test80-20220419-0117.tar 19-Apr-2022 01:20 1320949760 +training-run1-test80-20220419-0217.tar 19-Apr-2022 02:20 1290874880 +training-run1-test80-20220419-0317.tar 19-Apr-2022 03:21 1308067840 +training-run1-test80-20220419-0417.tar 19-Apr-2022 04:20 1300039680 +training-run1-test80-20220419-0517.tar 19-Apr-2022 05:21 1335398400 +training-run1-test80-20220419-0617.tar 19-Apr-2022 06:22 1374689280 +training-run1-test80-20220419-0717.tar 19-Apr-2022 07:21 1415444480 +training-run1-test80-20220419-0817.tar 19-Apr-2022 08:21 1330513920 +training-run1-test80-20220419-0917.tar 19-Apr-2022 09:21 1229168640 +training-run1-test80-20220419-1017.tar 19-Apr-2022 10:21 1107087360 +training-run1-test80-20220419-1117.tar 19-Apr-2022 11:22 1349795840 +training-run1-test80-20220419-1217.tar 19-Apr-2022 12:21 1347799040 +training-run1-test80-20220419-1317.tar 19-Apr-2022 13:21 1293844480 +training-run1-test80-20220419-1417.tar 19-Apr-2022 14:21 1352693760 +training-run1-test80-20220419-1517.tar 19-Apr-2022 15:20 1243115520 +training-run1-test80-20220419-1617.tar 19-Apr-2022 16:21 1433856000 +training-run1-test80-20220419-1717.tar 19-Apr-2022 17:21 1326510080 +training-run1-test80-20220419-1817.tar 19-Apr-2022 18:21 1289789440 +training-run1-test80-20220419-1917.tar 19-Apr-2022 19:20 1292605440 +training-run1-test80-20220419-2017.tar 19-Apr-2022 20:21 1426944000 +training-run1-test80-20220419-2117.tar 19-Apr-2022 21:21 1445120000 +training-run1-test80-20220419-2217.tar 19-Apr-2022 22:21 1443215360 +training-run1-test80-20220419-2317.tar 19-Apr-2022 23:21 1462712320 +training-run1-test80-20220420-0017.tar 20-Apr-2022 00:21 1518970880 +training-run1-test80-20220420-0117.tar 20-Apr-2022 01:21 1451284480 +training-run1-test80-20220420-0217.tar 20-Apr-2022 02:21 1455267840 +training-run1-test80-20220420-0317.tar 20-Apr-2022 03:21 1462138880 +training-run1-test80-20220420-0417.tar 20-Apr-2022 04:21 1403084800 +training-run1-test80-20220420-0517.tar +training-run1-test80-20220420-0617.tar diff --git a/training-tools/untar.py b/training-tools/untar.py new file mode 100644 index 00000000..5fb8e393 --- /dev/null +++ b/training-tools/untar.py @@ -0,0 +1,43 @@ +import os +import argparse +import shutil +import tarfile +from tqdm import tqdm + +parser = argparse.ArgumentParser(description="Extract all .tar files in a directory and its subdirectories") +parser.add_argument("src_dir", metavar="directory", type=str, help="the directory containing the .tar files to extract") +args = parser.parse_args() + +src_dir = args.src_dir + +if not os.path.isdir(src_dir): + print("Error: {} is not a directory".format(src_dir)) + exit(1) + +# Get a list of all .tar files in src_dir and its subdirectories +tar_files = [] +for root, dirs, files in os.walk(src_dir): + for file in files: + if file.endswith(".tar"): + tar_files.append(os.path.join(root, file)) + +# Extract each .tar file and display a progress bar +overall_desc = "Extracting .tar files" +with tqdm(total=len(tar_files), desc=overall_desc, unit="files") as overall_progress: + for tar_file in tar_files: + with tarfile.open(tar_file, "r") as tar: + members = tar.getmembers() + tar_desc = "Extracting {}".format(tar_file) + with tqdm(total=len(members), desc=tar_desc, unit="files") as tar_progress: + for member in members: + # Check if the file or directory already exists before extracting + dst_path = os.path.join(os.path.dirname(tar_file), member.name) + if os.path.exists(dst_path): + if os.path.isdir(dst_path): + shutil.rmtree(dst_path) + else: + os.remove(dst_path) + tar.extract(member, os.path.dirname(tar_file)) + tar_progress.update() + overall_progress.update() +