From 5cbdd86e75d311651017af5d09e6ba6643b9e033 Mon Sep 17 00:00:00 2001 From: nbei Date: Sat, 23 Oct 2021 14:23:46 +0800 Subject: [PATCH 1/9] initial regress script --- .dev_scripts/regression_benchmark/test_benchmark.sh | 8 ++++++++ tools/slurm_eval.sh | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 .dev_scripts/regression_benchmark/test_benchmark.sh diff --git a/.dev_scripts/regression_benchmark/test_benchmark.sh b/.dev_scripts/regression_benchmark/test_benchmark.sh new file mode 100644 index 000000000..47e14de10 --- /dev/null +++ b/.dev_scripts/regression_benchmark/test_benchmark.sh @@ -0,0 +1,8 @@ +PARTITION=$1 +CHECKPOINT_DIR=$2 + +echo 'configs/styleganv2/stylegan2_c2_ffhq_256_b4x8_800k.py' & +# GPUS=1 for the test with data parallel +GPUS=1 tools/slurm_eval.sh ${PARTITION} test-benchmark \ + configs/styleganv2/stylegan2_c2_ffhq_256_b4x8_800k.py \ + ${CHECKPOINT_DIR}/styleganv2/stylegan2/stylegan2_c2_ffhq_256_b4x8_20210407_160709-7890ae1f.pth & diff --git a/tools/slurm_eval.sh b/tools/slurm_eval.sh index b3abee0bb..dc81142ff 100644 --- a/tools/slurm_eval.sh +++ b/tools/slurm_eval.sh @@ -7,7 +7,8 @@ JOB_NAME=$2 CONFIG=$3 CKPT=$4 GPUS=${GPUS:-1} -GPUS_PER_NODE=${GPUS_PER_NODE:-1} +GPUS_PER_NODE=${GPUS_PER_NODE:-8} +NTASK_PER_NODE=${NTASK_PER_NODE:-1} CPUS_PER_TASK=${CPUS_PER_TASK:-5} PY_ARGS=${@:5} SRUN_ARGS=${SRUN_ARGS:-""} @@ -17,7 +18,7 @@ srun -p ${PARTITION} \ --job-name=${JOB_NAME} \ --gres=gpu:${GPUS_PER_NODE} \ --ntasks=${GPUS} \ - --ntasks-per-node=${GPUS_PER_NODE} \ + --ntasks-per-node=${NTASK_PER_NODE} \ --cpus-per-task=${CPUS_PER_TASK} \ --kill-on-bad-exit=1 \ ${SRUN_ARGS} \ From 716da33574c6f7ec49783bf44cc49e73bed13ef9 Mon Sep 17 00:00:00 2001 From: nbei Date: Sat, 23 Oct 2021 14:51:30 +0800 Subject: [PATCH 2/9] revise running command --- .dev_scripts/regression_benchmark/test_benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.dev_scripts/regression_benchmark/test_benchmark.sh b/.dev_scripts/regression_benchmark/test_benchmark.sh index 47e14de10..1dcc10534 100644 --- a/.dev_scripts/regression_benchmark/test_benchmark.sh +++ b/.dev_scripts/regression_benchmark/test_benchmark.sh @@ -3,6 +3,6 @@ CHECKPOINT_DIR=$2 echo 'configs/styleganv2/stylegan2_c2_ffhq_256_b4x8_800k.py' & # GPUS=1 for the test with data parallel -GPUS=1 tools/slurm_eval.sh ${PARTITION} test-benchmark \ +GPUS=1 bash tools/slurm_eval.sh ${PARTITION} test-benchmark \ configs/styleganv2/stylegan2_c2_ffhq_256_b4x8_800k.py \ ${CHECKPOINT_DIR}/styleganv2/stylegan2/stylegan2_c2_ffhq_256_b4x8_20210407_160709-7890ae1f.pth & From e85c7b4087324fa3d7d02aee8b317b8275e43012 Mon Sep 17 00:00:00 2001 From: nbei Date: Sat, 23 Oct 2021 14:57:21 +0800 Subject: [PATCH 3/9] fix bug in ckpt path --- .dev_scripts/regression_benchmark/test_benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.dev_scripts/regression_benchmark/test_benchmark.sh b/.dev_scripts/regression_benchmark/test_benchmark.sh index 1dcc10534..c6a74bee6 100644 --- a/.dev_scripts/regression_benchmark/test_benchmark.sh +++ b/.dev_scripts/regression_benchmark/test_benchmark.sh @@ -5,4 +5,4 @@ echo 'configs/styleganv2/stylegan2_c2_ffhq_256_b4x8_800k.py' & # GPUS=1 for the test with data parallel GPUS=1 bash tools/slurm_eval.sh ${PARTITION} test-benchmark \ configs/styleganv2/stylegan2_c2_ffhq_256_b4x8_800k.py \ - ${CHECKPOINT_DIR}/styleganv2/stylegan2/stylegan2_c2_ffhq_256_b4x8_20210407_160709-7890ae1f.pth & + ${CHECKPOINT_DIR}/styleganv2/stylegan2_c2_ffhq_256_b4x8_20210407_160709-7890ae1f.pth & From 0191057aa2979748412be1dc3f496a975e89e351 Mon Sep 17 00:00:00 2001 From: nbei Date: Sat, 23 Oct 2021 14:59:05 +0800 Subject: [PATCH 4/9] fix bug in ckpt path --- .dev_scripts/regression_benchmark/test_benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.dev_scripts/regression_benchmark/test_benchmark.sh b/.dev_scripts/regression_benchmark/test_benchmark.sh index c6a74bee6..b809e9aa8 100644 --- a/.dev_scripts/regression_benchmark/test_benchmark.sh +++ b/.dev_scripts/regression_benchmark/test_benchmark.sh @@ -5,4 +5,4 @@ echo 'configs/styleganv2/stylegan2_c2_ffhq_256_b4x8_800k.py' & # GPUS=1 for the test with data parallel GPUS=1 bash tools/slurm_eval.sh ${PARTITION} test-benchmark \ configs/styleganv2/stylegan2_c2_ffhq_256_b4x8_800k.py \ - ${CHECKPOINT_DIR}/styleganv2/stylegan2_c2_ffhq_256_b4x8_20210407_160709-7890ae1f.pth & + ${CHECKPOINT_DIR}/stylegan2/stylegan2_c2_ffhq_256_b4x8_20210407_160709-7890ae1f.pth & From a878cf69b8289a7f7772aaec93065a05da6bb31f Mon Sep 17 00:00:00 2001 From: nbei Date: Sat, 23 Oct 2021 15:45:52 +0800 Subject: [PATCH 5/9] update command --- .dev_scripts/regression_benchmark/test_benchmark.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.dev_scripts/regression_benchmark/test_benchmark.sh b/.dev_scripts/regression_benchmark/test_benchmark.sh index b809e9aa8..a14429a4e 100644 --- a/.dev_scripts/regression_benchmark/test_benchmark.sh +++ b/.dev_scripts/regression_benchmark/test_benchmark.sh @@ -5,4 +5,5 @@ echo 'configs/styleganv2/stylegan2_c2_ffhq_256_b4x8_800k.py' & # GPUS=1 for the test with data parallel GPUS=1 bash tools/slurm_eval.sh ${PARTITION} test-benchmark \ configs/styleganv2/stylegan2_c2_ffhq_256_b4x8_800k.py \ - ${CHECKPOINT_DIR}/stylegan2/stylegan2_c2_ffhq_256_b4x8_20210407_160709-7890ae1f.pth & + ${CHECKPOINT_DIR}/stylegan2/stylegan2_c2_ffhq_256_b4x8_20210407_160709-7890ae1f.pth \ + --online & From 3381547520750c6bdbf4b5d0b4052126d7ece14e Mon Sep 17 00:00:00 2001 From: nbei Date: Sun, 24 Oct 2021 14:20:25 +0800 Subject: [PATCH 6/9] add multiple test --- .dev_scripts/regression_benchmark/test_benchmark.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.dev_scripts/regression_benchmark/test_benchmark.sh b/.dev_scripts/regression_benchmark/test_benchmark.sh index a14429a4e..089739d5c 100644 --- a/.dev_scripts/regression_benchmark/test_benchmark.sh +++ b/.dev_scripts/regression_benchmark/test_benchmark.sh @@ -7,3 +7,10 @@ GPUS=1 bash tools/slurm_eval.sh ${PARTITION} test-benchmark \ configs/styleganv2/stylegan2_c2_ffhq_256_b4x8_800k.py \ ${CHECKPOINT_DIR}/stylegan2/stylegan2_c2_ffhq_256_b4x8_20210407_160709-7890ae1f.pth \ --online & +y +echo 'configs/styleganv1/styleganv1_ffhq_256_g8_25Mimg.py' & +# GPUS=1 for the test with data parallel +GPUS=1 bash tools/slurm_eval.sh ${PARTITION} test-benchmark \ + configs/styleganv1/styleganv1_ffhq_256_g8_25Mimg.py \ + ${CHECKPOINT_DIR}/styleganv1/styleganv1_ffhq_1024_g8_25Mimg_20210407_161627-850a7234.pth \ + --online & From 1fa83fed27def005e4c2f351328c1890bcec1858 Mon Sep 17 00:00:00 2001 From: nbei Date: Sun, 24 Oct 2021 14:24:44 +0800 Subject: [PATCH 7/9] fix bug in styleganv1 256 --- .dev_scripts/regression_benchmark/test_benchmark.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.dev_scripts/regression_benchmark/test_benchmark.sh b/.dev_scripts/regression_benchmark/test_benchmark.sh index 089739d5c..87659433e 100644 --- a/.dev_scripts/regression_benchmark/test_benchmark.sh +++ b/.dev_scripts/regression_benchmark/test_benchmark.sh @@ -1,16 +1,18 @@ PARTITION=$1 CHECKPOINT_DIR=$2 +# stylegan2 echo 'configs/styleganv2/stylegan2_c2_ffhq_256_b4x8_800k.py' & # GPUS=1 for the test with data parallel GPUS=1 bash tools/slurm_eval.sh ${PARTITION} test-benchmark \ configs/styleganv2/stylegan2_c2_ffhq_256_b4x8_800k.py \ ${CHECKPOINT_DIR}/stylegan2/stylegan2_c2_ffhq_256_b4x8_20210407_160709-7890ae1f.pth \ --online & -y + +# stylegan v1 echo 'configs/styleganv1/styleganv1_ffhq_256_g8_25Mimg.py' & # GPUS=1 for the test with data parallel GPUS=1 bash tools/slurm_eval.sh ${PARTITION} test-benchmark \ configs/styleganv1/styleganv1_ffhq_256_g8_25Mimg.py \ - ${CHECKPOINT_DIR}/styleganv1/styleganv1_ffhq_1024_g8_25Mimg_20210407_161627-850a7234.pth \ + ${CHECKPOINT_DIR}/styleganv1/styleganv1_ffhq_256_g8_25Mimg_20210407_161748-0094da86.pth \ --online & From e03fcbb512cc840c0294331c4c16ccf430833daf Mon Sep 17 00:00:00 2001 From: nbei Date: Sun, 24 Oct 2021 21:14:39 +0800 Subject: [PATCH 8/9] update parse_test_log.py --- .../regression_benchmark/parse_test_log.py | 211 ++++++++++++++++++ setup.cfg | 2 +- 2 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 .dev_scripts/regression_benchmark/parse_test_log.py diff --git a/.dev_scripts/regression_benchmark/parse_test_log.py b/.dev_scripts/regression_benchmark/parse_test_log.py new file mode 100644 index 000000000..1061291a0 --- /dev/null +++ b/.dev_scripts/regression_benchmark/parse_test_log.py @@ -0,0 +1,211 @@ +import argparse +import os + +import mmcv +import yaml + +# --------------------------------------------------------------------------- # +# +# Tips for adopting the test_benchmark.sh and parse_test_log.py: +# +# 1. The directory for pre-trained checkpoints should follow the structure +# of ``./configs`` folder, especially the name of each method. +# 2. For the regression report, you can quickly check the ``regress_status`` +# a) ``Missing Eval Info``: Cannot read the information table from log file +# correctly. Please further check the log file and testing output. +# b) ``Failed``: The evaluation metric cannot match the value in our +# metafile. +# c) ``Pass``: Successfully pass the regression test. +# 3. We should check the representation of the result value in metafile and the +# generated log table. (Related to convert_str_metric_value() function) +# 4. The matric name should be mapped to the standard one. Please check +# ``metric_name_mapping`` to ensure the metric name in the keys of the dict. +# +# --------------------------------------------------------------------------- # + +# used to store log infos: The key indicates the path of each log file, while +# the value contains meta information (method category, ckpt name) and the +# parsed log info +data_dict = {} + +metric_name_mapping = { + 'FID50k': 'FID', + 'FID': 'FID', + 'P&R': 'PR', + 'P&R50k': 'PR', + 'PR': 'PR', + 'PR50k': 'PR' +} + + +def parse_args(): + parser = argparse.ArgumentParser(description='Train a GAN model') + parser.add_argument('logdir', help='evaluation log path') + parser.add_argument( + '--out', + type=str, + default='./work_dirs/', + help='output directory for benchmark information') + + args = parser.parse_args() + return args + + +def read_metafile(filepath): + with open(filepath, 'r') as f: + data = yaml.safe_load(f) + + return data + + +def read_table_from_log(filepath): + """Read the evaluation table from the log file. + + .. note:: We assume the log file only records one evaluation procedure. + """ + # table_info will contain 2 elements. The first element is the head of the + # table indicating the meaning of each column. The second element is the + # content of each column, e.g., training configuration, and fid value. + table_info = [] + + with open(filepath, 'r') as f: + for line in f.readlines(): + if 'mmgen' in line or 'INFO' in line: + # running message + continue + if line[0] == '|' and '+' in line: + # table split line + continue + if line[0] == '|': + # useful content + line = line.strip() + line_split = line.split('|') + line_split = [x.strip() for x in line_split] + table_info.append(line_split) + + if len(table_info) < 2: + print(f'Please check the log file: {filepath}. Cannot get the eval' + ' information correctly.') + elif len(table_info) > 3: + print(f'Got more than 2 lines from the eval table in {filepath}') + + return table_info + + +def convert_str_metric_value(value): + if isinstance(value, float): + return value + + if isinstance(value, int): + return float(value) + + # otherwise, we assume the value will be string format + + # Case: 3.42 (1.xxx/2.xxx) -- used in FID + if '(' in value and ')' in value: + split = value.split('(') + res = split[0].strip() + return float(res) + + # Case: 60.xx/40.xx -- used in PR metric + elif '/' in value: + split = value.split('/') + + return [float(x.strip()) for x in split] + else: + try: + res = float(value) + return res + except Exception as err: + print(f'Cannot convert str value {value} to float') + print(f'Unexpected {err}, {type(err)}') + raise err + + +def check_info_from_metafile(meta_info): + """Check whether eval information matches the description from metafile. + + TODO: Set a dict containing the tolerance for different configurations. + """ + method_cat = meta_info['method'] + ckpt_name = meta_info['ckpt_name'] + metafile_path = os.path.join('./configs', method_cat, 'metafile.yml') + + meta_data_orig = read_metafile(metafile_path)['Models'] + results_orig = None + + for info in meta_data_orig: + if ckpt_name in info['Weights']: + results_orig = info['Results'] + break + + if results_orig is None: + print(f'Cannot find related models for {ckpt_name}') + return False + + metric_value_orig = {} + # get the original metric value + for k, v in results_orig['Metrics']: + if k in metric_name_mapping: + metric_value_orig[ + metric_name_mapping[k]] = convert_str_metric_value(v) + + assert len(metric_value_orig + ) > 0, f'Cannot get metric value in metafile for {ckpt_name}' + + # get the metric value from evaluation table + eval_info = meta_info['eval_table'] + metric_value_eval = {} + for i, name in enumerate(eval_info[0]): + if name in metric_name_mapping: + metric_name_mapping[ + metric_value_eval[name]] = convert_str_metric_value( + eval_info[1]) + assert len(metric_value_eval + ) > 0, f'Cannot get metric value in eval table: {eval_info}' + + # compare eval info and the original info from metafile + + +def get_log_files(args): + """Got all of the log files from the given args.logdir. + + This function is used to initialize ``data_dict``. + """ + log_paths = mmcv.scandir(args.logdir, '.txt', recursive=True) + log_paths = [os.path.join(args.logdir, x) for x in log_paths] + + # construct data dict + for log in log_paths: + splits = log.split('/') + method = splits[-2] + log_name = splits[-1] + ckpt_name = log_name.replace('_eval_log.txt', '.pth') + + data_dict[log] = dict( + method=method, log_name=log_name, ckpt_name=ckpt_name) + + print(f'Got {len(data_dict)} logs from {args.logdir}') + + return data_dict + + +def main(): + args = parse_args() + get_log_files(args) + + # process log files + for log_path, meta_info in data_dict.items(): + table_info = read_table_from_log(log_path) + + # only deal with valid table_info + if len(table_info) == 2: + meta_info['eval_table'] = table_info + meta_info['regress_status'] = 'Pass' if check_info_from_metafile( + meta_info) else 'Failed' + else: + meta_info['regress_status'] = 'Missing Eval Info' + + +if __name__ == '__main__': + data = read_metafile('configs/styleganv2/metafile.yml') diff --git a/setup.cfg b/setup.cfg index 3c88b6c89..ce8fc1640 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,6 +14,6 @@ line_length=79 multi_line_output=0 known_standard_library=argparse,inspect,contextlib,hashlib,subprocess,unittest,tempfile,copy,pkg_resources,logging,pickle,platform,setuptools,abc,collections,functools,os,math,time,warnings,random,shutil,sys known_first_party=mmgen -known_third_party=PIL,click,cv2,m2r,mmcls,mmcv,numpy,prettytable,pytest,pytorch_sphinx_theme,recommonmark,requests,scipy,torch,torchvision,tqdm,ts +known_third_party=PIL,click,cv2,m2r,mmcls,mmcv,numpy,prettytable,pytest,pytorch_sphinx_theme,recommonmark,requests,scipy,torch,torchvision,tqdm,ts,yaml no_lines_before=STDLIB,LOCALFOLDER default_section=THIRDPARTY From b0d871ed1148f5be107bb3d16236f34fed9a71e5 Mon Sep 17 00:00:00 2001 From: nbei Date: Tue, 26 Oct 2021 15:49:32 +0800 Subject: [PATCH 9/9] update PR string format --- .../regression_benchmark/parse_test_log.py | 77 +++++++++++++++++-- mmgen/core/evaluation/metrics.py | 3 +- 2 files changed, 74 insertions(+), 6 deletions(-) diff --git a/.dev_scripts/regression_benchmark/parse_test_log.py b/.dev_scripts/regression_benchmark/parse_test_log.py index 1061291a0..3016a01cc 100644 --- a/.dev_scripts/regression_benchmark/parse_test_log.py +++ b/.dev_scripts/regression_benchmark/parse_test_log.py @@ -1,4 +1,5 @@ import argparse +import json import os import mmcv @@ -20,6 +21,8 @@ # generated log table. (Related to convert_str_metric_value() function) # 4. The matric name should be mapped to the standard one. Please check # ``metric_name_mapping`` to ensure the metric name in the keys of the dict. +# 5. Pay attention to the format of numerical value. For instance, the +# precision value can be 0.69xx or 69.xx. # # --------------------------------------------------------------------------- # @@ -37,6 +40,8 @@ 'PR50k': 'PR' } +tolerance = 2.0 + def parse_args(): parser = argparse.ArgumentParser(description='Train a GAN model') @@ -58,6 +63,16 @@ def read_metafile(filepath): return data +def is_number(s): + try: + float(s) + return True + except ValueError: + pass + + return False + + def read_table_from_log(filepath): """Read the evaluation table from the log file. @@ -88,6 +103,9 @@ def read_table_from_log(filepath): ' information correctly.') elif len(table_info) > 3: print(f'Got more than 2 lines from the eval table in {filepath}') + else: + assert len(table_info[0]) == len( + table_info[1]), 'The eval table cannot be aligned' return table_info @@ -112,6 +130,17 @@ def convert_str_metric_value(value): split = value.split('/') return [float(x.strip()) for x in split] + # Case: precision: 69.59999918937683, recall:40.200000643730164 + elif ',' in value: + split = [x.strip() for x in value.split(',')] + res = [] + for x in split: + if ':' in x: + num_str = x.split(':')[1].strip() + res.append(float(num_str)) + elif is_number(x): + res.append(float(x)) + return res else: try: res = float(value) @@ -122,6 +151,30 @@ def convert_str_metric_value(value): raise err +def compare_eval_orig_info(orig_info, eval_info): + flag = True + for k, v in eval_info.items(): + orig_value = orig_info[k] + if isinstance(v, float): + if abs(v - orig_value) > tolerance: + print(v, orig_value) + flag = False + break + elif isinstance(v, list): + for tmp_v, temp_orig in zip(v, orig_value): + if abs(tmp_v - temp_orig) > tolerance: + print(v, orig_value) + flag = False + break + if not flag: + break + else: + raise RuntimeError(f'Cannot parse compare eval_value: {v} and ' + f'orig_value: {orig_value}.') + + return flag + + def check_info_from_metafile(meta_info): """Check whether eval information matches the description from metafile. @@ -144,8 +197,14 @@ def check_info_from_metafile(meta_info): return False metric_value_orig = {} + results_metric_orig = None + for info in results_orig: + if 'Metrics' in info: + results_metric_orig = info['Metrics'] + break + assert results_metric_orig is not None, 'Cannot find Metrics in metafile.' # get the original metric value - for k, v in results_orig['Metrics']: + for k, v in results_metric_orig.items(): if k in metric_name_mapping: metric_value_orig[ metric_name_mapping[k]] = convert_str_metric_value(v) @@ -158,13 +217,14 @@ def check_info_from_metafile(meta_info): metric_value_eval = {} for i, name in enumerate(eval_info[0]): if name in metric_name_mapping: - metric_name_mapping[ - metric_value_eval[name]] = convert_str_metric_value( - eval_info[1]) + metric_value_eval[ + metric_name_mapping[name]] = convert_str_metric_value( + eval_info[1][i]) assert len(metric_value_eval ) > 0, f'Cannot get metric value in eval table: {eval_info}' # compare eval info and the original info from metafile + return compare_eval_orig_info(metric_value_orig, metric_value_eval) def get_log_files(args): @@ -206,6 +266,13 @@ def main(): else: meta_info['regress_status'] = 'Missing Eval Info' + mmcv.mkdir_or_exist(args.out) + with open(os.path.join(args.out, 'test_regression_report.json'), 'w') as f: + json.dump(data_dict, f) + + print('-------------- Regression Report --------------') + print(data_dict) + if __name__ == '__main__': - data = read_metafile('configs/styleganv2/metafile.yml') + main() diff --git a/mmgen/core/evaluation/metrics.py b/mmgen/core/evaluation/metrics.py index a94f71b36..97c6e309e 100644 --- a/mmgen/core/evaluation/metrics.py +++ b/mmgen/core/evaluation/metrics.py @@ -942,7 +942,8 @@ def summary(self): precision = self._result_dict['precision'] recall = self._result_dict['recall'] - self._result_str = f'precision: {precision}, recall:{recall}' + self._result_str = (f'precision: {precision * 100.}, ' + f'recall:{recall * 100.}') return self._result_dict def extract_features(self, images):