diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/README.md b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/README.md index 23762270..9a7435fb 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/README.md +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/README.md @@ -19,7 +19,7 @@ pip install -U cmind cm rm cache -f -cm pull repo mlcommons@mlperf-automations --checkout=225220c7d9bb7e66e5b9a1e1ebfc3e0180fbd094 +cm pull repo mlcommons@mlperf-automations --checkout=a90475d2de72bf0622cebe8d5ca8eb8c9d872fbd cm run script \ --tags=app,mlperf,inference,generic,_nvidia,_bert-99,_tensorrt,_cuda,_valid,_r4.1-dev_default,_offline \ @@ -71,7 +71,7 @@ cm run script \ --env.CM_DOCKER_REUSE_EXISTING_CONTAINER=yes \ --env.CM_DOCKER_DETACHED_MODE=yes \ --env.CM_MLPERF_INFERENCE_RESULTS_DIR_=/home/arjun/gh_action_results/valid_results \ - --env.CM_DOCKER_CONTAINER_ID=b8ce345e7cfa \ + --env.CM_DOCKER_CONTAINER_ID=c66a229e22cf \ --env.CM_MLPERF_LOADGEN_COMPLIANCE_TEST=TEST01 \ --add_deps_recursive.compiler.tags=gcc \ --add_deps_recursive.coco2014-original.tags=_full \ @@ -129,4 +129,4 @@ Model Precision: int8 `F1`: `90.15674`, Required accuracy for closed division `>= 89.96526` ### Performance Results -`Samples per second`: `4119.08` +`Samples per second`: `4125.31` diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy_console.out b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy_console.out index 43446219..a490bc2c 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy_console.out +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy_console.out @@ -1,7 +1,7 @@ -[2024-12-23 02:01:52,311 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x1 -[2024-12-23 02:01:52,813 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x1_TRT/bert-99/Offline -[2024-12-23 02:01:52,813 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="AccuracyOnly" --gpu_batch_size=256 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/adc9dc0382544ffd/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=false --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/eb9854064bde419c94548e4a59d8e6b4.conf" --gpu_inference_streams=2 --gpu_copy_streams=2 --gpu_engines="./build/engines/RTX4090x1/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan" --scenario Offline --model bert -[2024-12-23 02:01:52,813 __init__.py:53 INFO] Overriding Environment +[2024-12-24 23:00:52,890 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x1 +[2024-12-24 23:00:53,380 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x1_TRT/bert-99/Offline +[2024-12-24 23:00:53,380 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="AccuracyOnly" --gpu_batch_size=256 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/e824f0bb4919400f/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=false --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/aab71c24a07442c090544e000738c311.conf" --gpu_inference_streams=2 --gpu_copy_streams=2 --gpu_engines="./build/engines/RTX4090x1/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan" --scenario Offline --model bert +[2024-12-24 23:00:53,380 __init__.py:53 INFO] Overriding Environment benchmark : Benchmark.BERT buffer_manager_thread_count : 0 coalesced_tensor : True @@ -11,8 +11,8 @@ gpu_copy_streams : 2 gpu_inference_streams : 2 input_dtype : int32 input_format : linear -log_dir : /home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/logs/2024.12.23-02.01.51 -mlperf_conf_path : /home/cmuser/CM/repos/local/cache/adc9dc0382544ffd/inference/mlperf.conf +log_dir : /home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/logs/2024.12.24-23.00.51 +mlperf_conf_path : /home/cmuser/CM/repos/local/cache/e824f0bb4919400f/inference/mlperf.conf offline_expected_qps : 0.0 precision : int8 preprocessed_data_dir : /home/cmuser/CM/repos/local/cache/a8c152aef5494496/preprocessed_data @@ -21,7 +21,7 @@ system : SystemConfiguration(host_cpu_conf=CPUConfiguration(layout={CPU(name='AM tensor_path : build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy test_mode : AccuracyOnly use_graphs : False -user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/eb9854064bde419c94548e4a59d8e6b4.conf +user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/aab71c24a07442c090544e000738c311.conf system_id : RTX4090x1 config_name : RTX4090x1_bert_Offline workload_setting : WorkloadSetting(HarnessType.Custom, AccuracyTarget.k_99, PowerSetting.MaxP) @@ -34,40 +34,40 @@ skip_file_checks : True power_limit : None cpu_freq : None &&&& RUNNING BERT_HARNESS # ./build/bin/harness_bert -I1223 02:01:52.866398 19723 main_bert.cc:163] Found 1 GPUs -I1223 02:01:53.259613 19723 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x1/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan +I1224 23:00:53.437846 19726 main_bert.cc:163] Found 1 GPUs +I1224 23:00:53.830546 19726 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x1/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan [I] [TRT] Loaded engine size: 414 MiB -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +7, GPU +8, now: CPU 582, GPU 1232 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +1, GPU +10, now: CPU 583, GPU 1242 (MiB) +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +6, GPU +8, now: CPU 582, GPU 1232 (MiB) +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +2, GPU +10, now: CPU 584, GPU 1242 (MiB) [I] [TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +290, now: CPU 0, GPU 290 (MiB) -I1223 02:01:53.596822 19723 bert_server.cc:208] Engines Creation Completed -I1223 02:01:53.616966 19723 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 -I1223 02:01:53.616971 19723 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 -I1223 02:01:53.616976 19723 bert_core_vs.cc:415] Engine - Profile 0 maxDims 98304 Bmax=256 Smax=384 -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +1, GPU +8, now: CPU 170, GPU 1908 (MiB) +I1224 23:00:54.158447 19726 bert_server.cc:208] Engines Creation Completed +I1224 23:00:54.175813 19726 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 +I1224 23:00:54.175822 19726 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 +I1224 23:00:54.175827 19726 bert_core_vs.cc:415] Engine - Profile 0 maxDims 98304 Bmax=256 Smax=384 +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 170, GPU 1908 (MiB) [I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 170, GPU 1916 (MiB) -I1223 02:01:53.672140 19723 bert_core_vs.cc:426] Setting Opt.Prof. to 0 -I1223 02:01:53.672174 19723 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 +I1224 23:00:54.230412 19726 bert_core_vs.cc:426] Setting Opt.Prof. to 0 +I1224 23:00:54.230443 19726 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +0, now: CPU 0, GPU 290 (MiB) -I1223 02:01:53.673125 19723 bert_core_vs.cc:476] Setup complete -I1223 02:01:53.673296 19723 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 -I1223 02:01:53.673300 19723 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 -I1223 02:01:53.673302 19723 bert_core_vs.cc:415] Engine - Profile 1 maxDims 98304 Bmax=256 Smax=384 -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 292, GPU 2722 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +1, GPU +10, now: CPU 293, GPU 2732 (MiB) -I1223 02:01:53.727489 19723 bert_core_vs.cc:426] Setting Opt.Prof. to 1 +I1224 23:00:54.231410 19726 bert_core_vs.cc:476] Setup complete +I1224 23:00:54.231583 19726 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 +I1224 23:00:54.231587 19726 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 +I1224 23:00:54.231590 19726 bert_core_vs.cc:415] Engine - Profile 1 maxDims 98304 Bmax=256 Smax=384 +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 293, GPU 2722 (MiB) +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +10, now: CPU 293, GPU 2732 (MiB) +I1224 23:00:54.285146 19726 bert_core_vs.cc:426] Setting Opt.Prof. to 1 [I] [TRT] Could not set default profile 0 for execution context. Profile index must be set explicitly. [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +1, GPU +0, now: CPU 1, GPU 290 (MiB) -I1223 02:01:53.727843 19723 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 -I1223 02:01:53.728783 19723 bert_core_vs.cc:476] Setup complete -I1223 02:01:53.957628 19723 main_bert.cc:184] Starting running actual test. -I1223 02:01:56.618793 19723 main_bert.cc:190] Finished running actual test. +I1224 23:00:54.285508 19726 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 +I1224 23:00:54.286469 19726 bert_core_vs.cc:476] Setup complete +I1224 23:00:54.514549 19726 main_bert.cc:184] Starting running actual test. +I1224 23:00:57.166287 19726 main_bert.cc:190] Finished running actual test. No warnings encountered during test. No errors encountered during test. -[2024-12-23 02:01:56,881 run_harness.py:166 INFO] Result: Accuracy run detected. -[2024-12-23 02:01:56,881 __init__.py:46 INFO] Running command: PYTHONPATH=code/bert/tensorrt/helpers python3 /home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/language/bert/accuracy-squad.py --log_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_accuracy.json --vocab_file build/models/bert/vocab.txt --val_data /home/cmuser/CM/repos/local/cache/a8c152aef5494496/data/squad/dev-v1.1.json --out_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/predictions.json --output_dtype float16 +[2024-12-24 23:00:57,429 run_harness.py:166 INFO] Result: Accuracy run detected. +[2024-12-24 23:00:57,429 __init__.py:46 INFO] Running command: PYTHONPATH=code/bert/tensorrt/helpers python3 /home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/language/bert/accuracy-squad.py --log_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_accuracy.json --vocab_file build/models/bert/vocab.txt --val_data /home/cmuser/CM/repos/local/cache/a8c152aef5494496/data/squad/dev-v1.1.json --out_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/predictions.json --output_dtype float16 {"exact_match": 82.81929990539263, "f1": 90.15673510616978} Reading examples... Loading cached features from 'eval_features.pickle'... diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.mmd b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.mmd index 35f499e1..145361fd 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.mmd +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.mmd @@ -16,20 +16,20 @@ graph TD app-mlperf-inference,d775cac873ee4231_(_nvidia,_bert-99,_tensorrt,_cuda,_valid,_r4.1-dev_default,_offline_) --> get,cuda-devices,_with-pycuda app-mlperf-inference,d775cac873ee4231_(_nvidia,_bert-99,_tensorrt,_cuda,_valid,_r4.1-dev_default,_offline_) --> get,dataset,squad,language-processing app-mlperf-inference,d775cac873ee4231_(_nvidia,_bert-99,_tensorrt,_cuda,_valid,_r4.1-dev_default,_offline_) --> get,dataset-aux,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> detect,os + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> detect,cpu - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,sys-utils-cm - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,mlperf,inference,nvidia,scratch,space + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> detect,cpu + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,sys-utils-cm + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,mlperf,inference,nvidia,scratch,space get-generic-python-lib,94b62a682bc44791_(_mlperf_logging_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_mlperf_logging - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,ml-model,bert,_onnx,_fp32 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,ml-model,bert,_onnx,_int8 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,mlcommons,inference,src,_deeplearningexamples - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,nvidia,mlperf,inference,common-code,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,generic-python-lib,_mlperf_logging + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,ml-model,bert,_onnx,_fp32 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,ml-model,bert,_onnx,_int8 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,squad-vocab + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,mlcommons,inference,src,_deeplearningexamples + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,nvidia,mlperf,inference,common-code,_mlcommons pull-git-repo,c23132ed65c4421d --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> pull,git,repo + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> pull,git,repo generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,os detect-cpu,586c8a43320142f7 --> detect,os generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,cpu @@ -37,58 +37,58 @@ graph TD generate-mlperf-inference-user-conf,3af4475745964b93 --> get,mlcommons,inference,src,_deeplearningexamples get-mlperf-inference-sut-configs,c2fbf72009e2445b --> get,cache,dir,_name.mlperf-inference-sut-configs generate-mlperf-inference-user-conf,3af4475745964b93 --> get,sut,configs - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> generate,user-conf,mlperf,inference + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> generate,user-conf,mlperf,inference get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_package.pycuda - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,nvidia,mitten - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,cuda,_cudnn - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,tensorrt - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> build,nvidia,inference,server,_mlcommons - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> detect,os + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,generic-python-lib,_package.pycuda + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,nvidia,mitten + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,cuda,_cudnn + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,tensorrt + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> build,nvidia,inference,server,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> detect,cpu - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,sys-utils-cm - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,mlperf,inference,nvidia,scratch,space,_version.4_0 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> detect,cpu + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,sys-utils-cm + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,mlperf,inference,nvidia,scratch,space,_version.4_0 get-generic-python-lib,94b62a682bc44791_(_mlperf_logging_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_mlperf_logging - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,ml-model,bert,_onnx,_fp32 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,ml-model,bert,_onnx,_int8 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,mlcommons,inference,src,_deeplearningexamples - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,nvidia,mlperf,inference,common-code,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_mlperf_logging + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,ml-model,bert,_onnx,_fp32 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,ml-model,bert,_onnx,_int8 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,squad-vocab + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,mlcommons,inference,src,_deeplearningexamples + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,nvidia,mlperf,inference,common-code,_mlcommons pull-git-repo,c23132ed65c4421d --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> pull,git,repo + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> pull,git,repo get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_package.pycuda - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,nvidia,mitten - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,cuda,_cudnn - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,tensorrt - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> build,nvidia,inference,server,_mlcommons - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_package.pycuda + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,nvidia,mitten + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,cuda,_cudnn + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,tensorrt + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> build,nvidia,inference,server,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_bert-99,_cuda,_bert_,_v4.1-dev get-generic-python-lib,94b62a682bc44791_(_transformers_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_transformers + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_transformers get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_safetensors + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_safetensors get-generic-python-lib,94b62a682bc44791_(_onnx_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_onnx + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_onnx get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_package.sympy + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_package.sympy get-generic-python-lib,94b62a682bc44791_(_onnx-graphsurgeon_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_onnx-graphsurgeon - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev_) --> save,mlperf,inference,state - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> get,generic-python-lib,_onnx-graphsurgeon + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev_) --> save,mlperf,inference,state + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_bert-99,_cuda,_bert_,_v4.1-dev get-generic-python-lib,94b62a682bc44791_(_transformers_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_transformers + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,generic-python-lib,_transformers get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_safetensors + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,generic-python-lib,_safetensors get-generic-python-lib,94b62a682bc44791_(_onnx_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_onnx + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,generic-python-lib,_onnx get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_package.sympy + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,generic-python-lib,_package.sympy get-generic-python-lib,94b62a682bc44791_(_onnx-graphsurgeon_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_onnx-graphsurgeon + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> get,generic-python-lib,_onnx-graphsurgeon detect-cpu,586c8a43320142f7 --> detect,os benchmark-program,19f369ef47084895 --> detect,cpu benchmark-program-mlperf,cfff0132a8aa4018 --> benchmark-program,program - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090_) --> benchmark-mlperf \ No newline at end of file + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090_) --> benchmark-mlperf \ No newline at end of file diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.png b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.png index 02253b2d..66a02a1f 100644 Binary files a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.png and b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-deps.png differ diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-version-info.json b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-version-info.json index 10fa0226..e667a226 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-version-info.json +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/cm-version-info.json @@ -177,7 +177,7 @@ "script_tags": "detect-os,detect,os,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -197,7 +197,7 @@ "script_tags": "detect,cpu,detect-cpu,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -207,7 +207,7 @@ "script_tags": "get,sys-utils-cm", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -217,7 +217,7 @@ "script_tags": "get,mlperf,inference,nvidia,scratch,space", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -237,7 +237,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "mlperf_logging", "version": "4.1.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -247,7 +247,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,fp32", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -257,7 +257,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,int8", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -267,7 +267,7 @@ "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -277,7 +277,7 @@ "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", "script_variations": "deeplearningexamples", "version": "master-git-2fdb814e6a765d994872fb99ee38f8c43418d5b0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -287,7 +287,7 @@ "script_tags": "get,nvidia,mlperf,inference,common-code", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -307,7 +307,7 @@ "script_tags": "pull,git,repo,repository", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -387,7 +387,7 @@ "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -407,7 +407,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.pycuda", "version": "2022.2.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -417,7 +417,7 @@ "script_tags": "get,nvidia,mitten,nvidia-mitten", "script_variations": "", "version": "master", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -427,7 +427,7 @@ "script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d", "script_variations": "cudnn", "version": "12.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -437,7 +437,7 @@ "script_tags": "get,tensorrt,nvidia", "script_variations": "", "version": "vdetected", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -447,7 +447,7 @@ "script_tags": "build,mlcommons,mlperf,inference,inference-server,server,nvidia-harness,nvidia", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -457,7 +457,7 @@ "script_tags": "detect-os,detect,os,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -477,7 +477,7 @@ "script_tags": "detect,cpu,detect-cpu,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -487,7 +487,7 @@ "script_tags": "get,sys-utils-cm", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -497,7 +497,7 @@ "script_tags": "get,mlperf,inference,nvidia,scratch,space", "script_variations": "version.4_0", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -517,7 +517,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "mlperf_logging", "version": "4.1.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -527,7 +527,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,fp32", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -537,7 +537,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,int8", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -547,7 +547,7 @@ "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -557,7 +557,7 @@ "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", "script_variations": "deeplearningexamples", "version": "master-git-2fdb814e6a765d994872fb99ee38f8c43418d5b0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -567,7 +567,7 @@ "script_tags": "get,nvidia,mlperf,inference,common-code", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -587,7 +587,7 @@ "script_tags": "pull,git,repo,repository", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -607,7 +607,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.pycuda", "version": "2022.2.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -617,7 +617,7 @@ "script_tags": "get,nvidia,mitten,nvidia-mitten", "script_variations": "", "version": "master", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -627,7 +627,7 @@ "script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d", "script_variations": "cudnn", "version": "12.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -637,7 +637,7 @@ "script_tags": "get,tensorrt,nvidia", "script_variations": "", "version": "vdetected", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -647,17 +647,17 @@ "script_tags": "build,mlcommons,mlperf,inference,inference-server,server,nvidia-harness,nvidia", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { - "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev": { + "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_bert-99,_cuda,_bert_,_v4.1-dev": { "script_uid": "bc3b17fb430f4732", "script_alias": "app-mlperf-inference-nvidia", "script_tags": "reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia", - "script_variations": "preprocess_data,tensorrt,cuda,bert-99,bert_,v4.1-dev", + "script_variations": "preprocess_data,tensorrt,bert-99,cuda,bert_,v4.1-dev", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -677,7 +677,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "transformers", "version": "4.46.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -697,7 +697,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "safetensors", "version": "0.4.5", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -717,7 +717,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx", "version": "1.13.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -737,7 +737,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.sympy", "version": "1.13.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -757,7 +757,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx-graphsurgeon", "version": "0.3.27", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { @@ -767,27 +767,27 @@ "script_tags": "save,mlperf,inference,implementation,state", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev )" } }, { - "reproduce,mlperf,inference,nvidia,harness,_build_engine,_tensorrt,_offline,_cuda,_bert-99,_bert_,_batch_size.256,_v4.1-dev": { + "reproduce,mlperf,inference,nvidia,harness,_build_engine,_offline,_tensorrt,_bert-99,_cuda,_bert_,_batch_size.256,_v4.1-dev": { "script_uid": "bc3b17fb430f4732", "script_alias": "app-mlperf-inference-nvidia", "script_tags": "reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia", - "script_variations": "build_engine,tensorrt,offline,cuda,bert-99,bert_,batch_size.256,v4.1-dev", + "script_variations": "build_engine,offline,tensorrt,bert-99,cuda,bert_,batch_size.256,v4.1-dev", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { - "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev": { + "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_bert-99,_cuda,_bert_,_v4.1-dev": { "script_uid": "bc3b17fb430f4732", "script_alias": "app-mlperf-inference-nvidia", "script_tags": "reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia", - "script_variations": "preprocess_data,tensorrt,cuda,bert-99,bert_,v4.1-dev", + "script_variations": "preprocess_data,tensorrt,bert-99,cuda,bert_,v4.1-dev", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -807,7 +807,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "transformers", "version": "4.46.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -827,7 +827,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "safetensors", "version": "0.4.5", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -847,7 +847,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx", "version": "1.13.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -867,7 +867,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.sympy", "version": "1.13.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -887,7 +887,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx-graphsurgeon", "version": "0.3.27", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } }, { @@ -927,7 +927,7 @@ "script_tags": "mlperf,benchmark-mlperf", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_offline,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_offline,_tensorrt,_bert-99,_cuda,_rtx_4090 )" } } ] diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/os_info.json b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/os_info.json index b515d081..8cf0acab 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/os_info.json +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/os_info.json @@ -26,5 +26,5 @@ ], "CM_HOST_PLATFORM_FLAVOR": "x86_64", "CM_HOST_PYTHON_BITS": "64", - "CM_HOST_SYSTEM_NAME": "b8ce345e7cfa" + "CM_HOST_SYSTEM_NAME": "c66a229e22cf" } \ No newline at end of file diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance_console.out b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance_console.out index 991736eb..51a1700a 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance_console.out +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance_console.out @@ -1,7 +1,7 @@ -[2024-12-23 01:50:51,661 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x1 -[2024-12-23 01:50:52,167 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x1_TRT/bert-99/Offline -[2024-12-23 01:50:52,167 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="PerformanceOnly" --gpu_batch_size=256 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/adc9dc0382544ffd/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=false --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/fe89db42d7884eaf9c96a81fa861b63c.conf" --gpu_inference_streams=2 --gpu_copy_streams=2 --gpu_engines="./build/engines/RTX4090x1/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan" --scenario Offline --model bert -[2024-12-23 01:50:52,167 __init__.py:53 INFO] Overriding Environment +[2024-12-24 22:49:53,074 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x1 +[2024-12-24 22:49:53,596 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x1_TRT/bert-99/Offline +[2024-12-24 22:49:53,596 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="PerformanceOnly" --gpu_batch_size=256 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/e824f0bb4919400f/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=false --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/c4ba3fe7fb684a2cb69d0d659cb5408f.conf" --gpu_inference_streams=2 --gpu_copy_streams=2 --gpu_engines="./build/engines/RTX4090x1/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan" --scenario Offline --model bert +[2024-12-24 22:49:53,596 __init__.py:53 INFO] Overriding Environment benchmark : Benchmark.BERT buffer_manager_thread_count : 0 coalesced_tensor : True @@ -11,8 +11,8 @@ gpu_copy_streams : 2 gpu_inference_streams : 2 input_dtype : int32 input_format : linear -log_dir : /home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/logs/2024.12.23-01.50.50 -mlperf_conf_path : /home/cmuser/CM/repos/local/cache/adc9dc0382544ffd/inference/mlperf.conf +log_dir : /home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/logs/2024.12.24-22.49.52 +mlperf_conf_path : /home/cmuser/CM/repos/local/cache/e824f0bb4919400f/inference/mlperf.conf offline_expected_qps : 0.0 precision : int8 preprocessed_data_dir : /home/cmuser/CM/repos/local/cache/a8c152aef5494496/preprocessed_data @@ -21,7 +21,7 @@ system : SystemConfiguration(host_cpu_conf=CPUConfiguration(layout={CPU(name='AM tensor_path : build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy test_mode : PerformanceOnly use_graphs : False -user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/fe89db42d7884eaf9c96a81fa861b63c.conf +user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/c4ba3fe7fb684a2cb69d0d659cb5408f.conf system_id : RTX4090x1 config_name : RTX4090x1_bert_Offline workload_setting : WorkloadSetting(HarnessType.Custom, AccuracyTarget.k_99, PowerSetting.MaxP) @@ -34,41 +34,41 @@ skip_file_checks : True power_limit : None cpu_freq : None &&&& RUNNING BERT_HARNESS # ./build/bin/harness_bert -I1223 01:50:52.219789 18130 main_bert.cc:163] Found 1 GPUs -I1223 01:50:52.626741 18130 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x1/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan +I1224 22:49:53.656656 18133 main_bert.cc:163] Found 1 GPUs +I1224 22:49:54.055827 18133 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x1/bert/Offline/bert-Offline-gpu-int8_S_384_B_256_P_2_vs.custom_k_99_MaxP.plan [I] [TRT] Loaded engine size: 414 MiB -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +7, GPU +8, now: CPU 582, GPU 1232 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +1, GPU +10, now: CPU 583, GPU 1242 (MiB) +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +6, GPU +8, now: CPU 582, GPU 1232 (MiB) +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +2, GPU +10, now: CPU 584, GPU 1242 (MiB) [I] [TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +290, now: CPU 0, GPU 290 (MiB) -I1223 01:50:52.961704 18130 bert_server.cc:208] Engines Creation Completed -I1223 01:50:52.981645 18130 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 -I1223 01:50:52.981649 18130 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 -I1223 01:50:52.981654 18130 bert_core_vs.cc:415] Engine - Profile 0 maxDims 98304 Bmax=256 Smax=384 -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +1, GPU +8, now: CPU 170, GPU 1908 (MiB) +I1224 22:49:54.389321 18133 bert_server.cc:208] Engines Creation Completed +I1224 22:49:54.408313 18133 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 +I1224 22:49:54.408319 18133 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 +I1224 22:49:54.408324 18133 bert_core_vs.cc:415] Engine - Profile 0 maxDims 98304 Bmax=256 Smax=384 +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 170, GPU 1908 (MiB) [I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 170, GPU 1916 (MiB) -I1223 01:50:53.036955 18130 bert_core_vs.cc:426] Setting Opt.Prof. to 0 -I1223 01:50:53.036990 18130 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 +I1224 22:49:54.465111 18133 bert_core_vs.cc:426] Setting Opt.Prof. to 0 [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +0, now: CPU 0, GPU 290 (MiB) -I1223 01:50:53.037946 18130 bert_core_vs.cc:476] Setup complete -I1223 01:50:53.038131 18130 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 -I1223 01:50:53.038134 18130 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 -I1223 01:50:53.038137 18130 bert_core_vs.cc:415] Engine - Profile 1 maxDims 98304 Bmax=256 Smax=384 -[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 292, GPU 2722 (MiB) -[I] [TRT] [MemUsageChange] Init cuDNN: CPU +1, GPU +10, now: CPU 293, GPU 2732 (MiB) -I1223 01:50:53.092455 18130 bert_core_vs.cc:426] Setting Opt.Prof. to 1 +I1224 22:49:54.465152 18133 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 +I1224 22:49:54.466116 18133 bert_core_vs.cc:476] Setup complete +I1224 22:49:54.466291 18133 bert_core_vs.cc:385] Engine - Device Memory requirements: 704644608 +I1224 22:49:54.466296 18133 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 2 +I1224 22:49:54.466298 18133 bert_core_vs.cc:415] Engine - Profile 1 maxDims 98304 Bmax=256 Smax=384 +[I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 293, GPU 2722 (MiB) +[I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +10, now: CPU 293, GPU 2732 (MiB) +I1224 22:49:54.522089 18133 bert_core_vs.cc:426] Setting Opt.Prof. to 1 [I] [TRT] Could not set default profile 0 for execution context. Profile index must be set explicitly. [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +1, GPU +0, now: CPU 1, GPU 290 (MiB) -I1223 01:50:53.092808 18130 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 -I1223 01:50:53.093750 18130 bert_core_vs.cc:476] Setup complete -I1223 01:50:53.320848 18130 main_bert.cc:184] Starting running actual test. -I1223 02:01:41.051962 18130 main_bert.cc:190] Finished running actual test. +I1224 22:49:54.522451 18133 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 256 +I1224 22:49:54.523409 18133 bert_core_vs.cc:476] Setup complete +I1224 22:49:54.750171 18133 main_bert.cc:184] Starting running actual test. +I1224 23:00:41.524847 18133 main_bert.cc:190] Finished running actual test. ================================================ MLPerf Results Summary ================================================ SUT name : BERT SERVER Scenario : Offline Mode : PerformanceOnly -Samples per second: 4119.08 +Samples per second: 4125.31 Result is : VALID Min duration satisfied : Yes Min queries satisfied : Yes @@ -77,15 +77,15 @@ Result is : VALID ================================================ Additional Stats ================================================ -Min latency (ns) : 677666862 -Max latency (ns) : 647328935877 -Mean latency (ns) : 392402112422 -50.00 percentile latency (ns) : 417797635309 -90.00 percentile latency (ns) : 617279379522 -95.00 percentile latency (ns) : 635020341501 -97.00 percentile latency (ns) : 640878535042 -99.00 percentile latency (ns) : 645568927895 -99.90 percentile latency (ns) : 647187794972 +Min latency (ns) : 683962619 +Max latency (ns) : 646351685683 +Mean latency (ns) : 391912634857 +50.00 percentile latency (ns) : 417296842561 +90.00 percentile latency (ns) : 616408103246 +95.00 percentile latency (ns) : 634092175919 +97.00 percentile latency (ns) : 639925713545 +99.00 percentile latency (ns) : 644597785494 +99.90 percentile latency (ns) : 646210541816 ================================================ Test Parameters Used @@ -113,7 +113,7 @@ performance_sample_count : 10833 No warnings encountered during test. No errors encountered during test. -[2024-12-23 02:01:41,319 run_harness.py:166 INFO] Result: result_samples_per_second: 4119.08, Result is VALID +[2024-12-24 23:00:41,804 run_harness.py:166 INFO] Result: result_samples_per_second: 4125.31, Result is VALID ======================== Result summaries: ======================== diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/system_info.txt b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/system_info.txt index c0b339f4..98ebb3db 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/system_info.txt +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/system_info.txt @@ -2,10 +2,10 @@ Platform Details ------------------------------------------------------------ 1. uname -a -Linux b8ce345e7cfa 6.8.0-51-generic #52-Ubuntu SMP PREEMPT_DYNAMIC Thu Dec 5 13:09:44 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux +Linux c66a229e22cf 6.8.0-51-generic #52-Ubuntu SMP PREEMPT_DYNAMIC Thu Dec 5 13:09:44 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux ------------------------------------------------------------ 2. w - 02:01:42 up 4 days, 11:54, 0 users, load average: 1.92, 1.85, 1.53 + 23:00:43 up 6 days, 8:53, 0 users, load average: 2.00, 1.86, 1.52 USER TTY FROM LOGIN@ IDLE JCPU PCPU WHAT ------------------------------------------------------------ 3. Username @@ -40,7 +40,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5391.726 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -68,7 +68,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 5487.572 cache size : 1024 KB physical id : 0 siblings : 32 @@ -152,7 +152,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5489.169 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -292,7 +292,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5124.706 +cpu MHz : 5220.063 cache size : 1024 KB physical id : 0 siblings : 32 @@ -320,7 +320,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 5264.571 cache size : 1024 KB physical id : 0 siblings : 32 @@ -348,7 +348,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5227.404 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -376,7 +376,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 5168.243 cache size : 1024 KB physical id : 0 siblings : 32 @@ -404,7 +404,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5190.937 +cpu MHz : 5143.256 cache size : 1024 KB physical id : 0 siblings : 32 @@ -432,7 +432,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5174.849 +cpu MHz : 3184.783 cache size : 1024 KB physical id : 0 siblings : 32 @@ -516,7 +516,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 3136.822 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -656,7 +656,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 3424.963 cache size : 1024 KB physical id : 0 siblings : 32 @@ -740,7 +740,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5205.396 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -796,7 +796,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5171.254 +cpu MHz : 4289.925 cache size : 1024 KB physical id : 0 siblings : 32 @@ -852,7 +852,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 4996.870 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -880,7 +880,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 5162.061 cache size : 1024 KB physical id : 0 siblings : 32 @@ -908,7 +908,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5262.359 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -946,7 +946,7 @@ CPU family: 25 Model: 97 Model name: AMD Ryzen 9 7950X 16-Core Processor Stepping: 2 -CPU MHz: 5391.726 +CPU MHz: 400.000 CPU max MHz: 5881.0000 CPU min MHz: 400.0000 BogoMIPS: 8982.90 @@ -976,49 +976,49 @@ Flags: fpu vme de pse tsc msr pae mce cx8 apic se available: 1 nodes (0) node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 node 0 size: 128007 MB -node 0 free: 34062 MB +node 0 free: 34383 MB node distances: node 0 0: 10 ------------------------------------------------------------ 9. /proc/meminfo MemTotal: 131080068 kB -MemFree: 34880368 kB -MemAvailable: 127606568 kB -Buffers: 159080 kB -Cached: 90143832 kB -SwapCached: 134432 kB -Active: 3465384 kB -Inactive: 87187584 kB -Active(anon): 347660 kB -Inactive(anon): 42656 kB -Active(file): 3117724 kB -Inactive(file): 87144928 kB -Unevictable: 4360 kB +MemFree: 35209028 kB +MemAvailable: 127632692 kB +Buffers: 237596 kB +Cached: 91398868 kB +SwapCached: 117528 kB +Active: 28140332 kB +Inactive: 63854588 kB +Active(anon): 357348 kB +Inactive(anon): 40928 kB +Active(file): 27782984 kB +Inactive(file): 63813660 kB +Unevictable: 0 kB Mlocked: 0 kB SwapTotal: 683591672 kB -SwapFree: 683094776 kB +SwapFree: 683133576 kB Zswap: 0 kB Zswapped: 0 kB -Dirty: 40 kB +Dirty: 72 kB Writeback: 0 kB -AnonPages: 345272 kB -Mapped: 268584 kB -Shmem: 35900 kB -KReclaimable: 3711036 kB -Slab: 4479724 kB -SReclaimable: 3711036 kB -SUnreclaim: 768688 kB -KernelStack: 16896 kB -PageTables: 21988 kB +AnonPages: 347328 kB +Mapped: 254484 kB +Shmem: 39820 kB +KReclaimable: 2074508 kB +Slab: 2752660 kB +SReclaimable: 2074508 kB +SUnreclaim: 678152 kB +KernelStack: 15968 kB +PageTables: 19468 kB SecPageTables: 0 kB NFS_Unstable: 0 kB Bounce: 0 kB WritebackTmp: 0 kB CommitLimit: 749131704 kB -Committed_AS: 5421476 kB +Committed_AS: 4990308 kB VmallocTotal: 34359738367 kB -VmallocUsed: 253704 kB +VmallocUsed: 252676 kB VmallocChunk: 0 kB Percpu: 38400 kB HardwareCorrupted: 0 kB @@ -1034,9 +1034,9 @@ HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 2048 kB Hugetlb: 0 kB -DirectMap4k: 17725856 kB -DirectMap2M: 110419968 kB -DirectMap1G: 6291456 kB +DirectMap4k: 20498848 kB +DirectMap2M: 109744128 kB +DirectMap1G: 4194304 kB ------------------------------------------------------------ 10. who -r ------------------------------------------------------------ @@ -1290,16 +1290,16 @@ dev.tty.ldisc_autoload = 1 dev.tty.legacy_tiocsti = 0 fs.aio-max-nr = 65536 fs.aio-nr = 0 -fs.dentry-state = 770926 597070 45 0 188220 0 +fs.dentry-state = 364900 291568 45 0 178847 0 fs.dir-notify-enable = 1 fs.epoll.max_user_watches = 29163675 fs.fanotify.max_queued_events = 16384 fs.fanotify.max_user_groups = 128 fs.fanotify.max_user_marks = 1048576 fs.file-max = 9223372036854775807 -fs.file-nr = 6528 0 9223372036854775807 -fs.inode-nr = 1023495 473605 -fs.inode-state = 1023495 473605 0 0 0 0 0 +fs.file-nr = 5504 0 9223372036854775807 +fs.inode-nr = 200663 15671 +fs.inode-state = 200663 15671 0 0 0 0 0 fs.inotify.max_queued_events = 16384 fs.inotify.max_user_instances = 128 fs.inotify.max_user_watches = 65536 @@ -1327,7 +1327,7 @@ fs.quota.drops = 0 fs.quota.free_dquots = 0 fs.quota.lookups = 0 fs.quota.reads = 0 -fs.quota.syncs = 494 +fs.quota.syncs = 534 fs.quota.writes = 0 fs.suid_dumpable = 2 fs.verity.require_signatures = 0 @@ -1354,7 +1354,7 @@ kernel.ftrace_dump_on_oops = 0 kernel.ftrace_enabled = 1 kernel.hardlockup_all_cpu_backtrace = 0 kernel.hardlockup_panic = 0 -kernel.hostname = b8ce345e7cfa +kernel.hostname = c66a229e22cf kernel.hotplug = kernel.hung_task_all_cpu_backtrace = 0 kernel.hung_task_check_count = 4194304 @@ -1386,7 +1386,7 @@ kernel.msgmnb = 16384 kernel.msgmni = 32000 kernel.ngroups_max = 65536 kernel.nmi_watchdog = 1 -kernel.ns_last_pid = 18477 +kernel.ns_last_pid = 18480 kernel.numa_balancing = 0 kernel.numa_balancing_promote_rate_limit_MBps = 65536 kernel.oops_all_cpu_backtrace = 0 @@ -1411,19 +1411,19 @@ kernel.perf_event_paranoid = 4 kernel.pid_max = 4194304 kernel.poweroff_cmd = /sbin/poweroff kernel.print-fatal-signals = 0 -kernel.printk = 1 4 1 7 +kernel.printk = 4 4 1 7 kernel.printk_delay = 0 kernel.printk_devkmsg = on kernel.printk_ratelimit = 5 kernel.printk_ratelimit_burst = 10 kernel.pty.max = 4096 -kernel.pty.nr = 9 +kernel.pty.nr = 10 kernel.pty.reserve = 1024 kernel.random.boot_id = d972d8ae-c9e1-46e1-875d-9f419475517a kernel.random.entropy_avail = 256 kernel.random.poolsize = 256 kernel.random.urandom_min_reseed_secs = 60 -kernel.random.uuid = b82c9a0e-5e1b-468f-8cb1-9d06c34d6a84 +kernel.random.uuid = b0ec93ca-c4ad-47eb-9166-1d8aca982cd6 kernel.random.write_wakeup_threshold = 256 kernel.randomize_va_space = 2 kernel.real-root-dev = 0 diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/README.md b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/README.md index 4bddab29..5e422b5d 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/README.md +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/README.md @@ -19,7 +19,7 @@ pip install -U cmind cm rm cache -f -cm pull repo mlcommons@mlperf-automations --checkout=225220c7d9bb7e66e5b9a1e1ebfc3e0180fbd094 +cm pull repo mlcommons@mlperf-automations --checkout=a90475d2de72bf0622cebe8d5ca8eb8c9d872fbd cm run script \ --tags=app,mlperf,inference,generic,_nvidia,_bert-99,_tensorrt,_cuda,_valid,_r4.1-dev_default,_singlestream \ @@ -126,4 +126,4 @@ Model Precision: int8 `F1`: `90.26682`, Required accuracy for closed division `>= 89.96526` ### Performance Results -`90th percentile latency (ns)`: `1016588.0` +`90th percentile latency (ns)`: `1015707.0` diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy_console.out b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy_console.out index 8343dc47..eac9164b 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy_console.out +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy_console.out @@ -1,7 +1,7 @@ -[2024-12-23 01:36:08,298 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x1 -[2024-12-23 01:36:08,615 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x1_TRT/bert-99/SingleStream -[2024-12-23 01:36:08,615 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="AccuracyOnly" --gpu_batch_size=1 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/adc9dc0382544ffd/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=true --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/c2665c8eb600457988f55a7f549c2040.conf" --gpu_inference_streams=1 --gpu_copy_streams=1 --gpu_engines="./build/engines/RTX4090x1/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan" --scenario SingleStream --model bert -[2024-12-23 01:36:08,615 __init__.py:53 INFO] Overriding Environment +[2024-12-24 22:35:11,230 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x1 +[2024-12-24 22:35:11,542 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x1_TRT/bert-99/SingleStream +[2024-12-24 22:35:11,542 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="AccuracyOnly" --gpu_batch_size=1 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/e824f0bb4919400f/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=true --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/faa11389fb7c4ad482f5ed7b80125c7d.conf" --gpu_inference_streams=1 --gpu_copy_streams=1 --gpu_engines="./build/engines/RTX4090x1/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan" --scenario SingleStream --model bert +[2024-12-24 22:35:11,542 __init__.py:53 INFO] Overriding Environment benchmark : Benchmark.BERT buffer_manager_thread_count : 0 coalesced_tensor : True @@ -11,8 +11,8 @@ gpu_copy_streams : 1 gpu_inference_streams : 1 input_dtype : int32 input_format : linear -log_dir : /home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/logs/2024.12.23-01.36.07 -mlperf_conf_path : /home/cmuser/CM/repos/local/cache/adc9dc0382544ffd/inference/mlperf.conf +log_dir : /home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/logs/2024.12.24-22.35.10 +mlperf_conf_path : /home/cmuser/CM/repos/local/cache/e824f0bb4919400f/inference/mlperf.conf precision : int8 preprocessed_data_dir : /home/cmuser/CM/repos/local/cache/a8c152aef5494496/preprocessed_data scenario : Scenario.SingleStream @@ -22,7 +22,7 @@ tensor_path : build/preprocessed_data/squad_tokenized/input_ids.npy,build/prepro test_mode : AccuracyOnly use_graphs : True use_small_tile_gemm_plugin : False -user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/c2665c8eb600457988f55a7f549c2040.conf +user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/faa11389fb7c4ad482f5ed7b80125c7d.conf system_id : RTX4090x1 config_name : RTX4090x1_bert_SingleStream workload_setting : WorkloadSetting(HarnessType.Custom, AccuracyTarget.k_99, PowerSetting.MaxP) @@ -35,32 +35,32 @@ skip_file_checks : True power_limit : None cpu_freq : None &&&& RUNNING BERT_HARNESS # ./build/bin/harness_bert -I1223 01:36:08.668620 13610 main_bert.cc:163] Found 1 GPUs -I1223 01:36:09.059736 13610 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x1/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan +I1224 22:35:11.600999 13613 main_bert.cc:163] Found 1 GPUs +I1224 22:35:11.999068 13613 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x1/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan [I] [TRT] Loaded engine size: 352 MiB [I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +6, GPU +8, now: CPU 458, GPU 1170 (MiB) [I] [TRT] [MemUsageChange] Init cuDNN: CPU +2, GPU +10, now: CPU 460, GPU 1180 (MiB) [I] [TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +290, now: CPU 0, GPU 290 (MiB) -I1223 01:36:09.316869 13610 bert_server.cc:208] Engines Creation Completed -I1223 01:36:09.333741 13610 bert_server.cc:213] Use CUDA graphs -I1223 01:36:09.333932 13610 bert_core_vs.cc:385] Engine - Device Memory requirements: 2753024 -I1223 01:36:09.333936 13610 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 1 -I1223 01:36:09.333943 13610 bert_core_vs.cc:415] Engine - Profile 0 maxDims 384 Bmax=1 Smax=384 +I1224 22:35:12.251561 13613 bert_server.cc:208] Engines Creation Completed +I1224 22:35:12.267074 13613 bert_server.cc:213] Use CUDA graphs +I1224 22:35:12.267278 13613 bert_core_vs.cc:385] Engine - Device Memory requirements: 2753024 +I1224 22:35:12.267282 13613 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 1 +I1224 22:35:12.267287 13613 bert_core_vs.cc:415] Engine - Profile 0 maxDims 384 Bmax=1 Smax=384 [I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 107, GPU 1176 (MiB) [I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 107, GPU 1184 (MiB) -I1223 01:36:09.362983 13610 bert_core_vs.cc:426] Setting Opt.Prof. to 0 -I1223 01:36:09.363015 13610 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 1 +I1224 22:35:12.297389 13613 bert_core_vs.cc:426] Setting Opt.Prof. to 0 +I1224 22:35:12.297418 13613 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 1 [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +0, now: CPU 0, GPU 290 (MiB) -I1223 01:36:09.363065 13610 bert_core_vs.cc:476] Setup complete -I1223 01:36:09.554054 13617 bert_core_vs.cc:352] Created 96 CUDA graphs -I1223 01:36:09.555380 13610 main_bert.cc:184] Starting running actual test. -I1223 01:36:19.804809 13610 main_bert.cc:190] Finished running actual test. +I1224 22:35:12.297467 13613 bert_core_vs.cc:476] Setup complete +I1224 22:35:12.491748 13620 bert_core_vs.cc:352] Created 96 CUDA graphs +I1224 22:35:12.493032 13613 main_bert.cc:184] Starting running actual test. +I1224 22:35:22.694063 13613 main_bert.cc:190] Finished running actual test. No warnings encountered during test. No errors encountered during test. -[2024-12-23 01:36:20,058 run_harness.py:166 INFO] Result: Accuracy run detected. -[2024-12-23 01:36:20,058 __init__.py:46 INFO] Running command: PYTHONPATH=code/bert/tensorrt/helpers python3 /home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/language/bert/accuracy-squad.py --log_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_accuracy.json --vocab_file build/models/bert/vocab.txt --val_data /home/cmuser/CM/repos/local/cache/a8c152aef5494496/data/squad/dev-v1.1.json --out_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/predictions.json --output_dtype float16 +[2024-12-24 22:35:22,959 run_harness.py:166 INFO] Result: Accuracy run detected. +[2024-12-24 22:35:22,959 __init__.py:46 INFO] Running command: PYTHONPATH=code/bert/tensorrt/helpers python3 /home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/language/bert/accuracy-squad.py --log_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_accuracy.json --vocab_file build/models/bert/vocab.txt --val_data /home/cmuser/CM/repos/local/cache/a8c152aef5494496/data/squad/dev-v1.1.json --out_file /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/predictions.json --output_dtype float16 {"exact_match": 82.81929990539263, "f1": 90.26682135974633} Reading examples... No cached features at 'eval_features.pickle'... converting from examples... diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.mmd b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.mmd index 6cb8cdca..8481b604 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.mmd +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.mmd @@ -33,28 +33,28 @@ graph TD download-and-extract,c67e81a4ce2649f5_(_wget,_url.https://zenodo.org/record/3733868/files/vocab.txt_) --> download,file,_wget,_url.https://zenodo.org/record/3733868/files/vocab.txt get-dataset-squad-vocab,e38874fff5094577 --> download-and-extract,_wget,_url.https://zenodo.org/record/3733868/files/vocab.txt app-mlperf-inference,d775cac873ee4231_(_nvidia,_bert-99,_tensorrt,_cuda,_valid,_r4.1-dev_default,_singlestream_) --> get,dataset-aux,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> detect,os + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> detect,cpu - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,sys-utils-cm - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,mlperf,inference,nvidia,scratch,space + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> detect,cpu + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,sys-utils-cm + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,mlperf,inference,nvidia,scratch,space get-generic-python-lib,94b62a682bc44791_(_mlperf_logging_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_mlperf_logging + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,generic-python-lib,_mlperf_logging download-file,9cdc8dc41aae437e_(_cmutil,_url.https://armi.in/files/model.onnx_) --> detect,os download-and-extract,c67e81a4ce2649f5_(_url.https://armi.in/files/model.onnx_) --> download,file,_cmutil,_url.https://armi.in/files/model.onnx get-ml-model-bert-large-squad,5e865dbdc65949d2_(_onnx,_fp32_) --> download-and-extract,_url.https://armi.in/files/model.onnx get-ml-model-bert-large-squad,5e865dbdc65949d2_(_onnx,_fp32_) --> get,dataset-aux,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,ml-model,bert,_onnx,_fp32 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,ml-model,bert,_onnx,_fp32 download-file,9cdc8dc41aae437e_(_cmutil,_url.https://zenodo.org/record/3750364/files/bert_large_v1_1_fake_quant.onnx_) --> detect,os download-and-extract,c67e81a4ce2649f5_(_url.https://zenodo.org/record/3750364/files/bert_large_v1_1_fake_quant.onnx_) --> download,file,_cmutil,_url.https://zenodo.org/record/3750364/files/bert_large_v1_1_fake_quant.onnx get-ml-model-bert-large-squad,5e865dbdc65949d2_(_onnx,_int8_) --> download-and-extract,_url.https://zenodo.org/record/3750364/files/bert_large_v1_1_fake_quant.onnx get-ml-model-bert-large-squad,5e865dbdc65949d2_(_onnx,_int8_) --> get,dataset-aux,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,ml-model,bert,_onnx,_int8 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,mlcommons,inference,src,_deeplearningexamples - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,nvidia,mlperf,inference,common-code,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,ml-model,bert,_onnx,_int8 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,squad-vocab + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,mlcommons,inference,src,_deeplearningexamples + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,nvidia,mlperf,inference,common-code,_mlcommons pull-git-repo,c23132ed65c4421d --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> pull,git,repo + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> pull,git,repo generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,os detect-cpu,586c8a43320142f7 --> detect,os generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,cpu @@ -62,9 +62,9 @@ graph TD generate-mlperf-inference-user-conf,3af4475745964b93 --> get,mlcommons,inference,src,_deeplearningexamples get-mlperf-inference-sut-configs,c2fbf72009e2445b --> get,cache,dir,_name.mlperf-inference-sut-configs generate-mlperf-inference-user-conf,3af4475745964b93 --> get,sut,configs - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> generate,user-conf,mlperf,inference + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> generate,user-conf,mlperf,inference get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_package.pycuda + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,generic-python-lib,_package.pycuda get-nvidia-mitten,1c045f2902374de9 --> detect,os get-nvidia-mitten,1c045f2902374de9 --> get,python3 get-generic-python-lib,94b62a682bc44791_(_pycuda_) --> detect,os @@ -77,9 +77,9 @@ graph TD get-nvidia-mitten,1c045f2902374de9 --> get,generic-python-lib,_pycuda get-git-repo,ed603e7292974f10_(_repo.https://github.com/NVIDIA/mitten_) --> detect,os get-nvidia-mitten,1c045f2902374de9 --> get,git,_repo.https://github.com/NVIDIA/mitten - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,nvidia,mitten - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,cuda,_cudnn - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,tensorrt + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,nvidia,mitten + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,cuda,_cudnn + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,tensorrt build-mlperf-inference-server-nvidia,f37403af5e9f4541_(_mlcommons_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os build-mlperf-inference-server-nvidia,f37403af5e9f4541_(_mlcommons_) --> detect,cpu @@ -137,95 +137,95 @@ graph TD get-generic-python-lib,94b62a682bc44791_(_pycuda_) --> get,python3 add-custom-nvidia-system,b2e6c46c6e8745a3_(_mlcommons_) --> get,generic-python-lib,_pycuda build-mlperf-inference-server-nvidia,f37403af5e9f4541_(_mlcommons_) --> add,custom,system,nvidia,_mlcommons - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> build,nvidia,inference,server,_mlcommons - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> detect,os + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> build,nvidia,inference,server,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> detect,cpu - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,sys-utils-cm - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,mlperf,inference,nvidia,scratch,space,_version.4_0 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> detect,cpu + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,sys-utils-cm + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,mlperf,inference,nvidia,scratch,space,_version.4_0 get-generic-python-lib,94b62a682bc44791_(_mlperf_logging_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_mlperf_logging - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_fp32 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_int8 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,mlcommons,inference,src,_deeplearningexamples - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,nvidia,mlperf,inference,common-code,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_mlperf_logging + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_fp32 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_int8 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,squad-vocab + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,mlcommons,inference,src,_deeplearningexamples + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,nvidia,mlperf,inference,common-code,_mlcommons pull-git-repo,c23132ed65c4421d --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> pull,git,repo + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> pull,git,repo get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.pycuda - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,nvidia,mitten - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,cuda,_cudnn - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,tensorrt - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> build,nvidia,inference,server,_mlcommons - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> detect,os + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.pycuda + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,nvidia,mitten + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,cuda,_cudnn + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,tensorrt + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> build,nvidia,inference,server,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> detect,cpu - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,sys-utils-cm - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,mlperf,inference,nvidia,scratch,space,_version.4_0 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> detect,cpu + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,sys-utils-cm + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,mlperf,inference,nvidia,scratch,space,_version.4_0 get-generic-python-lib,94b62a682bc44791_(_mlperf_logging_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,generic-python-lib,_mlperf_logging - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_fp32 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_int8 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,squad-vocab - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,mlcommons,inference,src,_deeplearningexamples - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,nvidia,mlperf,inference,common-code,_mlcommons + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,generic-python-lib,_mlperf_logging + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_fp32 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,ml-model,bert,_onnx,_int8 + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,squad-vocab + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,mlcommons,inference,src,_deeplearningexamples + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,nvidia,mlperf,inference,common-code,_mlcommons pull-git-repo,c23132ed65c4421d --> detect,os - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> pull,git,repo + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> pull,git,repo get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.pycuda - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,nvidia,mitten + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.pycuda + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,nvidia,mitten get-generic-python-lib,94b62a682bc44791_(_transformers_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,generic-python-lib,_transformers + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,generic-python-lib,_transformers get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> detect,cpu get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> get,python3 get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3 get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> get,generic-python-lib,_pip - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,generic-python-lib,_safetensors + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,generic-python-lib,_safetensors get-generic-python-lib,94b62a682bc44791_(_onnx_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os get-generic-python-lib,94b62a682bc44791_(_onnx_) --> detect,cpu get-generic-python-lib,94b62a682bc44791_(_onnx_) --> get,python3 get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3 get-generic-python-lib,94b62a682bc44791_(_onnx_) --> get,generic-python-lib,_pip - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> detect,os detect-cpu,586c8a43320142f7 --> detect,os get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> detect,cpu get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> get,python3 get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3 get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> get,generic-python-lib,_pip - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.sympy + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.sympy get-generic-python-lib,94b62a682bc44791_(_onnx-graphsurgeon_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx-graphsurgeon - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev_) --> save,mlperf,inference,state - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx-graphsurgeon + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev_) --> save,mlperf,inference,state + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev get-generic-python-lib,94b62a682bc44791_(_transformers_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_transformers + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_transformers get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_safetensors + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_safetensors get-generic-python-lib,94b62a682bc44791_(_onnx_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.sympy + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_package.sympy get-generic-python-lib,94b62a682bc44791_(_onnx-graphsurgeon_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx-graphsurgeon - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev_) --> save,mlperf,inference,state - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> get,generic-python-lib,_onnx-graphsurgeon + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev_) --> save,mlperf,inference,state + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev get-generic-python-lib,94b62a682bc44791_(_transformers_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_transformers + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,generic-python-lib,_transformers get-generic-python-lib,94b62a682bc44791_(_safetensors_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_safetensors + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,generic-python-lib,_safetensors get-generic-python-lib,94b62a682bc44791_(_onnx_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_onnx + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,generic-python-lib,_onnx get-generic-python-lib,94b62a682bc44791_(_package.sympy_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_package.sympy + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,generic-python-lib,_package.sympy get-generic-python-lib,94b62a682bc44791_(_onnx-graphsurgeon_) --> get,python3 - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> get,generic-python-lib,_onnx-graphsurgeon + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> get,generic-python-lib,_onnx-graphsurgeon detect-cpu,586c8a43320142f7 --> detect,os benchmark-program,19f369ef47084895 --> detect,cpu benchmark-program-mlperf,cfff0132a8aa4018 --> benchmark-program,program - app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090_) --> benchmark-mlperf \ No newline at end of file + app-mlperf-inference-nvidia,bc3b17fb430f4732_(_run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090_) --> benchmark-mlperf \ No newline at end of file diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.png b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.png index 98d1816a..736ece1c 100644 Binary files a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.png and b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-deps.png differ diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-version-info.json b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-version-info.json index 0e1645b3..8f198f77 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-version-info.json +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/cm-version-info.json @@ -347,7 +347,7 @@ "script_tags": "detect-os,detect,os,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -367,7 +367,7 @@ "script_tags": "detect,cpu,detect-cpu,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -377,7 +377,7 @@ "script_tags": "get,sys-utils-cm", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -387,7 +387,7 @@ "script_tags": "get,mlperf,inference,nvidia,scratch,space", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -407,7 +407,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "mlperf_logging", "version": "4.1.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -457,7 +457,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,fp32", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -507,7 +507,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,int8", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -517,7 +517,7 @@ "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -527,7 +527,7 @@ "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", "script_variations": "deeplearningexamples", "version": "master-git-2fdb814e6a765d994872fb99ee38f8c43418d5b0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -537,7 +537,7 @@ "script_tags": "get,nvidia,mlperf,inference,common-code", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -557,7 +557,7 @@ "script_tags": "pull,git,repo,repository", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -637,7 +637,7 @@ "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -657,7 +657,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.pycuda", "version": "2022.2.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -787,7 +787,7 @@ "script_tags": "get,nvidia,mitten,nvidia-mitten", "script_variations": "", "version": "master", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -797,7 +797,7 @@ "script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d", "script_variations": "cudnn", "version": "12.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -807,7 +807,7 @@ "script_tags": "get,tensorrt,nvidia", "script_variations": "", "version": "vdetected", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -1387,7 +1387,7 @@ "script_tags": "build,mlcommons,mlperf,inference,inference-server,server,nvidia-harness,nvidia", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -1397,7 +1397,7 @@ "script_tags": "detect-os,detect,os,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1417,7 +1417,7 @@ "script_tags": "detect,cpu,detect-cpu,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1427,7 +1427,7 @@ "script_tags": "get,sys-utils-cm", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1437,7 +1437,7 @@ "script_tags": "get,mlperf,inference,nvidia,scratch,space", "script_variations": "version.4_0", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1457,7 +1457,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "mlperf_logging", "version": "4.1.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1467,7 +1467,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,fp32", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1477,7 +1477,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,int8", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1487,7 +1487,7 @@ "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1497,7 +1497,7 @@ "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", "script_variations": "deeplearningexamples", "version": "master-git-2fdb814e6a765d994872fb99ee38f8c43418d5b0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1507,7 +1507,7 @@ "script_tags": "get,nvidia,mlperf,inference,common-code", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1527,7 +1527,7 @@ "script_tags": "pull,git,repo,repository", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1547,7 +1547,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.pycuda", "version": "2022.2.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1557,7 +1557,7 @@ "script_tags": "get,nvidia,mitten,nvidia-mitten", "script_variations": "", "version": "master", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1567,7 +1567,7 @@ "script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d", "script_variations": "cudnn", "version": "12.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1577,7 +1577,7 @@ "script_tags": "get,tensorrt,nvidia", "script_variations": "", "version": "vdetected", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1587,7 +1587,7 @@ "script_tags": "build,mlcommons,mlperf,inference,inference-server,server,nvidia-harness,nvidia", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -1597,7 +1597,7 @@ "script_tags": "detect-os,detect,os,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1617,7 +1617,7 @@ "script_tags": "detect,cpu,detect-cpu,info", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1627,7 +1627,7 @@ "script_tags": "get,sys-utils-cm", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1637,7 +1637,7 @@ "script_tags": "get,mlperf,inference,nvidia,scratch,space", "script_variations": "version.4_0", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1657,7 +1657,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "mlperf_logging", "version": "4.1.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1667,7 +1667,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,fp32", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1677,7 +1677,7 @@ "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", "script_variations": "onnx,int8", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1687,7 +1687,7 @@ "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1697,7 +1697,7 @@ "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", "script_variations": "deeplearningexamples", "version": "master-git-2fdb814e6a765d994872fb99ee38f8c43418d5b0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1707,7 +1707,7 @@ "script_tags": "get,nvidia,mlperf,inference,common-code", "script_variations": "mlcommons", "version": "r4.0", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1727,7 +1727,7 @@ "script_tags": "pull,git,repo,repository", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1747,7 +1747,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.pycuda", "version": "2022.2.2", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1757,7 +1757,7 @@ "script_tags": "get,nvidia,mitten,nvidia-mitten", "script_variations": "", "version": "master", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1777,7 +1777,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "transformers", "version": "4.46.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1847,7 +1847,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "safetensors", "version": "0.4.5", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1917,7 +1917,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx", "version": "1.13.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -1987,7 +1987,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.sympy", "version": "1.13.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -2007,7 +2007,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx-graphsurgeon", "version": "0.3.27", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { @@ -2017,17 +2017,17 @@ "script_tags": "save,mlperf,inference,implementation,state", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev )" } }, { - "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev": { + "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev": { "script_uid": "bc3b17fb430f4732", "script_alias": "app-mlperf-inference-nvidia", "script_tags": "reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia", - "script_variations": "preprocess_data,tensorrt,cuda,bert-99,bert_,v4.1-dev", + "script_variations": "preprocess_data,bert-99,cuda,tensorrt,bert_,v4.1-dev", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -2047,7 +2047,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "transformers", "version": "4.46.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -2067,7 +2067,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "safetensors", "version": "0.4.5", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -2087,7 +2087,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx", "version": "1.13.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -2107,7 +2107,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.sympy", "version": "1.13.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -2127,7 +2127,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx-graphsurgeon", "version": "0.3.27", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { @@ -2137,27 +2137,27 @@ "script_tags": "save,mlperf,inference,implementation,state", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev )" } }, { - "reproduce,mlperf,inference,nvidia,harness,_build_engine,_tensorrt,_singlestream,_cuda,_bert-99,_batch_size.1,_bert_,_v4.1-dev": { + "reproduce,mlperf,inference,nvidia,harness,_build_engine,_bert-99,_cuda,_tensorrt,_singlestream,_batch_size.1,_bert_,_v4.1-dev": { "script_uid": "bc3b17fb430f4732", "script_alias": "app-mlperf-inference-nvidia", "script_tags": "reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia", - "script_variations": "build_engine,tensorrt,singlestream,cuda,bert-99,batch_size.1,bert_,v4.1-dev", + "script_variations": "build_engine,bert-99,cuda,tensorrt,singlestream,batch_size.1,bert_,v4.1-dev", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { - "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_tensorrt,_cuda,_bert-99,_bert_,_v4.1-dev": { + "reproduce,mlperf,inference,nvidia,harness,_preprocess_data,_bert-99,_cuda,_tensorrt,_bert_,_v4.1-dev": { "script_uid": "bc3b17fb430f4732", "script_alias": "app-mlperf-inference-nvidia", "script_tags": "reproduce,mlcommons,mlperf,inference,harness,nvidia-harness,nvidia", - "script_variations": "preprocess_data,tensorrt,cuda,bert-99,bert_,v4.1-dev", + "script_variations": "preprocess_data,bert-99,cuda,tensorrt,bert_,v4.1-dev", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -2177,7 +2177,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "transformers", "version": "4.46.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -2197,7 +2197,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "safetensors", "version": "0.4.5", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -2217,7 +2217,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx", "version": "1.13.1", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -2237,7 +2237,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "package.sympy", "version": "1.13.3", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -2257,7 +2257,7 @@ "script_tags": "get,install,generic,pip-package,generic-python-lib", "script_variations": "onnx-graphsurgeon", "version": "0.3.27", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } }, { @@ -2297,7 +2297,7 @@ "script_tags": "mlperf,benchmark-mlperf", "script_variations": "", "version": "", - "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_tensorrt,_singlestream,_cuda,_bert-99,_rtx_4090 )" + "parent": "app-mlperf-inference-nvidia,bc3b17fb430f4732 ( run_harness,_bert-99,_cuda,_tensorrt,_singlestream,_rtx_4090 )" } } ] diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/os_info.json b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/os_info.json index b515d081..8cf0acab 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/os_info.json +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/os_info.json @@ -26,5 +26,5 @@ ], "CM_HOST_PLATFORM_FLAVOR": "x86_64", "CM_HOST_PYTHON_BITS": "64", - "CM_HOST_SYSTEM_NAME": "b8ce345e7cfa" + "CM_HOST_SYSTEM_NAME": "c66a229e22cf" } \ No newline at end of file diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance_console.out b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance_console.out index 5431e6d1..69765187 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance_console.out +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance_console.out @@ -1,7 +1,7 @@ -[2024-12-23 01:25:26,929 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x1 -[2024-12-23 01:25:27,239 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x1_TRT/bert-99/SingleStream -[2024-12-23 01:25:27,239 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="PerformanceOnly" --gpu_batch_size=1 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/adc9dc0382544ffd/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=true --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/cb246d8885894431af2b1d28ee066c76.conf" --gpu_inference_streams=1 --gpu_copy_streams=1 --gpu_engines="./build/engines/RTX4090x1/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan" --scenario SingleStream --model bert -[2024-12-23 01:25:27,239 __init__.py:53 INFO] Overriding Environment +[2024-12-24 22:24:30,704 main.py:229 INFO] Detected system ID: KnownSystem.RTX4090x1 +[2024-12-24 22:24:31,012 generate_conf_files.py:107 INFO] Generated measurements/ entries for RTX4090x1_TRT/bert-99/SingleStream +[2024-12-24 22:24:31,012 __init__.py:46 INFO] Running command: ./build/bin/harness_bert --logfile_outdir="/cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1" --logfile_prefix="mlperf_log_" --performance_sample_count=10833 --test_mode="PerformanceOnly" --gpu_batch_size=1 --mlperf_conf_path="/home/cmuser/CM/repos/local/cache/e824f0bb4919400f/inference/mlperf.conf" --tensor_path="build/preprocessed_data/squad_tokenized/input_ids.npy,build/preprocessed_data/squad_tokenized/segment_ids.npy,build/preprocessed_data/squad_tokenized/input_mask.npy" --use_graphs=true --user_conf_path="/home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/f975e32037d54399a38fd9c5d80006c2.conf" --gpu_inference_streams=1 --gpu_copy_streams=1 --gpu_engines="./build/engines/RTX4090x1/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan" --scenario SingleStream --model bert +[2024-12-24 22:24:31,013 __init__.py:53 INFO] Overriding Environment benchmark : Benchmark.BERT buffer_manager_thread_count : 0 coalesced_tensor : True @@ -11,8 +11,8 @@ gpu_copy_streams : 1 gpu_inference_streams : 1 input_dtype : int32 input_format : linear -log_dir : /home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/logs/2024.12.23-01.25.26 -mlperf_conf_path : /home/cmuser/CM/repos/local/cache/adc9dc0382544ffd/inference/mlperf.conf +log_dir : /home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/logs/2024.12.24-22.24.29 +mlperf_conf_path : /home/cmuser/CM/repos/local/cache/e824f0bb4919400f/inference/mlperf.conf precision : int8 preprocessed_data_dir : /home/cmuser/CM/repos/local/cache/a8c152aef5494496/preprocessed_data scenario : Scenario.SingleStream @@ -22,7 +22,7 @@ tensor_path : build/preprocessed_data/squad_tokenized/input_ids.npy,build/prepro test_mode : PerformanceOnly use_graphs : True use_small_tile_gemm_plugin : False -user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/cb246d8885894431af2b1d28ee066c76.conf +user_conf_path : /home/cmuser/CM/repos/mlcommons@mlperf-automations/script/generate-mlperf-inference-user-conf/tmp/f975e32037d54399a38fd9c5d80006c2.conf system_id : RTX4090x1 config_name : RTX4090x1_bert_SingleStream workload_setting : WorkloadSetting(HarnessType.Custom, AccuracyTarget.k_99, PowerSetting.MaxP) @@ -35,58 +35,58 @@ skip_file_checks : True power_limit : None cpu_freq : None &&&& RUNNING BERT_HARNESS # ./build/bin/harness_bert -I1223 01:25:27.299499 10478 main_bert.cc:163] Found 1 GPUs -I1223 01:25:27.695150 10478 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x1/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan +I1224 22:24:31.073055 10481 main_bert.cc:163] Found 1 GPUs +I1224 22:24:31.468959 10481 bert_server.cc:147] Engine Path: ./build/engines/RTX4090x1/bert/SingleStream/bert-SingleStream-gpu-int8_S_384_B_1_P_1_vs.custom_k_99_MaxP.plan [I] [TRT] Loaded engine size: 352 MiB [I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +6, GPU +8, now: CPU 458, GPU 1170 (MiB) [I] [TRT] [MemUsageChange] Init cuDNN: CPU +2, GPU +10, now: CPU 460, GPU 1180 (MiB) [I] [TRT] [MemUsageChange] TensorRT-managed allocation in engine deserialization: CPU +0, GPU +290, now: CPU 0, GPU 290 (MiB) -I1223 01:25:27.956712 10478 bert_server.cc:208] Engines Creation Completed -I1223 01:25:27.974658 10478 bert_server.cc:213] Use CUDA graphs -I1223 01:25:27.974850 10478 bert_core_vs.cc:385] Engine - Device Memory requirements: 2753024 -I1223 01:25:27.974854 10478 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 1 -I1223 01:25:27.974859 10478 bert_core_vs.cc:415] Engine - Profile 0 maxDims 384 Bmax=1 Smax=384 +I1224 22:24:31.728627 10481 bert_server.cc:208] Engines Creation Completed +I1224 22:24:31.745921 10481 bert_server.cc:213] Use CUDA graphs +I1224 22:24:31.746117 10481 bert_core_vs.cc:385] Engine - Device Memory requirements: 2753024 +I1224 22:24:31.746120 10481 bert_core_vs.cc:393] Engine - Number of Optimization Profiles: 1 +I1224 22:24:31.746125 10481 bert_core_vs.cc:415] Engine - Profile 0 maxDims 384 Bmax=1 Smax=384 [I] [TRT] [MemUsageChange] Init cuBLAS/cuBLASLt: CPU +0, GPU +8, now: CPU 107, GPU 1176 (MiB) [I] [TRT] [MemUsageChange] Init cuDNN: CPU +0, GPU +8, now: CPU 107, GPU 1184 (MiB) -I1223 01:25:28.005026 10478 bert_core_vs.cc:426] Setting Opt.Prof. to 0 -I1223 01:25:28.005055 10478 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 1 +I1224 22:24:31.776012 10481 bert_core_vs.cc:426] Setting Opt.Prof. to 0 +I1224 22:24:31.776042 10481 bert_core_vs.cc:444] Context creation complete. Max supported batchSize: 1 [I] [TRT] [MemUsageChange] TensorRT-managed allocation in IExecutionContext creation: CPU +0, GPU +0, now: CPU 0, GPU 290 (MiB) -I1223 01:25:28.005106 10478 bert_core_vs.cc:476] Setup complete -I1223 01:25:28.196641 10485 bert_core_vs.cc:352] Created 96 CUDA graphs -I1223 01:25:28.197918 10478 main_bert.cc:184] Starting running actual test. -I1223 01:35:29.671032 10478 main_bert.cc:190] Finished running actual test. +I1224 22:24:31.776091 10481 bert_core_vs.cc:476] Setup complete +I1224 22:24:31.970160 10488 bert_core_vs.cc:352] Created 96 CUDA graphs +I1224 22:24:31.971451 10481 main_bert.cc:184] Starting running actual test. +I1224 22:34:33.450074 10481 main_bert.cc:190] Finished running actual test. ================================================ MLPerf Results Summary ================================================ SUT name : BERT SERVER Scenario : SingleStream Mode : PerformanceOnly -90th percentile latency (ns) : 1013732 +90th percentile latency (ns) : 1014324 Result is : VALID Min duration satisfied : Yes Min queries satisfied : Yes Early stopping satisfied: Yes Early Stopping Result: - * Processed at least 64 queries (637969). - * Would discard 63238 highest latency queries. - * Early stopping 90th percentile estimate: 1016588 - * Early stopping 99th percentile estimate: 1190854 + * Processed at least 64 queries (641819). + * Would discard 63621 highest latency queries. + * Early stopping 90th percentile estimate: 1015707 + * Early stopping 99th percentile estimate: 1190463 ================================================ Additional Stats ================================================ -QPS w/ loadgen overhead : 1063.28 -QPS w/o loadgen overhead : 1069.05 +QPS w/ loadgen overhead : 1069.70 +QPS w/o loadgen overhead : 1074.10 -Min latency (ns) : 872179 -Max latency (ns) : 1540105 -Mean latency (ns) : 935409 -50.00 percentile latency (ns) : 913296 -90.00 percentile latency (ns) : 1013732 -95.00 percentile latency (ns) : 1075228 -97.00 percentile latency (ns) : 1174433 -99.00 percentile latency (ns) : 1190824 -99.90 percentile latency (ns) : 1193097 +Min latency (ns) : 859605 +Max latency (ns) : 1297262 +Mean latency (ns) : 931012 +50.00 percentile latency (ns) : 909739 +90.00 percentile latency (ns) : 1014324 +95.00 percentile latency (ns) : 1082421 +97.00 percentile latency (ns) : 1174393 +99.00 percentile latency (ns) : 1190403 +99.90 percentile latency (ns) : 1206002 ================================================ Test Parameters Used @@ -114,7 +114,7 @@ performance_sample_count : 10833 No warnings encountered during test. No errors encountered during test. -[2024-12-23 01:35:29,950 run_harness.py:166 INFO] Result: result_90.00_percentile_latency_ns: 1013732, Result is VALID +[2024-12-24 22:34:33,740 run_harness.py:166 INFO] Result: result_90.00_percentile_latency_ns: 1014324, Result is VALID ======================== Result summaries: ======================== diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/system_info.txt b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/system_info.txt index 74edc986..14aa1012 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/system_info.txt +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/system_info.txt @@ -2,10 +2,10 @@ Platform Details ------------------------------------------------------------ 1. uname -a -Linux b8ce345e7cfa 6.8.0-51-generic #52-Ubuntu SMP PREEMPT_DYNAMIC Thu Dec 5 13:09:44 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux +Linux c66a229e22cf 6.8.0-51-generic #52-Ubuntu SMP PREEMPT_DYNAMIC Thu Dec 5 13:09:44 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux ------------------------------------------------------------ 2. w - 01:35:44 up 4 days, 11:28, 0 users, load average: 0.95, 1.21, 1.42 + 22:34:48 up 6 days, 8:27, 0 users, load average: 1.00, 1.18, 1.39 USER TTY FROM LOGIN@ IDLE JCPU PCPU WHAT ------------------------------------------------------------ 3. Username @@ -68,7 +68,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 5489.567 cache size : 1024 KB physical id : 0 siblings : 32 @@ -96,7 +96,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5489.281 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -124,7 +124,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 5478.657 cache size : 1024 KB physical id : 0 siblings : 32 @@ -152,7 +152,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5482.217 +cpu MHz : 5492.255 cache size : 1024 KB physical id : 0 siblings : 32 @@ -180,7 +180,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 4469.085 cache size : 1024 KB physical id : 0 siblings : 32 @@ -264,7 +264,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 4009.098 cache size : 1024 KB physical id : 0 siblings : 32 @@ -320,7 +320,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5313.596 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -348,7 +348,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 3540.414 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -376,7 +376,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 4557.887 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -432,7 +432,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5317.876 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -656,7 +656,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 3420.140 cache size : 1024 KB physical id : 0 siblings : 32 @@ -796,7 +796,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 5336.151 cache size : 1024 KB physical id : 0 siblings : 32 @@ -824,7 +824,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 5274.150 cache size : 1024 KB physical id : 0 siblings : 32 @@ -976,49 +976,49 @@ Flags: fpu vme de pse tsc msr pae mce cx8 apic se available: 1 nodes (0) node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 node 0 size: 128007 MB -node 0 free: 34660 MB +node 0 free: 34910 MB node distances: node 0 0: 10 ------------------------------------------------------------ 9. /proc/meminfo MemTotal: 131080068 kB -MemFree: 35492496 kB -MemAvailable: 127513976 kB -Buffers: 157048 kB -Cached: 89463208 kB -SwapCached: 134496 kB -Active: 3444796 kB -Inactive: 86506816 kB -Active(anon): 327480 kB -Inactive(anon): 42660 kB -Active(file): 3117316 kB -Inactive(file): 86464156 kB -Unevictable: 4360 kB +MemFree: 35747924 kB +MemAvailable: 127460884 kB +Buffers: 235532 kB +Cached: 90718596 kB +SwapCached: 117532 kB +Active: 28115316 kB +Inactive: 63173668 kB +Active(anon): 332588 kB +Inactive(anon): 40928 kB +Active(file): 27782728 kB +Inactive(file): 63132740 kB +Unevictable: 0 kB Mlocked: 0 kB SwapTotal: 683591672 kB -SwapFree: 683093240 kB +SwapFree: 683133576 kB Zswap: 0 kB Zswapped: 0 kB -Dirty: 21660 kB +Dirty: 21156 kB Writeback: 0 kB -AnonPages: 326384 kB -Mapped: 268356 kB -Shmem: 34360 kB -KReclaimable: 3687496 kB -Slab: 4456924 kB -SReclaimable: 3687496 kB -SUnreclaim: 769428 kB -KernelStack: 17392 kB -PageTables: 22844 kB +AnonPages: 323544 kB +Mapped: 254972 kB +Shmem: 38660 kB +KReclaimable: 2044980 kB +Slab: 2722948 kB +SReclaimable: 2044980 kB +SUnreclaim: 677968 kB +KernelStack: 16192 kB +PageTables: 19656 kB SecPageTables: 0 kB NFS_Unstable: 0 kB Bounce: 0 kB WritebackTmp: 0 kB CommitLimit: 749131704 kB -Committed_AS: 5389168 kB +Committed_AS: 4961692 kB VmallocTotal: 34359738367 kB -VmallocUsed: 253848 kB +VmallocUsed: 252804 kB VmallocChunk: 0 kB Percpu: 38400 kB HardwareCorrupted: 0 kB @@ -1034,9 +1034,9 @@ HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 2048 kB Hugetlb: 0 kB -DirectMap4k: 17674656 kB -DirectMap2M: 110471168 kB -DirectMap1G: 6291456 kB +DirectMap4k: 20453792 kB +DirectMap2M: 109789184 kB +DirectMap1G: 4194304 kB ------------------------------------------------------------ 10. who -r ------------------------------------------------------------ @@ -1290,16 +1290,16 @@ dev.tty.ldisc_autoload = 1 dev.tty.legacy_tiocsti = 0 fs.aio-max-nr = 65536 fs.aio-nr = 0 -fs.dentry-state = 753086 583308 45 0 180912 0 +fs.dentry-state = 347067 277812 45 0 171539 0 fs.dir-notify-enable = 1 fs.epoll.max_user_watches = 29163675 fs.fanotify.max_queued_events = 16384 fs.fanotify.max_user_groups = 128 fs.fanotify.max_user_marks = 1048576 fs.file-max = 9223372036854775807 -fs.file-nr = 6912 0 9223372036854775807 -fs.inode-nr = 1012969 473605 -fs.inode-state = 1012969 473605 0 0 0 0 0 +fs.file-nr = 5760 0 9223372036854775807 +fs.inode-nr = 190144 15671 +fs.inode-state = 190144 15671 0 0 0 0 0 fs.inotify.max_queued_events = 16384 fs.inotify.max_user_instances = 128 fs.inotify.max_user_watches = 65536 @@ -1327,7 +1327,7 @@ fs.quota.drops = 0 fs.quota.free_dquots = 0 fs.quota.lookups = 0 fs.quota.reads = 0 -fs.quota.syncs = 494 +fs.quota.syncs = 534 fs.quota.writes = 0 fs.suid_dumpable = 2 fs.verity.require_signatures = 0 @@ -1354,7 +1354,7 @@ kernel.ftrace_dump_on_oops = 0 kernel.ftrace_enabled = 1 kernel.hardlockup_all_cpu_backtrace = 0 kernel.hardlockup_panic = 0 -kernel.hostname = b8ce345e7cfa +kernel.hostname = c66a229e22cf kernel.hotplug = kernel.hung_task_all_cpu_backtrace = 0 kernel.hung_task_check_count = 4194304 @@ -1386,7 +1386,7 @@ kernel.msgmnb = 16384 kernel.msgmni = 32000 kernel.ngroups_max = 65536 kernel.nmi_watchdog = 1 -kernel.ns_last_pid = 12021 +kernel.ns_last_pid = 12024 kernel.numa_balancing = 0 kernel.numa_balancing_promote_rate_limit_MBps = 65536 kernel.oops_all_cpu_backtrace = 0 @@ -1411,19 +1411,19 @@ kernel.perf_event_paranoid = 4 kernel.pid_max = 4194304 kernel.poweroff_cmd = /sbin/poweroff kernel.print-fatal-signals = 0 -kernel.printk = 1 4 1 7 +kernel.printk = 4 4 1 7 kernel.printk_delay = 0 kernel.printk_devkmsg = on kernel.printk_ratelimit = 5 kernel.printk_ratelimit_burst = 10 kernel.pty.max = 4096 -kernel.pty.nr = 9 +kernel.pty.nr = 10 kernel.pty.reserve = 1024 kernel.random.boot_id = d972d8ae-c9e1-46e1-875d-9f419475517a kernel.random.entropy_avail = 256 kernel.random.poolsize = 256 kernel.random.urandom_min_reseed_secs = 60 -kernel.random.uuid = 1166150c-fd8f-4c6d-b011-56dc265dbb66 +kernel.random.uuid = 3dfd7f9a-eff3-4fd7-83ff-e181f3330221 kernel.random.write_wakeup_threshold = 256 kernel.randomize_va_space = 2 kernel.real-root-dev = 0 diff --git a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/system_info.txt b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/system_info.txt index c0b339f4..98ebb3db 100644 --- a/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/system_info.txt +++ b/open/MLCommons/measurements/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/system_info.txt @@ -2,10 +2,10 @@ Platform Details ------------------------------------------------------------ 1. uname -a -Linux b8ce345e7cfa 6.8.0-51-generic #52-Ubuntu SMP PREEMPT_DYNAMIC Thu Dec 5 13:09:44 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux +Linux c66a229e22cf 6.8.0-51-generic #52-Ubuntu SMP PREEMPT_DYNAMIC Thu Dec 5 13:09:44 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux ------------------------------------------------------------ 2. w - 02:01:42 up 4 days, 11:54, 0 users, load average: 1.92, 1.85, 1.53 + 23:00:43 up 6 days, 8:53, 0 users, load average: 2.00, 1.86, 1.52 USER TTY FROM LOGIN@ IDLE JCPU PCPU WHAT ------------------------------------------------------------ 3. Username @@ -40,7 +40,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5391.726 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -68,7 +68,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 5487.572 cache size : 1024 KB physical id : 0 siblings : 32 @@ -152,7 +152,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5489.169 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -292,7 +292,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5124.706 +cpu MHz : 5220.063 cache size : 1024 KB physical id : 0 siblings : 32 @@ -320,7 +320,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 5264.571 cache size : 1024 KB physical id : 0 siblings : 32 @@ -348,7 +348,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5227.404 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -376,7 +376,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 5168.243 cache size : 1024 KB physical id : 0 siblings : 32 @@ -404,7 +404,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5190.937 +cpu MHz : 5143.256 cache size : 1024 KB physical id : 0 siblings : 32 @@ -432,7 +432,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5174.849 +cpu MHz : 3184.783 cache size : 1024 KB physical id : 0 siblings : 32 @@ -516,7 +516,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 3136.822 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -656,7 +656,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 3424.963 cache size : 1024 KB physical id : 0 siblings : 32 @@ -740,7 +740,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5205.396 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -796,7 +796,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5171.254 +cpu MHz : 4289.925 cache size : 1024 KB physical id : 0 siblings : 32 @@ -852,7 +852,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 4996.870 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -880,7 +880,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 400.000 +cpu MHz : 5162.061 cache size : 1024 KB physical id : 0 siblings : 32 @@ -908,7 +908,7 @@ model : 97 model name : AMD Ryzen 9 7950X 16-Core Processor stepping : 2 microcode : 0xa601206 -cpu MHz : 5262.359 +cpu MHz : 400.000 cache size : 1024 KB physical id : 0 siblings : 32 @@ -946,7 +946,7 @@ CPU family: 25 Model: 97 Model name: AMD Ryzen 9 7950X 16-Core Processor Stepping: 2 -CPU MHz: 5391.726 +CPU MHz: 400.000 CPU max MHz: 5881.0000 CPU min MHz: 400.0000 BogoMIPS: 8982.90 @@ -976,49 +976,49 @@ Flags: fpu vme de pse tsc msr pae mce cx8 apic se available: 1 nodes (0) node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 node 0 size: 128007 MB -node 0 free: 34062 MB +node 0 free: 34383 MB node distances: node 0 0: 10 ------------------------------------------------------------ 9. /proc/meminfo MemTotal: 131080068 kB -MemFree: 34880368 kB -MemAvailable: 127606568 kB -Buffers: 159080 kB -Cached: 90143832 kB -SwapCached: 134432 kB -Active: 3465384 kB -Inactive: 87187584 kB -Active(anon): 347660 kB -Inactive(anon): 42656 kB -Active(file): 3117724 kB -Inactive(file): 87144928 kB -Unevictable: 4360 kB +MemFree: 35209028 kB +MemAvailable: 127632692 kB +Buffers: 237596 kB +Cached: 91398868 kB +SwapCached: 117528 kB +Active: 28140332 kB +Inactive: 63854588 kB +Active(anon): 357348 kB +Inactive(anon): 40928 kB +Active(file): 27782984 kB +Inactive(file): 63813660 kB +Unevictable: 0 kB Mlocked: 0 kB SwapTotal: 683591672 kB -SwapFree: 683094776 kB +SwapFree: 683133576 kB Zswap: 0 kB Zswapped: 0 kB -Dirty: 40 kB +Dirty: 72 kB Writeback: 0 kB -AnonPages: 345272 kB -Mapped: 268584 kB -Shmem: 35900 kB -KReclaimable: 3711036 kB -Slab: 4479724 kB -SReclaimable: 3711036 kB -SUnreclaim: 768688 kB -KernelStack: 16896 kB -PageTables: 21988 kB +AnonPages: 347328 kB +Mapped: 254484 kB +Shmem: 39820 kB +KReclaimable: 2074508 kB +Slab: 2752660 kB +SReclaimable: 2074508 kB +SUnreclaim: 678152 kB +KernelStack: 15968 kB +PageTables: 19468 kB SecPageTables: 0 kB NFS_Unstable: 0 kB Bounce: 0 kB WritebackTmp: 0 kB CommitLimit: 749131704 kB -Committed_AS: 5421476 kB +Committed_AS: 4990308 kB VmallocTotal: 34359738367 kB -VmallocUsed: 253704 kB +VmallocUsed: 252676 kB VmallocChunk: 0 kB Percpu: 38400 kB HardwareCorrupted: 0 kB @@ -1034,9 +1034,9 @@ HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 2048 kB Hugetlb: 0 kB -DirectMap4k: 17725856 kB -DirectMap2M: 110419968 kB -DirectMap1G: 6291456 kB +DirectMap4k: 20498848 kB +DirectMap2M: 109744128 kB +DirectMap1G: 4194304 kB ------------------------------------------------------------ 10. who -r ------------------------------------------------------------ @@ -1290,16 +1290,16 @@ dev.tty.ldisc_autoload = 1 dev.tty.legacy_tiocsti = 0 fs.aio-max-nr = 65536 fs.aio-nr = 0 -fs.dentry-state = 770926 597070 45 0 188220 0 +fs.dentry-state = 364900 291568 45 0 178847 0 fs.dir-notify-enable = 1 fs.epoll.max_user_watches = 29163675 fs.fanotify.max_queued_events = 16384 fs.fanotify.max_user_groups = 128 fs.fanotify.max_user_marks = 1048576 fs.file-max = 9223372036854775807 -fs.file-nr = 6528 0 9223372036854775807 -fs.inode-nr = 1023495 473605 -fs.inode-state = 1023495 473605 0 0 0 0 0 +fs.file-nr = 5504 0 9223372036854775807 +fs.inode-nr = 200663 15671 +fs.inode-state = 200663 15671 0 0 0 0 0 fs.inotify.max_queued_events = 16384 fs.inotify.max_user_instances = 128 fs.inotify.max_user_watches = 65536 @@ -1327,7 +1327,7 @@ fs.quota.drops = 0 fs.quota.free_dquots = 0 fs.quota.lookups = 0 fs.quota.reads = 0 -fs.quota.syncs = 494 +fs.quota.syncs = 534 fs.quota.writes = 0 fs.suid_dumpable = 2 fs.verity.require_signatures = 0 @@ -1354,7 +1354,7 @@ kernel.ftrace_dump_on_oops = 0 kernel.ftrace_enabled = 1 kernel.hardlockup_all_cpu_backtrace = 0 kernel.hardlockup_panic = 0 -kernel.hostname = b8ce345e7cfa +kernel.hostname = c66a229e22cf kernel.hotplug = kernel.hung_task_all_cpu_backtrace = 0 kernel.hung_task_check_count = 4194304 @@ -1386,7 +1386,7 @@ kernel.msgmnb = 16384 kernel.msgmni = 32000 kernel.ngroups_max = 65536 kernel.nmi_watchdog = 1 -kernel.ns_last_pid = 18477 +kernel.ns_last_pid = 18480 kernel.numa_balancing = 0 kernel.numa_balancing_promote_rate_limit_MBps = 65536 kernel.oops_all_cpu_backtrace = 0 @@ -1411,19 +1411,19 @@ kernel.perf_event_paranoid = 4 kernel.pid_max = 4194304 kernel.poweroff_cmd = /sbin/poweroff kernel.print-fatal-signals = 0 -kernel.printk = 1 4 1 7 +kernel.printk = 4 4 1 7 kernel.printk_delay = 0 kernel.printk_devkmsg = on kernel.printk_ratelimit = 5 kernel.printk_ratelimit_burst = 10 kernel.pty.max = 4096 -kernel.pty.nr = 9 +kernel.pty.nr = 10 kernel.pty.reserve = 1024 kernel.random.boot_id = d972d8ae-c9e1-46e1-875d-9f419475517a kernel.random.entropy_avail = 256 kernel.random.poolsize = 256 kernel.random.urandom_min_reseed_secs = 60 -kernel.random.uuid = b82c9a0e-5e1b-468f-8cb1-9d06c34d6a84 +kernel.random.uuid = b0ec93ca-c4ad-47eb-9166-1d8aca982cd6 kernel.random.write_wakeup_threshold = 256 kernel.randomize_va_space = 2 kernel.real-root-dev = 0 diff --git a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/accuracy.txt b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/accuracy.txt index 038b77ab..b8677b98 100644 --- a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/accuracy.txt +++ b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/accuracy.txt @@ -1,8 +1,8 @@ {"exact_match": 82.81929990539263, "f1": 90.15673510616978} Reading examples... -Loading cached features from '/home/cmuser/CM/repos/local/cache/adc9dc0382544ffd/inference/language/bert/eval_features.pickle'... +Loading cached features from '/home/cmuser/CM/repos/local/cache/e824f0bb4919400f/inference/language/bert/eval_features.pickle'... Loading LoadGen logs... Post-processing predictions... Writing predictions to: /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/predictions.json Evaluating predictions... -hash=9172b37731f86dfb2868a22518dacfdbc3fb3aae1f5e2ea876cf60b0b65313eb +hash=09fb7bff237e91bf3db6f82cfe619448a97e090e341cb9fba7c268db54fbaad2 diff --git a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_detail.txt b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_detail.txt index c3eb7e03..87ef8955 100644 --- a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_detail.txt +++ b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/accuracy/mlperf_log_detail.txt @@ -1,68 +1,68 @@ -:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.002524, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-23T01:23:31.577600", "time_ms": 0.002524, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-23T09:23:31.577604", "time_ms": 0.002524, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.002524, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.002524, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.002524, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.002524, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "test_datetime", "value": "2024-12-23T02:01:53Z", "time_ms": 0.010108, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.010108, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "get_sut_name_duration_ns", "value": 230, "time_ms": 0.010108, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.010108, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.010108, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.010108, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_test_mode", "value": "AccuracyOnly", "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_offline_expected_qps", "value": 4160.27, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 306, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_min_query_count", "value": 10833, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.014777, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_test_mode", "value": "AccuracyOnly", "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_samples_per_query", "value": 2745778, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_target_qps", "value": 4160.27, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_min_sample_count", "value": 2745778, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.014907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "generic_message", "value": "Starting accuracy mode", "time_ms": 0.015769, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1090, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.112089, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 39.741801, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "generated_samples_per_query", "value": 2745778, "time_ms": 39.741801, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "generated_query_duration", "value": 240368, "time_ms": 39.741801, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 2640.281596, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 2640.281596, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 2640.281596, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 2640.281596, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 2640.281596, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 2640.281596, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "power_begin", "value": "12-23-2024 10:01:53.997", "time_ms": 2640.282017, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 19723, "tid": 19723}} -:::MLLOG {"key": "power_end", "value": "12-23-2024 10:01:56.590", "time_ms": 2640.282017, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 19723, "tid": 19723}} +:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.006422, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-24T22:22:34.978859", "time_ms": 0.006422, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-25T06:22:34.978862", "time_ms": 0.006422, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.006422, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.006422, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.006422, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.006422, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "test_datetime", "value": "2024-12-24T23:00:54Z", "time_ms": 0.012424, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.012424, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "get_sut_name_duration_ns", "value": 350, "time_ms": 0.012424, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.012424, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.012424, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.012424, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_test_mode", "value": "AccuracyOnly", "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_offline_expected_qps", "value": 4166.56, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 306, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_min_query_count", "value": 10833, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.017583, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_test_mode", "value": "AccuracyOnly", "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_samples_per_query", "value": 2749931, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_target_qps", "value": 4166.56, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_min_sample_count", "value": 2749931, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.017924, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "generic_message", "value": "Starting accuracy mode", "time_ms": 0.018826, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1090, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.120004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 39.006454, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "generated_samples_per_query", "value": 2749931, "time_ms": 39.006454, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "generated_query_duration", "value": 240005, "time_ms": 39.006454, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 2630.046976, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 2630.046976, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 2630.046976, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 2630.046976, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 2630.046976, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 2630.046976, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "power_begin", "value": "12-25-2024 07:00:54.553", "time_ms": 2630.047527, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 19726, "tid": 19726}} +:::MLLOG {"key": "power_end", "value": "12-25-2024 07:00:57.141", "time_ms": 2630.047527, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 19726, "tid": 19726}} diff --git a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_detail.txt b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_detail.txt index 7b617810..8c697e33 100644 --- a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_detail.txt +++ b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_detail.txt @@ -1,85 +1,85 @@ -:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.002945, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-23T01:23:31.577600", "time_ms": 0.002945, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-23T09:23:31.577604", "time_ms": 0.002945, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.002945, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.002945, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.002945, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.002945, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "test_datetime", "value": "2024-12-23T01:50:53Z", "time_ms": 0.009488, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.009488, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "get_sut_name_duration_ns", "value": 240, "time_ms": 0.009488, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.009488, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.009488, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.009488, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_test_mode", "value": "PerformanceOnly", "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_offline_expected_qps", "value": 4040, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 306, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_min_query_count", "value": 10833, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.014136, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_test_mode", "value": "PerformanceOnly", "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_samples_per_query", "value": 2666400, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_target_qps", "value": 4040, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_min_sample_count", "value": 2666400, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.014287, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "generic_message", "value": "Starting performance mode", "time_ms": 0.018805, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 841, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.122128, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 114.315097, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "generated_samples_per_query", "value": 2666400, "time_ms": 114.315097, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "generated_query_duration", "value": 247524, "time_ms": 114.315097, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "generic_message", "value": "Ending naturally: Minimum query count and test duration met.", "time_ms": 561.281441, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/issue_query_controller.cc", "line_no": 482, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 647447.652720, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 647447.652720, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 647447.652720, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 647447.652720, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 647447.652720, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 647447.652720, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "power_begin", "value": "12-23-2024 09:50:53.436", "time_ms": 647447.653121, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "power_end", "value": "12-23-2024 10:01:40.765", "time_ms": 647447.653121, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "result_validity", "value": "VALID", "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 644, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "result_min_duration_met", "value": true, "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 649, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "result_min_queries_met", "value": true, "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 650, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "early_stopping_met", "value": true, "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 651, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "early_stopping_result", "value": "", "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 669, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "result_query_count", "value": 1, "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 679, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "result_samples_per_second", "value": 4119.08, "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 735, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "result_min_latency_ns", "value": 677666862, "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 741, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "result_max_latency_ns", "value": 647328935877, "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 742, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "result_mean_latency_ns", "value": 392402112422, "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 743, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "result_50.00_percentile_latency_ns", "value": 417797635309, "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "result_90.00_percentile_latency_ns", "value": 617279379522, "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "result_95.00_percentile_latency_ns", "value": 635020341501, "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "result_97.00_percentile_latency_ns", "value": 640878535042, "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "result_99.00_percentile_latency_ns", "value": 645568927895, "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18130, "tid": 18130}} -:::MLLOG {"key": "result_99.90_percentile_latency_ns", "value": 647187794972, "time_ms": 647464.753544, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18130, "tid": 18130}} +:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.002955, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-24T22:22:34.978859", "time_ms": 0.002955, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-25T06:22:34.978862", "time_ms": 0.002955, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.002955, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.002955, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.002955, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.002955, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "test_datetime", "value": "2024-12-24T22:49:54Z", "time_ms": 0.009587, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.009587, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "get_sut_name_duration_ns", "value": 331, "time_ms": 0.009587, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.009587, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.009587, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.009587, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_test_mode", "value": "PerformanceOnly", "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_offline_expected_qps", "value": 4040, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 306, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_min_query_count", "value": 10833, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.014376, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_test_mode", "value": "PerformanceOnly", "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_samples_per_query", "value": 2666400, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_target_qps", "value": 4040, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_min_sample_count", "value": 2666400, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.014537, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "generic_message", "value": "Starting performance mode", "time_ms": 0.019185, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 841, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.127277, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 119.352302, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "generated_samples_per_query", "value": 2666400, "time_ms": 119.352302, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "generated_query_duration", "value": 247524, "time_ms": 119.352302, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "generic_message", "value": "Ending naturally: Minimum query count and test duration met.", "time_ms": 582.179297, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/issue_query_controller.cc", "line_no": 482, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 646478.496511, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 646478.496511, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 646478.496511, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 646478.496511, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 646478.496511, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 646478.496511, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "power_begin", "value": "12-25-2024 06:49:54.870", "time_ms": 646478.496972, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "power_end", "value": "12-25-2024 07:00:41.222", "time_ms": 646478.496972, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "result_validity", "value": "VALID", "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 644, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "result_min_duration_met", "value": true, "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 649, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "result_min_queries_met", "value": true, "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 650, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "early_stopping_met", "value": true, "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 651, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "early_stopping_result", "value": "", "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 669, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "result_query_count", "value": 1, "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 679, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "result_samples_per_second", "value": 4125.31, "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 735, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "result_min_latency_ns", "value": 683962619, "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 741, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "result_max_latency_ns", "value": 646351685683, "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 742, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "result_mean_latency_ns", "value": 391912634857, "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 743, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "result_50.00_percentile_latency_ns", "value": 417296842561, "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "result_90.00_percentile_latency_ns", "value": 616408103246, "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "result_95.00_percentile_latency_ns", "value": 634092175919, "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "result_97.00_percentile_latency_ns", "value": 639925713545, "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "result_99.00_percentile_latency_ns", "value": 644597785494, "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18133, "tid": 18133}} +:::MLLOG {"key": "result_99.90_percentile_latency_ns", "value": 646210541816, "time_ms": 646496.759021, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 18133, "tid": 18133}} diff --git a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_summary.txt b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_summary.txt index 77f8b0d6..36379d95 100644 --- a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_summary.txt +++ b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/offline/performance/run_1/mlperf_log_summary.txt @@ -4,7 +4,7 @@ MLPerf Results Summary SUT name : BERT SERVER Scenario : Offline Mode : PerformanceOnly -Samples per second: 4119.08 +Samples per second: 4125.31 Result is : VALID Min duration satisfied : Yes Min queries satisfied : Yes @@ -13,15 +13,15 @@ Result is : VALID ================================================ Additional Stats ================================================ -Min latency (ns) : 677666862 -Max latency (ns) : 647328935877 -Mean latency (ns) : 392402112422 -50.00 percentile latency (ns) : 417797635309 -90.00 percentile latency (ns) : 617279379522 -95.00 percentile latency (ns) : 635020341501 -97.00 percentile latency (ns) : 640878535042 -99.00 percentile latency (ns) : 645568927895 -99.90 percentile latency (ns) : 647187794972 +Min latency (ns) : 683962619 +Max latency (ns) : 646351685683 +Mean latency (ns) : 391912634857 +50.00 percentile latency (ns) : 417296842561 +90.00 percentile latency (ns) : 616408103246 +95.00 percentile latency (ns) : 634092175919 +97.00 percentile latency (ns) : 639925713545 +99.00 percentile latency (ns) : 644597785494 +99.90 percentile latency (ns) : 646210541816 ================================================ Test Parameters Used diff --git a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/accuracy.txt b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/accuracy.txt index 56c04e3d..46eb1c2a 100644 --- a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/accuracy.txt +++ b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/accuracy.txt @@ -1,9 +1,9 @@ {"exact_match": 82.81929990539263, "f1": 90.26682135974633} Reading examples... -No cached features at '/home/cmuser/CM/repos/local/cache/adc9dc0382544ffd/inference/language/bert/eval_features.pickle'... converting from examples... +No cached features at '/home/cmuser/CM/repos/local/cache/e824f0bb4919400f/inference/language/bert/eval_features.pickle'... converting from examples... Creating tokenizer... Converting examples to features... -Caching features at '/home/cmuser/CM/repos/local/cache/adc9dc0382544ffd/inference/language/bert/eval_features.pickle'... +Caching features at '/home/cmuser/CM/repos/local/cache/e824f0bb4919400f/inference/language/bert/eval_features.pickle'... Loading LoadGen logs... Post-processing predictions... Writing predictions to: /cm-mount/home/arjun/gh_action_results/valid_results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/predictions.json diff --git a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_detail.txt b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_detail.txt index 379e8312..1eadfeb2 100644 --- a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_detail.txt +++ b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/accuracy/mlperf_log_detail.txt @@ -1,69 +1,69 @@ -:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.007033, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-23T01:23:31.577600", "time_ms": 0.007033, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-23T09:23:31.577604", "time_ms": 0.007033, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.007033, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.007033, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.007033, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.007033, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "test_datetime", "value": "2024-12-23T01:36:09Z", "time_ms": 0.013826, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.013826, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "get_sut_name_duration_ns", "value": 341, "time_ms": 0.013826, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.013826, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.013826, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.013826, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_scenario", "value": "SingleStream", "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_test_mode", "value": "AccuracyOnly", "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_single_stream_expected_latency_ns", "value": 374164, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 274, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_single_stream_target_latency_percentile", "value": 0.9, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 276, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_max_duration_ms", "value": 660000, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_min_query_count", "value": 100, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.018004, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_scenario", "value": "SingleStream", "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_test_mode", "value": "AccuracyOnly", "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_samples_per_query", "value": 1, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_target_qps", "value": 2672.63, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.9, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_target_duration_ms", "value": 600000, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_max_duration_ms", "value": 660000, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_min_query_count", "value": 100, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_min_sample_count", "value": 100, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.018134, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "generic_message", "value": "Starting accuracy mode", "time_ms": 0.019006, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1090, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.114554, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "generated_query_count", "value": 10833, "time_ms": 41.128295, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "generated_samples_per_query", "value": 1, "time_ms": 41.128295, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "generated_query_duration", "value": 4053307779, "time_ms": 41.128295, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 10228.439435, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 10228.439435, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 10228.439435, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 10228.439435, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 10228.439435, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 10228.439435, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "power_begin", "value": "12-23-2024 09:36:09.596", "time_ms": 10228.439716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 13610, "tid": 13610}} -:::MLLOG {"key": "power_end", "value": "12-23-2024 09:36:19.774", "time_ms": 10228.439716, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 13610, "tid": 13610}} +:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.003597, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-24T22:22:34.978859", "time_ms": 0.003597, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-25T06:22:34.978862", "time_ms": 0.003597, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.003597, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.003597, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.003597, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.003597, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "test_datetime", "value": "2024-12-24T22:35:12Z", "time_ms": 0.011592, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.011592, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "get_sut_name_duration_ns", "value": 301, "time_ms": 0.011592, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.011592, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.011592, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.011592, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_scenario", "value": "SingleStream", "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_test_mode", "value": "AccuracyOnly", "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_single_stream_expected_latency_ns", "value": 372405, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 274, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_single_stream_target_latency_percentile", "value": 0.9, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 276, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_max_duration_ms", "value": 660000, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_min_query_count", "value": 100, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.016631, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_scenario", "value": "SingleStream", "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_test_mode", "value": "AccuracyOnly", "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_samples_per_query", "value": 1, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_target_qps", "value": 2685.25, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.9, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_target_duration_ms", "value": 600000, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_max_duration_ms", "value": 660000, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_min_query_count", "value": 100, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_min_sample_count", "value": 100, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.016711, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "generic_message", "value": "Starting accuracy mode", "time_ms": 0.017603, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1090, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.118451, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "generated_query_count", "value": 10833, "time_ms": 41.390764, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "generated_samples_per_query", "value": 1, "time_ms": 41.390764, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "generated_query_duration", "value": 4034252532, "time_ms": 41.390764, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 10180.095628, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 10180.095628, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 10180.095628, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 10180.095628, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 10180.095628, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 10180.095628, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "power_begin", "value": "12-25-2024 06:35:12.534", "time_ms": 10180.095928, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 13613, "tid": 13613}} +:::MLLOG {"key": "power_end", "value": "12-25-2024 06:35:22.664", "time_ms": 10180.095928, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 13613, "tid": 13613}} diff --git a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_detail.txt b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_detail.txt index c98ae287..2d5472de 100644 --- a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_detail.txt +++ b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_detail.txt @@ -1,89 +1,89 @@ -:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.009157, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-23T01:23:31.577600", "time_ms": 0.009157, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-23T09:23:31.577604", "time_ms": 0.009157, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.009157, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.009157, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.009157, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.009157, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "test_datetime", "value": "2024-12-23T01:25:28Z", "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "get_sut_name_duration_ns", "value": 240, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.016411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_scenario", "value": "SingleStream", "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_test_mode", "value": "PerformanceOnly", "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_single_stream_expected_latency_ns", "value": 500000, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 274, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_single_stream_target_latency_percentile", "value": 0.9, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 276, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_max_duration_ms", "value": 660000, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_min_query_count", "value": 100, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.021120, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_scenario", "value": "SingleStream", "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_test_mode", "value": "PerformanceOnly", "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_samples_per_query", "value": 1, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_target_qps", "value": 2000, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.9, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_target_duration_ms", "value": 600000, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_max_duration_ms", "value": 660000, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_min_query_count", "value": 100, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_min_sample_count", "value": 100, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.021280, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "generic_message", "value": "Starting performance mode", "time_ms": 0.022582, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 841, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.127618, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "generated_query_count", "value": 2400001, "time_ms": 977.450597, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "generated_samples_per_query", "value": 1, "time_ms": 977.450597, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "generated_query_duration", "value": 1200000500000, "time_ms": 977.450597, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "generic_message", "value": "Ending naturally: Minimum query count and test duration met.", "time_ms": 600977.579346, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/issue_query_controller.cc", "line_no": 482, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 600984.804727, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 600984.804727, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 600984.804727, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 600984.804727, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 600984.804727, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 600984.804727, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "power_begin", "value": "12-23-2024 09:25:29.175", "time_ms": 600984.805008, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "power_end", "value": "12-23-2024 09:35:29.176", "time_ms": 600984.805008, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_validity", "value": "VALID", "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 644, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_min_duration_met", "value": true, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 649, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_min_queries_met", "value": true, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 650, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "early_stopping_met", "value": true, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 651, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "early_stopping_result", "value": " * Processed at least 64 queries (637969). * Would discard 63238 highest latency queries. * Early stopping 90th percentile estimate: 1016588 * Early stopping 99th percentile estimate: 1190854", "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 669, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_query_count", "value": 637969, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 679, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_qps_with_loadgen_overhead", "value": 1063.28, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 702, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_qps_without_loadgen_overhead", "value": 1069.05, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 703, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "early_stopping_latency_ss", "value": 1016588, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 704, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "early_stopping_latency_ms", "value": 1190854, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 706, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_min_latency_ns", "value": 872179, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 741, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_max_latency_ns", "value": 1540105, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 742, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_mean_latency_ns", "value": 935409, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 743, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_50.00_percentile_latency_ns", "value": 913296, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_90.00_percentile_latency_ns", "value": 1013732, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_95.00_percentile_latency_ns", "value": 1075228, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_97.00_percentile_latency_ns", "value": 1174433, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_99.00_percentile_latency_ns", "value": 1190824, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10478, "tid": 10478}} -:::MLLOG {"key": "result_99.90_percentile_latency_ns", "value": 1193097, "time_ms": 601267.694138, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10478, "tid": 10478}} +:::MLLOG {"key": "loadgen_version", "value": "4.0 @ 486a629ea4", "time_ms": 0.009598, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 53, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-12-24T22:22:34.978859", "time_ms": 0.009598, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 55, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-12-25T06:22:34.978862", "time_ms": 0.009598, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 56, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-02-15T15:46:50-05:00", "time_ms": 0.009598, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 57, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "loadgen_git_log_message", "value": "486a629ea4d5c5150f452d0b0a196bf71fd2021e Hotfix: DLRMv2 Audit Test01 fallback failure (#1626)\nf9a643c0a0e920588da1b51a1d822e1071a9dbec Add number of tokens to offline (#1623)\nf06b920eac2170df38b5502c7443f50c9afa6045 Bugfix: equal-issue mode on offline causing accuracy run to fail (3D-UNet) (#1624)\n0ed5190e82e1ca1cf95626e50e61cb6678c7f2ff Fix submission checker and TEST06 for Llama2 (#1616)\n817dd961221d3ca284b024368130bd5dc5e4b54a Turn equal issue mode off for TEST06 (#1615)\n840435a1abe8be823e0a5a18ad3173bc520aa24f Update README.md with CM commands to download stable-diffusion, gptj and dlrmv2 models (#1604)\n3190d09a148bc1d72c9d946e4cf3b54db0d0e60a Typo fix in README.md (#1588)\nb0777f0bd3b7e5cd6a51d670d8b4df656cf6fb29 Update README.md - remove unwanted lines in CM commands (#1601)\nd7dba0826ac96a0e8bcae525086961e47c409a38 Remove loadgen warnings (#1608)\nd45a66c33bb0b6548d6172b38645bf94ffe24b39 Add upper limit to tokens per sample (#1612)\n44285d9a44041c3d4f7df0acf1e72c375e9fcbd2 Set completed samples per second as llama metric (#1613)\n357ccefca4db55ff373e029d5d1e3e6246bde5d4 Enable equal issue mode for LLM benchmarks (#1610)\n104d85531af7ae2a07ded36dd6e8c7f2c7746d06 Add sample length check to test06 (#1603)\n473053fc9f1b8c5edf60e11ec74607126938fbc4 Fix loadgen token metrics latency constrains (#1596)\ncc3daae3c7f18d7eedc5ffbc271d324e5aca6233 #1598: fix token and sample logging for Llama2 when accuracy_log_sampling_target is enabled (#1599)\n3a902e598e5a17013c26fea975c0a60e6d7feae1 Add support to dump 10 compliance images during accuracy run for SDXL (#1591)", "time_ms": 0.009598, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 58, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.009598, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 60, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"05d4a62d355a90363847358e58afeee4a54ad838","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"1dc0865130371c6a8ddb97cd8039244d74d6fe46","/demos/py_demo_multi_stream.py":"6ccf7dcbe08f8382ed2d3618370c2b21e205ca13","/demos/py_demo_offline.py":"6c3075538e0a3df5e93edee917f4537099759e11","/demos/py_demo_server.py":"40e37ea121ac08b21ed6af41aca8eb5eda641b89","/demos/py_demo_single_stream.py":"4ee5f048fb0dffc415342a2bc2049fb155bd2894","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"974adb632af4b76a1c72b914cc2cf9b3cbc66aa3","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"a170f991de69f8a5e45616e871b39a30ffed5598","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"e3e707e1191478332c00f1b6ff806f0ef98e0f65","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"aaf3803e700f3431a5ada72559eef5cce996ea64","/test_settings_internal.cc":"42aa46b17ac3fb2748cd5cde60fa8742e916f3c8","/test_settings_internal.h":"73b11c02228f3afb4c924db7a86fd8a2b5b9e483","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"4c9d98360b507d5673049ed70154fcfb2934f8a6"}, "time_ms": 0.009598, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/version.cc", "line_no": 67, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "test_datetime", "value": "2024-12-24T22:24:31Z", "time_ms": 0.017613, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1198, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "sut_name", "value": "BERT SERVER", "time_ms": 0.017613, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1199, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "get_sut_name_duration_ns", "value": 241, "time_ms": 0.017613, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1200, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "qsl_name", "value": "BERT QSL", "time_ms": 0.017613, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1201, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.017613, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1202, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.017613, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 1203, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_scenario", "value": "SingleStream", "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 268, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_test_mode", "value": "PerformanceOnly", "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 269, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_single_stream_expected_latency_ns", "value": 500000, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 274, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_single_stream_target_latency_percentile", "value": 0.9, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 276, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 312, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_max_duration_ms", "value": 660000, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 313, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_min_query_count", "value": 100, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 314, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 315, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 316, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 317, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 319, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 320, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 322, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 324, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 326, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 327, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 329, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 331, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.022622, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 333, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_scenario", "value": "SingleStream", "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 409, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_test_mode", "value": "PerformanceOnly", "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 410, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_samples_per_query", "value": 1, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 412, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_target_qps", "value": 2000, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 413, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 414, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.9, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 415, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 417, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_target_duration_ms", "value": 600000, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 418, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 420, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_max_duration_ms", "value": 660000, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 421, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_min_query_count", "value": 100, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 422, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 423, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_min_sample_count", "value": 100, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 424, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 425, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 426, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 428, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 429, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 431, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 433, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 435, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 436, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 438, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 440, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.022792, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/test_settings_internal.cc", "line_no": 442, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "generic_message", "value": "Starting performance mode", "time_ms": 0.023884, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 841, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "loaded_qsl_set", "value": [4117,9486,2947,9270,8227,4487,9799,4629,8571,8839,7101,2191,3315,6375,207,10454,3122,9490,4083,6119,1890,7657,9347,344,201,4232,5741,5454,9533,7842,1596,5330,10615,8827,8094,6714,9169,977,6619,9924,4982,4894,6544,10118,783,766,2122,7163,396,4200,9864,4254,8440,1758,10722,2829,2788,9906,2719,5613,8726,7847,440,6103,8567,7497,4793,1184,8675,10587,8270,7074,7408,8740,9265,4855,1199,220,6556,6595,1226,6695,3066,2143,6264,9806,7122,5007,10075,2370,2185,7254,77,2806,6243,6687,1156,1802,7302,391,3348,8159,9708,251,3595,3947,2302,4690,122,3116,1300,3768,6388,2986,728,5127,3904,6807,6259,2218,7555,8821,3580,7804,1321,8460,5419,6542,7833,3365,2290,6346,7952,918,707,118,10469,9023,1304,5220,1891,6878,9524,6522,7867,4037,7698,8507,9634,2145,10057,6263,7486,7557,2401,6082,3434,5927,8708,10081,8114,1349,7742,2742,7157,5705,4638,1764,3651,920,4016,2744,5859,6740,5380,9649,2,5245,5230,6421,4071,8909,6319,288,9346,9671,9646,9360,4702,5044,9157,6296,8318,8971,8184,7702,10475,7318,4024,5530,3991,1171,1909,6230,8461,8099,10563,5398,10760,10693,3688,8170,4952,6827,5893,7767,2656,10032,9178,10480,7195,9359,10608,10261,8975,5432,363,3346,3049,3360,4432,9736,473,7587,9416,1021,6596,10352,2666,2325,973,2071,7368,4267,867,1040,5796,3491,8237,9597,3489,10568,3777,7899,856,4818,2858,4662,777,10776,10026,6065,6481,2590,6780,10553,8789,6032,930,1049,10178,6187,585,3359,5832,2014,480,5866,5017,5141,1525,8271,8132,1642,6270,2265,140,1174,4641,8017,8844,6569,5575,7081,3779,3845,467,9761,3314,8653,4250,7578,8907,664,217,8419,8757,683,4565,1638,5216,2004,6679,4485,7683,5180,7422,6070,6283,2909,6281,8933,10639,4340,3079,3513,10552,2291,7487,2680,8807,8370,6127,7434,1822,908,8101,451,10316,8282,10724,885,1834,1556,6121,1468,409,1908,10539,7238,2891,10312,9435,2178,10051,4522,9650,615,7193,7796,2008,6011,4347,435,6881,1396,844,971,4216,5001,7886,6875,1348,4300,9348,2709,10580,9993,6705,8784,10821,9232,7724,6581,9764,2913,10125,6914,7693,4568,10526,10231,8502,7825,1783,795,6828,8006,4595,3892,3075,1278,10050,8235,4679,9971,75,2164,1175,3684,9869,8414,9016,765,2721,5786,10703,1326,7463,237,836,3912,5482,7835,4374,7661,4070,7754,7180,9945,10415,3638,4208,939,1097,7628,5725,3723,4547,1994,5232,5279,4049,7309,5963,10478,9834,7758,9463,3666,4663,801,4030,3010,8236,9237,2470,3382,6336,7570,190,4552,4113,2229,9018,7038,5021,8701,8268,4490,1373,6092,10181,10445,3056,4333,10394,1640,2328,3349,10069,10683,6439,3191,2544,4760,1739,8900,3,258,5416,4671,2041,9375,2661,6376,2306,5897,6373,2125,1940,5109,4256,6203,6411,9311,8029,2200,6301,1734,6176,6443,3007,9583,1841,4627,2523,4306,2801,5208,8607,4961,1467,10700,4031,2181,9958,8194,10295,9043,5366,306,9056,162,3196,10759,4349,5401,7030,6184,9216,2069,6357,5640,3015,1752,406,10049,3983,791,5610,5944,304,6380,1937,7862,9366,9365,1607,9252,2698,10350,5221,4847,10707,3023,1615,7017,5990,4045,7231,1202,7182,5172,3176,9612,6693,1013,10334,1805,8730,8841,7372,10094,6010,3216,8177,7406,9472,9608,2533,7691,3959,1623,5144,1831,751,4912,2152,8144,241,4187,10001,1924,8402,7877,8928,7294,4915,3876,3242,8996,7153,1000,10658,3946,284,5088,959,10661,1018,8931,546,4262,6489,3786,5278,4308,8178,7638,5211,7142,5771,634,8273,263,2548,4674,9910,624,3290,5925,1524,2642,7198,10345,3539,6719,10735,2006,10233,3816,9578,8624,6769,9661,6468,8952,9702,8670,5175,3449,1773,6130,636,1622,1696,1414,6523,2898,10650,4420,1280,2538,7880,3743,1227,2757,6131,3371,7445,1680,2901,36,6868,10811,7140,1900,10398,3724,8765,8278,7379,6040,7914,1580,3535,8033,7536,6480,3861,2437,5082,2881,10596,4902,8162,9637,4886,411,7189,1789,6704,9176,10045,4107,6333,4583,3923,4843,9935,9955,9297,3222,8257,6365,3095,1995,7924,5428,8640,6628,4819,7152,5016,7932,3226,1267,1225,9025,7878,6321,2373,3801,1399,9063,6506,9436,5781,7500,423,3588,7110,4607,3476,2477,3635,2425,9394,10449,4054,8148,5510,2236,9543,10499,6153,780,8692,8105,10268,172,4128,841,8925,5453,8561,2135,2976,2845,2209,7418,10817,5620,7010,9960,474,6545,8223,9713,205,10548,905,2396,2914,5591,2887,3379,4865,3304,5849,1309,10171,608,7314,6228,6961,7068,6943,4613,1317,121,8611,5902,5827,8042,1139,3121,223,2984,2897,3144,5379,10590,6433,3871,5475,43,1026,1192,1599,5729,6194,5838,1254,2054,2706,7925,6060,2177,10647,7727,5681,10000,3549,10814,4433,2100,6869,6593,9404,5534,5801,10321,9145,8363,7451,9072,4600,5096,10372,3022,679,5631,2552,723,4668,9613,10153,7423,9114,2973,3071,1413,4956,2838,5084,6777,5196,539,5036,382,7480,9327,323,4766,2273,2684,8182,10134,9203,2667,9185,2658,9087,6286,4162,6620,10472,1095,6759,7216,8191,9738,9548,5029,10326,5478,4089,8710,5439,6126,6160,4828,1006,1677,3546,6930,425,2645,4573,2770,10248,358,4975,10603,8129,8614,10180,7902,4303,2447,3427,753,8920,9449,256,6859,5711,5246,623,370,10767,2563,8300,9591,9991,5126,9568,2647,484,8644,3654,9507,8632,10076,9813,10638,4326,3862,569,7020,4213,5628,9341,839,3341,8347,5163,5563,7339,6928,1197,9158,3099,6120,1483,9545,4608,4215,2761,1259,6802,4364,4001,1606,4222,4752,3057,10022,1108,10592,4398,10376,9888,10033,4273,1238,3428,6553,6145,5731,7273,604,2436,3170,8673,5901,10074,4513,8305,8697,7476,5015,2669,3100,6344,9338,10383,4972,8727,1363,3223,10495,897,1698,9553,5662,53,9328,3733,9932,7102,2212,7382,7159,4556,10748,9294,5895,2262,10512,5913,8314,10624,2530,7560,10464,2443,8787,3800,1510,2027,9795,9670,283,2351,4260,2577,1518,906,3377,4764,2113,10699,3948,8465,9264,9267,5600,2924,3783,9209,7966,6464,720,6973,155,6227,7994,2636,535,8846,2931,2046,4953,10058,8228,5200,2211,4234,2207,8121,2007,7268,3212,7772,6496,10201,2762,7534,5539,10555,10496,628,3653,702,8683,10325,8660,8411,9387,326,8842,837,9688,3855,7053,1826,7944,1053,3193,456,361,3206,10315,2059,6612,6173,4562,9003,8034,9162,377,7551,10769,5354,6299,5997,2080,6288,6068,653,9937,10825,8833,3247,3058,4923,9104,7983,3305,9322,8339,7711,131,7328,5212,8623,10761,9718,5402,948,10640,8659,6479,2768,4616,2223,10625,4252,9980,9807,5318,6188,159,10655,3565,1478,6860,3521,4646,9880,1608,10427,9804,2092,4897,10238,6020,7642,8091,7789,4058,6681,4173,7918,5523,2444,9898,7161,7736,4667,2603,9370,7403,10797,1857,3276,2276,3445,8197,991,1863,4266,2170,4331,3296,6658,1308,7542,8458,4533,10177,6873,4190,815,2726,5589,2846,269,2634,1560,7067,7554,3102,4908,10741,10086,5286,8734,3824,1744,4111,3264,7494,3825,8834,6139,8432,147,1589,1871,941,2911,6018,7906,2874,3713,9523,4023,7522,6116,6864,5698,5464,3924,4590,7345,7395,4790,10540,1020,5309,5950,2956,4983,6163,9194,3848,3612,10525,9069,5253,7232,6876,5003,5067,2350,10375,10523,850,6399,10002,9689,7255,1016,3707,3384,2137,4242,4077,5170,8388,2387,10577,2482,3735,4313,8862,2029,8452,10528,6058,6871,5233,1195,3301,8738,3194,2638,2488,5009,6981,7935,10684,10631,1043,1962,3541,4973,9581,4285,9118,9770,8279,1316,6284,1161,877,7894,3981,2423,6872,5033,1798,6181,1255,7517,4067,3319,7457,5164,3977,1719,4436,9317,4765,4137,6515,10025,4628,3745,7739,9445,7016,1980,4481,8580,175,10704,10653,245,8163,3834,7006,9135,1950,4401,2289,1412,7325,4821,6917,1361,7569,6782,2474,8075,5757,1034,3353,8570,2678,4922,4356,6831,3401,10393,10508,5161,2635,1472,3331,3048,9526,5383,5670,7508,5113,10148,2912,8547,1183,5496,5139,1153,6897,7997,6625,9819,5766,7460,10124,6637,327,3466,1848,2455,5939,2282,6371,4812,6721,5314,4780,3112,5154,250,7103,7158,463,6920,8368,6024,6877,342,6799,437,2876,9690,4889,5129,2631,3366,8380,1528,10135,584,6292,9180,5863,10031,2432,980,7211,3117,6909,1248,3761,3601,4976,9420,10602,178,10755,1150,2009,6653,7378,4422,8445,1978,2390,5417,4353,6452,2002,10595,1306,2817,8735,8424,8976,4396,4919,9488,534,4478,8211,8264,385,8288,3063,9825,6996,9726,7239,668,2896,6212,5761,6271,618,8216,6838,195,216,465,8870,6774,8556,3392,786,2791,1096,10565,9399,7603,1723,4133,6106,5491,10443,335,8387,5006,8131,966,7321,4664,5361,8778,8085,10370,512,4408,9227,4676,927,5805,10457,5444,5333,2951,6804,7955,888,3706,8788,2895,1902,9874,7221,5222,10628,6300,3522,65,10112,7284,6826,9762,10226,8642,8030,609,3812,9914,8276,5946,4130,767,4876,3831,9441,5515,7928,4838,4550,78,6682,63,600,3798,8226,3577,8618,8919,4716,10385,5560,7695,9677,9470,6351,10585,6031,2941,2010,7623,1735,6574,1513,2774,1490,2303,5429,141,1915,7737,9086,2205,32,7165,4564,3715,8209,8559,4979,4985,7143,1656,6789,8025,7208,6845,503,3687,7028,5471,9975,5664,3404,9068,5248,2433,322,10390,3840,8058,1297,3551,3819,3342,7265,3266,8914,4394,3583,9786,9198,5609,1444,8947,3060,10252,5594,1291,7319,9559,3166,4669,9592,661,6602,4789,9415,2424,4265,7098,1876,7535,1203,10053,9809,3104,1941,2705,6635,3148,3598,1502,7383,9073,10425,6590,5162,3529,7306,10800,6393,8751,387,10172,2692,5719,2741,7855,4845,7039,2459,6901,6624,3554,4237,3867,10566,2116,531,5943,6690,792,8441,6260,682,3995,6154,4165,9066,2717,1842,5340,8261,8539,1074,4259,701,9500,8648,513,3417,2155,9041,116,4010,1218,7514,3738,10176,8592,4659,8491,291,10417,8338,7244,862,5005,4735,1362,964,9714,10691,9829,10779,351,4930,16,3561,5388,5985,5239,6821,4320,10682,3154,4367,8961,50,8733,4288,9893,824,2504,404,10529,6560,1489,8125,48,76,909,2237,2691,2416,1987,7699,4599,5585,10503,7807,3639,3994,5708,7135,4625,4516,5540,2916,5528,5975,7413,8915,1347,10065,706,5304,3436,6607,8262,5645,3625,1376,4476,2934,1592,1935,3958,5821,2456,3362,5367,4151,981,9573,10610,9207,6190,5275,4631,1708,1921,9287,699,1447,1636,696,3740,4597,3373,9444,1221,3780,8124,5381,7648,3536,10571,654,9064,9044,3503,9854,9626,3378,10675,10629,6080,4445,9605,2646,6982,5074,3618,52,6561,6185,3026,5968,9742,989,5867,9190,1178,2689,2001,6149,8334,2049,3180,5759,111,4691,2650,8777,3149,804,1868,5328,840,8687,1382,57,2419,808,10355,5588,5215,1403,5701,9,7853,8520,8669,8019,4006,9097,226,2827,1135,1901,979,819,2561,7049,504,4391,4400,6739,1402,5264,2230,10666,2566,7430,8459,10720,8794,10310,10174,443,6349,7310,533,7609,6200,4108,613,7558,7709,1553,5076,10645,300,2842,6261,2324,588,2193,2945,7510,7396,3094,1889,10781,3791,554,9284,9148,3519,9995,3209,5810,8322,9519,2560,3073,947,2781,7004,5548,7116,10152,10121,552,1887,2952,8739,9225,5819,3712,6550,5446,574,3090,8327,428,1266,8433,8383,4645,8754,6898,2652,6822,1455,2176,1519,8566,10303,6049,10068,4977,3894,7561,7488,5536,7224,7793,1989,3985,8695,3424,7035,7612,9938,6775,39,1850,7803,9174,3031,6441,2974,1603,7941,7223,7923,2831,2977,3080,2075,9236,312,8565,4634,8814,1558,10484,1272,7389,8872,10418,4755,7869,2610,10633,5046,6960,4193,7852,1754,2878,1936,3784,10314,4025,4805,4997,10007,9863,7155,10101,1342,9395,10291,9385,10218,799,1123,5843,2562,4905,1377,9226,5937,1986,7591,9962,1189,8902,619,9067,3458,7586,7210,328,8506,3000,1004,9212,4591,8020,8887,4012,592,101,3415,2527,72,7452,9400,7809,1561,7032,4840,7008,1056,8,5752,9588,3101,3106,3609,4503,8312,6536,3446,8745,1437,7547,10287,4816,1795,8004,5837,1042,9329,9008,9433,2047,1087,3421,4324,10588,6699,10747,7765,9719,10438,887,9062,6820,2946,4741,527,3043,1648,1133,2918,882,684,4939,4386,8641,8154,1323,6436,7873,10752,3686,10680,807,9886,549,3230,6147,294,7120,354,8171,8493,2663,6863,7489,6037,410,9913,124,6312,541,3159,10304,2117,735,9882,199,4315,9250,4055,3739,3826,9558,10782,3699,542,6645,7593,5123,6413,7532,8805,1517,8514,3396,9239,8883,6748,6002,5319,8690,6308,1691,4655,589,10278,5680,9489,6806,10465,4251,10142,538,5740,194,9167,9974,2107,1480,8439,5722,5674,5712,7528,3385,1906,10567,9057,340,5707,9308,6207,3078,2818,7518,6794,9555,9983,1676,1678,6079,2038,1670,2608,2335,10021,10243,7257,9405,8165,2242,1247,8620,7479,1305,8818,8790,7865,4998,5653,2097,261,5270,2140,2105,789,6984,7501,8748,9538,8167,7566,3307,8768,5195,3932,1130,3930,6535,3665,6818,870,6657,7938,8656,2975,8108,4377,1332,1792,10027,3395,5078,10380,724,6034,3423,4788,4511,9362,2462,7760,7023,2497,9802,5504,6817,4263,3361,8164,8418,2478,2287,3457,1264,9796,7218,6591,6401,1683,3534,7474,5945,9722,7233,3719,7072,9915,42,6386,1481,3426,8808,4075,9665,132,3679,1418,6327,4854,5667,1228,5621,10063,9685,9641,3792,8509,10725,1726,3510,6733,5311,6890,1127,1372,4557,9076,4493,1319,4639,6503,5373,319,6252,5205,2568,3262,5611,3692,8016,2058,10708,1990,10246,5149,6730,4745,9617,4496,8093,3764,4587,9046,8240,5654,7484,9304,9356,9122,9012,2409,2879,1728,8443,6110,2374,2695,6156,6123,7266,9525,1770,9102,10660,6745,5128,5806,10298,1081,5852,10365,5171,3316,1949,2796,5938,2888,505,9040,10137,2063,8296,703,7982,5184,5822,6713,8356,1212,10214,1643,1344,1740,6170,2765,8513,7491,313,7468,5830,5188,7420,8246,7931,5592,2300,5803,1333,1904,10524,2557,1320,9801,3887,3372,4648,8123,10207,10570,2782,9208,3493,8993,4688,9635,2367,890,4834,4750,2186,4059,9823,383,5746,6601,6995,3696,10626,6202,4034,2588,1415,3128,5342,9471,8634,6712,7283,5689,7596,5728,9160,6465,1823,8542,2381,129,3703,5297,5845,10090,9892,8109,10676,1417,6171,9518,2483,5993,6066,10657,4131,7317,4665,10546,7047,6337,6183,3537,253,7720,5316,4382,2720,8286,7478,3902,7237,3088,5066,2131,2830,10694,3644,1407,5650,5733,7970,8970,2283,8951,9288,6643,10349,4772,6849,3295,4178,1354,8522,10736,1720,4463,10409,7172,6220,1998,6367,7464,1973,6738,4238,9065,9537,2528,6911,4247,8425,183,7919,4791,5494,3274,6583,4995,7128,869,5889,7655,5199,1555,6754,10636,3874,5094,545,3805,1611,9596,2168,6048,7912,1253,3675,10190,6051,6887,5414,2919,2397,8982,2189,5410,2589,6458,7272,8280,6078,2349,9268,9229,8372,9571,2129,9070,7371,5048,8593,4925,5490,1503,6584,4713,6442,3119,687,3013,10757,8401,6102,6842,6324,7114,10127,9879,9783,4963,1626,2785,3640,9163,4835,1390,5087,8073,10203,3068,5747,1958,49,4916,620,1548,3018,9306,184,10272,3878,3053,2239,10187,1477,2902,9456,7260,5051,7639,5386,4477,80,1151,7277,5579,5072,5596,2204,10199,8267,2624,5110,6272,1330,10059,757,9947,1357,7745,6159,9580,6926,1486,10062,10488,8712,6582,7733,8905,3935,8265,7334,5495,9170,1657,4880,1542,9546,5577,3169,2880,8287,2763,1465,1041,10545,7951,2814,10729,1234,7881,6985,6670,5055,4329,9749,4744,7621,9946,665,475,6655,6976,1327,9703,33,9513,9857,3248,2158,7640,3830,6910,6599,10785,3908,9303,6403,858,2074,3941,4068,1488,4929,9954,4872,10514,7838,4677,1877,7014,8113,6883,1511,7174,381,1523,10229,9627,6966,9767,4283,1167,4168,6977,3543,6953,2737,4698,7908,8759,2429,7706,9642,7199,2064,7087,557,8591,4609,3962,2795,8283,6449,9092,9964,4970,4529,3616,6475,7523,56,2238,6431,8256,640,3416,688,10471,8348,5541,2371,994,5570,82,6166,2259,7589,10479,5713,449,2308,130,6882,6298,6517,9870,2353,4316,5452,2792,7556,3803,822,9084,798,9477,9082,657,495,8781,10619,4328,2710,460,10133,3234,2555,6069,5255,348,5511,6934,9594,5120,6729,3956,6290,830,9948,5911,1731,6291,4993,1533,9758,5693,8152,4988,514,10270,6000,9682,6946,8344,8051,5320,8247,9020,3751,7864,10163,540,5933,3650,6107,2840,9372,3701,8955,4269,8511,1024,2165,4739,5474,10239,2466,4157,6533,10780,9408,3312,2549,7113,4480,7741,3467,5710,10381,2392,4166,7485,9902,10116,1089,5281,2157,9182,412,6056,2281,697,4383,6531,7473,9231,7411,7070,1493,3570,4122,3461,4849,5119,3174,3139,8068,7162,3184,3691,2316,18,650,2382,8612,9692,6483,2442,7718,488,6642,8681,5271,9004,3942,7986,5406,6744,9014,6975,8403,7449,8721,3240,8297,752,4471,2833,10812,202,7701,10064,9164,1565,2172,3613,6589,2843,1052,8359,1932,2389,127,5508,4782,1048,2729,6216,5692,5219,4730,8987,940,7810,3907,4399,1220,9901,1549,1242,9815,673,7459,3322,8550,6865,5229,8557,2257,898,2893,8674,2446,7645,6528,2809,4711,2106,6267,1110,6415,7998,2094,9509,6269,4475,3096,1567,7066,2464,8516,1843,9885,8995,2345,4940,1077,903,6886,347,8766,3343,8534,5871,5310,5738,10217,7539,2823,9867,8238,7093,10667,2332,5535,1419,3203,7540,2417,1619,10744,4592,8817,9219,2565,4276,9380,6329,10426,8853,8106,6765,6932,3822,5864,825,8244,9566,8333,5954,5351,2978,3376,6275,10290,1604,1067,7901,3143,7979,3630,5252,5642,6098,3077,1464,325,10481,5853,10012,4658,8315,10029,2812,4123,4967,6097,4038,1755,2873,2905,8098,6941,3681,1070,8555,4875,10340,9944,9694,2180,7441,2904,8054,2531,9330,6750,6945,10067,6012,5092,10772,73,6213,1504,6425,8835,2520,1401,10768,3368,4375,4941,685,9479,453,1268,2245,1694,4325,7705,892,10775,7450,2679,842,3996,6241,1117,7713,10343,2124,5857,5617,6594,7305,8770,5928,2197,9166,5956,9299,10095,6524,10531,4335,6622,10823,802,3120,2073,3864,3490,6957,9845,3972,6935,5543,9774,154,5250,10342,8137,8040,4224,10500,7327,5779,2755,5085,8243,4632,1952,8047,7056,6304,3241,6219,5353,10320,4559,6552,4748,9475,203,4551,8552,7187,3748,4271,3232,5903,9402,1566,6381,1569,6192,5338,2965,8023,2360,10386,6383,923,10618,9172,421,3412,7582,7685,7332,4138,2820,876,1356,1325,1012,3814,9584,3799,5041,9273,5764,9757,1655,6991,8588,9423,8001,3853,6191,8755,576,6629,4115,8878,4253,3245,1885,2505,5426,2621,260,8848,3873,2060,8352,4842,7686,5359,3600,6732,857,5014,9386,9173,8150,3065,7127,10754,2987,9907,7347,1051,6114,1609,221,6666,7851,5093,2471,2465,398,2077,6459,5799,8462,7943,10731,2227,9619,3813,9031,4753,7797,6347,3836,1806,9134,6150,5643,497,3868,6915,5071,2769,3717,8575,4426,5358,1587,7752,4942,8857,742,8464,3442,3050,9660,1165,3309,5293,3171,5099,1154,1827,7728,3140,5995,7719,601,9119,2053,10056,10487,4949,5307,7866,7076,8052,4372,6045,9586,7799,4800,4575,5458,316,5425,6361,4965,1187,10816,1907,7808,10551,4531,146,3884,8978,1271,3568,10668,179,6571,163,408,1190,10175,10716,3837,9340,303,6792,5936,5979,6330,8574,6129,5555,2385,10594,4980,10728,5842,4321,10444,427,5890,1776,4598,5235,7900,602,4763,6907,6379,4804,1960,924,6133,5932,8041,98,3622,476,2517,8161,1374,10787,556,493,5804,9109,8786,1872,3138,1337,2822,6289,7562,3005,1392,6866,2518,1140,5258,133,10331,8306,5313,9575,10717,3156,2103,8495,4799,4672,838,2249,4686,2233,4281,1475,9108,206,9453,5941,8379,5103,2868,9988,9732,7417,7071,1231,2515,9371,4498,3081,9352,6529,2313,1529,4229,771,1134,3690,9005,1128,9503,9776,2571,8369,10271,4387,3408,5533,8666,9035,8390,712,9255,6634,8299,3221,7005,6850,2854,3921,4548,3501,1593,1050,1692,3685,4879,4277,2805,8468,2867,46,1258,2272,1118,816,6796,7965,2036,1722,9448,9851,6940,6862,10538,2547,6437,5343,6727,1359,6444,6573,852,7574,10336,61,7085,6247,5134,106,9192,2369,7786,9009,670,8311,638,5288,8446,5463,296,7337,5018,3103,10423,9970,10008,944,3205,5435,5332,7921,871,967,4777,271,7214,4652,6630,8199,6526,1695,5140,6215,9277,5174,8504,5181,8193,2863,126,2377,691,9662,6495,233,565,6466,1771,5348,9411,5026,5537,4801,656,1384,8396,10541,2326,6958,675,402,10098,4203,6835,6258,6969,7950,5704,4047,10642,5038,3340,1820,6525,7354,8752,5131,3937,4369,3486,586,5305,4971,9773,5948,7618,9361,6790,8743,4309,3722,7721,3944,8836,4154,700,2935,8003,7351,5521,9464,4718,4888,5124,7841,3135,5697,4530,7276,1395,9615,4825,6224,198,7805,633,8442,1840,1261,1170,879,9725,3626,3870,5050,3901,3954,9332,2906,10073,7674,722,6843,4363,9289,5077,6598,3059,3928,1114,10034,6377,1380,4378,4418,10505,4869,7246,1292,9942,7092,9139,861,7416,2747,1245,6423,9095,9653,3130,8948,6737,2404,4026,726,6101,7726,4540,10308,4243,478,2980,3227,8985,6913,2675,4504,3753,4931,2434,6753,1045,7477,834,4181,7134,10507,9514,1747,714,7790,6755,1971,8301,6009,3584,8877,4361,2927,9027,599,8160,9059,4675,8090,10091,8192,5283,719,3145,8671,8569,6824,8639,9253,5815,10361,6988,2510,2585,8904,7405,2398,1768,7989,8696,9899,9805,2439,9412,2727,7927,6257,8746,3982,5679,2013,4579,4141,7634,7553,346,603,1522,8764,7001,3603,7775,9715,7678,8482,8630,10530,7905,9211,5308,6760,5266,7308,7377,3957,7058,6340,3602,3218,4945,239,8867,9918,8130,8000,4950,8913,275,8097,1029,4630,10774,3173,2808,3754,1591,9115,3593,755,3318,6902,5762,2104,8893,3749,10401,8946,2616,8510,8937,7975,1078,6335,6238,10733,2044,7885,4233,7426,3606,5280,9756,810,9844,9695,8053,4704,3852,1484,1031,8399,5467,7672,7590,8215,5263,8028,2657,8117,6142,6273,5561,9748,672,7229,3098,84,10405,4272,2800,6889,8792,6691,4204,5646,5345,5615,4278,5315,4486,1543,9467,5355,2797,5400,1953,8917,7806,7073,3807,7312,1406,5716,7086,1738,6847,9781,10212,9631,10442,7548,10115,4678,4097,5030,2247,3002,10293,492,10429,8944,5106,4009,10269,5760,859,1201,10771,2026,6508,2957,1780,317,6445,8139,1810,1520,8983,2354,490,7375,9866,6035,5393,4751,9810,7710,5155,6576,7367,4368,332,5870,7762,7292,10020,6076,8355,1216,2119,2362,6470,6180,500,9775,298,4319,4935,7585,9563,4182,5326,1970,9266,845,6756,2256,8541,8972,7094,2748,7353,5185,10664,10743,9565,6942,7743,9912,5135,1452,1618,7592,3844,7037,2048,946,7036,5780,3146,5973,5505,8810,2892,3682,9607,931,6852,8823,10136,10440,2615,10764,9421,7909,8964,8449,3160,2153,181,10421,3357,5638,9478,9572,5797,2020,5368,10014,2415,9659,1772,9831,7340,10777,2584,1967,7190,5789,9905,3909,5499,5879,9787,2921,2683,462,2535,10730,7492,6718,10492,7124,1148,1079,3986,6891,5696,10414,5855,5737,4214,9389,3253,1273,3869,2641,8329,7323,4120,7723,4798,1387,5285,6983,7132,8731,5299,3663,4249,3438,365,5100,8684,8801,1993,4506,1671,4566,5122,1686,3195,7916,3880,1101,9672,8036,4397,2701,1786,8120,10726,3429,5114,4588,5217,1057,2735,10794,1331,8815,2261,4703,1667,9574,10803,10054,2435,8067,4393,6362,2937,985,8750,5374,770,2463,1749,5922,1063,10335,4743,2305,10396,7546,257,3297,8553,1775,10197,7279,2690,496,3454,8616,10612,7857,7126,3107,2298,4072,8906,151,1581,8061,439,6856,7903,2142,4526,5244,9029,4714,8335,881,10363,7503,6647,935,5133,7649,10561,6108,933,2551,6870,4673,2826,10129,5906,626,7717,1082,3575,1172,9417,3456,5568,5914,5856,1851,2953,4881,5851,6239,2050,4371,6477,7366,4452,9849,2246,7930,988,1951,10789,5878,9853,2323,2850,1942,1136,7421,10230,5130,6313,6929,4846,10267,7890,4964,6829,4357,8582,649,7241,6716,8258,1176,9263,5390,6994,5921,3192,1069,10796,5908,6486,6138,8562,8578,6925,1408,7917,8045,10286,6671,2352,1312,10309,8210,800,6050,2686,6214,8456,10302,8858,3443,418,8005,1445,7115,1824,5690,2569,2917,5197,6749,8643,1537,9977,5648,3997,2600,10395,8360,2861,1014,9790,9865,4095,6124,1666,9189,5160,8186,5137,10828,6025,1847,4932,1249,1299,9530,7282,6990,9223,3720,3818,7987,3137,7670,286,4731,7654,805,9668,10018,230,2467,8638,10084,8422,3481,9811,3285,4877,9281,10307,10801,102,530,1819,4918,6546,928,2930,6678,10253,10151,4094,8430,5056,7400,1313,10461,6039,7992,6664,6502,10644,6424,3383,4302,7186,8342,3559,310,10379,9446,2277,1116,5778,6538,1963,3185,1487,3294,10362,1207,6921,8181,5236,10353,5709,7359,10011,4147,910,2476,5337,9213,9916,2696,2391,7286,8797,1796,2411,2570,7990,2889,10723,10150,171,472,6246,7280,10100,8298,4517,2775,7831,10582,2365,3183,3670,6432,9074,7993,4852,646,3645,4052,5323,5556,10807,4014,5349,7109,1296,7125,8147,10043,10085,10709,6906,2251,1058,3842,1132,10328,6005,7725,1853,1862,4528,10824,2837,9129,2214,4209,3557,8758,2605,6667,8761,7620,2156,8505,4322,5269,9818,4447,3186,4709,3920,10087,568,7985,8911,2331,6494,10654,7631,2875,5363,6151,6896,5481,9705,3286,7753,5449,6013,6874,5153,1779,1494,4722,6728,4726,8008,6316,430,4403,6359,8762,6936,8704,8677,1111,8153,8680,7700,2622,582,1462,1621,4373,5430,6218,1825,10038,5597,5578,6064,7907,7892,1469,9038,6763,7197,10543,9461,3881,1536,8992,7465,235,4240,7502,1541,1616,7207,6608,4245,3470,2961,5423,9460,3993,1979,3680,8998,6844,6370,8959,3488,3217,2604,3891,3369,6467,6320,4458,9248,1793,8676,7581,10502,730,566,3694,4699,4416,2284,8110,1933,9333,10388,9873,7454,9254,8881,775,3497,1712,7090,5158,10122,7483,10818,9973,9704,1969,4606,2682,3354,8869,1338,10215,4379,9793,694,3725,7365,4431,8302,2752,3255,5392,5468,68,10751,4871,390,457,4705,3181,4081,1491,9246,5982,2524,3721,2372,727,2163,4734,6689,10005,5237,5583,280,5207,3034,4022,5934,6326,2334,7209,2702,4643,7498,10119,5825,8962,8185,5440,2169,1803,1742,7316,5183,1177,2936,5020,6959,5063,698,8234,9307,3086,8831,919,2234,3787,5714,8263,8991,6104,8554,3350,5187,4092,8071,1898,1394,4992,5371,9200,486,7787,8939,6047,2190,4210,3142,4261,1583,1121,1927,270,4948,6204,10255,6811,315,8896,3477,3895,9606,8851,4623,4244,893,7009,10344,5404,2539,3448,8682,6105,8965,9706,2862,5375,3273,242,9218,3585,7027,1112,8232,7751,8332,9015,6507,8544,4428,884,3857,1086,8115,949,4160,4268,6867,9803,8337,6903,10165,5744,3284,5854,1785,2319,3089,8448,389,2066,7236,3351,1466,123,1956,10341,8785,1741,2310,1544,10547,526,4421,8916,7915,161,2972,5424,1301,222,4661,6438,2496,3109,8488,9354,743,9345,3615,10721,9622,51,8829,9000,4890,10182,3760,8921,1867,5754,3265,2399,6089,7313,6339,9285,212,3308,1886,10441,2341,2626,1284,8138,10424,10204,3572,3966,7296,7303,6846,1875,2758,9663,7173,2959,7261,6295,8303,9088,7669,2112,1328,809,4456,3132,2346,1241,958,339,7663,7531,2519,7509,6949,8340,9959,7063,3055,4636,4650,7680,7203,6030,2264,8979,8819,2500,7559,5606,9434,3256,10689,2321,3324,523,8078,992,2736,7513,6385,7052,5970,5483,278,6350,1730,9367,729,157,7960,4446,2110,10575,3544,10795,6554,4384,6279,6659,2623,2082,145,6372,3582,4127,1878,10826,4129,1017,1837,9788,3281,1663,1929,1597,7393,10473,5167,9599,1482,5394,4906,952,2098,3465,9240,5524,3459,1724,9454,6189,5210,8609,3447,252,20,8014,560,7346,8756,9939,6463,3810,8537,7240,6703,3435,2252,8404,4582,4011,285,3744,6435,8127,6676,3492,9245,7358,4079,3555,3329,9868,10154,1435,5189,8220,1275,3926,7007,4464,8647,10250,7756,10798,10221,8891,4723,10161,6297,9083,3765,95,3530,10346,9909,1532,3700,2541,9989,6816,10311,9640,6547,6384,2081,7112,5666,3974,10374,617,5691,6310,1702,2487,2285,4644,3336,3756,1224,4343,987,369,1093,5223,1610,9981,7227,4114,915,1186,3856,583,10701,6518,7343,860,956,3165,4762,2529,10281,1429,3413,4135,483,1449,739,4878,3697,769,5841,4621,999,10557,9149,6955,1997,1073,4172,7653,5518,1485,3287,5466,9746,7146,8953,8830,7521,8289,2198,5300,4404,1659,336,5723,2857,1499,8658,5105,6122,9601,7954,7573,5243,762,3418,6234,3161,2681,4441,4649,8345,3804,1818,3177,833,9727,10225,965,7635,3006,2772,1142,3208,1098,150,3943,4761,3460,148,4784,7496,7263,612,3524,984,1983,5502,1120,7200,4104,28,9675,9544,7936,8763,9603,4304,6014,8498,10131,1367,2580,5959,2016,1205,1351,3288,4457,10673,8116,4406,1206,7046,9177,5683,7707,3547,2045,4376,1516,2865,4622,3207,3659,9175,5121,2777,2653,8524,1157,218,6418,3321,10102,2380,7980,4617,4220,8859,2550,9473,3495,5408,10677,343,7398,380,2894,5325,10235,1329,2869,7202,10437,6613,3509,2703,6314,7226,5218,10184,10627,3526,7179,6801,6853,8560,962,10511,7729,4125,4134,4392,2766,2492,5971,6144,2358,2583,1122,9632,2992,747,5145,7409,8392,543,3133,7946,10450,754,4981,5441,2618,6462,1229,8587,6498,9512,954,5848,5688,4759,5331,356,8088,7814,4350,7795,8176,3631,2773,4999,8259,7381,744,6609,2564,5940,1594,3755,6476,6540,5057,10616,9221,621,597,7887,2606,81,3980,2220,6640,518,2659,4538,9926,1733,7948,8018,3004,7363,2021,6992,2342,8483,9373,4527,4419,10224,5816,10623,2693,2160,2963,6773,74,4684,104,4898,4809,1674,7499,10696,4695,3978,2699,5214,7315,829,10410,1145,803,5951,301,3763,4196,7746,8328,3070,7407,6055,938,10522,7977,9896,2243,7033,9814,4494,9676,6036,6100,4775,12,2195,10196,1103,5862,3333,3471,6764,10222,4174,8568,8249,8362,5492,4053,5630,7000,1966,3422,1711,7682,295,865,7361,9652,5790,5626,3821,8409,9859,6363,8860,5989,4087,6395,9848,8308,4341,7520,7616,2671,6836,4860,3014,9789,9205,1918,5053,3335,9542,320,8884,5204,1442,5558,8250,9872,5952,1424,3709,8195,6140,4057,6633,659,2383,5834,1508,4246,10010,7287,8076,4554,3599,10685,1431,4050,7264,8535,5166,8086,8133,6541,4201,3545,9923,5647,4035,7740,10183,4169,1689,2664,5770,1882,1295,6978,4921,1812,5412,5387,8668,8408,4781,3579,732,3899,9510,3082,1914,7242,1557,1912,3961,3250,1039,9611,6683,9820,7647,10220,5720,10173,10330,4909,2337,2654,7133,1896,2883,2448,10727,2793,578,3199,8572,2743,10223,10662,9378,10004,4008,9331,1675,718,4682,4274,6762,1240,3261,4046,1934,79,1988,9624,5601,6621,3153,3882,10144,7429,6430,8986,5860,3225,5873,3960,9397,7687,5080,10674,10778,6193,5531,5793,88,7583,4601,2248,5748,3393,6702,6857,6950,8635,2375,1141,4236,4145,2733,5629,401,8954,5777,6783,3337,1943,5918,7821,255,10692,1434,9058,9629,3168,360,1690,1230,9458,951,9271,6469,5967,3061,7579,1003,3108,7681,8486,6265,4409,6109,8412,5618,6487,5562,6276,5907,7575,5529,10601,1725,9096,6905,7801,8610,5277,2627,5035,9569,3157,9957,7322,1600,6157,796,4460,6627,5685,10805,394,3669,7330,2426,7744,10520,8645,8932,5209,5820,8391,4769,6132,8613,1836,3841,3041,1085,521,5186,9462,7136,6429,9951,5409,1269,8128,7516,9188,8354,10167,10216,9497,9972,7399,8910,6456,2810,10339,10819,1646,9616,2174,4175,6091,748,3661,2366,6654,587,3627,10097,8367,1109,1844,8652,1364,8151,5060,2480,7817,4284,1076,5257,4861,7355,5872,9709,1298,8145,8519,806,2427,4524,3660,6293,4444,1660,3989,9374,7192,5369,8292,3914,5608,210,1002,7154,1302,2855,2412,10211,8799,4612,9744,267,5877,7137,8426,2828,4192,2378,6236,4924,9921,2440,6378,3556,9511,1897,357,8104,2400,10609,2970,3499,1830,8321,4543,7794,2971,254,6223,27,9480,7003,8856,6954,7704,318,2407,1182,4624,644,9496,10318,7961,9257,5695,6231,8894,4186,7976,812,7331,4217,7106,8092,1126,4102,8980,10591,3734,1213,2670,9679,3172,10459,8626,6492,3323,6646,9539,355,9337,8716,3808,6165,10562,5658,818,9707,8958,2134,4866,10317,7818,274,5457,3388,4282,4330,6482,109,561,4955,4874,117,8038,8866,826,3938,7069,9357,4806,4903,8741,2688,6343,2402,9895,9561,4100,10491,2597,3533,1849,3190,8737,8967,9007,4468,1068,10277,2258,4360,59,107,1036,5769,8437,9843,9700,9127,3525,1,7297,1809,5972,2484,9840,9609,431,5800,671,3009,10159,481,5025,6254,9602,2025,1001,8679,7472,8929,3164,10786,5165,7177,9987,733,4959,10276,6199,8022,9001,10576,9036,8938,10189,9238,297,3231,9080,6512,6509,1568,997,3773,9643,1030,22,2192,9407,7798,2813,10039,2088,5173,7170,6652,8536,10436,3325,10338,9437,1977,1919,9808,2430,10240,1107,6451,2784,1817,8037,9633,143,7117,6964,721,4194,4954,9730,8203,9491,1807,5659,9116,9816,6808,1160,1358,4106,1166,7206,371,3636,7656,2148,1938,7394,3634,8204,4770,4938,3896,4577,8548,7530,6016,7461,324,10646,7307,7338,8436,4381,6615,6087,5668,942,9953,1899,5546,9406,15,9684,3552,7335,7392,450,4470,6201,4040,10641,8713,953,5675,4947,8715,3092,5422,6023,5783,3950,1423,677,4710,878,7130,10822,5512,7204,9929,7326,2019,7774,3197,10810,7732,8471,5052,10750,4482,5156,334,8526,6222,9214,6614,7258,6488,6680,8320,5259,3757,9549,9196,5715,3581,5413,8031,5224,6497,7651,1100,10110,4797,7839,3091,10123,10358,8188,2208,5447,4542,373,434,3293,4904,214,8494,5450,1721,9828,9961,8747,4365,110,4910,704,9155,3504,7235,7734,873,10448,7667,8824,3796,7971,4605,10003,917,4066,705,779,1612,8861,5260,7295,3027,8487,3823,9826,4459,8941,6715,2085,3517,4808,7075,8272,4500,9656,6927,8405,1386,4737,8475,8365,1813,10088,4996,6029,6993,3364,4491,4885,4287,3667,10170,9325,2314,1129,3911,7830,5924,2676,4934,9403,7940,1982,5054,3028,9693,8633,454,1944,8599,9833,8595,41,1400,9291,25,3594,2697,8945,8840,7290,8963,2043,8431,6618,7978,4635,1669,7879,4019,3811,8168,1791,7099,6588,9120,8107,2633,7467,2490,3905,4697,3652,6742,1718,3987,6206,8467,3716,8875,9504,7822,1874,2899,2537,1515,7730,9667,9388,8007,5294,7185,9778,9451,446,1512,4152,9894,6948,6416,8189,10042,7024,232,648,1425,9531,10391,10715,6568,6353,1441,4423,470,7962,5376,2725,4610,9079,678,24,794,963,5479,2339,5023,1627,9260,19,811,1410,2506,2086,4218,7015,9600,2022,9466,7222,7410,438,2732,4654,7252,8826,1766,3483,9081,3998,2315,2767,5503,9724,4060,4507,4857,5360,8310,10600,3211,6448,7386,3790,3872,1343,5636,2280,2384,4015,5068,6328,7697,7538,1588,2760,4701,5706,2338,3925,9966,2674,5599,1985,3879,6636,10366,9877,3042,544,2724,429,1957,8924,1634,10106,6242,9852,1884,1799,3610,299,7696,1144,5923,5194,4467,5813,1925,5177,5554,9889,886,6244,8968,9883,1910,1661,4438,7766,2194,5792,7610,8874,9729,7166,5700,8477,2091,4076,4962,9797,10400,2179,6539,1243,3452,6322,7091,10164,6564,8879,1637,2222,8434,6026,9485,4235,10564,6164,10599,186,7953,1223,6484,630,2034,6225,5991,5787,2344,7376,6979,4462,204,10621,6747,13,302,7329,4339,5983,6338,3113,3397,2213,5772,6603,4140,8357,8048,9752,3875,4177,1106,4179,2789,2995,3774,625,797,4188,333,4093,7495,6043,8057,5433,9839,1038,9750,1763,8427,10470,8601,1471,7956,8118,6848,6237,3794,5899,7245,1913,8242,6565,5329,442,2644,986,2983,5684,7060,368,89,10710,8027,2481,10632,9061,3313,10698,5357,6054,2221,4615,1578,9878,2509,4810,8126,848,1303,2489,2441,764,8255,7065,1527,553,2628,6282,4518,6331,8096,3017,2039,5350,3657,6795,2357,6577,4296,510,5627,338,393,10407,6951,9090,2798,4589,8606,6434,868,4778,631,2964,231,4594,6665,3945,5427,1293,8655,8499,388,3633,1393,2877,8319,6980,736,622,8813,3797,5883,3433,10160,2786,9567,9022,5739,7920,8966,1083,4771,3532,10697,4826,137,2182,522,8984,9982,10264,4017,1974,5652,1981,2910,7029,2990,8724,3505,4651,7819,4345,5497,1928,7782,8949,433,2403,10288,6751,6342,4712,7684,9683,1035,2844,2317,10157,3858,4370,3883,1554,7061,8373,8015,5686,8795,6278,3683,8686,5493,5168,4032,1748,10790,1458,1432,7436,8032,7625,8651,10815,5998,10506,5202,8688,3678,9343,6510,1629,3182,1353,7419,3785,2472,8393,7761,1355,6332,9681,9931,364,10678,854,9098,9691,10792,5289,7764,4230,5069,7055,3955,3576,4567,10194,6414,6766,8956,3269,2997,3888,144,1777,5477,7013,5192,3642,2311,1336,849,1601,4968,7482,8135,6404,3124,8622,2949,5403,9269,1256,10521,9091,5486,6970,2815,4311,1893,7183,85,2292,9856,7984,855,1750,4279,1713,7002,7626,1705,2090,105,4687,9153,2294,10411,7939,6707,1672,10284,8545,2609,10714,8589,1757,7469,7217,2263,5027,1745,853,4429,6758,9651,7144,9286,7676,8527,8155,420,4443,1845,8082,6611,0,2794,9547,639,1222,6115,1572,9903,9495,5545,3243,9621,5182,247,5581,3289,3729,776,5095,2379,243,8744,2295,3732,4725,10544,2745,1246,10024,3345,2076,1155,627,6939,2454,2839,10259,641,5988,1808,7401,9723,6734,1294,10155,5818,1260,7904,10671,7750,1061,8224,4013,7194,9030,9482,1505,8012,10185,2293,8551,520,4831,10451,5002,1628,244,23,5459,3152,3566,2923,4389,2508,5984,9363,9138,4756,9871,3200,1729,8922,3064,5090,4626,491,1968,7768,4039,7882,6317,4822,10244,6947,6559,5,188,4424,6963,1879,10447,8269,9576,3865,2632,4003,9368,3484,8089,2790,5472,1922,6723,5553,6057,2734,4989,4706,397,8384,8002,6233,3469,1576,8470,9846,2836,9521,1829,7442,8771,7731,2751,813,9051,5637,863,329,8056,5574,9779,10558,9234,8923,366,10036,7320,8997,9522,5301,3292,3327,1113,5242,5602,5876,3817,6972,1054,7870,4310,211,1290,4005,6888,8732,7384,9564,6837,6044,6205,5655,10017,4824,2938,7281,10462,5091,6918,5488,8481,7828,5498,7613,3405,3648,6007,5634,10209,10109,10516,10466,10688,6501,2023,400,10517,10399,2297,5079,4148,6248,3482,5231,441,7759,9244,1064,8564,8973,711,10711,5525,3420,2526,4757,3759,8142,5750,1787,6232,10806,3730,4483,4255,2811,2707,8295,9699,10739,9911,7622,1784,6592,9590,9556,1687,7643,10265,1620,1119,8451,4198,10519,10607,6446,4227,4943,6810,5061,9364,4570,2738,2640,10126,8694,9666,5043,1883,8233,904,6081,4532,3553,7533,3507,7636,6788,9128,995,4867,9560,5190,1115,2771,731,9777,1653,6919,6776,10262,9881,1196,3386,6390,3574,8693,913,2714,2713,8780,674,3478,550,686,4786,8990,5885,139,10416,1684,1811,5147,5614,5178,2662,4080,3141,6885,4211,307,1232,7565,2704,7598,9383,5930,7348,9999,8415,5980,6965,3236,6685,8251,2065,1573,4893,10245,2051,1239,4581,3425,2594,2740,3394,2607,384,3646,3567,5732,10820,7380,9440,10827,7490,1194,9984,4435,5587,901,5828,3008,7045,8749,6974,1547,8225,2202,6968,1602,9124,8466,509,10066,934,5022,2542,10193,1852,5339,8180,6639,3263,4159,5632,1590,8059,3952,7225,158,3494,2723,6447,5625,10732,2864,6698,3906,7974,4002,10044,2673,1008,266,9293,1999,7714,7968,4112,7779,9610,2266,7515,7274,8558,2139,8725,6067,4642,8930,374,8485,4717,173,2575,3832,34,2449,1346,7854,7675,8378,1743,7178,7022,2495,5542,1668,40,8230,3849,3363,10006,90,1143,8636,3591,2779,2994,9997,8281,10483,3463,4899,7722,10830,8492,4136,10368,7437,5013,2783,2592,9110,10227,6638,6650,7827,5455,10583,4338,7387,4158,1318,6757,4126,5434,1931,7856,9721,8597,1209,737,4474,2718,93,4337,5102,3317,10432,83,1339,10534,572,3788,3298,8604,4176,8889,7836,9505,8400,3246,9552,3390,7599,9630,7850,6075,6854,6077,2687,866,6668,9206,3781,3587,717,2312,6600,6407,6341,264,6952,1579,2558,2532,6562,5389,3970,7034,293,7777,9506,4873,7247,9140,10493,464,4225,5639,3134,9195,2386,5603,2578,8533,9782,436,6893,5590,3596,2083,9908,5891,5064,9589,8760,1693,5571,8837,168,6382,8512,4334,3711,10474,9655,1704,7057,5724,5673,9735,5522,7289,6944,2787,1709,6198,1131,4555,2999,555,7781,8812,632,5743,6513,3414,2336,1682,8988,8087,2343,4412,9424,99,444,5874,10460,9017,6094,7614,2147,5125,10117,7840,4637,5104,1285,8602,1091,10113,9887,7576,10605,3258,4018,276,9324,7641,9276,10477,3054,9256,9450,7105,3953,5909,8717,6504,2715,2154,9197,8828,5929,3621,3016,7042,419,3847,9320,1334,3782,9312,8699,3228,7715,7370,9313,8346,4425,3047,8838,9049,6352,1340,4492,3356,7095,663,4660,4119,10146,3129,7369,2040,6674,5527,9259,2322,64,7858,7019,8417,1746,7285,3332,7129,6814,4796,8274,3083,2824,3001,8621,3746,5344,1575,10357,4758,5436,4896,1060,8371,426,5031,7991,5916,1335,1345,8444,2162,10791,4539,9850,7051,2270,961,94,4578,6521,9290,6724,3809,6306,1644,447,7084,9623,7811,8895,2410,2493,4839,6003,7889,8892,3674,1283,790,2118,1473,5070,9296,469,9447,2128,3252,3793,10408,466,9013,2776,10452,6135,9187,6662,8183,1605,3695,8077,9430,5955,2376,7511,1894,7304,8703,6063,6672,5462,4405,8742,10120,5593,2475,208,2431,2672,1288,2582,5519,2586,7843,353,8981,896,10275,647,10456,3689,3802,4317,6663,6118,4185,4708,395,4829,3538,9278,471,5010,8531,6345,3586,6688,120,847,8804,2494,6987,1125,7031,6879,2730,6717,1033,6709,1562,2274,9686,7735,5942,4863,10572,10404,71,2968,2067,5717,92,10412,3311,6833,5509,5794,4785,5042,3656,8366,3020,10494,2405,5758,10536,9541,1920,6696,6287,2756,31,10430,349,3571,7708,3037,5256,5321,10554,7529,9733,10179,5227,3702,9785,680,10140,4195,3381,9638,7385,3643,9165,10604,3355,8044,4603,580,7455,9274,6389,6169,8254,1630,4696,7633,10327,2173,8039,9220,10247,7041,6706,1492,2217,9186,9024,2127,7947,3550,7563,4143,570,5633,1816,4510,5549,10139,4149,4450,9112,4078,5910,10132,4414,3806,9204,10713,9384,7250,3444,2330,10071,4990,7959,5900,8010,695,2418,9349,2079,793,6022,8438,3569,305,8863,8888,1397,596,6355,5604,3795,4291,2731,5203,2908,7568,3939,8175,4569,1762,2017,3766,3623,8062,138,54,5884,2240,6735,4455,5961,7527,6660,9060,8855,5785,6392,7688,551,4768,10367,2546,432,6143,5703,7278,547,4440,4614,2031,760,9107,164,921,1262,1858,6701,4692,9105,4354,768,2359,2012,6566,7820,759,2144,10251,9156,6096,3370,2226,708,1514,5835,8811,10060,4184,9763,9838,2553,6095,2739,7630,4521,10515,8615,5831,2939,7650,6491,8999,311,8060,8381,9298,7748,4314,10356,6823,788,3150,5169,2525,7913,8479,2035,9235,5396,3833,6781,5501,2126,8389,8729,5767,8543,8899,149,3676,5028,9026,8523,3229,1778,2279,3562,6931,3968,2388,1586,2084,7646,331,2309,9358,9410,6419,8661,7981,2872,5966,3249,1438,1179,5514,9442,1421,7544,10463,10092,6813,7402,10433,2096,4264,3254,2188,7571,4693,9111,2183,3233,1270,9930,7201,6177,9326,6128,9272,10283,1420,482,8918,5904,1717,10166,4410,7447,10504,1500,7996,3302,62,3277,9151,7220,9469,3629,3965,3506,508,8207,7776,8825,6989,831,3439,8361,1092,1570,7605,3019,7205,3437,6454,3328,5547,6280,10300,5808,9936,4473,7111,8532,1368,10527,3775,10439,8290,3605,10169,2393,4774,6741,7802,7435,1375,4064,9210,1010,1274,662,6711,8865,3214,4099,10299,7104,571,3178,8753,2625,6461,2033,1379,5641,2095,5965,7249,6797,1947,9280,2643,8791,4864,9821,749,10294,5385,4132,8897,5726,3564,3558,6499,2032,2834,10634,10389,4163,8820,3038,6585,7673,5247,7352,9830,4171,7151,7824,2244,4848,10371,1211,8876,9133,689,7397,4352,1769,10832,6356,8081,2637,4926,7169,3409,8450,3934,1080,10061,693,3030,4740,5784,2907,5576,7288,10670,9125,8903,5886,6548,907,9950,268,4553,7424,750,5947,5377,5663,2235,3769,7615,4239,2501,6572,1559,745,1495,10273,10681,8489,4395,10550,4749,7145,2825,8650,10485,8134,10023,8463,3291,6412,9078,9224,9431,4944,3105,1422,10455,5507,4180,9075,9123,9438,6986,3671,8386,7507,7716,9927,5284,3589,1448,8503,7832,1584,8714,362,5802,5564,4604,10718,5437,4307,7506,8890,4913,4146,7823,7181,4280,7362,5384,2890,3886,9117,9728,10687,182,4183,10770,3859,10228,676,5251,690,1443,4927,2540,5520,9837,598,2513,8190,637,823,5487,4275,4862,9484,6360,8063,3762,1474,5032,6158,10419,3431,7602,9279,2841,1635,9928,8617,1959,5836,3440,4197,6061,2932,3375,7550,9791,5273,7964,10249,2579,452,2271,10518,6111,4096,1647,1753,4212,6798,5378,1700,7876,8594,4033,1869,5024,2620,9011,8095,6478,5881,9319,8205,5978,6779,8200,4074,8850,2269,4073,9222,3487,4519,2885,7160,6172,936,2458,7834,9468,7763,3430,1707,187,6770,7770,1055,352,7783,10737,9401,7167,1991,5240,1180,9121,1861,8395,2268,8957,5774,996,8605,3772,8940,4836,891,321,7988,6534,5671,9144,9301,9754,8849,2554,1546,7425,4746,10070,10742,2061,1782,6038,6250,591,1257,1391,1954,8206,6307,1903,9941,3093,4348,7666,2996,6904,9100,5694,10143,10663,1426,607,8201,10035,7107,1371,7747,2042,10206,5327,1992,8420,5391,6752,8538,9396,3479,7849,6514,6892,477,8969,1585,192,5669,516,1237,1833,2979,4488,9753,4362,176,1916,7215,5365,3213,1563,7973,5557,8685,2619,1200,3502,10104,4851,7267,8100,6858,3062,4062,9876,2821,645,9604,4742,7118,507,5987,2668,579,9459,6125,10813,10046,10202,70,227,9483,10606,10332,3198,3044,10672,774,2601,1251,5953,559,4109,6563,8803,4832,1094,4911,4514,4549,5225,9428,3931,9381,10578,8473,10305,1506,10,5012,2587,10195,10434,1162,3347,2556,976,5152,414,5445,7588,4680,1210,5566,4027,1651,10013,180,9734,4150,2216,5584,9168,6426,10533,403,6391,8317,7471,8773,6772,1072,2835,5935,4951,8382,2799,9835,8079,629,2596,8525,7123,9678,113,3992,10149,8364,761,3111,519,8672,1521,7119,1011,2057,3940,7139,1236,1450,1930,1307,1191,1147,17,8074,8428,1509,9136,7293,4515,975,1388,4537,5672,5513,2611,4156,1071,10105,4453,2998,9717,875,8084,3750,10030,9142,9784,7311,6895,2514,494,415,6325,8698,6626,2460,9933,108,282,10589,6610,4323,1427,4830,6348,26,9527,2832,2015,1383,7344,1961,1088,2149,3496,8083,4294,4105,5850,1265,4969,8187,1137,8324,1625,2413,193,5964,9355,8629,4366,7933,7456,1821,4327,9053,7859,6520,103,2420,4882,3084,4653,562,9979,1381,10099,2925,8667,10048,10482,6210,10637,5861,5958,7601,1765,5176,6366,7512,10788,4048,10513,8935,10186,1460,8398,6532,6028,3029,10258,7891,3903,4402,537,2062,1281,4813,740,7324,3040,4299,7668,3275,9071,8275,10532,9628,4580,7969,6649,5485,6042,4596,4602,125,8722,6015,758,5287,3237,3391,9664,4563,2981,7427,3951,4618,4312,2803,5268,9261,5992,8960,8376,375,1688,610,9393,2210,8111,912,4994,1214,4355,4733,3278,1168,5352,7026,5812,7390,3929,3718,9800,9249,6656,4098,7543,5415,5334,5034,2521,5228,1828,273,8886,3604,9425,7156,4574,9965,10712,2886,7298,10613,9052,4820,778,135,2132,10831,1761,6851,5039,3179,9452,2929,2982,9037,4683,9106,5150,7958,4585,5298,2395,10130,4221,4738,287,4891,2203,4572,4508,1697,5651,9191,1832,4295,10556,8454,1378,9741,4110,4301,3155,9897,713,5136,4681,9141,5138,6088,5157,10773,2485,9499,5644,1664,536,7580,8598,4088,6422,10257,6400,5551,1440,1534,4084,4633,3051,3949,3767,1946,2780,4946,2940,899,2278,10289,4063,2660,9418,9045,2250,9494,309,8245,8691,6616,642,2639,2985,1459,3235,7388,455,6579,1315,10260,9618,6197,6460,498,5290,5322,337,3025,6245,7440,3922,7141,10354,4576,4933,10292,9582,9429,2102,7462,5919,5996,4978,2333,8043,8149,2296,10028,9657,3012,6768,7692,1550,9824,3528,8871,5347,9179,10579,10297,5159,1654,7082,4167,6812,8540,11,4207,345,3210,2576,5075,10114,605,2340,7664,6736,4747,3473,3747,9183,9171,3850,5276,8994,9310,7149,7253,4103,3573,1185,1633,773,8528,246,3890,10347,9305,5312,9520,9751,8898,9154,10337,4966,9391,3889,1790,3035,2516,9318,8323,7868,1023,10077,1727,3032,10329,2922,4571,3464,9759,2545,1277,6285,4957,8397,616,7861,3542,9680,4689,7088,8521,1895,3741,8457,2856,5112,10635,7624,7665,10808,2450,4297,2950,6168,6967,4584,6179,10420,3187,943,7443,1215,2962,5896,5484,8689,4469,7147,7945,3607,2111,4479,3136,5047,4051,8050,8336,10015,2394,2573,4305,9963,4509,6916,3462,5254,4318,265,58,422,8927,279,5718,8069,6527,10620,4795,9766,4535,3189,7712,6998,10263,7096,6186,960,8809,5324,4206,4858,5143,4427,9101,4118,3352,2848,4776,10649,3175,9536,10208,2591,2944,7230,8912,7660,1614,228,2115,1385,7043,4437,8662,8349,914,10319,2130,8649,5249,8294,9152,1461,10162,1169,5198,9535,7018,399,7658,9755,4451,10107,3866,506,1526,3283,6274,7184,7926,2870,611,2884,1865,1984,10758,4449,7617,1706,2728,9443,7374,651,3592,8476,9028,9516,7453,3752,8774,6731,7011,6912,9047,10763,8654,3789,6093,2543,1009,3647,2101,3518,8563,1662,6686,2716,10072,4495,8603,10387,5306,9696,8508,8775,998,3658,21,9772,7972,10274,9585,2572,4289,564,8579,970,6209,6358,1235,1250,894,9579,5949,7248,1856,3268,3202,525,3406,6090,872,3916,9262,5442,55,7089,8429,215,4461,5749,9351,5559,8783,3624,2598,7611,8141,3432,2807,5399,1645,7458,4004,9925,7703,6684,8374,5238,9540,5565,6226,8421,5782,9113,6632,5582,1428,5438,1350,2694,3338,9419,1501,7360,1800,10573,6001,3238,9847,7769,7848,9593,7594,10254,5191,7481,7963,4161,9551,2355,6651,7846,9455,3480,6086,376,6604,9858,2286,4041,5267,6315,5875,6999,2967,9344,4336,8490,1314,3827,3736,8628,6305,152,5586,4720,8351,7064,7373,4091,3303,4390,4727,3815,900,3115,142,8009,359,590,341,10413,1075,5869,9595,2711,5920,9769,4541,4986,3219,6761,7431,1470,573,5976,5840,593,9598,10829,3984,9532,972,9550,9126,8584,2866,2593,6841,1923,7999,1463,2602,1905,4707,7600,200,10237,2819,5960,6033,10489,9243,1059,9768,9033,7097,3127,5065,2215,10745,6364,1545,10598,7391,7412,9577,4228,4783,9342,874,6,4732,725,6323,8143,8478,563,1737,7679,4868,7541,3339,10428,8035,8307,3828,5292,2491,6861,3474,9376,4984,8926,5745,9562,594,4802,1598,3637,8779,4358,5108,6053,8709,6580,7910,10266,8253,1736,9390,4560,489,5768,2368,6743,262,10490,3851,2991,1027,2993,581,3300,6809,3358,8496,3655,7176,8501,2559,236,1577,10019,1233,1926,9426,9050,4332,10236,9089,9992,2133,9230,2617,1685,2422,2900,4736,5111,8720,7062,1289,4021,3123,7415,9150,8806,2136,9501,1873,2225,5894,7800,9377,7845,9006,3649,8266,5101,5469,6318,4219,5098,5847,4170,2093,2536,6398,6137,5912,1015,8718,9812,7629,1341,292,8172,136,8723,511,6933,4351,3976,1641,5420,189,9427,827,5370,9382,9529,4914,3563,7957,6710,7138,756,6017,1047,4293,6606,3988,7432,9822,197,8112,1405,2121,4199,4715,10476,4870,5735,6099,1370,5826,2507,7893,6648,10108,8484,1360,6661,6402,7414,7349,7433,9251,38,4499,4619,3399,6136,5986,1976,5833,2301,1948,167,4844,2327,2438,7444,3919,6396,5456,8590,1446,9861,4611,5058,10406,5656,6771,3516,902,8252,8212,851,2759,9021,6530,7577,1972,9201,9295,1219,5407,6493,2629,69,828,6146,9740,4086,3705,281,3975,5756,1244,5418,3704,8989,86,2926,1859,8158,3617,8800,4685,6262,1679,9940,937,5550,8102,1276,7350,1571,9969,4958,4434,8343,5607,4620,166,1456,567,6311,10542,8218,6767,9392,7219,7671,5179,2052,8174,8196,67,781,9413,5362,6840,643,4987,2612,6334,4817,5097,9712,955,2933,9275,8013,9334,10397,10535,37,1613,4085,2753,10784,6405,6631,2175,5059,1539,191,10242,6235,7228,9103,3131,6708,8213,10648,6410,5213,3334,4472,6277,2802,7632,4657,4442,2056,4153,2267,3632,1451,1476,3641,8313,2099,6923,8627,1022,6041,6387,9817,1756,379,8140,4887,6268,1917,9978,6062,10498,4029,1975,9922,10569,2665,2254,5206,3326,9282,3374,7677,8413,1855,2299,2408,7897,3778,6217,7077,2816,2473,5573,6249,9184,10079,7537,7300,3419,8664,289,7896,2468,249,4841,3267,7504,7545,4286,5552,128,6726,8577,2406,314,5839,3893,9215,5132,7875,2651,10559,8024,1497,9697,2241,8407,1964,499,6587,932,7341,2089,10213,1286,229,1433,479,7755,2581,6570,4124,969,7251,501,8260,2028,4534,6417,97,10145,8847,8816,5296,4292,1552,9161,3846,10765,1311,10296,8146,6473,7175,3693,330,4928,6052,9130,1208,3918,6785,4164,5763,4497,6397,3917,1198,8173,4090,3498,517,2969,10168,8156,1104,5622,10497,3515,8222,6551,10128,3380,3979,8882,10467,7829,3410,883,10431,9639,5791,9720,8700,4792,9247,5727,3771,5858,8546,468,548,3500,8646,6453,746,10446,459,9674,6253,2120,8472,1044,1650,3860,7778,9998,5829,10804,2498,9094,8331,10089,4561,2108,6815,667,4892,3389,6221,5341,2955,6174,10630,2451,6956,7773,10232,6549,248,528,6597,1411,60,6669,2114,8455,4920,6027,3726,6516,8474,9508,2363,9716,308,3039,4895,96,5476,10205,916,6195,5262,7131,3511,5516,1430,9217,4191,7301,4448,5598,9093,6161,7025,9710,4767,8055,5773,3843,4960,8705,7826,9457,7524,4359,4346,1102,4466,3310,9339,5282,10191,5460,7694,4042,7844,4502,9644,10333,3838,669,7428,515,7564,8854,6567,6725,177,635,367,6803,8358,10734,4586,10313,1124,9949,2453,7949,7050,5678,577,4640,44,5107,5661,4439,6409,8136,5356,1025,7595,8736,10384,7525,4728,8608,4290,2712,5538,1217,7872,5702,5635,1479,8072,6834,2928,9143,3770,9283,10200,3468,7333,6922,8663,5660,2595,9920,6428,3036,7012,5957,9315,3118,5788,10378,10659,595,6255,4779,3224,10510,4991,6175,4936,386,8901,6586,4837,4694,1324,8350,9647,3829,763,784,5687,4380,5969,10198,1732,8326,7191,1788,3282,880,2502,2461,2018,5962,5272,3776,7171,2003,1911,8239,2915,1794,5811,6839,3163,7475,2860,9085,889,1774,2958,5517,5532,10285,5775,9919,5335,7888,10738,8497,3758,5004,1498,9309,91,3677,9557,1617,9314,7597,9792,5037,8793,7644,7967,1365,5994,7079,3162,710,5926,2196,10651,5008,2852,5981,7911,9780,4811,3863,4833,4155,4298,4028,6294,5193,558,2574,7150,8423,2954,821,9669,9181,9241,4593,8702,3220,4505,10210,10705,846,1804,10746,9747,6938,2055,3611,1716,7874,6617,9745,5081,10009,1945,1152,8377,1658,929,5765,7567,4724,6374,7,10138,1007,5086,8179,2255,9614,6784,2853,2512,9010,2920,7871,6825,6697,10373,3485,8277,7100,3344,2037,782,7048,8166,9967,3614,4043,4385,8769,3204,9353,8285,10679,10219,10586,1066,6420,6537,6511,7608,2871,6819,8934,6083,1710,114,10793,4859,3472,6354,8229,9302,8873,9515,213,6472,3069,7188,7021,6505,5882,5336,8157,3067,1158,1496,8596,3936,7812,30,4850,10234,6368,2522,153,8049,7619,5241,6557,3087,10706,2166,6722,3835,2943,47,6519,9917,1866,5795,5448,6046,1439,3708,2882,3215,7291,4729,6786,2648,3608,3933,1703,7121,8625,5303,10597,814,100,3820,372,119,7357,8070,2649,10617,4856,3024,9534,7791,1436,8375,10695,5721,1105,6266,9054,10593,3512,7883,2072,8500,8309,5612,925,7466,10192,10040,5915,7078,3628,4342,1767,10377,3877,9587,461,9985,5500,8868,10111,2000,8248,2206,9409,655,8586,9321,8341,8832,1781,2078,6787,35,6558,9146,10082,2452,5000,5699,6485,3151,3011,5526,8469,6555,9335,3279,10665,6256,185,8316,277,7637,5619,4223,234,4388,8198,10435,5905,6021,10753,2847,3698,2253,1352,5624,832,6471,3662,134,1540,5657,4901,4205,715,4241,1715,7196,9765,4512,5736,10537,8573,7234,5730,10549,9968,926,9986,4413,5880,10809,5544,5892,3085,1880,10306,3259,3973,3367,7780,1839,982,9099,8581,8600,772,5302,2219,7863,3913,6085,5572,10369,4142,6134,2764,5291,9476,1551,6303,10702,4007,5317,2989,6937,1854,2201,6182,3074,4344,8782,6641,3999,5461,2150,8908,2677,5364,1965,1531,115,7549,10581,6019,8943,2749,2161,7470,7275,2151,5470,2851,1814,2445,10360,3520,5814,10055,1019,7607,3620,502,4883,10574,1389,3260,7108,2171,5411,1681,1409,8719,5755,10749,2655,9300,2511,10037,5011,9528,3508,4794,1084,3548,3125,2030,10643,6004,2534,1457,10256,8066,2232,4069,445,10766,5265,8169,9654,6113,2942,7995,6605,3403,8576,8291,9193,4036,2599,945,3244,4656,787,9645,2746,7749,10324,7299,6793,5089,8885,10348,7792,8529,3527,1888,9002,4666,7269,10622,7815,5846,3578,6406,692,3045,6369,8776,3072,6071,6746,7526,6900,5062,8011,9658,392,5817,5623,10453,8214,2722,4411,5443,1181,9498,1028,2138,66,5073,5605,3540,6440,7271,170,1188,1163,7934,1751,2224,6073,4,3052,1065,1369,1801,2499,7895,5346,10802,350,2988,2849,3619,4937,6455,10156,10041,7690,8103,9369,8619,4257,1282,5676,9827,1138,8631,5201,1099,9032,3668,9673,2146,5226,6251,895,9841,2005,6152,3927,7243,7448,4544,4231,7083,3731,3272,8293,5116,9350,8711,9137,2167,4407,4065,10047,7584,5931,8064,259,10301,9884,6692,6500,5844,1835,8021,1881,1673,6677,240,785,5465,4258,957,3742,4189,7040,1892,5083,606,9132,5506,2320,10501,3915,8530,9147,666,2187,225,10096,6074,4974,734,8852,8515,6694,7059,4056,2024,5798,10686,7659,3402,9990,9934,7364,3257,7438,165,8637,6240,4484,3126,3714,405,196,3514,3900,2804,8284,5045,1564,2754,10158,974,1173,2859,9233,10103,3839,3885,7606,9202,4044,2364,9798,7922,2318,3737,7164,8480,6543,9952,7860,1149,6778,4773,9474,6924,3407,983,9379,8728,7336,3076,8772,3110,8065,7080,10509,5148,7256,9648,9701,9832,8517,817,4202,3455,8453,681,458,8549,9836,3897,2356,10783,8974,9048,417,6302,8208,4721,5451,9620,1996,4139,7816,5421,8796,8119,3147,9465,6450,2068,6112,3167,10486,4807,532,5753,5824,9034,2304,6309,485,272,8802,416,5807,5742,8325,9739,9890,9625,7342,8585,3967,7757,843,4900,575,4814,9771,4523,6800,8942,5117,8707,10279,4430,6997,9199,1582,9292,7404,9976,3531,10093,3523,219,6155,6006,1632,9042,10422,10083,3673,9698,4546,7662,6059,1699,8880,8435,9994,922,2486,911,3114,660,29,1005,4520,1815,8202,10382,5234,8950,2503,8330,9019,5295,1860,1366,8221,614,7884,1955,174,4536,5974,2708,2307,7738,9242,8706,9039,7446,2159,10756,112,3003,9493,6167,9862,2903,9159,7519,7439,2614,6084,6072,8822,1838,3239,6675,9422,3964,3033,6899,4754,3963,4853,3710,5040,8080,4061,3597,2109,2123,1595,6211,5146,6791,4827,9996,4144,5569,7942,1159,9943,7572,1639,3672,10359,10690,4525,6196,9481,738,716,4489,2685,10147,4248,8583,6457,3387,10614,10364,993,1037,1404,10080,9258,5580,10403,3854,4116,5118,10669,9891,9737,5665,2469,10241,5489,6971,8936,6880,5977,5649,9228,3158,7784,5115,1574,9316,1797,8385,835,169,2288,8864,4121,6575,4101,407,2421,3898,5395,5616,3453,1454,5431,8122,156,1864,87,6008,7652,6427,7356,2260,1090,4823,8447,2457,10280,1631,1538,950,8406,1279,2966,7552,2347,7259,6394,10560,5868,3560,5823,3451,3046,5019,2414,7785,7929,8217,3320,4226,864,6148,6673,6490,3400,1310,9492,2348,4415,968,10323,3990,5776,9900,9336,1046,7148,8798,5151,2960,6178,6623,1204,4454,1760,7168,6117,1146,10719,10458,1649,5397,10282,10584,9794,3475,10402,10392,10762,3270,413,1759,9711,3664,3306,5274,8678,652,8665,8416,6408,8046,1624,5865,9687,8657,5888,1062,2184,1164,3280,9055,3097,5372,424,1287,5887,4647,7493,6962,10740,5382,6805,4082,8845,3590,7270,2228,2630,3021,9731,524,4700,378,7771,10656,3398,6644,990,1322,10016,6855,5261,10799,9323,6720,6162,1252,7898,487,4787,9439,5898,7813,3728,7689,8353,45,8231,2275,1665,709,2700,7044,6229,9875,3411,1846,5567,9855,6894,658,448,8410,1416,978,9570,5480,2329,290,4719,6830,7788,5473,8767,6832,2750,4270,2778,4501,1530,4417,7262,9398,3441,8026,2141,6700,4803,4545,3201,8518,9432,5734,3969,9904,8241,9760,10652,4558,5677,1535,3251,3971,10052,2199,5751,9956,7627,3299,10078,6578,4000,2567,1193,1453,2231,1398,1714,1507,2613,8977,1701,4465,5809,820,5049,9517,5595,5917,529,4815,224,3910,7054,3727,1939,7937,6884,9636,8394,5682,5405,9487,4907,9554,9743,10322,6208,9414,8843,7213,7505,10188,7604,7837,9860,10351,2087,14,7212,5999,6474,209,9842,10468,3271,741,10611,5142,4020,1870,10141,2070,238,4670,3450,3188,2479,8304,1263,9502,3330,2361,9077,160,4884,2948,2011,1032,1652,9131,2428,4917,6908,8219,6141], "time_ms": 0.144850, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 613, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "generated_query_count", "value": 2400001, "time_ms": 969.687501, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 428, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "generated_samples_per_query", "value": 1, "time_ms": 969.687501, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 429, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "generated_query_duration", "value": 1200000500000, "time_ms": 969.687501, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 430, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "generic_message", "value": "Ending naturally: Minimum query count and test duration met.", "time_ms": 600970.564171, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/issue_query_controller.cc", "line_no": 482, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 600980.355756, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 898, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 600980.355756, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 900, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 600980.355756, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 902, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 600980.355756, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 904, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 600980.355756, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 906, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 600980.355756, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/logging.cc", "line_no": 908, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "power_begin", "value": "12-25-2024 06:24:32.941", "time_ms": 600980.356026, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 564, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "power_end", "value": "12-25-2024 06:34:32.943", "time_ms": 600980.356026, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/loadgen.cc", "line_no": 566, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_validity", "value": "VALID", "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 644, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_min_duration_met", "value": true, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 649, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_min_queries_met", "value": true, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 650, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "early_stopping_met", "value": true, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 651, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "early_stopping_result", "value": " * Processed at least 64 queries (641819). * Would discard 63621 highest latency queries. * Early stopping 90th percentile estimate: 1015707 * Early stopping 99th percentile estimate: 1190463", "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 669, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_query_count", "value": 641819, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 679, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_qps_with_loadgen_overhead", "value": 1069.7, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 702, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_qps_without_loadgen_overhead", "value": 1074.1, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 703, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "early_stopping_latency_ss", "value": 1015707, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 704, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "early_stopping_latency_ms", "value": 1190463, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 706, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_min_latency_ns", "value": 859605, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 741, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_max_latency_ns", "value": 1297262, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 742, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_mean_latency_ns", "value": 931012, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 743, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_50.00_percentile_latency_ns", "value": 909739, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_90.00_percentile_latency_ns", "value": 1014324, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_95.00_percentile_latency_ns", "value": 1082421, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_97.00_percentile_latency_ns", "value": 1174393, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_99.00_percentile_latency_ns", "value": 1190403, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10481, "tid": 10481}} +:::MLLOG {"key": "result_99.90_percentile_latency_ns", "value": 1206002, "time_ms": 601275.566111, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "/home/cmuser/CM/repos/local/cache/ba8d5f2a6bc546f9/repo/closed/NVIDIA/build/inference/loadgen/results.cc", "line_no": 745, "pid": 10481, "tid": 10481}} diff --git a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_summary.txt b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_summary.txt index c3ab6e8d..b6c35779 100644 --- a/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_summary.txt +++ b/open/MLCommons/results/RTX4090x1-nvidia_original-gpu-tensorrt-vdefault-default_config/bert-99/singlestream/performance/run_1/mlperf_log_summary.txt @@ -4,32 +4,32 @@ MLPerf Results Summary SUT name : BERT SERVER Scenario : SingleStream Mode : PerformanceOnly -90th percentile latency (ns) : 1013732 +90th percentile latency (ns) : 1014324 Result is : VALID Min duration satisfied : Yes Min queries satisfied : Yes Early stopping satisfied: Yes Early Stopping Result: - * Processed at least 64 queries (637969). - * Would discard 63238 highest latency queries. - * Early stopping 90th percentile estimate: 1016588 - * Early stopping 99th percentile estimate: 1190854 + * Processed at least 64 queries (641819). + * Would discard 63621 highest latency queries. + * Early stopping 90th percentile estimate: 1015707 + * Early stopping 99th percentile estimate: 1190463 ================================================ Additional Stats ================================================ -QPS w/ loadgen overhead : 1063.28 -QPS w/o loadgen overhead : 1069.05 +QPS w/ loadgen overhead : 1069.70 +QPS w/o loadgen overhead : 1074.10 -Min latency (ns) : 872179 -Max latency (ns) : 1540105 -Mean latency (ns) : 935409 -50.00 percentile latency (ns) : 913296 -90.00 percentile latency (ns) : 1013732 -95.00 percentile latency (ns) : 1075228 -97.00 percentile latency (ns) : 1174433 -99.00 percentile latency (ns) : 1190824 -99.90 percentile latency (ns) : 1193097 +Min latency (ns) : 859605 +Max latency (ns) : 1297262 +Mean latency (ns) : 931012 +50.00 percentile latency (ns) : 909739 +90.00 percentile latency (ns) : 1014324 +95.00 percentile latency (ns) : 1082421 +97.00 percentile latency (ns) : 1174393 +99.00 percentile latency (ns) : 1190403 +99.90 percentile latency (ns) : 1206002 ================================================ Test Parameters Used