diff --git a/open/scc132/code/bert-99/README.md b/open/scc132/code/bert-99/README.md new file mode 100644 index 0000000..2fd9f95 --- /dev/null +++ b/open/scc132/code/bert-99/README.md @@ -0,0 +1 @@ +TBD \ No newline at end of file diff --git a/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/README.md b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/README.md new file mode 100644 index 0000000..d72cf72 --- /dev/null +++ b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/README.md @@ -0,0 +1,3 @@ +| Model | Scenario | Accuracy | Throughput | Latency (in ms) | +|---------|------------|------------|--------------|-------------------| +| bert-99 | offline | 90.8749 | 1.97 | - | \ No newline at end of file diff --git a/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/README.md b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/README.md new file mode 100644 index 0000000..6f4be74 --- /dev/null +++ b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/README.md @@ -0,0 +1,54 @@ +This experiment is generated using the [MLCommons Collective Mind automation framework (CM)](https://github.com/mlcommons/cm4mlops). + +*Check [CM MLPerf docs](https://docs.mlcommons.org/inference) for more details.* + +## Host platform + +* OS version: Linux-6.1.110-1.el9.elrepo.x86_64-x86_64-with-glibc2.34 +* CPU version: x86_64 +* Python version: 3.11.7 (main, Oct 21 2024, 23:35:59) [GCC 11.4.1 20231218 (Red Hat 11.4.1-3)] +* MLCommons CM version: 3.2.6 + +## CM Run Command + +See [CM installation guide](https://docs.mlcommons.org/inference/install/). + +```bash +pip install -U cmind + +cm rm cache -f + +cm pull repo mlcommons@cm4mlops --checkout=6dbb26a3da6b8ebdbc96be3be3a0e9817d3b6d26 + +cm run script \ + --tags=run-mlperf,inference,_r4.1-dev \ + --model=bert-99 \ + --implementation=reference \ + --framework=pytorch \ + --category=edge \ + --scenario=Offline \ + --execution_mode=valid \ + --device=cpu \ + --quiet +``` +*Note that if you want to use the [latest automation recipes](https://docs.mlcommons.org/inference) for MLPerf (CM scripts), + you should simply reload mlcommons@cm4mlops without checkout and clean CM cache as follows:* + +```bash +cm rm repo mlcommons@cm4mlops +cm pull repo mlcommons@cm4mlops +cm rm cache -f + +``` + +## Results + +Platform: scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config + +Model Precision: fp32 + +### Accuracy Results +`F1`: `90.87487`, Required accuracy for closed division `>= 89.96526` + +### Performance Results +`Samples per second`: `1.96994` diff --git a/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy_console.out b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy_console.out new file mode 100644 index 0000000..e69de29 diff --git a/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/cm-version-info.json b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/cm-version-info.json new file mode 100644 index 0000000..96803c7 --- /dev/null +++ b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/cm-version-info.json @@ -0,0 +1,374 @@ +{ + "app-mlperf-inference,d775cac873ee4231:reference,bert-99,pytorch,cpu,valid,r4.1-dev_default,offline": [ + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cpu,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "get-sys-utils-cm,bc90993277e84b8e" + } + }, + { + "get,python": { + "script_uid": "d0b5dd74373f4a62", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_variations": "", + "version": "3.11.7", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cpu,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,mlcommons,inference,src,_deeplearningexamples": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "deeplearningexamples", + "version": "master-git-f5c8f1758374aeaba26b2e84d31690111cfdf054", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cpu,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,mlperf,inference,src,_deeplearningexamples": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "deeplearningexamples", + "version": "master-git-f5c8f1758374aeaba26b2e84d31690111cfdf054", + "parent": "get-mlperf-inference-utils,e341e5f86d8342e5" + } + }, + { + "get,mlperf,inference,utils": { + "script_uid": "e341e5f86d8342e5", + "script_alias": "get-mlperf-inference-utils", + "script_tags": "get,mlperf,inference,util,utils,functions", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cpu,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,dataset,squad,language-processing": { + "script_uid": "6651c119c3ae49b3", + "script_alias": "get-dataset-squad", + "script_tags": "get,dataset,squad,language-processing,validation,original", + "script_variations": "", + "version": "1.1", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cpu,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "get,dataset-aux,squad-vocab": { + "script_uid": "e38874fff5094577", + "script_alias": "get-dataset-squad-vocab", + "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_bert-99,_pytorch,_cpu,_valid,_r4.1-dev_default,_offline )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "detect-cpu,586c8a43320142f7" + } + }, + { + "detect,cpu": { + "script_uid": "586c8a43320142f7", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "get-sys-utils-cm,bc90993277e84b8e" + } + }, + { + "get,python": { + "script_uid": "d0b5dd74373f4a62", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_variations": "", + "version": "3.11.7", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_torch": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "torch", + "version": "2.5.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_torch": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "torch", + "version": "2.5.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_torchvision": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "torchvision", + "version": "0.20.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_transformers": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "transformers", + "version": "4.45.2", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,ml-model,language-processing,bert-large,raw,_fp32,_pytorch": { + "script_uid": "5e865dbdc65949d2", + "script_alias": "get-ml-model-bert-large-squad", + "script_tags": "get,ml-model,raw,bert,bert-large,bert-squad,language,language-processing", + "script_variations": "fp32,pytorch", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,dataset,squad,original": { + "script_uid": "6651c119c3ae49b3", + "script_alias": "get-dataset-squad", + "script_tags": "get,dataset,squad,language-processing,validation,original", + "script_variations": "", + "version": "1.1", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,dataset-aux,squad-vocab": { + "script_uid": "e38874fff5094577", + "script_alias": "get-dataset-squad-vocab", + "script_tags": "get,aux,squad,dataset-aux,language-processing,squad-aux,vocab,squad-vocab", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "detect,os": { + "script_uid": "863735b7db8c44fc", + "script_alias": "detect-os", + "script_tags": "detect-os,detect,os,info", + "script_variations": "", + "version": "", + "parent": "detect-cpu,586c8a43320142f7" + } + }, + { + "detect,cpu": { + "script_uid": "586c8a43320142f7", + "script_alias": "detect-cpu", + "script_tags": "detect,cpu,detect-cpu,info", + "script_variations": "", + "version": "", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "get,python": { + "script_uid": "d0b5dd74373f4a62", + "script_alias": "get-python3", + "script_tags": "get,python,python3,get-python,get-python3", + "script_variations": "", + "version": "3.11.7", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "get,mlcommons,inference,src,_deeplearningexamples": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "deeplearningexamples", + "version": "master-git-f5c8f1758374aeaba26b2e84d31690111cfdf054", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "get,cache,dir,_name.mlperf-inference-sut-configs": { + "script_uid": "48f4622e059b45ce", + "script_alias": "get-cache-dir", + "script_tags": "get,cache,dir,directory", + "script_variations": "name.mlperf-inference-sut-configs", + "version": "", + "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b" + } + }, + { + "get,sut,configs": { + "script_uid": "c2fbf72009e2445b", + "script_alias": "get-mlperf-inference-sut-configs", + "script_tags": "get,mlperf,inference,sut,configs,sut-configs", + "script_variations": "", + "version": "", + "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" + } + }, + { + "generate,user-conf,mlperf,inference": { + "script_uid": "3af4475745964b93", + "script_alias": "generate-mlperf-inference-user-conf", + "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", + "script_variations": "", + "version": "", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,loadgen": { + "script_uid": "64c3d98d0ba04950", + "script_alias": "get-mlperf-inference-loadgen", + "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", + "script_variations": "", + "version": "master", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,mlcommons,inference,src,_deeplearningexamples": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "deeplearningexamples", + "version": "master-git-f5c8f1758374aeaba26b2e84d31690111cfdf054", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,mlcommons,inference,src": { + "script_uid": "4b57186581024797", + "script_alias": "get-mlperf-inference-src", + "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", + "script_variations": "", + "version": "master-git-f5c8f1758374aeaba26b2e84d31690111cfdf054", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_package.psutil": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "package.psutil", + "version": "6.1.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_package.pydantic": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "package.pydantic", + "version": "2.9.2", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_tokenization": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "tokenization", + "version": "1.0.7", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_six": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "six", + "version": "1.16.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_package.absl-py": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "package.absl-py", + "version": "2.1.0", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + }, + { + "get,generic-python-lib,_boto3": { + "script_uid": "94b62a682bc44791", + "script_alias": "get-generic-python-lib", + "script_tags": "get,install,generic,generic-python-lib", + "script_variations": "boto3", + "version": "1.35.45", + "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( bert-99,_cpu,_pytorch,_offline,_fp32 )" + } + } + ] +} \ No newline at end of file diff --git a/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/cpu_info.json b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/cpu_info.json new file mode 100644 index 0000000..8012edd --- /dev/null +++ b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/cpu_info.json @@ -0,0 +1,26 @@ +{ + "CM_HOST_CPU_WRITE_PROTECT_SUPPORT": "yes", + "CM_HOST_CPU_MICROCODE": "0x1000065", + "CM_HOST_CPU_FPU_SUPPORT": "yes", + "CM_HOST_CPU_FPU_EXCEPTION_SUPPORT": "yes", + "CM_HOST_CPU_BUGS": "sysret_ss_attrs null_seg spectre_v1 spectre_v2 spec_store_bypass srso", + "CM_HOST_CPU_TLB_SIZE": "1024 4K pages", + "CM_HOST_CPU_CFLUSH_SIZE": "64", + "CM_HOST_CPU_ARCHITECTURE": "x86_64", + "CM_HOST_CPU_TOTAL_CORES": "16", + "CM_HOST_CPU_ON_LINE_CPUS_LIST": "0-15", + "CM_HOST_CPU_VENDOR_ID": "AuthenticAMD", + "CM_HOST_CPU_MODEL_NAME": "AMD EPYC-Milan Processor", + "CM_HOST_CPU_FAMILY": "25", + "CM_HOST_CPU_THREADS_PER_CORE": "1", + "CM_HOST_CPU_PHYSICAL_CORES_PER_SOCKET": "1", + "CM_HOST_CPU_SOCKETS": "16", + "CM_HOST_CPU_L1D_CACHE_SIZE": "512 KiB (16 instances)", + "CM_HOST_CPU_L1I_CACHE_SIZE": "512 KiB (16 instances)", + "CM_HOST_CPU_L2_CACHE_SIZE": "8 MiB (16 instances)", + "CM_HOST_CPU_L3_CACHE_SIZE": "512 MiB (16 instances)", + "CM_HOST_CPU_NUMA_NODES": "1", + "CM_HOST_CPU_TOTAL_LOGICAL_CORES": "16", + "CM_HOST_MEMORY_CAPACITY": "60G", + "CM_HOST_DISK_CAPACITY": "107G" +} \ No newline at end of file diff --git a/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/mlperf.conf b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/mlperf.conf new file mode 100644 index 0000000..10f7ae7 --- /dev/null +++ b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/mlperf.conf @@ -0,0 +1,98 @@ +# The format of this config file is 'key = value'. +# The key has the format 'model.scenario.key'. Value is mostly int64_t. +# Model maybe '*' as wildcard. In that case the value applies to all models. +# All times are in milli seconds + +# Set performance_sample_count for each model. +# User can optionally set this to higher values in user.conf. +resnet50.*.performance_sample_count_override = 1024 +ssd-mobilenet.*.performance_sample_count_override = 256 +retinanet.*.performance_sample_count_override = 64 +bert.*.performance_sample_count_override = 10833 +dlrm.*.performance_sample_count_override = 204800 +dlrm-v2.*.performance_sample_count_override = 204800 +rnnt.*.performance_sample_count_override = 2513 +gptj.*.performance_sample_count_override = 13368 +llama2-70b.*.performance_sample_count_override = 24576 +stable-diffusion-xl.*.performance_sample_count_override = 5000 +# set to 0 to let entire sample set to be performance sample +3d-unet.*.performance_sample_count_override = 0 + +# Set seeds. The seeds will be distributed two weeks before the submission. +*.*.qsl_rng_seed = 3066443479025735752 +*.*.sample_index_rng_seed = 10688027786191513374 +*.*.schedule_rng_seed = 14962580496156340209 +# Set seeds for TEST_05. The seeds will be distributed two weeks before the submission. +*.*.test05_qsl_rng_seed = 16799458546791641818 +*.*.test05_sample_index_rng_seed = 5453809927556429288 +*.*.test05_schedule_rng_seed = 5435552105434836064 + + +*.SingleStream.target_latency_percentile = 90 +*.SingleStream.min_duration = 600000 + +*.MultiStream.target_latency_percentile = 99 +*.MultiStream.samples_per_query = 8 +*.MultiStream.min_duration = 600000 +*.MultiStream.min_query_count = 662 +retinanet.MultiStream.target_latency = 528 + +# 3D-UNet uses equal issue mode because it has non-uniform inputs +3d-unet.*.sample_concatenate_permutation = 1 + +# LLM benchmarks have non-uniform inputs and outputs, and use equal issue mode for all latency scenario +gptj.*.sample_concatenate_permutation = 1 +llama2-70b.*.sample_concatenate_permutation = 1 +mixtral-8x7b.*.sample_concatenate_permutation = 1 + +*.Server.target_latency = 10 +*.Server.target_latency_percentile = 99 +*.Server.target_duration = 0 +*.Server.min_duration = 600000 +resnet50.Server.target_latency = 15 +retinanet.Server.target_latency = 100 +bert.Server.target_latency = 130 +dlrm.Server.target_latency = 60 +dlrm-v2.Server.target_latency = 60 +rnnt.Server.target_latency = 1000 +gptj.Server.target_latency = 20000 +stable-diffusion-xl.Server.target_latency = 20000 +# Llama2-70b benchmarks measures token latencies +llama2-70b.*.use_token_latencies = 1 +mixtral-8x7b.*.use_token_latencies = 1 +# gptj benchmark infers token latencies +gptj.*.infer_token_latencies = 1 +gptj.*.token_latency_scaling_factor = 69 +# Only ttft and tpot are tracked for the llama2-70b & mixtral-8x7B benchmark therefore target_latency = 0 +llama2-70b.Server.target_latency = 0 +llama2-70b.Server.ttft_latency = 2000 +llama2-70b.Server.tpot_latency = 200 + +mixtral-8x7b.Server.target_latency = 0 +mixtral-8x7b.Server.ttft_latency = 2000 +mixtral-8x7b.Server.tpot_latency = 200 + +*.Offline.target_latency_percentile = 90 +*.Offline.min_duration = 600000 + +# In Offline scenario, we always have one query. But LoadGen maps this to +# min_sample_count internally in Offline scenario. If the dataset size is larger +# than 24576 we limit the min_query_count to 24576 and otherwise we use +# the dataset size as the limit + +resnet50.Offline.min_query_count = 24576 +retinanet.Offline.min_query_count = 24576 +dlrm-v2.Offline.min_query_count = 24576 +bert.Offline.min_query_count = 10833 +gptj.Offline.min_query_count = 13368 +rnnt.Offline.min_query_count = 2513 +3d-unet.Offline.min_query_count = 43 +stable-diffusion-xl.Offline.min_query_count = 5000 +llama2-70b.Offline.min_query_count = 24576 +mixtral-8x7b.Offline.min_query_count = 15000 + +# These fields should be defined and overridden by user.conf. +*.SingleStream.target_latency = 10 +*.MultiStream.target_latency = 80 +*.Server.target_qps = 1.0 +*.Offline.target_qps = 1.0 diff --git a/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/os_info.json b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/os_info.json new file mode 100644 index 0000000..2201983 --- /dev/null +++ b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/os_info.json @@ -0,0 +1,26 @@ +{ + "CM_HOST_OS_TYPE": "linux", + "CM_HOST_OS_BITS": "64", + "CM_HOST_OS_FLAVOR": "rocky", + "CM_HOST_OS_FLAVOR_LIKE": "rhel centos fedora", + "CM_HOST_OS_VERSION": "9.4", + "CM_HOST_OS_KERNEL_VERSION": "6.1.110-1.el9.elrepo.x86_64", + "CM_HOST_OS_GLIBC_VERSION": "2.34", + "CM_HOST_OS_MACHINE": "x86_64", + "CM_HOST_OS_PACKAGE_MANAGER": "dnf", + "CM_HOST_OS_PACKAGE_MANAGER_INSTALL_CMD": "dnf install -y", + "CM_HOST_OS_PACKAGE_MANAGER_UPDATE_CMD": "dnf update -y", + "+CM_HOST_OS_DEFAULT_LIBRARY_PATH": [ + "/usr/x86_64-redhat-linux/lib64", + "/usr/lib64", + "/usr/local/lib64", + "/lib64", + "/usr/x86_64-redhat-linux/lib", + "/usr/local/lib", + "/lib", + "/usr/lib" + ], + "CM_HOST_PLATFORM_FLAVOR": "x86_64", + "CM_HOST_PYTHON_BITS": "64", + "CM_HOST_SYSTEM_NAME": "scc132-gpu0.novalocal" +} \ No newline at end of file diff --git a/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance_console.out b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance_console.out new file mode 100644 index 0000000..e69de29 diff --git a/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/pip_freeze.json b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/pip_freeze.json new file mode 100644 index 0000000..3c5a576 --- /dev/null +++ b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/pip_freeze.json @@ -0,0 +1,46 @@ +{ + "pip_freeze": { + "absl-py": "2.1.0", + "annotated-types": "0.7.0", + "boto3": "1.35.45", + "botocore": "1.35.45", + "certifi": "2024.8.30", + "charset-normalizer": "3.4.0", + "dmiparser": "5.1", + "filelock": "3.16.1", + "fsspec": "2024.10.0", + "huggingface-hub": "0.26.1", + "idna": "3.10", + "Jinja2": "3.1.4", + "jmespath": "1.0.1", + "MarkupSafe": "3.0.2", + "mpmath": "1.3.0", + "networkx": "3.4.2", + "numpy": "2.1.2", + "packaging": "24.1", + "pillow": "11.0.0", + "pip": "23.2.1", + "psutil": "6.1.0", + "pybind11": "2.13.6", + "pydantic": "2.9.2", + "pydantic_core": "2.23.4", + "python-dateutil": "2.9.0.post0", + "PyYAML": "6.0.2", + "regex": "2024.9.11", + "requests": "2.32.3", + "s3transfer": "0.10.3", + "safetensors": "0.4.5", + "setuptools": "65.5.0", + "six": "1.16.0", + "sympy": "1.13.1", + "tokenization": "1.0.7", + "tokenizers": "0.20.1", + "torch": "2.5.0+cpu", + "torchvision": "0.20.0+cpu", + "tqdm": "4.66.5", + "transformers": "4.45.2", + "typing_extensions": "4.12.2", + "urllib3": "2.2.3", + "wheel": "0.44.0" + } +} \ No newline at end of file diff --git a/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config.json b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config.json new file mode 100644 index 0000000..46db96a --- /dev/null +++ b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config.json @@ -0,0 +1,7 @@ +{ + "starting_weights_filename": "https://armi.in/files/fp32/model.pytorch", + "retraining": "no", + "input_data_types": "fp32", + "weight_data_types": "fp32", + "weight_transformations": "none" +} \ No newline at end of file diff --git a/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/user.conf b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/user.conf new file mode 100644 index 0000000..ae73424 --- /dev/null +++ b/open/scc132/measurements/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/user.conf @@ -0,0 +1 @@ +bert.Offline.target_qps = 1.9896394 diff --git a/open/scc132/model_mapping.json b/open/scc132/model_mapping.json new file mode 100644 index 0000000..0f13f35 --- /dev/null +++ b/open/scc132/model_mapping.json @@ -0,0 +1,3 @@ +{ + "bert-99": "bert-99" +} \ No newline at end of file diff --git a/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/accuracy.txt b/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/accuracy.txt new file mode 100644 index 0000000..fac4236 --- /dev/null +++ b/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/accuracy.txt @@ -0,0 +1,8 @@ +{"exact_match": 83.68968779564806, "f1": 90.87487229720105} +Reading examples... +Loading cached features from '/home/rocky/CM/repos/local/cache/f085891876124ec9/inference/language/bert/eval_features.pickle'... +Loading LoadGen logs... +Post-processing predictions... +Writing predictions to: /home/rocky/CM/repos/local/cache/65a2412105e94e95/valid_results/scc132_gpu0.novalocal-reference-cpu-pytorch-v2.5.0-default_config/bert-99/offline/accuracy/predictions.json +Evaluating predictions... +hash=e95733a12d75d696f73d0b0de16ea6d8763b083013c9aeea30fdb9aae5146cad diff --git a/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_accuracy.json b/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_accuracy.json new file mode 100644 index 0000000..02b04b2 --- /dev/null +++ b/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_accuracy.json @@ -0,0 +1,7 @@ +[ +{ "seq_id" : 0, "qsl_idx" : 4119, "data" : "0AC4EEC01169FAC0EB16FDC0CD2225C1DDA9D0C08B3F24C1642802C14BA216C1945913C1FC0E18C194300FC1495620C14E730CC18C831EC16BBB16C1845814C1627A16C1C3BE13C1239D13C1452A12C1E61F14C1951418C104B110C133311FC1C5F219C1051D10C1E45E1DC1C3521DC106DB14C166AB18C1BCF60CC07F28A7C0DFEDACC0E959A2C0178AB0C00B819EC0E35BFEC0F59EF3C0A67CF3C07022E4C00C8503C1955C03C1A9DAE7C0EA1C07C1C760EAC0D470EEC0041F0CC17E5B10C14E6EC4C0A7EFF1C0AE32DFC06329C0C072B715C1FD190FC1F499D1C0DE810AC1DDE4C8C06E2307C1825510C13DEE1BC1ABD2E3C0D99DCCC0DBC8FFC0C9A70CC1AA8CD0C0DDE905C1D8E595C0D8A7D5C06A65CFC0333FABC0BD0DC9C0D3C7C4C08BA7BCC06B2001C13670BDC08BD208C1F85EBBC077B5F0C0FCA009C1B55213C1A523A4C0410604C12D47DFC0C4D5C1C02D26DDC00D3E13C1B41E02C1C84B12C1E17CCBC04B5703C1D715E8C06F25C3C0871216C009BFC3C06346BBC0BF74AAC0175FB1C0BF078EC00CE6FDC0FB2108C11CFC0BC1794104C1ECD201C12F1EE8C0666004C1D69B09C1E1C6E7C028FC09C18375EAC01D59ECC008BC0EC1137312C127D2D7C03FCF05C1E387E8C07B1DA4C06B3DCAC0FD79EAC04CA67BC0E7A4EBC0F9A5A9C0224FE0C0F0CFFBC063851CC1303234C0A075AAC0A7DED0C0C156B1C05D3EBBC0D15A70C015CE03C1BFE0F1C0C71BE9C0560FEEC056B806C11B3600C128CD0AC1280FFCC0BD3CDAC019FE10C1C441FCC0DF58CBC0A587FEC076D407C1D193E8C0990C0DC1E1C5E7C0AB0EE1C0305110C1BD4E10C17484B8C00D38F2C0652FCDC0F9716CC046F2EDC02F4F97C0233DBFC0D76AFAC0F9F5E6C0B12D01C18EEC8CC0B4F9C7C0E37DB7C0756979C06706B9C05943BDC022FF6AC04CA2E2C0942729C0F5E8D2C0ABFFA2C0C7DB9CC05D62AAC0AD21C9C0CF09C2C089DEE5C0E72DA6C02DDCB5C01F3ACDC034E6DAC0954EBBC07D5FE3C08A449CC07F15E2C02556CDC0553E9BC07F35C3BEAD9BC4C0F1CAAA40E9E987C0FF7AB44002B66FC0C98B573F083AA4C0B0A0B93E0F7FB0C007FC3CBF426561C07B338FC005BBD3C00883C8BF9DE6ABC09E752AC0F93E3EC0F4852FC04DD2C740573260C048F14E406224ACC0148200C109B096C0C8E2EBC0615FF0C08A90F6C009CE12C1948C15C1C18FF5C0715CB6C079FA09C1495D07C16C2E00C1471C0FC1A3FFFBC0EF6B08C1EFB5F8C04D80F8C0AD4604C1E3C20AC104D90EC148EF13C12461FBC0CB100FC1BE1F5AC03BD0F2C0AFA5E5C06784EFC04563F9C0E1F4E0C0DF23F0C03106DFC0F053E1C0F62F0EC1AA9B08C190A80CC10D0014C1C62E1AC195DFA1C04118F7C055D8F3C04F07EBC0E992FAC0291DE3C092D50CC1C5F116C19FECEDC0556614C1310509C1072002C1CF7410C1133E17C1548201C1EA4B15C10B92FAC0F00C13C1B6DC12C105051BC1CB2705C1A536E3C02B630FC15FD412C18B32E2C0FDAD0FC110A306C1A74CF5C00C5B0FC1515719C1AE96EEC093AF13C1625900C16EF50AC1CC4B0BC1A6921FC13BB482C0A5F7DDC0A742E9C0FFACEEC0ED73FFC03465F1C0291E92C03131ED3EAEC602C1D4A80DC168651DC163AA21C1B2711DC134A021C146861DC1718F21C14C821DC1C98321C1A6651DC1BAA121C198281DC1ADE221C140391DC1E5D621C1BE2D1DC1C7DC21C1834D1DC145C221C116971DC1516D21C10E841DC1E18221C1E1661DC111B121C1AC5E1DC183B621C1BE3E1DC1E1D121C1088C1DC1997B21C14C921DC1C37021C1C88F1DC1716B21C190C81DC1B33021C1F8C61DC1CB3F21C1DA9C1DC1077021C1C49C1DC1C26F21C114931DC1617421C1AD8C1DC15F7B21C189391DC1FBD621C1AD1D1DC107F421C1DF411DC125CA21C11C331DC145D721C1C4741DC11E9F21C1DD6C1DC1DF9E21C19CED1DC1E06920C1757A1DC1597E21C14E681DC1DFA621C142C11DC13BFB20C1F86B1DC1BE9A21C13C671DC17AA621C1DF721DC1BDA521C110621DC1AAB321C138731DC1619521C1BE611DC11FB321C1B45E1DC13FAD21C1BA9C1DC1076B21C1728D1DC1848621C17CB21DC11A5321C19EA51DC18E5E21C154941DC1057421C15A4E1DC100C721C171531DC1D7B921C174461DC101C921C1842C1DC1FBE121C1F94E1DC100C221C1F0761DC1139521C17A671DC157AF21C1F26C1DC19C9F21C100521DC14CBF21C1685A1DC12BB021C10E581DC1EEBA21C1C8791DC19B8521C1865F1DC14EA721C1D0781DC1159121C153721DC1CA9A21C1DB741DC1709021C1B5841DC10B7D21C12D3B1DC1A3CE21C1B42C1DC187E221C1A9441DC148C921C101761DC1B39721C11AD51DC12ACB20C195861DC1177921C140651DC169B021C1A1F81DC1434320C1946B1DC1859E21C19E5D1DC1B7AF21C1A7761DC16B9821C1FE661DC15EAB21C179561DC1B1BD21C1B6601DC161B421C1FF5F1DC1B3B821C13D821DC1F19121C102631DC177BA21C157661DC123AA21C1664C1DC149B821C15A6C1DC1F59C21C1E0A01DC18D5A21C171781DC1779221C108AF1DC1755021C1F1A41DC1846521C1C89E1DC1976A21C1B4711DC1D39C21C1D25B1DC187B321C140761DC13F9021C130441DC107CD21C138421DC1A1CB21C1C85B1DC1F0AF21C1FE481DC15FC821C1BE7F1DC17F8821C1CC8F1DC1937621C100C91DC1E4E320C1E0661DC1A8AB21C11F851DC1337F21C1AA4C1DC12DC821C164411DC1A4D821C1E1531DC159B921C1FE661DC106B221C137681DC1C1B421C1BA871DC1A98021C132701DC1EBA521C1D7A61DC16D3421C157 + +... + +C1E93B1FC154A41EC1BD501FC11D861EC1D6511FC1EDD51EC18B011FC1EAE71EC1E5ED1EC1ACFB1EC135E21EC1A0BC1EC1831F1FC144D71EC19F0D1FC1E0D31EC1760D1FC1F1D01EC1D1151FC1D3D11EC1FF041FC156E31EC125FC1EC18FE01EC1B7031FC153F71EC1B4E51EC11EE41EC193021FC198F41EC1A3EA1EC1A50B1FC163D51EC184F11EC1F1F11EC1A6F91EC157F71EC1484B1FC149941EC120131FC128D81EC18E1C1FC109B71EC119F41EC19AE11EC1C9E31EC1DBF31EC1E4DA1EC125071FC108EB1EC1BBDF1EC18F1F1FC19CB71EC17A451FC1D9821EC156E71EC1E4F31EC194CE1EC1D31F1FC13EAE1EC1B7391FC1BFD01EC1D30F1FC1E2C71EC141181FC108D81EC15C021FC184E11EC15FF41EC1F4FE1EC154C71EC14AFD1EC112DD1EC17AEF1EC18FF01EC1A00F1FC13FC21EC13CE31EC1E4031FC1A9E61EC1FD051FC150ED1EC140F91EC146D61EC1980D1FC159CE1EC1D9181FC14CE71EC198F01EC1AEF51EC153E41EC169EC1EC111F31EC102011FC1DDD61EC107241FC103AE1EC1CB0C1FC1E7CD1EC1620E1FC192CC1EC106171FC10DC11EC1F4101FC1A9E01EC117FE1EC117F31EC1ACF41EC12BF61EC10C0A1FC17BE11EC154E61EC144F21EC1242C1FC1BFA91EC1392C1FC12FB81EC1FAEA1EC191FD1EC151F71EC1E1E91EC173D41EC129141FC1E2DA1EC151061FC1B2DF1EC119001FC139EF1EC1DCEC1EC1F0001FC18FDA1EC187F41EC165E71EC13E161FC1BAC31EC1C4EE1EC1DFF81EC120DC1EC1E9091FC1FBF01EC12FF51EC1C0F01EC1AFF21EC138EB1EC19CFC1EC172021FC1EEDC1EC11D4B1FC1A98D1EC141E51EC1D5191FC12CDA1EC183201FC139FF1EC155E21EC1CFC91EC14D301FC11CE51EC1CA071FC1A3E41EC127F61EC118DF1EC15B011FC1E2C31EC189351FC12AC11EC136421FC1CD1A1FC1F5B41EC186091FC127CE1EC162E41EC144071FC1E2101FC14BBF1EC1B6081FC16BCD1EC1A6E61EC1D6031FC160031FC1ADED1EC13B711FC1A7701EC1F0F71EC137FF1EC1B60F1FC1FDDD1EC1B4001FC111FA1EC1704A1FC1F18C1EC110C91EC166271FC11EBF1EC1A62B1FC115DA1EC1080C1FC119EC1EC126F21EC1FDF61EC1CFE61EC1E5001FC135DC1EC17A0C1FC1E6CF1EC1E8EE1EC1A1FD1EC1A5E11EC1410D1FC165E81EC1EB031FC165EA1EC107021FC1FACC1EC1D3291FC110F01EC12DF61EC144EC1EC1FFFD1EC140DD1EC1ED191FC160F81EC19AF21EC1CEC71EC1F0321FC1EAC01EC1DD391FC1110E1FC16AC51EC111021FC1DBD71EC165EF1EC1ADF01EC1A8141FC185B81EC155071FC157CE1EC1FAF01EC15B031FC197EE1EC106051FC1F7141FC183C71EC1C4221FC1CDC71EC196D91EC1951C1FC1660D1FC10BC31EC1E6041FC12DCF1EC1F6F41EC191031FC134FD1EC180E71EC1A6631FC138741EC1E2391FC145A61EC1FEFB1EC1A0EF1EC1C9201FC107BC1EC142081FC1BCDB1EC138EB1EC107001FC132031FC1DFDD1EC132F81EC1F6ED1EC118EC1EC1B8011FC120F41EC1F7F21EC15D031FC1C9E41EC190D21EC1191F1FC1FFF21EC1AFF31EC1AEEB1EC152F61EC144CB1EC159291FC189061FC1D6D81EC1E2151FC133D11EC16FEF1EC10C071FC1B7E91EC1FF091FC1BDF81EC17EEC1EC10BDA1EC1F81A1FC104F91EC129F21EC11BFC1EC19BE61EC106C21EC1F7321FC140F51EC1DBF11EC1D6071FC143CD1EC162F51EC1E3DF1EC1B40A1FC19DD71EC198581FC15C8B1EC1F44F1FC1C1961EC100831FC19B3D1EC182F01EC19AFF1EC152EA1EC13F0E1FC12CEE1EC148FE1EC1F2EE1EC1B7F11EC17CD11EC1F71B1FC1D7DE1EC1250F1FC1B7FF1EC1DAE01EC15FE21EC17D041FC17CC91EC1A11D1FC147ED1EC195FB1EC15A0F1FC17CD31EC17FDC1EC145171FC12AD81EC15D211FC199EE1EC1DEFA1EC1FAE91EC13D091FC16EDB1EC13D221FC108EC1EC12E041FC128F21EC148F71EC10DD61EC1601D1FC113EB1EC175FC1EC1DCF71EC1A1E61EC101E01EC167061FC164E01EC129201FC143231FC15BB81EC10CC71EC195421FC16DE61EC1900B1FC138F81EC1C0E31EC191DB1EC12D081FC17EDA1EC161161FC18DF21EC141F41EC1EBE61EC133111FC13A021FC147FA1EC12DF01EC14DF21EC10BFB1EC12FE11EC1A2E91EC143F91EC15FD21EC10D211FC13AFD1EC108E41EC19A0E1FC18FD11EC1CCEB1EC1F3181FC13EFF1EC15BF41EC17A441FC18CA41EC14AF91EC143FA1EC181E11EC128181FC1F5C81EC1F21E1FC11ADF1EC1DE161FC10AD41EC175281FC10AE11EC1771E1FC11FBD1EC1BD451FC137FF1EC115E11EC1A4001FC1A3DB1EC1D9D81EC1AD0F1FC171E61EC175081FC192041FC170DE1EC146D61EC1272A1FC138D41EC12E2A1FC1EBE21EC13D061FC100CA1EC14D1F1FC160BF1EC1F52D1FC1A0C11EC11E2C1FC145EC1EC1CFFD1EC133F61EC1DBF91EC134F91EC19EF61EC163181FC1D7D41EC179C81EC17D421FC1DAEA1EC1590E1FC1B9ED1EC133091FC1B4CE1EC150251FC19CF31EC18DF61EC1FAE91EC1EFFA1EC12FDC1EC1D7161FC154CB1EC1D9351FC1C6051FC1B1D01EC17BFD1EC1E1D41EC14FE11EC167081FC1D80E1FC1A9F01EC180691FC13E791EC15C6D1FC1DA691EC1BADD1EC129181FC10BFC1EC121F91EC134441FC1F1A41EC188D01EC1332F1FC16EE71EC1640A1FC181031FC1CDE11EC176EA1EC1C80C1FC1FFDE1EC1EF1B1FC120141FC11BDF1EC1300D1FC12BE71EC1CEC71EC103391FC137E51EC14A0C1FC158ED1EC14FF81EC18AD41EC169221FC1F3E21EC1B3271FC1FA3D1FC193A31EC1B2001FC1DE021FC1E6271FC1E6C71EC156021FC141E51EC11CF01EC13DF31EC1" } +] diff --git a/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_detail.txt b/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_detail.txt new file mode 100644 index 0000000..f27f4ba --- /dev/null +++ b/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_detail.txt @@ -0,0 +1,70 @@ +:::MLLOG {"key": "loadgen_version", "value": "4.1 @ f5c8f17583", "time_ms": 0.004398, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 53, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-10-21T23:43:42.761999", "time_ms": 0.004398, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 55, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-10-21T23:43:42.762007", "time_ms": 0.004398, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 56, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-10-08T18:30:16+01:00", "time_ms": 0.004398, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 57, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "loadgen_git_log_message", "value": "f5c8f1758374aeaba26b2e84d31690111cfdf054 Fix bug: Loadgen ignoring token latency targets in user conf (#1874)\n976bb1ad9c7946be79507f3ff67955c27426af52 Set correct remote repo (#1871)\n41fa8aadd1ba0ecc97f6a519d8b42b04278e5f24 Add format files github action (#1682)\n518b454fd8647bfbd23a074e875e87353f33393e Tflite tpu (#1449)\ne0fdec1c7a75c98cfc194f13d62ac4388d419c8a Fix link in GettingStarted.ipynb (#1512)", "time_ms": 0.004398, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 58, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.004398, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 60, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"d5274ff0b56e8d3cdb273174628a4461fca6f02a","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"ea4c89decad19eaf3217bfa2fb757d3b83a561d6","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"ca17720f9c8246e821331946d893e830fc88f8bd","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"34e2d2a44324cb07c884f92146ecbb8ef9d704e2","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"13c49b028b22749b5f3c44f3d9bb489e8c0574e9","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"c15c3e150030089a8d634bd2ad6d4b644002e613","/test_settings_internal.cc":"e21febd60f9b5bedd1fc81bb990f09c34b32043c","/test_settings_internal.h":"f1d5335b53ca610c30e0edc5d07999a27b5b4b9a","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"eea9b9cb1a06cd1abe1bbdaee82f9af31527fedb"}, "time_ms": 0.004398, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 67, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "test_datetime", "value": "2024-10-22T01:21:15Z", "time_ms": 0.013385, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1198, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "sut_name", "value": "PySUT", "time_ms": 0.013385, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1199, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "get_sut_name_duration_ns", "value": 531, "time_ms": 0.013385, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1200, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "qsl_name", "value": "PyQSL", "time_ms": 0.013385, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1201, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.013385, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1202, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.013385, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1203, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 270, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_test_mode", "value": "AccuracyOnly", "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 271, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_offline_expected_qps", "value": 1.98964, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 308, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 314, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 315, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_min_query_count", "value": 10833, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 316, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 317, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 318, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 319, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 321, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 322, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 324, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 326, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 328, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 329, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 331, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 333, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 335, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "requested_sample_concatenate_permutation", "value": false, "time_ms": 0.020779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 337, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 413, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_test_mode", "value": "AccuracyOnly", "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 414, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_samples_per_query", "value": 10833, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 416, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_target_qps", "value": 1.98964, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 417, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 418, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 419, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 421, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 422, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 424, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 425, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 426, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 427, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_min_sample_count", "value": 10833, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 428, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 429, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 430, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 432, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 433, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 435, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 437, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 439, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 440, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 442, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 444, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 446, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "effective_sample_concatenate_permutation", "value": false, "time_ms": 0.020919, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 448, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "generic_message", "value": "Starting accuracy mode", "time_ms": 0.031629, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1090, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "loaded_qsl_set", "value": [4119,3771,2947,5174,8221,10562,9791,4633,8569,8837,7091,5810,6371,10778,540,1732,2832,4085,676,6115,887,9339,10819,344,9312,1951,5739,9692,9525,4397,638,4375,10601,8825,6906,3471,9163,7272,9940,1207,7738,1663,9660,5625,2468,396,7153,8514,1418,8510,4187,1103,9970,10200,2141,6364,9748,8412,2719,5615,2838,7835,752,6099,10486,8982,3673,7292,10573,3118,9998,6825,5831,4451,9259,4859,1199,7842,6587,920,6687,3869,1033,6262,1223,30,5009,10063,2956,1007,2185,64,77,3470,6239,6679,1470,62,3074,1622,8153,202,9235,1614,9392,4912,4968,1648,122,3772,902,3526,1772,858,265,5129,6799,6255,523,10796,8819,1230,9400,3732,1321,7882,7821,9768,6056,4282,4617,6107,10130,7756,3036,10451,2201,3976,1086,3668,5394,3177,5128,8384,7234,4039,8503,3640,10043,5766,6810,1011,7547,1718,2401,3909,8343,5923,10069,9886,532,3270,7147,10062,9680,5432,9974,7736,3655,811,1105,1820,7465,2267,9641,4046,3956,5247,5181,7625,4073,9663,288,426,6762,1617,7914,4014,2904,9151,4181,6042,8967,6981,5547,10457,9322,500,938,131,8457,1171,3814,10549,8093,10675,2159,5807,7567,8366,6518,943,6819,5889,5108,7755,5382,8456,9756,10247,9351,3150,8971,1231,2990,5380,510,4324,3049,1165,6116,8684,6992,1829,139,1021,1602,5805,4226,5966,1040,3274,4269,10376,5316,8756,9587,8231,2809,2270,7887,7390,485,8028,7880,328,6236,514,4644,6061,1522,10537,9446,8787,8577,6774,2023,7474,4904,5064,6183,5196,3361,8268,7606,5968,6182,5019,8265,4621,1525,8181,163,1352,5785,8632,1074,2392,8398,6561,125,7071,5577,995,3783,3849,9753,8651,1051,1159,3971,8903,8413,162,5854,2133,1416,9098,1759,633,5800,7671,4731,10623,5586,10578,8929,5894,6833,2909,5063,3163,5002,6278,4148,7477,8408,10104,8805,1504,8105,5746,3572,8095,312,7436,6165,2786,6286,2079,4807,10523,6117,3840,4169,8750,3016,10238,1137,10037,9427,8449,4908,3020,7183,1557,10500,2982,8020,599,1519,6873,10276,7350,8224,719,8474,9981,9198,2802,5756,1897,10801,7682,91,4160,6697,2989,10113,5765,982,48,6573,4412,5148,7681,2514,6348,620,6702,250,10219,7813,4512,3988,7538,3272,4433,4597,7580,8229,603,9284,9959,4683,711,7542,5450,1175,9859,10685,5041,9760,765,6303,3395,2908,7823,7516,237,105,8034,1757,10401,9935,5256,7800,2642,276,8636,5297,10766,5954,5725,8144,6660,3901,8078,4158,9455,7297,5959,8088,8583,4733,10345,5232,9216,8902,4667,1449,801,9231,1792,1211,6013,8566,4988,2938,4730,404,8699,2346,3059,672,3701,7231,10169,10431,5806,1373,4193,10665,10030,4335,7950,10150,842,5753,6435,5096,5922,5497,3193,9367,2158,3816,5356,65,258,4675,1277,2041,6032,10088,7814,8726,2036,7885,9303,7440,8023,1151,5749,9573,7776,572,1240,6297,6439,1343,8605,3386,3251,4631,8140,3760,2068,4957,2623,4961,10281,1467,9037,10546,4415,215,2194,3273,387,3327,2225,10739,300,4467,10823,10668,710,4228,4517,2069,8709,2409,2573,6144,9357,4844,3985,2864,2814,2615,8952,5085,5090,1937,1854,10689,5061,10374,1580,5223,4851,7888,869,3023,10178,1615,7566,7219,5276,6685,8839,10534,10532,5826,1251,9124,8308,8702,1805,3148,3529,9942,8171,2450,6782,7679,2503,9989,2748,3961,3660,9466,6832,1635,1831,9298,6457,3165,6633,6513,10354,7865,1888,1182,10643,5477,3120,7676,6178,2833,9468,4910,2657,9723,492,3347,6481,2822,4426,1018,486,826,3455,1535,1539,5782,8267,10102,2714,5573,5482,2442,537,6934,9420,1925,10526,8158,583,5921,5634,10715,2063,2865,4722,6711,3033,5830,6761,10221,3292,5093,3487,9844,5192,3127,3166,5517,8842,5986,6515,8806,4368,3880,682,1506,8542,6600,5522,10789,2775,6979,7375,9732,9488,7435,6127,2058,8763,5776,2901,551,2871,7369,6929,4164,307,7840,861,2670,8027,9410,2386,2513,1063,5122,8218,4457,1100,2437,462,5774,9629,10033,5794,4106,3616,7179,5414,4598,9925,9289,69,9948,9945,4585,6214,1953,6363,7744,6030,3012,37,4855,10794,2109,5500,1652,9019,7047,3763,10114,2735,4137,7192,6450,7896,6319,2559,9057,4150,1626,1399,3805,10481,7054,3160,3802,2753,643,4212,10435,3360,157,8424,6667,1686,8112,1983,6648,9535,2567,6149,8099,319,10795,2980,8559,3326,2116,2360,7544,5778,2589,3310,3751,3352,4366,9705,368,6088,8217,2399,6537,10120,6669,3061,1457,5252,7052,10159,6880,5593,8852,3506,5847,4877,4944,8609,5364,1816,5290,634,1661,1221,6935,702,4824,1453,8062,7092,345,7030,9428,204,3121,2188,6429,7446,10631,2354,2569,1219,5475,5727,6274,1556,8968,7730,10342,7917,1985,6856,5692,7132,1146,10468,6861,1550,2928,6585,5681,5352,5348,10265,10307,322,9139,8357,9486,7441,4023,10141,10170,1441,165,9603,5998,10586,9022,5633,1234,6283,2131,7996,5042,6246,4201,3064,3071,5674,6769,9698,386,9319,10686,4,9197,3510,428,382,6614,7688,6154,10078,2876,8185,5240,1006,8696,4353,5760,7961,8376,6405,2107,5804,9860,3812,1679,2756,9652,7964,750,1997,4091,1899,1308,4035,6656,8123,10248,1510,2846,5638,477,4076,10730,10589,8266,3218,4977,2536,5334,10054,8762,8598,3929,3811,9581,7752,552,9441,5711,6642,1129,284,976,10747,933,7929,7607,9805,5102,9979,2939,6910,2647,4835,7528,4876,6734,2706,4394,4926,8341,8734,1135,3753,9333,1291,6011,3867,5563,3343,7612,1027,5165,9700,6340,6104,3099,9537,3428,1612,1606,10634,10021,2761,1042,9310,3601,6806,4003,8336,1320,8874,6545,3842,3715,4916,177,6141,922,10477,10369,3372,8706,7261,9000,2944,1770,4537,8671,8299,4515,6709,7300,5017,1499,3864,4145,8725,2669,6292,6724,7960,6080,4257,743,1363,10444,4518,9545,10388,2296,3870,3737,984,4590,5924,149,1966,10084,6018,1340,8785,1678,4211,8663,6542,1475,6302,7202,1850,9261,10681,2339,8461,2664,7011,3441,9549,1482,9314,6828,1290,9203,7486,209,5215,2113,714,6092,3889,825,7966,5609,3787,6223,207,6965,2644,4612,9140,10280,472,5726,5208,8772,2044,2931,1775,4134,10539,7508,172,8681,8240,10189,4870,8908,7684,9379,3266,3376,8405,10311,453,8213,2034,2190,2294,8580,4421,4725,3952,2918,8999,8545,7043,542,10301,9146,9032,3597,4112,2062,3195,2850,6169,10805,4900,8382,2059,5703,10074,7541,10662,1760,2114,10749,5993,6295,5211,7792,4639,8333,3833,1847,83,10770,8312,8831,9004,371,4923,7699,3307,7834,5879,7458,6735,9799,10741,1304,1576,8657,3329,8860,6081,1744,7308,10564,7102,10611,7126,10413,6653,6846,1371,6864,2078,8362,3021,3569,10777,1939,2861,8085,4899,3288,7360,7690,1489,5526,116,7151,5145,6673,5523,9390,8191,1004,10528,8958,9118,7393,4684,7266,6966,5995,8508,3778,313,2324,3282,10165,3447,6294,6865,9833,2202,6316,365,3052,7057,10821,1194,7974,1410,1782,9986,1123,5591,7286,7552,10721,1088,1793,6524,117,1763,8854,2199,3366,6230,6135,4113,4154,2326,7696,10438,4546,6268,4991,8670,9515,9500,6272,5253,2634,10092,965,6654,316,7333,10361,7385,2359,2521,10505,6418,3731,6452,4151,2002,4407,9193,5715,5311,9681,1461,6752,10196,9063,7064,7760,10356,9534,6401,9570,6213,7243,1969,5069,4463,8379,2228,10617,10563,4001,7533,4118,8634,303,4758,6480,6863,4079,3551,2387,959,10384,8204,3739,4388,9571,5235,890,3448,3380,4022,3407,3303,8225,1710,3134,8655,8273,1442,9126,9090,767,2352,5036,6959,1116,7507,8320,206,5100,10580,4657,10013,591,5970,3983,10637,3489,6177,4329,9309,598,3799,7447,4206,7727,6138,480,3483,3979,1719,5575,5814,6507,10156,4769,9437,3749,949,10166,9129,1387,7559,21,3606,2329,10379,71,6909,4268,3231,7838,4403,4880,2289,4825,7313,3855,9482,5669,5755,1472,8584,145,7282,4105,5546,9644,1811,10620,8543,8840,6823,3084,9331,7771,5570,2129,3333,9811,1567,9316,6617,7070,2971,5115,9806,8920,6889,6597,9550,7991,10076,1289,8872,1156,963,6369,353,4520,4406,4510,327,2672,10123,881,4197,10616,7244,6791,4055,9060,7901,4398,10019,5895,6816,724,6400,3142,2649,727,7658,4411,4131,4893,3203,6901,5131,5905,7199,3408,4562,101,9214,54,2388,5032,6645,6310,973,7184,5595,4177,3578,10735,6783,1468,7622,854,1425,1272,8441,10581,2009,3999,4258,1929,7531,8205,610,8733,2698,10366,9817,7227,10638,1446,4587,10052,9114,3354,10140,677,5614,10068,3206,10826,5460,3063,1093,5083,10551,502,9391,7170,6267,10812,1555,9356,8910,4623,3970,10429,4096,9172,522,7591,8773,2791,5612,3922,6555,8079,6802,5491,4135,6640,7203,373,9221,2355,7947,5119,5940,5363,10443,646,8044,3473,2214,5335,5803,10138,5180,5525,927,5272,2372,2951,526,7209,5972,1753,6877,1496,185,843,1124,8942,5229,1918,10371,1282,6424,10228,3253,354,7137,9669,5618,2061,632,986,1991,9433,5261,5130,2246,6315,3835,5515,2976,187,7121,2230,7683,930,10571,2538,3106,1630,8453,5267,6534,6349,8372,6532,7725,8522,8596,4984,6027,1483,2281,10284,4300,7155,8557,1842,6925,251,4156,2883,10632,4008,8994,3844,3112,5932,6837,4987,4751,7133,9724,4981,6781,5405,8770,1012,9963,9551,9840,5471,3562,454,4696,198,1902,1516,8943,4082,7253,497,281,8464,3677,8635,10172,7900,5238,2747,4914,2487,9376,2618,6526,9801,463,10039,1226,7307,4673,7525,8661,1010,6727,9665,10411,1307,4267,6543,9564,4793,9552,6627,3638,3785,2658,3384,3158,9720,9818,10790,6483,9067,669,3930,7843,5427,7587,7536,7029,3125,3531,6385,1565,8749,1736,3512,5719,4868,888,8255,8535,6194,10174,8304,5939,5070,6040,588,8437,10759,8538,8206,1458,5159,3997,1259,4858,7176,9035,10403,2868,1122,5888,9842,4622,4261,10673,9324,8487,600,9479,6220,3419,195,2155,399,287,9830,9883,9328,2055,7836,10511,1451,6850,5911,3337,9778,10660,4739,5981,8347,3588,9865,4238,8957,5074,1691,6813,3860,3883,8878,7674,8731,7336,9136,7178,4565,7820,10508,2051,8164,823,7624,2648,3128,7150,7687,4830,10350,8119,416,735,4601,5587,120,2691,3920,4192,7125,10053,25,808,9563,4299,10112,5112,3643,5446,7403,8391,7114,1542,2628,8911,9279,2886,3221,10056,1912,3291,5819,3830,8896,10788,4000,5815,1183,355,9074,8086,5140,10657,2345,2550,8728,8468,697,10557,10322,8004,1473,1698,4574,4635,1921,261,5606,3409,9595,8966,5447,4599,6622,7714,5381,2778,9396,7374,10654,1346,5113,1536,6553,1306,1932,3216,656,10615,6053,96,4447,8775,8624,8685,2235,7956,4390,1420,220,6145,2255,2478,2730,7084,1155,9091,5757,8924,2955,7628,2093,1334,1068,8013,1927,7841,9731,10436,809,6519,5217,174,2169,7952,10341,1776,10629,3169,8108,3635,5671,8667,1653,10761,1309,3659,7039,8056,2827,42,524,10646,921,2283,449,9983,2561,4542,4260,1725,9512,4540,1378,7252,2161,6347,937,8455,7862,5008,10109,1246,7697,6176,2816,1824,1746,734,7320,3564,3892,9662,2526,1553,8188,7826,3837,429,6706,2945,9511,3466,9636,6688,2453,8254,8438,8321,2699,5091,3521,3229,9219,4792,1859,9449,5190,8178,3073,2781,5817,8737,1855,2849,3602,79,6723,10289,8429,3626,7006,5884,3852,2315,5409,8693,1762,4978,8377,9892,3182,1361,2626,2581,10619,115,7025,9240,2236,2317,3641,1835,7551,4979,7791,6767,8032,7159,3000,1145,2175,549,3987,1748,8906,7038,7915,9995,8563,5741,9152,5391,7122,7933,5747,1511,8215,948,15,1029,6760,6874,6170,1801,9164,9377,8369,3748,10277,7497,7145,4724,17,2694,1041,1682,3788,8184,1015,8296,2167,4999,4809,10396,9853,4018,2086,10584,10089,3002,5841,5318,689,7581,7680,3427,950,10466,2087,9890,7666,4523,8530,1538,10784,247,6301,7460,3972,684,7646,8883,8024,1238,6393,9716,618,5104,6728,6168,7797,10273,7164,670,10252,85,6602,1693,2527,4274,3552,2284,4290,6156,8784,274,1620,9382,5000,110,10290,1326,4785,8644,9321,8743,9,1878,9711,9425,8764,6691,4962,9498,10727,2959,5262,10066,372,1336,4845,7753,5342,2047,4024,2708,7710,8639,7472,8668,337,3094,4172,9903,7644,10831,10082,4625,3043,4471,9277,1133,9577,1523,8489,1841,7878,2932,9616,7479,2448,8165,6033,4630,1946,5486,9327,9442,6824,2104,6855,8319,8378,6456,6623,3915,988,2662,7009,1331,4094,2812,796,9735,2396,3797,9481,8897,3743,8803,1808,8879,6637,9233,3703,10278,3136,10094,81,2825,23,5436,5321,3824,4891,10231,1138,6722,5433,10553,2430,2984,1167,7110,9750,10009,8406,4659,8768,5678,8435,6988,1727,9161,9051,8582,4297,1815,5707,4350,243,1708,567,4490,9397,4652,10440,667,2578,9330,7469,8600,7356,9971,8486,1243,9547,6352,8556,19,9780,4502,2964,10460,2851,7626,2804,3746,58,2929,2472,4481,10015,4514,5012,6768,8136,7804,8161,5094,1206,7412,87,1022,3011,1545,6527,4404,1587,219,6490,3575,2132,2170,8614,4632,1562,7478,1779,2568,2949,10606,5660,1261,8182,8310,9657,2565,10346,822,2574,9905,4513,942,7786,1994,3533,6809,6583,3425,6015,7013,6397,4259,6497,9677,2497,3363,1940,4265,6946,3088,1401,10722,10732,5925,2685,6250,3914,4958,1876,10051,7221,26,84,2499,10705,4400,3723,5910,10250,5623,6564,4077,6725,6325,9633,9992,437,9142,605,3683,8832,8260,9272,4071,5646,3230,3596,6495,5696,4247,10351,5313,5373,4495,9682,9607,6146,358,4643,8625,3709,5207,5780,2967,5613,9068,3608,10710,6708,1898,5621,3131,4750,10125,3496,103,5264,6034,10680,9190,1349,3828,5604,9517,9102,8439,9832,8173,6119,3318,5786,3492,401,9618,6737,9246,10146,3680,7759,8436,1478,3378,3003,1660,925,9796,5173,1044,2008,3475,865,6017,718,9793,1592,9430,6229,529,7481,95,7128,9088,8572,4171,2859,8960,8950,3438,7131,5,8015,10358,67,8337,807,6106,1295,6866,9463,2720,3919,5147,1892,9815,10195,9138,266,3848,3780,1271,9627,5332,1521,10639,3556,9904,7271,3891,1213,8989,4234,563,4263,3480,10700,5545,7916,9350,9786,6511,4021,6922,1984,4254,755,6461,3650,5958,5689,3435,7761,3648,8174,8881,2716,6508,6386,5843,3153,6167,5457,8103,7101,5161,7225,3116,6335,6232,3624,6746,7037,10058,489,7847,812,6876,2727,7260,10814,4133,9238,10395,8947,4124,4669,875,4427,2431,4726,3733,3001,3539,768,8472,4334,6094,7578,6868,7276,9502,6084,7087,402,9412,9610,10335,6365,2290,9256,6903,7270,1972,9424,10448,3560,6410,8419,4920,5914,5584,9059,9529,562,4294,5088,6380,5179,5616,9946,6684,4795,4616,9561,7288,1987,7946,7108,8662,5201,7766,6575,1062,3497,9869,9122,961,8317,10115,4600,1059,8,6879,3847,7515,9775,3184,3741,3809,341,9223,6047,8395,2605,4141,2368,1109,181,2919,2252,321,8946,2397,1328,2189,2233,8146,4925,3129,5624,8067,8220,2629,8591,6744,8973,3534,1933,10737,10754,1081,4428,7665,2095,5946,8168,4176,8325,5243,10191,5745,627,2313,5089,9157,10175,1803,8250,4839,4344,1397,2785,8976,4182,7627,2439,8767,992,3594,818,2102,2101,7732,2136,10045,6248,1701,7480,10187,9937,3843,4786,8259,7733,7721,5053,6924,7265,10529,10608,8077,7425,10765,813,4107,5464,8698,8647,3665,581,980,8450,10412,9757,6155,1706,1941,8281,6977,5637,7609,8901,10709,5763,7148,7382,9695,2239,2800,9120,5495,7097,3339,9044,827,50,1384,5652,4019,9507,9295,2200,7869,9741,548,5114,8388,2264,1407,4688,395,5627,7012,2351,9849,3137,6676,1406,8049,10738,7938,6844,8757,1906,1720,824,152,1742,32,8444,3287,6875,521,4929,860,1248,7532,9617,94,6945,8107,8890,2118,7189,6022,154,1980,3854,3140,4681,4570,1392,10217,4753,9345,8277,6969,10410,6610,9759,6796,742,9170,7334,7780,717,5580,5154,4276,4306,8589,6427,10628,915,3066,1846,10121,10220,2508,4302,1543,4611,10453,2889,7579,5234,4531,3198,10398,10020,6225,5270,10618,1303,10518,8779,10461,891,4256,10506,5200,4219,9264,8721,3488,2880,7936,6805,2890,8052,6522,6509,2171,2995,1433,1839,10227,3882,6721,3807,2408,2490,1655,10605,7259,1887,1633,5018,5907,10151,3302,7634,3450,7198,10367,5511,8730,747,9227,1485,916,2122,3422,1130,5716,2056,9262,5693,6798,6626,8951,2751,4995,8045,3931,10260,3342,862,1645,3349,8241,7103,5082,6862,6982,3208,7729,4230,8507,2794,6525,5350,6103,3755,6140,7475,3405,4271,9560,8504,10318,4354,4804,4034,7982,4743,10081,5034,5204,4456,2219,5283,9225,10570,2549,7837,8810,7726,8525,4911,3469,1584,1560,2451,5714,10472,1089,7401,4989,4385,3767,8397,5864,4789,3259,4890,5461,5320,1786,1140,3776,3696,9036,5564,2049,6468,9156,10126,1935,234,9858,9891,8536,1209,4892,8258,8291,6581,6296,10502,3412,10768,1198,188,4816,2370,6120,8353,3903,2653,4473,9807,3336,10450,336,9875,10364,5056,853,9406,2684,2793,40,6549,4146,5640,8555,5304,7418,471,8630,5934,2035,8983,3445,290,3371,4401,8316,10649,5231,164,9503,7449,5802,2757,8777,816,3911,2,6857,9514,960,1621,5752,1908,466,709,3191,9230,6834,6411,6592,7152,4715,8364,918,7236,1398,262,6157,6360,2921,7615,5288,8106,8849,3554,10572,8991,2539,5869,9213,1216,10205,561,8327,7083,6887,3060,4662,10044,4191,6757,6073,7207,560,4918,5459,8732,10017,7135,7652,5535,7610,495,273,5312,3205,6574,2417,584,5836,10755,5353,759,3625,8620,420,7973,2307,66,5398,2085,10086,723,7014,519,5236,5632,8553,6916,6998,8152,1275,3831,10010,10463,838,2524,4967,9471,5851,6093,4443,4646,8309,7431,2676,10055,6830,4125,3918,5589,3010,6421,2099,6933,6436,72,8166,8163,10515,4243,5788,450,10803,9002,6970,1769,2469,3200,8104,9015,1690,6937,855,6928,9291,4941,839,9930,7156,7706,4081,3157,8792,8833,10389,3563,10,4377,3576,4697,7293,1518,6860,10329,9176,9462,8131,9614,5770,7315,192,8974,6949,9467,9960,8516,6686,10083,9837,4736,5790,5619,3704,5021,9476,2830,7667,7796,4310,5560,10482,6927,1338,9168,6215,199,6636,3334,736,2197,4126,5777,4790,8780,8041,4551,9267,2103,7162,828,8017,7190,9703,2127,2927,147,7206,2050,9966,3873,8688,7673,5456,135,431,673,7177,6377,5127,4934,6254,5626,5899,4771,473,7437,1169,6379,6983,2858,196,1437,7010,9086,3399,10046,6596,8526,9415,9897,111,5043,4298,8918,6621,935,6991,3676,9578,6304,4155,9749,4602,7003,2347,2356,3803,4396,3893,2328,8008,9080,3092,6,3868,5686,6187,8753,4117,3827,7839,7802,886,7335,6504,5407,8562,9025,8690,2280,7174,6455,4088,4782,10711,4028,9609,503,9260,9167,1101,6139,6477,4806,9054,8853,6694,2675,6907,5095,3365,10135,2635,2179,6345,3916,10650,5588,3946,7935,9270,8573,1954,2668,7883,5073,9113,7274,8506,5789,1170,3817,2025,2558,2930,6677,1237,10535,3048,5337,5718,4716,3595,5829,4484,1296,1869,1917,8270,1571,2788,5991,1215,1784,4026,10806,3362,10767,8060,7707,4475,9900,10469,10163,7968,647,5425,10308,6002,6493,1364,5762,6698,2545,3856,3924,8431,6247,4392,6777,9694,9804,2686,6563,4577,5772,3144,6822,6884,2836,8850,8094,2821,1948,9586,4533,6110,3712,4199,9103,129,10164,1348,4064,9565,6125,4138,6980,847,5975,169,10600,4323,10072,5913,6375,8035,8155,1563,8328,7201,9934,2410,9856,4423,4637,2398,6290,9363,10699,1833,4767,6129,7556,1208,3982,6244,191,7024,3183,1735,2193,10317,10626,505,445,10257,2710,10330,8720,2534,1699,7061,275,3942,2477,4419,10426,1517,2622,9286,9030,7554,179,7188,5937,9497,1586,9211,5323,6521,9632,999,405,8373,9292,8812,4986,9445,153,3298,2973,8030,9150,7407,2234,863,1548,5328,4231,1669,8363,1276,9774,6514,5054,384,802,2896,940,8293,5167,4544,10091,8984,4821,7220,7959,9005,9249,4416,4721,5592,6076,4389,45,6716,3519,10820,6719,1781,3161,5442,1313,7141,9096,10636,9843,4279,7790,10409,6433,3351,1938,374,5402,9271,7075,2805,5431,7325,6565,2773,7340,2485,8318,8193,461,5345,8187,412,7342,2476,1702,4996,8452,5548,7214,2260,5900,10525,8305,1342,7913,253,2981,9746,2958,1421,2627,1341,9765,1549,929,1840,1178,2466,9353,3252,9403,6487,4854,597,6938,6488,286,6792,762,5919,2092,2884,1724,475,6827,6211,1333,9454,6986,13,2168,1391,2067,609,5116,7378,4960,9290,1657,9620,5336,8686,3243,5885,7257,6961,201,9658,8788,3097,8502,4805,10487,911,7829,5956,5366,8415,5651,1578,6314,4237,2562,5501,2366,5504,9881,6217,4973,3028,5307,9605,1460,5583,9812,9281,8741,5133,5521,1791,4066,2285,9188,9954,7788,9717,5022,5218,4311,790,9089,2367,7997,10489,2272,5410,99,10061,8346,4930,6835,6048,7535,3091,9401,2593,2937,4829,10714,1996,10064,3424,9182,941,9928,1082,10347,3145,979,9133,3454,1711,946,7088,7078,5079,3933,4365,5938,3905,2650,1863,10688,6729,7242,6905,3301,6078,7467,4208,4240,102,2898,7866,3908,0,2320,4564,9645,3130,5168,730,9889,1750,5874,8513,2268,5242,493,8981,3411,8873,9200,6005,9460,5665,5708,9797,6328,1036,6745,4405,8295,1758,6789,468,10742,10566,6780,8567,4284,7106,3888,6747,10134,6950,10079,4363,7898,5374,1257,7745,2380,3605,895,2687,3690,6462,2529,8692,9247,7395,3283,8669,9707,9205,7983,6253,3503,7919,1053,704,2244,6779,3846,6993,3850,4924,1971,10387,240,7572,8863,7864,663,4581,1409,2378,9100,10684,1465,8200,5679,8430,5866,10602,4143,4065,7157,8933,7763,756,8091,2488,3115,8248,4356,4813,2288,9687,1393,5999,1844,7255,5349,10830,4116,1112,8544,3895,6576,8194,8834,10391,7969,10713,270,8909,4111,137,9459,7322,10103,8581,3155,7142,7191,5400,8913,8209,2912,3434,9773,879,4719,2807,9704,53,6242,6241,10666,1322,7298,10008,6086,680,8047,8898,5265,4202,5467,2493,7408,4157,10255,7722,4414,2878,8074,7421,9232,7063,8111,1670,10266,8674,7217,10279,2979,5574,6269,5317,10415,5184,2429,6674,419,1028,602,3727,7429,8638,6839,4843,9767,9621,5347,8133,7694,5617,1227,9555,3029,7363,6500,1191,3770,569,9048,5524,4940,10368,3132,7358,6003,9402,5732,2310,9557,4538,4345,8550,2484,7195,2504,7417,9600,6031,5846,6812,2555,4214,6655,8537,10751,6918,10757,8349,3244,4011,2276,1882,4060,3935,5393,7871,8386,7808,6472,3279,9192,7420,5157,8294,9566,3586,6818,3461,2903,10582,4755,932,6547,7731,7116,6503,4569,2923,910,3633,10723,9597,7027,4013,318,4554,10480,4935,6482,9895,3968,9779,6338,5187,3697,4582,1668,5137,8926,2100,8939,421,7079,8746,9413,4702,4901,3314,4101,5505,4007,1945,2963,5969,4783,1783,9823,256,4620,9153,9013,4927,9344,381,3998,9990,9381,1974,343,1916,7817,8445,664,7711,1362,956,5597,4417,9924,10746,5499,8223,10152,10186,7344,10783,919,5740,2709,10804,3018,9651,1095,6801,2717,695,1039,4704,1968,7498,5877,6665,6062,6342,1369,7364,3518,1077,3477,167,6184,5735,10006,7140,142,10370,7238,8799,8729,8082,9052,7067,8401,10349,9366,2312,1977,687,2772,2054,1413,612,3524,9634,9184,487,2026,2950,10136,2795,9524,4779,2019,5694,9852,557,4399,3006,9302,448,9064,5979,614,8551,10117,5001,8813,2045,2004,1154,2660,2894,10769,10256,218,4627,5322,6193,4038,1047,10321,10185,8329,7893,3951,2841,5700,10545,1978,8269,7370,7944,1003,7894,5343,6838,3515,2592,5720,8210,4921,1411,8822,5033,5904,726,5850,10080,7768,8330,1761,10253,1999,2046,912,7247,388,909,7637,2885,6014,556,9409,8055,3013,2750,356,2013,7530,10229,7411,7058,3058,5964,661,5758,5075,9836,6536,7720,10652,7524,4127,5014,4102,7705,3151,5532,9841,10340,7154,3284,7345,3375,6469,5518,4207,9845,400,1689,9398,10817,8610,6210,1751,10381,8534,6921,7999,6198,579,5124,5576,5607,5123,2421,10295,936,9870,7568,7811,7124,7907,411,3853,9183,106,6917,264,10050,10827,7105,8068,10781,1848,8039,6663,2847,9257,216,1113,4974,3196,611,304,2473,2790,10718,8641,10239,6376,1673,6374,9374,10406,2266,1651,9275,1450,5199,3493,729,8786,3965,2164,10365,10704,10208,1957,9965,9217,10139,733,5282,8080,3543,5275,590,1160,3542,9855,2518,2115,4956,10293,406,4864,5139,90,2580,370,9999,4425,7981,6406,4233,1031,2241,3237,1368,2321,6915,6584,6024,9548,2178,6387,649,754,8490,3414,7263,9642,2974,1402,9436,1217,3607,1,10445,2177,5224,10785,6035,7483,4860,9638,2206,1664,3234,2369,9648,2988,4099,9207,8862,1325,3217,9838,24,8606,6913,613,4303,5931,2463,2389,4701,3517,3541,10703,76,3694,9364,1845,8650,8520,7539,1716,7934,7906,1623,8466,2147,8175,10031,6726,329,10339,4814,7099,3759,10691,2464,6133,7713,3511,8759,8141,8368,10168,10610,9278,2522,10242,8130,224,2991,7810,2665,1142,6650,7115,5876,7737,5600,1412,8703,2590,9123,9408,9072,7819,10100,8038,5736,4266,2379,6486,7044,1502,484,4061,7076,4856,830,9697,361,8006,4689,9128,6434,5473,4976,7287,6089,9453,8501,2261,694,1665,5015,9854,9116,6001,238,5344,7822,10075,520,10527,944,5415,5578,3430,8048,4248,2837,5481,4618,1894,6009,6394,6147,8147,4351,4142,5620,3634,5536,8907,8980,3468,5449,7897,2920,5155,5395,3019,5222,2840,6755,7988,2654,4044,6114,3682,666,1302,737,3689,9752,1569,1785,6357,1114,675,108,10494,3388,513,2607,7824,7604,8675,6630,8212,5132,4706,9440,8955,2474,997,3170,7381,5811,5423,9952,4352,536,6807,6608,4381,3613,6090,2043,9993,8022,6463,5233,5538,3567,8628,10107,4752,2176,2510,9131,608,1934,10326,6049,7473,10002,9980,9325,8922,173,4485,2029,6396,8838,3465,5068,8116,1752,3938,2702,3111,4095,2364,5833,3051,1076,9518,9346,9961,4445,10800,1787,9308,436,9226,2645,1335,6681,10203,2632,109,189,6794,6162,3330,4732,10731,2570,2874,7353,9202,3047,1624,1049,10760,4875,1474,3649,9608,8257,5856,2064,8798,9299,2151,8005,4950,8407,2553,10736,5585,3257,2732,3721,1488,9661,4482,8935,10132,7630,5372,893,7775,10774,5823,6158,5209,3451,6891,3300,8847,8532,6843,8934,6898,6790,2606,8179,4049,8802,2680,1512,906,1111,4647,3762,6238,8988,6754,5832,6252,5058,3387,5371,4464,3207,3791,8987,1915,9554,504,6612,9795,7017,2817,6043,3181,9908,7232,8996,4938,2603,10153,2996,8961,8102,8494,8546,7196,9426,10531,5274,5378,9572,9076,7582,7739,4030,9819,2759,4619,5997,2557,8196,5858,1822,8552,5375,792,9222,3432,6101,8917,6499,2065,8660,7774,9932,1634,2053,460,2302,6753,9132,3549,8109,4132,2226,35,856,7415,1789,10690,1117,7113,10701,4075,10160,5071,6859,6895,10327,3925,1886,4449,7509,3389,1094,3621,3138,8654,4811,8794,3825,8282,3255,10427,1318,7657,3724,4749,3832,7691,3700,654,10775,6337,9337,5942,2010,9884,1312,4170,6284,4315,1647,8322,413,5086,7301,10671,10286,57,7291,8493,8827,5516,5906,5944,3083,5859,7521,8744,9754,7249,4505,4325,3481,2395,7226,7462,4472,5996,7193,8297,6354,5730,3544,3775,9949,6291,3664,2797,2215,2382,576,8029,527,964,6438,1643,3289,9708,1424,6054,10708,7651,772,10561,1065,884,8975,4503,1814,4889,6641,4252,7430,8817,140,4691,4309,9589,9736,8607,10522,6358,5483,5424,10622,8007,6096,3368,2576,6381,872,952,8533,4866,8454,1462,525,8302,8570,1305,8578,10192,3851,8998,2077,4583,7603,1981,1950,6336,9929,5882,1017,6446,10118,1091,10455,3910,1910,3168,5820,7254,9582,2248,10158,4501,3219,7586,3332,2495,10149,1400,9536,8198,1825,8617,7892,6695,1674,564,1837,8354,7450,3757,3350,6431,8710,2218,883,7237,6185,9864,4220,4166,8345,3899,1141,6516,9239,2828,8645,2299,280,10297,4149,6999,3323,6334,6890,5010,533,7576,8887,3871,10267,151,10683,8121,241,1263,4798,8306,7032,7161,10209,8138,8648,3886,2731,3246,6008,6293,8100,3735,535,10541,2575,194,398,9077,4511,2897,2801,1107,9448,4698,9977,56,5691,2799,6299,1877,2546,249,6840,369,8154,4543,1030,7511,6672,2479,3693,4817,7215,1959,9969,8892,8588,8949,2597,9591,5225,5839,8656,9544,10360,1379,770,4033,9667,1889,10119,9584,8283,1173,1268,3054,7641,1976,1180,186,1242,8339,10012,10655,10510,3278,686,4677,8761,9171,546,6318,1990,10316,7623,119,297,10223,3981,3652,70,2530,367,2505,5067,3426,5107,9593,5494,4975,6102,10542,9640,6174,2993,993,10810,7304,6270,3623,6960,2789,10659,9465,7208,232,1797,9084,4634,10213,2892,1531,10538,6240,4048,434,1696,2096,2154,5955,3906,1023,764,7620,4200,1480,5444,8855,2875,4680,2373,2509,415,4847,244,3536,7396,7854,6402,6764,5717,2612,7518,1388,2494,7990,7695,4699,725,10423,6758,1529,10825,10028,4651,9574,3622,7717,6038,34,782,3761,1360,1252,469,1332,8916,7484,6163,1355,6793,2856,5510,3793,2097,3537,9431,7678,9169,4833,6330,5980,3209,7638,4552,4100,4097,6541,5193,8253,8484,4713,1619,8236,10598,7169,5420,7904,7461,2877,6963,4387,5027,7978,7922,5764,868,9365,5698,10613,1471,7376,2475,7724,5441,7876,7772,4810,5974,6180,2341,7399,4869,8272,2551,9540,3663,4661,7387,9215,678,559,8585,9556,1172,8642,10262,1629,758,5978,5650,1058,4005,8479,3393,2314,3109,688,4087,1385,3656,5084,3632,6601,2240,773,3705,6186,6718,10490,6206,4953,5333,3990,5296,7962,4700,3875,3379,4566,6968,10025,2445,3345,3024,6765,7875,4867,7351,8380,4679,3322,10035,9288,6748,1947,7616,7548,4123,8016,7489,3823,4872,1408,5880,9180,10609,8791,9910,2762,501,2636,7971,7303,3710,8084,1871,1192,7080,1610,8560,6046,3286,7608,8403,1405,1456,171,9912,5947,6118,3610,2547,1024,9199,538,75,2767,7471,9726,3927,2537,10667,4043,6391,3214,721,6063,655,8208,2262,8748,8869,8531,2027,3236,3406,442,1572,745,2271,8858,8243,9847,5287,8127,6546,9913,7275,644,7994,1440,10414,3423,7031,1962,4244,6559,5916,5549,1383,1046,7830,8063,2015,7986,3304,8612,6704,6361,7166,3462,4209,10171,1685,7002,6730,308,9110,577,10568,7015,7048,6200,7212,7204,3804,3527,4437,4759,4010,4020,10664,6710,10161,3553,10282,9968,3260,3077,6533,7424,2986,2071,441,5647,6703,2446,4080,8054,10182,5896,7928,3584,362,8050,2586,8843,10707,5492,2815,10658,7635,9592,8608,6908,7434,7085,4774,1327,835,4693,3188,6956,8886,2162,9389,2679,7123,4884,7419,6613,2599,2770,2512,9265,9323,6675,315,4508,994,1605,2256,6321,7675,1002,1016,5303,7584,939,9626,476,10211,7573,1394,2212,6189,5687,5309,2519,2186,9559,1936,10268,6558,1365,5791,778,8678,5601,2292,5025,1235,1149,7433,4062,2881,5871,4555,39,78,9166,3311,8126,6888,10408,2625,3960,4656,4561,6849,2338,4591,5531,9619,6036,10464,4378,5775,5278,8385,5514,814,6775,3691,1153,5631,1738,10799,4966,8611,566,9820,5957,100,5963,7789,2157,945,2454,9093,7849,4225,6444,1374,10587,975,7098,8058,9947,6619,2301,5426,7998,5117,10155,5206,4371,4802,3685,9491,592,4838,3861,7949,8643,10780,5685,1218,6105,8629,550,4773,3887,1085,9941,9690,6897,4089,7284,2403,3185,7042,3189,4886,8571,10372,4446,10204,6191,10190,5220,892,681,1956,4588,7534,16,9701,3355,7389,5463,8145,8026,10246,6004,8139,8300,9676,6087,10325,7770,1641,5543,8985,2344,3897,3463,4337,5949,121,9911,5948,5163,380,4338,5743,2588,7318,3309,1269,7850,10085,9008,7879,3726,8774,4465,1582,1084,7094,2400,1591,3299,3681,3079,4841,8797,4760,1087,9671,4295,9505,193,6289,8515,3661,4952,3713,1229,2917,5257,1283,10199,392,9175,5151,5186,5259,2742,5558,7953,4610,7127,6288,7319,4110,4571,6219,2481,6926,9553,6990,3912,5875,7281,10595,9108,1438,1721,2385,1677,1680,1319,9504,615,1423,2962,9699,3025,1598,4435,383,1376,1500,10577,3504,3614,2769,6227,9003,3580,9666,7605,565,10263,2243,4723,6161,3943,2985,5072,10303,214,662,9251,7059,1594,10716,8301,8575,4253,851,9329,221,4931,4304,5659,9531,4453,5603,5149,9034,8677,3769,1067,2273,8997,10147,9404,2376,2891,7600,3841,8925,665,2306,9250,5455,9480,867,6341,9121,3774,1526,6615,9599,8963,5503,7574,6773,6067,4849,7428,10274,7285,5767,2348,9835,10216,6878,5135,1952,4374,8735,5508,7856,9349,2843,1163,8433,10027,8160,1666,10473,2184,5213,3742,3312,4222,8097,7886,4280,9485,9868,1202,10177,10625,6589,6484,9399,7670,928,1799,4665,2656,6195,2070,3590,4204,7167,6582,9483,2334,708,4965,10394,5443,1132,6578,8197,5453,6137,2391,2305,4122,10702,10596,8626,9975,846,2613,7107,5840,6501,10762,8233,9635,5216,4642,981,1273,5860,7778,8031,3295,3978,3377,1692,9623,2022,5006,2933,2362,6505,7846,8760,4139,3926,8228,4993,955,7036,5748,5838,4895,2873,4320,6831,2316,10111,4289,821,6447,7323,1992,8371,4121,1486,2222,2729,9106,1726,9943,9438,6083,7096,9919,7295,7958,8467,8711,4069,2961,1279,3075,9506,8713,4595,5744,4442,1632,2425,9991,4850,2145,4173,7451,2060,6197,2208,5824,2153,8172,10574,2182,1108,7870,6607,2471,7827,375,350,1422,10352,3592,8700,4654,5258,1317,741,6720,6234,4016,8176,3364,6644,1640,6309,4506,1505,8432,5960,8540,8037,5684,3482,9664,6398,2776,6996,4042,29,4796,9606,4589,7588,5677,9996,223,10604,9268,9332,1000,5030,8857,4104,2253,6858,8937,803,7668,2554,3619,3806,9541,9951,4563,3124,8399,7965,357,518,8483,6068,7283,6756,4235,4742,4717,8592,2142,9685,8548,7402,1547,9149,5784,3989,3932,4074,2048,5281,9318,9283,926,7932,8745,970,7942,9395,2038,9659,4474,2826,1014,1256,7223,6836,5031,10434,6919,3313,4292,1196,3720,9317,7640,9160,4194,1494,4862,6707,6512,9825,4448,10377,6057,4671,8463,2263,5369,4103,3525,1964,4949,3722,628,2595,9388,2771,8001,6221,8959,794,8167,5818,6785,10198,2327,6074,7405,104,2796,8427,5164,3194,4163,2428,4530,9301,1853,3810,3369,3566,8871,3593,5162,10696,3950,9776,776,8593,793,4384,6886,8597,10697,6506,7867,7662,6932,3070,5849,1299,9313,7089,1188,7527,8274,7617,9763,1965,1382,1278,7165,4754,132,3934,5773,3618,1861,4879,4232,1608,7457,9306,8350,6811,331,5542,8990,10048,2922,2424,9307,7175,9873,2420,1541,4434,9879,5961,379,9443,5898,8122,5731,5295,10776,4342,5821,6351,9794,9510,3798,3885,4822,5020,8183,6732,8244,786,2465,6664,7517,580,1414,5712,9073,4036,3581,5520,1967,5666,7007,805,7398,1375,2287,3014,1514,7082,2948,4840,5182,7685,9902,3858,2726,5489,3321,1069,2160,7404,7495,774,3015,1566,2165,10288,7464,4720,4575,1314,4215,9784,6609,3315,9477,8473,4763,5845,4286,10726,1603,2744,5759,7388,7496,443,5962,2544,4507,3383,10748,977,3133,5066,7945,7020,7372,7908,9721,3396,3044,4705,5051,2867,4909,3792,10614,8740,9867,6736,5040,4029,1286,6666,8875,7317,3955,5706,5928,2204,2970,9492,1771,1577,4196,1796,6150,6332,8066,10210,10514,6531,1597,3550,1998,10585,10338,3467,3513,7648,7130,478,3766,1589,8418,7858,3941,252,1390,5599,5989,10670,4660,2121,8351,10824,1083,5326,3784,2480,4330,6742,1055,6714,9734,7618,6628,6739,3459,7735,7233,896,9831,1034,6971,9972,10607,6012,1828,1179,7485,8423,2000,10488,10393,8148,306,6307,5723,3736,8021,545,1800,8561,4709,4341,5076,6557,49,5452,3152,9361,7324,5291,10230,1166,5673,10554,9590,2402,6817,6282,4560,6787,2619,3082,7172,378,9055,3528,4948,2721,3878,6323,465,1459,4690,5881,8599,6943,3078,4573,10188,989,6954,7512,2440,10679,1955,5738,4278,6300,4090,418,2325,797,7902,5562,7500,5812,6426,9675,10422,5734,5466,9828,4321,1741,10793,8344,5110,9826,5798,3173,8495,934,7692,9878,6594,10145,10478,359,1168,2900,7019,5060,10029,9714,10294,2461,6362,5496,4818,5045,8945,1395,467,5935,7392,8811,4144,138,3996,9673,22,6571,9148,3490,5469,8279,4128,10552,5965,3222,8387,4735,5675,587,4483,732,9006,8313,9916,10503,8672,8904,8717,3879,804,6395,6853,8793,5645,9263,4326,7807,703,9920,9921,621,1152,2080,9094,1654,9085,5434,228,6097,9710,6478,3745,292,1884,6743,7614,227,283,342,8367,5329,335,9729,8649,3535,7069,9212,8129,631,8568,3410,7051,3356,7090,9978,3476,6530,5512,558,1920,5688,874,10397,9335,3501,6682,2456,7749,2711,4236,3796,1144,2238,6882,9740,1559,9354,2183,6050,578,7734,7779,908,10302,3754,8541,2030,2217,2677,7045,8230,4045,7459,2594,245,1546,7210,2489,1050,10245,2211,6196,7005,3105,2394,8442,2643,1681,683,10344,3821,1581,1329,3027,10653,4438,2855,5299,9489,4828,6646,7762,4189,8143,2232,3949,1858,6854,4653,543,7173,7312,2808,8149,9147,5657,10143,10744,9355,5572,5005,6795,2371,9706,5397,114,5550,5722,4002,2902,9611,4624,1249,5994,3057,7931,8587,128,3617,3834,3936,2579,3505,5039,2734,1604,10004,9028,6222,134,2754,9643,6359,5667,5834,7338,9715,6603,8549,7546,8064,7406,9539,9079,3267,8523,5474,8278,749,10173,3890,8826,1232,2152,3609,6052,4636,547,6111,4470,7391,9771,3495,10437,4357,5642,7073,3199,5636,7976,3039,7306,8263,2432,6931,6939,1810,6231,6064,2910,5622,210,3667,7592,4179,6544,8919,553,6261,6344,7743,5103,8769,6550,10433,3442,6715,5078,2994,156,10005,8083,459,10499,7719,6126,4902,882,346,496,9696,7967,9186,6019,8594,6353,5210,1989,4180,3440,9062,9026,9861,1765,2083,7647,4668,5988,10385,236,8142,9513,3647,9762,6634,707,4780,4349,9788,6037,1973,7316,3644,7269,1293,4331,5920,2824,784,8808,7636,10501,4346,9604,3583,89,815,5118,8724,8820,622,3558,5367,3268,10496,10251,10612,9358,4492,3439,2528,4578,4998,9901,4834,6670,2381,452,728,2414,2144,205,9871,10693,2531,2693,10512,4273,9691,6586,8465,9769,9712,8864,2535,2511,4098,7560,5136,2297,8993,1174,127,8876,2274,6451,2258,7504,8488,10197,2124,3458,7321,530,5044,10362,5873,6408,9987,1128,3718,2187,8072,2935,1162,8969,4865,10233,5870,9814,2766,11,3017,9046,269,10719,2081,4459,55,10097,1148,7631,699,1631,9964,233,5950,10034,630,4183,507,146,6343,636,5796,2173,8298,6538,4678,2936,1875,5183,9375,5050,9418,4161,10593,1823,4761,2322,4641,6692,1037,10807,2458,2715,5655,594,8227,6911,6800,92,2723,6060,9962,8447,10758,364,5943,4409,1330,9276,2608,3262,5754,771,10332,2577,6814,12,1452,2768,6277,8409,4685,701,5324,5690,5340,4536,8970,1380,5185,3324,3789,8640,2084,3096,3642,6870,9730,3877,3730,2074,9798,1919,3884,6570,2954,10828,4606,6346,456,1222,9125,6957,9985,8392,6713,8226,8090,2426,9478,2977,3055,7816,2895,1435,5144,9285,7337,7948,4494,2337,2021,3233,6847,7767,5141,4072,4942,4519,8219,3957,7035,5737,10181,1857,573,6453,8070,1487,8245,3401,74,2412,10024,1767,2340,3582,6259,4382,4391,9071,8524,3277,7663,4461,6690,8738,4640,1057,2411,1766,3958,10475,2331,6209,5120,4241,3620,7851,8747,8014,4615,5327,6661,5724,7213,4174,3042,3781,8256,9050,6121,1687,8723,8928,8216,4012,1609,1025,10815,1627,2712,3460,1819,528,8275,3103,6324,6258,1714,3494,339,2746,4748,1558,5927,4130,255,8202,5952,4136,3744,8776,1493,10404,4959,4666,10569,4604,1258,7505,9352,9742,2210,9713,5191,1157,3921,4608,3631,3994,6122,9702,783,1638,7712,4431,9499,10236,7925,3175,3839,7765,3228,967,10465,8682,9189,5901,6216,8835,6577,10105,6803,4545,10101,5035,8664,3247,7289,6285,9579,2335,4441,5540,5314,10215,7815,1561,1515,8885,5594,6631,9877,6205,1817,4745,7235,8356,1570,5325,2137,7589,4823,5413,870,9038,8595,6638,1396,8242,6569,9562,5783,6639,5695,7180,1728,9872,10476,7661,1224,6893,1241,9378,3256,4203,9933,183,1530,7277,6528,5438,7181,1982,6972,2295,1503,334,4532,2293,6171,4842,4728,8492,124,7444,3752,5825,31,8000,2163,10530,1366,2548,8081,10771,63,5827,9191,10218,3258,1045,1656,3215,3630,2150,3276,4800,9414,5226,2673,7677,4255,5266,2072,6953,5004,6562,5365,1818,3162,6766,4452,10524,6985,3637,9850,10000,9672,3876,9647,1575,3699,1905,7629,8972,9744,2924,9452,10234,4737,9010,4288,1843,897,2435,7348,9305,978,2138,10591,3862,10540,4897,8402,8932,2443,2279,438,5038,3574,6535,7273,541,9598,8619,10459,3306,9043,5605,595,1270,309,9280,3080,8411,7825,6006,7801,7664,7599,9458,2506,7764,798,1520,1616,6378,3245,7877,2953,8480,819,5608,9181,3729,61,1072,9009,7482,6788,7703,136,7700,254,9615,2540,1717,3947,7794,6947,836,5519,2444,511,3548,1804,7296,5388,8948,3707,1683,4664,4175,6204,10261,7357,2251,7104,3675,9253,6265,9613,8071,880,8365,1019,10270,4469,2357,9646,6892,3507,9792,7985,8527,8701,5552,8180,7799,685,5787,10343,3628,2637,4607,6202,7831,1747,8342,8891,8110,4050,366,5387,6974,3666,6611,761,2852,1372,2423,2697,4355,4272,3122,1161,8618,3711,5037,2076,6160,5662,4277,2651,5470,7848,3954,10235,8938,8288,9439,7828,1147,9241,931,2983,2602,3639,10306,4791,7723,217,1464,4162,5951,9082,8177,1064,391,4460,4955,2377,8880,6173,10304,5903,4972,9018,8977,3457,6329,9922,10740,3241,3917,1061,1885,8708,1200,422,10127,10300,4057,9790,616,10594,876,9033,8848,5862,1729,3265,10167,2659,9386,160,7519,2905,7659,2866,7245,5302,10353,8856,7921,10474,4053,5354,2011,9422,4729,5751,3319,7514,9944,9950,4440,2560,330,6772,9725,8422,4070,8694,6370,82,1377,983,5023,4812,923,2223,9533,2839,6942,1434,8796,6440,7757,4383,2743,10695,3570,3836,2997,1901,10313,8818,282,7540,7951,640,4658,9066,6712,3190,6649,18,7251,8497,10497,3530,8859,7445,293,3135,3285,5228,7818,9159,1642,212,5077,10059,10224,9474,38,3687,8851,5630,4078,5205,2674,5273,9822,9101,5376,8046,10206,9099,10624,8498,3325,9800,2610,10036,5867,958,107,8096,3239,2779,7466,7553,257,5710,4982,7049,3335,5857,5359,427,5097,1659,9772,1903,7040,7989,3671,3800,4015,2571,9341,184,9530,6705,7872,713,10630,2143,5392,4797,9516,7023,8169,1432,998,10446,2128,1143,2641,5346,6883,2829,9336,531,2566,2192,7022,7377,3100,4287,2722,348,6510,2462,8539,4168,8882,8284,4946,8475,325,7060,5026,8396,242,6696,9738,8290,3456,974,3611,3034,3400,9070,3913,9058,5565,9612,491,9628,7903,8809,5909,4190,133,2459,10390,9461,6598,3296,1484,6651,1189,1136,10269,8716,2604,9105,2737,9229,9906,4458,4593,10509,2300,10419,2978,3444,5399,7138,4718,8884,878,9755,8727,6201,913,5246,8460,3046,10378,5541,1509,8375,10416,2966,5271,5412,6459,7443,423,4115,8895,5842,8995,1870,10678,7194,4184,5171,5134,1386,10357,10447,1588,8201,1476,3085,9936,4819,6112,6423,4114,5793,8659,10375,10392,10226,6228,10520,9994,6659,4213,8586,10157,5178,968,6417,7026,1891,5152,9371,7758,9926,1713,3758,1986,7000,10425,5742,5202,6059,301,8124,10285,9862,5195,6313,1125,9145,8861,1904,3437,966,6740,6131,4547,748,6188,1778,3108,2707,4740,1281,5530,2220,3117,267,2298,9813,4067,410,5319,6671,1650,2031,8162,4250,3449,2343,4319,1120,4328,8823,1497,8676,4534,4493,7543,1637,10786,6778,9204,1922,9654,7216,10663,9984,9722,10259,2842,5479,903,8705,10644,340,7186,10011,8075,10402,3227,3486,8059,1508,1928,2913,5628,3948,9127,834,2148,6044,10047,4301,5883,8459,10071,1777,347,9119,10550,7146,8893,6026,6382,987,1958,9674,1439,4336,617,10110,8751,3865,1739,9637,7028,6264,544,9315,7881,1646,9678,3604,3429,5902,1131,3146,425,7593,4580,3394,9917,10669,51,4054,5421,2972,4784,4216,10237,9244,9117,263,2703,4153,829,2671,9144,1300,3538,2713,969,2940,5837,123,4246,2242,3397,3559,9273,2862,8899,10122,4756,10441,1345,2552,8012,2803,9783,2374,113,10808,7117,2887,5370,9423,7118,10405,2042,5214,7520,6305,5230,5702,5893,5160,8239,8443,1963,3056,8615,1988,5048,8374,10467,3263,1351,8824,4051,3390,9007,7844,1417,3179,4370,7476,5408,9069,2486,3857,9829,3149,5458,6826,3702,7502,1931,7305,2646,3186,10418,4308,9668,4614,5683,2725,5028,2075,10096,5579,7891,1740,3443,4395,5792,1806,2934,1358,446,8199,2600,7575,4686,8603,10645,2365,6077,27,8416,8389,3532,10452,9112,6458,779,3706,5602,3491,1214,9766,10558,7160,985,9311,4747,7365,9393,8192,10131,352,3555,8718,2318,6298,9115,6414,5059,4762,1043,10292,3719,6000,9293,7993,5809,9174,6588,7874,1285,7895,4497,490,3603,4468,10023,4990,8098,8953,7955,7585,9538,10717,4198,7379,7347,6404,2452,10791,9451,9434,1900,6207,8195,444,4707,7438,3945,7309,9508,3636,9931,751,1513,7330,4526,7302,6181,4836,2969,8683,2434,10420,1255,7561,5451,1354,3156,8512,6041,7410,2695,1052,5733,326,8002,2911,6192,1186,4794,3102,2780,10124,6902,7751,1860,7119,9475,1080,2501,6738,657,3040,1404,6243,4638,7596,8286,10001,1995,4885,5016,700,1096,9373,10380,6442,8010,8622,5590,5555,4420,2265,10517,360,4766,3698,6413,5355,2098,2682,6007,7884,7455,6407,5992,9866,2449,9770,3026,8469,6652,3474,2728,6460,8092,4744,6701,483,4307,6416,1197,6172,9248,3679,9624,3141,10291,4778,7741,6399,10137,5490,1537,2502,4058,5728,6731,7569,9370,4584,9650,8771,9296,4853,6281,3213,3991,10214,4402,3308,5861,8801,2541,2806,8246,7693,7163,4479,10183,4499,2870,2818,433,5280,7423,3523,277,9234,3359,6266,2652,8400,3509,6474,6899,3627,4408,4857,2763,2692,951,5878,93,7602,3866,6123,9827,4827,4296,6166,9155,1573,1187,7248,5668,5248,10495,6987,1832,7795,3031,6226,10462,5930,4682,2556,7669,10829,3232,4195,1466,658,2736,1005,7371,6604,5653,9709,6620,203,9923,9596,9134,10184,1097,2006,841,5306,6568,1774,953,5445,6271,10536,6632,5237,3317,7016,1838,1599,9473,7491,9956,6082,6567,10498,6069,1316,9300,7654,1245,8348,200,3338,10575,8521,8420,4422,6023,4084,2433,629,8627,2765,3348,8868,9228,6100,8931,6058,7149,7228,7187,8836,8870,9519,1583,6952,2907,1879,2174,3280,5649,2941,9958,5417,2835,1228,10093,8120,6580,1267,10753,8830,889,1533,8040,451,9047,7555,6662,3248,2254,1102,5454,2792,760,555,5644,5852,7394,7299,7034,3995,10424,8434,6449,3385,9049,176,8115,7940,2120,7349,5390,1849,9385,1809,7557,4293,5338,2195,9782,1060,3418,2741,8426,6276,10811,6551,9347,9967,417,2752,653,5143,10764,10087,9520,7863,7890,1212,363,8665,8428,10298,4025,9887,6616,9693,8458,7386,6976,8752,3838,10243,2149,7229,8451,208,8888,6658,6151,6997,9185,4898,1254,3235,7523,9065,1205,3187,3872,3316,3579,4480,6989,9622,3600,7742,904,3750,7977,894,1671,753,4343,8574,2696,575,9888,1479,9031,2140,3571,3119,738,1618,36,4165,8790,7056,775,9417,5933,5664,2091,7923,6491,289,389,4738,3275,9130,6657,7650,9976,9493,7264,4439,6025,5092,6529,10275,5286,5439,6153,8944,4549,2311,4848,175,8689,7442,1737,10334,9384,8394,10599,7656,2139,1056,9173,3210,7453,8334,4218,7781,2663,458,4826,1253,1287,9135,9542,2213,5029,5983,8326,9733,8956,10621,4450,2018,10264,4032,6941,4832,9224,3782,7416,9039,5279,9177,8940,3107,8604,8065,5872,6263,4936,2231,5699,1658,9528,4516,6143,5057,2024,1723,2787,8421,6871,9097,6717,10283,8554,311,6930,8742,3598,1695,8025,5125,3072,2863,10772,4239,3795,6784,4808,1507,6842,8992,2384,8867,8410,10565,3264,5172,1066,6964,5639,1071,7503,3037,7185,3993,791,10129,9421,4291,6224,2040,1121,7182,2117,660,7773,2992,9297,5098,5635,260,41,2308,4477,6829,10003,2353,8547,10633,3779,8471,2221,7454,3331,3404,2418,6306,3098,9490,2906,10202,8331,8448,2688,1419,7833,2491,2134,10430,5537,10249,3708,2915,8687,4746,1181,1098,2275,5656,5176,1454,4557,9649,5855,3662,9686,226,8923,5848,2245,691,506,7852,625,3859,3822,10065,4907,4896,7095,2309,10323,10315,4815,8238,2330,6372,8754,9914,5816,7672,10333,9743,1008,720,298,9165,5430,10336,645,7004,10040,2404,6075,1185,2172,1079,1150,9024,1856,7211,9810,7230,231,2135,2166,3577,5488,7158,3953,4500,2250,3297,479,2482,1001,7728,9751,9484,1712,4787,2415,4339,9988,10373,7494,1443,1359,7359,3734,1350,6388,7452,9953,8101,10682,6113,1070,1429,4283,2304,2180,9416,5990,9728,7168,7120,6689,4915,7910,3290,4224,9470,9684,8758,601,6944,8564,7963,3963,8114,5865,8087,1284,8807,10305,6479,5221,8359,8011,4943,6494,2542,1715,5559,9575,4093,2621,8073,6280,5529,3101,408,7564,10648,3269,4476,795,3975,6028,6275,88,7927,5484,3340,7979,10590,2005,1104,7643,1463,3479,6973,6502,5701,1236,2585,1389,9527,2631,3065,10144,9764,7611,3293,4186,6643,8222,8338,1867,539,10676,2819,8370,1075,3358,698,5361,6443,4799,4649,9747,4281,6751,3089,10324,7046,9915,5245,3147,1310,4831,6492,8695,2823,9761,10095,3874,3686,9107,10116,7139,2090,2705,8511,9137,10225,2224,4509,9583,230,10470,3415,10743,10752,799,1106,112,3067,7889,8280,5158,8646,4846,1544,3211,5416,6948,924,8912,4525,10559,3143,10232,5254,8476,9368,409,2319,1203,2237,397,947,810,2946,10809,10399,1600,1239,295,3373,586,7134,9411,693,7262,2639,1534,9472,320,2942,155,4548,7427,1353,9585,3022,4550,2943,7570,5654,3478,279,2869,2014,619,10180,9839,5377,235,6848,2667,6079,9567,5768,170,5853,1812,845,3904,3674,8936,4947,10142,2975,4027,5709,7448,8156,6759,1118,5055,4954,2413,7053,5971,8135,8477,9982,2033,8789,7256,2624,9432,5761,3522,2564,817,3249,1649,1551,4937,2916,877,1370,9027,4063,8234,5177,8841,9543,8118,8089,8894,1970,499,8781,2611,4210,3940,5797,10007,864,4852,9326,9209,9688,9083,9522,7704,7686,9014,3845,5697,8323,4951,5676,6333,2661,10194,2683,3353,712,5437,6233,5241,4152,7501,3164,8707,9042,8264,3126,2783,6470,4362,2926,5973,4521,455,10725,8186,1942,9237,9369,6683,5672,637,7362,3391,9898,7409,6317,6437,2810,2007,8889,8125,6599,4596,7632,9040,5918,6556,8783,182,8844,1636,2587,7562,7702,624,10471,7899,740,8235,2375,9444,2701,2278,9340,8712,6951,1426,626,1667,4734,9495,8631,246,4980,4264,1481,5485,4380,1260,9918,8509,5929,4041,7750,8262,1579,3557,10822,763,3765,2525,5476,2517,6016,3974,10428,3254,1009,2700,9348,3271,2999,47,2760,9208,7086,8261,2106,8251,4710,1852,7708,7487,1501,5013,7346,6355,8829,5065,6593,9258,3413,8043,5188,9078,3068,4711,148,5953,6678,1851,8491,2704,3587,6776,3801,7613,3863,7343,5926,8930,3114,4985,3589,5890,6770,2349,314,10328,4992,5506,4873,7072,2745,439,403,9419,5493,3242,2001,1339,457,1204,9521,5330,9287,6465,5472,4223,5301,4504,2507,447,4803,6647,5513,1294,2782,1492,6441,8954,7222,8061,8637,1709,10640,2457,4006,5977,5146,1444,4270,10383,9372,2094,10519,8033,6356,7329,957,582,8390,7246,6466,2957,5138,2854,4971,6312,596,7995,2777,291,6471,7414,4004,4322,1961,474,1381,6422,4466,2407,3261,4361,2405,9245,1427,4567,1639,1590,3004,6540,5263,3695,4340,3794,5528,10038,6552,10458,6485,7909,3202,1733,6605,512,9092,130,2543,668,787,7368,7987,1265,769,5011,4613,7619,8157,2774,9304,9405,6373,3967,3433,7008,7093,5844,5799,6476,2358,5331,873,4801,8355,7250,1448,46,440,9957,1158,2089,259,7583,1491,2112,197,3226,3568,7279,7144,1292,10479,7857,1979,7980,7050,4418,674,3547,3416,1415,6415,3514,1704,2156,10016,8979,9081,10555,1827,905,8440,4251,7355,3728,917,10022,299,20,3069,1532,4592,9362,8528,6496,4983,5003,3937,5771,2755,1880,10133,5465,9158,9689,10543,10712,5985,651,5315,5249,482,3176,98,9834,3367,2196,3907,3093,8381,8722,1611,9894,8679,1428,6287,4496,5534,5987,4863,4672,10516,5721,1564,9938,1247,6179,5868,6432,5298,7111,5781,6881,1347,7754,6820,781,5533,5284,10067,2111,3446,3502,8866,5915,4359,996,4770,7785,1595,3035,3969,1834,3894,10060,8237,1794,285,2198,3898,2191,585,7384,3768,5892,7861,3172,9201,3973,8360,9342,6750,5498,9320,3139,9195,271,8978,2498,3090,9029,8057,5487,7859,5886,2422,6572,4524,3341,10417,3420,2125,9679,8478,10296,4837,2146,10201,4572,7845,432,4056,6428,6786,5383,8941,5658,3110,3684,6070,7510,8914,10042,859,4068,2249,4663,8335,3453,2952,7639,7327,5126,3180,1297,6085,6914,7645,3692,5197,4462,3670,8821,2088,8579,7490,7747,1909,6872,2681,8113,10745,6445,7205,6128,1745,10449,623,1826,2520,2860,4903,8795,2126,7311,43,10400,832,4454,9274,6159,6680,9631,6808,5611,1527,9011,5539,3818,5795,5294,3417,1220,5251,10641,705,731,4052,659,6164,3599,8211,7918,414,5863,6203,4906,3646,3087,2784,9252,5435,9360,10077,3962,10597,568,7493,2844,1498,6635,2020,9896,2965,3007,9816,5198,8816,2614,10724,7426,5268,8446,5403,7001,1874,9220,8332,1949,10750,554,3431,850,8986,8616,5396,10179,1773,4905,6851,4712,4871,8715,8602,6473,33,7331,1430,10299,2572,5750,5351,746,10635,849,4964,7698,1628,481,9939,7701,9494,2303,1528,642,2016,8927,6467,8499,10319,6885,8311,3714,954,8404,6367,9487,962,5111,4317,5150,4777,7109,4603,2853,3520,7240,7400,1722,757,8128,9243,5801,8462,1048,9218,3498,4386,3612,5024,6841,10271,7171,844,6606,8417,9909,3819,3645,2342,2282,6771,1176,2203,4765,1195,10359,7129,6350,9997,5643,4312,2277,3964,3346,5897,7970,7911,8613,4692,5080,3725,9380,8921,4376,10773,8271,8482,5509,2205,7855,8340,604,8190,9469,8159,3095,1868,10544,126,4528,8170,5189,7468,4360,1730,2406,8633,7463,5887,6108,716,7373,6962,9580,4262,4650,5357,1177,2039,2110,5109,3081,9569,4488,10687,6520,6045,639,2447,5169,840,10661,4655,2914,4314,2617,10818,7954,1139,10816,4159,4963,8800,2888,213,6936,4444,1895,4037,272,7413,4249,7339,7366,3808,7197,1703,278,1274,3500,6039,6237,3786,8778,6912,2460,7565,6517,6920,5419,9429,9625,2105,7278,44,5835,1731,2470,3716,5105,3086,10588,4129,144,2492,4881,1035,10813,6693,6804,2893,3197,9021,5596,3452,470,3320,1210,10363,5976,14,10026,1127,7112,10310,5566,5663,4313,6308,570,8905,9464,696,9907,9143,9739,534,5567,9737,8697,8009,6595,7748,6175,2811,1795,323,5705,9266,8804,3777,2857,4429,848,10309,4432,6124,9294,9758,4888,5462,10734,60,6955,7033,10162,10782,5704,10677,6489,239,1688,2845,7642,5557,8276,10148,4919,715,1613,7832,7782,4369,4221,10098,7332,2563,1821,7380,9670,7590,1672,8134,4047,8414,1447,4883,5300,10207,333,7805,5046,10355,10106,10787,8425,118,2057,5194,1013,785,3756,8500,1705,7621,1644,7926,5106,10337,5047,6368,8076,10556,4887,3167,1092,2286,6560,9254,10521,6051,6958,6256,9338,4372,2598,1596,10583,2798,1250,1090,3250,1193,4922,7367,6904,1032,2633,3984,3678,10579,9630,7571,4424,5891,9045,10533,3944,8151,6326,349,6091,10032,7920,5682,1625,6700,5362,10627,390,2960,10442,9501,2740,332,6412,5368,7522,7456,4776,5568,10176,10603,2834,9456,494,1184,7598,10802,6199,6212,4285,8470,5081,4933,8900,1926,6071,4553,3615,4333,10240,8739,3508,2738,6320,606,4676,6448,7062,3923,9653,9846,6235,4436,7653,6591,7352,5142,2582,141,7984,7937,1700,5360,9973,8303,2216,3392,5680,4674,898,7812,5440,10154,5908,6940,1495,6403,1960,143,10733,1115,385,2003,3658,6218,9387,3104,3224,9821,6590,6995,1119,7558,1749,6419,1201,6475,7268,2028,4645,159,4539,650,8755,914,5101,6625,10041,4939,5386,4379,10272,10567,6251,4413,1836,8361,2724,10090,9095,5561,9683,2123,5828,9016,7290,3041,6152,2523,6430,6136,10244,8247,1743,2496,1756,7074,9655,2609,168,2012,1883,7018,4120,8666,4874,6425,9334,1431,2247,6331,2119,10692,4109,5227,7218,7957,9785,789,7793,9255,3565,6148,4764,1357,3900,10483,5553,310,6566,9803,5087,1038,9601,5212,7798,97,5341,8719,2052,2879,9955,6539,7136,6190,8865,10729,3813,4393,4913,971,3881,296,8207,1344,4147,5982,8505,3540,692,571,3436,10049,4969,7746,2500,7224,8877,607,2764,9718,9457,9447,7873,7941,4775,4586,10698,3113,1684,2739,7397,8815,7939,80,1455,68,1790,324,1324,166,7055,508,4188,10439,10456,2733,3154,7294,8691,8828,5121,4478,8150,10672,4373,10763,6245,10348,9857,509,3045,1893,9523,393,6821,2257,10491,4882,1798,6020,3826,6109,3516,4486,1802,7354,7041,8285,9526,1190,2813,2082,1020,5289,10287,5967,1788,2181,7526,1913,6629,10108,161,5813,3201,10756,777,2616,9787,3171,9236,5769,1054,9576,10386,3381,7594,4648,7314,9435,1755,8289,7709,73,10073,5670,706,1298,6797,6390,7549,9558,4178,7597,10720,10254,4629,10212,6749,4997,3038,5219,9808,5984,8232,9546,9111,2455,6021,5912,2427,1924,2227,9383,7068,8845,5239,6273,1134,4576,593,9745,2640,3398,2363,10832,10258,6618,9041,10560,7100,4894,10312,3030,10057,1445,8018,1356,4009,424,7326,1568,9899,52,7740,6498,4092,2987,9882,8623,1830,4741,7803,6132,4605,10728,766,8252,6098,9594,2831,1862,5556,3050,9343,6894,2383,8358,2515,9023,8652,5581,229,2207,4205,2332,7383,1585,338,1907,3370,6464,1315,2666,3790,5170,9206,2333,1944,1768,2438,4695,6741,4242,2467,1574,6815,5502,2073,3076,9242,9824,6994,225,9602,10432,4714,9719,1233,1490,7529,2209,8915,7280,8518,9056,6763,7601,871,9777,5468,7267,6130,5156,4708,2017,3178,4626,4727,2899,2259,4527,268,1126,2690,8846,4768,990,7470,464,8962,9588,5052,498,5422,3374,222,8315,9194,5648,3740,5358,9450,5404,800,7633,7513,1911,10798,4108,6668,9880,6896,10382,972,2638,5292,2848,8203,1662,10651,7066,9075,1244,7537,5305,8307,2416,8736,6072,5507,2269,10314,8042,5527,4970,9893,4059,6257,3546,2718,1288,8132,10694,1469,8314,7328,7065,7972,10070,2689,9001,5203,7595,4017,7716,5277,4579,8558,3992,1707,5641,1676,4430,1754,739,4781,8393,10484,7806,1993,3657,1865,10642,8287,3204,8680,1675,6066,1601,1734,5936,9282,5478,7930,4559,991,10018,690,9154,5255,2655,7,190,7563,3717,7868,4358,7924,1301,4491,1780,1110,4275,2925,10548,3212,5661,1607,648,3977,377,5569,8069,6249,635,7992,9496,1881,831,9851,6923,2872,2229,5544,4541,8383,5285,3344,4031,8003,8292,3980,2591,6389,7649,4318,4167,4367,5554,9509,10507,7912,3421,7077,158,10779,1764,1975,3545,2350,2601,9141,3829,488,2532,7550,1593,3403,515,1078,1026,8352,28,5244,1436,3651,10513,7655,4609,7718,722,2066,8653,5428,2630,10421,4348,3225,2483,2390,407,4347,6548,3357,9017,3959,1311,4687,1864,3815,7860,5808,833,7853,5293,2032,4185,8576,8529,7310,5411,5448,9196,6311,1403,5260,3192,3485,8481,7769,5729,8249,9809,10407,5779,9162,4861,3747,5007,8565,150,1930,59,9012,6420,8601,1323,3382,10547,9104,2516,1866,7239,5308,6733,6967,4529,305,1914,5713,10320,4772,4140,2749,3591,5250,899,7081,9020,6409,8965,10656,9269,1896,3,3240,885,837,5062,3654,8658,7200,9568,4820,4917,3032,10576,3305,7492,394,5822,1262,4455,1266,4332,10128,8704,1943,901,4994,2291,852,5385,4703,7488,5551,6142,8051,2323,3484,900,7787,6383,5099,7499,4489,3561,3005,3672,9053,2678,2620,5480,6523,3986,8496,8673,8036,10492,3123,4932,6279,9394,2583,8766,3820,8782,3223,3939,6554,2596,6975,6208,8519,5429,5598,4556,3773,9885,5418,5269,6845,5945,6978,671,7361,4878,857,6065,3669,5153,6339,9781,7506,5582,10674,7783,1164,6454,6869,744,5571,1337,178,3159,4040,4487,10592,3464,4364,9407,3896,2419,351,5379,8019,10792,4217,1280,7422,10222,3499,8117,7432,435,5339,7143,9876,6029,6260,3573,7943,2998,4757,9359,86,4594,6384,5384,2037,806,1813,10493,10485,9639,9927,788,4945,6055,3688,8189,5401,8485,3053,6984,652,9087,1697,4410,7577,10647,7975,1225,10797,4670,4305,1554,9109,3402,9179,1367,1807,5406,7905,9802,9532,5049,1890,3281,8324,820,4227,10504,8964,6579,3009,7784,6134,516,6010,2584,574,3764,6900,6852,2336,6366,2108,10331,4628,1923,2441,5175,5310,2361,1099,7341,1540,7715,4245,430,9210,6867,7021,866,5941,3220,2820,10706,211,6392,2533,589,5389,9848,7241,1073,6699,4694,2436,9187,3328,3174,10193,1477,8517,2758,641,6327,1264,9178,3902,376,10241,517,3294,9789,3966,4568,6624,1873,7689,10014,8621,9727,679,1872,4229,1524,10454,7777,5629,2130,1552,907,1694,4086,9061,6322,9874,294,3629,3738,780,7258,3008,180,4316,2968,7545,5166,7660,7809,9863,4788,248,2393,3585,3472,6095,4522,3928,3238,4498,7439,4083,10099,8137,8053,4558,4928,9656,3062,302,4535,8590,8214,8765,5610,317,5917,4327,8714,8814,3653,2882], "time_ms": 0.108735, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 613, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 0.814401, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 428, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "generated_samples_per_query", "value": 10833, "time_ms": 0.814401, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 429, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "generated_query_duration", "value": 502603637, "time_ms": 0.814401, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 430, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 5439043.514937, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 898, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 5439043.514937, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 900, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 5439043.514937, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 902, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 5439043.514937, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 904, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 5439043.514937, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 906, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 5439043.514937, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 908, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "power_begin", "value": "10-22-2024 01:21:15.388", "time_ms": 5439043.515948, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 564, "pid": 57394, "tid": 57394}} +:::MLLOG {"key": "power_end", "value": "10-22-2024 02:51:54.421", "time_ms": 5439043.515948, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 566, "pid": 57394, "tid": 57394}} diff --git a/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_summary.txt b/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_summary.txt new file mode 100644 index 0000000..a05d865 --- /dev/null +++ b/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/accuracy/mlperf_log_summary.txt @@ -0,0 +1,4 @@ + +No warnings encountered during test. + +No errors encountered during test. diff --git a/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_accuracy.json b/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_accuracy.json new file mode 100644 index 0000000..0d4f101 --- /dev/null +++ b/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_accuracy.json @@ -0,0 +1,2 @@ +[ +] diff --git a/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_detail.txt b/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_detail.txt new file mode 100644 index 0000000..ecda366 --- /dev/null +++ b/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_detail.txt @@ -0,0 +1,87 @@ +:::MLLOG {"key": "loadgen_version", "value": "4.1 @ f5c8f17583", "time_ms": 0.006562, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 53, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "loadgen_build_date_local", "value": "2024-10-21T23:43:42.761999", "time_ms": 0.006562, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 55, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "loadgen_build_date_utc", "value": "2024-10-21T23:43:42.762007", "time_ms": 0.006562, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 56, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "loadgen_git_commit_date", "value": "2024-10-08T18:30:16+01:00", "time_ms": 0.006562, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 57, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "loadgen_git_log_message", "value": "f5c8f1758374aeaba26b2e84d31690111cfdf054 Fix bug: Loadgen ignoring token latency targets in user conf (#1874)\n976bb1ad9c7946be79507f3ff67955c27426af52 Set correct remote repo (#1871)\n41fa8aadd1ba0ecc97f6a519d8b42b04278e5f24 Add format files github action (#1682)\n518b454fd8647bfbd23a074e875e87353f33393e Tflite tpu (#1449)\ne0fdec1c7a75c98cfc194f13d62ac4388d419c8a Fix link in GettingStarted.ipynb (#1512)", "time_ms": 0.006562, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 58, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.006562, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 60, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"d5274ff0b56e8d3cdb273174628a4461fca6f02a","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/bindings/c_api.cc":"32181da9e161c285f8fe46ddaa49e6cba2f9f918","/bindings/c_api.h":"91f58bd79b83b278f3240174a9af747fc38aff74","/bindings/python_api.cc":"ea4c89decad19eaf3217bfa2fb757d3b83a561d6","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"126e952d00f4ea9efd12405fb209aa3ed585e4b2","/issue_query_controller.h":"923d9d5cdf598e3ec33d7a1110a31f7e11527ec7","/loadgen.cc":"6650091ba7a918f343b06eb7a5aa540eae87275f","/loadgen.h":"e00fdc6dbc85a8c9a8485dbcbfe2944f81251c4e","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"197efc96d178e5d33a750d07fa7b2966417506ea","/logging.h":"ddb961df7bcc145bcd7cce8c21f7cf075350dcbe","/pyproject.toml":"ca17720f9c8246e821331946d893e830fc88f8bd","/query_dispatch_library.h":"13ad6d842200cb161d6927eb74a3fafd79c46c75","/query_sample.h":"e9187c8612bbdc972305b789feb6e15c26e96cfe","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"34e2d2a44324cb07c884f92146ecbb8ef9d704e2","/results.h":"d82500c326c2de83db411f1146882aa4692b419c","/setup.py":"13c49b028b22749b5f3c44f3d9bb489e8c0574e9","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"c15c3e150030089a8d634bd2ad6d4b644002e613","/test_settings_internal.cc":"e21febd60f9b5bedd1fc81bb990f09c34b32043c","/test_settings_internal.h":"f1d5335b53ca610c30e0edc5d07999a27b5b4b9a","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"eea9b9cb1a06cd1abe1bbdaee82f9af31527fedb"}, "time_ms": 0.006562, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 67, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "test_datetime", "value": "2024-10-21T23:49:24Z", "time_ms": 0.021230, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1198, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "sut_name", "value": "PySUT", "time_ms": 0.021230, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1199, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "get_sut_name_duration_ns", "value": 441, "time_ms": 0.021230, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1200, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "qsl_name", "value": "PyQSL", "time_ms": 0.021230, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1201, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "qsl_reported_total_count", "value": 10833, "time_ms": 0.021230, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1202, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "qsl_reported_performance_count", "value": 10833, "time_ms": 0.021230, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1203, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 270, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_test_mode", "value": "PerformanceOnly", "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 271, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_offline_expected_qps", "value": 2.06111, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 308, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_min_duration_ms", "value": 600000, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 314, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 315, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_min_query_count", "value": 10833, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 316, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_max_query_count", "value": 0, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 317, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 318, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 319, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 321, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 322, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 324, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 326, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 328, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 329, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 331, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 333, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_performance_sample_count_override", "value": 10833, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 335, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "requested_sample_concatenate_permutation", "value": false, "time_ms": 0.030418, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 337, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 413, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_test_mode", "value": "PerformanceOnly", "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 414, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_samples_per_query", "value": 10833, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 416, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_target_qps", "value": 2.06111, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 417, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 418, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 419, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 421, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 422, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_min_duration_ms", "value": 600000, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 424, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 425, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 426, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_max_query_count", "value": 0, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 427, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_min_sample_count", "value": 10833, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 428, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_qsl_rng_seed", "value": 3066443479025735752, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 429, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 10688027786191513374, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 430, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_schedule_rng_seed", "value": 14962580496156340209, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 432, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 433, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 435, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 437, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 439, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 440, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 442, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 444, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_performance_sample_count", "value": 10833, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 446, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "effective_sample_concatenate_permutation", "value": false, "time_ms": 0.030909, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 448, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "generic_message", "value": "Starting performance mode", "time_ms": 0.043632, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 841, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "loaded_qsl_set", "value": [4119,3771,2947,5174,8221,10562,9791,4633,8569,8837,7091,5810,6371,10778,540,1732,2832,4085,676,6115,887,9339,10819,344,9312,1951,5739,9692,9525,4397,638,4375,10601,8825,6906,3471,9163,7272,9940,1207,7738,1663,9660,5625,2468,396,7153,8514,1418,8510,4187,1103,9970,10200,2141,6364,9748,8412,2719,5615,2838,7835,752,6099,10486,8982,3673,7292,10573,3118,9998,6825,5831,4451,9259,4859,1199,7842,6587,920,6687,3869,1033,6262,1223,30,5009,10063,2956,1007,2185,64,77,3470,6239,6679,1470,62,3074,1622,8153,202,9235,1614,9392,4912,4968,1648,122,3772,902,3526,1772,858,265,5129,6799,6255,523,10796,8819,1230,9400,3732,1321,7882,7821,9768,6056,4282,4617,6107,10130,7756,3036,10451,2201,3976,1086,3668,5394,3177,5128,8384,7234,4039,8503,3640,10043,5766,6810,1011,7547,1718,2401,3909,8343,5923,10069,9886,532,3270,7147,10062,9680,5432,9974,7736,3655,811,1105,1820,7465,2267,9641,4046,3956,5247,5181,7625,4073,9663,288,426,6762,1617,7914,4014,2904,9151,4181,6042,8967,6981,5547,10457,9322,500,938,131,8457,1171,3814,10549,8093,10675,2159,5807,7567,8366,6518,943,6819,5889,5108,7755,5382,8456,9756,10247,9351,3150,8971,1231,2990,5380,510,4324,3049,1165,6116,8684,6992,1829,139,1021,1602,5805,4226,5966,1040,3274,4269,10376,5316,8756,9587,8231,2809,2270,7887,7390,485,8028,7880,328,6236,514,4644,6061,1522,10537,9446,8787,8577,6774,2023,7474,4904,5064,6183,5196,3361,8268,7606,5968,6182,5019,8265,4621,1525,8181,163,1352,5785,8632,1074,2392,8398,6561,125,7071,5577,995,3783,3849,9753,8651,1051,1159,3971,8903,8413,162,5854,2133,1416,9098,1759,633,5800,7671,4731,10623,5586,10578,8929,5894,6833,2909,5063,3163,5002,6278,4148,7477,8408,10104,8805,1504,8105,5746,3572,8095,312,7436,6165,2786,6286,2079,4807,10523,6117,3840,4169,8750,3016,10238,1137,10037,9427,8449,4908,3020,7183,1557,10500,2982,8020,599,1519,6873,10276,7350,8224,719,8474,9981,9198,2802,5756,1897,10801,7682,91,4160,6697,2989,10113,5765,982,48,6573,4412,5148,7681,2514,6348,620,6702,250,10219,7813,4512,3988,7538,3272,4433,4597,7580,8229,603,9284,9959,4683,711,7542,5450,1175,9859,10685,5041,9760,765,6303,3395,2908,7823,7516,237,105,8034,1757,10401,9935,5256,7800,2642,276,8636,5297,10766,5954,5725,8144,6660,3901,8078,4158,9455,7297,5959,8088,8583,4733,10345,5232,9216,8902,4667,1449,801,9231,1792,1211,6013,8566,4988,2938,4730,404,8699,2346,3059,672,3701,7231,10169,10431,5806,1373,4193,10665,10030,4335,7950,10150,842,5753,6435,5096,5922,5497,3193,9367,2158,3816,5356,65,258,4675,1277,2041,6032,10088,7814,8726,2036,7885,9303,7440,8023,1151,5749,9573,7776,572,1240,6297,6439,1343,8605,3386,3251,4631,8140,3760,2068,4957,2623,4961,10281,1467,9037,10546,4415,215,2194,3273,387,3327,2225,10739,300,4467,10823,10668,710,4228,4517,2069,8709,2409,2573,6144,9357,4844,3985,2864,2814,2615,8952,5085,5090,1937,1854,10689,5061,10374,1580,5223,4851,7888,869,3023,10178,1615,7566,7219,5276,6685,8839,10534,10532,5826,1251,9124,8308,8702,1805,3148,3529,9942,8171,2450,6782,7679,2503,9989,2748,3961,3660,9466,6832,1635,1831,9298,6457,3165,6633,6513,10354,7865,1888,1182,10643,5477,3120,7676,6178,2833,9468,4910,2657,9723,492,3347,6481,2822,4426,1018,486,826,3455,1535,1539,5782,8267,10102,2714,5573,5482,2442,537,6934,9420,1925,10526,8158,583,5921,5634,10715,2063,2865,4722,6711,3033,5830,6761,10221,3292,5093,3487,9844,5192,3127,3166,5517,8842,5986,6515,8806,4368,3880,682,1506,8542,6600,5522,10789,2775,6979,7375,9732,9488,7435,6127,2058,8763,5776,2901,551,2871,7369,6929,4164,307,7840,861,2670,8027,9410,2386,2513,1063,5122,8218,4457,1100,2437,462,5774,9629,10033,5794,4106,3616,7179,5414,4598,9925,9289,69,9948,9945,4585,6214,1953,6363,7744,6030,3012,37,4855,10794,2109,5500,1652,9019,7047,3763,10114,2735,4137,7192,6450,7896,6319,2559,9057,4150,1626,1399,3805,10481,7054,3160,3802,2753,643,4212,10435,3360,157,8424,6667,1686,8112,1983,6648,9535,2567,6149,8099,319,10795,2980,8559,3326,2116,2360,7544,5778,2589,3310,3751,3352,4366,9705,368,6088,8217,2399,6537,10120,6669,3061,1457,5252,7052,10159,6880,5593,8852,3506,5847,4877,4944,8609,5364,1816,5290,634,1661,1221,6935,702,4824,1453,8062,7092,345,7030,9428,204,3121,2188,6429,7446,10631,2354,2569,1219,5475,5727,6274,1556,8968,7730,10342,7917,1985,6856,5692,7132,1146,10468,6861,1550,2928,6585,5681,5352,5348,10265,10307,322,9139,8357,9486,7441,4023,10141,10170,1441,165,9603,5998,10586,9022,5633,1234,6283,2131,7996,5042,6246,4201,3064,3071,5674,6769,9698,386,9319,10686,4,9197,3510,428,382,6614,7688,6154,10078,2876,8185,5240,1006,8696,4353,5760,7961,8376,6405,2107,5804,9860,3812,1679,2756,9652,7964,750,1997,4091,1899,1308,4035,6656,8123,10248,1510,2846,5638,477,4076,10730,10589,8266,3218,4977,2536,5334,10054,8762,8598,3929,3811,9581,7752,552,9441,5711,6642,1129,284,976,10747,933,7929,7607,9805,5102,9979,2939,6910,2647,4835,7528,4876,6734,2706,4394,4926,8341,8734,1135,3753,9333,1291,6011,3867,5563,3343,7612,1027,5165,9700,6340,6104,3099,9537,3428,1612,1606,10634,10021,2761,1042,9310,3601,6806,4003,8336,1320,8874,6545,3842,3715,4916,177,6141,922,10477,10369,3372,8706,7261,9000,2944,1770,4537,8671,8299,4515,6709,7300,5017,1499,3864,4145,8725,2669,6292,6724,7960,6080,4257,743,1363,10444,4518,9545,10388,2296,3870,3737,984,4590,5924,149,1966,10084,6018,1340,8785,1678,4211,8663,6542,1475,6302,7202,1850,9261,10681,2339,8461,2664,7011,3441,9549,1482,9314,6828,1290,9203,7486,209,5215,2113,714,6092,3889,825,7966,5609,3787,6223,207,6965,2644,4612,9140,10280,472,5726,5208,8772,2044,2931,1775,4134,10539,7508,172,8681,8240,10189,4870,8908,7684,9379,3266,3376,8405,10311,453,8213,2034,2190,2294,8580,4421,4725,3952,2918,8999,8545,7043,542,10301,9146,9032,3597,4112,2062,3195,2850,6169,10805,4900,8382,2059,5703,10074,7541,10662,1760,2114,10749,5993,6295,5211,7792,4639,8333,3833,1847,83,10770,8312,8831,9004,371,4923,7699,3307,7834,5879,7458,6735,9799,10741,1304,1576,8657,3329,8860,6081,1744,7308,10564,7102,10611,7126,10413,6653,6846,1371,6864,2078,8362,3021,3569,10777,1939,2861,8085,4899,3288,7360,7690,1489,5526,116,7151,5145,6673,5523,9390,8191,1004,10528,8958,9118,7393,4684,7266,6966,5995,8508,3778,313,2324,3282,10165,3447,6294,6865,9833,2202,6316,365,3052,7057,10821,1194,7974,1410,1782,9986,1123,5591,7286,7552,10721,1088,1793,6524,117,1763,8854,2199,3366,6230,6135,4113,4154,2326,7696,10438,4546,6268,4991,8670,9515,9500,6272,5253,2634,10092,965,6654,316,7333,10361,7385,2359,2521,10505,6418,3731,6452,4151,2002,4407,9193,5715,5311,9681,1461,6752,10196,9063,7064,7760,10356,9534,6401,9570,6213,7243,1969,5069,4463,8379,2228,10617,10563,4001,7533,4118,8634,303,4758,6480,6863,4079,3551,2387,959,10384,8204,3739,4388,9571,5235,890,3448,3380,4022,3407,3303,8225,1710,3134,8655,8273,1442,9126,9090,767,2352,5036,6959,1116,7507,8320,206,5100,10580,4657,10013,591,5970,3983,10637,3489,6177,4329,9309,598,3799,7447,4206,7727,6138,480,3483,3979,1719,5575,5814,6507,10156,4769,9437,3749,949,10166,9129,1387,7559,21,3606,2329,10379,71,6909,4268,3231,7838,4403,4880,2289,4825,7313,3855,9482,5669,5755,1472,8584,145,7282,4105,5546,9644,1811,10620,8543,8840,6823,3084,9331,7771,5570,2129,3333,9811,1567,9316,6617,7070,2971,5115,9806,8920,6889,6597,9550,7991,10076,1289,8872,1156,963,6369,353,4520,4406,4510,327,2672,10123,881,4197,10616,7244,6791,4055,9060,7901,4398,10019,5895,6816,724,6400,3142,2649,727,7658,4411,4131,4893,3203,6901,5131,5905,7199,3408,4562,101,9214,54,2388,5032,6645,6310,973,7184,5595,4177,3578,10735,6783,1468,7622,854,1425,1272,8441,10581,2009,3999,4258,1929,7531,8205,610,8733,2698,10366,9817,7227,10638,1446,4587,10052,9114,3354,10140,677,5614,10068,3206,10826,5460,3063,1093,5083,10551,502,9391,7170,6267,10812,1555,9356,8910,4623,3970,10429,4096,9172,522,7591,8773,2791,5612,3922,6555,8079,6802,5491,4135,6640,7203,373,9221,2355,7947,5119,5940,5363,10443,646,8044,3473,2214,5335,5803,10138,5180,5525,927,5272,2372,2951,526,7209,5972,1753,6877,1496,185,843,1124,8942,5229,1918,10371,1282,6424,10228,3253,354,7137,9669,5618,2061,632,986,1991,9433,5261,5130,2246,6315,3835,5515,2976,187,7121,2230,7683,930,10571,2538,3106,1630,8453,5267,6534,6349,8372,6532,7725,8522,8596,4984,6027,1483,2281,10284,4300,7155,8557,1842,6925,251,4156,2883,10632,4008,8994,3844,3112,5932,6837,4987,4751,7133,9724,4981,6781,5405,8770,1012,9963,9551,9840,5471,3562,454,4696,198,1902,1516,8943,4082,7253,497,281,8464,3677,8635,10172,7900,5238,2747,4914,2487,9376,2618,6526,9801,463,10039,1226,7307,4673,7525,8661,1010,6727,9665,10411,1307,4267,6543,9564,4793,9552,6627,3638,3785,2658,3384,3158,9720,9818,10790,6483,9067,669,3930,7843,5427,7587,7536,7029,3125,3531,6385,1565,8749,1736,3512,5719,4868,888,8255,8535,6194,10174,8304,5939,5070,6040,588,8437,10759,8538,8206,1458,5159,3997,1259,4858,7176,9035,10403,2868,1122,5888,9842,4622,4261,10673,9324,8487,600,9479,6220,3419,195,2155,399,287,9830,9883,9328,2055,7836,10511,1451,6850,5911,3337,9778,10660,4739,5981,8347,3588,9865,4238,8957,5074,1691,6813,3860,3883,8878,7674,8731,7336,9136,7178,4565,7820,10508,2051,8164,823,7624,2648,3128,7150,7687,4830,10350,8119,416,735,4601,5587,120,2691,3920,4192,7125,10053,25,808,9563,4299,10112,5112,3643,5446,7403,8391,7114,1542,2628,8911,9279,2886,3221,10056,1912,3291,5819,3830,8896,10788,4000,5815,1183,355,9074,8086,5140,10657,2345,2550,8728,8468,697,10557,10322,8004,1473,1698,4574,4635,1921,261,5606,3409,9595,8966,5447,4599,6622,7714,5381,2778,9396,7374,10654,1346,5113,1536,6553,1306,1932,3216,656,10615,6053,96,4447,8775,8624,8685,2235,7956,4390,1420,220,6145,2255,2478,2730,7084,1155,9091,5757,8924,2955,7628,2093,1334,1068,8013,1927,7841,9731,10436,809,6519,5217,174,2169,7952,10341,1776,10629,3169,8108,3635,5671,8667,1653,10761,1309,3659,7039,8056,2827,42,524,10646,921,2283,449,9983,2561,4542,4260,1725,9512,4540,1378,7252,2161,6347,937,8455,7862,5008,10109,1246,7697,6176,2816,1824,1746,734,7320,3564,3892,9662,2526,1553,8188,7826,3837,429,6706,2945,9511,3466,9636,6688,2453,8254,8438,8321,2699,5091,3521,3229,9219,4792,1859,9449,5190,8178,3073,2781,5817,8737,1855,2849,3602,79,6723,10289,8429,3626,7006,5884,3852,2315,5409,8693,1762,4978,8377,9892,3182,1361,2626,2581,10619,115,7025,9240,2236,2317,3641,1835,7551,4979,7791,6767,8032,7159,3000,1145,2175,549,3987,1748,8906,7038,7915,9995,8563,5741,9152,5391,7122,7933,5747,1511,8215,948,15,1029,6760,6874,6170,1801,9164,9377,8369,3748,10277,7497,7145,4724,17,2694,1041,1682,3788,8184,1015,8296,2167,4999,4809,10396,9853,4018,2086,10584,10089,3002,5841,5318,689,7581,7680,3427,950,10466,2087,9890,7666,4523,8530,1538,10784,247,6301,7460,3972,684,7646,8883,8024,1238,6393,9716,618,5104,6728,6168,7797,10273,7164,670,10252,85,6602,1693,2527,4274,3552,2284,4290,6156,8784,274,1620,9382,5000,110,10290,1326,4785,8644,9321,8743,9,1878,9711,9425,8764,6691,4962,9498,10727,2959,5262,10066,372,1336,4845,7753,5342,2047,4024,2708,7710,8639,7472,8668,337,3094,4172,9903,7644,10831,10082,4625,3043,4471,9277,1133,9577,1523,8489,1841,7878,2932,9616,7479,2448,8165,6033,4630,1946,5486,9327,9442,6824,2104,6855,8319,8378,6456,6623,3915,988,2662,7009,1331,4094,2812,796,9735,2396,3797,9481,8897,3743,8803,1808,8879,6637,9233,3703,10278,3136,10094,81,2825,23,5436,5321,3824,4891,10231,1138,6722,5433,10553,2430,2984,1167,7110,9750,10009,8406,4659,8768,5678,8435,6988,1727,9161,9051,8582,4297,1815,5707,4350,243,1708,567,4490,9397,4652,10440,667,2578,9330,7469,8600,7356,9971,8486,1243,9547,6352,8556,19,9780,4502,2964,10460,2851,7626,2804,3746,58,2929,2472,4481,10015,4514,5012,6768,8136,7804,8161,5094,1206,7412,87,1022,3011,1545,6527,4404,1587,219,6490,3575,2132,2170,8614,4632,1562,7478,1779,2568,2949,10606,5660,1261,8182,8310,9657,2565,10346,822,2574,9905,4513,942,7786,1994,3533,6809,6583,3425,6015,7013,6397,4259,6497,9677,2497,3363,1940,4265,6946,3088,1401,10722,10732,5925,2685,6250,3914,4958,1876,10051,7221,26,84,2499,10705,4400,3723,5910,10250,5623,6564,4077,6725,6325,9633,9992,437,9142,605,3683,8832,8260,9272,4071,5646,3230,3596,6495,5696,4247,10351,5313,5373,4495,9682,9607,6146,358,4643,8625,3709,5207,5780,2967,5613,9068,3608,10710,6708,1898,5621,3131,4750,10125,3496,103,5264,6034,10680,9190,1349,3828,5604,9517,9102,8439,9832,8173,6119,3318,5786,3492,401,9618,6737,9246,10146,3680,7759,8436,1478,3378,3003,1660,925,9796,5173,1044,2008,3475,865,6017,718,9793,1592,9430,6229,529,7481,95,7128,9088,8572,4171,2859,8960,8950,3438,7131,5,8015,10358,67,8337,807,6106,1295,6866,9463,2720,3919,5147,1892,9815,10195,9138,266,3848,3780,1271,9627,5332,1521,10639,3556,9904,7271,3891,1213,8989,4234,563,4263,3480,10700,5545,7916,9350,9786,6511,4021,6922,1984,4254,755,6461,3650,5958,5689,3435,7761,3648,8174,8881,2716,6508,6386,5843,3153,6167,5457,8103,7101,5161,7225,3116,6335,6232,3624,6746,7037,10058,489,7847,812,6876,2727,7260,10814,4133,9238,10395,8947,4124,4669,875,4427,2431,4726,3733,3001,3539,768,8472,4334,6094,7578,6868,7276,9502,6084,7087,402,9412,9610,10335,6365,2290,9256,6903,7270,1972,9424,10448,3560,6410,8419,4920,5914,5584,9059,9529,562,4294,5088,6380,5179,5616,9946,6684,4795,4616,9561,7288,1987,7946,7108,8662,5201,7766,6575,1062,3497,9869,9122,961,8317,10115,4600,1059,8,6879,3847,7515,9775,3184,3741,3809,341,9223,6047,8395,2605,4141,2368,1109,181,2919,2252,321,8946,2397,1328,2189,2233,8146,4925,3129,5624,8067,8220,2629,8591,6744,8973,3534,1933,10737,10754,1081,4428,7665,2095,5946,8168,4176,8325,5243,10191,5745,627,2313,5089,9157,10175,1803,8250,4839,4344,1397,2785,8976,4182,7627,2439,8767,992,3594,818,2102,2101,7732,2136,10045,6248,1701,7480,10187,9937,3843,4786,8259,7733,7721,5053,6924,7265,10529,10608,8077,7425,10765,813,4107,5464,8698,8647,3665,581,980,8450,10412,9757,6155,1706,1941,8281,6977,5637,7609,8901,10709,5763,7148,7382,9695,2239,2800,9120,5495,7097,3339,9044,827,50,1384,5652,4019,9507,9295,2200,7869,9741,548,5114,8388,2264,1407,4688,395,5627,7012,2351,9849,3137,6676,1406,8049,10738,7938,6844,8757,1906,1720,824,152,1742,32,8444,3287,6875,521,4929,860,1248,7532,9617,94,6945,8107,8890,2118,7189,6022,154,1980,3854,3140,4681,4570,1392,10217,4753,9345,8277,6969,10410,6610,9759,6796,742,9170,7334,7780,717,5580,5154,4276,4306,8589,6427,10628,915,3066,1846,10121,10220,2508,4302,1543,4611,10453,2889,7579,5234,4531,3198,10398,10020,6225,5270,10618,1303,10518,8779,10461,891,4256,10506,5200,4219,9264,8721,3488,2880,7936,6805,2890,8052,6522,6509,2171,2995,1433,1839,10227,3882,6721,3807,2408,2490,1655,10605,7259,1887,1633,5018,5907,10151,3302,7634,3450,7198,10367,5511,8730,747,9227,1485,916,2122,3422,1130,5716,2056,9262,5693,6798,6626,8951,2751,4995,8045,3931,10260,3342,862,1645,3349,8241,7103,5082,6862,6982,3208,7729,4230,8507,2794,6525,5350,6103,3755,6140,7475,3405,4271,9560,8504,10318,4354,4804,4034,7982,4743,10081,5034,5204,4456,2219,5283,9225,10570,2549,7837,8810,7726,8525,4911,3469,1584,1560,2451,5714,10472,1089,7401,4989,4385,3767,8397,5864,4789,3259,4890,5461,5320,1786,1140,3776,3696,9036,5564,2049,6468,9156,10126,1935,234,9858,9891,8536,1209,4892,8258,8291,6581,6296,10502,3412,10768,1198,188,4816,2370,6120,8353,3903,2653,4473,9807,3336,10450,336,9875,10364,5056,853,9406,2684,2793,40,6549,4146,5640,8555,5304,7418,471,8630,5934,2035,8983,3445,290,3371,4401,8316,10649,5231,164,9503,7449,5802,2757,8777,816,3911,2,6857,9514,960,1621,5752,1908,466,709,3191,9230,6834,6411,6592,7152,4715,8364,918,7236,1398,262,6157,6360,2921,7615,5288,8106,8849,3554,10572,8991,2539,5869,9213,1216,10205,561,8327,7083,6887,3060,4662,10044,4191,6757,6073,7207,560,4918,5459,8732,10017,7135,7652,5535,7610,495,273,5312,3205,6574,2417,584,5836,10755,5353,759,3625,8620,420,7973,2307,66,5398,2085,10086,723,7014,519,5236,5632,8553,6916,6998,8152,1275,3831,10010,10463,838,2524,4967,9471,5851,6093,4443,4646,8309,7431,2676,10055,6830,4125,3918,5589,3010,6421,2099,6933,6436,72,8166,8163,10515,4243,5788,450,10803,9002,6970,1769,2469,3200,8104,9015,1690,6937,855,6928,9291,4941,839,9930,7156,7706,4081,3157,8792,8833,10389,3563,10,4377,3576,4697,7293,1518,6860,10329,9176,9462,8131,9614,5770,7315,192,8974,6949,9467,9960,8516,6686,10083,9837,4736,5790,5619,3704,5021,9476,2830,7667,7796,4310,5560,10482,6927,1338,9168,6215,199,6636,3334,736,2197,4126,5777,4790,8780,8041,4551,9267,2103,7162,828,8017,7190,9703,2127,2927,147,7206,2050,9966,3873,8688,7673,5456,135,431,673,7177,6377,5127,4934,6254,5626,5899,4771,473,7437,1169,6379,6983,2858,196,1437,7010,9086,3399,10046,6596,8526,9415,9897,111,5043,4298,8918,6621,935,6991,3676,9578,6304,4155,9749,4602,7003,2347,2356,3803,4396,3893,2328,8008,9080,3092,6,3868,5686,6187,8753,4117,3827,7839,7802,886,7335,6504,5407,8562,9025,8690,2280,7174,6455,4088,4782,10711,4028,9609,503,9260,9167,1101,6139,6477,4806,9054,8853,6694,2675,6907,5095,3365,10135,2635,2179,6345,3916,10650,5588,3946,7935,9270,8573,1954,2668,7883,5073,9113,7274,8506,5789,1170,3817,2025,2558,2930,6677,1237,10535,3048,5337,5718,4716,3595,5829,4484,1296,1869,1917,8270,1571,2788,5991,1215,1784,4026,10806,3362,10767,8060,7707,4475,9900,10469,10163,7968,647,5425,10308,6002,6493,1364,5762,6698,2545,3856,3924,8431,6247,4392,6777,9694,9804,2686,6563,4577,5772,3144,6822,6884,2836,8850,8094,2821,1948,9586,4533,6110,3712,4199,9103,129,10164,1348,4064,9565,6125,4138,6980,847,5975,169,10600,4323,10072,5913,6375,8035,8155,1563,8328,7201,9934,2410,9856,4423,4637,2398,6290,9363,10699,1833,4767,6129,7556,1208,3982,6244,191,7024,3183,1735,2193,10317,10626,505,445,10257,2710,10330,8720,2534,1699,7061,275,3942,2477,4419,10426,1517,2622,9286,9030,7554,179,7188,5937,9497,1586,9211,5323,6521,9632,999,405,8373,9292,8812,4986,9445,153,3298,2973,8030,9150,7407,2234,863,1548,5328,4231,1669,8363,1276,9774,6514,5054,384,802,2896,940,8293,5167,4544,10091,8984,4821,7220,7959,9005,9249,4416,4721,5592,6076,4389,45,6716,3519,10820,6719,1781,3161,5442,1313,7141,9096,10636,9843,4279,7790,10409,6433,3351,1938,374,5402,9271,7075,2805,5431,7325,6565,2773,7340,2485,8318,8193,461,5345,8187,412,7342,2476,1702,4996,8452,5548,7214,2260,5900,10525,8305,1342,7913,253,2981,9746,2958,1421,2627,1341,9765,1549,929,1840,1178,2466,9353,3252,9403,6487,4854,597,6938,6488,286,6792,762,5919,2092,2884,1724,475,6827,6211,1333,9454,6986,13,2168,1391,2067,609,5116,7378,4960,9290,1657,9620,5336,8686,3243,5885,7257,6961,201,9658,8788,3097,8502,4805,10487,911,7829,5956,5366,8415,5651,1578,6314,4237,2562,5501,2366,5504,9881,6217,4973,3028,5307,9605,1460,5583,9812,9281,8741,5133,5521,1791,4066,2285,9188,9954,7788,9717,5022,5218,4311,790,9089,2367,7997,10489,2272,5410,99,10061,8346,4930,6835,6048,7535,3091,9401,2593,2937,4829,10714,1996,10064,3424,9182,941,9928,1082,10347,3145,979,9133,3454,1711,946,7088,7078,5079,3933,4365,5938,3905,2650,1863,10688,6729,7242,6905,3301,6078,7467,4208,4240,102,2898,7866,3908,0,2320,4564,9645,3130,5168,730,9889,1750,5874,8513,2268,5242,493,8981,3411,8873,9200,6005,9460,5665,5708,9797,6328,1036,6745,4405,8295,1758,6789,468,10742,10566,6780,8567,4284,7106,3888,6747,10134,6950,10079,4363,7898,5374,1257,7745,2380,3605,895,2687,3690,6462,2529,8692,9247,7395,3283,8669,9707,9205,7983,6253,3503,7919,1053,704,2244,6779,3846,6993,3850,4924,1971,10387,240,7572,8863,7864,663,4581,1409,2378,9100,10684,1465,8200,5679,8430,5866,10602,4143,4065,7157,8933,7763,756,8091,2488,3115,8248,4356,4813,2288,9687,1393,5999,1844,7255,5349,10830,4116,1112,8544,3895,6576,8194,8834,10391,7969,10713,270,8909,4111,137,9459,7322,10103,8581,3155,7142,7191,5400,8913,8209,2912,3434,9773,879,4719,2807,9704,53,6242,6241,10666,1322,7298,10008,6086,680,8047,8898,5265,4202,5467,2493,7408,4157,10255,7722,4414,2878,8074,7421,9232,7063,8111,1670,10266,8674,7217,10279,2979,5574,6269,5317,10415,5184,2429,6674,419,1028,602,3727,7429,8638,6839,4843,9767,9621,5347,8133,7694,5617,1227,9555,3029,7363,6500,1191,3770,569,9048,5524,4940,10368,3132,7358,6003,9402,5732,2310,9557,4538,4345,8550,2484,7195,2504,7417,9600,6031,5846,6812,2555,4214,6655,8537,10751,6918,10757,8349,3244,4011,2276,1882,4060,3935,5393,7871,8386,7808,6472,3279,9192,7420,5157,8294,9566,3586,6818,3461,2903,10582,4755,932,6547,7731,7116,6503,4569,2923,910,3633,10723,9597,7027,4013,318,4554,10480,4935,6482,9895,3968,9779,6338,5187,3697,4582,1668,5137,8926,2100,8939,421,7079,8746,9413,4702,4901,3314,4101,5505,4007,1945,2963,5969,4783,1783,9823,256,4620,9153,9013,4927,9344,381,3998,9990,9381,1974,343,1916,7817,8445,664,7711,1362,956,5597,4417,9924,10746,5499,8223,10152,10186,7344,10783,919,5740,2709,10804,3018,9651,1095,6801,2717,695,1039,4704,1968,7498,5877,6665,6062,6342,1369,7364,3518,1077,3477,167,6184,5735,10006,7140,142,10370,7238,8799,8729,8082,9052,7067,8401,10349,9366,2312,1977,687,2772,2054,1413,612,3524,9634,9184,487,2026,2950,10136,2795,9524,4779,2019,5694,9852,557,4399,3006,9302,448,9064,5979,614,8551,10117,5001,8813,2045,2004,1154,2660,2894,10769,10256,218,4627,5322,6193,4038,1047,10321,10185,8329,7893,3951,2841,5700,10545,1978,8269,7370,7944,1003,7894,5343,6838,3515,2592,5720,8210,4921,1411,8822,5033,5904,726,5850,10080,7768,8330,1761,10253,1999,2046,912,7247,388,909,7637,2885,6014,556,9409,8055,3013,2750,356,2013,7530,10229,7411,7058,3058,5964,661,5758,5075,9836,6536,7720,10652,7524,4127,5014,4102,7705,3151,5532,9841,10340,7154,3284,7345,3375,6469,5518,4207,9845,400,1689,9398,10817,8610,6210,1751,10381,8534,6921,7999,6198,579,5124,5576,5607,5123,2421,10295,936,9870,7568,7811,7124,7907,411,3853,9183,106,6917,264,10050,10827,7105,8068,10781,1848,8039,6663,2847,9257,216,1113,4974,3196,611,304,2473,2790,10718,8641,10239,6376,1673,6374,9374,10406,2266,1651,9275,1450,5199,3493,729,8786,3965,2164,10365,10704,10208,1957,9965,9217,10139,733,5282,8080,3543,5275,590,1160,3542,9855,2518,2115,4956,10293,406,4864,5139,90,2580,370,9999,4425,7981,6406,4233,1031,2241,3237,1368,2321,6915,6584,6024,9548,2178,6387,649,754,8490,3414,7263,9642,2974,1402,9436,1217,3607,1,10445,2177,5224,10785,6035,7483,4860,9638,2206,1664,3234,2369,9648,2988,4099,9207,8862,1325,3217,9838,24,8606,6913,613,4303,5931,2463,2389,4701,3517,3541,10703,76,3694,9364,1845,8650,8520,7539,1716,7934,7906,1623,8466,2147,8175,10031,6726,329,10339,4814,7099,3759,10691,2464,6133,7713,3511,8759,8141,8368,10168,10610,9278,2522,10242,8130,224,2991,7810,2665,1142,6650,7115,5876,7737,5600,1412,8703,2590,9123,9408,9072,7819,10100,8038,5736,4266,2379,6486,7044,1502,484,4061,7076,4856,830,9697,361,8006,4689,9128,6434,5473,4976,7287,6089,9453,8501,2261,694,1665,5015,9854,9116,6001,238,5344,7822,10075,520,10527,944,5415,5578,3430,8048,4248,2837,5481,4618,1894,6009,6394,6147,8147,4351,4142,5620,3634,5536,8907,8980,3468,5449,7897,2920,5155,5395,3019,5222,2840,6755,7988,2654,4044,6114,3682,666,1302,737,3689,9752,1569,1785,6357,1114,675,108,10494,3388,513,2607,7824,7604,8675,6630,8212,5132,4706,9440,8955,2474,997,3170,7381,5811,5423,9952,4352,536,6807,6608,4381,3613,6090,2043,9993,8022,6463,5233,5538,3567,8628,10107,4752,2176,2510,9131,608,1934,10326,6049,7473,10002,9980,9325,8922,173,4485,2029,6396,8838,3465,5068,8116,1752,3938,2702,3111,4095,2364,5833,3051,1076,9518,9346,9961,4445,10800,1787,9308,436,9226,2645,1335,6681,10203,2632,109,189,6794,6162,3330,4732,10731,2570,2874,7353,9202,3047,1624,1049,10760,4875,1474,3649,9608,8257,5856,2064,8798,9299,2151,8005,4950,8407,2553,10736,5585,3257,2732,3721,1488,9661,4482,8935,10132,7630,5372,893,7775,10774,5823,6158,5209,3451,6891,3300,8847,8532,6843,8934,6898,6790,2606,8179,4049,8802,2680,1512,906,1111,4647,3762,6238,8988,6754,5832,6252,5058,3387,5371,4464,3207,3791,8987,1915,9554,504,6612,9795,7017,2817,6043,3181,9908,7232,8996,4938,2603,10153,2996,8961,8102,8494,8546,7196,9426,10531,5274,5378,9572,9076,7582,7739,4030,9819,2759,4619,5997,2557,8196,5858,1822,8552,5375,792,9222,3432,6101,8917,6499,2065,8660,7774,9932,1634,2053,460,2302,6753,9132,3549,8109,4132,2226,35,856,7415,1789,10690,1117,7113,10701,4075,10160,5071,6859,6895,10327,3925,1886,4449,7509,3389,1094,3621,3138,8654,4811,8794,3825,8282,3255,10427,1318,7657,3724,4749,3832,7691,3700,654,10775,6337,9337,5942,2010,9884,1312,4170,6284,4315,1647,8322,413,5086,7301,10671,10286,57,7291,8493,8827,5516,5906,5944,3083,5859,7521,8744,9754,7249,4505,4325,3481,2395,7226,7462,4472,5996,7193,8297,6354,5730,3544,3775,9949,6291,3664,2797,2215,2382,576,8029,527,964,6438,1643,3289,9708,1424,6054,10708,7651,772,10561,1065,884,8975,4503,1814,4889,6641,4252,7430,8817,140,4691,4309,9589,9736,8607,10522,6358,5483,5424,10622,8007,6096,3368,2576,6381,872,952,8533,4866,8454,1462,525,8302,8570,1305,8578,10192,3851,8998,2077,4583,7603,1981,1950,6336,9929,5882,1017,6446,10118,1091,10455,3910,1910,3168,5820,7254,9582,2248,10158,4501,3219,7586,3332,2495,10149,1400,9536,8198,1825,8617,7892,6695,1674,564,1837,8354,7450,3757,3350,6431,8710,2218,883,7237,6185,9864,4220,4166,8345,3899,1141,6516,9239,2828,8645,2299,280,10297,4149,6999,3323,6334,6890,5010,533,7576,8887,3871,10267,151,10683,8121,241,1263,4798,8306,7032,7161,10209,8138,8648,3886,2731,3246,6008,6293,8100,3735,535,10541,2575,194,398,9077,4511,2897,2801,1107,9448,4698,9977,56,5691,2799,6299,1877,2546,249,6840,369,8154,4543,1030,7511,6672,2479,3693,4817,7215,1959,9969,8892,8588,8949,2597,9591,5225,5839,8656,9544,10360,1379,770,4033,9667,1889,10119,9584,8283,1173,1268,3054,7641,1976,1180,186,1242,8339,10012,10655,10510,3278,686,4677,8761,9171,546,6318,1990,10316,7623,119,297,10223,3981,3652,70,2530,367,2505,5067,3426,5107,9593,5494,4975,6102,10542,9640,6174,2993,993,10810,7304,6270,3623,6960,2789,10659,9465,7208,232,1797,9084,4634,10213,2892,1531,10538,6240,4048,434,1696,2096,2154,5955,3906,1023,764,7620,4200,1480,5444,8855,2875,4680,2373,2509,415,4847,244,3536,7396,7854,6402,6764,5717,2612,7518,1388,2494,7990,7695,4699,725,10423,6758,1529,10825,10028,4651,9574,3622,7717,6038,34,782,3761,1360,1252,469,1332,8916,7484,6163,1355,6793,2856,5510,3793,2097,3537,9431,7678,9169,4833,6330,5980,3209,7638,4552,4100,4097,6541,5193,8253,8484,4713,1619,8236,10598,7169,5420,7904,7461,2877,6963,4387,5027,7978,7922,5764,868,9365,5698,10613,1471,7376,2475,7724,5441,7876,7772,4810,5974,6180,2341,7399,4869,8272,2551,9540,3663,4661,7387,9215,678,559,8585,9556,1172,8642,10262,1629,758,5978,5650,1058,4005,8479,3393,2314,3109,688,4087,1385,3656,5084,3632,6601,2240,773,3705,6186,6718,10490,6206,4953,5333,3990,5296,7962,4700,3875,3379,4566,6968,10025,2445,3345,3024,6765,7875,4867,7351,8380,4679,3322,10035,9288,6748,1947,7616,7548,4123,8016,7489,3823,4872,1408,5880,9180,10609,8791,9910,2762,501,2636,7971,7303,3710,8084,1871,1192,7080,1610,8560,6046,3286,7608,8403,1405,1456,171,9912,5947,6118,3610,2547,1024,9199,538,75,2767,7471,9726,3927,2537,10667,4043,6391,3214,721,6063,655,8208,2262,8748,8869,8531,2027,3236,3406,442,1572,745,2271,8858,8243,9847,5287,8127,6546,9913,7275,644,7994,1440,10414,3423,7031,1962,4244,6559,5916,5549,1383,1046,7830,8063,2015,7986,3304,8612,6704,6361,7166,3462,4209,10171,1685,7002,6730,308,9110,577,10568,7015,7048,6200,7212,7204,3804,3527,4437,4759,4010,4020,10664,6710,10161,3553,10282,9968,3260,3077,6533,7424,2986,2071,441,5647,6703,2446,4080,8054,10182,5896,7928,3584,362,8050,2586,8843,10707,5492,2815,10658,7635,9592,8608,6908,7434,7085,4774,1327,835,4693,3188,6956,8886,2162,9389,2679,7123,4884,7419,6613,2599,2770,2512,9265,9323,6675,315,4508,994,1605,2256,6321,7675,1002,1016,5303,7584,939,9626,476,10211,7573,1394,2212,6189,5687,5309,2519,2186,9559,1936,10268,6558,1365,5791,778,8678,5601,2292,5025,1235,1149,7433,4062,2881,5871,4555,39,78,9166,3311,8126,6888,10408,2625,3960,4656,4561,6849,2338,4591,5531,9619,6036,10464,4378,5775,5278,8385,5514,814,6775,3691,1153,5631,1738,10799,4966,8611,566,9820,5957,100,5963,7789,2157,945,2454,9093,7849,4225,6444,1374,10587,975,7098,8058,9947,6619,2301,5426,7998,5117,10155,5206,4371,4802,3685,9491,592,4838,3861,7949,8643,10780,5685,1218,6105,8629,550,4773,3887,1085,9941,9690,6897,4089,7284,2403,3185,7042,3189,4886,8571,10372,4446,10204,6191,10190,5220,892,681,1956,4588,7534,16,9701,3355,7389,5463,8145,8026,10246,6004,8139,8300,9676,6087,10325,7770,1641,5543,8985,2344,3897,3463,4337,5949,121,9911,5948,5163,380,4338,5743,2588,7318,3309,1269,7850,10085,9008,7879,3726,8774,4465,1582,1084,7094,2400,1591,3299,3681,3079,4841,8797,4760,1087,9671,4295,9505,193,6289,8515,3661,4952,3713,1229,2917,5257,1283,10199,392,9175,5151,5186,5259,2742,5558,7953,4610,7127,6288,7319,4110,4571,6219,2481,6926,9553,6990,3912,5875,7281,10595,9108,1438,1721,2385,1677,1680,1319,9504,615,1423,2962,9699,3025,1598,4435,383,1376,1500,10577,3504,3614,2769,6227,9003,3580,9666,7605,565,10263,2243,4723,6161,3943,2985,5072,10303,214,662,9251,7059,1594,10716,8301,8575,4253,851,9329,221,4931,4304,5659,9531,4453,5603,5149,9034,8677,3769,1067,2273,8997,10147,9404,2376,2891,7600,3841,8925,665,2306,9250,5455,9480,867,6341,9121,3774,1526,6615,9599,8963,5503,7574,6773,6067,4849,7428,10274,7285,5767,2348,9835,10216,6878,5135,1952,4374,8735,5508,7856,9349,2843,1163,8433,10027,8160,1666,10473,2184,5213,3742,3312,4222,8097,7886,4280,9485,9868,1202,10177,10625,6589,6484,9399,7670,928,1799,4665,2656,6195,2070,3590,4204,7167,6582,9483,2334,708,4965,10394,5443,1132,6578,8197,5453,6137,2391,2305,4122,10702,10596,8626,9975,846,2613,7107,5840,6501,10762,8233,9635,5216,4642,981,1273,5860,7778,8031,3295,3978,3377,1692,9623,2022,5006,2933,2362,6505,7846,8760,4139,3926,8228,4993,955,7036,5748,5838,4895,2873,4320,6831,2316,10111,4289,821,6447,7323,1992,8371,4121,1486,2222,2729,9106,1726,9943,9438,6083,7096,9919,7295,7958,8467,8711,4069,2961,1279,3075,9506,8713,4595,5744,4442,1632,2425,9991,4850,2145,4173,7451,2060,6197,2208,5824,2153,8172,10574,2182,1108,7870,6607,2471,7827,375,350,1422,10352,3592,8700,4654,5258,1317,741,6720,6234,4016,8176,3364,6644,1640,6309,4506,1505,8432,5960,8540,8037,5684,3482,9664,6398,2776,6996,4042,29,4796,9606,4589,7588,5677,9996,223,10604,9268,9332,1000,5030,8857,4104,2253,6858,8937,803,7668,2554,3619,3806,9541,9951,4563,3124,8399,7965,357,518,8483,6068,7283,6756,4235,4742,4717,8592,2142,9685,8548,7402,1547,9149,5784,3989,3932,4074,2048,5281,9318,9283,926,7932,8745,970,7942,9395,2038,9659,4474,2826,1014,1256,7223,6836,5031,10434,6919,3313,4292,1196,3720,9317,7640,9160,4194,1494,4862,6707,6512,9825,4448,10377,6057,4671,8463,2263,5369,4103,3525,1964,4949,3722,628,2595,9388,2771,8001,6221,8959,794,8167,5818,6785,10198,2327,6074,7405,104,2796,8427,5164,3194,4163,2428,4530,9301,1853,3810,3369,3566,8871,3593,5162,10696,3950,9776,776,8593,793,4384,6886,8597,10697,6506,7867,7662,6932,3070,5849,1299,9313,7089,1188,7527,8274,7617,9763,1965,1382,1278,7165,4754,132,3934,5773,3618,1861,4879,4232,1608,7457,9306,8350,6811,331,5542,8990,10048,2922,2424,9307,7175,9873,2420,1541,4434,9879,5961,379,9443,5898,8122,5731,5295,10776,4342,5821,6351,9794,9510,3798,3885,4822,5020,8183,6732,8244,786,2465,6664,7517,580,1414,5712,9073,4036,3581,5520,1967,5666,7007,805,7398,1375,2287,3014,1514,7082,2948,4840,5182,7685,9902,3858,2726,5489,3321,1069,2160,7404,7495,774,3015,1566,2165,10288,7464,4720,4575,1314,4215,9784,6609,3315,9477,8473,4763,5845,4286,10726,1603,2744,5759,7388,7496,443,5962,2544,4507,3383,10748,977,3133,5066,7945,7020,7372,7908,9721,3396,3044,4705,5051,2867,4909,3792,10614,8740,9867,6736,5040,4029,1286,6666,8875,7317,3955,5706,5928,2204,2970,9492,1771,1577,4196,1796,6150,6332,8066,10210,10514,6531,1597,3550,1998,10585,10338,3467,3513,7648,7130,478,3766,1589,8418,7858,3941,252,1390,5599,5989,10670,4660,2121,8351,10824,1083,5326,3784,2480,4330,6742,1055,6714,9734,7618,6628,6739,3459,7735,7233,896,9831,1034,6971,9972,10607,6012,1828,1179,7485,8423,2000,10488,10393,8148,306,6307,5723,3736,8021,545,1800,8561,4709,4341,5076,6557,49,5452,3152,9361,7324,5291,10230,1166,5673,10554,9590,2402,6817,6282,4560,6787,2619,3082,7172,378,9055,3528,4948,2721,3878,6323,465,1459,4690,5881,8599,6943,3078,4573,10188,989,6954,7512,2440,10679,1955,5738,4278,6300,4090,418,2325,797,7902,5562,7500,5812,6426,9675,10422,5734,5466,9828,4321,1741,10793,8344,5110,9826,5798,3173,8495,934,7692,9878,6594,10145,10478,359,1168,2900,7019,5060,10029,9714,10294,2461,6362,5496,4818,5045,8945,1395,467,5935,7392,8811,4144,138,3996,9673,22,6571,9148,3490,5469,8279,4128,10552,5965,3222,8387,4735,5675,587,4483,732,9006,8313,9916,10503,8672,8904,8717,3879,804,6395,6853,8793,5645,9263,4326,7807,703,9920,9921,621,1152,2080,9094,1654,9085,5434,228,6097,9710,6478,3745,292,1884,6743,7614,227,283,342,8367,5329,335,9729,8649,3535,7069,9212,8129,631,8568,3410,7051,3356,7090,9978,3476,6530,5512,558,1920,5688,874,10397,9335,3501,6682,2456,7749,2711,4236,3796,1144,2238,6882,9740,1559,9354,2183,6050,578,7734,7779,908,10302,3754,8541,2030,2217,2677,7045,8230,4045,7459,2594,245,1546,7210,2489,1050,10245,2211,6196,7005,3105,2394,8442,2643,1681,683,10344,3821,1581,1329,3027,10653,4438,2855,5299,9489,4828,6646,7762,4189,8143,2232,3949,1858,6854,4653,543,7173,7312,2808,8149,9147,5657,10143,10744,9355,5572,5005,6795,2371,9706,5397,114,5550,5722,4002,2902,9611,4624,1249,5994,3057,7931,8587,128,3617,3834,3936,2579,3505,5039,2734,1604,10004,9028,6222,134,2754,9643,6359,5667,5834,7338,9715,6603,8549,7546,8064,7406,9539,9079,3267,8523,5474,8278,749,10173,3890,8826,1232,2152,3609,6052,4636,547,6111,4470,7391,9771,3495,10437,4357,5642,7073,3199,5636,7976,3039,7306,8263,2432,6931,6939,1810,6231,6064,2910,5622,210,3667,7592,4179,6544,8919,553,6261,6344,7743,5103,8769,6550,10433,3442,6715,5078,2994,156,10005,8083,459,10499,7719,6126,4902,882,346,496,9696,7967,9186,6019,8594,6353,5210,1989,4180,3440,9062,9026,9861,1765,2083,7647,4668,5988,10385,236,8142,9513,3647,9762,6634,707,4780,4349,9788,6037,1973,7316,3644,7269,1293,4331,5920,2824,784,8808,7636,10501,4346,9604,3583,89,815,5118,8724,8820,622,3558,5367,3268,10496,10251,10612,9358,4492,3439,2528,4578,4998,9901,4834,6670,2381,452,728,2414,2144,205,9871,10693,2531,2693,10512,4273,9691,6586,8465,9769,9712,8864,2535,2511,4098,7560,5136,2297,8993,1174,127,8876,2274,6451,2258,7504,8488,10197,2124,3458,7321,530,5044,10362,5873,6408,9987,1128,3718,2187,8072,2935,1162,8969,4865,10233,5870,9814,2766,11,3017,9046,269,10719,2081,4459,55,10097,1148,7631,699,1631,9964,233,5950,10034,630,4183,507,146,6343,636,5796,2173,8298,6538,4678,2936,1875,5183,9375,5050,9418,4161,10593,1823,4761,2322,4641,6692,1037,10807,2458,2715,5655,594,8227,6911,6800,92,2723,6060,9962,8447,10758,364,5943,4409,1330,9276,2608,3262,5754,771,10332,2577,6814,12,1452,2768,6277,8409,4685,701,5324,5690,5340,4536,8970,1380,5185,3324,3789,8640,2084,3096,3642,6870,9730,3877,3730,2074,9798,1919,3884,6570,2954,10828,4606,6346,456,1222,9125,6957,9985,8392,6713,8226,8090,2426,9478,2977,3055,7816,2895,1435,5144,9285,7337,7948,4494,2337,2021,3233,6847,7767,5141,4072,4942,4519,8219,3957,7035,5737,10181,1857,573,6453,8070,1487,8245,3401,74,2412,10024,1767,2340,3582,6259,4382,4391,9071,8524,3277,7663,4461,6690,8738,4640,1057,2411,1766,3958,10475,2331,6209,5120,4241,3620,7851,8747,8014,4615,5327,6661,5724,7213,4174,3042,3781,8256,9050,6121,1687,8723,8928,8216,4012,1609,1025,10815,1627,2712,3460,1819,528,8275,3103,6324,6258,1714,3494,339,2746,4748,1558,5927,4130,255,8202,5952,4136,3744,8776,1493,10404,4959,4666,10569,4604,1258,7505,9352,9742,2210,9713,5191,1157,3921,4608,3631,3994,6122,9702,783,1638,7712,4431,9499,10236,7925,3175,3839,7765,3228,967,10465,8682,9189,5901,6216,8835,6577,10105,6803,4545,10101,5035,8664,3247,7289,6285,9579,2335,4441,5540,5314,10215,7815,1561,1515,8885,5594,6631,9877,6205,1817,4745,7235,8356,1570,5325,2137,7589,4823,5413,870,9038,8595,6638,1396,8242,6569,9562,5783,6639,5695,7180,1728,9872,10476,7661,1224,6893,1241,9378,3256,4203,9933,183,1530,7277,6528,5438,7181,1982,6972,2295,1503,334,4532,2293,6171,4842,4728,8492,124,7444,3752,5825,31,8000,2163,10530,1366,2548,8081,10771,63,5827,9191,10218,3258,1045,1656,3215,3630,2150,3276,4800,9414,5226,2673,7677,4255,5266,2072,6953,5004,6562,5365,1818,3162,6766,4452,10524,6985,3637,9850,10000,9672,3876,9647,1575,3699,1905,7629,8972,9744,2924,9452,10234,4737,9010,4288,1843,897,2435,7348,9305,978,2138,10591,3862,10540,4897,8402,8932,2443,2279,438,5038,3574,6535,7273,541,9598,8619,10459,3306,9043,5605,595,1270,309,9280,3080,8411,7825,6006,7801,7664,7599,9458,2506,7764,798,1520,1616,6378,3245,7877,2953,8480,819,5608,9181,3729,61,1072,9009,7482,6788,7703,136,7700,254,9615,2540,1717,3947,7794,6947,836,5519,2444,511,3548,1804,7296,5388,8948,3707,1683,4664,4175,6204,10261,7357,2251,7104,3675,9253,6265,9613,8071,880,8365,1019,10270,4469,2357,9646,6892,3507,9792,7985,8527,8701,5552,8180,7799,685,5787,10343,3628,2637,4607,6202,7831,1747,8342,8891,8110,4050,366,5387,6974,3666,6611,761,2852,1372,2423,2697,4355,4272,3122,1161,8618,3711,5037,2076,6160,5662,4277,2651,5470,7848,3954,10235,8938,8288,9439,7828,1147,9241,931,2983,2602,3639,10306,4791,7723,217,1464,4162,5951,9082,8177,1064,391,4460,4955,2377,8880,6173,10304,5903,4972,9018,8977,3457,6329,9922,10740,3241,3917,1061,1885,8708,1200,422,10127,10300,4057,9790,616,10594,876,9033,8848,5862,1729,3265,10167,2659,9386,160,7519,2905,7659,2866,7245,5302,10353,8856,7921,10474,4053,5354,2011,9422,4729,5751,3319,7514,9944,9950,4440,2560,330,6772,9725,8422,4070,8694,6370,82,1377,983,5023,4812,923,2223,9533,2839,6942,1434,8796,6440,7757,4383,2743,10695,3570,3836,2997,1901,10313,8818,282,7540,7951,640,4658,9066,6712,3190,6649,18,7251,8497,10497,3530,8859,7445,293,3135,3285,5228,7818,9159,1642,212,5077,10059,10224,9474,38,3687,8851,5630,4078,5205,2674,5273,9822,9101,5376,8046,10206,9099,10624,8498,3325,9800,2610,10036,5867,958,107,8096,3239,2779,7466,7553,257,5710,4982,7049,3335,5857,5359,427,5097,1659,9772,1903,7040,7989,3671,3800,4015,2571,9341,184,9530,6705,7872,713,10630,2143,5392,4797,9516,7023,8169,1432,998,10446,2128,1143,2641,5346,6883,2829,9336,531,2566,2192,7022,7377,3100,4287,2722,348,6510,2462,8539,4168,8882,8284,4946,8475,325,7060,5026,8396,242,6696,9738,8290,3456,974,3611,3034,3400,9070,3913,9058,5565,9612,491,9628,7903,8809,5909,4190,133,2459,10390,9461,6598,3296,1484,6651,1189,1136,10269,8716,2604,9105,2737,9229,9906,4458,4593,10509,2300,10419,2978,3444,5399,7138,4718,8884,878,9755,8727,6201,913,5246,8460,3046,10378,5541,1509,8375,10416,2966,5271,5412,6459,7443,423,4115,8895,5842,8995,1870,10678,7194,4184,5171,5134,1386,10357,10447,1588,8201,1476,3085,9936,4819,6112,6423,4114,5793,8659,10375,10392,10226,6228,10520,9994,6659,4213,8586,10157,5178,968,6417,7026,1891,5152,9371,7758,9926,1713,3758,1986,7000,10425,5742,5202,6059,301,8124,10285,9862,5195,6313,1125,9145,8861,1904,3437,966,6740,6131,4547,748,6188,1778,3108,2707,4740,1281,5530,2220,3117,267,2298,9813,4067,410,5319,6671,1650,2031,8162,4250,3449,2343,4319,1120,4328,8823,1497,8676,4534,4493,7543,1637,10786,6778,9204,1922,9654,7216,10663,9984,9722,10259,2842,5479,903,8705,10644,340,7186,10011,8075,10402,3227,3486,8059,1508,1928,2913,5628,3948,9127,834,2148,6044,10047,4301,5883,8459,10071,1777,347,9119,10550,7146,8893,6026,6382,987,1958,9674,1439,4336,617,10110,8751,3865,1739,9637,7028,6264,544,9315,7881,1646,9678,3604,3429,5902,1131,3146,425,7593,4580,3394,9917,10669,51,4054,5421,2972,4784,4216,10237,9244,9117,263,2703,4153,829,2671,9144,1300,3538,2713,969,2940,5837,123,4246,2242,3397,3559,9273,2862,8899,10122,4756,10441,1345,2552,8012,2803,9783,2374,113,10808,7117,2887,5370,9423,7118,10405,2042,5214,7520,6305,5230,5702,5893,5160,8239,8443,1963,3056,8615,1988,5048,8374,10467,3263,1351,8824,4051,3390,9007,7844,1417,3179,4370,7476,5408,9069,2486,3857,9829,3149,5458,6826,3702,7502,1931,7305,2646,3186,10418,4308,9668,4614,5683,2725,5028,2075,10096,5579,7891,1740,3443,4395,5792,1806,2934,1358,446,8199,2600,7575,4686,8603,10645,2365,6077,27,8416,8389,3532,10452,9112,6458,779,3706,5602,3491,1214,9766,10558,7160,985,9311,4747,7365,9393,8192,10131,352,3555,8718,2318,6298,9115,6414,5059,4762,1043,10292,3719,6000,9293,7993,5809,9174,6588,7874,1285,7895,4497,490,3603,4468,10023,4990,8098,8953,7955,7585,9538,10717,4198,7379,7347,6404,2452,10791,9451,9434,1900,6207,8195,444,4707,7438,3945,7309,9508,3636,9931,751,1513,7330,4526,7302,6181,4836,2969,8683,2434,10420,1255,7561,5451,1354,3156,8512,6041,7410,2695,1052,5733,326,8002,2911,6192,1186,4794,3102,2780,10124,6902,7751,1860,7119,9475,1080,2501,6738,657,3040,1404,6243,4638,7596,8286,10001,1995,4885,5016,700,1096,9373,10380,6442,8010,8622,5590,5555,4420,2265,10517,360,4766,3698,6413,5355,2098,2682,6007,7884,7455,6407,5992,9866,2449,9770,3026,8469,6652,3474,2728,6460,8092,4744,6701,483,4307,6416,1197,6172,9248,3679,9624,3141,10291,4778,7741,6399,10137,5490,1537,2502,4058,5728,6731,7569,9370,4584,9650,8771,9296,4853,6281,3213,3991,10214,4402,3308,5861,8801,2541,2806,8246,7693,7163,4479,10183,4499,2870,2818,433,5280,7423,3523,277,9234,3359,6266,2652,8400,3509,6474,6899,3627,4408,4857,2763,2692,951,5878,93,7602,3866,6123,9827,4827,4296,6166,9155,1573,1187,7248,5668,5248,10495,6987,1832,7795,3031,6226,10462,5930,4682,2556,7669,10829,3232,4195,1466,658,2736,1005,7371,6604,5653,9709,6620,203,9923,9596,9134,10184,1097,2006,841,5306,6568,1774,953,5445,6271,10536,6632,5237,3317,7016,1838,1599,9473,7491,9956,6082,6567,10498,6069,1316,9300,7654,1245,8348,200,3338,10575,8521,8420,4422,6023,4084,2433,629,8627,2765,3348,8868,9228,6100,8931,6058,7149,7228,7187,8836,8870,9519,1583,6952,2907,1879,2174,3280,5649,2941,9958,5417,2835,1228,10093,8120,6580,1267,10753,8830,889,1533,8040,451,9047,7555,6662,3248,2254,1102,5454,2792,760,555,5644,5852,7394,7299,7034,3995,10424,8434,6449,3385,9049,176,8115,7940,2120,7349,5390,1849,9385,1809,7557,4293,5338,2195,9782,1060,3418,2741,8426,6276,10811,6551,9347,9967,417,2752,653,5143,10764,10087,9520,7863,7890,1212,363,8665,8428,10298,4025,9887,6616,9693,8458,7386,6976,8752,3838,10243,2149,7229,8451,208,8888,6658,6151,6997,9185,4898,1254,3235,7523,9065,1205,3187,3872,3316,3579,4480,6989,9622,3600,7742,904,3750,7977,894,1671,753,4343,8574,2696,575,9888,1479,9031,2140,3571,3119,738,1618,36,4165,8790,7056,775,9417,5933,5664,2091,7923,6491,289,389,4738,3275,9130,6657,7650,9976,9493,7264,4439,6025,5092,6529,10275,5286,5439,6153,8944,4549,2311,4848,175,8689,7442,1737,10334,9384,8394,10599,7656,2139,1056,9173,3210,7453,8334,4218,7781,2663,458,4826,1253,1287,9135,9542,2213,5029,5983,8326,9733,8956,10621,4450,2018,10264,4032,6941,4832,9224,3782,7416,9039,5279,9177,8940,3107,8604,8065,5872,6263,4936,2231,5699,1658,9528,4516,6143,5057,2024,1723,2787,8421,6871,9097,6717,10283,8554,311,6930,8742,3598,1695,8025,5125,3072,2863,10772,4239,3795,6784,4808,1507,6842,8992,2384,8867,8410,10565,3264,5172,1066,6964,5639,1071,7503,3037,7185,3993,791,10129,9421,4291,6224,2040,1121,7182,2117,660,7773,2992,9297,5098,5635,260,41,2308,4477,6829,10003,2353,8547,10633,3779,8471,2221,7454,3331,3404,2418,6306,3098,9490,2906,10202,8331,8448,2688,1419,7833,2491,2134,10430,5537,10249,3708,2915,8687,4746,1181,1098,2275,5656,5176,1454,4557,9649,5855,3662,9686,226,8923,5848,2245,691,506,7852,625,3859,3822,10065,4907,4896,7095,2309,10323,10315,4815,8238,2330,6372,8754,9914,5816,7672,10333,9743,1008,720,298,9165,5430,10336,645,7004,10040,2404,6075,1185,2172,1079,1150,9024,1856,7211,9810,7230,231,2135,2166,3577,5488,7158,3953,4500,2250,3297,479,2482,1001,7728,9751,9484,1712,4787,2415,4339,9988,10373,7494,1443,1359,7359,3734,1350,6388,7452,9953,8101,10682,6113,1070,1429,4283,2304,2180,9416,5990,9728,7168,7120,6689,4915,7910,3290,4224,9470,9684,8758,601,6944,8564,7963,3963,8114,5865,8087,1284,8807,10305,6479,5221,8359,8011,4943,6494,2542,1715,5559,9575,4093,2621,8073,6280,5529,3101,408,7564,10648,3269,4476,795,3975,6028,6275,88,7927,5484,3340,7979,10590,2005,1104,7643,1463,3479,6973,6502,5701,1236,2585,1389,9527,2631,3065,10144,9764,7611,3293,4186,6643,8222,8338,1867,539,10676,2819,8370,1075,3358,698,5361,6443,4799,4649,9747,4281,6751,3089,10324,7046,9915,5245,3147,1310,4831,6492,8695,2823,9761,10095,3874,3686,9107,10116,7139,2090,2705,8511,9137,10225,2224,4509,9583,230,10470,3415,10743,10752,799,1106,112,3067,7889,8280,5158,8646,4846,1544,3211,5416,6948,924,8912,4525,10559,3143,10232,5254,8476,9368,409,2319,1203,2237,397,947,810,2946,10809,10399,1600,1239,295,3373,586,7134,9411,693,7262,2639,1534,9472,320,2942,155,4548,7427,1353,9585,3022,4550,2943,7570,5654,3478,279,2869,2014,619,10180,9839,5377,235,6848,2667,6079,9567,5768,170,5853,1812,845,3904,3674,8936,4947,10142,2975,4027,5709,7448,8156,6759,1118,5055,4954,2413,7053,5971,8135,8477,9982,2033,8789,7256,2624,9432,5761,3522,2564,817,3249,1649,1551,4937,2916,877,1370,9027,4063,8234,5177,8841,9543,8118,8089,8894,1970,499,8781,2611,4210,3940,5797,10007,864,4852,9326,9209,9688,9083,9522,7704,7686,9014,3845,5697,8323,4951,5676,6333,2661,10194,2683,3353,712,5437,6233,5241,4152,7501,3164,8707,9042,8264,3126,2783,6470,4362,2926,5973,4521,455,10725,8186,1942,9237,9369,6683,5672,637,7362,3391,9898,7409,6317,6437,2810,2007,8889,8125,6599,4596,7632,9040,5918,6556,8783,182,8844,1636,2587,7562,7702,624,10471,7899,740,8235,2375,9444,2701,2278,9340,8712,6951,1426,626,1667,4734,9495,8631,246,4980,4264,1481,5485,4380,1260,9918,8509,5929,4041,7750,8262,1579,3557,10822,763,3765,2525,5476,2517,6016,3974,10428,3254,1009,2700,9348,3271,2999,47,2760,9208,7086,8261,2106,8251,4710,1852,7708,7487,1501,5013,7346,6355,8829,5065,6593,9258,3413,8043,5188,9078,3068,4711,148,5953,6678,1851,8491,2704,3587,6776,3801,7613,3863,7343,5926,8930,3114,4985,3589,5890,6770,2349,314,10328,4992,5506,4873,7072,2745,439,403,9419,5493,3242,2001,1339,457,1204,9521,5330,9287,6465,5472,4223,5301,4504,2507,447,4803,6647,5513,1294,2782,1492,6441,8954,7222,8061,8637,1709,10640,2457,4006,5977,5146,1444,4270,10383,9372,2094,10519,8033,6356,7329,957,582,8390,7246,6466,2957,5138,2854,4971,6312,596,7995,2777,291,6471,7414,4004,4322,1961,474,1381,6422,4466,2407,3261,4361,2405,9245,1427,4567,1639,1590,3004,6540,5263,3695,4340,3794,5528,10038,6552,10458,6485,7909,3202,1733,6605,512,9092,130,2543,668,787,7368,7987,1265,769,5011,4613,7619,8157,2774,9304,9405,6373,3967,3433,7008,7093,5844,5799,6476,2358,5331,873,4801,8355,7250,1448,46,440,9957,1158,2089,259,7583,1491,2112,197,3226,3568,7279,7144,1292,10479,7857,1979,7980,7050,4418,674,3547,3416,1415,6415,3514,1704,2156,10016,8979,9081,10555,1827,905,8440,4251,7355,3728,917,10022,299,20,3069,1532,4592,9362,8528,6496,4983,5003,3937,5771,2755,1880,10133,5465,9158,9689,10543,10712,5985,651,5315,5249,482,3176,98,9834,3367,2196,3907,3093,8381,8722,1611,9894,8679,1428,6287,4496,5534,5987,4863,4672,10516,5721,1564,9938,1247,6179,5868,6432,5298,7111,5781,6881,1347,7754,6820,781,5533,5284,10067,2111,3446,3502,8866,5915,4359,996,4770,7785,1595,3035,3969,1834,3894,10060,8237,1794,285,2198,3898,2191,585,7384,3768,5892,7861,3172,9201,3973,8360,9342,6750,5498,9320,3139,9195,271,8978,2498,3090,9029,8057,5487,7859,5886,2422,6572,4524,3341,10417,3420,2125,9679,8478,10296,4837,2146,10201,4572,7845,432,4056,6428,6786,5383,8941,5658,3110,3684,6070,7510,8914,10042,859,4068,2249,4663,8335,3453,2952,7639,7327,5126,3180,1297,6085,6914,7645,3692,5197,4462,3670,8821,2088,8579,7490,7747,1909,6872,2681,8113,10745,6445,7205,6128,1745,10449,623,1826,2520,2860,4903,8795,2126,7311,43,10400,832,4454,9274,6159,6680,9631,6808,5611,1527,9011,5539,3818,5795,5294,3417,1220,5251,10641,705,731,4052,659,6164,3599,8211,7918,414,5863,6203,4906,3646,3087,2784,9252,5435,9360,10077,3962,10597,568,7493,2844,1498,6635,2020,9896,2965,3007,9816,5198,8816,2614,10724,7426,5268,8446,5403,7001,1874,9220,8332,1949,10750,554,3431,850,8986,8616,5396,10179,1773,4905,6851,4712,4871,8715,8602,6473,33,7331,1430,10299,2572,5750,5351,746,10635,849,4964,7698,1628,481,9939,7701,9494,2303,1528,642,2016,8927,6467,8499,10319,6885,8311,3714,954,8404,6367,9487,962,5111,4317,5150,4777,7109,4603,2853,3520,7240,7400,1722,757,8128,9243,5801,8462,1048,9218,3498,4386,3612,5024,6841,10271,7171,844,6606,8417,9909,3819,3645,2342,2282,6771,1176,2203,4765,1195,10359,7129,6350,9997,5643,4312,2277,3964,3346,5897,7970,7911,8613,4692,5080,3725,9380,8921,4376,10773,8271,8482,5509,2205,7855,8340,604,8190,9469,8159,3095,1868,10544,126,4528,8170,5189,7468,4360,1730,2406,8633,7463,5887,6108,716,7373,6962,9580,4262,4650,5357,1177,2039,2110,5109,3081,9569,4488,10687,6520,6045,639,2447,5169,840,10661,4655,2914,4314,2617,10818,7954,1139,10816,4159,4963,8800,2888,213,6936,4444,1895,4037,272,7413,4249,7339,7366,3808,7197,1703,278,1274,3500,6039,6237,3786,8778,6912,2460,7565,6517,6920,5419,9429,9625,2105,7278,44,5835,1731,2470,3716,5105,3086,10588,4129,144,2492,4881,1035,10813,6693,6804,2893,3197,9021,5596,3452,470,3320,1210,10363,5976,14,10026,1127,7112,10310,5566,5663,4313,6308,570,8905,9464,696,9907,9143,9739,534,5567,9737,8697,8009,6595,7748,6175,2811,1795,323,5705,9266,8804,3777,2857,4429,848,10309,4432,6124,9294,9758,4888,5462,10734,60,6955,7033,10162,10782,5704,10677,6489,239,1688,2845,7642,5557,8276,10148,4919,715,1613,7832,7782,4369,4221,10098,7332,2563,1821,7380,9670,7590,1672,8134,4047,8414,1447,4883,5300,10207,333,7805,5046,10355,10106,10787,8425,118,2057,5194,1013,785,3756,8500,1705,7621,1644,7926,5106,10337,5047,6368,8076,10556,4887,3167,1092,2286,6560,9254,10521,6051,6958,6256,9338,4372,2598,1596,10583,2798,1250,1090,3250,1193,4922,7367,6904,1032,2633,3984,3678,10579,9630,7571,4424,5891,9045,10533,3944,8151,6326,349,6091,10032,7920,5682,1625,6700,5362,10627,390,2960,10442,9501,2740,332,6412,5368,7522,7456,4776,5568,10176,10603,2834,9456,494,1184,7598,10802,6199,6212,4285,8470,5081,4933,8900,1926,6071,4553,3615,4333,10240,8739,3508,2738,6320,606,4676,6448,7062,3923,9653,9846,6235,4436,7653,6591,7352,5142,2582,141,7984,7937,1700,5360,9973,8303,2216,3392,5680,4674,898,7812,5440,10154,5908,6940,1495,6403,1960,143,10733,1115,385,2003,3658,6218,9387,3104,3224,9821,6590,6995,1119,7558,1749,6419,1201,6475,7268,2028,4645,159,4539,650,8755,914,5101,6625,10041,4939,5386,4379,10272,10567,6251,4413,1836,8361,2724,10090,9095,5561,9683,2123,5828,9016,7290,3041,6152,2523,6430,6136,10244,8247,1743,2496,1756,7074,9655,2609,168,2012,1883,7018,4120,8666,4874,6425,9334,1431,2247,6331,2119,10692,4109,5227,7218,7957,9785,789,7793,9255,3565,6148,4764,1357,3900,10483,5553,310,6566,9803,5087,1038,9601,5212,7798,97,5341,8719,2052,2879,9955,6539,7136,6190,8865,10729,3813,4393,4913,971,3881,296,8207,1344,4147,5982,8505,3540,692,571,3436,10049,4969,7746,2500,7224,8877,607,2764,9718,9457,9447,7873,7941,4775,4586,10698,3113,1684,2739,7397,8815,7939,80,1455,68,1790,324,1324,166,7055,508,4188,10439,10456,2733,3154,7294,8691,8828,5121,4478,8150,10672,4373,10763,6245,10348,9857,509,3045,1893,9523,393,6821,2257,10491,4882,1798,6020,3826,6109,3516,4486,1802,7354,7041,8285,9526,1190,2813,2082,1020,5289,10287,5967,1788,2181,7526,1913,6629,10108,161,5813,3201,10756,777,2616,9787,3171,9236,5769,1054,9576,10386,3381,7594,4648,7314,9435,1755,8289,7709,73,10073,5670,706,1298,6797,6390,7549,9558,4178,7597,10720,10254,4629,10212,6749,4997,3038,5219,9808,5984,8232,9546,9111,2455,6021,5912,2427,1924,2227,9383,7068,8845,5239,6273,1134,4576,593,9745,2640,3398,2363,10832,10258,6618,9041,10560,7100,4894,10312,3030,10057,1445,8018,1356,4009,424,7326,1568,9899,52,7740,6498,4092,2987,9882,8623,1830,4741,7803,6132,4605,10728,766,8252,6098,9594,2831,1862,5556,3050,9343,6894,2383,8358,2515,9023,8652,5581,229,2207,4205,2332,7383,1585,338,1907,3370,6464,1315,2666,3790,5170,9206,2333,1944,1768,2438,4695,6741,4242,2467,1574,6815,5502,2073,3076,9242,9824,6994,225,9602,10432,4714,9719,1233,1490,7529,2209,8915,7280,8518,9056,6763,7601,871,9777,5468,7267,6130,5156,4708,2017,3178,4626,4727,2899,2259,4527,268,1126,2690,8846,4768,990,7470,464,8962,9588,5052,498,5422,3374,222,8315,9194,5648,3740,5358,9450,5404,800,7633,7513,1911,10798,4108,6668,9880,6896,10382,972,2638,5292,2848,8203,1662,10651,7066,9075,1244,7537,5305,8307,2416,8736,6072,5507,2269,10314,8042,5527,4970,9893,4059,6257,3546,2718,1288,8132,10694,1469,8314,7328,7065,7972,10070,2689,9001,5203,7595,4017,7716,5277,4579,8558,3992,1707,5641,1676,4430,1754,739,4781,8393,10484,7806,1993,3657,1865,10642,8287,3204,8680,1675,6066,1601,1734,5936,9282,5478,7930,4559,991,10018,690,9154,5255,2655,7,190,7563,3717,7868,4358,7924,1301,4491,1780,1110,4275,2925,10548,3212,5661,1607,648,3977,377,5569,8069,6249,635,7992,9496,1881,831,9851,6923,2872,2229,5544,4541,8383,5285,3344,4031,8003,8292,3980,2591,6389,7649,4318,4167,4367,5554,9509,10507,7912,3421,7077,158,10779,1764,1975,3545,2350,2601,9141,3829,488,2532,7550,1593,3403,515,1078,1026,8352,28,5244,1436,3651,10513,7655,4609,7718,722,2066,8653,5428,2630,10421,4348,3225,2483,2390,407,4347,6548,3357,9017,3959,1311,4687,1864,3815,7860,5808,833,7853,5293,2032,4185,8576,8529,7310,5411,5448,9196,6311,1403,5260,3192,3485,8481,7769,5729,8249,9809,10407,5779,9162,4861,3747,5007,8565,150,1930,59,9012,6420,8601,1323,3382,10547,9104,2516,1866,7239,5308,6733,6967,4529,305,1914,5713,10320,4772,4140,2749,3591,5250,899,7081,9020,6409,8965,10656,9269,1896,3,3240,885,837,5062,3654,8658,7200,9568,4820,4917,3032,10576,3305,7492,394,5822,1262,4455,1266,4332,10128,8704,1943,901,4994,2291,852,5385,4703,7488,5551,6142,8051,2323,3484,900,7787,6383,5099,7499,4489,3561,3005,3672,9053,2678,2620,5480,6523,3986,8496,8673,8036,10492,3123,4932,6279,9394,2583,8766,3820,8782,3223,3939,6554,2596,6975,6208,8519,5429,5598,4556,3773,9885,5418,5269,6845,5945,6978,671,7361,4878,857,6065,3669,5153,6339,9781,7506,5582,10674,7783,1164,6454,6869,744,5571,1337,178,3159,4040,4487,10592,3464,4364,9407,3896,2419,351,5379,8019,10792,4217,1280,7422,10222,3499,8117,7432,435,5339,7143,9876,6029,6260,3573,7943,2998,4757,9359,86,4594,6384,5384,2037,806,1813,10493,10485,9639,9927,788,4945,6055,3688,8189,5401,8485,3053,6984,652,9087,1697,4410,7577,10647,7975,1225,10797,4670,4305,1554,9109,3402,9179,1367,1807,5406,7905,9802,9532,5049,1890,3281,8324,820,4227,10504,8964,6579,3009,7784,6134,516,6010,2584,574,3764,6900,6852,2336,6366,2108,10331,4628,1923,2441,5175,5310,2361,1099,7341,1540,7715,4245,430,9210,6867,7021,866,5941,3220,2820,10706,211,6392,2533,589,5389,9848,7241,1073,6699,4694,2436,9187,3328,3174,10193,1477,8517,2758,641,6327,1264,9178,3902,376,10241,517,3294,9789,3966,4568,6624,1873,7689,10014,8621,9727,679,1872,4229,1524,10454,7777,5629,2130,1552,907,1694,4086,9061,6322,9874,294,3629,3738,780,7258,3008,180,4316,2968,7545,5166,7660,7809,9863,4788,248,2393,3585,3472,6095,4522,3928,3238,4498,7439,4083,10099,8137,8053,4558,4928,9656,3062,302,4535,8590,8214,8765,5610,317,5917,4327,8714,8814,3653,2882], "time_ms": 0.123783, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 613, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 0.769907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 428, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "generated_samples_per_query", "value": 10833, "time_ms": 0.769907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 429, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "generated_query_duration", "value": 485176169, "time_ms": 0.769907, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 430, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "generic_message", "value": "Ending naturally: Minimum query count and test duration met.", "time_ms": 5499143.693067, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "issue_query_controller.cc", "line_no": 482, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 5499143.704779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 898, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 5499143.704779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 900, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 5499143.704779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 902, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 5499143.704779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 904, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 5499143.704779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 906, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 5499143.704779, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 908, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "power_begin", "value": "10-21-2024 23:49:24.519", "time_ms": 5499143.704870, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 564, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "power_end", "value": "10-22-2024 01:21:03.658", "time_ms": 5499143.704870, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 566, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "result_validity", "value": "VALID", "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 655, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "result_min_duration_met", "value": true, "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 660, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "result_min_queries_met", "value": true, "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 661, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "early_stopping_met", "value": true, "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 662, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "early_stopping_result", "value": "", "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 682, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "result_query_count", "value": 1, "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 692, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "result_samples_per_second", "value": 1.96994, "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 748, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "result_min_latency_ns", "value": 537701299, "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 754, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "result_max_latency_ns", "value": 5499139384710, "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 755, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "result_mean_latency_ns", "value": 2758490972516, "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 756, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "result_50.00_percentile_latency_ns", "value": 2766519343874, "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "result_90.00_percentile_latency_ns", "value": 4952943609397, "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "result_95.00_percentile_latency_ns", "value": 5229045346642, "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "result_97.00_percentile_latency_ns", "value": 5336432580843, "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "result_99.00_percentile_latency_ns", "value": 5444675049450, "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 56281, "tid": 56281}} +:::MLLOG {"key": "result_99.90_percentile_latency_ns", "value": 5494101834853, "time_ms": 5499143.801882, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 758, "pid": 56281, "tid": 56281}} diff --git a/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_summary.txt b/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_summary.txt new file mode 100644 index 0000000..2282233 --- /dev/null +++ b/open/scc132/results/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config/bert-99/offline/performance/run_1/mlperf_log_summary.txt @@ -0,0 +1,51 @@ +================================================ +MLPerf Results Summary +================================================ +SUT name : PySUT +Scenario : Offline +Mode : PerformanceOnly +Samples per second: 1.96994 +Result is : VALID + Min duration satisfied : Yes + Min queries satisfied : Yes + Early stopping satisfied: Yes + +================================================ +Additional Stats +================================================ +Min latency (ns) : 537701299 +Max latency (ns) : 5499139384710 +Mean latency (ns) : 2758490972516 +50.00 percentile latency (ns) : 2766519343874 +90.00 percentile latency (ns) : 4952943609397 +95.00 percentile latency (ns) : 5229045346642 +97.00 percentile latency (ns) : 5336432580843 +99.00 percentile latency (ns) : 5444675049450 +99.90 percentile latency (ns) : 5494101834853 + +================================================ +Test Parameters Used +================================================ +samples_per_query : 10833 +target_qps : 2.06111 +target_latency (ns): 0 +max_async_queries : 1 +min_duration (ms): 600000 +max_duration (ms): 0 +min_query_count : 1 +max_query_count : 0 +qsl_rng_seed : 3066443479025735752 +sample_index_rng_seed : 10688027786191513374 +schedule_rng_seed : 14962580496156340209 +accuracy_log_rng_seed : 0 +accuracy_log_probability : 0 +accuracy_log_sampling_target : 0 +print_timestamps : 0 +performance_issue_unique : 0 +performance_issue_same : 0 +performance_issue_same_index : 0 +performance_sample_count : 10833 + +No warnings encountered during test. + +No errors encountered during test. diff --git a/open/scc132/systems/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config.json b/open/scc132/systems/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config.json new file mode 100644 index 0000000..e74cba9 --- /dev/null +++ b/open/scc132/systems/scc132_gpu0.novalocal-reference-cpu-pytorch_v2.5.0-default_config.json @@ -0,0 +1,37 @@ +{ + "accelerator_frequency": "", + "accelerator_host_interconnect": "N/A", + "accelerator_interconnect": "N/A", + "accelerator_interconnect_topology": "", + "accelerator_memory_capacity": "N/A", + "accelerator_memory_configuration": "N/A", + "accelerator_model_name": "N/A", + "accelerator_on-chip_memories": "", + "accelerators_per_node": "0", + "cooling": "air", + "division": "open", + "framework": "pytorch v2.5.0", + "host_memory_capacity": "60G", + "host_memory_configuration": "undefined", + "host_network_card_count": "1", + "host_networking": "Gig Ethernet", + "host_networking_topology": "N/A", + "host_processor_caches": "L1d cache: 512 KiB (16 instances), L1i cache: 512 KiB (16 instances), L2 cache: 8 MiB (16 instances), L3 cache: 512 MiB (16 instances)", + "host_processor_core_count": "1", + "host_processor_frequency": "undefined", + "host_processor_interconnect": "", + "host_processor_model_name": "AMD EPYC-Milan Processor", + "host_processors_per_node": "16", + "host_storage_capacity": "107G", + "host_storage_type": "SSD", + "hw_notes": "", + "number_of_nodes": "1", + "operating_system": "Rocky 9.4 (linux-6.1.110-1.el9.elrepo.x86_64-glibc2.34)", + "other_software_stack": "Python: 3.11.7, LLVM-15.0.6", + "status": "available", + "submitter": "scc132", + "sw_notes": "Automated by MLCommons CM v3.2.6. ", + "system_name": "scc132_gpu0.novalocal", + "system_type": "edge", + "system_type_detail": "edge server" +} \ No newline at end of file