Merge branch 'mlcommons:master' into fix-import

cm-mlops/script/app-mlperf-inference/_cm.yaml

@@ -290,6 +290,9 @@ variations:
         - all
         CM_MLPERF_ACCURACY_RESULTS_DIR:
         - 'on'
+      skip_if_env:
+        CM_MLPERF_IMPLEMENTATION:
+        - nvidia-original
       names:
       - mlperf-accuracy-script
       - openimages-accuracy-script
@@ -327,6 +330,9 @@ variations:
         - all
         CM_MLPERF_ACCURACY_RESULTS_DIR:
         - 'on'
+      skip_if_env:
+        CM_MLPERF_IMPLEMENTATION:
+        - nvidia-original
       names:
       - mlperf-accuracy-script
       - 3d-unet-accuracy-script

cm-mlops/script/app-mlperf-inference/customize.py

@@ -29,6 +29,7 @@ def postprocess(i):
 
     env = i['env']
     inp = i['input']
+    env['CMD'] = ''
     state = i['state']
 
     if env.get('CM_MLPERF_USER_CONF', '') == '':

cm-mlops/script/app-mlperf-inference/run.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+cmd="${CMD}"
+if [[ -n ${cmd} ]]; then
+  echo "$cmd"
+  eval "$cmd"
+  test $? -eq 0 || exit $?
+fi

cm-mlops/script/generate-mlperf-inference-user-conf/customize.py

@@ -260,7 +260,7 @@ def preprocess(i):
 
         print("Output Dir: '" + OUTPUT_DIR + "'")
         print(user_conf)
-        if env.get('CM_MLPERF_POWER','') == "yes" and os.path.exists(env['CM_MLPERF_POWER_LOG_DIR']):
+        if env.get('CM_MLPERF_POWER','') == "yes" and os.path.exists(env.get('CM_MLPERF_POWER_LOG_DIR', '')):
             shutil.rmtree(env['CM_MLPERF_POWER_LOG_DIR'])
     else:
         print("Run files exist, skipping run...\n")

cm-mlops/script/get-git-repo/README.md

@@ -183,6 +183,7 @@ ___
 * `--folder=value`  →  `CM_GIT_CHECKOUT_FOLDER=value`
 * `--patch=value`  →  `CM_GIT_PATCH=value`
 * `--submodules=value`  →  `CM_GIT_RECURSE_SUBMODULES=value`
+* `--update=value`  →  `CM_GIT_REPO_PULL=value`
 
 **Above CLI flags can be used in the Python CM API as follows:**
 

cm-mlops/script/get-git-repo/_cm.json

@@ -16,6 +16,7 @@
     "depth": "CM_GIT_DEPTH",
     "folder": "CM_GIT_CHECKOUT_FOLDER",
     "patch": "CM_GIT_PATCH",
+    "update": "CM_GIT_REPO_PULL",
     "submodules": "CM_GIT_RECURSE_SUBMODULES"
   },
   "default_variation": "default",

cm-mlops/script/install-cuda-prebuilt/customize.py

@@ -17,10 +17,10 @@ def preprocess(i):
 
     env['WGET_URL']="https://developer.download.nvidia.com/compute/cuda/"+env['CM_VERSION']+"/local_installers/"+env['CM_CUDA_LINUX_FILENAME']
 
+    extra_options = env.get('CUDA_ADDITIONAL_INSTALL_OPTIONS', '')
     if env.get('CM_CUDA_INSTALL_DRIVER','') == "yes":
-        env['CUDA_ADDITIONAL_INSTALL_OPTIONS'] = " --driver"
-    else:
-        env['CUDA_ADDITIONAL_INSTALL_OPTIONS'] = ""
+        extra_options += " --driver"
+    env['CUDA_ADDITIONAL_INSTALL_OPTIONS'] = extra_options
 
     env['CM_CUDA_INSTALLED_PATH'] = os.path.join(os.getcwd(), 'install')
     env['CM_NVCC_BIN_WITH_PATH'] = os.path.join(os.getcwd(), 'install', 'bin', nvcc_bin)

cm-mlops/script/install-cuda-prebuilt/run.sh

@@ -4,5 +4,7 @@ CUR=${PWD}
 
 INSTALL_DIR=${CUR}/install
 
-${CM_SUDO} bash ${CM_CUDA_RUN_FILE_PATH} --toolkitpath=${INSTALL_DIR} --defaultroot=${INSTALL_DIR} --toolkit ${CUDA_ADDITIONAL_INSTALL_OPTIONS} --silent
+cmd="${CM_SUDO} bash ${CM_CUDA_RUN_FILE_PATH} --toolkitpath=${INSTALL_DIR} --defaultroot=${INSTALL_DIR} --toolkit ${CUDA_ADDITIONAL_INSTALL_OPTIONS} --silent "
+echo "${cmd}"
+eval "${cmd}"
 

cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml

@@ -31,7 +31,9 @@ default_env:
   CM_SKIP_PREPROCESS_DATASET: 'no'
   CM_SKIP_MODEL_DOWNLOAD: 'no'
   CM_MLPERF_SUT_NAME_IMPLEMENTATION_PREFIX: nvidia_original
-  CM_CALL_RUNNER: 'yes'
+  CM_MLPERF_SKIP_RUN: 'no'
+env:
+  CM_CALL_MLPERF_RUNNER: 'no'
 
 # Map script inputs to environment variables
 input_mapping:
@@ -40,7 +42,7 @@ input_mapping:
   mlperf_conf: CM_MLPERF_CONF
   mode: CM_MLPERF_LOADGEN_MODE
   output_dir: CM_MLPERF_OUTPUT_DIR
-  performance_sample_count: CM_MLPERF_LOADGEN_PERFORMANCE_SAMPLE_COUNT
+  performance_sample_count: CM_MLPERF_PERFORMANCE_SAMPLE_COUNT
   scenario: CM_MLPERF_LOADGEN_SCENARIO
   user_conf: CM_MLPERF_USER_CONF
   devices: CM_MLPERF_NVIDIA_HARNESS_DEVICES
@@ -250,14 +252,14 @@ post_deps:
 
   - names:
     - runner
+    - mlperf-runner
     skip_if_env:
       CM_MLPERF_SKIP_RUN:
+        - 'yes'
         - yes
-      CM_MLPERF_NVIDIA_HARNESS_RUN_MODE:
-        - run_harness
     tags: benchmark-mlperf
     enable_if_env:
-      CM_CALL_RUNNER:
+      CM_CALL_MLPERF_RUNNER:
         - yes
 
 
@@ -293,6 +295,7 @@ variations:
     group: model
     env:
       CM_MODEL: retinanet
+      CM_ML_MODEL_STARTING_WEIGHTS_FILENAME: "https://zenodo.org/record/6617981/files/resnext50_32x4d_fpn.pth"
     deps:
     - tags: get,generic-python-lib,_Pillow
     - tags: get,generic-python-lib,_torch
@@ -310,36 +313,43 @@ variations:
     - bert_
     env:
       CM_MODEL: bert-99
+      CM_NOT_ML_MODEL_STARTING_WEIGHTS_FILENAME: "https://zenodo.org/record/3750364/files/bert_large_v1_1_fake_quant.onnx"
 
   bert-99.9:
     group: model
     base:
     - bert_
     env:
       CM_MODEL: bert-99.9
+      CM_NOT_ML_MODEL_STARTING_WEIGHTS_FILENAME: "https://zenodo.org/record/3733910/files/model.onnx"
 
   3d-unet_:
     deps:
     - tags: get,generic-python-lib,_transformers
+    - tags: get,generic-python-lib,_pandas
+      version_max: "1.5.3"
 
   3d-unet-99:
     group: model
     base:
     - 3d-unet_
     env:
       CM_MODEL: 3d-unet-99
+      CM_ML_MODEL_STARTING_WEIGHTS_FILENAME: "https://zenodo.org/record/5597155/files/3dunet_kits19_128x128x128.onnx"
 
   3d-unet-99.9:
     group: model
     base:
     - 3d-unet_
     env:
       CM_MODEL: 3d-unet-99.9
+      CM_ML_MODEL_STARTING_WEIGHTS_FILENAME: "https://zenodo.org/record/5597155/files/3dunet_kits19_128x128x128.onnx"
 
   rnnt:
     group: model
     env:
       CM_MODEL: rnnt
+      CM_ML_MODEL_STARTING_WEIGHTS_FILENAME: "https://zenodo.org/record/3662521/files/DistributedDataParallel_1576581068.9962234-epoch-100.pt"
     deps:
     - tags: get,generic-python-lib,_toml
     - tags: get,generic-python-lib,_torchvision
@@ -479,9 +489,9 @@ variations:
             - dlrm-99.9
       - tags: reproduce,mlperf,inference,nvidia,harness,_calibrate
         inherit_variation_tags: true
-        skip_if_env:
+        enable_if_env:
           CM_MODEL:
-            - rnnt
+            - retinanet
         force_cache: true
         skip_inherit_variation_groups:
           - run-mode
@@ -557,6 +567,7 @@ variations:
     env:
       CM_MLPERF_NVIDIA_HARNESS_RUN_MODE: run_harness
       MLPERF_NVIDIA_RUN_COMMAND: run_harness
+      CM_CALL_MLPERF_RUNNER: 'yes'
     new_env_keys:
       - CM_MLPERF_*
       - CM_DATASET_*
@@ -897,12 +908,20 @@ variations:
     env:
       CM_NVIDIA_CUSTOM_GPU: "yes"
 
+  l4,resnet50:
+    default_env:
+      CM_MLPERF_LOADGEN_OFFLINE_TARGET_QPS: 10500
+      CM_MLPERF_LOADGEN_SERVER_TARGET_QPS: 9000
+      CM_MLPERF_LOADGEN_SINGLESTREAM_TARGET_LATENCY: 0.35
+      CM_MLPERF_LOADGEN_MULTISTREAM_TARGET_LATENCY: 1
+
   l4,resnet50,offline,run_harness:
     add_deps_recursive:
       build-engine:
-        tags: _batch_size.16
+        tags: _batch_size.32
+
     env:
-      CM_MODEL_BATCH_SIZE: "16"
+      CM_MODEL_BATCH_SIZE: "32"
       CM_MLPERF_NVIDIA_HARNESS_GPU_COPY_STREAMS: "2"
       CM_MLPERF_NVIDIA_HARNESS_GPU_INFERENCE_STREAMS: "1"
       CM_MLPERF_NVIDIA_HARNESS_USE_GRAPHS: 'True'
@@ -952,10 +971,18 @@ variations:
     env:
       CM_NVIDIA_CUSTOM_GPU: "yes"
 
+  t4,resnet50:
+    default_env:
+      CM_MLPERF_LOADGEN_OFFLINE_TARGET_QPS: 4900
+      CM_MLPERF_LOADGEN_SERVER_TARGET_QPS: 4000
+      CM_MLPERF_LOADGEN_SINGLESTREAM_TARGET_LATENCY: 0.6
+      CM_MLPERF_LOADGEN_MULTISTREAM_TARGET_LATENCY: 2
+
   t4,resnet50,offline,run_harness:
     add_deps_recursive:
       build-engine:
         tags: _batch_size.256
+
     env:
       CM_MODEL_BATCH_SIZE: "256"
       CM_MLPERF_NVIDIA_HARNESS_GPU_COPY_STREAMS: "4"

cm-mlops/script/reproduce-mlperf-inference-nvidia/customize.py

@@ -143,7 +143,6 @@ def preprocess(i):
         if not os.path.exists(model_path):
             cmds.append(f"make download_model BENCHMARKS='{model_name}'")
         else:
-            env['CM_CALL_RUNNER'] = "no"
             return {'return':0}
 
     elif make_command == "preprocess_data":
@@ -200,20 +199,28 @@ def preprocess(i):
             multistream_target_latency_ns = int(float(multistream_target_latency) * 1000000)
             run_config += f" --multi_stream_expected_latency_ns={multistream_target_latency_ns}"
 
+        high_accuracy = "99.9" in env['CM_MODEL']
+
         use_triton = env.get('CM_MLPERF_NVIDIA_HARNESS_USE_TRITON')
         if use_triton:
-            run_config += f" --use_triton --config_ver=triton"
+            if high_accuracy:
+                run_config += f" --use_triton --config_ver=high_accuracy_triton"
+            else:
+                run_config += f" --use_triton --config_ver=triton"
+        else:
+            if high_accuracy:
+                run_config += f" --config_ver=high_accuracy"
 
         user_conf_path = env.get('CM_MLPERF_USER_CONF')
-        if user_conf_path:
+        if user_conf_path and env['CM_MLPERF_NVIDIA_HARNESS_RUN_MODE'] == "run_harness":
             run_config += f" --user_conf_path={user_conf_path}"
 
         mlperf_conf_path = env.get('CM_MLPERF_INFERENCE_CONF_PATH')
-        if mlperf_conf_path:
+        if mlperf_conf_path and env['CM_MLPERF_NVIDIA_HARNESS_RUN_MODE'] == "run_harness":
             run_config += f" --mlperf_conf_path={mlperf_conf_path}"
 
         power_setting = env.get('CM_MLPERF_NVIDIA_HARNESS_POWER_SETTING')
-        if power_setting:
+        if power_setting and env['CM_MLPERF_NVIDIA_HARNESS_RUN_MODE'] == "run_harness":
             run_config += f" --power_setting={power_setting}"
 
         gpu_copy_streams = env.get('CM_MLPERF_NVIDIA_HARNESS_GPU_COPY_STREAMS')
@@ -240,7 +247,7 @@ def preprocess(i):
         if input_format:
             run_config += f" --input_format={input_format}"
 
-        performance_sample_count = env.get('CM_MLPERF_LOADGEN_PERFORMANCE_SAMPLE_COUNT')
+        performance_sample_count = env.get('CM_MLPERF_PERFORMANCE_SAMPLE_COUNT')
         if performance_sample_count:
             run_config += f" --performance_sample_count={performance_sample_count}"
 

cm-mlops/script/reproduce-mlperf-inference-nvidia/run.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+if [[ ${CM_CALL_MLPERF_RUNNER} == "no" ]]; then
+  cd ${CM_RUN_DIR}
+  cmd=${CM_RUN_CMD}
+  echo "${cmd}"
+  eval "${cmd}"
+  test $? -eq 0 || exit $?
+fi