From adc207c196f982f0a0337dd19a6411361402256a Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 21 Jul 2023 16:22:35 +0200 Subject: [PATCH 1/8] minor clean up --- cm-mlops/automation/experiment/module.py | 4 ++ .../README.md | 20 ++++--- .../_cm.json | 2 +- .../README.md | 2 +- .../README.md | 2 +- .../README.md | 8 +-- .../README.md | 2 +- .../README.md | 2 +- .../README.md | 2 +- .../README.md | 3 +- .../README.md | 2 +- .../README.md | 2 +- .../README.md | 2 +- .../README.md | 2 +- .../README.md | 2 +- cm-mlops/script/gui/playground_challenges.py | 6 +- docs/news.md | 5 ++ platform/register.md | 59 ++----------------- platform/register2.md | 59 +++++++++++++++++++ 19 files changed, 103 insertions(+), 83 deletions(-) create mode 100644 platform/register2.md diff --git a/cm-mlops/automation/experiment/module.py b/cm-mlops/automation/experiment/module.py index 0c580b86db..203574d5e5 100644 --- a/cm-mlops/automation/experiment/module.py +++ b/cm-mlops/automation/experiment/module.py @@ -57,6 +57,10 @@ def test(self, i): return {'return':0} + + + + ############################################################ def run(self, i): """ diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-2023/README.md b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-2023/README.md index a4b229eeb8..532cbf49e1 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-2023/README.md +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-2023/README.md @@ -1,5 +1,8 @@ ### Introduction +Our goal is to help the community benchmark and optimize various AI/ML applications +across diverse software and hardware provided by volunteers similar to SETI@home! + Open-source [MLPerf inference benchmarks](https://arxiv.org/abs/1911.02549) were developed by a [consortium of 50+ companies and universities (MLCommons)](https://mlcommons.org) to enable trustable and reproducible comparison of AI/ML systems @@ -7,14 +10,14 @@ in terms of latency, throughput, power consumption, accuracy and other metrics across diverse software/hardware stacks from different vendors. However, running MLPerf inference benchmarks and submitting results [turned out to be a challenge](https://doi.org/10.5281/zenodo.8144274) -even for experts and could easily take many weeks. That's why MLCommons, +even for experts and could easily take many weeks. That's why [MLCommons](https://mlcommons.org), [cTuning.org](https://www.linkedin.com/company/ctuning-foundation) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge) decided to develop an open-source, technology-agnostic and non-intrusive [Collective Mind automation language (CM)](https://github.com/mlcommons/ck) and [Collective Knowledge Playground (CK)](https://access.cknowledge.org/playground/?action=experiments) -to run, reproduce, optimize and compare MLPerf inference benchmarks out-of-the-box -across diverse software, hardware, models and data sets from any vendor. +to help anyone run, reproduce, optimize and compare MLPerf inference benchmarks out-of-the-box +across diverse software, hardware, models and data sets. You can read more about our vision, open-source technology and future plans in this [presentation](https://doi.org/10.5281/zenodo.8105339). @@ -23,19 +26,20 @@ in this [presentation](https://doi.org/10.5281/zenodo.8105339). ### Challenge -We would like you to run as many MLPerf inference benchmarks on as many CPUs (Intel, AMD, Arm) and Nvidia GPUs -as possible across different framework (ONNX, PyTorch, TF, TFLite) +We would like to help volunteers run various MLPerf inference benchmarks +on diverse CPUs (Intel, AMD, Arm) and Nvidia GPUs similar to SETI@home +across different framework (ONNX, PyTorch, TF, TFLite) either natively or in a cloud (AWS, Azure, GCP, Alibaba, Oracle, OVHcloud, ...) -and submit official results to MLPerf inference v3.1. +and submit results to MLPerf inference v3.1. However, since some benchmarks may take 1 day to run, we suggest to start in the following order: * [CPU: Reference implementation of Image Classification with ResNet50 (open and then closed division)](https://github.com/mlcommons/ck/blob/master/docs/mlperf/inference/resnet50/README_reference.md) +* [CPU: TFLite C++ implementation of Image classification with variations of MobileNets and EfficientNets (open division)](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/run-mlperf-inference-mobilenet-models/README-about.md) * [Nvidia GPU: Nvidia optimized implementation of Image Classification with ResNet50 (open and then closed division)](https://github.com/mlcommons/ck/blob/master/docs/mlperf/inference/resnet50/README_nvidia.md) * [Nvidia GPU: Nvidia optimized implementation of Language processing with BERT large (open and then closed division)](https://github.com/mlcommons/ck/blob/master/docs/mlperf/inference/bert/README_nvidia.md) * [Nvidia GPU: Reference implementation of Image Classification with ResNet50 (open and then closed division)](https://github.com/mlcommons/ck/blob/master/docs/mlperf/inference/bert/README_nvidia.md) * [Nvidia GPU: Reference implementation of Language processing with BERT large (open and then closed division)](https://github.com/mlcommons/ck/blob/master/docs/mlperf/inference/resnet50/README_reference.md) * [Nvidia GPU (24GB of memory min): Reference implementation of Language processing with GPT-J 6B (open)](https://github.com/mlcommons/ck/blob/master/docs/mlperf/inference/gpt-j/README_reference.md) -* [CPU: TFLite C++ implementation of Image classification with variations of MobileNets and EfficientNets (open division)](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/run-mlperf-inference-mobilenet-models/README-about.md) * [Nvidia GPU: Nvidia optimized implementation of all other models (open and closed division)](https://github.com/ctuning/mlcommons-ck/blob/master/docs/mlperf/inference/README.md#run-benchmarks-and-submit-results) Please read [this documentation](https://github.com/mlcommons/ck/blob/master/docs/mlperf/inference/README.md) @@ -60,7 +64,7 @@ Looking forward to your submissions and happy hacking! * *All submitters will participate in writing a common white paper about running and comparing MLPerf inference benchmarks out-of-the-box.* * *All submitters will receive 1 point for submitting valid results for 1 complete benchmark on one system.* * *All submitters will receive an official MLCommons Collective Knowledge contributor award (see [this example](https://ctuning.org/awards/ck-award-202307-zhu.pdf)).* -* *The top 3 submitters by points will receive a prize of 200$ each.* +* *The top contributors will receive cash prizes from [MLCommons organizations](https://mlcommons.org) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge)*. ### Organizers diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-2023/_cm.json b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-2023/_cm.json index 5e623b6184..2184ae4558 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-2023/_cm.json +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-2023/_cm.json @@ -22,6 +22,6 @@ "mlperf-inference-v3.1-2023", "v3.1" ], - "title": "Run and optimize MLPerf inference v3.1 benchmarks (latency, throughput, power consumption, accuracy, cost)", + "title": "Participate in collaborative benchmarking of AI/ML systems similar to SETI@home (latency, throughput, power consumption, accuracy, cost)", "uid": "3e971d8089014d1f" } diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-amazon-inferentia-2023/README.md b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-amazon-inferentia-2023/README.md index 3aab421937..4baa492ebd 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-amazon-inferentia-2023/README.md +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-amazon-inferentia-2023/README.md @@ -18,7 +18,7 @@ Check [this ACM REP'23 keynote](https://doi.org/10.5281/zenodo.8105339) to learn * *All contributors will participate in writing a common white paper about running and comparing MLPerf inference benchmarks out-of-the-box.* * *All contributors will receive an official MLCommons Collective Knowledge contributor award (see [this example](https://ctuning.org/awards/ck-award-202307-zhu.pdf)).* -* *The first implementation will receive 3 points and a prize of 400$ and the fastest implementation will receive a prize of 300$.* +* *The top contributors will receive cash prizes from [MLCommons organizations](https://mlcommons.org) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge)*. diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-android/README.md b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-android/README.md index 2ed9150e18..1af3d813a0 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-android/README.md +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-android/README.md @@ -19,7 +19,7 @@ Check [this ACM REP'23 keynote](https://doi.org/10.5281/zenodo.8105339) to learn * *All contributors will participate in writing a common white paper about running and comparing MLPerf inference benchmarks out-of-the-box.* * *All contributors will receive an official MLCommons Collective Knowledge contributor award (see [this example](https://ctuning.org/awards/ck-award-202307-zhu.pdf)).* -* *The first implementation will receive 2 points and a prize of 300$.* +* *The top contributors will receive cash prizes from [MLCommons organizations](https://mlcommons.org) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge)*. ### Organizers diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-create-end-to-end-app/README.md b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-create-end-to-end-app/README.md index c3f3771d93..4dabd3ae3c 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-create-end-to-end-app/README.md +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-create-end-to-end-app/README.md @@ -13,10 +13,10 @@ Looking forward to your submissions and happy hacking! ### Prizes -*All submitters will participate in writing a common white paper about running and comparing MLPerf inference benchmarks out-of-the-box.* -*All submitters will receive 1 point for submitting valid results for 1 complete benchmark on one system.* -*All submitters will receive an official MLCommons Collective Knowledge contributor award (see [this example](https://ctuning.org/awards/ck-award-202307-zhu.pdf)).* -*The top 3 submitters by points will receive a prize of 200$ each.* +* *All submitters will participate in writing a common white paper about running and comparing MLPerf inference benchmarks out-of-the-box.* +* *All submitters will receive 1 point for submitting valid results for 1 complete benchmark on one system.* +* *All submitters will receive an official MLCommons Collective Knowledge contributor award (see [this example](https://ctuning.org/awards/ck-award-202307-zhu.pdf)).* +* *The top contributors will receive cash prizes from [MLCommons organizations](https://mlcommons.org) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge)*. ### Organizers diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-deepsparse/README.md b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-deepsparse/README.md index 9359166640..b4b709ecd1 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-deepsparse/README.md +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-deepsparse/README.md @@ -20,7 +20,7 @@ Check [this ACM REP'23 keynote](https://doi.org/10.5281/zenodo.8105339) to learn * *All contributors will participate in writing a common white paper about running and comparing MLPerf inference benchmarks out-of-the-box.* * *All contributors will receive 1 point for submitting valid results for 1 complete benchmark on one system.* * *All contributors will receive an official MLCommons Collective Knowledge contributor award (see [this example](https://ctuning.org/awards/ck-award-202307-zhu.pdf)).* -* *The top 3 submitters by points will receive a prize of 150$ each.* +* *The top contributors will receive cash prizes from [MLCommons organizations](https://mlcommons.org) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge)*. ### Organizers diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-google-tpu-2023/README.md b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-google-tpu-2023/README.md index 997164bacc..70f1f34f1a 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-google-tpu-2023/README.md +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-google-tpu-2023/README.md @@ -20,7 +20,7 @@ Check [this ACM REP'23 keynote](https://doi.org/10.5281/zenodo.8105339) to learn * *All contributors will participate in writing a common white paper about running and comparing MLPerf inference benchmarks out-of-the-box.* * *All contributors will receive an official MLCommons Collective Knowledge contributor award (see [this example](https://ctuning.org/awards/ck-award-202307-zhu.pdf)).* -* *The first implementation will receive 3 points and a prize of 400$ and the fastest implementation will receive a prize of 300$.* +* *The top contributors will receive cash prizes from [MLCommons organizations](https://mlcommons.org) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge)*. ### Organizers diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-hugging-face-models-2023/README.md b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-hugging-face-models-2023/README.md index 98bedc8d99..7d162aca35 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-hugging-face-models-2023/README.md +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-hugging-face-models-2023/README.md @@ -45,7 +45,7 @@ Check [this ACM REP'23 keynote](https://doi.org/10.5281/zenodo.8105339) to learn * *All contributors will participate in writing a common white paper about running and comparing MLPerf inference benchmarks out-of-the-box.* * *All contributors will receive 1 point for submitting valid results for 1 complete benchmark on one system.* * *All contributors will receive an official MLCommons Collective Knowledge contributor award (see [this example](https://ctuning.org/awards/ck-award-202307-zhu.pdf)).* -* *The first implementation will receive 3 points and a prize of 300$.* +* *The top contributors will receive cash prizes from [MLCommons organizations](https://mlcommons.org) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge)*. ### Organizers diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-intel-2023/README.md b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-intel-2023/README.md index a1fafbe3e1..40e0949a51 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-intel-2023/README.md +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-intel-2023/README.md @@ -19,8 +19,7 @@ Check [this ACM REP'23 keynote](https://doi.org/10.5281/zenodo.8105339) to learn * *All contributors will participate in writing a common white paper about running and comparing MLPerf inference benchmarks out-of-the-box.* * *All contributors will receive an official MLCommons Collective Knowledge contributor award (see [this example](https://ctuning.org/awards/ck-award-202307-zhu.pdf)).* -* *The first implementation will receive 2 points and a prize of 200$ and the fastest implementation will receive a prize of 200$.* - +* *The top contributors will receive cash prizes from [MLCommons organizations](https://mlcommons.org) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge)*. ### Organizers diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-modular-mojo-2023/README.md b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-modular-mojo-2023/README.md index 47f6741d88..7198895785 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-modular-mojo-2023/README.md +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-modular-mojo-2023/README.md @@ -23,7 +23,7 @@ Check [this ACM REP'23 keynote](https://doi.org/10.5281/zenodo.8105339) to learn * *The first implementation will receive a cache prize from organizers.* * *All contributors will participate in writing a common white paper about running and comparing MLPerf inference benchmarks out-of-the-box.* * *All contributors will receive 1 point for submitting valid results for 1 complete benchmark on one system.* -* *All contributors will receive an official MLCommons Collective Knowledge contributor award (see [this example](https://ctuning.org/awards/ck-award-202307-zhu.pdf)).* +* *The top contributors will receive cash prizes from [MLCommons organizations](https://mlcommons.org) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge)*. ### Organizers diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-qualcomm-ai100-2023/README.md b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-qualcomm-ai100-2023/README.md index 2d845b6656..a91c819826 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-qualcomm-ai100-2023/README.md +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-qualcomm-ai100-2023/README.md @@ -19,7 +19,7 @@ Check [this ACM REP'23 keynote](https://doi.org/10.5281/zenodo.8105339) to learn * *All contributors will participate in writing a common white paper about running and comparing MLPerf inference benchmarks out-of-the-box.* * *All contributors will receive an official MLCommons Collective Knowledge contributor award (see [this example](https://ctuning.org/awards/ck-award-202307-zhu.pdf)).* -* *The first implementation will receive 3 points and a prize of 200$ and the fastest implementation will receive a prize of 200$.* +* *The top contributors will receive cash prizes from [MLCommons organizations](https://mlcommons.org) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge)*. diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-tvm-2023/README.md b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-tvm-2023/README.md index a9ccecb38a..fcec854ba3 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-tvm-2023/README.md +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-tvm-2023/README.md @@ -20,7 +20,7 @@ Check [this ACM REP'23 keynote](https://doi.org/10.5281/zenodo.8105339) to learn * *All contributors will participate in writing a common white paper about running and comparing MLPerf inference benchmarks out-of-the-box.* * *All contributors will receive 1 point for submitting valid results for 1 complete benchmark on one system.* * *All contributors will receive an official MLCommons Collective Knowledge contributor award (see [this example](https://ctuning.org/awards/ck-award-202307-zhu.pdf)).* -* *The top 3 submitters by points will receive a prize of 150$ each.* +* *The top contributors will receive cash prizes from [MLCommons organizations](https://mlcommons.org) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge)*. ### Organizers diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-universal-cpp-implementation-2023/README.md b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-universal-cpp-implementation-2023/README.md index d082feaabc..014ae17137 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-universal-cpp-implementation-2023/README.md +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-universal-cpp-implementation-2023/README.md @@ -18,7 +18,7 @@ Check [this ACM REP'23 keynote](https://doi.org/10.5281/zenodo.8105339) to learn * *All contributors will participate in writing a common white paper about running and comparing MLPerf inference benchmarks out-of-the-box.* * *All contributors will receive an official MLCommons Collective Knowledge contributor award (see [this example](https://ctuning.org/awards/ck-award-202307-zhu.pdf)).* -* *The first implementation will receive 2 points and a prize of 200$ and the fastest implementation will receive a prize of 200$.* +* *The top contributors will receive cash prizes from [MLCommons organizations](https://mlcommons.org) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge)*. diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-windows-2023/README.md b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-windows-2023/README.md index dc5ba58802..179961b5b6 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-windows-2023/README.md +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-windows-2023/README.md @@ -18,7 +18,7 @@ Check [this ACM REP'23 keynote](https://doi.org/10.5281/zenodo.8105339) to learn * *All contributors will participate in writing a common white paper about running and comparing MLPerf inference benchmarks out-of-the-box.* * *All contributors will receive an official MLCommons Collective Knowledge contributor award (see [this example](https://ctuning.org/awards/ck-award-202307-zhu.pdf)).* -* *The first implementation will receive 2 points and a prize of 200$.* +* *The top contributors will receive cash prizes from [MLCommons organizations](https://mlcommons.org) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge)*. ### Organizers diff --git a/cm-mlops/script/gui/playground_challenges.py b/cm-mlops/script/gui/playground_challenges.py index 06e635f346..efb755aec3 100644 --- a/cm-mlops/script/gui/playground_challenges.py +++ b/cm-mlops/script/gui/playground_challenges.py @@ -347,9 +347,9 @@ def page(st, params): if prize_short!='': z+='* **Prizes:** {}\n'.format(prize_short) - prize = meta.get('prize','') - if prize!='': - z+='* **Student prizes:** {}\n'.format(prize) +# prize = meta.get('prize','') +# if prize!='': +# z+='* **Student prizes:** {}\n'.format(prize) urls = meta.get('urls',[]) diff --git a/docs/news.md b/docs/news.md index 1e714aecad..84f4315c03 100644 --- a/docs/news.md +++ b/docs/news.md @@ -2,6 +2,11 @@ ## MLCommons CK and CM news +### 202307 + +The overview of the MedPerf project was published in Nature: +[Federated benchmarking of medical artificial intelligence with MedPerf](https://www.nature.com/articles/s42256-023-00652-2)! + ### 202306 We were honored to give a [keynote](https://doi.org/10.5281/zenodo.8105338) about our MLCommons automation and reproducibility language diff --git a/platform/register.md b/platform/register.md index 1d9a9fde01..358f3e7555 100644 --- a/platform/register.md +++ b/platform/register.md @@ -1,59 +1,8 @@ # Register for Collective Knowledge challenges -Since the [MLCommons CK playground](https://access.cKnowledge.org) -is still in the heavy development stage, the registration is not yet automated via CK GUI. - -You can simply add add your name, organization and URL in this [GitHub ticket](https://github.com/mlcommons/ck/issues/855). - -You name will be added to the [CK leaderboard](https://access.cknowledge.org/playground) -with 1 point after your PR is accepted (to support your intent to participate in our collaborative effort). - -You can add yourself to this [GitHub repository](https://github.com/mlcommons/ck/tree/master/cm-mlops/contributor) -using our [CM automation language](https://doi.org/10.5281/zenodo.8105339) from the command line as follows. - -Install [CM](../docs/installation.md) on your system. - -Fork https://github.com/mlcommons/ck . - -Pull it via CM as follows: - -```bash -cm pull repo --url={URL of the fork of github.com/mlcommons/ck} -``` - -Note that if you already have `mlcommons@ck` repository installed via CM, -you need to delete it and then install your fork: -```bash -cm rm repo mlcommons@ck --all -cm pull repo --url={URL of the fork of github.com/mlcommons/ck} -``` -Create a new contributor with your name: -```bash -cm add contributor "your name" -``` - -CM will ask you a few questions and will create a new CM contributor entry with your name. - -You can commit this entry to your fork and create a PR to https://github.com/mlcommons/ck . - -*Note that you will need to sign MLCommons CLA to contribute to MLCommons projects - it may take a few days to approve it by MLCommons*. - -Note that you will need CM and your fork of https://github.com/mlcommons/ck to participate in challenges, -so please keep and use it. - -Happy hacking! - -## Discussions - -You can now join the [public Discord server](https://discord.gg/JjWNWXKxwT) +Please join the [public Discord server](https://discord.gg/JjWNWXKxwT) from the [MLCommons Task Force on Automation and Reproducibility](../docs/taskforce.md) -to ask any questions, provide feedback and discuss challenges! - -## Our mission - -You can learn more about our mission [here](https://doi.org/10.5281/zenodo.8105339). - -## Organizers +and send your name, organization and URL to @gfursin and @arjunsuresh +(task force co-chairs and organizers of open challenges). -* [Grigori Fursin](https://cKnowledge.org/gfursin) and [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh) - ([MLCommons](https://mlcommons.org), [cTuning.org](https://cTuning.org) and [cKnowledge.org](https://cKnowledge.org)) +In the future, we plan to add a registeration GUI to our [MLCommons Collective Knowledge playground](https://access.cKnowledge.org). diff --git a/platform/register2.md b/platform/register2.md new file mode 100644 index 0000000000..1d9a9fde01 --- /dev/null +++ b/platform/register2.md @@ -0,0 +1,59 @@ +# Register for Collective Knowledge challenges + +Since the [MLCommons CK playground](https://access.cKnowledge.org) +is still in the heavy development stage, the registration is not yet automated via CK GUI. + +You can simply add add your name, organization and URL in this [GitHub ticket](https://github.com/mlcommons/ck/issues/855). + +You name will be added to the [CK leaderboard](https://access.cknowledge.org/playground) +with 1 point after your PR is accepted (to support your intent to participate in our collaborative effort). + +You can add yourself to this [GitHub repository](https://github.com/mlcommons/ck/tree/master/cm-mlops/contributor) +using our [CM automation language](https://doi.org/10.5281/zenodo.8105339) from the command line as follows. + +Install [CM](../docs/installation.md) on your system. + +Fork https://github.com/mlcommons/ck . + +Pull it via CM as follows: + +```bash +cm pull repo --url={URL of the fork of github.com/mlcommons/ck} +``` + +Note that if you already have `mlcommons@ck` repository installed via CM, +you need to delete it and then install your fork: +```bash +cm rm repo mlcommons@ck --all +cm pull repo --url={URL of the fork of github.com/mlcommons/ck} +``` +Create a new contributor with your name: +```bash +cm add contributor "your name" +``` + +CM will ask you a few questions and will create a new CM contributor entry with your name. + +You can commit this entry to your fork and create a PR to https://github.com/mlcommons/ck . + +*Note that you will need to sign MLCommons CLA to contribute to MLCommons projects - it may take a few days to approve it by MLCommons*. + +Note that you will need CM and your fork of https://github.com/mlcommons/ck to participate in challenges, +so please keep and use it. + +Happy hacking! + +## Discussions + +You can now join the [public Discord server](https://discord.gg/JjWNWXKxwT) +from the [MLCommons Task Force on Automation and Reproducibility](../docs/taskforce.md) +to ask any questions, provide feedback and discuss challenges! + +## Our mission + +You can learn more about our mission [here](https://doi.org/10.5281/zenodo.8105339). + +## Organizers + +* [Grigori Fursin](https://cKnowledge.org/gfursin) and [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh) + ([MLCommons](https://mlcommons.org), [cTuning.org](https://cTuning.org) and [cKnowledge.org](https://cKnowledge.org)) From 7fda850d1014444a525d7a147079a4cc26c7a7db Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 21 Jul 2023 16:28:32 +0200 Subject: [PATCH 2/8] clean up --- cm-mlops/script/gui/playground_challenges.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cm-mlops/script/gui/playground_challenges.py b/cm-mlops/script/gui/playground_challenges.py index efb755aec3..0efc88f559 100644 --- a/cm-mlops/script/gui/playground_challenges.py +++ b/cm-mlops/script/gui/playground_challenges.py @@ -204,7 +204,7 @@ def page(st, params): prize = row.get('prize_short','') if prize!='': - x += '   Prizes from [MLCommons organizations]({}): **{}**\n'.format('https://mlcommons.org', prize) + x += '   Prizes from [MLCommons organizations](https://mlcommons.org) and [cKnowledge.org](https://www.linkedin.com/company/cknowledge): **{}**\n'.format(prize) if awards!='': awards+=' , ' awards += prize @@ -225,7 +225,7 @@ def page(st, params): import numpy as np df = pd.DataFrame(data, - columns=['Challenge', 'Closing date', 'Extension', 'Points', 'Contributor award and prizes from MLCommons organizations']) + columns=['Challenge', 'Closing date', 'Extension', 'Points', 'Contributor award and prizes from MLCommons organizations and cKnowledge.org']) df.index+=1 From 4e8453f8eead2b345f8ebd8b90d53338a9be90a9 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 21 Jul 2023 16:38:28 +0200 Subject: [PATCH 3/8] clean up --- .../challenge/optimize-mlperf-inference-v3.1-2023/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-2023/README.md b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-2023/README.md index 532cbf49e1..af11fc8f4d 100644 --- a/cm-mlops/challenge/optimize-mlperf-inference-v3.1-2023/README.md +++ b/cm-mlops/challenge/optimize-mlperf-inference-v3.1-2023/README.md @@ -32,7 +32,7 @@ across different framework (ONNX, PyTorch, TF, TFLite) either natively or in a cloud (AWS, Azure, GCP, Alibaba, Oracle, OVHcloud, ...) and submit results to MLPerf inference v3.1. -However, since some benchmarks may take 1 day to run, we suggest to start in the following order: +However, since some benchmarks may take 1..2 days to run, we suggest to start in the following order (these links describe CM commands to run benchmarks and submit results): * [CPU: Reference implementation of Image Classification with ResNet50 (open and then closed division)](https://github.com/mlcommons/ck/blob/master/docs/mlperf/inference/resnet50/README_reference.md) * [CPU: TFLite C++ implementation of Image classification with variations of MobileNets and EfficientNets (open division)](https://github.com/mlcommons/ck/blob/master/cm-mlops/script/run-mlperf-inference-mobilenet-models/README-about.md) * [Nvidia GPU: Nvidia optimized implementation of Image Classification with ResNet50 (open and then closed division)](https://github.com/mlcommons/ck/blob/master/docs/mlperf/inference/resnet50/README_nvidia.md) From 31d39841405c55374bdb1db755683f84fa9d7043 Mon Sep 17 00:00:00 2001 From: Arjun Date: Fri, 21 Jul 2023 21:52:22 +0000 Subject: [PATCH 4/8] Fix gpu size for R50 offline, L4 --- .../script/reproduce-mlperf-inference-nvidia/_cm.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml b/cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml index 3f931ad58a..e5f1d2e1f1 100644 --- a/cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml +++ b/cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml @@ -880,10 +880,12 @@ variations: l4,resnet50,offline,run_harness: add_deps_recursive: build-engine: - tags: _batch_size.32 + tags: _batch_size.16 env: - CM_MODEL_BATCH_SIZE: "32" - CM_MLPERF_NVIDIA_HARNESS_GPU_COPY_STREAMS: "4" + CM_MODEL_BATCH_SIZE: "16" + CM_MLPERF_NVIDIA_HARNESS_GPU_COPY_STREAMS: "2" + CM_MLPERF_NVIDIA_HARNESS_GPU_INFERENCE_STREAMS: "1" + CM_MLPERF_NVIDIA_HARNESS_USE_GRAPHS: 'True' CM_MLPERF_NVIDIA_HARNESS_GPU_BATCH_SIZE: "<<>>" l4,retinanet,offline,run_harness: From 6663c49e4f3de55520059fc64057ffdb8f991333 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 22 Jul 2023 00:59:08 +0100 Subject: [PATCH 5/8] Added name for cuda in get-cuda-devices --- cm-mlops/script/get-cuda-devices/_cm.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cm-mlops/script/get-cuda-devices/_cm.json b/cm-mlops/script/get-cuda-devices/_cm.json index 455b5d9a10..ffa5ede93d 100644 --- a/cm-mlops/script/get-cuda-devices/_cm.json +++ b/cm-mlops/script/get-cuda-devices/_cm.json @@ -14,6 +14,9 @@ ], "deps": [ { + "names": [ + "cuda" + ], "tags": "get,cuda,_toolkit" } ], From e249b899cd3597f80895a54dd16513b4cc3757f2 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 22 Jul 2023 01:19:39 +0100 Subject: [PATCH 6/8] Use nvidia script to download kits dataset --- cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml b/cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml index e5f1d2e1f1..35277a5e3f 100644 --- a/cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml +++ b/cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml @@ -137,8 +137,8 @@ deps: - enable_if_env: CM_MODEL: - - 3d-unet-99 - - 3d-unet-99.9 + - 3d-unet-99-disabled + - 3d-unet-99.9-disabled names: - kits19-original tags: get,dataset,original,kits19 From e159fb624de0ca2bb568c60d6a544c9a9e69087b Mon Sep 17 00:00:00 2001 From: Arjun Date: Sat, 22 Jul 2023 09:52:36 +0000 Subject: [PATCH 7/8] support extra build-engine options for nvidia-harness --- .../script/reproduce-mlperf-inference-nvidia/_cm.yaml | 11 +++++++++++ .../reproduce-mlperf-inference-nvidia/customize.py | 6 +++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml b/cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml index 35277a5e3f..db7df9c7c5 100644 --- a/cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml +++ b/cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml @@ -71,6 +71,7 @@ input_mapping: power_setting: CM_MLPERF_NVIDIA_HARNESS_POWER_SETTING make_cmd: MLPERF_NVIDIA_RUN_COMMAND rerun: CM_RERUN + extra_run_options: CM_MLPERF_NVIDIA_HARNESS_EXTRA_RUN_OPTIONS new_state_keys: - mlperf-inference-implementation @@ -453,6 +454,7 @@ variations: - device-memory - gpu-name - batch-size + - build-engine-options - tags: reproduce,mlperf,inference,nvidia,harness,_download_model inherit_variation_tags: true force_cache: true @@ -462,6 +464,7 @@ variations: - device-memory - gpu-name - batch-size + - build-engine-options skip_if_env: CM_MODEL: - retinanet_old @@ -480,6 +483,7 @@ variations: - device-type - gpu-name - batch-size + - build-engine-options singlestream: group: loadgen-scenario @@ -521,6 +525,7 @@ variations: - device-memory - gpu_name - batch-size + - build-engine-options force_cache: true - tags: reproduce,mlperf,inference,nvidia,harness,_download_model inherit_variation_tags: true @@ -530,6 +535,7 @@ variations: - device-memory - gpu_name - batch-size + - build-engine-options force_cache: true skip_if_env: CM_MODEL: @@ -549,6 +555,11 @@ variations: - CM_HW_NAME - CM_MAX_EXAMPLES + build_engine_options.#: + group: build-engine-options + env: + CM_MLPERF_NVIDIA_HARNESS_EXTRA_BUILD_ENGINE_OPTIONS: "#" + gpu_memory.16: group: device-memory env: diff --git a/cm-mlops/script/reproduce-mlperf-inference-nvidia/customize.py b/cm-mlops/script/reproduce-mlperf-inference-nvidia/customize.py index 5cf8a400a7..baa2b8b148 100644 --- a/cm-mlops/script/reproduce-mlperf-inference-nvidia/customize.py +++ b/cm-mlops/script/reproduce-mlperf-inference-nvidia/customize.py @@ -285,9 +285,13 @@ def preprocess(i): else: test_mode_string = "" + extra_build_engine_options_string = env.get('CM_MLPERF_NVIDIA_HARNESS_EXTRA_BUILD_ENGINE_OPTIONS', '') + + extra_run_options_string = env.get('CM_MLPERF_NVIDIA_HARNESS_EXTRA_RUN_OPTIONS', '') #will be ignored during build engine + run_config += " --no_audit_verify" - cmds.append(f"make {make_command} RUN_ARGS=' --benchmarks={model_name} --scenarios={scenario} {test_mode_string} {run_config}'") + cmds.append(f"make {make_command} RUN_ARGS=' --benchmarks={model_name} --scenarios={scenario} {test_mode_string} {run_config} {extra_build_engine_options_string} {extra_run_options_string}'") run_cmd = " && ".join(cmds) env['CM_MLPERF_RUN_CMD'] = run_cmd From b989c4ec5087174dd179255c8c25470fe4073bfa Mon Sep 17 00:00:00 2001 From: Arjun Date: Sat, 22 Jul 2023 10:04:54 +0000 Subject: [PATCH 8/8] Support cuda 12.2.0 --- cm-mlops/script/install-cuda-prebuilt/_cm.json | 5 +++++ cm-mlops/script/install-cuda-prebuilt/customize.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cm-mlops/script/install-cuda-prebuilt/_cm.json b/cm-mlops/script/install-cuda-prebuilt/_cm.json index 764eda9a99..5d6609ff12 100644 --- a/cm-mlops/script/install-cuda-prebuilt/_cm.json +++ b/cm-mlops/script/install-cuda-prebuilt/_cm.json @@ -68,6 +68,11 @@ "env": { "CM_CUDA_LINUX_FILENAME": "cuda_12.0.0_525.60.13_linux.run" } + }, + "12.2.0": { + "env": { + "CM_CUDA_LINUX_FILENAME": "cuda_12.2.0_535.54.03_linux.run" + } } }, "variations": { diff --git a/cm-mlops/script/install-cuda-prebuilt/customize.py b/cm-mlops/script/install-cuda-prebuilt/customize.py index 4a839183a2..c5dcbc99ff 100644 --- a/cm-mlops/script/install-cuda-prebuilt/customize.py +++ b/cm-mlops/script/install-cuda-prebuilt/customize.py @@ -10,7 +10,7 @@ def preprocess(i): automation = i['automation'] version = env.get('CM_VERSION') if version not in env.get('CM_CUDA_LINUX_FILENAME', ''): - return {'return': 1, 'error': "Only CUDA versions 11.7.0 and 11.8.0 are supported now!"} + return {'return': 1, 'error': "Only CUDA versions 11.7.0, 11.8.0, 12.0.0 and 12.2.0 are supported now!"} recursion_spaces = i['recursion_spaces'] nvcc_bin = "nvcc"