From cc1950627002dedf0c8527aa83b4d08a8eabf409 Mon Sep 17 00:00:00 2001 From: JoseSantosAMD Date: Mon, 7 Aug 2023 11:23:25 -0500 Subject: [PATCH 01/34] Adding min/max/avg breakdown across dispatches for instruction mix panels Signed-off-by: JoseSantosAMD --- .../1000_compute-unit-instruction-mix.yaml | 37 ++++++++++---- .../1000_compute-unit-instruction-mix.yaml | 36 ++++++++++---- .../1000_compute-unit-instruction-mix.yaml | 36 ++++++++++---- .../1100_compute-unit-compute-pipeline.yaml | 48 ++++++++++++++++--- 4 files changed, 122 insertions(+), 35 deletions(-) diff --git a/src/omniperf_analyze/configs/gfx906/1000_compute-unit-instruction-mix.yaml b/src/omniperf_analyze/configs/gfx906/1000_compute-unit-instruction-mix.yaml index b72344f3b..6d28cb2a4 100644 --- a/src/omniperf_analyze/configs/gfx906/1000_compute-unit-instruction-mix.yaml +++ b/src/omniperf_analyze/configs/gfx906/1000_compute-unit-instruction-mix.yaml @@ -12,41 +12,58 @@ Panel Config: id: 1001 title: Instruction Mix header: - metric: Metric - count: Count + avg: Avg + min: Min + max: Max unit: Unit tips: Tips metric: VALU - Vector: - count: None # No HW module + avg: None # No HW module + min: None # No HW module + max: None # No HW module unit: (instr + $normUnit) tips: VMEM: - count: None # No HW module + avg: None # No HW module + min: None # No HW module + max: None # No HW module unit: (instr + $normUnit) tips: LDS: - count: AVG((SQ_INSTS_LDS / $denom)) + avg: AVG((SQ_INSTS_LDS / $denom)) + min: MIN((SQ_INSTS_LDS / $denom)) + max: MAX((SQ_INSTS_LDS / $denom)) unit: $normUnit tips: VALU - MFMA: - count: None # No HW module + avg: None # No HW module + min: None # No HW module + max: None # No HW module unit: (instr + $normUnit) tips: SALU: - count: AVG((SQ_INSTS_SALU / $denom)) + avg: AVG((SQ_INSTS_SALU / $denom)) + min: MIN((SQ_INSTS_SALU / $denom)) + max: MAX((SQ_INSTS_SALU / $denom)) unit: (instr + $normUnit) tips: SMEM: - count: AVG((SQ_INSTS_SMEM / $denom)) + avg: AVG((SQ_INSTS_SMEM / $denom)) + min: MIN((SQ_INSTS_SMEM / $denom)) + max: MAX((SQ_INSTS_SMEM / $denom)) unit: (instr + $normUnit) tips: Branch: - count: AVG((SQ_INSTS_BRANCH / $denom)) + avg: AVG((SQ_INSTS_BRANCH / $denom)) + min: MIN((SQ_INSTS_BRANCH / $denom)) + max: MAX((SQ_INSTS_BRANCH / $denom)) unit: (instr + $normUnit) tips: GDS: - count: AVG((SQ_INSTS_GDS / $denom)) + avg: AVG((SQ_INSTS_GDS / $denom)) + min: MIN((SQ_INSTS_GDS / $denom)) + max: MAX((SQ_INSTS_GDS / $denom)) unit: (instr + $normUnit) tips: diff --git a/src/omniperf_analyze/configs/gfx908/1000_compute-unit-instruction-mix.yaml b/src/omniperf_analyze/configs/gfx908/1000_compute-unit-instruction-mix.yaml index b1977ce49..5a9bdb456 100644 --- a/src/omniperf_analyze/configs/gfx908/1000_compute-unit-instruction-mix.yaml +++ b/src/omniperf_analyze/configs/gfx908/1000_compute-unit-instruction-mix.yaml @@ -13,40 +13,58 @@ Panel Config: title: Instruction Mix header: metric: Metric - count: Count + avg: Avg + min: Min + max: Max unit: Unit tips: Tips metric: VALU - Vector: - count: AVG(((SQ_INSTS_VALU - SQ_INSTS_MFMA) / $denom)) + avg: AVG(((SQ_INSTS_VALU - SQ_INSTS_MFMA) / $denom)) + min: MIN(((SQ_INSTS_VALU - SQ_INSTS_MFMA) / $denom)) + max: MAX(((SQ_INSTS_VALU - SQ_INSTS_MFMA) / $denom)) unit: (instr + $normUnit) tips: VMEM: - count: AVG(((SQ_INSTS_VMEM - SQ_INSTS_FLAT_LDS_ONLY) / $denom)) + avg: AVG(((SQ_INSTS_VMEM - SQ_INSTS_FLAT_LDS_ONLY) / $denom)) + min: MIN(((SQ_INSTS_VMEM - SQ_INSTS_FLAT_LDS_ONLY) / $denom) + max: MAX(((SQ_INSTS_VMEM - SQ_INSTS_FLAT_LDS_ONLY) / $denom) unit: (instr + $normUnit) tips: LDS: - count: AVG((SQ_INSTS_LDS / $denom)) + avg: AVG((SQ_INSTS_LDS / $denom)) + min: MIN((SQ_INSTS_LDS / $denom)) + max: MAX((SQ_INSTS_LDS / $denom)) unit: (instr + $normUnit) tips: VALU - MFMA: - count: None # No HW module + avg: None # No HW module + min: None # No HW module + max: None # No HW module unit: (instr + $normUnit) tips: SALU: - count: AVG((SQ_INSTS_SALU / $denom)) + avg: AVG((SQ_INSTS_SALU / $denom)) + min: MIN((SQ_INSTS_SALU / $denom)) + max: MAX((SQ_INSTS_SALU / $denom)) unit: (instr + $normUnit) tips: SMEM: - count: AVG((SQ_INSTS_SMEM / $denom)) + avg: AVG((SQ_INSTS_SMEM / $denom)) + min: MIN((SQ_INSTS_SMEM / $denom)) + max: MAX((SQ_INSTS_SMEM / $denom)) unit: (instr + $normUnit) tips: Branch: - count: AVG((SQ_INSTS_BRANCH / $denom)) + avg: AVG((SQ_INSTS_BRANCH / $denom)) + min: MIN((SQ_INSTS_BRANCH / $denom)) + max: MAX((SQ_INSTS_BRANCH / $denom)) unit: (instr + $normUnit) tips: GDS: - count: AVG((SQ_INSTS_GDS / $denom)) + avg: AVG((SQ_INSTS_GDS / $denom)) + min: MIN((SQ_INSTS_GDS / $denom)) + max: MAX((SQ_INSTS_GDS / $denom)) unit: (instr + $normUnit) tips: diff --git a/src/omniperf_analyze/configs/gfx90a/1000_compute-unit-instruction-mix.yaml b/src/omniperf_analyze/configs/gfx90a/1000_compute-unit-instruction-mix.yaml index cd3f6161c..999d25e19 100644 --- a/src/omniperf_analyze/configs/gfx90a/1000_compute-unit-instruction-mix.yaml +++ b/src/omniperf_analyze/configs/gfx90a/1000_compute-unit-instruction-mix.yaml @@ -13,40 +13,58 @@ Panel Config: title: Instruction Mix header: metric: Metric - count: Count + avg: Avg + min: Min + max: Max unit: Unit tips: Tips metric: VALU - Vector: - count: AVG(((SQ_INSTS_VALU - SQ_INSTS_MFMA) / $denom)) + avg: AVG(((SQ_INSTS_VALU - SQ_INSTS_MFMA) / $denom)) + min: MIN(((SQ_INSTS_VALU - SQ_INSTS_MFMA) / $denom)) + max: MAX(((SQ_INSTS_VALU - SQ_INSTS_MFMA) / $denom)) unit: (instr + $normUnit) tips: VMEM: - count: AVG(((SQ_INSTS_VMEM - SQ_INSTS_FLAT_LDS_ONLY) / $denom)) + avg: AVG(((SQ_INSTS_VMEM - SQ_INSTS_FLAT_LDS_ONLY) / $denom)) + min: MIN(((SQ_INSTS_VMEM - SQ_INSTS_FLAT_LDS_ONLY) / $denom)) + max: MAX(((SQ_INSTS_VMEM - SQ_INSTS_FLAT_LDS_ONLY) / $denom)) unit: (instr + $normUnit) tips: LDS: - count: AVG((SQ_INSTS_LDS / $denom)) + avg: AVG((SQ_INSTS_LDS / $denom)) + min: MIN((SQ_INSTS_LDS / $denom)) + max: MAX((SQ_INSTS_LDS / $denom)) unit: (instr + $normUnit) tips: VALU - MFMA: - count: AVG((SQ_INSTS_MFMA / $denom)) + avg: AVG((SQ_INSTS_MFMA / $denom)) + min: MIN((SQ_INSTS_MFMA / $denom)) + max: MAX((SQ_INSTS_MFMA / $denom)) unit: (instr + $normUnit) tips: SALU: - count: AVG((SQ_INSTS_SALU / $denom)) + avg: AVG((SQ_INSTS_SALU / $denom)) + min: MIN((SQ_INSTS_SALU / $denom)) + max: MAX((SQ_INSTS_SALU / $denom)) unit: (instr + $normUnit) tips: SMEM: - count: AVG((SQ_INSTS_SMEM / $denom)) + avg: AVG((SQ_INSTS_SMEM / $denom)) + min: MIN((SQ_INSTS_SMEM / $denom)) + max: MAX((SQ_INSTS_SMEM / $denom)) unit: (instr + $normUnit) tips: Branch: - count: AVG((SQ_INSTS_BRANCH / $denom)) + avg: AVG((SQ_INSTS_BRANCH / $denom)) + min: MIN((SQ_INSTS_BRANCH / $denom)) + max: MAX((SQ_INSTS_BRANCH / $denom)) unit: (instr + $normUnit) tips: GDS: - count: AVG((SQ_INSTS_GDS / $denom)) + avg: AVG((SQ_INSTS_GDS / $denom)) + min: MIN((SQ_INSTS_GDS / $denom)) + max: MAX((SQ_INSTS_GDS / $denom)) unit: (instr + $normUnit) tips: diff --git a/src/omniperf_analyze/configs/gfx90a/1100_compute-unit-compute-pipeline.yaml b/src/omniperf_analyze/configs/gfx90a/1100_compute-unit-compute-pipeline.yaml index 4ed332e3f..2866f9652 100644 --- a/src/omniperf_analyze/configs/gfx90a/1100_compute-unit-compute-pipeline.yaml +++ b/src/omniperf_analyze/configs/gfx90a/1100_compute-unit-compute-pipeline.yaml @@ -13,11 +13,25 @@ Panel Config: title: Speed-of-Light header: metric: Metric - value: Value + avg: Avg + min: Min + max: Max tips: Tips metric: valu_flops_pop: - value: ((100 * AVG(((((64 * (((SQ_INSTS_VALU_ADD_F16 + SQ_INSTS_VALU_MUL_F16) + avg: ((100 * AVG(((((64 * (((SQ_INSTS_VALU_ADD_F16 + SQ_INSTS_VALU_MUL_F16) + + SQ_INSTS_VALU_TRANS_F16) + (2 * SQ_INSTS_VALU_FMA_F16))) + (64 * (((SQ_INSTS_VALU_ADD_F32 + + SQ_INSTS_VALU_MUL_F32) + SQ_INSTS_VALU_TRANS_F32) + (2 * SQ_INSTS_VALU_FMA_F32)))) + + (64 * (((SQ_INSTS_VALU_ADD_F64 + SQ_INSTS_VALU_MUL_F64) + SQ_INSTS_VALU_TRANS_F64) + + (2 * SQ_INSTS_VALU_FMA_F64)))) / (EndNs - BeginNs)))) / (((($sclk + * $numCU) * 64) * 2) / 1000)) + min: ((100 * MIN(((((64 * (((SQ_INSTS_VALU_ADD_F16 + SQ_INSTS_VALU_MUL_F16) + + SQ_INSTS_VALU_TRANS_F16) + (2 * SQ_INSTS_VALU_FMA_F16))) + (64 * (((SQ_INSTS_VALU_ADD_F32 + + SQ_INSTS_VALU_MUL_F32) + SQ_INSTS_VALU_TRANS_F32) + (2 * SQ_INSTS_VALU_FMA_F32)))) + + (64 * (((SQ_INSTS_VALU_ADD_F64 + SQ_INSTS_VALU_MUL_F64) + SQ_INSTS_VALU_TRANS_F64) + + (2 * SQ_INSTS_VALU_FMA_F64)))) / (EndNs - BeginNs)))) / (((($sclk + * $numCU) * 64) * 2) / 1000)) + max: ((100 * MAX(((((64 * (((SQ_INSTS_VALU_ADD_F16 + SQ_INSTS_VALU_MUL_F16) + SQ_INSTS_VALU_TRANS_F16) + (2 * SQ_INSTS_VALU_FMA_F16))) + (64 * (((SQ_INSTS_VALU_ADD_F32 + SQ_INSTS_VALU_MUL_F32) + SQ_INSTS_VALU_TRANS_F32) + (2 * SQ_INSTS_VALU_FMA_F32)))) + (64 * (((SQ_INSTS_VALU_ADD_F64 + SQ_INSTS_VALU_MUL_F64) + SQ_INSTS_VALU_TRANS_F64) @@ -25,23 +39,43 @@ Panel Config: * $numCU) * 64) * 2) / 1000)) tips: mfma_flops_bf16_pop: - value: ((100 * AVG(((SQ_INSTS_VALU_MFMA_MOPS_BF16 * 512) / (EndNs - BeginNs)))) + avg: ((100 * AVG(((SQ_INSTS_VALU_MFMA_MOPS_BF16 * 512) / (EndNs - BeginNs)))) + / ((($sclk * $numCU) * 512) / 1000)) + min: ((100 * MIN(((SQ_INSTS_VALU_MFMA_MOPS_BF16 * 512) / (EndNs - BeginNs)))) + / ((($sclk * $numCU) * 512) / 1000)) + max: ((100 * MAX(((SQ_INSTS_VALU_MFMA_MOPS_BF16 * 512) / (EndNs - BeginNs)))) / ((($sclk * $numCU) * 512) / 1000)) tips: mfma_flops_f16_pop: - value: ((100 * AVG(((SQ_INSTS_VALU_MFMA_MOPS_F16 * 512) / (EndNs - BeginNs)))) + avg: ((100 * AVG(((SQ_INSTS_VALU_MFMA_MOPS_F16 * 512) / (EndNs - BeginNs)))) + / ((($sclk * $numCU) * 1024) / 1000)) + min: ((100 * MIN(((SQ_INSTS_VALU_MFMA_MOPS_F16 * 512) / (EndNs - BeginNs)))) + / ((($sclk * $numCU) * 1024) / 1000)) + max: ((100 * MAX(((SQ_INSTS_VALU_MFMA_MOPS_F16 * 512) / (EndNs - BeginNs)))) / ((($sclk * $numCU) * 1024) / 1000)) tips: mfma_flops_f32_pop: - value: ((100 * AVG(((SQ_INSTS_VALU_MFMA_MOPS_F32 * 512) / (EndNs - BeginNs)))) + avg: ((100 * AVG(((SQ_INSTS_VALU_MFMA_MOPS_F32 * 512) / (EndNs - BeginNs)))) + / ((($sclk * $numCU) * 256) / 1000)) + min: ((100 * MIN(((SQ_INSTS_VALU_MFMA_MOPS_F32 * 512) / (EndNs - BeginNs)))) + / ((($sclk * $numCU) * 256) / 1000)) + max: ((100 * MAX(((SQ_INSTS_VALU_MFMA_MOPS_F32 * 512) / (EndNs - BeginNs)))) / ((($sclk * $numCU) * 256) / 1000)) tips: mfma_flops_f64_pop: - value: ((100 * AVG(((SQ_INSTS_VALU_MFMA_MOPS_F64 * 512) / (EndNs - BeginNs)))) + avg: ((100 * AVG(((SQ_INSTS_VALU_MFMA_MOPS_F64 * 512) / (EndNs - BeginNs)))) + / ((($sclk * $numCU) * 256) / 1000)) + min: ((100 * MIN(((SQ_INSTS_VALU_MFMA_MOPS_F64 * 512) / (EndNs - BeginNs)))) + / ((($sclk * $numCU) * 256) / 1000)) + max: ((100 * MAX(((SQ_INSTS_VALU_MFMA_MOPS_F64 * 512) / (EndNs - BeginNs)))) / ((($sclk * $numCU) * 256) / 1000)) tips: mfma_flops_i8_pop: - value: ((100 * AVG(((SQ_INSTS_VALU_MFMA_MOPS_I8 * 512) / (EndNs - BeginNs)))) + avg: ((100 * AVG(((SQ_INSTS_VALU_MFMA_MOPS_I8 * 512) / (EndNs - BeginNs)))) + / ((($sclk * $numCU) * 1024) / 1000)) + min: ((100 * MIN(((SQ_INSTS_VALU_MFMA_MOPS_I8 * 512) / (EndNs - BeginNs)))) + / ((($sclk * $numCU) * 1024) / 1000)) + max: ((100 * MAX(((SQ_INSTS_VALU_MFMA_MOPS_I8 * 512) / (EndNs - BeginNs)))) / ((($sclk * $numCU) * 1024) / 1000)) tips: From c16656eb8713ed248ae867f2ff90d6f286c06f26 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Fri, 9 Jun 2023 10:00:56 -0500 Subject: [PATCH 02/34] Fix VGPR issue (#139) Signed-off-by: colramos-amd --- src/utils/perfagg.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/utils/perfagg.py b/src/utils/perfagg.py index e965b784c..651bcb86d 100755 --- a/src/utils/perfagg.py +++ b/src/utils/perfagg.py @@ -127,11 +127,17 @@ def join_prof(workload_dir, join_type, log_file, verbose, out=None): "wgr": [col for col in df.columns if "wgr" in col], "lds": [col for col in df.columns if "lds" in col], "scr": [col for col in df.columns if "scr" in col], - "arch_vgpr": [col for col in df.columns if "arch_vgpr" in col], - "accum_vgpr": [col for col in df.columns if "accum_vgpr" in col], "spgr": [col for col in df.columns if "sgpr" in col], } + # Check for vgpr counter in ROCm < 5.3 + if "vgpr" in df.columns: + duplicate_cols["vgpr"] = [col for col in df.columns if "vgpr" in col] + # Check for vgpr counter in ROCm >= 5.3 + else: + duplicate_cols["arch_vgpr"] = [col for col in df.columns if "arch_vgpr" in col] + duplicate_cols["accum_vgpr"] = [col for col in df.columns if "accum_vgpr" in col] for key, cols in duplicate_cols.items(): + print("Key is ", key) _df = df[cols] if not test_df_column_equality(_df): msg = ( From 599bc01310dd875ab6c1d68bc3fc93d384cb5b83 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Fri, 9 Jun 2023 10:01:37 -0500 Subject: [PATCH 03/34] Omniperf rocomni changes Signed-off-by: colramos-amd --- src/omniperf_analyze/omniperf_analyze.py | 38 +++++++++++++++------- src/omniperf_analyze/utils/parser.py | 41 ++++++++++++++++++++++-- src/omniperf_analyze/utils/schema.py | 2 ++ 3 files changed, 66 insertions(+), 15 deletions(-) diff --git a/src/omniperf_analyze/omniperf_analyze.py b/src/omniperf_analyze/omniperf_analyze.py index 58991e8b3..c15181c6f 100644 --- a/src/omniperf_analyze/omniperf_analyze.py +++ b/src/omniperf_analyze/omniperf_analyze.py @@ -47,36 +47,50 @@ from omniperf_analyze.utils.gui_components.roofline import get_roofline -def initialize_run(args, normalization_filter=None): - import pandas as pd - from collections import OrderedDict +################################################ +# Helper Functions +################################################ +def generate_configs(config_dir, list_kernels, filter_metrics): from omniperf_analyze.utils import schema - from tabulate import tabulate - # Fixme: cur_root.parent.joinpath('soc_params') - soc_params_dir = os.path.join(os.path.dirname(__file__), "..", "soc_params") - soc_spec_df = file_io.load_soc_params(soc_params_dir) - - single_panel_config = file_io.is_single_panel_config(Path(args.config_dir)) + single_panel_config = file_io.is_single_panel_config(Path(config_dir)) global archConfigs archConfigs = {} for arch in file_io.supported_arch.keys(): ac = schema.ArchConfig() - if args.list_kernels: + if list_kernels: ac.panel_configs = file_io.top_stats_build_in_config else: arch_panel_config = ( - args.config_dir if single_panel_config else args.config_dir.joinpath(arch) + config_dir if single_panel_config else config_dir.joinpath(arch) ) ac.panel_configs = file_io.load_panel_configs(arch_panel_config) # TODO: filter_metrics should/might be one per arch # print(ac) - parser.build_dfs(ac, args.filter_metrics) + parser.build_dfs(ac, filter_metrics) archConfigs[arch] = ac + return archConfigs # Note: This return comes in handy for rocScope which borrows generate_configs() in its rocomni plugin + + +################################################ +# Core Functions +################################################ +def initialize_run(args, normalization_filter=None): + import pandas as pd + from collections import OrderedDict + from tabulate import tabulate + from omniperf_analyze.utils import schema + + # Fixme: cur_root.parent.joinpath('soc_params') + soc_params_dir = os.path.join(os.path.dirname(__file__), "..", "soc_params") + soc_spec_df = file_io.load_soc_params(soc_params_dir) + + generate_configs(args.config_dir, args.list_kernels, args.filter_metrics) + if args.list_metrics in file_io.supported_arch.keys(): print( tabulate( diff --git a/src/omniperf_analyze/utils/parser.py b/src/omniperf_analyze/utils/parser.py index d11cbbbfc..5fb03c39a 100644 --- a/src/omniperf_analyze/utils/parser.py +++ b/src/omniperf_analyze/utils/parser.py @@ -320,6 +320,26 @@ def update_normUnit_string(equation, unit): str(equation), ).capitalize() +def gen_counter_list(formula): + function_filter = {"MIN": None, "MAX": None, "AVG": None, "ROUND": None, "TO_INT": None, "GB": None, "STD": None, "GFLOP": None, "GOP": None, "OP": None, "CU": None, "NC": None, "UC": None, "CC": None, "RW": None, "GIOP": None} + + counters = [] + if not isinstance(formula,str): + return counters + try: + tree = ast.parse( + formula + .replace("$normUnit", "SQ_WAVES") + .replace("$denom", "SQ_WAVES") + .replace("$","") + ) + for node in ast.walk(tree): + if isinstance(node, ast.Name) and node.id.rstrip("_sum").isupper() and node.id not in function_filter: + counters.append(node.id.rstrip("_sum")) + except: + pass + return counters + def build_dfs(archConfigs, filter_metrics): """ @@ -338,6 +358,7 @@ def build_dfs(archConfigs, filter_metrics): d = {} metric_list = {} dfs_type = {} + metric_counters = {} for panel_id, panel in archConfigs.panel_configs.items(): for data_source in panel["data source"]: for type, data_cofig in data_source.items(): @@ -362,6 +383,7 @@ def build_dfs(archConfigs, filter_metrics): ) metric_idx = data_source_idx + "." + str(i) values = [] + eqn_content = [] if ( (not filter_metrics) @@ -378,6 +400,7 @@ def build_dfs(archConfigs, filter_metrics): for k, v in entries.items(): if k != "tips" and k != "coll_level" and k != "alias": values.append(v) + eqn_content.append(v) if "alias" in entries.keys(): values.append(entries["alias"]) @@ -396,6 +419,15 @@ def build_dfs(archConfigs, filter_metrics): # collect metric_list metric_list[metric_idx] = key.replace(" ", "_") + # generate mapping of counters and metrics + filter = {} + for formula in eqn_content: + if formula is not None and formula != "None": + for k in gen_counter_list(formula): + filter[k] = None + if len(filter) > 0: + metric_counters[key] = list(filter) + i += 1 df.set_index("Index", inplace=True) @@ -431,6 +463,7 @@ def build_dfs(archConfigs, filter_metrics): setattr(archConfigs, "dfs", d) setattr(archConfigs, "metric_list", metric_list) setattr(archConfigs, "dfs_type", dfs_type) + setattr(archConfigs, "metric_counters", metric_counters) def build_metric_value_string(dfs, dfs_type, normal_unit): @@ -469,7 +502,8 @@ def eval_metric(dfs, dfs_type, sys_info, soc_spec, raw_pmc_df, debug): # confirm no illogical counter values (only consider non-roofline runs) roof_only_run = sys_info.ip_blocks == "roofline" - if not roof_only_run and (raw_pmc_df["pmc_perf"]["GRBM_GUI_ACTIVE"] == 0).any(): + rocscope_run = sys_info.ip_blocks == "rocscope" + if not rocscope_run and not roof_only_run and (raw_pmc_df["pmc_perf"]["GRBM_GUI_ACTIVE"] == 0).any(): print("WARNING: Dectected GRBM_GUI_ACTIVE == 0\nHaulting execution.") sys.exit(1) @@ -711,12 +745,13 @@ def load_kernel_top(workload, dir): workload.dfs.update(tmp) -def load_table_data(workload, dir, is_gui, debug, verbose): +def load_table_data(workload, dir, is_gui, debug, verbose, skipKernelTop=False): """ Load data for all "raw_csv_table". Calculate mertric value for all "metric_table". """ - load_kernel_top(workload, dir) + if not skipKernelTop: + load_kernel_top(workload, dir) eval_metric( workload.dfs, diff --git a/src/omniperf_analyze/utils/schema.py b/src/omniperf_analyze/utils/schema.py index bcfc0bff5..6e147fcae 100644 --- a/src/omniperf_analyze/utils/schema.py +++ b/src/omniperf_analyze/utils/schema.py @@ -52,6 +52,8 @@ class ArchConfig: # [Index: Metric name] pairs metric_list: Dict[str, str] = field(default_factory=dict) + # [Metric name: Counters] pairs + metric_counters: Dict[str, list] = field(default_factory=dict) @dataclass class Workload: From 48f037e2423e579c8156470abcac65530eacfc27 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Fri, 9 Jun 2023 10:04:32 -0500 Subject: [PATCH 04/34] Comply to Python formatting Signed-off-by: colramos-amd --- src/omniperf_analyze/omniperf_analyze.py | 2 +- src/omniperf_analyze/utils/parser.py | 41 +++++++++++++++++++----- src/omniperf_analyze/utils/schema.py | 1 + src/utils/perfagg.py | 2 +- 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/omniperf_analyze/omniperf_analyze.py b/src/omniperf_analyze/omniperf_analyze.py index c15181c6f..6415ed285 100644 --- a/src/omniperf_analyze/omniperf_analyze.py +++ b/src/omniperf_analyze/omniperf_analyze.py @@ -73,7 +73,7 @@ def generate_configs(config_dir, list_kernels, filter_metrics): archConfigs[arch] = ac - return archConfigs # Note: This return comes in handy for rocScope which borrows generate_configs() in its rocomni plugin + return archConfigs # Note: This return comes in handy for rocScope which borrows generate_configs() in its rocomni plugin ################################################ diff --git a/src/omniperf_analyze/utils/parser.py b/src/omniperf_analyze/utils/parser.py index 5fb03c39a..b6573566b 100644 --- a/src/omniperf_analyze/utils/parser.py +++ b/src/omniperf_analyze/utils/parser.py @@ -320,26 +320,47 @@ def update_normUnit_string(equation, unit): str(equation), ).capitalize() + def gen_counter_list(formula): - function_filter = {"MIN": None, "MAX": None, "AVG": None, "ROUND": None, "TO_INT": None, "GB": None, "STD": None, "GFLOP": None, "GOP": None, "OP": None, "CU": None, "NC": None, "UC": None, "CC": None, "RW": None, "GIOP": None} + function_filter = { + "MIN": None, + "MAX": None, + "AVG": None, + "ROUND": None, + "TO_INT": None, + "GB": None, + "STD": None, + "GFLOP": None, + "GOP": None, + "OP": None, + "CU": None, + "NC": None, + "UC": None, + "CC": None, + "RW": None, + "GIOP": None, + } counters = [] - if not isinstance(formula,str): + if not isinstance(formula, str): return counters try: tree = ast.parse( - formula - .replace("$normUnit", "SQ_WAVES") + formula.replace("$normUnit", "SQ_WAVES") .replace("$denom", "SQ_WAVES") - .replace("$","") + .replace("$", "") ) for node in ast.walk(tree): - if isinstance(node, ast.Name) and node.id.rstrip("_sum").isupper() and node.id not in function_filter: + if ( + isinstance(node, ast.Name) + and node.id.rstrip("_sum").isupper() + and node.id not in function_filter + ): counters.append(node.id.rstrip("_sum")) except: pass return counters - + def build_dfs(archConfigs, filter_metrics): """ @@ -503,7 +524,11 @@ def eval_metric(dfs, dfs_type, sys_info, soc_spec, raw_pmc_df, debug): # confirm no illogical counter values (only consider non-roofline runs) roof_only_run = sys_info.ip_blocks == "roofline" rocscope_run = sys_info.ip_blocks == "rocscope" - if not rocscope_run and not roof_only_run and (raw_pmc_df["pmc_perf"]["GRBM_GUI_ACTIVE"] == 0).any(): + if ( + not rocscope_run + and not roof_only_run + and (raw_pmc_df["pmc_perf"]["GRBM_GUI_ACTIVE"] == 0).any() + ): print("WARNING: Dectected GRBM_GUI_ACTIVE == 0\nHaulting execution.") sys.exit(1) diff --git a/src/omniperf_analyze/utils/schema.py b/src/omniperf_analyze/utils/schema.py index 6e147fcae..f9b59868f 100644 --- a/src/omniperf_analyze/utils/schema.py +++ b/src/omniperf_analyze/utils/schema.py @@ -55,6 +55,7 @@ class ArchConfig: # [Metric name: Counters] pairs metric_counters: Dict[str, list] = field(default_factory=dict) + @dataclass class Workload: sys_info: pd.DataFrame = None diff --git a/src/utils/perfagg.py b/src/utils/perfagg.py index 651bcb86d..109fdecda 100755 --- a/src/utils/perfagg.py +++ b/src/utils/perfagg.py @@ -135,7 +135,7 @@ def join_prof(workload_dir, join_type, log_file, verbose, out=None): # Check for vgpr counter in ROCm >= 5.3 else: duplicate_cols["arch_vgpr"] = [col for col in df.columns if "arch_vgpr" in col] - duplicate_cols["accum_vgpr"] = [col for col in df.columns if "accum_vgpr" in col] + duplicate_cols["accum_vgpr"] = [col for col in df.columns if "accum_vgpr" in col] for key, cols in duplicate_cols.items(): print("Key is ", key) _df = df[cols] From 0b135be559666c99b133167417e1888bc0ad5948 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Wed, 21 Jun 2023 11:06:03 -0500 Subject: [PATCH 05/34] Add subsection title to System Speed-of-Light Signed-off-by: colramos-amd --- .../configs/gfx906/0200_system-speed-of-light.yaml | 1 + .../configs/gfx908/0200_system-speed-of-light.yaml | 1 + .../configs/gfx90a/0200_system-speed-of-light.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml b/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml index 74de040b2..986b2f0ae 100644 --- a/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml +++ b/src/omniperf_analyze/configs/gfx906/0200_system-speed-of-light.yaml @@ -11,6 +11,7 @@ Panel Config: data source: - metric_table: id: 201 + title: Speed-of-Light header: metric: Metric value: Value diff --git a/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml b/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml index 74de040b2..986b2f0ae 100644 --- a/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml +++ b/src/omniperf_analyze/configs/gfx908/0200_system-speed-of-light.yaml @@ -11,6 +11,7 @@ Panel Config: data source: - metric_table: id: 201 + title: Speed-of-Light header: metric: Metric value: Value diff --git a/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml b/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml index f10d7630f..20721ee1f 100644 --- a/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml +++ b/src/omniperf_analyze/configs/gfx90a/0200_system-speed-of-light.yaml @@ -11,6 +11,7 @@ Panel Config: data source: - metric_table: id: 201 + title: Speed-of-Light header: metric: Metric value: Value From 5e52983bbf800eed0dfd32b4e0a27c3781713e57 Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Mon, 26 Jun 2023 15:30:38 -0500 Subject: [PATCH 06/34] Extend filtering into timestamps.csv (#80) Signed-off-by: coleramos425 --- src/omniperf | 34 ++-------------------------------- src/utils/perfagg.py | 9 +++++++++ 2 files changed, 11 insertions(+), 32 deletions(-) diff --git a/src/omniperf b/src/omniperf index 3b38e419b..e611547d4 100755 --- a/src/omniperf +++ b/src/omniperf @@ -439,23 +439,7 @@ def characterize_app(args, VER): else: run_prof(fname, workload_dir, perfmon_dir, app_cmd, args.target, log, args.verbose) - - - # run again with timestamps - success, output = capture_subprocess_output( - [ - rocprof_cmd, - # "-i", fname, - # "-m", perfmon_dir + "/" + "metrics.xml", - "--timestamp", - "on", - "-o", - workload_dir + "/" + "timestamps.csv", - '"' + app_cmd + '"', - ] - ) - log.write(output) - # Update pmc_perf.csv timestamps + # Update timestamps replace_timestamps(workload_dir, log) # Manually join each pmc_perf*.csv output @@ -676,21 +660,7 @@ def omniperf_profile(args, VER): else: run_prof(fname, workload_dir, perfmon_dir, args.remaining, args.target, log, args.verbose) - # run again with timestamps - success, output = capture_subprocess_output( - [ - rocprof_cmd, - # "-i", fname, - # "-m", perfmon_dir + "/" + "metrics.xml", - "--timestamp", - "on", - "-o", - workload_dir + "/" + "timestamps.csv", - '"' + args.remaining + '"', - ] - ) - log.write(output) - # Update pmc_perf.csv timestamps + # Update timestamps replace_timestamps(workload_dir, log) # Manually join each pmc_perf*.csv output diff --git a/src/utils/perfagg.py b/src/utils/perfagg.py index 109fdecda..1c80a22a9 100755 --- a/src/utils/perfagg.py +++ b/src/utils/perfagg.py @@ -345,6 +345,15 @@ def perfmon_coalesce(pmc_files_list, workload_dir, soc): # initial counter in this channel pmc_list["TCC2"][str(ch)] = [counter] + + # add a timestamp file + fd = open(workload_perfmon_dir + "/timestamps.txt", "w") + fd.write("pmc:\n\n") + fd.write("gpu:\n") + fd.write("range:\n") + fd.write("kernel:\n") + fd.close() + # sort the per channel counter, so that same counter in all channels can be aligned for ch in range(perfmon_config[soc]["TCC_channels"]): pmc_list["TCC2"][str(ch)].sort() From 8ae839533db7874880200c1fd98c0ee145e1dd4b Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Mon, 26 Jun 2023 15:38:51 -0500 Subject: [PATCH 07/34] Comply to Python formatting Signed-off-by: coleramos425 --- src/utils/perfagg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/utils/perfagg.py b/src/utils/perfagg.py index 1c80a22a9..59460bc80 100755 --- a/src/utils/perfagg.py +++ b/src/utils/perfagg.py @@ -345,7 +345,6 @@ def perfmon_coalesce(pmc_files_list, workload_dir, soc): # initial counter in this channel pmc_list["TCC2"][str(ch)] = [counter] - # add a timestamp file fd = open(workload_perfmon_dir + "/timestamps.txt", "w") fd.write("pmc:\n\n") @@ -353,7 +352,7 @@ def perfmon_coalesce(pmc_files_list, workload_dir, soc): fd.write("range:\n") fd.write("kernel:\n") fd.close() - + # sort the per channel counter, so that same counter in all channels can be aligned for ch in range(perfmon_config[soc]["TCC_channels"]): pmc_list["TCC2"][str(ch)].sort() From 607d9f79d3345abc60b1cc9d79b1ea4d6a98730c Mon Sep 17 00:00:00 2001 From: Nicholas Curtis Date: Wed, 7 Jun 2023 10:23:49 -0400 Subject: [PATCH 08/34] Add options to enable latexpdf builds Signed-off-by: Nicholas Curtis --- src/docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/docs/conf.py b/src/docs/conf.py index af0003fb7..d97f79bb7 100644 --- a/src/docs/conf.py +++ b/src/docs/conf.py @@ -90,8 +90,8 @@ def install(package): pygments_style = None # options for latex output -latex_engine = "lualatex" -latex_show_urls = "footnote" +latex_engine = 'lualatex' +latex_show_urls = 'footnote' # -- Options for HTML output ------------------------------------------------- From a6a0fc4d5ea9faa7dd05da11e470296bbf252a63 Mon Sep 17 00:00:00 2001 From: Nicholas Curtis Date: Wed, 7 Jun 2023 15:19:56 -0400 Subject: [PATCH 09/34] apply formatting Signed-off-by: Nicholas Curtis --- src/docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/docs/conf.py b/src/docs/conf.py index d97f79bb7..af0003fb7 100644 --- a/src/docs/conf.py +++ b/src/docs/conf.py @@ -90,8 +90,8 @@ def install(package): pygments_style = None # options for latex output -latex_engine = 'lualatex' -latex_show_urls = 'footnote' +latex_engine = "lualatex" +latex_show_urls = "footnote" # -- Options for HTML output ------------------------------------------------- From 931a333be16d53b1c3c586e87a02f0ee20f6447f Mon Sep 17 00:00:00 2001 From: "Karl W. Schulz" Date: Fri, 30 Jun 2023 15:01:57 -0500 Subject: [PATCH 10/34] updating path for rocm repo to supported rhel8 release (8.8) Signed-off-by: Karl W. Schulz --- docker/rhel8/rocm.repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/rhel8/rocm.repo b/docker/rhel8/rocm.repo index 17171d755..8b2048978 100644 --- a/docker/rhel8/rocm.repo +++ b/docker/rhel8/rocm.repo @@ -7,7 +7,7 @@ gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key [amdgpu] name=amdgpu -baseurl=https://repo.radeon.com/amdgpu/latest/rhel/8.5/main/x86_64 +baseurl=https://repo.radeon.com/amdgpu/latest/rhel/8.8/main/x86_64 enabled=1 gpgcheck=1 gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key \ No newline at end of file From dcd2595412d085886c771d4cef6ddbfda20e73e8 Mon Sep 17 00:00:00 2001 From: JoseSantosAMD Date: Mon, 10 Jul 2023 16:26:56 -0500 Subject: [PATCH 11/34] Fixed Units inconsistencies - Table 10: Units were output as "$normUnit" now they are instr + normUnit - Table 16: Changed to Req per $normUnit Signed-off-by: JoseSantosAMD --- .../gfx906/1000_compute-unit-instruction-mix.yaml | 4 ++-- .../configs/gfx906/1600_L1_cache.yaml | 8 ++++---- .../configs/gfx906/1800_L2_cache_per_channel.yaml | 14 +++++++------- .../configs/gfx908/1600_L1_cache.yaml | 8 ++++---- .../configs/gfx908/1800_L2_cache_per_channel.yaml | 14 +++++++------- .../configs/gfx90a/1600_L1_cache.yaml | 8 ++++---- .../configs/gfx90a/1800_L2_cache_per_channel.yaml | 14 +++++++------- 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/omniperf_analyze/configs/gfx906/1000_compute-unit-instruction-mix.yaml b/src/omniperf_analyze/configs/gfx906/1000_compute-unit-instruction-mix.yaml index 6d28cb2a4..c2e2f7f3a 100644 --- a/src/omniperf_analyze/configs/gfx906/1000_compute-unit-instruction-mix.yaml +++ b/src/omniperf_analyze/configs/gfx906/1000_compute-unit-instruction-mix.yaml @@ -34,7 +34,7 @@ Panel Config: avg: AVG((SQ_INSTS_LDS / $denom)) min: MIN((SQ_INSTS_LDS / $denom)) max: MAX((SQ_INSTS_LDS / $denom)) - unit: $normUnit + unit: (instr + $normUnit) tips: VALU - MFMA: avg: None # No HW module @@ -78,7 +78,7 @@ Panel Config: metric: INT-32: count: None # No perf counter - unit: $normUnit + unit: (instr + $normUnit) tips: INT-64: count: None # No perf counter diff --git a/src/omniperf_analyze/configs/gfx906/1600_L1_cache.yaml b/src/omniperf_analyze/configs/gfx906/1600_L1_cache.yaml index 1713068d2..1e05b3e4c 100644 --- a/src/omniperf_analyze/configs/gfx906/1600_L1_cache.yaml +++ b/src/omniperf_analyze/configs/gfx906/1600_L1_cache.yaml @@ -361,7 +361,7 @@ Panel Config: mean: AVG((TCP_UTCL1_REQUEST_sum / $denom)) min: MIN((TCP_UTCL1_REQUEST_sum / $denom)) max: MAX((TCP_UTCL1_REQUEST_sum / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: Hit Ratio: mean: AVG((((100 * TCP_UTCL1_TRANSLATION_HIT_sum) / TCP_UTCL1_REQUEST_sum) if @@ -376,17 +376,17 @@ Panel Config: mean: AVG((TCP_UTCL1_TRANSLATION_HIT_sum / $denom)) min: MIN((TCP_UTCL1_TRANSLATION_HIT_sum / $denom)) max: MAX((TCP_UTCL1_TRANSLATION_HIT_sum / $denom)) - units: ( + $normUnit) + units: (Hits + $normUnit) tips: Misses (Translation): mean: AVG((TCP_UTCL1_TRANSLATION_MISS_sum / $denom)) min: MIN((TCP_UTCL1_TRANSLATION_MISS_sum / $denom)) max: MAX((TCP_UTCL1_TRANSLATION_MISS_sum / $denom)) - units: ( + $normUnit) + units: (Misses + $normUnit) tips: Misses (Permission): mean: AVG((TCP_UTCL1_PERMISSION_MISS_sum / $denom)) min: MIN((TCP_UTCL1_PERMISSION_MISS_sum / $denom)) max: MAX((TCP_UTCL1_PERMISSION_MISS_sum / $denom)) - units: ( + $normUnit) + units: (Misses + $normUnit) tips: diff --git a/src/omniperf_analyze/configs/gfx906/1800_L2_cache_per_channel.yaml b/src/omniperf_analyze/configs/gfx906/1800_L2_cache_per_channel.yaml index 95bba22e8..08a9a9f76 100644 --- a/src/omniperf_analyze/configs/gfx906/1800_L2_cache_per_channel.yaml +++ b/src/omniperf_analyze/configs/gfx906/1800_L2_cache_per_channel.yaml @@ -204,7 +204,7 @@ Panel Config: + TO_INT(TCC_REQ[22])) + TO_INT(TCC_REQ[23])) + TO_INT(TCC_REQ[24])) + TO_INT(TCC_REQ[25])) + TO_INT(TCC_REQ[26])) + TO_INT(TCC_REQ[27])) + TO_INT(TCC_REQ[28])) + TO_INT(TCC_REQ[29])) + TO_INT(TCC_REQ[30])) + TO_INT(TCC_REQ[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: L1 - L2 Read Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_READ[0]) + TO_INT(TCC_READ[1])) @@ -247,7 +247,7 @@ Panel Config: + TO_INT(TCC_READ[24])) + TO_INT(TCC_READ[25])) + TO_INT(TCC_READ[26])) + TO_INT(TCC_READ[27])) + TO_INT(TCC_READ[28])) + TO_INT(TCC_READ[29])) + TO_INT(TCC_READ[30])) + TO_INT(TCC_READ[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: L1 - L2 Write Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_WRITE[0]) + TO_INT(TCC_WRITE[1])) @@ -294,7 +294,7 @@ Panel Config: + TO_INT(TCC_WRITE[24])) + TO_INT(TCC_WRITE[25])) + TO_INT(TCC_WRITE[26])) + TO_INT(TCC_WRITE[27])) + TO_INT(TCC_WRITE[28])) + TO_INT(TCC_WRITE[29])) + TO_INT(TCC_WRITE[30])) + TO_INT(TCC_WRITE[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: L1 - L2 Atomic Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_ATOMIC[0]) + TO_INT(TCC_ATOMIC[1])) @@ -345,7 +345,7 @@ Panel Config: + TO_INT(TCC_ATOMIC[26])) + TO_INT(TCC_ATOMIC[27])) + TO_INT(TCC_ATOMIC[28])) + TO_INT(TCC_ATOMIC[29])) + TO_INT(TCC_ATOMIC[30])) + TO_INT(TCC_ATOMIC[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: L2 - EA Read Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_EA_RDREQ[0]) + TO_INT(TCC_EA_RDREQ[1])) @@ -396,7 +396,7 @@ Panel Config: + TO_INT(TCC_EA_RDREQ[26])) + TO_INT(TCC_EA_RDREQ[27])) + TO_INT(TCC_EA_RDREQ[28])) + TO_INT(TCC_EA_RDREQ[29])) + TO_INT(TCC_EA_RDREQ[30])) + TO_INT(TCC_EA_RDREQ[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: L2 - EA Write Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_EA_WRREQ[0]) + TO_INT(TCC_EA_WRREQ[1])) @@ -447,7 +447,7 @@ Panel Config: + TO_INT(TCC_EA_WRREQ[26])) + TO_INT(TCC_EA_WRREQ[27])) + TO_INT(TCC_EA_WRREQ[28])) + TO_INT(TCC_EA_WRREQ[29])) + TO_INT(TCC_EA_WRREQ[30])) + TO_INT(TCC_EA_WRREQ[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: L2 - EA Atomic Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_EA_ATOMIC[0]) + TO_INT(TCC_EA_ATOMIC[1])) @@ -498,7 +498,7 @@ Panel Config: + TO_INT(TCC_EA_ATOMIC[26])) + TO_INT(TCC_EA_ATOMIC[27])) + TO_INT(TCC_EA_ATOMIC[28])) + TO_INT(TCC_EA_ATOMIC[29])) + TO_INT(TCC_EA_ATOMIC[30])) + TO_INT(TCC_EA_ATOMIC[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: L2 - EA Read Lat: mean: AVG((((((((((((((((((((((((((((((((((TCC_EA_RDREQ_LEVEL[0] + TCC_EA_RDREQ_LEVEL[1]) diff --git a/src/omniperf_analyze/configs/gfx908/1600_L1_cache.yaml b/src/omniperf_analyze/configs/gfx908/1600_L1_cache.yaml index 4ff3fd4d4..f65309a31 100644 --- a/src/omniperf_analyze/configs/gfx908/1600_L1_cache.yaml +++ b/src/omniperf_analyze/configs/gfx908/1600_L1_cache.yaml @@ -361,7 +361,7 @@ Panel Config: mean: AVG((TCP_UTCL1_REQUEST_sum / $denom)) min: MIN((TCP_UTCL1_REQUEST_sum / $denom)) max: MAX((TCP_UTCL1_REQUEST_sum / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: Hit Ratio: mean: AVG((((100 * TCP_UTCL1_TRANSLATION_HIT_sum) / TCP_UTCL1_REQUEST_sum) if @@ -376,17 +376,17 @@ Panel Config: mean: AVG((TCP_UTCL1_TRANSLATION_HIT_sum / $denom)) min: MIN((TCP_UTCL1_TRANSLATION_HIT_sum / $denom)) max: MAX((TCP_UTCL1_TRANSLATION_HIT_sum / $denom)) - units: ( + $normUnit) + units: (Hits + $normUnit) tips: Misses (Translation): mean: AVG((TCP_UTCL1_TRANSLATION_MISS_sum / $denom)) min: MIN((TCP_UTCL1_TRANSLATION_MISS_sum / $denom)) max: MAX((TCP_UTCL1_TRANSLATION_MISS_sum / $denom)) - units: ( + $normUnit) + units: (Misses + $normUnit) tips: Misses (Permission): mean: AVG((TCP_UTCL1_PERMISSION_MISS_sum / $denom)) min: MIN((TCP_UTCL1_PERMISSION_MISS_sum / $denom)) max: MAX((TCP_UTCL1_PERMISSION_MISS_sum / $denom)) - units: ( + $normUnit) + units: (Misses + $normUnit) tips: diff --git a/src/omniperf_analyze/configs/gfx908/1800_L2_cache_per_channel.yaml b/src/omniperf_analyze/configs/gfx908/1800_L2_cache_per_channel.yaml index e68511e9e..3acee5740 100644 --- a/src/omniperf_analyze/configs/gfx908/1800_L2_cache_per_channel.yaml +++ b/src/omniperf_analyze/configs/gfx908/1800_L2_cache_per_channel.yaml @@ -204,7 +204,7 @@ Panel Config: + TO_INT(TCC_REQ[22])) + TO_INT(TCC_REQ[23])) + TO_INT(TCC_REQ[24])) + TO_INT(TCC_REQ[25])) + TO_INT(TCC_REQ[26])) + TO_INT(TCC_REQ[27])) + TO_INT(TCC_REQ[28])) + TO_INT(TCC_REQ[29])) + TO_INT(TCC_REQ[30])) + TO_INT(TCC_REQ[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: L1 - L2 Read Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_READ[0]) + TO_INT(TCC_READ[1])) @@ -247,7 +247,7 @@ Panel Config: + TO_INT(TCC_READ[24])) + TO_INT(TCC_READ[25])) + TO_INT(TCC_READ[26])) + TO_INT(TCC_READ[27])) + TO_INT(TCC_READ[28])) + TO_INT(TCC_READ[29])) + TO_INT(TCC_READ[30])) + TO_INT(TCC_READ[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: L1 - L2 Write Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_WRITE[0]) + TO_INT(TCC_WRITE[1])) @@ -294,7 +294,7 @@ Panel Config: + TO_INT(TCC_WRITE[24])) + TO_INT(TCC_WRITE[25])) + TO_INT(TCC_WRITE[26])) + TO_INT(TCC_WRITE[27])) + TO_INT(TCC_WRITE[28])) + TO_INT(TCC_WRITE[29])) + TO_INT(TCC_WRITE[30])) + TO_INT(TCC_WRITE[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: L1 - L2 Atomic Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_ATOMIC[0]) + TO_INT(TCC_ATOMIC[1])) @@ -345,7 +345,7 @@ Panel Config: + TO_INT(TCC_ATOMIC[26])) + TO_INT(TCC_ATOMIC[27])) + TO_INT(TCC_ATOMIC[28])) + TO_INT(TCC_ATOMIC[29])) + TO_INT(TCC_ATOMIC[30])) + TO_INT(TCC_ATOMIC[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: L2 - EA Read Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_EA_RDREQ[0]) + TO_INT(TCC_EA_RDREQ[1])) @@ -396,7 +396,7 @@ Panel Config: + TO_INT(TCC_EA_RDREQ[26])) + TO_INT(TCC_EA_RDREQ[27])) + TO_INT(TCC_EA_RDREQ[28])) + TO_INT(TCC_EA_RDREQ[29])) + TO_INT(TCC_EA_RDREQ[30])) + TO_INT(TCC_EA_RDREQ[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: L2 - EA Write Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_EA_WRREQ[0]) + TO_INT(TCC_EA_WRREQ[1])) @@ -447,7 +447,7 @@ Panel Config: + TO_INT(TCC_EA_WRREQ[26])) + TO_INT(TCC_EA_WRREQ[27])) + TO_INT(TCC_EA_WRREQ[28])) + TO_INT(TCC_EA_WRREQ[29])) + TO_INT(TCC_EA_WRREQ[30])) + TO_INT(TCC_EA_WRREQ[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: L2 - EA Atomic Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_EA_ATOMIC[0]) + TO_INT(TCC_EA_ATOMIC[1])) @@ -498,7 +498,7 @@ Panel Config: + TO_INT(TCC_EA_ATOMIC[26])) + TO_INT(TCC_EA_ATOMIC[27])) + TO_INT(TCC_EA_ATOMIC[28])) + TO_INT(TCC_EA_ATOMIC[29])) + TO_INT(TCC_EA_ATOMIC[30])) + TO_INT(TCC_EA_ATOMIC[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: L2 - EA Read Lat: mean: AVG((((((((((((((((((((((((((((((((((TCC_EA_RDREQ_LEVEL[0] + TCC_EA_RDREQ_LEVEL[1]) diff --git a/src/omniperf_analyze/configs/gfx90a/1600_L1_cache.yaml b/src/omniperf_analyze/configs/gfx90a/1600_L1_cache.yaml index 985be3803..917cb3aa0 100644 --- a/src/omniperf_analyze/configs/gfx90a/1600_L1_cache.yaml +++ b/src/omniperf_analyze/configs/gfx90a/1600_L1_cache.yaml @@ -361,7 +361,7 @@ Panel Config: mean: AVG((TCP_UTCL1_REQUEST_sum / $denom)) min: MIN((TCP_UTCL1_REQUEST_sum / $denom)) max: MAX((TCP_UTCL1_REQUEST_sum / $denom)) - units: ( + $normUnit) + units: (Req + $normUnit) tips: Hit Ratio: mean: AVG((((100 * TCP_UTCL1_TRANSLATION_HIT_sum) / TCP_UTCL1_REQUEST_sum) if @@ -376,17 +376,17 @@ Panel Config: mean: AVG((TCP_UTCL1_TRANSLATION_HIT_sum / $denom)) min: MIN((TCP_UTCL1_TRANSLATION_HIT_sum / $denom)) max: MAX((TCP_UTCL1_TRANSLATION_HIT_sum / $denom)) - units: ( + $normUnit) + units: (Hits + $normUnit) tips: Misses (Translation): mean: AVG((TCP_UTCL1_TRANSLATION_MISS_sum / $denom)) min: MIN((TCP_UTCL1_TRANSLATION_MISS_sum / $denom)) max: MAX((TCP_UTCL1_TRANSLATION_MISS_sum / $denom)) - units: ( + $normUnit) + units: (Misses + $normUnit) tips: Misses (Permission): mean: AVG((TCP_UTCL1_PERMISSION_MISS_sum / $denom)) min: MIN((TCP_UTCL1_PERMISSION_MISS_sum / $denom)) max: MAX((TCP_UTCL1_PERMISSION_MISS_sum / $denom)) - units: ( + $normUnit) + units: (Misses + $normUnit) tips: diff --git a/src/omniperf_analyze/configs/gfx90a/1800_L2_cache_per_channel.yaml b/src/omniperf_analyze/configs/gfx90a/1800_L2_cache_per_channel.yaml index a5bf6fa25..094df5b19 100644 --- a/src/omniperf_analyze/configs/gfx90a/1800_L2_cache_per_channel.yaml +++ b/src/omniperf_analyze/configs/gfx90a/1800_L2_cache_per_channel.yaml @@ -204,7 +204,7 @@ Panel Config: + TO_INT(TCC_REQ[22])) + TO_INT(TCC_REQ[23])) + TO_INT(TCC_REQ[24])) + TO_INT(TCC_REQ[25])) + TO_INT(TCC_REQ[26])) + TO_INT(TCC_REQ[27])) + TO_INT(TCC_REQ[28])) + TO_INT(TCC_REQ[29])) + TO_INT(TCC_REQ[30])) + TO_INT(TCC_REQ[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (req + $normUnit) tips: L1 - L2 Read Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_READ[0]) + TO_INT(TCC_READ[1])) @@ -247,7 +247,7 @@ Panel Config: + TO_INT(TCC_READ[24])) + TO_INT(TCC_READ[25])) + TO_INT(TCC_READ[26])) + TO_INT(TCC_READ[27])) + TO_INT(TCC_READ[28])) + TO_INT(TCC_READ[29])) + TO_INT(TCC_READ[30])) + TO_INT(TCC_READ[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (req + $normUnit) tips: L1 - L2 Write Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_WRITE[0]) + TO_INT(TCC_WRITE[1])) @@ -294,7 +294,7 @@ Panel Config: + TO_INT(TCC_WRITE[24])) + TO_INT(TCC_WRITE[25])) + TO_INT(TCC_WRITE[26])) + TO_INT(TCC_WRITE[27])) + TO_INT(TCC_WRITE[28])) + TO_INT(TCC_WRITE[29])) + TO_INT(TCC_WRITE[30])) + TO_INT(TCC_WRITE[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (req + $normUnit) tips: L1 - L2 Atomic Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_ATOMIC[0]) + TO_INT(TCC_ATOMIC[1])) @@ -345,7 +345,7 @@ Panel Config: + TO_INT(TCC_ATOMIC[26])) + TO_INT(TCC_ATOMIC[27])) + TO_INT(TCC_ATOMIC[28])) + TO_INT(TCC_ATOMIC[29])) + TO_INT(TCC_ATOMIC[30])) + TO_INT(TCC_ATOMIC[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (req + $normUnit) tips: L2 - EA Read Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_EA_RDREQ[0]) + TO_INT(TCC_EA_RDREQ[1])) @@ -396,7 +396,7 @@ Panel Config: + TO_INT(TCC_EA_RDREQ[26])) + TO_INT(TCC_EA_RDREQ[27])) + TO_INT(TCC_EA_RDREQ[28])) + TO_INT(TCC_EA_RDREQ[29])) + TO_INT(TCC_EA_RDREQ[30])) + TO_INT(TCC_EA_RDREQ[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (req + $normUnit) tips: L2 - EA Write Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_EA_WRREQ[0]) + TO_INT(TCC_EA_WRREQ[1])) @@ -447,7 +447,7 @@ Panel Config: + TO_INT(TCC_EA_WRREQ[26])) + TO_INT(TCC_EA_WRREQ[27])) + TO_INT(TCC_EA_WRREQ[28])) + TO_INT(TCC_EA_WRREQ[29])) + TO_INT(TCC_EA_WRREQ[30])) + TO_INT(TCC_EA_WRREQ[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (req + $normUnit) tips: L2 - EA Atomic Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_EA_ATOMIC[0]) + TO_INT(TCC_EA_ATOMIC[1])) @@ -498,7 +498,7 @@ Panel Config: + TO_INT(TCC_EA_ATOMIC[26])) + TO_INT(TCC_EA_ATOMIC[27])) + TO_INT(TCC_EA_ATOMIC[28])) + TO_INT(TCC_EA_ATOMIC[29])) + TO_INT(TCC_EA_ATOMIC[30])) + TO_INT(TCC_EA_ATOMIC[31])) / 32) / $denom)) - units: ( + $normUnit) + units: (req + $normUnit) tips: L2 - EA Read Lat: mean: AVG((((((((((((((((((((((((((((((((((TCC_EA_RDREQ_LEVEL[0] + TCC_EA_RDREQ_LEVEL[1]) From 02c647301d817092a3a2176550278efc26701a40 Mon Sep 17 00:00:00 2001 From: Cole Ramos Date: Tue, 11 Jul 2023 13:27:46 -0500 Subject: [PATCH 12/34] Update 1800_L2_cache_per_channel.yaml Capitalizing for consistency Signed-off-by: Cole Ramos --- .../configs/gfx90a/1800_L2_cache_per_channel.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/omniperf_analyze/configs/gfx90a/1800_L2_cache_per_channel.yaml b/src/omniperf_analyze/configs/gfx90a/1800_L2_cache_per_channel.yaml index 094df5b19..93fc2b412 100644 --- a/src/omniperf_analyze/configs/gfx90a/1800_L2_cache_per_channel.yaml +++ b/src/omniperf_analyze/configs/gfx90a/1800_L2_cache_per_channel.yaml @@ -204,7 +204,7 @@ Panel Config: + TO_INT(TCC_REQ[22])) + TO_INT(TCC_REQ[23])) + TO_INT(TCC_REQ[24])) + TO_INT(TCC_REQ[25])) + TO_INT(TCC_REQ[26])) + TO_INT(TCC_REQ[27])) + TO_INT(TCC_REQ[28])) + TO_INT(TCC_REQ[29])) + TO_INT(TCC_REQ[30])) + TO_INT(TCC_REQ[31])) / 32) / $denom)) - units: (req + $normUnit) + units: (Req + $normUnit) tips: L1 - L2 Read Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_READ[0]) + TO_INT(TCC_READ[1])) @@ -294,7 +294,7 @@ Panel Config: + TO_INT(TCC_WRITE[24])) + TO_INT(TCC_WRITE[25])) + TO_INT(TCC_WRITE[26])) + TO_INT(TCC_WRITE[27])) + TO_INT(TCC_WRITE[28])) + TO_INT(TCC_WRITE[29])) + TO_INT(TCC_WRITE[30])) + TO_INT(TCC_WRITE[31])) / 32) / $denom)) - units: (req + $normUnit) + units: (Req + $normUnit) tips: L1 - L2 Atomic Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_ATOMIC[0]) + TO_INT(TCC_ATOMIC[1])) @@ -396,7 +396,7 @@ Panel Config: + TO_INT(TCC_EA_RDREQ[26])) + TO_INT(TCC_EA_RDREQ[27])) + TO_INT(TCC_EA_RDREQ[28])) + TO_INT(TCC_EA_RDREQ[29])) + TO_INT(TCC_EA_RDREQ[30])) + TO_INT(TCC_EA_RDREQ[31])) / 32) / $denom)) - units: (req + $normUnit) + units: (Req + $normUnit) tips: L2 - EA Write Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_EA_WRREQ[0]) + TO_INT(TCC_EA_WRREQ[1])) @@ -447,7 +447,7 @@ Panel Config: + TO_INT(TCC_EA_WRREQ[26])) + TO_INT(TCC_EA_WRREQ[27])) + TO_INT(TCC_EA_WRREQ[28])) + TO_INT(TCC_EA_WRREQ[29])) + TO_INT(TCC_EA_WRREQ[30])) + TO_INT(TCC_EA_WRREQ[31])) / 32) / $denom)) - units: (req + $normUnit) + units: (Req + $normUnit) tips: L2 - EA Atomic Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_EA_ATOMIC[0]) + TO_INT(TCC_EA_ATOMIC[1])) @@ -498,7 +498,7 @@ Panel Config: + TO_INT(TCC_EA_ATOMIC[26])) + TO_INT(TCC_EA_ATOMIC[27])) + TO_INT(TCC_EA_ATOMIC[28])) + TO_INT(TCC_EA_ATOMIC[29])) + TO_INT(TCC_EA_ATOMIC[30])) + TO_INT(TCC_EA_ATOMIC[31])) / 32) / $denom)) - units: (req + $normUnit) + units: (Req + $normUnit) tips: L2 - EA Read Lat: mean: AVG((((((((((((((((((((((((((((((((((TCC_EA_RDREQ_LEVEL[0] + TCC_EA_RDREQ_LEVEL[1]) From 3daad292f92d7fec2eaf0150abb09b8f0aee3ef8 Mon Sep 17 00:00:00 2001 From: Cole Ramos Date: Tue, 11 Jul 2023 13:29:07 -0500 Subject: [PATCH 13/34] Update 1800_L2_cache_per_channel.yaml Capitalizing for consistency Signed-off-by: Cole Ramos --- .../configs/gfx90a/1800_L2_cache_per_channel.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/omniperf_analyze/configs/gfx90a/1800_L2_cache_per_channel.yaml b/src/omniperf_analyze/configs/gfx90a/1800_L2_cache_per_channel.yaml index 93fc2b412..f13647847 100644 --- a/src/omniperf_analyze/configs/gfx90a/1800_L2_cache_per_channel.yaml +++ b/src/omniperf_analyze/configs/gfx90a/1800_L2_cache_per_channel.yaml @@ -247,7 +247,7 @@ Panel Config: + TO_INT(TCC_READ[24])) + TO_INT(TCC_READ[25])) + TO_INT(TCC_READ[26])) + TO_INT(TCC_READ[27])) + TO_INT(TCC_READ[28])) + TO_INT(TCC_READ[29])) + TO_INT(TCC_READ[30])) + TO_INT(TCC_READ[31])) / 32) / $denom)) - units: (req + $normUnit) + units: (Req + $normUnit) tips: L1 - L2 Write Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_WRITE[0]) + TO_INT(TCC_WRITE[1])) @@ -345,7 +345,7 @@ Panel Config: + TO_INT(TCC_ATOMIC[26])) + TO_INT(TCC_ATOMIC[27])) + TO_INT(TCC_ATOMIC[28])) + TO_INT(TCC_ATOMIC[29])) + TO_INT(TCC_ATOMIC[30])) + TO_INT(TCC_ATOMIC[31])) / 32) / $denom)) - units: (req + $normUnit) + units: (Req + $normUnit) tips: L2 - EA Read Req: mean: AVG((((((((((((((((((((((((((((((((((TO_INT(TCC_EA_RDREQ[0]) + TO_INT(TCC_EA_RDREQ[1])) From 33ad9ffca327aec1e72bbfd8c87cd7ad056343e5 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Tue, 11 Jul 2023 14:11:38 -0500 Subject: [PATCH 14/34] Rearranging build_df func to optimize ArchConfig for rocomni plugin Signed-off-by: colramos-amd --- src/omniperf_analyze/utils/parser.py | 57 +++++++++++++++++++--------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/src/omniperf_analyze/utils/parser.py b/src/omniperf_analyze/utils/parser.py index b6573566b..025745b17 100644 --- a/src/omniperf_analyze/utils/parser.py +++ b/src/omniperf_analyze/utils/parser.py @@ -339,11 +339,25 @@ def gen_counter_list(formula): "CC": None, "RW": None, "GIOP": None, + "GFLOPs": None, } + built_in_counter=[ + "lds", + "grd", + "wgr", + "arch_vgpr", + "accum_vgpr", + "sgpr", + "scr", + "BeginNs", + "EndNs" + ] + + visited = False counters = [] if not isinstance(formula, str): - return counters + return visited, counters try: tree = ast.parse( formula.replace("$normUnit", "SQ_WAVES") @@ -351,15 +365,17 @@ def gen_counter_list(formula): .replace("$", "") ) for node in ast.walk(tree): - if ( - isinstance(node, ast.Name) - and node.id.rstrip("_sum").isupper() - and node.id not in function_filter - ): - counters.append(node.id.rstrip("_sum")) + if isinstance(node, ast.Name): + val = str(node.id)[:-4] if str(node.id).endswith("_sum") else str(node.id) + if (val.isupper() and val not in function_filter): + counters.append(val) + visited = True + if val in built_in_counter: + visited = True except: pass - return counters + + return visited, counters def build_dfs(archConfigs, filter_metrics): @@ -381,9 +397,14 @@ def build_dfs(archConfigs, filter_metrics): dfs_type = {} metric_counters = {} for panel_id, panel in archConfigs.panel_configs.items(): + panel_idx = str(panel_id // 100) for data_source in panel["data source"]: for type, data_cofig in data_source.items(): if type == "metric_table": + metric_list[panel_idx] = panel["title"] + table_idx = panel_idx + "." + str(data_cofig["id"] % 100) + metric_list[table_idx] = data_cofig["title"] + headers = ["Index"] for key, tile in data_cofig["header"].items(): if key != "tips": @@ -397,12 +418,7 @@ def build_dfs(archConfigs, filter_metrics): i = 0 for key, entries in data_cofig["metric"].items(): - data_source_idx = ( - str(data_cofig["id"] // 100) - + "." - + str(data_cofig["id"] % 100) - ) - metric_idx = data_source_idx + "." + str(i) + metric_idx = table_idx + "." + str(i) values = [] eqn_content = [] @@ -411,7 +427,7 @@ def build_dfs(archConfigs, filter_metrics): or (metric_idx in filter_metrics) # no filter or # metric in filter # the whole table in filter - (data_source_idx in filter_metrics) + (table_idx in filter_metrics) or # the whole IP block in filter (str(panel_id // 100) in filter_metrics) @@ -439,14 +455,19 @@ def build_dfs(archConfigs, filter_metrics): df = pd.concat([df, df_new_row]) # collect metric_list - metric_list[metric_idx] = key.replace(" ", "_") + metric_list[metric_idx] = key # generate mapping of counters and metrics filter = {} + _visited = False for formula in eqn_content: if formula is not None and formula != "None": - for k in gen_counter_list(formula): + visited, counters = gen_counter_list(formula) + if visited: + _visited = True + for k in counters: filter[k] = None - if len(filter) > 0: + + if len(filter) > 0 or _visited: metric_counters[key] = list(filter) i += 1 From 2c2b384ca236dd8c9b56f7740c5eb3fff816121d Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Tue, 11 Jul 2023 14:13:09 -0500 Subject: [PATCH 15/34] Abstract perfmon coalesing for useage in rocomni plugin Signed-off-by: colramos-amd --- src/utils/perfagg.py | 173 ++++++++++++++++++++++++++++--------------- 1 file changed, 115 insertions(+), 58 deletions(-) diff --git a/src/utils/perfagg.py b/src/utils/perfagg.py index 59460bc80..8e95482c5 100755 --- a/src/utils/perfagg.py +++ b/src/utils/perfagg.py @@ -256,6 +256,96 @@ def pmc_perf_split(workload_dir): os.remove(workload_perfmon_dir + "/pmc_perf.txt") +def update_pmc_bucket( + counters, + save_file, + soc, + pmc_list=None, + stext=None, + workload_perfmon_dir=None + ): + # Verify inputs. + # If save_file is True, we're being called internally, from perfmon_coalesce + # Else we're being called externally, from rocomni + detected_extermal_call = False + if save_file and (stext is None or workload_perfmon_dir is None): + raise ValueError("stext and workload_perfmon_dir must be specified if save_file is True") + if pmc_list is None: + detected_extermal_call = True + pmc_list = dict( + [ + ("SQ", []), + ("GRBM", []), + ("TCP", []), + ("TA", []), + ("TD", []), + ("TCC", []), + ("SPI", []), + ("CPC", []), + ("CPF", []), + ("GDS", []), + ("TCC2", {}), # per-channel TCC perfmon + ] + ) + for ch in range(perfmon_config[soc]["TCC_channels"]): + pmc_list["TCC2"][str(ch)] = [] + + if "SQ_ACCUM_PREV_HIRES" in counters: + # save all level counters separately + nindex = counters.index("SQ_ACCUM_PREV_HIRES") + level_counter = counters[nindex - 1] + + if save_file: + # Save to level counter file, file name = level counter name + fd = open(workload_perfmon_dir + "/" + level_counter + ".txt", "w") + fd.write(stext + "\n\n") + fd.write("gpu:\n") + fd.write("range:\n") + fd.write("kernel:\n") + fd.close() + + return pmc_list + + # save normal pmc counters in matching buckets + for counter in counters: + IP_block = counter.split(sep="_")[0].upper() + # SQC and SQ belong to the IP block, coalesce them + if IP_block == "SQC": + IP_block = "SQ" + + if IP_block != "TCC": + # Insert unique pmc counters into its bucket + if counter not in pmc_list[IP_block]: + pmc_list[IP_block].append(counter) + + else: + # TCC counters processing + m = re.match(r"[\s\S]+\[(\d+)\]", counter) + if m is None: + # Aggregated TCC counters + if counter not in pmc_list[IP_block]: + pmc_list[IP_block].append(counter) + + else: + # TCC channel ID + ch = m.group(1) + + # fake IP block for per channel TCC + if str(ch) in pmc_list["TCC2"]: + # append unique counter into the channel + if counter not in pmc_list["TCC2"][str(ch)]: + pmc_list["TCC2"][str(ch)].append(counter) + else: + # initial counter in this channel + pmc_list["TCC2"][str(ch)] = [counter] + + if detected_extermal_call: + # sort the per channel counter, so that same counter in all channels can be aligned + for ch in range(perfmon_config[soc]["TCC_channels"]): + pmc_list["TCC2"][str(ch)].sort() + return pmc_list + + def perfmon_coalesce(pmc_files_list, workload_dir, soc): workload_perfmon_dir = workload_dir + "/perfmon" @@ -296,55 +386,11 @@ def perfmon_coalesce(pmc_files_list, workload_dir, soc): # we have found all the counters, store them in buckets counters = m.group(1).split() - if "SQ_ACCUM_PREV_HIRES" in counters: - # save all level counters separately - - nindex = counters.index("SQ_ACCUM_PREV_HIRES") - level_counter = counters[nindex - 1] - - # Save to level counter file, file name = level counter name - fd = open(workload_perfmon_dir + "/" + level_counter + ".txt", "w") - fd.write(stext + "\n\n") - fd.write("gpu:\n") - fd.write("range:\n") - fd.write("kernel:\n") - fd.close() - - continue - - # save normal pmc counters in matching buckets - for counter in counters: - IP_block = counter.split(sep="_")[0].upper() - # SQC and SQ belong to the IP block, coalesce them - if IP_block == "SQC": - IP_block = "SQ" - - if IP_block != "TCC": - # Insert unique pmc counters into its bucket - if counter not in pmc_list[IP_block]: - pmc_list[IP_block].append(counter) - - else: - # TCC counters processing - m = re.match(r"[\s\S]+\[(\d+)\]", counter) - if m is None: - # Aggregated TCC counters - if counter not in pmc_list[IP_block]: - pmc_list[IP_block].append(counter) - - else: - # TCC channel ID - ch = m.group(1) - - # fake IP block for per channel TCC - if str(ch) in pmc_list["TCC2"]: - # append unique counter into the channel - if counter not in pmc_list["TCC2"][str(ch)]: - pmc_list["TCC2"][str(ch)].append(counter) - else: - # initial counter in this channel - pmc_list["TCC2"][str(ch)] = [counter] - + + # Utilitze helper function once a list of counters has be extracted + save_file = True + pmc_list = update_pmc_bucket(counters, save_file, soc, pmc_list, stext, workload_perfmon_dir) + # add a timestamp file fd = open(workload_perfmon_dir + "/timestamps.txt", "w") fd.write("pmc:\n\n") @@ -360,9 +406,7 @@ def perfmon_coalesce(pmc_files_list, workload_dir, soc): return pmc_list -def perfmon_emit(pmc_list, workload_dir, soc): - workload_perfmon_dir = workload_dir + "/perfmon" - +def perfmon_emit(pmc_list, soc, save_file=True, workload_dir=None): # Calculate the minimum number of iteration to save the pmc counters # non-TCC counters pmc_cnt = [ @@ -384,7 +428,12 @@ def perfmon_emit(pmc_list, workload_dir, soc): niter = max(math.ceil(max(pmc_cnt)), math.ceil(tcc_cnt) + math.ceil(max(tcc2_cnt))) # Emit PMC counters into pmc config file - fd = open(workload_perfmon_dir + "/pmc_perf.txt", "w") + if save_file: + workload_perfmon_dir = workload_dir + "/perfmon" + fd = open(workload_perfmon_dir + "/pmc_perf.txt", "w") + else: + batches = [] + tcc2_index = 0 for iter in range(niter): @@ -414,12 +463,20 @@ def perfmon_emit(pmc_list, workload_dir, soc): # TCC aggregated counters line = line + " " + " ".join(tcc_counters) - fd.write(line + "\n") + if save_file: + fd.write(line + "\n") + else: + b = line.split() + b.remove("pmc:") + batches.append(b) - fd.write("\ngpu:\n") - fd.write("range:\n") - fd.write("kernel:\n") - fd.close() + if save_file: + fd.write("\ngpu:\n") + fd.write("range:\n") + fd.write("kernel:\n") + fd.close() + else: + return batches def perfmon_filter(workload_dir, perfmon_dir, args): From 35daf8c120260121802d100e21f1e664b4a2202b Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Tue, 11 Jul 2023 14:14:10 -0500 Subject: [PATCH 16/34] Comply to Python formatting Signed-off-by: colramos-amd --- src/omniperf_analyze/utils/parser.py | 8 ++++---- src/utils/perfagg.py | 26 ++++++++++++-------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/omniperf_analyze/utils/parser.py b/src/omniperf_analyze/utils/parser.py index 025745b17..0328d7aa8 100644 --- a/src/omniperf_analyze/utils/parser.py +++ b/src/omniperf_analyze/utils/parser.py @@ -342,7 +342,7 @@ def gen_counter_list(formula): "GFLOPs": None, } - built_in_counter=[ + built_in_counter = [ "lds", "grd", "wgr", @@ -351,7 +351,7 @@ def gen_counter_list(formula): "sgpr", "scr", "BeginNs", - "EndNs" + "EndNs", ] visited = False @@ -367,7 +367,7 @@ def gen_counter_list(formula): for node in ast.walk(tree): if isinstance(node, ast.Name): val = str(node.id)[:-4] if str(node.id).endswith("_sum") else str(node.id) - if (val.isupper() and val not in function_filter): + if val.isupper() and val not in function_filter: counters.append(val) visited = True if val in built_in_counter: @@ -404,7 +404,7 @@ def build_dfs(archConfigs, filter_metrics): metric_list[panel_idx] = panel["title"] table_idx = panel_idx + "." + str(data_cofig["id"] % 100) metric_list[table_idx] = data_cofig["title"] - + headers = ["Index"] for key, tile in data_cofig["header"].items(): if key != "tips": diff --git a/src/utils/perfagg.py b/src/utils/perfagg.py index 8e95482c5..04658795c 100755 --- a/src/utils/perfagg.py +++ b/src/utils/perfagg.py @@ -257,19 +257,16 @@ def pmc_perf_split(workload_dir): def update_pmc_bucket( - counters, - save_file, - soc, - pmc_list=None, - stext=None, - workload_perfmon_dir=None - ): + counters, save_file, soc, pmc_list=None, stext=None, workload_perfmon_dir=None +): # Verify inputs. # If save_file is True, we're being called internally, from perfmon_coalesce # Else we're being called externally, from rocomni detected_extermal_call = False if save_file and (stext is None or workload_perfmon_dir is None): - raise ValueError("stext and workload_perfmon_dir must be specified if save_file is True") + raise ValueError( + "stext and workload_perfmon_dir must be specified if save_file is True" + ) if pmc_list is None: detected_extermal_call = True pmc_list = dict( @@ -289,7 +286,7 @@ def update_pmc_bucket( ) for ch in range(perfmon_config[soc]["TCC_channels"]): pmc_list["TCC2"][str(ch)] = [] - + if "SQ_ACCUM_PREV_HIRES" in counters: # save all level counters separately nindex = counters.index("SQ_ACCUM_PREV_HIRES") @@ -305,7 +302,7 @@ def update_pmc_bucket( fd.close() return pmc_list - + # save normal pmc counters in matching buckets for counter in counters: IP_block = counter.split(sep="_")[0].upper() @@ -386,11 +383,13 @@ def perfmon_coalesce(pmc_files_list, workload_dir, soc): # we have found all the counters, store them in buckets counters = m.group(1).split() - + # Utilitze helper function once a list of counters has be extracted save_file = True - pmc_list = update_pmc_bucket(counters, save_file, soc, pmc_list, stext, workload_perfmon_dir) - + pmc_list = update_pmc_bucket( + counters, save_file, soc, pmc_list, stext, workload_perfmon_dir + ) + # add a timestamp file fd = open(workload_perfmon_dir + "/timestamps.txt", "w") fd.write("pmc:\n\n") @@ -434,7 +433,6 @@ def perfmon_emit(pmc_list, soc, save_file=True, workload_dir=None): else: batches = [] - tcc2_index = 0 for iter in range(niter): # Prefix From 626fc7f18f04af78d7d66de0860c4f1d4b528476 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Mon, 17 Jul 2023 13:12:22 -0500 Subject: [PATCH 17/34] Filter additional ops in gen_counter_list fucn Signed-off-by: colramos-amd --- src/omniperf_analyze/utils/parser.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/omniperf_analyze/utils/parser.py b/src/omniperf_analyze/utils/parser.py index 0328d7aa8..bff3314b2 100644 --- a/src/omniperf_analyze/utils/parser.py +++ b/src/omniperf_analyze/utils/parser.py @@ -340,6 +340,8 @@ def gen_counter_list(formula): "RW": None, "GIOP": None, "GFLOPs": None, + "CONCAT": None, + "MOD": None, } built_in_counter = [ @@ -362,6 +364,12 @@ def gen_counter_list(formula): tree = ast.parse( formula.replace("$normUnit", "SQ_WAVES") .replace("$denom", "SQ_WAVES") + .replace( + "$numActiveCUs", + "TO_INT(MIN((((ROUND(AVG(((4 * SQ_BUSY_CU_CYCLES) / GRBM_GUI_ACTIVE)), \ + 0) / $maxWavesPerCU) * 8) + MIN(MOD(ROUND(AVG(((4 * SQ_BUSY_CU_CYCLES) \ + / GRBM_GUI_ACTIVE)), 0), $maxWavesPerCU), 8)), $numCU))", + ) .replace("$", "") ) for node in ast.walk(tree): From e6ca8e0361aabbdaedd58128426c0e0370894f85 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Mon, 17 Jul 2023 13:12:56 -0500 Subject: [PATCH 18/34] Enable join_prof() merge util to be called from outside Omniperf Signed-off-by: colramos-amd --- src/utils/perfagg.py | 57 +++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/src/utils/perfagg.py b/src/utils/perfagg.py index 04658795c..1c21b1736 100755 --- a/src/utils/perfagg.py +++ b/src/utils/perfagg.py @@ -95,13 +95,19 @@ def test_df_column_equality(df): # joins disparate runs less dumbly than rocprof def join_prof(workload_dir, join_type, log_file, verbose, out=None): # Set default output directory if not specified - if out == None: - out = workload_dir + "/pmc_perf.csv" - files = glob.glob(workload_dir + "/" + "pmc_perf_*.csv") - df = None + if type(workload_dir) == str: + if out is None: + out = workload_dir + "/pmc_perf.csv" + files = glob.glob(workload_dir + "/" + "pmc_perf_*.csv") + elif type(workload_dir) == list: + files = workload_dir + else: + print("ERROR: Invalid workload_dir") + sys.exit(1) + df = None for i, file in enumerate(files): - _df = pd.read_csv(file) + _df = pd.read_csv(file) if type(workload_dir) == str else file if join_type == "kernel": key = _df.groupby("KernelName").cumcount() _df["key"] = _df.KernelName + " - " + key.astype(str) @@ -137,7 +143,6 @@ def join_prof(workload_dir, join_type, log_file, verbose, out=None): duplicate_cols["arch_vgpr"] = [col for col in df.columns if "arch_vgpr" in col] duplicate_cols["accum_vgpr"] = [col for col in df.columns if "accum_vgpr" in col] for key, cols in duplicate_cols.items(): - print("Key is ", key) _df = df[cols] if not test_df_column_equality(_df): msg = ( @@ -146,10 +151,12 @@ def join_prof(workload_dir, join_type, log_file, verbose, out=None): ) ) warnings.warn(msg) - log_file.write(msg + "\n") + if log_file: + log_file.write(msg + "\n") else: msg = "Successfully joined {} in pmc_perf.csv".format(key) - log_file.write(msg + "\n") + if log_file: + log_file.write(msg + "\n") if test_df_column_equality(_df) and verbose: print(msg) @@ -179,6 +186,8 @@ def join_prof(workload_dir, join_type, log_file, verbose, out=None): "fbar", "sig", "obj", + # rocscope specific merged counters, keep original + "dispatch_", ] ) ] @@ -189,7 +198,15 @@ def join_prof(workload_dir, join_type, log_file, verbose, out=None): [ k for k in df.keys() - if not any(check in k for check in ["DispatchNs", "CompleteNs"]) + if not any( + check in k + for check in [ + "DispatchNs", + "CompleteNs", + # rocscope specific timestamp + "HostDuration", + ] + ) ] ] #   C) sanity check the name and key @@ -216,12 +233,14 @@ def join_prof(workload_dir, join_type, log_file, verbose, out=None): df["EndNs"] = endNs # finally, join the drop key df = df.drop(columns=["key"]) - # and save to file - df.to_csv(out, index=False) - # and delete old file(s) - if not verbose: - for file in files: - os.remove(file) + # save to file and delete old file(s), skip if we're being called outside of Omniperf + if type(workload_dir) == str: + df.to_csv(out, index=False) + if not verbose: + for file in files: + os.remove(file) + else: + return df def pmc_perf_split(workload_dir): @@ -262,13 +281,13 @@ def update_pmc_bucket( # Verify inputs. # If save_file is True, we're being called internally, from perfmon_coalesce # Else we're being called externally, from rocomni - detected_extermal_call = False + detected_external_call = False if save_file and (stext is None or workload_perfmon_dir is None): raise ValueError( "stext and workload_perfmon_dir must be specified if save_file is True" ) if pmc_list is None: - detected_extermal_call = True + detected_external_call = True pmc_list = dict( [ ("SQ", []), @@ -287,7 +306,7 @@ def update_pmc_bucket( for ch in range(perfmon_config[soc]["TCC_channels"]): pmc_list["TCC2"][str(ch)] = [] - if "SQ_ACCUM_PREV_HIRES" in counters: + if "SQ_ACCUM_PREV_HIRES" in counters and not detected_external_call: # save all level counters separately nindex = counters.index("SQ_ACCUM_PREV_HIRES") level_counter = counters[nindex - 1] @@ -336,7 +355,7 @@ def update_pmc_bucket( # initial counter in this channel pmc_list["TCC2"][str(ch)] = [counter] - if detected_extermal_call: + if detected_external_call: # sort the per channel counter, so that same counter in all channels can be aligned for ch in range(perfmon_config[soc]["TCC_channels"]): pmc_list["TCC2"][str(ch)].sort() From 424dd6a4a8a65d8c4a1647ff29208b743ab066d3 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Tue, 18 Jul 2023 16:03:21 -0500 Subject: [PATCH 19/34] Add figure styling to YML configs Signed-off-by: colramos-amd --- .../configs/gfx906/1000_compute-unit-instruction-mix.yaml | 6 ++++++ .../configs/gfx906/1100_compute-unit-compute-pipeline.yaml | 5 +++++ src/omniperf_analyze/configs/gfx906/1200_lds.yaml | 5 +++++ .../configs/gfx906/1300_instruction-cache.yaml | 5 +++++ .../configs/gfx906/1400_constant-cache.yaml | 5 +++++ src/omniperf_analyze/configs/gfx906/1600_L1_cache.yaml | 7 +++++++ src/omniperf_analyze/configs/gfx906/1700_L2_cache.yaml | 4 ++++ .../configs/gfx908/1000_compute-unit-instruction-mix.yaml | 6 ++++++ .../configs/gfx908/1100_compute-unit-compute-pipeline.yaml | 5 +++++ src/omniperf_analyze/configs/gfx908/1200_lds.yaml | 5 +++++ .../configs/gfx908/1300_instruction-cache.yaml | 5 +++++ .../configs/gfx908/1400_constant-cache.yaml | 5 +++++ src/omniperf_analyze/configs/gfx908/1600_L1_cache.yaml | 7 +++++++ src/omniperf_analyze/configs/gfx908/1700_L2_cache.yaml | 4 ++++ .../configs/gfx90a/1000_compute-unit-instruction-mix.yaml | 6 ++++++ .../configs/gfx90a/1100_compute-unit-compute-pipeline.yaml | 5 +++++ src/omniperf_analyze/configs/gfx90a/1200_lds.yaml | 5 +++++ .../configs/gfx90a/1300_instruction-cache.yaml | 5 +++++ .../configs/gfx90a/1400_constant-cache.yaml | 5 +++++ src/omniperf_analyze/configs/gfx90a/1600_L1_cache.yaml | 7 +++++++ src/omniperf_analyze/configs/gfx90a/1700_L2_cache.yaml | 4 ++++ 21 files changed, 111 insertions(+) diff --git a/src/omniperf_analyze/configs/gfx906/1000_compute-unit-instruction-mix.yaml b/src/omniperf_analyze/configs/gfx906/1000_compute-unit-instruction-mix.yaml index c2e2f7f3a..eeec25221 100644 --- a/src/omniperf_analyze/configs/gfx906/1000_compute-unit-instruction-mix.yaml +++ b/src/omniperf_analyze/configs/gfx906/1000_compute-unit-instruction-mix.yaml @@ -17,6 +17,9 @@ Panel Config: max: Max unit: Unit tips: Tips + style: + type: simple_bar + label_txt: (# of instr + $normUnit) metric: VALU - Vector: avg: None # No HW module @@ -75,6 +78,9 @@ Panel Config: count: Count unit: Unit tips: Tips + style: + type: simple_bar + label_txt: (# of instr + $normUnit) metric: INT-32: count: None # No perf counter diff --git a/src/omniperf_analyze/configs/gfx906/1100_compute-unit-compute-pipeline.yaml b/src/omniperf_analyze/configs/gfx906/1100_compute-unit-compute-pipeline.yaml index e7f537290..8cffb24c7 100644 --- a/src/omniperf_analyze/configs/gfx906/1100_compute-unit-compute-pipeline.yaml +++ b/src/omniperf_analyze/configs/gfx906/1100_compute-unit-compute-pipeline.yaml @@ -15,6 +15,11 @@ Panel Config: metric: Metric value: Value tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: valu_flops_pop: value: None # No perf counter diff --git a/src/omniperf_analyze/configs/gfx906/1200_lds.yaml b/src/omniperf_analyze/configs/gfx906/1200_lds.yaml index 4f12a2ac3..218ad2cda 100644 --- a/src/omniperf_analyze/configs/gfx906/1200_lds.yaml +++ b/src/omniperf_analyze/configs/gfx906/1200_lds.yaml @@ -15,6 +15,11 @@ Panel Config: metric: Metric value: Value tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: Utilization: value: AVG(((100 * SQ_LDS_IDX_ACTIVE) / (GRBM_GUI_ACTIVE * $numCU))) diff --git a/src/omniperf_analyze/configs/gfx906/1300_instruction-cache.yaml b/src/omniperf_analyze/configs/gfx906/1300_instruction-cache.yaml index f40d3546e..fb9f384e1 100644 --- a/src/omniperf_analyze/configs/gfx906/1300_instruction-cache.yaml +++ b/src/omniperf_analyze/configs/gfx906/1300_instruction-cache.yaml @@ -15,6 +15,11 @@ Panel Config: metric: Metric value: Value tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: Bandwidth: value: AVG(((SQC_ICACHE_REQ * 100000) / (($sclk * $numSQC) diff --git a/src/omniperf_analyze/configs/gfx906/1400_constant-cache.yaml b/src/omniperf_analyze/configs/gfx906/1400_constant-cache.yaml index 4583c1012..91a2d6c9f 100644 --- a/src/omniperf_analyze/configs/gfx906/1400_constant-cache.yaml +++ b/src/omniperf_analyze/configs/gfx906/1400_constant-cache.yaml @@ -15,6 +15,11 @@ Panel Config: mertic: Metric value: Value tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: Bandwidth: value: AVG(((SQC_DCACHE_REQ * 100000) / (($sclk * $numSQC) diff --git a/src/omniperf_analyze/configs/gfx906/1600_L1_cache.yaml b/src/omniperf_analyze/configs/gfx906/1600_L1_cache.yaml index 1e05b3e4c..c734e21c8 100644 --- a/src/omniperf_analyze/configs/gfx906/1600_L1_cache.yaml +++ b/src/omniperf_analyze/configs/gfx906/1600_L1_cache.yaml @@ -15,6 +15,11 @@ Panel Config: metric: Metric value: Value tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: Buffer Coalescing: value: AVG(((((TA_TOTAL_WAVEFRONTS_sum * 64) * 100) / (TCP_TOTAL_ACCESSES_sum @@ -248,6 +253,8 @@ Panel Config: max: Max unit: Unit tips: Tips + style: + type: simple_multi_bar metric: NC - Read: xfer: Read diff --git a/src/omniperf_analyze/configs/gfx906/1700_L2_cache.yaml b/src/omniperf_analyze/configs/gfx906/1700_L2_cache.yaml index 53235ca14..0b5f5e827 100644 --- a/src/omniperf_analyze/configs/gfx906/1700_L2_cache.yaml +++ b/src/omniperf_analyze/configs/gfx906/1700_L2_cache.yaml @@ -16,6 +16,8 @@ Panel Config: value: Value unit: Unit tips: Tips + style: + type: simple_bar metric: L2 Util: value: AVG(((TCC_BUSY_sum * 100) / (TO_INT($L2Banks) * GRBM_GUI_ACTIVE))) @@ -301,6 +303,8 @@ Panel Config: max: Max unit: Unit tips: Tips + style: + type: simple_multi_bar metric: Read - Remote Socket Stall: type: Remote Socket Stall diff --git a/src/omniperf_analyze/configs/gfx908/1000_compute-unit-instruction-mix.yaml b/src/omniperf_analyze/configs/gfx908/1000_compute-unit-instruction-mix.yaml index 5a9bdb456..75eca82b8 100644 --- a/src/omniperf_analyze/configs/gfx908/1000_compute-unit-instruction-mix.yaml +++ b/src/omniperf_analyze/configs/gfx908/1000_compute-unit-instruction-mix.yaml @@ -18,6 +18,9 @@ Panel Config: max: Max unit: Unit tips: Tips + style: + type: simple_bar + label_txt: (# of instr + $normUnit) metric: VALU - Vector: avg: AVG(((SQ_INSTS_VALU - SQ_INSTS_MFMA) / $denom)) @@ -76,6 +79,9 @@ Panel Config: count: Count unit: Unit tips: Tips + style: + type: simple_bar + label_txt: (# of instr + $normUnit) metric: INT-32: count: None # No perf counter diff --git a/src/omniperf_analyze/configs/gfx908/1100_compute-unit-compute-pipeline.yaml b/src/omniperf_analyze/configs/gfx908/1100_compute-unit-compute-pipeline.yaml index 9316d1d25..061311d62 100644 --- a/src/omniperf_analyze/configs/gfx908/1100_compute-unit-compute-pipeline.yaml +++ b/src/omniperf_analyze/configs/gfx908/1100_compute-unit-compute-pipeline.yaml @@ -15,6 +15,11 @@ Panel Config: metric: Metric value: Value tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: valu_flops_pop: value: None # No perf counter diff --git a/src/omniperf_analyze/configs/gfx908/1200_lds.yaml b/src/omniperf_analyze/configs/gfx908/1200_lds.yaml index 4f12a2ac3..218ad2cda 100644 --- a/src/omniperf_analyze/configs/gfx908/1200_lds.yaml +++ b/src/omniperf_analyze/configs/gfx908/1200_lds.yaml @@ -15,6 +15,11 @@ Panel Config: metric: Metric value: Value tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: Utilization: value: AVG(((100 * SQ_LDS_IDX_ACTIVE) / (GRBM_GUI_ACTIVE * $numCU))) diff --git a/src/omniperf_analyze/configs/gfx908/1300_instruction-cache.yaml b/src/omniperf_analyze/configs/gfx908/1300_instruction-cache.yaml index f40d3546e..fb9f384e1 100644 --- a/src/omniperf_analyze/configs/gfx908/1300_instruction-cache.yaml +++ b/src/omniperf_analyze/configs/gfx908/1300_instruction-cache.yaml @@ -15,6 +15,11 @@ Panel Config: metric: Metric value: Value tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: Bandwidth: value: AVG(((SQC_ICACHE_REQ * 100000) / (($sclk * $numSQC) diff --git a/src/omniperf_analyze/configs/gfx908/1400_constant-cache.yaml b/src/omniperf_analyze/configs/gfx908/1400_constant-cache.yaml index 4583c1012..91a2d6c9f 100644 --- a/src/omniperf_analyze/configs/gfx908/1400_constant-cache.yaml +++ b/src/omniperf_analyze/configs/gfx908/1400_constant-cache.yaml @@ -15,6 +15,11 @@ Panel Config: mertic: Metric value: Value tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: Bandwidth: value: AVG(((SQC_DCACHE_REQ * 100000) / (($sclk * $numSQC) diff --git a/src/omniperf_analyze/configs/gfx908/1600_L1_cache.yaml b/src/omniperf_analyze/configs/gfx908/1600_L1_cache.yaml index f65309a31..7eeed0477 100644 --- a/src/omniperf_analyze/configs/gfx908/1600_L1_cache.yaml +++ b/src/omniperf_analyze/configs/gfx908/1600_L1_cache.yaml @@ -15,6 +15,11 @@ Panel Config: metric: Metric value: Value tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: Buffer Coalescing: value: AVG(((((TA_TOTAL_WAVEFRONTS_sum * 64) * 100) / (TCP_TOTAL_ACCESSES_sum @@ -248,6 +253,8 @@ Panel Config: max: Max unit: Unit tips: Tips + style: + type: simple_multi_bar metric: NC - Read: xfer: Read diff --git a/src/omniperf_analyze/configs/gfx908/1700_L2_cache.yaml b/src/omniperf_analyze/configs/gfx908/1700_L2_cache.yaml index 9e76a39b6..0c7b03811 100644 --- a/src/omniperf_analyze/configs/gfx908/1700_L2_cache.yaml +++ b/src/omniperf_analyze/configs/gfx908/1700_L2_cache.yaml @@ -16,6 +16,8 @@ Panel Config: value: Value unit: Unit tips: Tips + style: + type: simple_bar metric: L2 Util: value: AVG(((TCC_BUSY_sum * 100) / (TO_INT($L2Banks) * GRBM_GUI_ACTIVE))) @@ -301,6 +303,8 @@ Panel Config: max: Max unit: Unit tips: Tips + style: + type: simple_multi_bar metric: Read - Remote Socket Stall: type: Remote Socket Stall diff --git a/src/omniperf_analyze/configs/gfx90a/1000_compute-unit-instruction-mix.yaml b/src/omniperf_analyze/configs/gfx90a/1000_compute-unit-instruction-mix.yaml index 999d25e19..811ebca11 100644 --- a/src/omniperf_analyze/configs/gfx90a/1000_compute-unit-instruction-mix.yaml +++ b/src/omniperf_analyze/configs/gfx90a/1000_compute-unit-instruction-mix.yaml @@ -18,6 +18,9 @@ Panel Config: max: Max unit: Unit tips: Tips + style: + type: simple_bar + label_txt: (# of instr + $normUnit) metric: VALU - Vector: avg: AVG(((SQ_INSTS_VALU - SQ_INSTS_MFMA) / $denom)) @@ -76,6 +79,9 @@ Panel Config: count: Count unit: Unit tips: Tips + style: + type: simple_bar + label_txt: (# of instr + $normUnit) metric: INT32: count: AVG((SQ_INSTS_VALU_INT32 / $denom)) diff --git a/src/omniperf_analyze/configs/gfx90a/1100_compute-unit-compute-pipeline.yaml b/src/omniperf_analyze/configs/gfx90a/1100_compute-unit-compute-pipeline.yaml index 2866f9652..301217fde 100644 --- a/src/omniperf_analyze/configs/gfx90a/1100_compute-unit-compute-pipeline.yaml +++ b/src/omniperf_analyze/configs/gfx90a/1100_compute-unit-compute-pipeline.yaml @@ -17,6 +17,11 @@ Panel Config: min: Min max: Max tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: valu_flops_pop: avg: ((100 * AVG(((((64 * (((SQ_INSTS_VALU_ADD_F16 + SQ_INSTS_VALU_MUL_F16) diff --git a/src/omniperf_analyze/configs/gfx90a/1200_lds.yaml b/src/omniperf_analyze/configs/gfx90a/1200_lds.yaml index 4f12a2ac3..218ad2cda 100644 --- a/src/omniperf_analyze/configs/gfx90a/1200_lds.yaml +++ b/src/omniperf_analyze/configs/gfx90a/1200_lds.yaml @@ -15,6 +15,11 @@ Panel Config: metric: Metric value: Value tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: Utilization: value: AVG(((100 * SQ_LDS_IDX_ACTIVE) / (GRBM_GUI_ACTIVE * $numCU))) diff --git a/src/omniperf_analyze/configs/gfx90a/1300_instruction-cache.yaml b/src/omniperf_analyze/configs/gfx90a/1300_instruction-cache.yaml index cd0af3bfa..1a7000e93 100644 --- a/src/omniperf_analyze/configs/gfx90a/1300_instruction-cache.yaml +++ b/src/omniperf_analyze/configs/gfx90a/1300_instruction-cache.yaml @@ -15,6 +15,11 @@ Panel Config: metric: Metric value: Value tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: Bandwidth: value: AVG(((SQC_ICACHE_REQ * 100000) / (($sclk * $numSQC) diff --git a/src/omniperf_analyze/configs/gfx90a/1400_constant-cache.yaml b/src/omniperf_analyze/configs/gfx90a/1400_constant-cache.yaml index 4583c1012..91a2d6c9f 100644 --- a/src/omniperf_analyze/configs/gfx90a/1400_constant-cache.yaml +++ b/src/omniperf_analyze/configs/gfx90a/1400_constant-cache.yaml @@ -15,6 +15,11 @@ Panel Config: mertic: Metric value: Value tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: Bandwidth: value: AVG(((SQC_DCACHE_REQ * 100000) / (($sclk * $numSQC) diff --git a/src/omniperf_analyze/configs/gfx90a/1600_L1_cache.yaml b/src/omniperf_analyze/configs/gfx90a/1600_L1_cache.yaml index 917cb3aa0..7ea26db05 100644 --- a/src/omniperf_analyze/configs/gfx90a/1600_L1_cache.yaml +++ b/src/omniperf_analyze/configs/gfx90a/1600_L1_cache.yaml @@ -15,6 +15,11 @@ Panel Config: metric: Metric value: Value tips: Tips + style: + type: simple_bar + range_color: [1, 100] + label_txt: (%) + xrange: [0, 110] metric: Buffer Coalescing: value: AVG(((((TA_TOTAL_WAVEFRONTS_sum * 64) * 100) / (TCP_TOTAL_ACCESSES_sum @@ -248,6 +253,8 @@ Panel Config: max: Max unit: Unit tips: Tips + style: + type: simple_multi_bar metric: NC - Read: xfer: Read diff --git a/src/omniperf_analyze/configs/gfx90a/1700_L2_cache.yaml b/src/omniperf_analyze/configs/gfx90a/1700_L2_cache.yaml index 7be18091a..ddbaf9155 100644 --- a/src/omniperf_analyze/configs/gfx90a/1700_L2_cache.yaml +++ b/src/omniperf_analyze/configs/gfx90a/1700_L2_cache.yaml @@ -16,6 +16,8 @@ Panel Config: value: Value unit: Unit tips: Tips + style: + type: simple_bar metric: L2 Util: value: AVG(((TCC_BUSY_sum * 100) / (TO_INT($L2Banks) * GRBM_GUI_ACTIVE))) @@ -301,6 +303,8 @@ Panel Config: max: Max unit: Unit tips: Tips + style: + type: simple_multi_bar metric: Read - Remote Socket Stall: type: Remote Socket Stall From a61d509739b5f8ff86be95c93cb1854095ee6019 Mon Sep 17 00:00:00 2001 From: colramos-amd Date: Tue, 18 Jul 2023 16:04:31 -0500 Subject: [PATCH 20/34] New simple_charts utility Signed-off-by: colramos-amd --- src/omniperf_analyze/utils/simple_charts.py | 119 ++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 src/omniperf_analyze/utils/simple_charts.py diff --git a/src/omniperf_analyze/utils/simple_charts.py b/src/omniperf_analyze/utils/simple_charts.py new file mode 100644 index 000000000..8a853f987 --- /dev/null +++ b/src/omniperf_analyze/utils/simple_charts.py @@ -0,0 +1,119 @@ +##############################################################################bl +# MIT License +# +# Copyright (c) 2021 - 2023 Advanced Micro Devices, Inc. All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +##############################################################################el + +import plotly.express as px +import pandas as pd + + +# Notes: +# This file includes implementation of a few simple but common charts in CLI. +# We try to auto-size the layout to cover most of the cases as default. If it +# doesn't work, apply style config in yaml for each dashboard. + + +def simple_bar(df, title: str = None, id=None, style: dict = None, orientation="h"): + """ + Plot data with simple bar chart + """ + + # TODO: handle None properly + if "Metric" in df.columns and ("Count" in df.columns or "Value" in df.columns): + detected_label = "Count" if "Count" in df.columns else "Value" + df[detected_label] = [ + x.astype(int) if x != "" else int(0) for x in df[detected_label] + ] + else: + raise NameError("simple_bar: No Metric or Count in df columns!") + + # Assign figure characteristics + range_color = style.get("range_color", None) + label_txt = style.get("label_txt", None) + xrange = style.get("xrange", None) + if label_txt is not None: + label_txt = label_txt.strip("()") + try: + label_txt = label_txt.replace("+ $normUnit", df["Unit"][0]) + except KeyError: + print("No units found in df. Auto labeling.") + + # Overrides for figure chatacteristics + if id == 1701.1: + label_txt = "%" + range_color = [0, 100] + xrange = [0, 110] + if id == 1701.2: + label_txt = "Gb/s" + range_color = [0, 1638] + xrange = [0, 1638] + + fig = px.bar( + df, + title=title, + x=detected_label, + y="Metric", + color=detected_label, + range_color=range_color, + labels={detected_label: label_txt}, + orientation=orientation, + ).update_xaxes(range=xrange) + + return fig + + +def simple_multi_bar(df, title=None, id=None): + """ + Plot data with simple multiple bar chart + """ + + # TODO: handle Nan and None properly + if "Metric" in df.columns and "Avg" in df.columns: + df["Avg"] = [x.astype(int) if x != "" else int(0) for x in df["Avg"]] + else: + raise NameError("simple_multi_bar: No Metric or Count in df columns!") + + dfigs = [] + nested_bar = {} + df_unit = df["Unit"][0] + if id == 1604: + nested_bar = {"NC": {}, "UC": {}, "RW": {}, "CC": {}} + for index, row in df.iterrows(): + nested_bar[row["Coherency"]][row["Xfer"]] = row["Avg"] + if id == 1704: + nested_bar = {"Read": {}, "Write": {}} + for index, row in df.iterrows(): + nested_bar[row["Transaction"]][row["Type"]] = row["Avg"] + + for group, metric in nested_bar.items(): + dfigs.append( + px.bar( + title=group, + x=metric.values(), + y=metric.keys(), + labels={"x": df_unit, "y": ""}, + text=metric.values(), + ) + .update_xaxes(showgrid=False, rangemode="nonnegative") + .update_yaxes(showgrid=False) + ) + return dfigs From 08f4684620aad6bb1203887f0ddeaa99a3747eb5 Mon Sep 17 00:00:00 2001 From: JoseSantosAMD Date: Mon, 17 Jul 2023 12:46:05 -0500 Subject: [PATCH 21/34] Migrate to @grafana/create-plugin Signed-off-by: JoseSantosAMD --- grafana_plugins/svg_plugin/.prettierrc.js | 5 +- grafana_plugins/svg_plugin/package.json | 65 +++++++++++++++++++---- grafana_plugins/svg_plugin/tsconfig.json | 12 +---- 3 files changed, 59 insertions(+), 23 deletions(-) diff --git a/grafana_plugins/svg_plugin/.prettierrc.js b/grafana_plugins/svg_plugin/.prettierrc.js index f60eb1d25..aaa5045c6 100644 --- a/grafana_plugins/svg_plugin/.prettierrc.js +++ b/grafana_plugins/svg_plugin/.prettierrc.js @@ -1,3 +1,4 @@ module.exports = { - ...require("./node_modules/@grafana/toolkit/src/config/prettier.plugin.config.json"), - }; \ No newline at end of file + // Prettier configuration provided by Grafana scaffolding + ...require("./.config/.prettierrc.js") +}; \ No newline at end of file diff --git a/grafana_plugins/svg_plugin/package.json b/grafana_plugins/svg_plugin/package.json index fb88025c1..aa445bd25 100644 --- a/grafana_plugins/svg_plugin/package.json +++ b/grafana_plugins/svg_plugin/package.json @@ -3,29 +3,72 @@ "version": "1.0.0", "description": "", "scripts": { - "build": "grafana-toolkit plugin:build", - "test": "grafana-toolkit plugin:test", - "dev": "grafana-toolkit plugin:dev", - "watch": "grafana-toolkit plugin:dev --watch", - "sign": "grafana-toolkit plugin:sign", - "start": "yarn watch" + "build": "webpack -c ./.config/webpack/webpack.config.ts --env production", + "dev": "webpack -w -c ./.config/webpack/webpack.config.ts --env development", + "e2e": "yarn exec cypress install && yarn exec grafana-e2e run", + "e2e:update": "yarn exec cypress install && yarn exec grafana-e2e run --update-screenshots", + "lint": "eslint --cache --ignore-path ./.gitignore --ext .js,.jsx,.ts,.tsx .", + "lint:fix": "yarn run lint --fix", + "server": "docker-compose up --build", + "sign": "npx --yes @grafana/sign-plugin@latest", + "start": "yarn watch", + "test": "jest --watch --onlyChanged", + "test:ci": "jest --passWithNoTests --maxWorkers 4", + "typecheck": "tsc --noEmit" }, "author": "Audacious Software Group", "license": "MIT", "devDependencies": { - "@grafana/toolkit": "latest", + "@babel/core": "^7.21.4", + "@grafana/e2e": "9.5.3", + "@grafana/e2e-selectors": "9.5.3", + "@grafana/eslint-config": "^6.0.0", + "@grafana/tsconfig": "^1.2.0-rc1", + "@swc/core": "^1.3.62", + "@swc/helpers": "^0.5.0", + "@swc/jest": "^0.2.26", + "@testing-library/jest-dom": "^5.16.5", + "@testing-library/react": "^12.1.4", + "@types/jest": "^29.5.0", + "@types/lodash": "^4.14.194", + "@types/node": "^18.15.11", + "copy-webpack-plugin": "^11.0.0", + "css-loader": "^6.7.3", "emotion": "10.0.27", + "eslint-webpack-plugin": "^4.0.1", + "fork-ts-checker-webpack-plugin": "^8.0.0", + "glob": "^10.2.7", + "identity-obj-proxy": "3.0.0", + "jest": "^29.5.0", + "jest-environment-jsdom": "^29.5.0", + "prettier": "^2.8.7", "react-monaco-editor": "^0.44.0", - "tslib": "^2.3.1" + "replace-in-file-webpack-plugin": "^1.0.6", + "sass": "1.63.2", + "sass-loader": "13.3.1", + "style-loader": "3.3.3", + "swc-loader": "^0.2.3", + "ts-node": "^10.9.1", + "tsconfig-paths": "^4.2.0", + "tslib": "^2.3.1", + "typescript": "4.8.4", + "webpack": "^5.86.0", + "webpack-cli": "^5.1.4", + "webpack-livereload-plugin": "^3.0.2" }, "engines": { "node": ">=14" }, "dependencies": { - "@grafana/runtime": "9.1.2", + "@emotion/css": "^11.1.3", "@grafana/data": "9.1.2", + "@grafana/runtime": "9.1.2", "@grafana/ui": "9.1.2", - "@svgdotjs/svg.js": "^3.1.1" + "@svgdotjs/svg.js": "^3.1.1", + "react": "17.0.2", + "react-dom": "17.0.2", + "tslib": "2.5.3" }, - "_comments": "Dependencies are not included as part of Omniperf. It's the user's responsibility to accept any licensing implications before building the project." + "_comments": "Dependencies are not included as part of Omniperf. It's the user's responsibility to accept any licensing implications before building the project.", + "packageManager": "yarn@1.22.19" } diff --git a/grafana_plugins/svg_plugin/tsconfig.json b/grafana_plugins/svg_plugin/tsconfig.json index 7e6657d2f..d294745aa 100644 --- a/grafana_plugins/svg_plugin/tsconfig.json +++ b/grafana_plugins/svg_plugin/tsconfig.json @@ -1,11 +1,3 @@ { - "extends": "./node_modules/@grafana/toolkit/src/config/tsconfig.plugin.json", - "include": ["src", "types"], - "compilerOptions": { - "types": ["@emotion/core"], - "rootDir": "./src", - "baseUrl": "./src", - "typeRoots": ["./node_modules/@types"], - "jsx": "react" - } -} + "extends": "./.config/tsconfig.json" +} \ No newline at end of file From c209583e9878de76914693f1ca8def68efc39f1a Mon Sep 17 00:00:00 2001 From: JoseSantosAMD Date: Mon, 17 Jul 2023 13:02:00 -0500 Subject: [PATCH 22/34] Adding config files Signed-off-by: JoseSantosAMD --- .../svg_plugin/.config/tsconfig.json | 26 +++ .../.config/webpack/webpack.config.ts | 201 ++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 grafana_plugins/svg_plugin/.config/tsconfig.json create mode 100644 grafana_plugins/svg_plugin/.config/webpack/webpack.config.ts diff --git a/grafana_plugins/svg_plugin/.config/tsconfig.json b/grafana_plugins/svg_plugin/.config/tsconfig.json new file mode 100644 index 000000000..64b376907 --- /dev/null +++ b/grafana_plugins/svg_plugin/.config/tsconfig.json @@ -0,0 +1,26 @@ +/* + * ⚠️⚠️⚠️ THIS FILE WAS SCAFFOLDED BY `@grafana/create-plugin`. DO NOT EDIT THIS FILE DIRECTLY. ⚠️⚠️⚠️ + * + * In order to extend the configuration follow the steps in + * https://grafana.github.io/plugin-tools/docs/advanced-configuration#extending-the-typescript-config + */ + { + "compilerOptions": { + "alwaysStrict": true, + "declaration": false, + "rootDir": "../src", + "baseUrl": "../src", + "typeRoots": ["../node_modules/@types"], + "resolveJsonModule": true + }, + "ts-node": { + "compilerOptions": { + "module": "commonjs", + "target": "es5", + "esModuleInterop": true + }, + "transpileOnly": true + }, + "include": ["../src", "./types"], + "extends": "@grafana/tsconfig" +} diff --git a/grafana_plugins/svg_plugin/.config/webpack/webpack.config.ts b/grafana_plugins/svg_plugin/.config/webpack/webpack.config.ts new file mode 100644 index 000000000..22cb86ca4 --- /dev/null +++ b/grafana_plugins/svg_plugin/.config/webpack/webpack.config.ts @@ -0,0 +1,201 @@ +/* + * ⚠️⚠️⚠️ THIS FILE WAS SCAFFOLDED BY `@grafana/create-plugin`. DO NOT EDIT THIS FILE DIRECTLY. ⚠️⚠️⚠️ + * + * In order to extend the configuration follow the steps in + * https://grafana.github.io/plugin-tools/docs/advanced-configuration#extending-the-webpack-config + */ + +import CopyWebpackPlugin from 'copy-webpack-plugin'; +import ESLintPlugin from 'eslint-webpack-plugin'; +import ForkTsCheckerWebpackPlugin from 'fork-ts-checker-webpack-plugin'; +import LiveReloadPlugin from 'webpack-livereload-plugin'; +import path from 'path'; +import ReplaceInFileWebpackPlugin from 'replace-in-file-webpack-plugin'; +import { Configuration } from 'webpack'; + +import { getPackageJson, getPluginJson, hasReadme, getEntries } from './utils'; +import { SOURCE_DIR, DIST_DIR } from './constants'; + +const pluginJson = getPluginJson(); + +const config = async (env): Promise => ({ + cache: { + type: 'filesystem', + buildDependencies: { + config: [__filename], + }, + }, + + context: path.join(process.cwd(), SOURCE_DIR), + + devtool: env.production ? 'source-map' : 'eval-source-map', + + entry: await getEntries(), + + externals: [ + 'lodash', + 'jquery', + 'moment', + 'slate', + 'emotion', + '@emotion/react', + '@emotion/css', + 'prismjs', + 'slate-plain-serializer', + '@grafana/slate-react', + 'react', + 'react-dom', + 'react-redux', + 'redux', + 'rxjs', + 'react-router', + 'react-router-dom', + 'd3', + 'angular', + '@grafana/ui', + '@grafana/runtime', + '@grafana/data', + + // Mark legacy SDK imports as external if their name starts with the "grafana/" prefix + ({ request }, callback) => { + const prefix = 'grafana/'; + const hasPrefix = (request) => request.indexOf(prefix) === 0; + const stripPrefix = (request) => request.substr(prefix.length); + + if (hasPrefix(request)) { + return callback(undefined, stripPrefix(request)); + } + + callback(); + }, + ], + + mode: env.production ? 'production' : 'development', + + module: { + rules: [ + { + exclude: /(node_modules)/, + test: /\.[tj]sx?$/, + use: { + loader: 'swc-loader', + options: { + jsc: { + baseUrl: './src', + target: 'es2015', + loose: false, + parser: { + syntax: 'typescript', + tsx: true, + decorators: false, + dynamicImport: true, + }, + }, + }, + }, + }, + { + test: /\.css$/, + use: ["style-loader", "css-loader"] + }, + { + test: /\.s[ac]ss$/, + use: ['style-loader', 'css-loader', 'sass-loader'], + }, + { + test: /\.(png|jpe?g|gif|svg)$/, + type: 'asset/resource', + generator: { + // Keep publicPath relative for host.com/grafana/ deployments + publicPath: `public/plugins/${pluginJson.id}/img/`, + outputPath: 'img/', + filename: Boolean(env.production) ? '[hash][ext]' : '[name][ext]', + }, + }, + { + test: /\.(woff|woff2|eot|ttf|otf)(\?v=\d+\.\d+\.\d+)?$/, + type: 'asset/resource', + generator: { + // Keep publicPath relative for host.com/grafana/ deployments + publicPath: `public/plugins/${pluginJson.id}/fonts/`, + outputPath: 'fonts/', + filename: Boolean(env.production) ? '[hash][ext]' : '[name][ext]', + }, + }, + ], + }, + + output: { + clean: { + keep: new RegExp(`.*?_(amd64|arm(64)?)(.exe)?`), + }, + filename: '[name].js', + library: { + type: 'amd', + }, + path: path.resolve(process.cwd(), DIST_DIR), + publicPath: '/', + }, + + plugins: [ + new CopyWebpackPlugin({ + patterns: [ + // If src/README.md exists use it; otherwise the root README + // To `compiler.options.output` + { from: hasReadme() ? 'README.md' : '../README.md', to: '.', force: true }, + { from: 'plugin.json', to: '.' }, + { from: '../LICENSE', to: '.' }, + { from: '../CHANGELOG.md', to: '.', force: true }, + { from: '**/*.json', to: '.' }, // TODO + { from: '**/*.svg', to: '.', noErrorOnMissing: true }, // Optional + { from: '**/*.png', to: '.', noErrorOnMissing: true }, // Optional + { from: '**/*.html', to: '.', noErrorOnMissing: true }, // Optional + { from: 'img/**/*', to: '.', noErrorOnMissing: true }, // Optional + { from: 'libs/**/*', to: '.', noErrorOnMissing: true }, // Optional + { from: 'static/**/*', to: '.', noErrorOnMissing: true }, // Optional + ], + }), + // Replace certain template-variables in the README and plugin.json + new ReplaceInFileWebpackPlugin([ + { + dir: DIST_DIR, + files: ['plugin.json', 'README.md'], + rules: [ + { + search: /\%VERSION\%/g, + replace: getPackageJson().version, + }, + { + search: /\%TODAY\%/g, + replace: new Date().toISOString().substring(0, 10), + }, + { + search: /\%PLUGIN_ID\%/g, + replace: pluginJson.id, + }, + ], + }, + ]), + new ForkTsCheckerWebpackPlugin({ + async: Boolean(env.development), + issue: { + include: [{ file: '**/*.{ts,tsx}' }], + }, + typescript: { configFile: path.join(process.cwd(), 'tsconfig.json') }, + }), + new ESLintPlugin({ + extensions: ['.ts', '.tsx'], + lintDirtyModulesOnly: Boolean(env.development), // don't lint on start, only lint changed files + }), + ...(env.development ? [new LiveReloadPlugin()] : []), + ], + + resolve: { + extensions: ['.js', '.jsx', '.ts', '.tsx'], + // handle resolving "rootDir" paths + modules: [path.resolve(process.cwd(), 'src'), 'node_modules'], + unsafeCache: true, + }, +}); + +export default config; From 7ed6c620aa25505da968e1db4f3f10504b4b1536 Mon Sep 17 00:00:00 2001 From: JoseSantosAMD Date: Mon, 17 Jul 2023 13:19:03 -0500 Subject: [PATCH 23/34] Adding config files Signed-off-by: JoseSantosAMD --- grafana_plugins/svg_plugin/.config/.eslintrc | 13 ++ .../svg_plugin/.config/.prettierrc.js | 16 ++ grafana_plugins/svg_plugin/.config/Dockerfile | 16 ++ grafana_plugins/svg_plugin/.config/README.md | 164 ++++++++++++++++++ .../svg_plugin/.config/jest-setup.js | 25 +++ .../svg_plugin/.config/jest.config.js | 43 +++++ .../.config/jest/mocks/react-inlinesvg.tsx | 25 +++ .../svg_plugin/.config/jest/utils.js | 31 ++++ .../svg_plugin/.config/types/custom.d.ts | 37 ++++ .../svg_plugin/.config/webpack/constants.ts | 2 + .../svg_plugin/.config/webpack/utils.ts | 40 +++++ grafana_plugins/svg_plugin/.eslintrc | 3 + grafana_plugins/svg_plugin/.nvmrc | 1 + .../svg_plugin/docker-compose.yaml | 15 ++ grafana_plugins/svg_plugin/jest-setup.js | 2 + grafana_plugins/svg_plugin/jest.config.js | 8 + 16 files changed, 441 insertions(+) create mode 100644 grafana_plugins/svg_plugin/.config/.eslintrc create mode 100644 grafana_plugins/svg_plugin/.config/.prettierrc.js create mode 100644 grafana_plugins/svg_plugin/.config/Dockerfile create mode 100644 grafana_plugins/svg_plugin/.config/README.md create mode 100644 grafana_plugins/svg_plugin/.config/jest-setup.js create mode 100644 grafana_plugins/svg_plugin/.config/jest.config.js create mode 100644 grafana_plugins/svg_plugin/.config/jest/mocks/react-inlinesvg.tsx create mode 100644 grafana_plugins/svg_plugin/.config/jest/utils.js create mode 100644 grafana_plugins/svg_plugin/.config/types/custom.d.ts create mode 100644 grafana_plugins/svg_plugin/.config/webpack/constants.ts create mode 100644 grafana_plugins/svg_plugin/.config/webpack/utils.ts create mode 100644 grafana_plugins/svg_plugin/.eslintrc create mode 100644 grafana_plugins/svg_plugin/.nvmrc create mode 100644 grafana_plugins/svg_plugin/docker-compose.yaml create mode 100644 grafana_plugins/svg_plugin/jest-setup.js create mode 100644 grafana_plugins/svg_plugin/jest.config.js diff --git a/grafana_plugins/svg_plugin/.config/.eslintrc b/grafana_plugins/svg_plugin/.config/.eslintrc new file mode 100644 index 000000000..3f8c381a4 --- /dev/null +++ b/grafana_plugins/svg_plugin/.config/.eslintrc @@ -0,0 +1,13 @@ +/* + * ⚠️⚠️⚠️ THIS FILE WAS SCAFFOLDED BY `@grafana/create-plugin`. DO NOT EDIT THIS FILE DIRECTLY. ⚠️⚠️⚠️ + * + * In order to extend the configuration follow the steps in + * https://grafana.github.io/plugin-tools/docs/advanced-configuration#extending-the-eslint-config + */ + { + "extends": ["@grafana/eslint-config"], + "root": true, + "rules": { + "react/prop-types": "off" + } +} diff --git a/grafana_plugins/svg_plugin/.config/.prettierrc.js b/grafana_plugins/svg_plugin/.config/.prettierrc.js new file mode 100644 index 000000000..66a76ec5b --- /dev/null +++ b/grafana_plugins/svg_plugin/.config/.prettierrc.js @@ -0,0 +1,16 @@ +/* + * ⚠️⚠️⚠️ THIS FILE WAS SCAFFOLDED BY `@grafana/create-plugin`. DO NOT EDIT THIS FILE DIRECTLY. ⚠️⚠️⚠️ + * + * In order to extend the configuration follow the steps in .config/README.md + */ + +module.exports = { + "endOfLine": "auto", + "printWidth": 120, + "trailingComma": "es5", + "semi": true, + "jsxSingleQuote": false, + "singleQuote": true, + "useTabs": false, + "tabWidth": 2 +}; \ No newline at end of file diff --git a/grafana_plugins/svg_plugin/.config/Dockerfile b/grafana_plugins/svg_plugin/.config/Dockerfile new file mode 100644 index 000000000..35d89bd1c --- /dev/null +++ b/grafana_plugins/svg_plugin/.config/Dockerfile @@ -0,0 +1,16 @@ +ARG grafana_version=latest +ARG grafana_image=grafana-enterprise + +FROM grafana/${grafana_image}:${grafana_version} + +# Make it as simple as possible to access the grafana instance for development purposes +# Do NOT enable these settings in a public facing / production grafana instance +ENV GF_AUTH_ANONYMOUS_ORG_ROLE "Admin" +ENV GF_AUTH_ANONYMOUS_ENABLED "true" +ENV GF_AUTH_BASIC_ENABLED "false" +# Set development mode so plugins can be loaded without the need to sign +ENV GF_DEFAULT_APP_MODE "development" + +# Inject livereload script into grafana index.html +USER root +RUN sed -i 's/<\/body><\/html>/