[sai-gen] Create SAI spec for SAI API generation. (#573)

## Problem

Currently, the SAI API generation is facing 2 issues: ABI compatibility and code review with SAI API change.

To maintain the ABI compatibility, it is required to maintain the order of API, attributes, stats in the order of its creation. The newer ones must comes later. Before the "order" annotation is introduced, this problem was never solved. With "order" annotation, although it kinda helps some cases, but it is still not good - it is hacky, and it is not generically applicable. E.g., currently, the keys and action parameters are not sorted together, counters can be introduced in multiple files making "order" not intuitive.

Furthermore, with P4 updates, it is hard to review the SAI API changes, because these changes are not captured in the PR.

## What we are doing in this change

The SAI spec YAML file is introduced for helping these cases.

First all, all changes to the SAI API will be reflected in the SAI spec YAML file, allow us to review the PR without building the SAI branch. Once the CI passes, we will know how the SAI APIs are being changed, and it works.

Secondly, it helps maintaining the ABI compatibility in future. Instead of using "order" to explicitly order the attributes, we can merge the new API spec with the old API spec by insert any new things in the end of the spec. This ensures any new changes will be put in the end, and no hacky solution needs to be introduced again.

This change will not change any generated SAI headers and libs.
![image](https://github.com/sonic-net/DASH/assets/1533278/279af16d-e0d7-4ea0-bc68-4595545e1dfc)

dash-pipeline/Makefile

@@ -176,7 +176,12 @@ sai: sai-headers sai-meta libsai
 
 sai-headers: p4 docker-saithrift-bldr-image-exists | SAI/SAI
 	@echo "Generate SAI library headers and implementation..."
+
+	# Once the specs are checked in, we can use this to revert any local changes before generating the new specs.
+	# git checkout SAI/specs/*
+
 	mkdir -p SAI/lib
+
 	$(DOCKER_RUN) \
 		$(DOCKER_FLAGS) \
 		--name build_sai-$(USER) \

dash-pipeline/SAI/Makefile

@@ -4,6 +4,7 @@ all: copysrc
 		/bmv2/dash_pipeline.bmv2/dash_pipeline_p4rt.json \
 		--ir /bmv2/dash_pipeline.bmv2/dash_pipeline_ir.json \
 		--ignore-tables=appliance,eni_meter,slb_decap \
+		--sai-spec-dir=/SAI/specs \
 		dash
 
 copysrc:

dash-pipeline/SAI/sai_api_gen.py

@@ -9,10 +9,13 @@
     import jinja2
     import typing
     import base64
+    import yaml
+    import yaml_include
     import jsonpath_ng.ext as jsonpath_ext
     import jsonpath_ng as jsonpath
     from utils.dash_p4 import DashP4SAIExtensions
     from utils.p4ir import P4IRTree, P4VarRefGraph
+    from utils.sai_spec import SaiSpec
     from utils.sai_gen import SAIGenerator
 except ImportError as ie:
     print("Import failed for " + ie.name)
@@ -27,15 +30,16 @@
     parser.add_argument("--ir", type=str, help="Path to P4 program IR JSON file")
     parser.add_argument("--print-sai-lib", type=bool)
     parser.add_argument("--ignore-tables", type=str, default="", help="Comma separated list of tables to ignore")
+    parser.add_argument("--sai-spec-dir", type=str, required=True, help="Path to output SAI spec file")
     args = parser.parse_args()
 
+    os.chdir(os.path.dirname(os.path.realpath(__file__)))
+
     p4rt_file_path = os.path.realpath(args.filepath)
     if not os.path.isfile(p4rt_file_path):
         print("File " + p4rt_file_path + " does not exist")
         exit(1)
 
-    os.chdir(os.path.dirname(os.path.realpath(__file__)))
-
     p4ir = P4IRTree.from_file(args.ir)
     var_ref_graph = P4VarRefGraph(p4ir)
 
@@ -49,5 +53,22 @@
         print("Dumping parsed SAI data:")
         print(json.dumps(dash_sai_exts, indent=2))
 
+    # Initialize YAML loader and dumper
+    sai_spec_dir = os.path.realpath(args.sai_spec_dir)
+    yaml_inc_ctor = yaml_include.Constructor(base_dir=sai_spec_dir, autoload=True)
+    yaml.add_constructor("!inc", yaml_inc_ctor)
+    yaml_inc_rpr = yaml_include.Representer("inc")
+    yaml.add_representer(yaml_include.Data, yaml_inc_rpr)
+
+    # Ensure the current SAI spec can be loaded
+    print("Loading SAI spec from " + sai_spec_dir)
+    sai_spec = SaiSpec.deserialize(sai_spec_dir)
+
+    # Output the new SAI spec
+    print("Outputting new SAI spec to " + sai_spec_dir)
+    yaml_inc_ctor.autoload = False
+    new_sai_spec = dash_sai_exts.to_sai()
+    new_sai_spec.serialize(sai_spec_dir)
+
     # Generate and update all SAI files
     SAIGenerator(dash_sai_exts).generate()

dash-pipeline/SAI/specs/dash_acl.yaml

@@ -0,0 +1,100 @@
+!!python/object:utils.sai_spec.sai_api_group.SaiApiGroup
+name: dash_acl
+description: ''
+sai_apis:
+- !!python/object:utils.sai_spec.sai_api.SaiApi
+  name: dash_acl_group
+  description: ''
+  is_object: true
+  enums: []
+  structs: []
+  attributes:
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_DASH_ACL_GROUP_IP_ADDR_FAMILY
+    description: Action parameter IP_ADDR_FAMILY
+    type: sai_ip_addr_family_t
+    attr_value_field: u32
+    default: SAI_IP_ADDR_FAMILY_IPV4
+    isresourcetype: true
+    flags: CREATE_AND_SET
+    object_name: null
+    allow_null: false
+    valid_only: null
+    deprecated: null
+  stats: []
+- !!python/object:utils.sai_spec.sai_api.SaiApi
+  name: dash_acl_rule
+  description: ''
+  is_object: true
+  enums:
+  - !!python/object:utils.sai_spec.sai_enum.SaiEnum
+    name: sai_dash_acl_rule_action_t
+    description: Attribute data for SAI_DASH_ACL_RULE_ATTR_ACTION
+    members:
+    - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
+      name: SAI_DASH_ACL_RULE_ACTION_PERMIT
+      description: ''
+      value: '0'
+    - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
+      name: SAI_DASH_ACL_RULE_ACTION_PERMIT_AND_CONTINUE
+      description: ''
+      value: '0'
+    - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
+      name: SAI_DASH_ACL_RULE_ACTION_DENY
+      description: ''
+      value: '0'
+    - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
+      name: SAI_DASH_ACL_RULE_ACTION_DENY_AND_CONTINUE
+      description: ''
+      value: '0'
+  structs: []
+  attributes:
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_DASH_ACL_RULE_ATTR_ACTION
+    description: Action
+    type: sai_dash_acl_rule_action_t
+    attr_value_field: null
+    default: SAI_DASH_ACL_RULE_ACTION_PERMIT
+    isresourcetype: false
+    flags: MANDATORY_ON_CREATE | CREATE_ONLY
+    object_name: null
+    allow_null: false
+    valid_only: null
+    deprecated: null
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_DASH_ACL_RULE_ATTR_COUNTER_ID
+    description: Attach a counter. When it is empty, then packet hits won't be counted.
+    type: sai_object_id_t
+    attr_value_field: null
+    default: SAI_NULL_OBJECT_ID
+    isresourcetype: false
+    flags: CREATE_AND_SET
+    object_name: SAI_OBJECT_TYPE_COUNTER
+    allow_null: true
+    valid_only: null
+    deprecated: null
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_DASH_ACL_RULE_ATTR_PRIORITY
+    description: Rule priority in table
+    type: sai_uint32_t
+    attr_value_field: null
+    default: null
+    isresourcetype: false
+    flags: MANDATORY_ON_CREATE | CREATE_ONLY
+    object_name: null
+    allow_null: false
+    valid_only: null
+    deprecated: null
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_DASH_ACL_RULE_ATTR_IP_ADDR_FAMILY
+    description: IP address family for resource accounting
+    type: sai_ip_addr_family_t
+    attr_value_field: null
+    default: null
+    isresourcetype: true
+    flags: READ_ONLY
+    object_name: null
+    allow_null: false
+    valid_only: null
+    deprecated: null
+  stats: []

dash-pipeline/SAI/specs/dash_direction_lookup.yaml

@@ -0,0 +1,42 @@
+!!python/object:utils.sai_spec.sai_api_group.SaiApiGroup
+name: dash_direction_lookup
+description: ''
+sai_apis:
+- !!python/object:utils.sai_spec.sai_api.SaiApi
+  name: direction_lookup_entry
+  description: ''
+  is_object: false
+  enums:
+  - !!python/object:utils.sai_spec.sai_enum.SaiEnum
+    name: sai_direction_lookup_entry_action_t
+    description: Attribute data for SAI_DIRECTION_LOOKUP_ENTRY_ATTR_ACTION
+    members:
+    - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
+      name: SAI_DIRECTION_LOOKUP_ENTRY_ACTION_SET_OUTBOUND_DIRECTION
+      description: ''
+      value: '0'
+  structs:
+  - !!python/object:utils.sai_spec.sai_struct.SaiStruct
+    name: sai_direction_lookup_entry_entry_t
+    description: Entry for direction_lookup_entry
+    members:
+    - !!python/object:utils.sai_spec.sai_struct_entry.SaiStructEntry
+      name: VNI
+      description: Exact matched key VNI
+      type: sai_uint32_t
+      objects: null
+      valid_only: null
+  attributes:
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_DIRECTION_LOOKUP_ENTRY_ATTR_ACTION
+    description: Action
+    type: sai_direction_lookup_entry_action_t
+    attr_value_field: null
+    default: SAI_DIRECTION_LOOKUP_ENTRY_ACTION_SET_OUTBOUND_DIRECTION
+    isresourcetype: false
+    flags: MANDATORY_ON_CREATE | CREATE_ONLY
+    object_name: null
+    allow_null: false
+    valid_only: null
+    deprecated: null
+  stats: []