Handle patch error messages with SDK

src/cript/api/api.py

@@ -4,7 +4,7 @@
 import uuid
 import warnings
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Set, Union
+from typing import Any, Dict, List, Optional, Union
 
 import boto3
 import jsonschema
@@ -21,7 +21,12 @@
 )
 from cript.api.paginator import Paginator
 from cript.api.utils.get_host_token import resolve_host_and_token
-from cript.api.utils.save_helper import fix_node_save
+from cript.api.utils.save_helper import (
+    _fix_node_save,
+    _get_uuid_from_error_message,
+    _identify_suppress_attributes,
+    _InternalSaveValues,
+)
 from cript.api.valid_search_modes import SearchModes
 from cript.api.vocabulary_categories import ControlledVocabularyCategories
 from cript.nodes.exceptions import CRIPTJsonNodeError, CRIPTNodeSchemaError
@@ -514,11 +519,14 @@ def save(self, project: Project) -> None:
         """
         try:
             self._internal_save(project)
-        except Exception as exc:
-            # TODO remove all pre-handled nodes.
+        except CRIPTAPISaveError as exc:
+            if exc.pre_saved_nodes:
+                for node_uuid in exc.pre_saved_nodes:
+                    # TODO remove all pre-saved nodes by their uuid.
+                    pass
             raise exc from exc
 
-    def _internal_save(self, node, known_uuid: Optional[Set[str]] = None) -> Optional[Set[str]]:
+    def _internal_save(self, node, save_values: Optional[_InternalSaveValues] = None) -> _InternalSaveValues:
         """
         Internal helper function that handles the saving of different nodes (not just project).
 
@@ -528,42 +536,77 @@ def _internal_save(self, node, known_uuid: Optional[Set[str]] = None) -> Optiona
         This works, because we keep track of "Bad UUID" handled nodes, and represent them in the JSON only as the UUID.
         """
 
-        # known_uuid are node, that we have saved to the back end before.
-        # We keep track of it, so that we can condense them to UUID only in the JSON.
-        if known_uuid is None:
-            known_uuid = set()
+        if save_values is None:
+            save_values = _InternalSaveValues()
 
         node.validate()
         # saves all the local files to cloud storage right before saving the Project node
         # Ensure that all file nodes have uploaded there payload before actual save.
         for file_node in node.find_children({"node": ["File"]}):
             file_node.ensure_uploaded(api=self)
 
-        # We assemble the JSON to be saved to back end.
-        # Note how we exclude pre-saved uuid nodes.
-        json_data = node.get_json(known_uuid=known_uuid).json
-
-        # This checks if the current node exists on the back end.
-        # if it does exist we use `patch` if it doesn't `post`.
-        node_known = len(self.search(type(node), SearchModes.UUID, str(node.uuid)).current_page_results) == 1
-        if node_known:
-            response: Dict = requests.patch(url=f"{self._host}/{node.node_type.lower()}/{str(node.uuid)}", headers=self._http_headers, data=json_data).json()
-        else:
-            response: Dict = requests.post(url=f"{self._host}/{node.node_type.lower()}", headers=self._http_headers, data=json_data).json()  # type: ignore
-
-        # If we get an error we may be able to fix, we to handle this extra and save the bad node first.
-        # Errors with this code, may be fixable
-        if response["code"] in (400, 409):
-            nodes_fixed = fix_node_save(self, node, response, known_uuid)
-            # In case of a success, we return the know uuid
-            if nodes_fixed is not False:
-                return nodes_fixed
-            # if not successful, we escalate the problem further
+        # Dummy response to have a virtual do-while loop, instead of while loop.
+        response = {"code": -1}
+        # TODO remove once get works properly
+        force_patch = False
+
+        while response["code"] != 200:
+            # Keep a record of how the state was before the loop
+            old_save_values = copy.deepcopy(save_values)
+            # We assemble the JSON to be saved to back end.
+            # Note how we exclude pre-saved uuid nodes.
+            json_data = node.get_json(known_uuid=save_values.saved_uuid, suppress_attributes=save_values.suppress_attributes).json
+
+            # This checks if the current node exists on the back end.
+            # if it does exist we use `patch` if it doesn't `post`.
+            test_get_response: Dict = requests.get(url=f"{self._host}/{node.node_type_snake_case}/{str(node.uuid)}", headers=self._http_headers).json()
+            patch_request = test_get_response["code"] == 200
+
+            # TODO remove once get works properly
+            if not patch_request and force_patch:
+                patch_request = True
+                force_patch = False
+
+            # If all that is left is a UUID, we don't need to save it, we can just exit the loop.
+            if patch_request and len(json.loads(json_data)) == 1:
+                response = {"code": 200}
+                break
+
+            if patch_request:
+                response: Dict = requests.patch(url=f"{self._host}/{node.node_type_snake_case}/{str(node.uuid)}", headers=self._http_headers, data=json_data).json()  # type: ignore
+            else:
+                response: Dict = requests.post(url=f"{self._host}/{node.node_type_snake_case}", headers=self._http_headers, data=json_data).json()  # type: ignore
+
+            # If we get an error we may be able to fix, we to handle this extra and save the bad node first.
+            # Errors with this code, may be fixable
+            if response["code"] in (400, 409):
+                returned_save_values = _fix_node_save(self, node, response, save_values)
+                save_values += returned_save_values
+
+            # Handle errors from patching with too many attributes
+            if patch_request and response["code"] in (400,):
+                suppress_attributes = _identify_suppress_attributes(node, response)
+                new_save_values = _InternalSaveValues(save_values.saved_uuid, suppress_attributes)
+                save_values += new_save_values
+
+            # It is only worthwhile repeating the attempted save loop if our state has improved.
+            # Aka we did something to fix the occurring error
+            if not save_values > old_save_values:
+                # TODO remove once get works properly
+                if not patch_request and response["code"] == 409 and response["error"].strip().startswith("Duplicate uuid:"):  # type: ignore
+                    duplicate_uuid = _get_uuid_from_error_message(response["error"])  # type: ignore
+                    if str(node.uuid) == duplicate_uuid:
+                        print("force_patch", node.uuid)
+                        force_patch = True
+                        continue
+
+                break
 
         if response["code"] != 200:
-            raise CRIPTAPISaveError(api_host_domain=self._host, http_code=response["code"], api_response=response["error"])
+            raise CRIPTAPISaveError(api_host_domain=self._host, http_code=response["code"], api_response=response["error"], patch_request=patch_request, pre_saved_nodes=save_values.saved_uuid, json_data=json_data)  # type: ignore
 
-        return known_uuid
+        save_values.saved_uuid.add(str(node.uuid))
+        return save_values
 
     def upload_file(self, file_path: Union[Path, str]) -> str:
         # trunk-ignore-begin(cspell)
@@ -717,7 +760,8 @@ def search(
         """
 
         # get node typ from class
-        node_type = node_type.node_type.lower()
+        node_type = node_type.node_type_snake_case
+        print(node_type)
 
         # always putting a page parameter of 0 for all search URLs
         page_number = 0

src/cript/api/exceptions.py

@@ -1,4 +1,4 @@
-from typing import List
+from typing import List, Optional, Set
 
 from cript.exceptions import CRIPTException
 
@@ -110,13 +110,21 @@ class CRIPTAPISaveError(CRIPTException):
     http_code: str
     api_response: str
 
-    def __init__(self, api_host_domain: str, http_code: str, api_response: str):
+    def __init__(self, api_host_domain: str, http_code: str, api_response: str, patch_request: bool, pre_saved_nodes: Optional[Set[str]] = None, json_data: Optional[str] = None):
         self.api_host_domain = api_host_domain
         self.http_code = http_code
         self.api_response = api_response
+        self.patch_request = patch_request
+        self.pre_saved_nodes = pre_saved_nodes
+        self.json_data = json_data
 
     def __str__(self) -> str:
-        error_message = f"API responded with 'http:{self.http_code} {self.api_response}'"
+        type = "POST"
+        if self.patch_request:
+            type = "PATCH"
+        error_message = f"API responded to {type} with 'http:{self.http_code} {self.api_response}'"
+        if self.json_data:
+            error_message += f" data: {self.json_data}"
 
         return error_message
 

src/cript/api/utils/save_helper.py

@@ -1,29 +1,114 @@
-def fix_node_save(api, node, response, known_uuid):
+import json
+import re
+import uuid
+from dataclasses import dataclass, field
+from typing import Dict, Set
+
+
+@dataclass
+class _InternalSaveValues:
+    """
+    Class that carries attributes to be carried through recursive calls of _internal_save.
+    """
+
+    saved_uuid: Set[str] = field(default_factory=set)
+    suppress_attributes: Dict[str, Set[str]] = field(default_factory=dict)
+
+    def __add__(self, other: "_InternalSaveValues") -> "_InternalSaveValues":
+        """
+        Implement a short hand to combine two of these save values, with `+`.
+        This unions, the `saved_uuid`.
+        And safely unions `suppress_attributes` too.
+        """
+        # Make a manual copy of `self`.
+        return_value = _InternalSaveValues(self.saved_uuid.union(other.saved_uuid), self.suppress_attributes)
+
+        # Union the dictionary.
+        for uuid_str in other.suppress_attributes:
+            try:
+                # If the uuid exists in both `suppress_attributes` union the value sets
+                return_value.suppress_attributes[uuid_str] = return_value.suppress_attributes[uuid_str].union(other.suppress_attributes[uuid_str])
+            except KeyError:
+                # If it only exists in one, just copy the set into the new one.
+                return_value.suppress_attributes[uuid_str] = other.suppress_attributes[uuid_str]
+        return return_value
+
+    def __gt__(self, other):
+        """
+        A greater comparison to see if something was added to the info.
+        """
+        if len(self.saved_uuid) > len(other.saved_uuid):
+            return True
+        if len(self.suppress_attributes) > len(other.suppress_attributes):
+            return True
+        # If the two dicts have the same key, make sure at least one key has more suppressed attributes
+        if self.suppress_attributes.keys() == other.suppress_attributes.keys():
+            longer_set_found = False
+            for key in other.suppress_attributes:
+                if len(self.suppress_attributes[key]) < len(other.suppress_attributes[key]):
+                    return False
+                if self.suppress_attributes[key] > other.suppress_attributes[key]:
+                    longer_set_found = True
+            return longer_set_found
+        return False
+
+
+def _fix_node_save(api, node, response, save_values: _InternalSaveValues) -> _InternalSaveValues:
     """
     Helper function, that attempts to fix a bad node.
     And if it is fixable, we resave the entire node.
 
     Returns set of known uuids, if fixable, otherwise False.
     """
-    assert response["code"] in (400, 409)
+    if response["code"] not in (400, 409):
+        raise RuntimeError(f"The internal helper function `_fix_node_save` has been called for an error that is not yet implemented to be handled {response}.")
+
     if response["error"].startswith("Bad uuid:") or response["error"].strip().startswith("Duplicate uuid:"):
-        missing_uuid = get_uuid_from_error_message(response["error"])
+        missing_uuid = _get_uuid_from_error_message(response["error"])
         missing_node = find_node_by_uuid(node, missing_uuid)
-
+        # If the missing node, is the same as the one we are trying to save, this not working.
+        # We end the infinite loop here.
+        if missing_uuid == str(node.uuid):
+            return save_values
         # Now we save the bad node extra.
         # So it will be known when we attempt to save the graph again.
         # Since we pre-saved this node, we want it to be UUID edge only the next JSON.
         # So we add it to the list of known nodes
-        known_uuid.union(api._internal_save(missing_node, known_uuid))  # type: ignore
+        returned_save_values = api._internal_save(missing_node, save_values)
+        save_values += returned_save_values
         # The missing node, is now known to the API
-        known_uuid.add(missing_uuid)
-        # Recursive call.
-        # Since we should have fixed the "Bad UUID" now, we can try to save the node again
-        return api._internal_save(node, known_uuid)
-    return False
+        save_values.saved_uuid.add(missing_uuid)
+
+    # Handle all duplicate items warnings if possible
+    if response["error"].startswith("duplicate item"):
+        for search_dict_str in re.findall(r"\{(.*?)\}", response["error"]):  # Regular expression finds all text elements enclosed in `{}`. In the error message this is the dictionary describing the duplicated item.
+            # The error message contains a description of the offending elements.
+            search_dict_str = "{" + search_dict_str + "}"
+            search_dict_str = search_dict_str.replace("'", '"')
+            search_dict = json.loads(search_dict_str)
+            # These are in the exact format to use with `find_children` so we find all the offending children.
+            all_duplicate_nodes = node.find_children(search_dict)
+            for duplicate_node in all_duplicate_nodes:
+                # Unfortunately, even patch errors if you patch with an offending element.
+                # So we remove the offending element from the JSON
+                # TODO IF THIS IS A TRUE DUPLICATE NAME ERROR, IT WILL ERROR AS THE NAME ATTRIBUTE IS MISSING.
+                try:
+                    # the search_dict convenient list all the attributes that are offending in the keys.
+                    # So if we haven't listed the current node in the suppress attribute dict, we add the node with the offending attributes to suppress.
+                    save_values.suppress_attributes[str(duplicate_node.uuid)] = set(search_dict.keys())
+                except KeyError:
+                    # If we have the current node in the dict, we just add the new elements to the list of suppressed attributes for it.
+                    save_values.suppress_attributes[str(duplicate_node.uuid)].add(set(search_dict.keys()))  # type: ignore
+
+                # Attempts to save the duplicate items element.
+                save_values += api._internal_save(duplicate_node, save_values)
+                # After the save, we can reduce it to just a UUID edge in the graph (avoiding the duplicate issues).
+                save_values.saved_uuid.add(str(duplicate_node.uuid))
+
+    return save_values
 
 
-def get_uuid_from_error_message(error_message: str) -> str:
+def _get_uuid_from_error_message(error_message: str) -> str:
     """
     takes an CRIPTAPISaveError and tries to get the UUID that the API is having trouble with
     and return that
@@ -37,19 +122,43 @@ def get_uuid_from_error_message(error_message: str) -> str:
     UUID
         the UUID the API had trouble with
     """
+    bad_uuid = None
     if error_message.startswith("Bad uuid: "):
         bad_uuid = error_message[len("Bad uuid: ") : -len(" provided")].strip()
     if error_message.strip().startswith("Duplicate uuid:"):
         bad_uuid = error_message[len(" Duplicate uuid:") : -len("provided")].strip()
+    if bad_uuid is None or len(bad_uuid) != len(str(uuid.uuid4())):  # Ensure we found a full UUID describing string (here tested against a random new uuid length.)
+        raise RuntimeError(f"The internal helper function `_get_uuid_from_error_message` has been called for an error message that is not yet implemented to be handled. error message {error_message}, found uuid {bad_uuid}.")
 
     return bad_uuid
 
 
-def find_node_by_uuid(node, uuid: str):
+def find_node_by_uuid(node, uuid_str: str):
     # Use the find_children functionality to find that node in our current tree
     # We can have multiple occurrences of the node,
     # but it doesn't matter which one we save
     # TODO some error handling, for the BUG case of not finding the UUID
-    missing_node = node.find_children({"uuid": uuid})[0]
+    missing_node = node.find_children({"uuid": uuid_str})[0]
 
     return missing_node
+
+
+def _identify_suppress_attributes(node, response: Dict) -> Dict[str, Set[str]]:
+    suppress_attributes: Dict[str, Set[str]] = {}
+    if response["error"].startswith("Additional properties are not allowed"):
+        # Find all the attributes, that are listed in the error message with regex
+        attributes = set(re.findall(r"'(.*?)'", response["error"]))  # regex finds all attributes in enclosing `'`. This is how the error message lists them.
+
+        # At the end of the error message the offending path is given.
+        # The structure of the error message is such, that is is after `path:`, so we find and strip the path out of the message.
+        path = response["error"][response["error"].rfind("path:") + len("path:") :].strip()
+
+        if path != "/":
+            # TODO find the UUID this belongs to
+            raise RuntimeError("Fixing non-root objects for patch, not implemented yet. This is a bug, please report it on https://github.com/C-Accel-CRIPT/Python-SDK/ .")
+
+        try:
+            suppress_attributes[str(node.uuid)].add(attributes)  # type: ignore
+        except KeyError:
+            suppress_attributes[str(node.uuid)] = attributes
+    return suppress_attributes