Merge branch 'develop' into keep-alive

docs/examples/synthesis.md

@@ -132,7 +132,7 @@ These materials are simple, notice how we use the SMILES notation here as an ide
 Similarly, we can create more initial materials.
 
 ```python
-toluene = cript.Material(name="toluene", identifier=[{"smiles": "Cc1ccccc1"}, {"pubchem_id": 1140}])
+toluene = cript.Material(name="toluene", identifier=[{"smiles": "Cc1ccccc1"}, {"pubchem_cid": 1140}])
 styrene = cript.Material(name="styrene", identifier=[{"smiles": "c1ccccc1C=C"}, {"inchi": "InChI=1S/C8H8/c1-2-8-6-4-3-5-7-8/h2-7H,1H2"}])
 butanol = cript.Material(name="1-butanol", identifier=[{"smiles": "OCCCC"}, {"inchi_key": "InChIKey=LRHPLDYGYMQRHN-UHFFFAOYSA-N"}])
 methanol = cript.Material(name="methanol", identifier=[{"smiles": "CO"}, {"names": ["Butan-1-ol", "Butyric alcohol", "Methylolpropane", "n-Butan-1-ol", "methanol"]}])

src/cript/api/data_schema.py

@@ -211,11 +211,21 @@ def is_node_schema_valid(self, node_json: str, is_patch: bool = False, force_val
 
         # logging out info to the terminal for the user feedback
         # (improve UX because the program is currently slow)
-        log_message = f"Validating {node_type} graph..."
+        log_message = f"Validating {node_type} graph"
+        try:
+            log_message += " '" + str(node_dict["name"]) + "'"
+        except KeyError:
+            try:
+                log_message += " '" + str(node_dict["uid"]) + "'"
+            except KeyError:
+                pass
+
+        log_message += " ... "
+
         if force_validation:
-            log_message = "Forced: " + log_message + " if error occur, try setting `cript.API.skip_validation = False` for debugging."
+            log_message = "Forced: " + log_message + " if error occur, try setting `cript_api.schema.skip_validation = False` for debugging."
         else:
-            log_message += " (Can be disabled by setting `cript.API.skip_validation = True`.)"
+            log_message += " (Can be disabled by setting `cript_api.schema.skip_validation = True`.)"
 
         self._api.logger.info(log_message)
 

src/cript/api/paginator.py

@@ -33,6 +33,8 @@ class Paginator:
     _current_position: int
     _fetched_nodes: list
     _number_fetched_pages: int = 0
+    _limit_page_fetches: Union[int, None] = None
+    _num_skip_pages: int = 0
     auto_load_nodes: bool = True
 
     @beartype
@@ -103,11 +105,15 @@ def _fetch_next_page(self) -> None:
              None
         """
 
+        # Check if we are supposed to fetch more pages
+        if self._limit_page_fetches and self._number_fetched_pages >= self._limit_page_fetches:
+            raise StopIteration
+
         # Composition of the query URL
         temp_url_path: str = self._url_path
         temp_url_path += f"/?q={self._query}"
         if self._initial_page_number is not None:
-            temp_url_path += f"&page={self._initial_page_number + self._number_fetched_pages}"
+            temp_url_path += f"&page={self.page_number}"
         self._number_fetched_pages += 1
 
         response: requests.Response = self._api._capsule_request(url_path=temp_url_path, method="GET")
@@ -136,9 +142,9 @@ def _fetch_next_page(self) -> None:
 
         if api_response["code"] == 404 and api_response["error"] == "The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.":
             current_page_results = []
-
+            self._api.logger.debug(f"The paginator hit a 404 HTTP for requesting this {temp_url_path} with GET. We interpret it as no nodes present, but this is brittle at the moment.")
         # if API response is not 200 raise error for the user to debug
-        if api_response["code"] != 200:
+        elif api_response["code"] != 200:
             raise APIError(api_error=str(response.json()), http_method="GET", api_url=temp_url_path)
 
         # Here we only load the JSON into the temporary results.
@@ -174,3 +180,66 @@ def __next__(self):
     def __iter__(self):
         self._current_position = 0
         return self
+
+    @property
+    def page_number(self) -> Union[int, None]:
+        """Obtain the current page number the paginator is fetching next.
+
+        Returns
+        -------
+        int
+          positive number of the next page this paginator is fetching.
+        None
+          if no page number is associated with the pagination
+        """
+        page_number = self._num_skip_pages + self._number_fetched_pages
+        if self._initial_page_number is not None:
+            page_number += self._initial_page_number
+        return page_number
+
+    @beartype
+    def limit_page_fetches(self, max_num_pages: Union[int, None]) -> None:
+        """Limit pagination to a maximum number of pages.
+
+        This can be used for very large searches with the paginator, so the search can be split into
+        smaller portions.
+
+        Parameters
+        ----------
+        max_num_pages: Union[int, None],
+          positive integer with maximum number of page fetches.
+          or None, indicating unlimited number of page fetches are permitted.
+        """
+        self._limit_page_fetches = max_num_pages
+
+    def skip_pages(self, skip_pages: int) -> int:
+        """Skip pages in the pagination.
+
+        Warning this function is advanced usage and may not produce the results you expect.
+        In particular, every search is different, even if we search for the same values there is
+        no guarantee that the results are in the same order. (And results can change if data is
+        added or removed from CRIPT.) So if you break up your search with `limit_page_fetches` and
+        `skip_pages` there is no guarantee that it is the same as one continuous search.
+        If the paginator associated search does not accept pages, there is no effect.
+
+        Parameters
+        ----------
+        skip_pages:int
+          Number of pages that the paginator skips now before fetching the next page.
+          The parameter is added to the internal state, so repeated calls skip more pages.
+
+        Returns
+        -------
+        int
+          The number this paginator is skipping. Internal skip count.
+
+        Raises
+        ------
+        RuntimeError
+          If the total number of skipped pages is negative.
+        """
+        num_skip_pages = self._num_skip_pages + skip_pages
+        if self._num_skip_pages < 0:
+            RuntimeError(f"Invalid number of skipped pages. The total number of pages skipped is negative {num_skip_pages}, requested to skip {skip_pages}.")
+        self._num_skip_pages = num_skip_pages
+        return self._num_skip_pages

src/cript/nodes/primary_nodes/collection.py

@@ -105,16 +105,15 @@ def __init__(self, name: str, experiment: Optional[List[Any]] = None, inventory:
         if citation is None:
             citation = []
 
-        self._json_attrs = replace(
+        new_json_attrs = replace(
             self._json_attrs,
             name=name,
             experiment=experiment,
             inventory=inventory,
             doi=doi,
             citation=citation,
         )
-
-        self.validate()
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     @property
     @beartype

src/cript/nodes/primary_nodes/computation.py

@@ -141,7 +141,7 @@ def __init__(
         if citation is None:
             citation = []
 
-        self._json_attrs = replace(
+        new_json_attrs = replace(
             self._json_attrs,
             type=type,
             input_data=input_data,
@@ -151,8 +151,7 @@ def __init__(
             prerequisite_computation=prerequisite_computation,
             citation=citation,
         )
-
-        self.validate()
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     # ------------------ Properties ------------------
 

src/cript/nodes/primary_nodes/computation_process.py

@@ -154,7 +154,7 @@ def __init__(
         >>> input_data = cript.Data(name="my data name", type="afm_amp", file=[data_files])
         >>> my_material = cript.Material(
         ...     name="my material",
-        ...     identifier=[{"alternative_names": "my material alternative name"}]
+        ...     identifier=[{"names": ["my material alternative name"]}]
         ... )
         >>> my_quantity = cript.Quantity(key="mass", value=1.23, unit="kg")
         >>> ingredient = cript.Ingredient(
@@ -220,7 +220,7 @@ def __init__(
         if citation is None:
             citation = []
 
-        self._json_attrs = replace(
+        new_json_attrs = replace(
             self._json_attrs,
             type=type,
             input_data=input_data,
@@ -231,8 +231,7 @@ def __init__(
             property=property,
             citation=citation,
         )
-
-        # self.validate()
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     @property
     @beartype

src/cript/nodes/primary_nodes/data.py

@@ -167,7 +167,7 @@ def __init__(
         if citation is None:
             citation = []
 
-        self._json_attrs = replace(
+        new_json_attrs = replace(
             self._json_attrs,
             type=type,
             file=file,
@@ -178,8 +178,7 @@ def __init__(
             process=process,
             citation=citation,
         )
-
-        self.validate()
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     @property
     @beartype

src/cript/nodes/primary_nodes/experiment.py

@@ -135,7 +135,7 @@ def __init__(
 
         super().__init__(name=name, notes=notes, **kwargs)
 
-        self._json_attrs = replace(
+        new_json_attrs = replace(
             self._json_attrs,
             name=name,
             process=process,
@@ -147,8 +147,7 @@ def __init__(
             notes=notes,
         )
 
-        # check if the code is still valid
-        self.validate()
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     @property
     @beartype

src/cript/nodes/primary_nodes/inventory.py

@@ -99,7 +99,8 @@ def __init__(self, name: str, material: List[Material], notes: str = "", **kwarg
 
         super().__init__(name=name, notes=notes, **kwargs)
 
-        self._json_attrs = replace(self._json_attrs, material=material)
+        new_json_attrs = replace(self._json_attrs, material=material)
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     @property
     @beartype

src/cript/nodes/primary_nodes/material.py

@@ -126,7 +126,7 @@ def __init__(
         if keyword is None:
             keyword = []
 
-        self._json_attrs = replace(
+        new_json_attrs = replace(
             self._json_attrs,
             name=name,
             identifier=identifier,
@@ -137,6 +137,7 @@ def __init__(
             computational_forcefield=computational_forcefield,
             keyword=keyword,
         )
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     @property
     @beartype

src/cript/nodes/primary_nodes/project.py

@@ -101,8 +101,8 @@ def __init__(self, name: str, collection: Optional[List[Collection]] = None, mat
         if material is None:
             material = []
 
-        self._json_attrs = replace(self._json_attrs, name=name, collection=collection, material=material)
-        self.validate()
+        new_json_attrs = replace(self._json_attrs, name=name, collection=collection, material=material)
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     def validate(self, api=None, is_patch=False, force_validation: bool = False):
         from cript.nodes.exceptions import (

src/cript/nodes/primary_nodes/reference.py

@@ -168,7 +168,6 @@ def __init__(
         new_attrs = replace(self._json_attrs, type=type, title=title, author=author, journal=journal, publisher=publisher, year=year, volume=volume, issue=issue, pages=pages, doi=doi, issn=issn, arxiv_id=arxiv_id, pmid=pmid, website=website)
 
         self._update_json_attrs_if_valid(new_attrs)
-        self.validate()
 
     @property
     @beartype

src/cript/nodes/subobjects/algorithm.py

@@ -106,8 +106,8 @@ def __init__(self, key: str, type: str, parameter: Optional[List[Parameter]] = N
         if citation is None:
             citation = []
         super().__init__(**kwargs)
-        self._json_attrs = replace(self._json_attrs, key=key, type=type, parameter=parameter)
-        self.validate()
+        new_json_attrs = replace(self._json_attrs, key=key, type=type, parameter=parameter)
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     @property
     def key(self) -> str:

src/cript/nodes/subobjects/citation.py

@@ -104,8 +104,8 @@ def __init__(self, type: str, reference: Reference, **kwargs):
             Instantiate citation subobject
         """
         super().__init__(**kwargs)
-        self._json_attrs = replace(self._json_attrs, type=type, reference=reference)
-        self.validate()
+        new_json_attrs = replace(self._json_attrs, type=type, reference=reference)
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     @property
     @beartype

src/cript/nodes/subobjects/computational_forcefield.py

@@ -140,7 +140,7 @@ def __init__(self, key: str, building_block: str, coarse_grained_mapping: str =
         if data is None:
             data = []
 
-        self._json_attrs = replace(
+        new_json_attrs = replace(
             self._json_attrs,
             key=key,
             building_block=building_block,
@@ -151,7 +151,7 @@ def __init__(self, key: str, building_block: str, coarse_grained_mapping: str =
             data=data,
             citation=citation,
         )
-        self.validate()
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     @property
     @beartype

src/cript/nodes/subobjects/condition.py

@@ -151,7 +151,7 @@ def __init__(
         if data is None:
             data = []
 
-        self._json_attrs = replace(
+        new_json_attrs = replace(
             self._json_attrs,
             key=key,
             type=type,
@@ -164,7 +164,7 @@ def __init__(
             measurement_id=measurement_id,
             data=data,
         )
-        self.validate()
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     @property
     @beartype

src/cript/nodes/subobjects/equipment.py

@@ -94,8 +94,8 @@ def __init__(self, key: str, description: str = "", condition: Union[List[Condit
         if citation is None:
             citation = []
         super().__init__(**kwargs)
-        self._json_attrs = replace(self._json_attrs, key=key, description=description, condition=condition, file=file, citation=citation)
-        self.validate()
+        new_json_attrs = replace(self._json_attrs, key=key, description=description, condition=condition, file=file, citation=citation)
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     @property
     @beartype

src/cript/nodes/subobjects/ingredient.py

@@ -106,8 +106,8 @@ def __init__(self, material: Material, quantity: List[Quantity], keyword: Option
         super().__init__(**kwargs)
         if keyword is None:
             keyword = []
-        self._json_attrs = replace(self._json_attrs, material=material, quantity=quantity, keyword=keyword)
-        self.validate()
+        new_json_attrs = replace(self._json_attrs, material=material, quantity=quantity, keyword=keyword)
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     @classmethod
     def _from_json(cls, json_dict: dict):

src/cript/nodes/subobjects/parameter.py

@@ -92,8 +92,8 @@ def __init__(self, key: str, value: Number, unit: Optional[str] = None, **kwargs
             create Parameter sub-object
         """
         super().__init__(**kwargs)
-        self._json_attrs = replace(self._json_attrs, key=key, value=value, unit=unit)
-        self.validate()
+        new_json_attrs = replace(self._json_attrs, key=key, value=value, unit=unit)
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     @classmethod
     def _from_json(cls, json_dict: dict):

src/cript/nodes/subobjects/property.py

@@ -167,7 +167,7 @@ def __init__(
             citation = []
 
         super().__init__(**kwargs)
-        self._json_attrs = replace(
+        new_json_attrs = replace(
             self._json_attrs,
             key=key,
             type=type,
@@ -185,7 +185,7 @@ def __init__(
             citation=citation,
             notes=notes,
         )
-        self.validate()
+        self._update_json_attrs_if_valid(new_json_attrs)
 
     @property
     @beartype