GH Issue #3: digest calculation (#13)

Manifest has unique tag, use unique tag to download manifest and then create hash for response content from registry
the response from the registry SHOULD exactly be the same content used by the registry to calculate the digest.

This way, we do not have different order of attributes, different whitespaces or other stuff that changes the hash.

Currently, I do not find another way to make sure that the hash used to reference the manifest in the index is compliant to
what the registry calculated.

src/gloci/oras/crypto.py

@@ -44,3 +44,18 @@ def calculate_sha256(file_path):
     except FileNotFoundError:
         print(f"File {file_path} not found.")
         return None
+
+
+def calculate_sha256(file_path):
+    """Calculate the SHA256 checksum of a file."""
+    sha256_hash = hashlib.sha256()
+    try:
+        with open(file_path, "rb") as f:
+            # Read the file in chunks of 4KB
+            for byte_block in iter(lambda: f.read(4096), b""):
+                sha256_hash.update(byte_block)
+        # Return the hexadecimal digest of the checksum
+        return sha256_hash.hexdigest()
+    except FileNotFoundError:
+        print(f"File {file_path} not found.")
+        return None

src/gloci/oras/helper.py

@@ -1,4 +1,13 @@
 import re
+import json
+import os
+
+
+def write_dict_to_json_file(input, output_path):
+    if os.path.exists(output_path):
+        raise ValueError(f"{output_path} already exists")
+    with open(output_path, "w") as fp:
+        json.dump(input, fp)
 
 
 def get_uri_for_digest(uri, digest):

src/gloci/oras/registry.py

@@ -9,6 +9,7 @@
 from oras.logger import setup_logger, logger
 
 import jsonschema
+import json
 import requests
 
 import os
@@ -18,6 +19,7 @@
 import uuid
 import re
 from enum import Enum, auto
+from hashlib import sha256
 
 from gloci.oras.crypto import calculate_sha1, calculate_md5, calculate_sha256
 from gloci.oras.schemas import index as indexSchema
@@ -102,6 +104,13 @@ def get_digest(self, container, allowed_media_type=None):
         self._check_200_response(response)
         return f"sha256:{hashlib.sha256(response.content).hexdigest()}"
 
+    def calculate_manifest_digest(self, manifest_image):
+        # Make sure manifest contains required attributes
+        jsonschema.validate(manifest_image, schema=oras.schemas.manifest)
+        content = json.dumps(manifest_image, sort_keys=True).encode()
+        digest = sha256(content).hexdigest()
+        return f"sha256:{digest}"
+
     @ensure_container
     def get_index(self, container, allowed_media_type=None):
         """
@@ -139,6 +148,7 @@ def get_manifest_by_cname(self, container, cname, arch, allowed_media_type=None)
             return None
 
         for manifest_meta in index["manifests"]:
+            logger.debug(manifest_meta)
             if "annotations" not in manifest_meta:
                 logger.debug("Manifest annotations was none, which is invalid")
                 return None
@@ -155,7 +165,8 @@ def get_manifest_by_cname(self, container, cname, arch, allowed_media_type=None)
                 manifest_meta["annotations"]["cname"] == cname
                 and manifest_meta["annotations"]["architecture"] == arch
             ):
-                manifest_digest = manifest_meta["digest"]
+                manifest_digest = f"{manifest_meta['digest']}"
+                logger.debug(f"manifest meta: {manifest_meta}")
                 logger.debug(f"registry: {container.registry}")
                 logger.debug(f"repository: {container.repository}")
                 logger.debug(f"tag: {container.tag}")
@@ -245,6 +256,26 @@ def _check_if_manifest_exists(self, index, manifest_meta):
 
         return False
 
+    def create_config_from_dict(self, conf: dict, annotations: dict):
+        """
+        Write a new OCI configuration to file, and generate oci meta data for it
+        For reference see https://github.com/opencontainers/image-spec/blob/main/config.md
+        annotations, mediatrype, size, digest are not part of digest and size calculation,
+        and therefore must be attached to the output dict and not written to the file.
+
+        :param conf: dict with custom configuration (the payload of the configuration)
+        :param annotations: dict with custom annotations to be attached to metadata part of config
+
+        """
+        config_path = os.path.join(os.path.curdir, str(uuid.uuid4()))
+        with open(config_path, "w") as fp:
+            json.dump(conf, fp)
+        conf["annotations"] = annotations
+        conf["mediaType"] = oras.defaults.unknown_config_media_type
+        conf["size"] = oras.utils.get_size(config_path)
+        conf["digest"] = f"sha256:{oras.utils.get_file_hash(config_path)}"
+        return conf, config_path
+
     def _get_index(self, container):
         """
         Ensures an oci index exists for the container, and returns it
@@ -269,30 +300,17 @@ def push_image_manifest(self, container_name, architecture, cname, info_yaml):
         creates and pushes an image manifest
         """
         container = oras.container.Container(container_name)
-        logger.debug("start push image manifest")
+        logger.debug(f"container name: {container_name}")
         assert info_yaml is not None, "error: info_yaml is None"
         with open(info_yaml, "r") as f:
             info_data = yaml.safe_load(f)
             base_path = os.path.join(os.path.dirname(info_yaml))
-        conf, config_file = oras.oci.ManifestConfig()
 
         image_index = self._get_index(container)
 
         logger.debug("Creating new Manifest")
         manifest_image = oras.oci.NewManifest()
 
-        logger.debug("Create metadata info Layer")
-        layer = oras.oci.NewLayer(
-            info_yaml, "application/vnd.gardenlinux.metadata.info", is_dir=False
-        )
-        manifest_image["layers"].append(layer)
-
-        logger.debug("Upload metadata info Layer")
-        assert container is not None, "error: container is none"
-        assert layer is not None, "error: layer is none"
-        # response = self.upload_blob(info_yaml, container, layer)
-        # self._check_200_response(response)
-
         missing_layer_detected = False
 
         logger.debug("Iterate over all artifacts specified in info yaml...")
@@ -329,7 +347,7 @@ def push_image_manifest(self, container_name, architecture, cname, info_yaml):
             layer["annotations"] = {
                 oras.defaults.annotation_title: file_name,
                 "application/vnd.gardenlinux.image.checksum.sha256": checksum_sha256,
-                "application/vnd.gardenlinux.image.checksum.sha1": checksum_md5,
+                "application/vnd.gardenlinux.image.checksum.sha1": checksum_sha1,
                 "application/vnd.gardenlinux.image.checksum.md5": checksum_md5,
             }
             if annotations:
@@ -346,58 +364,68 @@ def push_image_manifest(self, container_name, architecture, cname, info_yaml):
             if cleanup_blob and os.path.exists(file_path):
                 os.remove(file_path)
 
-        config_file = os.path.join(os.path.curdir, str(uuid.uuid4()))
-        if not os.path.exists(config_file):
-            with open(config_file, "w"):
-                pass
+        logger.debug("Create metadata info Layer")
+        layer = oras.oci.NewLayer(
+            info_yaml, "application/vnd.gardenlinux.metadata.info", is_dir=False
+        )
+        manifest_image["layers"].append(layer)
 
-        conf, _ = oras.oci.ManifestConfig(path=config_file)
-        manifest_digest = conf["digest"]
-        logger.debug(f"manifest digest: {manifest_digest}")
-        conf["annotations"] = {}
+        logger.debug("Upload metadata info Layer")
+        assert container is not None, "error: container is none"
+        assert layer is not None, "error: layer is none"
+        response = self.upload_blob(info_yaml, container, layer)
+        self._check_200_response(response)
 
-        # TODO: annotations may be part of digest calculation. We need to consider how digest is caclulated. according to specs its:
-        #   1. content addressable, so it should be the digest of the target manifest_digest
-        #   2. based on conf content and layers of image, so we need know the digest of the manifest and its config before we upload it
-        conf["annotations"]["cname"] = cname
+        config_annotations = {}
+        config_annotations["cname"] = cname
+        config_annotations["architecture"] = architecture
+        conf, config_file = self.create_config_from_dict(dict(), config_annotations)
 
-        # Config is just another layer blob!
+        # manifest_digest = self.calculate_manifest_digest(manifest_image, conf["digest"])
+        # logger.debug(f"manifest digest: {manifest_digest}")
+        logger.debug("upload config blob")
         response = self.upload_blob(config_file, container, conf)
         self._check_200_response(response)
+        logger.debug(response)
 
-        os.remove(config_file)
-        # Final upload of the manifest
+        logger.debug("upload manifest")
         manifest_image["config"] = conf
 
-        manifest_container = oras.container.Container(f"{container_name}-{cname}")
+        manifest_container_name = f"{container_name}-{cname}"
+        manifest_container = oras.container.Container(
+            f"{container_name}-{cname}-{architecture}"
+        )
+        logger.debug(f"Manifest image tag {manifest_container_name}")
 
-        logger.debug(f"Manifest Digest: {manifest_image['config']['digest']}")
+        self._check_200_response(
+            self.upload_manifest(manifest_image, manifest_container)
+        )
+        manifest_image["digest"] = self.get_digest(manifest_container)
+
+        logger.debug(f"Manifest Digest in manifest: {manifest_image['digest']}")
+
+        self._check_200_response(
+            self.upload_manifest(manifest_image, manifest_container)
+        )
+
+        logger.debug("TODO: get digest of manifest")
 
         # attach Manifest to oci-index
         manifest_index_metadata = NewManifestMetadata()
         manifest_index_metadata["mediaType"] = (
             "application/vnd.oci.image.manifest.v1+json"
         )
-        manifest_index_metadata["digest"] = manifest_digest
+        manifest_index_metadata["digest"] = manifest_image["digest"]
+        logger.debug(
+            f"Manifest Digest in MetaData: {manifest_index_metadata['digest']}"
+        )
         manifest_index_metadata["size"] = 0
         manifest_index_metadata["annotations"] = {}
         manifest_index_metadata["annotations"]["cname"] = cname
         manifest_index_metadata["annotations"]["architecture"] = architecture
         manifest_index_metadata["annotations"]["version"] = "experimental"
         manifest_index_metadata["platform"] = NewPlatform(architecture)
         manifest_index_metadata["artifactType"] = ""
-
-        if self._check_if_manifest_exists(image_index, manifest_index_metadata):
-            logger.debug(
-                f"Manifest with digest {manifest_digest} already exists. Not uploading again."
-            )
-            logger.debug(manifest_index_metadata)
-            return
-
-        self._check_200_response(
-            self.upload_manifest(manifest_image, manifest_container)
-        )
-
         image_index["manifests"].append(manifest_index_metadata)
         logger.debug("Show Image Index")
         logger.debug(image_index)