Use Stable Storage for the canister Credential data. Retrieves the da…

…ta as needed from storage without holding everything in memory at once (#25)

* use ic-stable-structes::BTreeMap for to store the credential data and retrieve it as needed to the canister memory

* fix the byte CredentialList size (in bytes) to Unbounded

* fix

* add the hooks for serialization; missing Serialize/Deserialize traits for the data types

* restore SignaturesMap through storing and applying individual signatures; Id alias error

* cleanup

* use upgrade script to pass the right config arguments upon canister re-initialization when upgrading

* print all the credentials of user

* add logs where serialization is happening

* fix the storage conflict by declaring the config memory  space with the memory manager

* cleanup

scripts/upgrade-civic.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# source .env.local 
+
+#########
+# USAGE #
+#########
+
+function title() {
+    echo "Provisioning Civic canister" >&2
+}
+
+function usage() {
+    cat >&2 << EOF
+
+Usage:
+  $0 [--ii-canister-id CANISTER_ID] [--dfx-network NETWORK]
+
+Options:
+  --ii-canister-id CANISTER_ID  The canister ID to use as IDP, defaults to the local internet_identity canister
+  --dfx-network NETWORK         The network to use (typically "local" or "ic"), defaults to "local"
+  --issuer-canister CANISTER    The canister to configure (name or canister ID), defaults to "issuer"
+EOF
+}
+
+function help() {
+    cat >&2 << EOF
+
+The issuer canister needs some information to operate correctly. This reads data
+from the replica to ensure the issuer is provisioned correctly.
+EOF
+}
+
+II_CANISTER_ID=
+DFX_NETWORK=
+ADMIN_PRINCIPAL_ID=tglqb-kbqlj-to66e-3w5sg-kkz32-c6ffi-nsnta-vj2gf-vdcc5-5rzjk-jae
+
+while [[ $# -gt 0  ]]
+do
+    case "$1" in
+        -h|--help)
+            title
+            usage
+            help
+            exit 0
+            ;;
+        --ii-canister-id)
+            II_CANISTER_ID="${2:?missing value for '--ii-canister-id'}"
+            shift; # shift past --ii-canister-id & value
+            shift;
+            ;;
+        --dfx-network)
+            DFX_NETWORK="${2:?missing value for '--dfx-network'}"
+            shift; # shift past --dfx-network & value
+            shift;
+            ;;
+        --issuer-canister)
+            ISSUER_CANISTER_ID="${2:?missing value for '--issuer-canister-id'}"
+            shift; # shift past --issuer-canister & value
+            shift;
+            ;;
+        *)
+            echo "ERROR: unknown argument $1"
+            usage
+            echo
+            echo "Use '$0 --help' for more information"
+            exit 1
+            ;;
+    esac
+done
+
+# Make sure we always run from the repo's root
+SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$SCRIPTS_DIR/.."
+
+DFX_NETWORK="${DFX_NETWORK:-local}"
+II_CANISTER_ID="${II_CANISTER_ID:-$(dfx canister id internet_identity --network "$DFX_NETWORK")}"
+ISSUER_CANISTER_ID="${ISSUER_CANISTER_ID:-$(dfx canister id civic_canister_backend --network "$DFX_NETWORK")}"
+CIVIC_FRONTEND_CANISTER_ID="${CIVIC_FRONTEND_CANISTER_ID:-$(dfx canister id civic_canister_frontend --network "$DFX_NETWORK")}"
+if [ "$DFX_NETWORK" = "local" ]; then
+  REPLICA_SERVER_PORT=$(dfx info webserver-port)
+  ISSUER_DERIVATION_ORIGIN="http://${ISSUER_CANISTER_ID}.localhost:${REPLICA_SERVER_PORT}"
+  ISSUER_FRONTEND_HOSTNAME="http://${ISSUER_CANISTER_ID}.localhost:${REPLICA_SERVER_PORT}"
+fi
+if [ "$DFX_NETWORK" = "ic" ]; then
+  ISSUER_DERIVATION_ORIGIN="https://${ISSUER_CANISTER_ID}.icp0.io"
+  ISSUER_FRONTEND_HOSTNAME="https://${ISSUER_CANISTER_ID}.icp0.io"
+fi
+
+
+echo "Using DFX network: $DFX_NETWORK" >&2
+echo "Using II canister: $II_CANISTER_ID" >&2
+echo "Using issuer canister: $ISSUER_CANISTER_ID" >&2
+echo "Using derivation origin: $ISSUER_DERIVATION_ORIGIN" >&2
+echo "Using frontend hostname: $ISSUER_FRONTEND_HOSTNAME" >&2
+
+# At the time of writing dfx outputs incorrect JSON with dfx ping (commas between object
+# entries are missing).
+# In order to read the root key we grab the array from the '"root_key": [...]' bit, the brackets
+# to match what candid expects ({}), replace the commas between array entries to match
+# what candid expects (semicolon) and annotate the numbers with their type (otherwise dfx assumes 'nat'
+# instead of 'nat8').
+rootkey_did=$(dfx ping "$DFX_NETWORK" \
+    | sed -n 's/.*"root_key": \[\(.*\)\].*/{\1}/p' \
+    | sed 's/\([0-9][0-9]*\)/\1:nat8/g' \
+    | sed 's/,/;/g')
+
+echo "Parsed rootkey: ${rootkey_did:0:20}..." >&2
+
+# Add also dev server to alternative origins when deploying locally
+if [ "$DFX_NETWORK" = "local" ]; then
+  ALTERNATIVE_ORIGINS="\"http://$CIVIC_FRONTEND_CANISTER_ID.localhost:4943\""
+  else
+  ALTERNATIVE_ORIGINS="\"https://$CIVIC_FRONTEND_CANISTER_ID.icp0.io\""
+fi
+
+echo "Using Alternative Origin: $ALTERNATIVE_ORIGINS $ISSUER_FRONTEND_HOSTNAME"
+
+# Adjust issuer's .well-known/ii-alternative-origins to contain FE-hostname of local/dev deployments.
+# We had a problem with `sed` command in CI. This is a hack to make it work locally and in CI.
+mv src/civic_canister_backend/dist/.well-known/ii-alternative-origins ./ii-alternative-origins-template
+cat ./ii-alternative-origins-template | sed "s+ISSUER_FE_HOSTNAME_PLACEHOLDER+$ALTERNATIVE_ORIGINS+g"  > src/civic_canister_backend/dist/.well-known/ii-alternative-origins
+rm ./ii-alternative-origins-template
+
+dfx deploy --upgrade-unchanged civic_canister_backend --network "$DFX_NETWORK" --argument '(
+    opt record {
+        idp_canister_ids = vec { principal "'"$II_CANISTER_ID"'" };
+        ic_root_key_der = vec '"$rootkey_did"';
+        derivation_origin = "'"$ISSUER_DERIVATION_ORIGIN"'";
+        frontend_hostname = "'"$ISSUER_FRONTEND_HOSTNAME"'";
+        admin = principal "'"$ADMIN_PRINCIPAL_ID"'";
+        authorized_issuers = vec { principal "'"$ADMIN_PRINCIPAL_ID"'" };
+    }
+)'
+# Revert changes
+git checkout src/civic_canister_backend/dist/.well-known/ii-alternative-origins

src/civic_canister_backend/Cargo.toml

@@ -42,6 +42,7 @@ lazy_static = "1.4"
 mockall = "0.10.2"
 serde_json.workspace = true
 sha2.workspace = true
+ciborium = "0.2.2"
 
 
 

src/civic_canister_backend/dist/.well-known/ii-alternative-origins

@@ -1,3 +1,3 @@
 {
-    "alternativeOrigins": [ISSUER_FE_HOSTNAME_PLACEHOLDER]
+    "alternativeOrigins": ["http://bd3sg-teaaa-aaaaa-qaaba-cai.localhost:4943"]
 }

src/civic_canister_backend/src/config.rs

@@ -6,7 +6,7 @@
 //! - Managing assets and their certification.
 //! - Handling HTTP requests with CORS support.
 
-use crate::credential::{update_root_hash, StoredCredential};
+use crate::credential::{update_root_hash, CredentialList, CANISTER_SIG_SEED};
 use asset_util::{collect_assets, CertifiedAssets};
 use candid::{candid_method, CandidType, Deserialize, Principal};
 use canister_sig_util::signature_map::{SignatureMap, LABEL_SIG};
@@ -15,34 +15,50 @@ use ic_cdk::api;
 use ic_cdk_macros::{init, post_upgrade, query, update};
 use ic_certification::{labeled_hash, pruned};
 use ic_stable_structures::storable::Bound;
-use ic_stable_structures::{DefaultMemoryImpl, RestrictedMemory, StableCell, Storable};
+use ic_stable_structures::{
+    memory_manager::{MemoryId, MemoryManager, VirtualMemory},
+    DefaultMemoryImpl, StableBTreeMap, StableCell, StableVec, Storable,
+};
 use include_dir::{include_dir, Dir};
 use serde_bytes::ByteBuf;
 use std::borrow::Cow;
 use std::cell::RefCell;
-use std::collections::HashMap;
 use vc_util::issuer_api::{DerivationOriginData, DerivationOriginError, DerivationOriginRequest};
 
 const PROD_II_CANISTER_ID: &str = "rdmx6-jaaaa-aaaaa-aaadq-cai";
 
+// A memory for config, where data from the heap can be serialized/deserialized.
+const CONF: MemoryId = MemoryId::new(0);
+// A memory for Signatures, where data from the heap can be serialized/deserialized.
+const SIG: MemoryId = MemoryId::new(1);
+
+// A memory for the Credential data
+const CREDENTIAL: MemoryId = MemoryId::new(2);
+
+type ConfigCell = StableCell<IssuerConfig, VirtualMemory<DefaultMemoryImpl>>;
+
 thread_local! {
-    /// Static configuration of the canister set by init() or post_upgrade().
-    pub(crate) static CONFIG: RefCell<ConfigCell> = RefCell::new(ConfigCell::init(config_memory(), IssuerConfig::default()).expect("failed to initialize stable cell"));
+    static MEMORY_MANAGER: RefCell<MemoryManager<DefaultMemoryImpl>> =
+        RefCell::new(MemoryManager::init(DefaultMemoryImpl::default()));
+
+    pub(crate) static CONFIG: RefCell<ConfigCell> = RefCell::new(ConfigCell::init(MEMORY_MANAGER.with(|m| m.borrow().get(CONF)), IssuerConfig::default()).expect("failed to initialize stable cell"));
     pub(crate) static SIGNATURES : RefCell<SignatureMap> = RefCell::new(SignatureMap::default());
 
-    pub static CREDENTIALS : RefCell<HashMap<Principal, Vec<StoredCredential>>> = RefCell::new(HashMap::new());
+    pub(crate) static CREDENTIALS: RefCell<StableBTreeMap<Principal, CredentialList, VirtualMemory<DefaultMemoryImpl>>> = RefCell::new(
+        StableBTreeMap::init(
+            MEMORY_MANAGER.with(|m| m.borrow().get(CREDENTIAL))
+        )
+    );
     // Assets for the management app
-    pub static ASSETS: RefCell<CertifiedAssets> = RefCell::new(CertifiedAssets::default());
-}
+    pub(crate) static ASSETS: RefCell<CertifiedAssets> = RefCell::new(CertifiedAssets::default());
 
-/// We use restricted memory in order to ensure the separation between non-managed config memory (first page)
-/// and the managed memory for potential other data of the canister.
-type Memory = RestrictedMemory<DefaultMemoryImpl>;
-type ConfigCell = StableCell<IssuerConfig, Memory>;
 
-/// Reserve the first stable memory page for the configuration stable cell.
-fn config_memory() -> Memory {
-    RestrictedMemory::new(DefaultMemoryImpl::default(), 0..1)
+    // Stable vector to restore the signatures when the canister is upgraded
+    pub(crate) static MSG_HASHES: RefCell<StableVec<[u8; 32], VirtualMemory<DefaultMemoryImpl>>> = RefCell::new(
+        StableVec::init(
+            MEMORY_MANAGER.with(|m| m.borrow().get(SIG))
+        ).expect("failed to initialize stable vector")
+    );
 }
 
 #[cfg(target_arch = "wasm32")]
@@ -132,7 +148,7 @@ fn init(init_arg: Option<IssuerInit>) {
         let default_config = IssuerConfig::default();
         CONFIG.with(|config_cell| {
             let mut config = config_cell.borrow_mut();
-            *config = ConfigCell::init(config_memory(), default_config)
+            *config = ConfigCell::init(MEMORY_MANAGER.with(|m| m.borrow().get(CONF)), default_config)
                 .expect("Failed to initialize config");
         });
     }
@@ -194,7 +210,20 @@ fn get_admin() -> Principal {
 /// Called when the canister is upgraded.
 #[post_upgrade]
 fn post_upgrade(init_arg: Option<IssuerInit>) {
+    // Initialize the CONFIG
     init(init_arg);
+
+    // Restore the signatures
+    SIGNATURES.with(|sigs| {
+        let mut sigs = sigs.borrow_mut();
+        MSG_HASHES.with(|hashes| {
+            hashes.borrow().iter().for_each(|hash| {
+                sigs.add_signature(&CANISTER_SIG_SEED, hash);
+            })
+        });
+    });
+
+    update_root_hash();
 }
 
 /// Called when the canister is configured.