From 764ccde05e8e866d1ab79620b2aa3f3aedd66dad Mon Sep 17 00:00:00 2001 From: Calascibetta Romain Date: Wed, 24 Apr 2024 18:21:58 +0200 Subject: [PATCH 01/12] Add an implementation of mirage-crypto-rng-miou to initialize the RNG with Miou --- mirage-crypto-rng-miou-unix.opam | 28 ++++++ rng/miou/dune | 5 + rng/miou/mirage_crypto_rng_miou_unix.ml | 97 ++++++++++++++++++ rng/miou/mirage_crypto_rng_miou_unix.mli | 47 +++++++++ rng/miou/pfortuna.ml | 120 +++++++++++++++++++++++ rng/miou/pfortuna.mli | 1 + rng/mirage_crypto_rng.mli | 3 +- tests/dune | 16 +++ tests/test_miou_entropy_collection.ml | 31 ++++++ tests/test_miou_rng.ml | 16 +++ 10 files changed, 363 insertions(+), 1 deletion(-) create mode 100644 mirage-crypto-rng-miou-unix.opam create mode 100644 rng/miou/dune create mode 100644 rng/miou/mirage_crypto_rng_miou_unix.ml create mode 100644 rng/miou/mirage_crypto_rng_miou_unix.mli create mode 100644 rng/miou/pfortuna.ml create mode 100644 rng/miou/pfortuna.mli create mode 100644 tests/test_miou_entropy_collection.ml create mode 100644 tests/test_miou_rng.ml diff --git a/mirage-crypto-rng-miou-unix.opam b/mirage-crypto-rng-miou-unix.opam new file mode 100644 index 00000000..22422b94 --- /dev/null +++ b/mirage-crypto-rng-miou-unix.opam @@ -0,0 +1,28 @@ +opam-version: "2.0" +homepage: "https://github.com/mirage/mirage-crypto" +dev-repo: "git+https://github.com/mirage/mirage-crypto.git" +bug-reports: "https://github.com/mirage/mirage-crypto/issues" +doc: "https://mirage.github.io/mirage-crypto/doc" +authors: ["Romain Calascibetta " ] +maintainer: "Romain Calascibetta " +license: "ISC" +synopsis: "Feed the entropy source in an miou.unix-friendly way" + +build: [ ["dune" "subst"] {dev} + ["dune" "build" "-p" name "-j" jobs ] + ["dune" "runtest" "-p" name "-j" jobs] {with-test} ] + +depends: [ + "ocaml" {>= "5.0.0"} + "dune" {>= "2.7"} + "miou" + "logs" + "mirage-crypto-rng" {=version} + "duration" + "mtime" + "ohex" {with-test & >= "0.2.0"} +] +description: """ +Mirage-crypto-rng-miou-unix feeds the entropy source for Mirage_crypto_rng-based +random number generator implementations, in an miou.unix-friendly way. +""" diff --git a/rng/miou/dune b/rng/miou/dune new file mode 100644 index 00000000..7c97c9e9 --- /dev/null +++ b/rng/miou/dune @@ -0,0 +1,5 @@ +(library + (name mirage_crypto_rng_miou_unix) + (public_name mirage-crypto-rng-miou-unix) + (libraries miou miou.unix mirage-crypto mirage-crypto-rng mirage-crypto-rng.unix digestif duration mtime.clock.os logs) + (modules mirage_crypto_rng_miou_unix pfortuna)) diff --git a/rng/miou/mirage_crypto_rng_miou_unix.ml b/rng/miou/mirage_crypto_rng_miou_unix.ml new file mode 100644 index 00000000..446ea357 --- /dev/null +++ b/rng/miou/mirage_crypto_rng_miou_unix.ml @@ -0,0 +1,97 @@ +open Mirage_crypto_rng + +module Pfortuna = Pfortuna + +type _ Effect.t += Spawn : (unit -> unit) -> unit Effect.t +external reraise : exn -> 'a = "%reraise" + +let periodic fn delta = + let rec one () = + fn (); + Miou_unix.sleep (Duration.to_f delta); + one () in + Effect.perform (Spawn one) + +let getrandom delta source = + let fn () = + let per_pool = 8 in + let size = per_pool * pools None in + let random = Mirage_crypto_rng_unix.getrandom size in + let idx = ref 0 in + let fn () = incr idx; String.sub random (per_pool * (pred !idx)) per_pool in + Entropy.feed_pools None source fn in + periodic fn delta + +let getrandom_init i = + let data = Mirage_crypto_rng_unix.getrandom 128 in + Entropy.header i data + +let rdrand delta = + match Entropy.cpu_rng with + | Error `Not_supported -> () + | Ok cpu_rng -> periodic (cpu_rng None) delta + +let running = Atomic.make false + +let switch fn = + let orphans = Miou.orphans () in + let open Effect.Deep in + let retc = Fun.id in + let exnc = reraise in + let effc : type c. c Effect.t -> ((c, 'r) continuation -> 'r) option + = function + | Spawn fn -> + ignore (Miou.call_cc ~orphans fn); + Some (fun k -> continue k ()) + | _ -> None in + match_with fn orphans { retc; exnc; effc } + +let default_generator_already_set = + "Mirage_crypto_rng.default_generator has already \ + been set (but not via Mirage_crypto_rng_miou). Please check \ + that this is intentional" + +let miou_generator_already_launched = + "Mirage_crypto_rng_miou.initialize has already been launched \ + and a task is already seeding the RNG." + +type rng = unit Miou.t + +let rec compare_and_set ?(backoff= Backoff.default) t a b = + if Atomic.compare_and_set t a b = false + then compare_and_set ~backoff:(Backoff.once backoff) t a b + +let rec clean_up sleep orphans = match Miou.care orphans with + | Some None | None -> Miou_unix.sleep (Duration.to_f sleep); clean_up sleep orphans + | Some (Some prm) -> Miou.await_exn prm; clean_up sleep orphans + +let call_if_domain_available fn = + let available = Miou.Domain.available () in + let current = (Stdlib.Domain.self () :> int) in + if current = 0 && available > 0 + || current <> 0 && available > 1 + then Miou.call fn + else Miou.call_cc fn + +let initialize (type a) ?g ?(sleep= Duration.of_sec 1) (rng : a generator) = + if Atomic.compare_and_set running false true + then begin + let seed = + let init = Entropy.[ bootstrap; whirlwind_bootstrap; bootstrap; getrandom_init ] in + List.mapi (fun i fn -> fn i) init |> String.concat "" in + let rng = create ?g ~seed ~time:Mtime_clock.elapsed_ns rng in + set_default_generator rng; + call_if_domain_available @@ fun () -> + let finally () = compare_and_set running true false in + Fun.protect ~finally @@ fun () -> switch @@ fun orphans -> + let () = + try let _ = default_generator () in + Logs.warn (fun m -> m "%s" default_generator_already_set) + with No_default_generator -> () in + rdrand sleep; + let source = Entropy.register_source "getrandom" in + getrandom (Int64.mul sleep 10L) source; + clean_up sleep orphans + end else invalid_arg miou_generator_already_launched + +let kill prm = Miou.cancel prm diff --git a/rng/miou/mirage_crypto_rng_miou_unix.mli b/rng/miou/mirage_crypto_rng_miou_unix.mli new file mode 100644 index 00000000..057afbc8 --- /dev/null +++ b/rng/miou/mirage_crypto_rng_miou_unix.mli @@ -0,0 +1,47 @@ +(** {b RNG} seeding on {b Miou_unix}. + + This module initializes a RNG with [getrandom()], and CPU RNG. On BSD system + (FreeBSD, OpenBSD, MacOS) [getentropy()] is used instead of [getrandom()]. + On Windows 10 or higher, [BCryptGenRandom()] is used with the default RNG. + Windows 8 or lower are not supported by this library. +*) + +module Pfortuna : Mirage_crypto_rng.Generator +(** {b Pfortuna}, a {b domain-safe} CSPRNG + {{: https://www.schneier.com/fortuna.html} proposed} by Schneier. *) + +type rng +(** Type of tasks seeding the RNG. *) + +val initialize : ?g:'a -> ?sleep:int64 -> 'a Mirage_crypto_rng.generator -> rng +(** [initialize ?g ?sleep (module Generator)] will allow the RNG to operate in a + returned task. This task periodically launches sub-tasks that seed the + engine (using [getrandom()], [getentropy()] or [BCryptGenRandom()] depending + on the system). These sub-tasks must be cleaned periodically (in seconds) + according to the [sleep] parameter given (defaults to 1 second). + + The user must then {!val:kill} the returned task at the end of the program + to be sure to clean everything. Otherwise, Miou will complain with the + exception [Still_has_children]. + + We strongly recommend using {!module:Pfortuna} as an RNG engine rather than + {!module:Mirage_crypto_rng.Fortuna}. The engine is launched in parallel with + the other tasks if at least one domain is available. To ensure that there is + no compromise in the values generated by a {i data-race}, [Pfortuna] is an + {b domain-safe} implementation of Fortuna. + + The user cannot make any subsequent calls to [initialize]. In other words, + you can only initialise a single {!type:rng} task. You must {!val:kill} the + returned {!type:rng} if you want to re-initialise the RNG. + + A basic usage of [mirage-crypto-rng-miou-unix] is: + {[ + let () = Miou_unix.run @@ fun () -> + let rng = Mirage_crypto_rng_miou_unix.(initialize (module Pfortuna)) in + let str = Mirage_crypto_rng.generate 16 in + Format.printf "random: %S\n%!" str; + Mirage_crypto_rng_miou_unix.kill rng + ]} *) + +val kill : rng -> unit +(** [kill rng] terminates the {i background} task which seeds the RNG. *) diff --git a/rng/miou/pfortuna.ml b/rng/miou/pfortuna.ml new file mode 100644 index 00000000..44a3182b --- /dev/null +++ b/rng/miou/pfortuna.ml @@ -0,0 +1,120 @@ +open Mirage_crypto +open Mirage_crypto.Uncommon + +module SHAd256 = struct + open Digestif + type ctx = SHA256.ctx + let empty = SHA256.empty + let get t = SHA256.(get t |> to_raw_string |> digest_string |> to_raw_string) + let digesti i = SHA256.(digesti_string i |> to_raw_string |> digest_string |> to_raw_string) + let feedi = SHA256.feedi_string +end + +let block = 16 + +(* the minimal amount of bytes in a pool to trigger a reseed *) +let min_pool_size = 64 +(* the minimal duration between two reseeds *) +let min_time_duration = 1_000_000_000L +(* number of pools *) +let pools = 32 + +type t = + { ctr : AES.CTR.ctr + ; secret : string + ; key : AES.CTR.key + ; pools : SHAd256.ctx array + ; pool0_size : int + ; reseed_count : int + ; last_reseed : int64 + ; time : (unit -> int64) option + } + +type g = Miou.Mutex.t * t ref + +let update (m, g) fn = Miou.Mutex.protect m @@ fun () -> g := fn !g +let get (m, g) fn = Miou.Mutex.protect m @@ fun () -> fn !g + +let create ?time () = + let secret = String.make 32 '\000' in + let m = Miou.Mutex.create () in + let t = + { ctr= (0L, 0L); secret; key= AES.CTR.of_secret secret + ; pools= Array.make pools SHAd256.empty + ; pool0_size= 0 + ; reseed_count= 0 + ; last_reseed= 0L + ; time } in + (m, { contents= t }) + +let seeded ~t = + let lo, hi = t.ctr in + not (Int64.equal lo 0L && Int64.equal hi 0L) + +let set_key ~t secret = + { t with secret; key= AES.CTR.of_secret secret } + +let reseedi ~t iter = + let t = set_key ~t (SHAd256.digesti (fun fn -> fn t.secret; iter fn)) in + { t with ctr= AES.CTR.add_ctr t.ctr 1L } + +let iter1 a f = f a +let reseed ~t cs = reseedi ~t (iter1 cs) + +let generate_rekey ~t buf ~off len = + let b = len // block* 2 in + let n = b * block in + let r = AES.CTR.stream ~key:t.key ~ctr:t.ctr n in + Bytes.unsafe_blit_string r 0 buf off len; + let r2 = String.sub r (n - 32) 32 in + let t = set_key ~t r2 in + { t with ctr= AES.CTR.add_ctr t.ctr (Int64.of_int b) } + +let add_pool_entropy t = + if t.pool0_size > min_pool_size then + let should_reseed, now = match t.time with + | None -> true, 0L + | Some fn -> + let now = fn () in + Int64.(sub now t.last_reseed > min_time_duration), now in + if should_reseed then begin + let t = { t with reseed_count= t.reseed_count + 1 + ; last_reseed= now + ; pool0_size= 0 } in + reseedi ~t @@ fun add -> + for i = 0 to pools - 1 do + if t.reseed_count land ((1 lsl i) - 1) = 0 + then (SHAd256.get t.pools.(i) |> add; t.pools.(i) <- SHAd256.empty) + done + end else t else t + +let generate_into ~t buf ~off len = + let t = add_pool_entropy t in + if not (seeded ~t) then raise Mirage_crypto_rng.Unseeded_generator; + let rec chunk t off = function + | i when i <= 0 -> t + | n -> + let n' = imin n 0x10000 in + let t = generate_rekey ~t buf ~off n' in + chunk t (off + n') (n - n') in + chunk t off len + +let add ~t source ~pool data = + let buf = Bytes.create 2 + and pool = pool land (pools - 1) + and source = Mirage_crypto_rng.Entropy.id source land 0xff in + Bytes.set_uint8 buf 0 source; + Bytes.set_uint8 buf 1 (String.length data); + t.pools.(pool) <- SHAd256.feedi t.pools.(pool) (iter2 (Bytes.unsafe_to_string buf) data); + if pool = 0 then { t with pool0_size= t.pool0_size + String.length data } else t + +let accumulate ~g source = + let pool = ref 0 in + `Acc (fun buf -> + update g @@ fun t -> + let t = add ~t source ~pool:!pool buf in + incr pool; t) + +let reseed ~g cs = update g @@ fun t -> reseed ~t cs +let generate_into ~g buf ~off len = update g @@ fun t -> generate_into ~t buf ~off len +let seeded ~g = get g @@ fun t -> seeded ~t diff --git a/rng/miou/pfortuna.mli b/rng/miou/pfortuna.mli new file mode 100644 index 00000000..398c3c05 --- /dev/null +++ b/rng/miou/pfortuna.mli @@ -0,0 +1 @@ +include Mirage_crypto_rng.Generator diff --git a/rng/mirage_crypto_rng.mli b/rng/mirage_crypto_rng.mli index fd171f52..3c7855a6 100644 --- a/rng/mirage_crypto_rng.mli +++ b/rng/mirage_crypto_rng.mli @@ -25,7 +25,8 @@ {{!Mirage_crypto_rng_async}mirage-crypto-rng-async} (for Async), {{!Mirage_crypto_rng_mirage}mirage-crypto-rng-mirage} (for MirageOS), {{!Mirage_crypto_rng_unix}mirage-crypto-rng.unix}, - and {{!Mirage_crypto_rng_eio}mirage-crypto-rng-eio} (for Eio). + {{!Mirage_crypto_rng_eio}mirage-crypto-rng-eio} (for Eio), + and {{!Mirage_crypto_rng_miou_unix}mirage-crypto-miou-unix} (for Miou_unix). The intention is that "initialize" in the respective sub-library is called once, which sets the default generator and registers entropy diff --git a/tests/dune b/tests/dune index 05ae7adb..0d28646c 100644 --- a/tests/dune +++ b/tests/dune @@ -6,12 +6,14 @@ (test (name test_symmetric_runner) + (modes native) (libraries test_common mirage-crypto ounit2) (package mirage-crypto) (modules test_base test_cipher test_symmetric_runner)) (test (name test_random_runner) + (modes native) (libraries test_common mirage-crypto mirage-crypto-rng mirage-crypto-rng.unix randomconv ounit2) (package mirage-crypto-rng) @@ -19,6 +21,7 @@ (test (name test_pk_runner) + (modes native) (libraries test_common mirage-crypto-pk mirage-crypto-rng.unix randomconv ounit2) (package mirage-crypto-pk) @@ -26,6 +29,7 @@ (test (name test_entropy_collection) + (modes native) (modules test_entropy_collection) (package mirage-crypto-rng-mirage) (libraries mirage-crypto-rng-mirage mirage-unix mirage-time-unix @@ -33,24 +37,28 @@ (test (name test_entropy_collection_async) + (modes native) (modules test_entropy_collection_async) (package mirage-crypto-rng-async) (libraries mirage-crypto-rng-async ohex)) (test (name test_entropy) + (modes native) (modules test_entropy) (package mirage-crypto-rng) (libraries mirage-crypto-rng ohex)) (test (name test_ec) + (modes native) (modules test_ec) (libraries test_common alcotest mirage-crypto-ec mirage-crypto-rng.unix) (package mirage-crypto-ec)) (test (name test_ec_wycheproof) + (modes native) (modules test_ec_wycheproof) (deps ecdh_secp256r1_test.json ecdsa_secp256r1_sha256_test.json ecdsa_secp256r1_sha512_test.json ecdh_secp384r1_test.json @@ -62,6 +70,14 @@ (tests (names test_eio_rng test_eio_entropy_collection) + (modes native) (modules test_eio_rng test_eio_entropy_collection) (libraries mirage-crypto-rng-eio duration eio_main ohex) (package mirage-crypto-rng-eio)) + +(tests + (names test_miou_rng test_miou_entropy_collection) + (modes native) + (modules test_miou_rng test_miou_entropy_collection) + (libraries mirage-crypto-rng-miou-unix duration ohex) + (package mirage-crypto-rng-miou-unix)) diff --git a/tests/test_miou_entropy_collection.ml b/tests/test_miou_entropy_collection.ml new file mode 100644 index 00000000..57ad3eab --- /dev/null +++ b/tests/test_miou_entropy_collection.ml @@ -0,0 +1,31 @@ +module Printing_rng = struct + type g = unit + + let block = 16 + let create ?time:_ () = () + let generate_into ~g:_ _buf ~off:_ _len = assert false + let seeded ~g:_ = true + let pools = 1 + + let reseed ~g:_ data = + Format.printf "reseeding:@.%a@.%!" (Ohex.pp_hexdump ()) data + + let accumulate ~g:_ source = + let print data = + Format.printf "accumulate: (src: %a) %a@.%!" + Mirage_crypto_rng.Entropy.pp_source source Ohex.pp data + in + `Acc print +end + +let () = + Miou_unix.run @@ fun () -> + let rng = Mirage_crypto_rng_miou_unix.initialize (module Printing_rng) in + Format.printf "entropy sources: %a@,%!" + (fun ppf -> List.iter (fun x -> + Mirage_crypto_rng.Entropy.pp_source ppf x; + Format.pp_print_space ppf ())) + (Mirage_crypto_rng.Entropy.sources ()); + let sleep = Duration.(of_sec 2 |> to_f) in + Miou_unix.sleep sleep; + Mirage_crypto_rng_miou_unix.kill rng diff --git a/tests/test_miou_rng.ml b/tests/test_miou_rng.ml new file mode 100644 index 00000000..1cd5dcb8 --- /dev/null +++ b/tests/test_miou_rng.ml @@ -0,0 +1,16 @@ +let () = Miou_unix.run @@ fun () -> + let rng = Mirage_crypto_rng_miou_unix.(initialize (module Pfortuna)) in + let random_num = Mirage_crypto_rng.generate 32 in + assert (String.length random_num = 32); + Printf.printf "32 bit random number: %s\n%!" (Ohex.encode random_num); + let random_num = Mirage_crypto_rng.generate 16 in + assert (String.length random_num = 16); + Printf.printf "16 bit random number: %s\n%!" (Ohex.encode random_num); + (* NOTE(dinosaure): the test below shows that [Pfortuna] is domain-safe when + run with TSan. If we use the Fortuna engine, TSan will report invalid + accesses between the domain that seeds the RNG and [dom0]. *) + for _ = 0 to 4 do + let _ = Mirage_crypto_rng.generate 16 in + Miou_unix.sleep 0.5; + done; + Mirage_crypto_rng_miou_unix.kill rng From 4169f33cb2c217a566069199b574bd66ceb0f1fc Mon Sep 17 00:00:00 2001 From: Calascibetta Romain Date: Thu, 25 Apr 2024 12:11:10 +0200 Subject: [PATCH 02/12] Update GitHub actions and CirrusCI --- .github/workflows/test.yml | 12 +++++++----- .github/workflows/windows.yml | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c1a8bb0e..ca16e909 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,7 @@ jobs: opam-local-packages: | *.opam !mirage-crypto-rng-eio.opam + !mirage-crypto-rng-miou-unix.opam ocaml-compiler: ${{ matrix.ocaml-version }} - name: Install dependencies @@ -35,8 +36,8 @@ jobs: - name: Test run: opam exec -- dune runtest -p mirage-crypto,mirage-crypto-rng,mirage-crypto-rng-lwt,mirage-crypto-rng-mirage,mirage-crypto-pk,mirage-crypto-ec,mirage-crypto-rng-async - build-test-unix-eio: - name : Unix (eio) + build-test-ocaml-5: + name : Tests with OCaml 5 strategy: fail-fast: false @@ -57,13 +58,14 @@ jobs: mirage-crypto.opam mirage-crypto-rng.opam mirage-crypto-rng-eio.opam + mirage-crypto-rng-miou-unix.opam ocaml-compiler: ${{ matrix.ocaml-version }} - name: Install dependencies - run: opam install --deps-only -t mirage-crypto mirage-crypto-rng mirage-crypto-rng-eio + run: opam install --deps-only -t mirage-crypto mirage-crypto-rng mirage-crypto-rng-eio mirage-crypto-rng-miou-unix - name: Build - run: opam exec -- dune build -p mirage-crypto,mirage-crypto-rng,mirage-crypto-rng-eio + run: opam exec -- dune build -p mirage-crypto,mirage-crypto-rng,mirage-crypto-rng-eio,mirage-crypto-rng-miou-unix - name: Test - run: opam exec -- dune runtest -p mirage-crypto,mirage-crypto-rng,mirage-crypto-rng-eio + run: opam exec -- dune runtest -p mirage-crypto,mirage-crypto-rng,mirage-crypto-rng-eio,mirage-crypto-rng-miou-unix diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 4f97944f..43b28cf3 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -28,6 +28,7 @@ jobs: *.opam !mirage-crypto-rng-async.opam !mirage-crypto-rng-eio.opam + !mirage-crypto-rng-miou.opam ocaml-compiler: ${{ matrix.ocaml-version }} - name: Install dependencies From 7970f35b15392d927ffd8596150cc6abeb669988 Mon Sep 17 00:00:00 2001 From: Calascibetta Romain Date: Fri, 3 May 2024 15:00:39 +0200 Subject: [PATCH 03/12] Use the last version of Miou --- .github/workflows/windows.yml | 2 +- mirage-crypto-rng-miou-unix.opam | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 43b28cf3..3bf05ee7 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -28,7 +28,7 @@ jobs: *.opam !mirage-crypto-rng-async.opam !mirage-crypto-rng-eio.opam - !mirage-crypto-rng-miou.opam + !mirage-crypto-rng-miou-unix.opam ocaml-compiler: ${{ matrix.ocaml-version }} - name: Install dependencies diff --git a/mirage-crypto-rng-miou-unix.opam b/mirage-crypto-rng-miou-unix.opam index 22422b94..682a8a2c 100644 --- a/mirage-crypto-rng-miou-unix.opam +++ b/mirage-crypto-rng-miou-unix.opam @@ -26,3 +26,7 @@ description: """ Mirage-crypto-rng-miou-unix feeds the entropy source for Mirage_crypto_rng-based random number generator implementations, in an miou.unix-friendly way. """ + +pin-depends: [ + [ "miou.dev" "git+https://github.com/robur-coop/miou.git#ed5087b832797616df073bd8ec9baed2ec4e474c" ] +] From 154a3ecfddd63ec9fa18c4a6d7c2c1a1554ce2e3 Mon Sep 17 00:00:00 2001 From: Calascibetta Romain Date: Fri, 3 May 2024 18:31:58 +0200 Subject: [PATCH 04/12] Prefer to follow the user's control flow than the cancellation control flow to set correctly global variables --- rng/miou/mirage_crypto_rng_miou_unix.ml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/rng/miou/mirage_crypto_rng_miou_unix.ml b/rng/miou/mirage_crypto_rng_miou_unix.ml index 446ea357..fcc2eff7 100644 --- a/rng/miou/mirage_crypto_rng_miou_unix.ml +++ b/rng/miou/mirage_crypto_rng_miou_unix.ml @@ -79,19 +79,20 @@ let initialize (type a) ?g ?(sleep= Duration.of_sec 1) (rng : a generator) = let seed = let init = Entropy.[ bootstrap; whirlwind_bootstrap; bootstrap; getrandom_init ] in List.mapi (fun i fn -> fn i) init |> String.concat "" in - let rng = create ?g ~seed ~time:Mtime_clock.elapsed_ns rng in - set_default_generator rng; - call_if_domain_available @@ fun () -> - let finally () = compare_and_set running true false in - Fun.protect ~finally @@ fun () -> switch @@ fun orphans -> let () = try let _ = default_generator () in Logs.warn (fun m -> m "%s" default_generator_already_set) with No_default_generator -> () in + let rng = create ?g ~seed ~time:Mtime_clock.elapsed_ns rng in + set_default_generator rng; + call_if_domain_available @@ fun () -> switch @@ fun orphans -> rdrand sleep; let source = Entropy.register_source "getrandom" in getrandom (Int64.mul sleep 10L) source; clean_up sleep orphans end else invalid_arg miou_generator_already_launched -let kill prm = Miou.cancel prm +let kill prm = + Miou.cancel prm; + compare_and_set running true false; + unset_default_generator () From ff544f044fb69ea9cc24e51714a8f25f491ef326 Mon Sep 17 00:00:00 2001 From: Calascibetta Romain Date: Fri, 17 May 2024 10:41:46 +0200 Subject: [PATCH 05/12] Use Miou_backoff instead of Backoff and avoid a conflict with the backoff package --- mirage-crypto-rng-miou-unix.opam | 2 +- rng/miou/mirage_crypto_rng_miou_unix.ml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mirage-crypto-rng-miou-unix.opam b/mirage-crypto-rng-miou-unix.opam index 682a8a2c..74ce24a4 100644 --- a/mirage-crypto-rng-miou-unix.opam +++ b/mirage-crypto-rng-miou-unix.opam @@ -28,5 +28,5 @@ random number generator implementations, in an miou.unix-friendly way. """ pin-depends: [ - [ "miou.dev" "git+https://github.com/robur-coop/miou.git#ed5087b832797616df073bd8ec9baed2ec4e474c" ] + [ "miou.dev" "git+https://github.com/robur-coop/miou.git#b43d607e45fe35c2ab5e9ad1c50f22ada6d11c2a" ] ] diff --git a/rng/miou/mirage_crypto_rng_miou_unix.ml b/rng/miou/mirage_crypto_rng_miou_unix.ml index fcc2eff7..a95fa161 100644 --- a/rng/miou/mirage_crypto_rng_miou_unix.ml +++ b/rng/miou/mirage_crypto_rng_miou_unix.ml @@ -57,9 +57,9 @@ let miou_generator_already_launched = type rng = unit Miou.t -let rec compare_and_set ?(backoff= Backoff.default) t a b = +let rec compare_and_set ?(backoff= Miou_backoff.default) t a b = if Atomic.compare_and_set t a b = false - then compare_and_set ~backoff:(Backoff.once backoff) t a b + then compare_and_set ~backoff:(Miou_backoff.once backoff) t a b let rec clean_up sleep orphans = match Miou.care orphans with | Some None | None -> Miou_unix.sleep (Duration.to_f sleep); clean_up sleep orphans From 7b211cdc110af092fb229855ed7d9319b6a83a2d Mon Sep 17 00:00:00 2001 From: Calascibetta Romain Date: Thu, 6 Jun 2024 11:17:12 +0200 Subject: [PATCH 06/12] Upgrade the PR with miou.0.2.0 and delete the pin-depends --- bench/dune | 2 +- bench/fortuna.ml | 0 bench/speed.ml | 12 ++++++++++++ mirage-crypto-rng-miou-unix.opam | 6 +----- rng/miou/mirage_crypto_rng_miou_unix.ml | 4 ++-- 5 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 bench/fortuna.ml diff --git a/bench/dune b/bench/dune index 63558d89..71ad5387 100644 --- a/bench/dune +++ b/bench/dune @@ -2,4 +2,4 @@ (names speed) (modules speed) (libraries mirage-crypto mirage-crypto-rng mirage-crypto-rng.unix - mirage-crypto-pk mirage-crypto-ec)) + mirage-crypto-pk mirage-crypto-ec mirage-crypto-rng-miou-unix)) diff --git a/bench/fortuna.ml b/bench/fortuna.ml new file mode 100644 index 00000000..e69de29b diff --git a/bench/speed.ml b/bench/speed.ml index 90d44425..74f20bf1 100644 --- a/bench/speed.ml +++ b/bench/speed.ml @@ -404,6 +404,18 @@ let benchmarks = [ throughput name (fun buf -> let buf = Bytes.unsafe_of_string buf in generate_into ~g buf ~off:0 (Bytes.length buf))) ; + + bm "pfortuna" (fun name -> + let open Mirage_crypto_rng_miou_unix.Pfortuna in + Miou_unix.run ~domains:2 @@ fun () -> + let rng = Mirage_crypto_rng_miou_unix.(initialize (module Pfortuna)) in + let g = create () in + reseed ~g "abcd" ; + throughput name (fun buf -> + let buf = Bytes.unsafe_of_string buf in + generate_into ~g buf ~off:0 (Bytes.length buf)); + Mirage_crypto_rng_miou_unix.kill rng) ; + ] let help () = diff --git a/mirage-crypto-rng-miou-unix.opam b/mirage-crypto-rng-miou-unix.opam index 74ce24a4..1db4e7b8 100644 --- a/mirage-crypto-rng-miou-unix.opam +++ b/mirage-crypto-rng-miou-unix.opam @@ -15,7 +15,7 @@ build: [ ["dune" "subst"] {dev} depends: [ "ocaml" {>= "5.0.0"} "dune" {>= "2.7"} - "miou" + "miou" {>= "0.2.0"} "logs" "mirage-crypto-rng" {=version} "duration" @@ -26,7 +26,3 @@ description: """ Mirage-crypto-rng-miou-unix feeds the entropy source for Mirage_crypto_rng-based random number generator implementations, in an miou.unix-friendly way. """ - -pin-depends: [ - [ "miou.dev" "git+https://github.com/robur-coop/miou.git#b43d607e45fe35c2ab5e9ad1c50f22ada6d11c2a" ] -] diff --git a/rng/miou/mirage_crypto_rng_miou_unix.ml b/rng/miou/mirage_crypto_rng_miou_unix.ml index a95fa161..82ecdbf7 100644 --- a/rng/miou/mirage_crypto_rng_miou_unix.ml +++ b/rng/miou/mirage_crypto_rng_miou_unix.ml @@ -41,7 +41,7 @@ let switch fn = let effc : type c. c Effect.t -> ((c, 'r) continuation -> 'r) option = function | Spawn fn -> - ignore (Miou.call_cc ~orphans fn); + ignore (Miou.async ~orphans fn); Some (fun k -> continue k ()) | _ -> None in match_with fn orphans { retc; exnc; effc } @@ -71,7 +71,7 @@ let call_if_domain_available fn = if current = 0 && available > 0 || current <> 0 && available > 1 then Miou.call fn - else Miou.call_cc fn + else Miou.async fn let initialize (type a) ?g ?(sleep= Duration.of_sec 1) (rng : a generator) = if Atomic.compare_and_set running false true From 3723068d69d17594d66dc6d37d61873eb90ff467 Mon Sep 17 00:00:00 2001 From: Calascibetta Romain Date: Sun, 9 Jun 2024 04:19:07 +0200 Subject: [PATCH 07/12] Provide a separate executable to bench pfortuna with miou --- bench/dune | 7 +++- bench/fortuna.ml | 0 bench/miou.ml | 91 ++++++++++++++++++++++++++++++++++++++++++++++++ bench/speed.ml | 12 ------- 4 files changed, 97 insertions(+), 13 deletions(-) delete mode 100644 bench/fortuna.ml create mode 100644 bench/miou.ml diff --git a/bench/dune b/bench/dune index 71ad5387..f0abf3e3 100644 --- a/bench/dune +++ b/bench/dune @@ -2,4 +2,9 @@ (names speed) (modules speed) (libraries mirage-crypto mirage-crypto-rng mirage-crypto-rng.unix - mirage-crypto-pk mirage-crypto-ec mirage-crypto-rng-miou-unix)) + mirage-crypto-pk mirage-crypto-ec)) + +(executables + (names miou) + (modules miou) + (libraries mirage-crypto-rng-miou-unix)) diff --git a/bench/fortuna.ml b/bench/fortuna.ml deleted file mode 100644 index e69de29b..00000000 diff --git a/bench/miou.ml b/bench/miou.ml new file mode 100644 index 00000000..27120c32 --- /dev/null +++ b/bench/miou.ml @@ -0,0 +1,91 @@ +open Mirage_crypto + +module Time = struct + + let time ~n f a = + let t1 = Sys.time () in + for _ = 1 to n do ignore (f a) done ; + let t2 = Sys.time () in + (t2 -. t1) + + let warmup () = + let x = ref 0 in + let rec go start = + if Sys.time () -. start < 1. then begin + for i = 0 to 10000 do x := !x + i done ; + go start + end in + go (Sys.time ()) + +end + +let burn_period = 2.0 + +let sizes = [16; 64; 256; 1024; 8192] +(* let sizes = [16] *) + +let burn f n = + let buf = Mirage_crypto_rng.generate n in + let (t1, i1) = + let rec loop it = + let t = Time.time ~n:it f buf in + if t > 0.2 then (t, it) else loop (it * 10) in + loop 10 in + let iters = int_of_float (float i1 *. burn_period /. t1) in + let time = Time.time ~n:iters f buf in + (iters, time, float (n * iters) /. time) + +let mb = 1024. *. 1024. + +let throughput title f = + Printf.printf "\n* [%s]\n%!" title ; + sizes |> List.iter @@ fun size -> + Gc.full_major () ; + let (iters, time, bw) = burn f size in + Printf.printf " % 5d: %04f MB/s (%d iters in %.03f s)\n%!" + size (bw /. mb) iters time + +let bm name f = (name, fun () -> f name) + +let benchmarks = [ + bm "pfortuna" (fun name -> + let open Mirage_crypto_rng_miou_unix.Pfortuna in + Miou_unix.run ~domains:2 @@ fun () -> + let rng = Mirage_crypto_rng_miou_unix.(initialize (module Pfortuna)) in + let g = create () in + reseed ~g "abcd" ; + throughput name (fun buf -> + let buf = Bytes.unsafe_of_string buf in + generate_into ~g buf ~off:0 (Bytes.length buf)); + Mirage_crypto_rng_miou_unix.kill rng) ; +] + +let help () = + Printf.printf "available benchmarks:\n "; + List.iter (fun (n, _) -> Printf.printf "%s " n) benchmarks ; + Printf.printf "\n%!" + +let runv fs = + Format.printf "accel: %a\n%!" + (fun ppf -> List.iter @@ fun x -> + Format.fprintf ppf "%s " @@ + match x with `XOR -> "XOR" | `AES -> "AES" | `GHASH -> "GHASH") + accelerated; + Time.warmup () ; + List.iter (fun f -> f ()) fs + + +let () = + let seed = "abcd" in + let g = Mirage_crypto_rng.(create ~seed (module Fortuna)) in + Mirage_crypto_rng.set_default_generator g; + match Array.to_list Sys.argv with + | _::(_::_ as args) -> begin + try + let fs = + args |> List.map @@ fun n -> + snd (benchmarks |> List.find @@ fun (n1, _) -> n = n1) in + runv fs + with Not_found -> help () + end + | _ -> help () diff --git a/bench/speed.ml b/bench/speed.ml index 74f20bf1..90d44425 100644 --- a/bench/speed.ml +++ b/bench/speed.ml @@ -404,18 +404,6 @@ let benchmarks = [ throughput name (fun buf -> let buf = Bytes.unsafe_of_string buf in generate_into ~g buf ~off:0 (Bytes.length buf))) ; - - bm "pfortuna" (fun name -> - let open Mirage_crypto_rng_miou_unix.Pfortuna in - Miou_unix.run ~domains:2 @@ fun () -> - let rng = Mirage_crypto_rng_miou_unix.(initialize (module Pfortuna)) in - let g = create () in - reseed ~g "abcd" ; - throughput name (fun buf -> - let buf = Bytes.unsafe_of_string buf in - generate_into ~g buf ~off:0 (Bytes.length buf)); - Mirage_crypto_rng_miou_unix.kill rng) ; - ] let help () = From 14029b6a8b0b205171e7ee357610da90ab45cde3 Mon Sep 17 00:00:00 2001 From: Calascibetta Romain Date: Sun, 9 Jun 2024 04:29:06 +0200 Subject: [PATCH 08/12] Add a comment about the goal of Pfortuna --- rng/miou/pfortuna.ml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/rng/miou/pfortuna.ml b/rng/miou/pfortuna.ml index 44a3182b..6d840308 100644 --- a/rng/miou/pfortuna.ml +++ b/rng/miou/pfortuna.ml @@ -1,3 +1,17 @@ +(* Pfortuna is a re-implementation of Fortuna with a mutex. The goal of this + module is to provide a global and domain-safe RNG. The implementation use + [Miou.Mutex] instead of [Mutex] - [Pfortuna] is only available as part of + the [mirage-crypto-rng-miou-unix] package. Thus, in the context of Miou, + [Pfortuna] can be used and recommended in place of [Fortuna], so that the + user can generate random numbers in parallel in several domains. + + {[ + let () = Miou_unix.run @@ fun () -> + let rng = Mirage_crypto_rng_miou_unix.(initialize (module Pfortuna)) in + ... + Mirage_crypto_rng_miou_unix.kill rng + ]} *) + open Mirage_crypto open Mirage_crypto.Uncommon From 8c5d585d0b8aa4c2895ac10341ab81486c0f1204 Mon Sep 17 00:00:00 2001 From: Hannes Mehnert Date: Sun, 9 Jun 2024 14:28:47 +0200 Subject: [PATCH 09/12] note to sync fortuna and pfortuna --- rng/fortuna.ml | 3 +++ rng/miou/pfortuna.ml | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/rng/fortuna.ml b/rng/fortuna.ml index db494da2..e83feab5 100644 --- a/rng/fortuna.ml +++ b/rng/fortuna.ml @@ -1,3 +1,6 @@ +(* NOTE: when modifying this file, please also check whether + rng/miou/pfortuna.ml needs to be updated. *) + open Mirage_crypto open Mirage_crypto.Uncommon diff --git a/rng/miou/pfortuna.ml b/rng/miou/pfortuna.ml index 6d840308..fa1ec409 100644 --- a/rng/miou/pfortuna.ml +++ b/rng/miou/pfortuna.ml @@ -10,7 +10,10 @@ let rng = Mirage_crypto_rng_miou_unix.(initialize (module Pfortuna)) in ... Mirage_crypto_rng_miou_unix.kill rng - ]} *) + ]} + + NOTE: when modifying this file, please also check whether rng/fortuna.ml + needs to be updated. *) open Mirage_crypto open Mirage_crypto.Uncommon From 857e7770f97f2c082f13cab0f6e9e10aacd92849 Mon Sep 17 00:00:00 2001 From: Hannes Mehnert Date: Sun, 9 Jun 2024 15:20:04 +0200 Subject: [PATCH 10/12] fix opam-lint check: add digestif to dependencies of mirage-crypto-rng-miou-unix --- mirage-crypto-rng-miou-unix.opam | 1 + 1 file changed, 1 insertion(+) diff --git a/mirage-crypto-rng-miou-unix.opam b/mirage-crypto-rng-miou-unix.opam index 1db4e7b8..4a04a259 100644 --- a/mirage-crypto-rng-miou-unix.opam +++ b/mirage-crypto-rng-miou-unix.opam @@ -20,6 +20,7 @@ depends: [ "mirage-crypto-rng" {=version} "duration" "mtime" + "digestif" {>= "1.2.0"} "ohex" {with-test & >= "0.2.0"} ] description: """ From f750706e008c0da9deed69e9ba9e55afc56134ad Mon Sep 17 00:00:00 2001 From: Hannes Mehnert Date: Sun, 9 Jun 2024 14:43:01 +0200 Subject: [PATCH 11/12] comment out bench/miou in dune to avoid CI build failures since (package mirage-crypto-rng-miou-unix) is not supported without (public_names ..) in dune, there's no easy alternative. Marking it (optional) still results in failures with OCaml-CI --- bench/dune | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bench/dune b/bench/dune index f0abf3e3..dec1e4f9 100644 --- a/bench/dune +++ b/bench/dune @@ -4,7 +4,9 @@ (libraries mirage-crypto mirage-crypto-rng mirage-crypto-rng.unix mirage-crypto-pk mirage-crypto-ec)) -(executables - (names miou) - (modules miou) - (libraries mirage-crypto-rng-miou-unix)) +; marking as "(optional)" leads to OCaml-CI failures +; marking with "(package mirage-crypto-rng-miou-unix)" only has an effect with a "public_name" +;(executables +; (names miou) +; (modules miou) +; (libraries mirage-crypto-rng-miou-unix)) From 2ee08f31e39a249d12ae8efa8980d5cb9691fc57 Mon Sep 17 00:00:00 2001 From: Hannes Mehnert Date: Sun, 9 Jun 2024 19:53:51 +0200 Subject: [PATCH 12/12] no need for (modes native), this is known as https://github.com/ocaml/dune/issues/9979 --- tests/dune | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/dune b/tests/dune index 0d28646c..b06e9201 100644 --- a/tests/dune +++ b/tests/dune @@ -6,14 +6,12 @@ (test (name test_symmetric_runner) - (modes native) (libraries test_common mirage-crypto ounit2) (package mirage-crypto) (modules test_base test_cipher test_symmetric_runner)) (test (name test_random_runner) - (modes native) (libraries test_common mirage-crypto mirage-crypto-rng mirage-crypto-rng.unix randomconv ounit2) (package mirage-crypto-rng) @@ -21,7 +19,6 @@ (test (name test_pk_runner) - (modes native) (libraries test_common mirage-crypto-pk mirage-crypto-rng.unix randomconv ounit2) (package mirage-crypto-pk) @@ -29,7 +26,6 @@ (test (name test_entropy_collection) - (modes native) (modules test_entropy_collection) (package mirage-crypto-rng-mirage) (libraries mirage-crypto-rng-mirage mirage-unix mirage-time-unix @@ -37,28 +33,24 @@ (test (name test_entropy_collection_async) - (modes native) (modules test_entropy_collection_async) (package mirage-crypto-rng-async) (libraries mirage-crypto-rng-async ohex)) (test (name test_entropy) - (modes native) (modules test_entropy) (package mirage-crypto-rng) (libraries mirage-crypto-rng ohex)) (test (name test_ec) - (modes native) (modules test_ec) (libraries test_common alcotest mirage-crypto-ec mirage-crypto-rng.unix) (package mirage-crypto-ec)) (test (name test_ec_wycheproof) - (modes native) (modules test_ec_wycheproof) (deps ecdh_secp256r1_test.json ecdsa_secp256r1_sha256_test.json ecdsa_secp256r1_sha512_test.json ecdh_secp384r1_test.json @@ -70,14 +62,12 @@ (tests (names test_eio_rng test_eio_entropy_collection) - (modes native) (modules test_eio_rng test_eio_entropy_collection) (libraries mirage-crypto-rng-eio duration eio_main ohex) (package mirage-crypto-rng-eio)) (tests (names test_miou_rng test_miou_entropy_collection) - (modes native) (modules test_miou_rng test_miou_entropy_collection) (libraries mirage-crypto-rng-miou-unix duration ohex) (package mirage-crypto-rng-miou-unix))