From 820447db84465789238f6b81692ea4b4ad91f01a Mon Sep 17 00:00:00 2001 From: Steven Barrett Date: Wed, 1 Apr 2020 16:42:07 -0500 Subject: [PATCH 01/34] Revert "Add WireGuard for v5.5 - 2020-03-30" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit af2ea5dbfdc06d07182b39cfb738a7ceb9194eb2. Doesn't build with 5.5.14, did with 5.5.13. Let's wait for an official patch from the WireGuard team considering both Zen and Liquorix haven't had a successful release with WireGuard yet. Errors from build log: In file included from drivers/net/wireguard/main.c:9: drivers/net/wireguard/queueing.h: In function ‘wg_reset_packet’: drivers/net/wireguard/queueing.h:103:2: error: implicit declaration of function ‘skb_reset_tc’; did you mean ‘nf_reset_ct’? [-Werror=implicit-function-declaration] 103 | skb_reset_tc(skb); | ^~~~~~~~~~~~ | nf_reset_ct In file included from drivers/net/wireguard/noise.c:10: drivers/net/wireguard/queueing.h: In function ‘wg_reset_packet’: drivers/net/wireguard/queueing.h:103:2: error: implicit declaration of function ‘skb_reset_tc’; did you mean ‘nf_reset_ct’? [-Werror=implicit-function-declaration] 103 | skb_reset_tc(skb); | ^~~~~~~~~~~~ | nf_reset_ct CC [M] drivers/net/eql.o cc1: some warnings being treated as errors In file included from drivers/net/wireguard/device.c:6: drivers/net/wireguard/queueing.h: In function ‘wg_reset_packet’: drivers/net/wireguard/queueing.h:103:2: error: implicit declaration of function ‘skb_reset_tc’; did you mean ‘nf_reset_ct’? [-Werror=implicit-function-declaration] 103 | skb_reset_tc(skb); | ^~~~~~~~~~~~ | nf_reset_ct make[3]: *** [scripts/Makefile.build:266: drivers/net/wireguard/main.o] Error 1 make[3]: *** Waiting for unfinished jobs.... CC [M] drivers/net/ifb.o In file included from drivers/net/wireguard/peer.c:8: drivers/net/wireguard/queueing.h: In function ‘wg_reset_packet’: drivers/net/wireguard/queueing.h:103:2: error: implicit declaration of function ‘skb_reset_tc’; did you mean ‘nf_reset_ct’? [-Werror=implicit-function-declaration] 103 | skb_reset_tc(skb); | ^~~~~~~~~~~~ | nf_reset_ct CC [M] drivers/net/macsec.o cc1: some warnings being treated as errors make[3]: *** [scripts/Makefile.build:266: drivers/net/wireguard/peer.o] Error 1 CC [M] drivers/net/wireless/ath/ath5k/caps.o cc1: some warnings being treated as errors make[3]: *** [scripts/Makefile.build:266: drivers/net/wireguard/device.o] Error 1 CC [M] drivers/net/wireless/ath/ath6kl/debug.o cc1: some warnings being treated as errors make[3]: *** [scripts/Makefile.build:266: drivers/net/wireguard/noise.o] Error 1 make[2]: *** [scripts/Makefile.build:503: drivers/net/wireguard] Error 2 make[2]: *** Waiting for unfinished jobs.... --- MAINTAINERS | 8 - arch/arm/crypto/.gitignore | 1 - arch/arm/crypto/poly1305-glue.c | 18 +- arch/arm64/crypto/.gitignore | 1 - arch/arm64/crypto/chacha-neon-glue.c | 8 +- arch/arm64/crypto/poly1305-glue.c | 18 +- arch/mips/crypto/poly1305-glue.c | 18 +- arch/x86/Makefile | 5 +- arch/x86/crypto/.gitignore | 1 - arch/x86/crypto/Makefile | 18 +- arch/x86/crypto/curve25519-x86_64.c | 3546 +++++++++----- arch/x86/crypto/poly1305-avx2-x86_64.S | 390 ++ arch/x86/crypto/poly1305-sse2-x86_64.S | 590 +++ arch/x86/crypto/poly1305-x86_64-cryptogams.pl | 4265 ----------------- arch/x86/crypto/poly1305_glue.c | 304 +- crypto/Kconfig | 4 + crypto/adiantum.c | 4 +- crypto/nhpoly1305.c | 2 +- crypto/poly1305_generic.c | 25 +- drivers/net/Kconfig | 43 - drivers/net/Makefile | 1 - drivers/net/wireguard/Makefile | 18 - drivers/net/wireguard/allowedips.c | 377 -- drivers/net/wireguard/allowedips.h | 59 - drivers/net/wireguard/cookie.c | 236 - drivers/net/wireguard/cookie.h | 59 - drivers/net/wireguard/device.c | 459 -- drivers/net/wireguard/device.h | 65 - drivers/net/wireguard/main.c | 63 - drivers/net/wireguard/messages.h | 128 - drivers/net/wireguard/netlink.c | 636 --- drivers/net/wireguard/netlink.h | 12 - drivers/net/wireguard/noise.c | 835 ---- drivers/net/wireguard/noise.h | 137 - drivers/net/wireguard/peer.c | 237 - drivers/net/wireguard/peer.h | 83 - drivers/net/wireguard/peerlookup.c | 221 - drivers/net/wireguard/peerlookup.h | 64 - drivers/net/wireguard/queueing.c | 53 - drivers/net/wireguard/queueing.h | 200 - drivers/net/wireguard/ratelimiter.c | 223 - drivers/net/wireguard/ratelimiter.h | 19 - drivers/net/wireguard/receive.c | 597 --- drivers/net/wireguard/selftest/allowedips.c | 683 --- drivers/net/wireguard/selftest/counter.c | 104 - drivers/net/wireguard/selftest/ratelimiter.c | 226 - drivers/net/wireguard/send.c | 419 -- drivers/net/wireguard/socket.c | 437 -- drivers/net/wireguard/socket.h | 44 - drivers/net/wireguard/timers.c | 243 - drivers/net/wireguard/timers.h | 31 - drivers/net/wireguard/version.h | 1 - include/crypto/curve25519.h | 6 +- include/crypto/internal/poly1305.h | 45 +- include/crypto/nhpoly1305.h | 4 +- include/crypto/poly1305.h | 26 +- include/linux/icmpv6.h | 10 - include/linux/skbuff.h | 5 - include/net/icmp.h | 6 - include/uapi/linux/wireguard.h | 196 - lib/crypto/Kconfig | 2 +- lib/crypto/Makefile | 4 +- lib/crypto/chacha20poly1305-selftest.c | 1717 +------ lib/crypto/poly1305-donna32.c | 204 - lib/crypto/poly1305-donna64.c | 185 - lib/crypto/poly1305.c | 169 +- net/ipv4/icmp.c | 33 - net/ipv6/ip6_icmp.c | 34 - tools/testing/selftests/wireguard/netns.sh | 555 --- .../selftests/wireguard/qemu/.gitignore | 2 - .../testing/selftests/wireguard/qemu/Makefile | 377 -- .../wireguard/qemu/arch/aarch64.config | 5 - .../wireguard/qemu/arch/aarch64_be.config | 6 - .../selftests/wireguard/qemu/arch/arm.config | 9 - .../wireguard/qemu/arch/armeb.config | 10 - .../selftests/wireguard/qemu/arch/i686.config | 5 - .../selftests/wireguard/qemu/arch/m68k.config | 9 - .../selftests/wireguard/qemu/arch/mips.config | 11 - .../wireguard/qemu/arch/mips64.config | 14 - .../wireguard/qemu/arch/mips64el.config | 15 - .../wireguard/qemu/arch/mipsel.config | 12 - .../wireguard/qemu/arch/powerpc.config | 10 - .../wireguard/qemu/arch/powerpc64le.config | 12 - .../wireguard/qemu/arch/x86_64.config | 5 - .../selftests/wireguard/qemu/debug.config | 67 - tools/testing/selftests/wireguard/qemu/init.c | 284 -- .../selftests/wireguard/qemu/kernel.config | 88 - 87 files changed, 3646 insertions(+), 16735 deletions(-) delete mode 100644 arch/x86/crypto/.gitignore create mode 100644 arch/x86/crypto/poly1305-avx2-x86_64.S create mode 100644 arch/x86/crypto/poly1305-sse2-x86_64.S delete mode 100644 arch/x86/crypto/poly1305-x86_64-cryptogams.pl delete mode 100644 drivers/net/wireguard/Makefile delete mode 100644 drivers/net/wireguard/allowedips.c delete mode 100644 drivers/net/wireguard/allowedips.h delete mode 100644 drivers/net/wireguard/cookie.c delete mode 100644 drivers/net/wireguard/cookie.h delete mode 100644 drivers/net/wireguard/device.c delete mode 100644 drivers/net/wireguard/device.h delete mode 100644 drivers/net/wireguard/main.c delete mode 100644 drivers/net/wireguard/messages.h delete mode 100644 drivers/net/wireguard/netlink.c delete mode 100644 drivers/net/wireguard/netlink.h delete mode 100644 drivers/net/wireguard/noise.c delete mode 100644 drivers/net/wireguard/noise.h delete mode 100644 drivers/net/wireguard/peer.c delete mode 100644 drivers/net/wireguard/peer.h delete mode 100644 drivers/net/wireguard/peerlookup.c delete mode 100644 drivers/net/wireguard/peerlookup.h delete mode 100644 drivers/net/wireguard/queueing.c delete mode 100644 drivers/net/wireguard/queueing.h delete mode 100644 drivers/net/wireguard/ratelimiter.c delete mode 100644 drivers/net/wireguard/ratelimiter.h delete mode 100644 drivers/net/wireguard/receive.c delete mode 100644 drivers/net/wireguard/selftest/allowedips.c delete mode 100644 drivers/net/wireguard/selftest/counter.c delete mode 100644 drivers/net/wireguard/selftest/ratelimiter.c delete mode 100644 drivers/net/wireguard/send.c delete mode 100644 drivers/net/wireguard/socket.c delete mode 100644 drivers/net/wireguard/socket.h delete mode 100644 drivers/net/wireguard/timers.c delete mode 100644 drivers/net/wireguard/timers.h delete mode 100644 drivers/net/wireguard/version.h delete mode 100644 include/uapi/linux/wireguard.h delete mode 100644 lib/crypto/poly1305-donna32.c delete mode 100644 lib/crypto/poly1305-donna64.c delete mode 100755 tools/testing/selftests/wireguard/netns.sh delete mode 100644 tools/testing/selftests/wireguard/qemu/.gitignore delete mode 100644 tools/testing/selftests/wireguard/qemu/Makefile delete mode 100644 tools/testing/selftests/wireguard/qemu/arch/aarch64.config delete mode 100644 tools/testing/selftests/wireguard/qemu/arch/aarch64_be.config delete mode 100644 tools/testing/selftests/wireguard/qemu/arch/arm.config delete mode 100644 tools/testing/selftests/wireguard/qemu/arch/armeb.config delete mode 100644 tools/testing/selftests/wireguard/qemu/arch/i686.config delete mode 100644 tools/testing/selftests/wireguard/qemu/arch/m68k.config delete mode 100644 tools/testing/selftests/wireguard/qemu/arch/mips.config delete mode 100644 tools/testing/selftests/wireguard/qemu/arch/mips64.config delete mode 100644 tools/testing/selftests/wireguard/qemu/arch/mips64el.config delete mode 100644 tools/testing/selftests/wireguard/qemu/arch/mipsel.config delete mode 100644 tools/testing/selftests/wireguard/qemu/arch/powerpc.config delete mode 100644 tools/testing/selftests/wireguard/qemu/arch/powerpc64le.config delete mode 100644 tools/testing/selftests/wireguard/qemu/arch/x86_64.config delete mode 100644 tools/testing/selftests/wireguard/qemu/debug.config delete mode 100644 tools/testing/selftests/wireguard/qemu/init.c delete mode 100644 tools/testing/selftests/wireguard/qemu/kernel.config diff --git a/MAINTAINERS b/MAINTAINERS index d9adb2152f683..61856805620eb 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17872,14 +17872,6 @@ L: linux-gpio@vger.kernel.org S: Maintained F: drivers/gpio/gpio-ws16c48.c -WIREGUARD SECURE NETWORK TUNNEL -M: Jason A. Donenfeld -S: Maintained -F: drivers/net/wireguard/ -F: tools/testing/selftests/wireguard/ -L: wireguard@lists.zx2c4.com -L: netdev@vger.kernel.org - WISTRON LAPTOP BUTTON DRIVER M: Miloslav Trmac S: Maintained diff --git a/arch/arm/crypto/.gitignore b/arch/arm/crypto/.gitignore index a3c7ad52a469f..31e1f538df7de 100644 --- a/arch/arm/crypto/.gitignore +++ b/arch/arm/crypto/.gitignore @@ -1,4 +1,3 @@ aesbs-core.S sha256-core.S sha512-core.S -poly1305-core.S diff --git a/arch/arm/crypto/poly1305-glue.c b/arch/arm/crypto/poly1305-glue.c index ceec04ec2f405..abe3f2d587dcb 100644 --- a/arch/arm/crypto/poly1305-glue.c +++ b/arch/arm/crypto/poly1305-glue.c @@ -20,7 +20,7 @@ void poly1305_init_arm(void *state, const u8 *key); void poly1305_blocks_arm(void *state, const u8 *src, u32 len, u32 hibit); -void poly1305_emit_arm(void *state, u8 *digest, const u32 *nonce); +void poly1305_emit_arm(void *state, __le32 *digest, const u32 *nonce); void __weak poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit) { @@ -179,6 +179,9 @@ EXPORT_SYMBOL(poly1305_update_arch); void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) { + __le32 digest[4]; + u64 f = 0; + if (unlikely(dctx->buflen)) { dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, @@ -186,7 +189,18 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) poly1305_blocks_arm(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); } - poly1305_emit_arm(&dctx->h, dst, dctx->s); + poly1305_emit_arm(&dctx->h, digest, dctx->s); + + /* mac = (h + s) % (2^128) */ + f = (f >> 32) + le32_to_cpu(digest[0]); + put_unaligned_le32(f, dst); + f = (f >> 32) + le32_to_cpu(digest[1]); + put_unaligned_le32(f, dst + 4); + f = (f >> 32) + le32_to_cpu(digest[2]); + put_unaligned_le32(f, dst + 8); + f = (f >> 32) + le32_to_cpu(digest[3]); + put_unaligned_le32(f, dst + 12); + *dctx = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL(poly1305_final_arch); diff --git a/arch/arm64/crypto/.gitignore b/arch/arm64/crypto/.gitignore index e403b1343328d..879df8781ed56 100644 --- a/arch/arm64/crypto/.gitignore +++ b/arch/arm64/crypto/.gitignore @@ -1,3 +1,2 @@ sha256-core.S sha512-core.S -poly1305-core.S diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/crypto/chacha-neon-glue.c index 37ca3e8898484..c1f9660d104cf 100644 --- a/arch/arm64/crypto/chacha-neon-glue.c +++ b/arch/arm64/crypto/chacha-neon-glue.c @@ -55,10 +55,10 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, break; } chacha_4block_xor_neon(state, dst, src, nrounds, l); - bytes -= l; - src += l; - dst += l; - state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); + bytes -= CHACHA_BLOCK_SIZE * 5; + src += CHACHA_BLOCK_SIZE * 5; + dst += CHACHA_BLOCK_SIZE * 5; + state[12] += 5; } } diff --git a/arch/arm64/crypto/poly1305-glue.c b/arch/arm64/crypto/poly1305-glue.c index e97b092f56b8f..83a2338a88263 100644 --- a/arch/arm64/crypto/poly1305-glue.c +++ b/arch/arm64/crypto/poly1305-glue.c @@ -21,7 +21,7 @@ asmlinkage void poly1305_init_arm64(void *state, const u8 *key); asmlinkage void poly1305_blocks(void *state, const u8 *src, u32 len, u32 hibit); asmlinkage void poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit); -asmlinkage void poly1305_emit(void *state, u8 *digest, const u32 *nonce); +asmlinkage void poly1305_emit(void *state, __le32 *digest, const u32 *nonce); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); @@ -162,6 +162,9 @@ EXPORT_SYMBOL(poly1305_update_arch); void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) { + __le32 digest[4]; + u64 f = 0; + if (unlikely(dctx->buflen)) { dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, @@ -169,7 +172,18 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) poly1305_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); } - poly1305_emit(&dctx->h, dst, dctx->s); + poly1305_emit(&dctx->h, digest, dctx->s); + + /* mac = (h + s) % (2^128) */ + f = (f >> 32) + le32_to_cpu(digest[0]); + put_unaligned_le32(f, dst); + f = (f >> 32) + le32_to_cpu(digest[1]); + put_unaligned_le32(f, dst + 4); + f = (f >> 32) + le32_to_cpu(digest[2]); + put_unaligned_le32(f, dst + 8); + f = (f >> 32) + le32_to_cpu(digest[3]); + put_unaligned_le32(f, dst + 12); + *dctx = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL(poly1305_final_arch); diff --git a/arch/mips/crypto/poly1305-glue.c b/arch/mips/crypto/poly1305-glue.c index fc881b46d9111..b37d29cf5d0a8 100644 --- a/arch/mips/crypto/poly1305-glue.c +++ b/arch/mips/crypto/poly1305-glue.c @@ -15,7 +15,7 @@ asmlinkage void poly1305_init_mips(void *state, const u8 *key); asmlinkage void poly1305_blocks_mips(void *state, const u8 *src, u32 len, u32 hibit); -asmlinkage void poly1305_emit_mips(void *state, u8 *digest, const u32 *nonce); +asmlinkage void poly1305_emit_mips(void *state, __le32 *digest, const u32 *nonce); void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 *key) { @@ -134,6 +134,9 @@ EXPORT_SYMBOL(poly1305_update_arch); void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) { + __le32 digest[4]; + u64 f = 0; + if (unlikely(dctx->buflen)) { dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, @@ -141,7 +144,18 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) poly1305_blocks_mips(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); } - poly1305_emit_mips(&dctx->h, dst, dctx->s); + poly1305_emit_mips(&dctx->h, digest, dctx->s); + + /* mac = (h + s) % (2^128) */ + f = (f >> 32) + le32_to_cpu(digest[0]); + put_unaligned_le32(f, dst); + f = (f >> 32) + le32_to_cpu(digest[1]); + put_unaligned_le32(f, dst + 4); + f = (f >> 32) + le32_to_cpu(digest[2]); + put_unaligned_le32(f, dst + 8); + f = (f >> 32) + le32_to_cpu(digest[3]); + put_unaligned_le32(f, dst + 12); + *dctx = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL(poly1305_final_arch); diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 800e049850ec1..dcbed7e3a0700 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -234,10 +234,9 @@ avx2_instr :=$(call as-instr,vpbroadcastb %xmm0$(comma)%ymm1,-DCONFIG_AS_AVX2=1) avx512_instr :=$(call as-instr,vpmovm2b %k1$(comma)%zmm5,-DCONFIG_AS_AVX512=1) sha1_ni_instr :=$(call as-instr,sha1msg1 %xmm0$(comma)%xmm1,-DCONFIG_AS_SHA1_NI=1) sha256_ni_instr :=$(call as-instr,sha256msg1 %xmm0$(comma)%xmm1,-DCONFIG_AS_SHA256_NI=1) -adx_instr := $(call as-instr,adox %r10$(comma)%r10,-DCONFIG_AS_ADX=1) -KBUILD_AFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(avx512_instr) $(sha1_ni_instr) $(sha256_ni_instr) $(adx_instr) -KBUILD_CFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(avx512_instr) $(sha1_ni_instr) $(sha256_ni_instr) $(adx_instr) +KBUILD_AFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(avx512_instr) $(sha1_ni_instr) $(sha256_ni_instr) +KBUILD_CFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(avx512_instr) $(sha1_ni_instr) $(sha256_ni_instr) KBUILD_LDFLAGS := -m elf_$(UTS_MACHINE) diff --git a/arch/x86/crypto/.gitignore b/arch/x86/crypto/.gitignore deleted file mode 100644 index 30be0400a4392..0000000000000 --- a/arch/x86/crypto/.gitignore +++ /dev/null @@ -1 +0,0 @@ -poly1305-x86_64-cryptogams.S diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 8c2e9eadee8a0..958440eae27ec 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -11,7 +11,6 @@ avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\ avx512_supported :=$(call as-instr,vpmovm2b %k1$(comma)%zmm5,yes,no) sha1_ni_supported :=$(call as-instr,sha1msg1 %xmm0$(comma)%xmm1,yes,no) sha256_ni_supported :=$(call as-instr,sha256msg1 %xmm0$(comma)%xmm1,yes,no) -adx_supported := $(call as-instr,adox %r10$(comma)%r10,yes,no) obj-$(CONFIG_CRYPTO_GLUE_HELPER_X86) += glue_helper.o @@ -40,11 +39,7 @@ obj-$(CONFIG_CRYPTO_AEGIS128_AESNI_SSE2) += aegis128-aesni.o obj-$(CONFIG_CRYPTO_NHPOLY1305_SSE2) += nhpoly1305-sse2.o obj-$(CONFIG_CRYPTO_NHPOLY1305_AVX2) += nhpoly1305-avx2.o - -# These modules require the assembler to support ADX. -ifeq ($(adx_supported),yes) - obj-$(CONFIG_CRYPTO_CURVE25519_X86) += curve25519-x86_64.o -endif +obj-$(CONFIG_CRYPTO_CURVE25519_X86) += curve25519-x86_64.o # These modules require assembler to support AVX. ifeq ($(avx_supported),yes) @@ -78,10 +73,6 @@ aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o blake2s-x86_64-y := blake2s-core.o blake2s-glue.o -poly1305-x86_64-y := poly1305-x86_64-cryptogams.o poly1305_glue.o -ifneq ($(CONFIG_CRYPTO_POLY1305_X86_64),) -targets += poly1305-x86_64-cryptogams.S -endif ifeq ($(avx_supported),yes) camellia-aesni-avx-x86_64-y := camellia-aesni-avx-asm_64.o \ @@ -110,8 +101,10 @@ aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o aesni-intel-$(CONFIG_64BIT) += aesni-intel_avx-x86_64.o aes_ctrby8_avx-x86_64.o ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o +poly1305-x86_64-y := poly1305-sse2-x86_64.o poly1305_glue.o ifeq ($(avx2_supported),yes) sha1-ssse3-y += sha1_avx2_x86_64_asm.o +poly1305-x86_64-y += poly1305-avx2-x86_64.o endif ifeq ($(sha1_ni_supported),yes) sha1-ssse3-y += sha1_ni_asm.o @@ -125,8 +118,3 @@ sha256-ssse3-y += sha256_ni_asm.o endif sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o crct10dif-pclmul-y := crct10dif-pcl-asm_64.o crct10dif-pclmul_glue.o - -quiet_cmd_perlasm = PERLASM $@ - cmd_perlasm = $(PERL) $< > $@ -$(obj)/%.S: $(src)/%.pl FORCE - $(call if_changed,perlasm) diff --git a/arch/x86/crypto/curve25519-x86_64.c b/arch/x86/crypto/curve25519-x86_64.c index 8a17621f7d3a3..eec7d2d242396 100644 --- a/arch/x86/crypto/curve25519-x86_64.c +++ b/arch/x86/crypto/curve25519-x86_64.c @@ -1,7 +1,8 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2020 Jason A. Donenfeld . All Rights Reserved. - * Copyright (c) 2016-2020 INRIA, CMU and Microsoft Corporation + * Copyright (c) 2017 Armando Faz . All Rights Reserved. + * Copyright (C) 2018-2019 Jason A. Donenfeld . All Rights Reserved. + * Copyright (C) 2018 Samuel Neves . All Rights Reserved. */ #include @@ -15,1378 +16,2337 @@ #include #include -static __always_inline u64 eq_mask(u64 a, u64 b) +static __ro_after_init DEFINE_STATIC_KEY_FALSE(curve25519_use_bmi2); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(curve25519_use_adx); + +enum { NUM_WORDS_ELTFP25519 = 4 }; +typedef __aligned(32) u64 eltfp25519_1w[NUM_WORDS_ELTFP25519]; +typedef __aligned(32) u64 eltfp25519_1w_buffer[2 * NUM_WORDS_ELTFP25519]; + +#define mul_eltfp25519_1w_adx(c, a, b) do { \ + mul_256x256_integer_adx(m.buffer, a, b); \ + red_eltfp25519_1w_adx(c, m.buffer); \ +} while (0) + +#define mul_eltfp25519_1w_bmi2(c, a, b) do { \ + mul_256x256_integer_bmi2(m.buffer, a, b); \ + red_eltfp25519_1w_bmi2(c, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_1w_adx(a) do { \ + sqr_256x256_integer_adx(m.buffer, a); \ + red_eltfp25519_1w_adx(a, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_1w_bmi2(a) do { \ + sqr_256x256_integer_bmi2(m.buffer, a); \ + red_eltfp25519_1w_bmi2(a, m.buffer); \ +} while (0) + +#define mul_eltfp25519_2w_adx(c, a, b) do { \ + mul2_256x256_integer_adx(m.buffer, a, b); \ + red_eltfp25519_2w_adx(c, m.buffer); \ +} while (0) + +#define mul_eltfp25519_2w_bmi2(c, a, b) do { \ + mul2_256x256_integer_bmi2(m.buffer, a, b); \ + red_eltfp25519_2w_bmi2(c, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_2w_adx(a) do { \ + sqr2_256x256_integer_adx(m.buffer, a); \ + red_eltfp25519_2w_adx(a, m.buffer); \ +} while (0) + +#define sqr_eltfp25519_2w_bmi2(a) do { \ + sqr2_256x256_integer_bmi2(m.buffer, a); \ + red_eltfp25519_2w_bmi2(a, m.buffer); \ +} while (0) + +#define sqrn_eltfp25519_1w_adx(a, times) do { \ + int ____counter = (times); \ + while (____counter-- > 0) \ + sqr_eltfp25519_1w_adx(a); \ +} while (0) + +#define sqrn_eltfp25519_1w_bmi2(a, times) do { \ + int ____counter = (times); \ + while (____counter-- > 0) \ + sqr_eltfp25519_1w_bmi2(a); \ +} while (0) + +#define copy_eltfp25519_1w(C, A) do { \ + (C)[0] = (A)[0]; \ + (C)[1] = (A)[1]; \ + (C)[2] = (A)[2]; \ + (C)[3] = (A)[3]; \ +} while (0) + +#define setzero_eltfp25519_1w(C) do { \ + (C)[0] = 0; \ + (C)[1] = 0; \ + (C)[2] = 0; \ + (C)[3] = 0; \ +} while (0) + +__aligned(32) static const u64 table_ladder_8k[252 * NUM_WORDS_ELTFP25519] = { + /* 1 */ 0xfffffffffffffff3UL, 0xffffffffffffffffUL, + 0xffffffffffffffffUL, 0x5fffffffffffffffUL, + /* 2 */ 0x6b8220f416aafe96UL, 0x82ebeb2b4f566a34UL, + 0xd5a9a5b075a5950fUL, 0x5142b2cf4b2488f4UL, + /* 3 */ 0x6aaebc750069680cUL, 0x89cf7820a0f99c41UL, + 0x2a58d9183b56d0f4UL, 0x4b5aca80e36011a4UL, + /* 4 */ 0x329132348c29745dUL, 0xf4a2e616e1642fd7UL, + 0x1e45bb03ff67bc34UL, 0x306912d0f42a9b4aUL, + /* 5 */ 0xff886507e6af7154UL, 0x04f50e13dfeec82fUL, + 0xaa512fe82abab5ceUL, 0x174e251a68d5f222UL, + /* 6 */ 0xcf96700d82028898UL, 0x1743e3370a2c02c5UL, + 0x379eec98b4e86eaaUL, 0x0c59888a51e0482eUL, + /* 7 */ 0xfbcbf1d699b5d189UL, 0xacaef0d58e9fdc84UL, + 0xc1c20d06231f7614UL, 0x2938218da274f972UL, + /* 8 */ 0xf6af49beff1d7f18UL, 0xcc541c22387ac9c2UL, + 0x96fcc9ef4015c56bUL, 0x69c1627c690913a9UL, + /* 9 */ 0x7a86fd2f4733db0eUL, 0xfdb8c4f29e087de9UL, + 0x095e4b1a8ea2a229UL, 0x1ad7a7c829b37a79UL, + /* 10 */ 0x342d89cad17ea0c0UL, 0x67bedda6cced2051UL, + 0x19ca31bf2bb42f74UL, 0x3df7b4c84980acbbUL, + /* 11 */ 0xa8c6444dc80ad883UL, 0xb91e440366e3ab85UL, + 0xc215cda00164f6d8UL, 0x3d867c6ef247e668UL, + /* 12 */ 0xc7dd582bcc3e658cUL, 0xfd2c4748ee0e5528UL, + 0xa0fd9b95cc9f4f71UL, 0x7529d871b0675ddfUL, + /* 13 */ 0xb8f568b42d3cbd78UL, 0x1233011b91f3da82UL, + 0x2dce6ccd4a7c3b62UL, 0x75e7fc8e9e498603UL, + /* 14 */ 0x2f4f13f1fcd0b6ecUL, 0xf1a8ca1f29ff7a45UL, + 0xc249c1a72981e29bUL, 0x6ebe0dbb8c83b56aUL, + /* 15 */ 0x7114fa8d170bb222UL, 0x65a2dcd5bf93935fUL, + 0xbdc41f68b59c979aUL, 0x2f0eef79a2ce9289UL, + /* 16 */ 0x42ecbf0c083c37ceUL, 0x2930bc09ec496322UL, + 0xf294b0c19cfeac0dUL, 0x3780aa4bedfabb80UL, + /* 17 */ 0x56c17d3e7cead929UL, 0xe7cb4beb2e5722c5UL, + 0x0ce931732dbfe15aUL, 0x41b883c7621052f8UL, + /* 18 */ 0xdbf75ca0c3d25350UL, 0x2936be086eb1e351UL, + 0xc936e03cb4a9b212UL, 0x1d45bf82322225aaUL, + /* 19 */ 0xe81ab1036a024cc5UL, 0xe212201c304c9a72UL, + 0xc5d73fba6832b1fcUL, 0x20ffdb5a4d839581UL, + /* 20 */ 0xa283d367be5d0fadUL, 0x6c2b25ca8b164475UL, + 0x9d4935467caaf22eUL, 0x5166408eee85ff49UL, + /* 21 */ 0x3c67baa2fab4e361UL, 0xb3e433c67ef35cefUL, + 0x5259729241159b1cUL, 0x6a621892d5b0ab33UL, + /* 22 */ 0x20b74a387555cdcbUL, 0x532aa10e1208923fUL, + 0xeaa17b7762281dd1UL, 0x61ab3443f05c44bfUL, + /* 23 */ 0x257a6c422324def8UL, 0x131c6c1017e3cf7fUL, + 0x23758739f630a257UL, 0x295a407a01a78580UL, + /* 24 */ 0xf8c443246d5da8d9UL, 0x19d775450c52fa5dUL, + 0x2afcfc92731bf83dUL, 0x7d10c8e81b2b4700UL, + /* 25 */ 0xc8e0271f70baa20bUL, 0x993748867ca63957UL, + 0x5412efb3cb7ed4bbUL, 0x3196d36173e62975UL, + /* 26 */ 0xde5bcad141c7dffcUL, 0x47cc8cd2b395c848UL, + 0xa34cd942e11af3cbUL, 0x0256dbf2d04ecec2UL, + /* 27 */ 0x875ab7e94b0e667fUL, 0xcad4dd83c0850d10UL, + 0x47f12e8f4e72c79fUL, 0x5f1a87bb8c85b19bUL, + /* 28 */ 0x7ae9d0b6437f51b8UL, 0x12c7ce5518879065UL, + 0x2ade09fe5cf77aeeUL, 0x23a05a2f7d2c5627UL, + /* 29 */ 0x5908e128f17c169aUL, 0xf77498dd8ad0852dUL, + 0x74b4c4ceab102f64UL, 0x183abadd10139845UL, + /* 30 */ 0xb165ba8daa92aaacUL, 0xd5c5ef9599386705UL, + 0xbe2f8f0cf8fc40d1UL, 0x2701e635ee204514UL, + /* 31 */ 0x629fa80020156514UL, 0xf223868764a8c1ceUL, + 0x5b894fff0b3f060eUL, 0x60d9944cf708a3faUL, + /* 32 */ 0xaeea001a1c7a201fUL, 0xebf16a633ee2ce63UL, + 0x6f7709594c7a07e1UL, 0x79b958150d0208cbUL, + /* 33 */ 0x24b55e5301d410e7UL, 0xe3a34edff3fdc84dUL, + 0xd88768e4904032d8UL, 0x131384427b3aaeecUL, + /* 34 */ 0x8405e51286234f14UL, 0x14dc4739adb4c529UL, + 0xb8a2b5b250634ffdUL, 0x2fe2a94ad8a7ff93UL, + /* 35 */ 0xec5c57efe843faddUL, 0x2843ce40f0bb9918UL, + 0xa4b561d6cf3d6305UL, 0x743629bde8fb777eUL, + /* 36 */ 0x343edd46bbaf738fUL, 0xed981828b101a651UL, + 0xa401760b882c797aUL, 0x1fc223e28dc88730UL, + /* 37 */ 0x48604e91fc0fba0eUL, 0xb637f78f052c6fa4UL, + 0x91ccac3d09e9239cUL, 0x23f7eed4437a687cUL, + /* 38 */ 0x5173b1118d9bd800UL, 0x29d641b63189d4a7UL, + 0xfdbf177988bbc586UL, 0x2959894fcad81df5UL, + /* 39 */ 0xaebc8ef3b4bbc899UL, 0x4148995ab26992b9UL, + 0x24e20b0134f92cfbUL, 0x40d158894a05dee8UL, + /* 40 */ 0x46b00b1185af76f6UL, 0x26bac77873187a79UL, + 0x3dc0bf95ab8fff5fUL, 0x2a608bd8945524d7UL, + /* 41 */ 0x26449588bd446302UL, 0x7c4bc21c0388439cUL, + 0x8e98a4f383bd11b2UL, 0x26218d7bc9d876b9UL, + /* 42 */ 0xe3081542997c178aUL, 0x3c2d29a86fb6606fUL, + 0x5c217736fa279374UL, 0x7dde05734afeb1faUL, + /* 43 */ 0x3bf10e3906d42babUL, 0xe4f7803e1980649cUL, + 0xe6053bf89595bf7aUL, 0x394faf38da245530UL, + /* 44 */ 0x7a8efb58896928f4UL, 0xfbc778e9cc6a113cUL, + 0x72670ce330af596fUL, 0x48f222a81d3d6cf7UL, + /* 45 */ 0xf01fce410d72caa7UL, 0x5a20ecc7213b5595UL, + 0x7bc21165c1fa1483UL, 0x07f89ae31da8a741UL, + /* 46 */ 0x05d2c2b4c6830ff9UL, 0xd43e330fc6316293UL, + 0xa5a5590a96d3a904UL, 0x705edb91a65333b6UL, + /* 47 */ 0x048ee15e0bb9a5f7UL, 0x3240cfca9e0aaf5dUL, + 0x8f4b71ceedc4a40bUL, 0x621c0da3de544a6dUL, + /* 48 */ 0x92872836a08c4091UL, 0xce8375b010c91445UL, + 0x8a72eb524f276394UL, 0x2667fcfa7ec83635UL, + /* 49 */ 0x7f4c173345e8752aUL, 0x061b47feee7079a5UL, + 0x25dd9afa9f86ff34UL, 0x3780cef5425dc89cUL, + /* 50 */ 0x1a46035a513bb4e9UL, 0x3e1ef379ac575adaUL, + 0xc78c5f1c5fa24b50UL, 0x321a967634fd9f22UL, + /* 51 */ 0x946707b8826e27faUL, 0x3dca84d64c506fd0UL, + 0xc189218075e91436UL, 0x6d9284169b3b8484UL, + /* 52 */ 0x3a67e840383f2ddfUL, 0x33eec9a30c4f9b75UL, + 0x3ec7c86fa783ef47UL, 0x26ec449fbac9fbc4UL, + /* 53 */ 0x5c0f38cba09b9e7dUL, 0x81168cc762a3478cUL, + 0x3e23b0d306fc121cUL, 0x5a238aa0a5efdcddUL, + /* 54 */ 0x1ba26121c4ea43ffUL, 0x36f8c77f7c8832b5UL, + 0x88fbea0b0adcf99aUL, 0x5ca9938ec25bebf9UL, + /* 55 */ 0xd5436a5e51fccda0UL, 0x1dbc4797c2cd893bUL, + 0x19346a65d3224a08UL, 0x0f5034e49b9af466UL, + /* 56 */ 0xf23c3967a1e0b96eUL, 0xe58b08fa867a4d88UL, + 0xfb2fabc6a7341679UL, 0x2a75381eb6026946UL, + /* 57 */ 0xc80a3be4c19420acUL, 0x66b1f6c681f2b6dcUL, + 0x7cf7036761e93388UL, 0x25abbbd8a660a4c4UL, + /* 58 */ 0x91ea12ba14fd5198UL, 0x684950fc4a3cffa9UL, + 0xf826842130f5ad28UL, 0x3ea988f75301a441UL, + /* 59 */ 0xc978109a695f8c6fUL, 0x1746eb4a0530c3f3UL, + 0x444d6d77b4459995UL, 0x75952b8c054e5cc7UL, + /* 60 */ 0xa3703f7915f4d6aaUL, 0x66c346202f2647d8UL, + 0xd01469df811d644bUL, 0x77fea47d81a5d71fUL, + /* 61 */ 0xc5e9529ef57ca381UL, 0x6eeeb4b9ce2f881aUL, + 0xb6e91a28e8009bd6UL, 0x4b80be3e9afc3fecUL, + /* 62 */ 0x7e3773c526aed2c5UL, 0x1b4afcb453c9a49dUL, + 0xa920bdd7baffb24dUL, 0x7c54699f122d400eUL, + /* 63 */ 0xef46c8e14fa94bc8UL, 0xe0b074ce2952ed5eUL, + 0xbea450e1dbd885d5UL, 0x61b68649320f712cUL, + /* 64 */ 0x8a485f7309ccbdd1UL, 0xbd06320d7d4d1a2dUL, + 0x25232973322dbef4UL, 0x445dc4758c17f770UL, + /* 65 */ 0xdb0434177cc8933cUL, 0xed6fe82175ea059fUL, + 0x1efebefdc053db34UL, 0x4adbe867c65daf99UL, + /* 66 */ 0x3acd71a2a90609dfUL, 0xe5e991856dd04050UL, + 0x1ec69b688157c23cUL, 0x697427f6885cfe4dUL, + /* 67 */ 0xd7be7b9b65e1a851UL, 0xa03d28d522c536ddUL, + 0x28399d658fd2b645UL, 0x49e5b7e17c2641e1UL, + /* 68 */ 0x6f8c3a98700457a4UL, 0x5078f0a25ebb6778UL, + 0xd13c3ccbc382960fUL, 0x2e003258a7df84b1UL, + /* 69 */ 0x8ad1f39be6296a1cUL, 0xc1eeaa652a5fbfb2UL, + 0x33ee0673fd26f3cbUL, 0x59256173a69d2cccUL, + /* 70 */ 0x41ea07aa4e18fc41UL, 0xd9fc19527c87a51eUL, + 0xbdaacb805831ca6fUL, 0x445b652dc916694fUL, + /* 71 */ 0xce92a3a7f2172315UL, 0x1edc282de11b9964UL, + 0xa1823aafe04c314aUL, 0x790a2d94437cf586UL, + /* 72 */ 0x71c447fb93f6e009UL, 0x8922a56722845276UL, + 0xbf70903b204f5169UL, 0x2f7a89891ba319feUL, + /* 73 */ 0x02a08eb577e2140cUL, 0xed9a4ed4427bdcf4UL, + 0x5253ec44e4323cd1UL, 0x3e88363c14e9355bUL, + /* 74 */ 0xaa66c14277110b8cUL, 0x1ae0391610a23390UL, + 0x2030bd12c93fc2a2UL, 0x3ee141579555c7abUL, + /* 75 */ 0x9214de3a6d6e7d41UL, 0x3ccdd88607f17efeUL, + 0x674f1288f8e11217UL, 0x5682250f329f93d0UL, + /* 76 */ 0x6cf00b136d2e396eUL, 0x6e4cf86f1014debfUL, + 0x5930b1b5bfcc4e83UL, 0x047069b48aba16b6UL, + /* 77 */ 0x0d4ce4ab69b20793UL, 0xb24db91a97d0fb9eUL, + 0xcdfa50f54e00d01dUL, 0x221b1085368bddb5UL, + /* 78 */ 0xe7e59468b1e3d8d2UL, 0x53c56563bd122f93UL, + 0xeee8a903e0663f09UL, 0x61efa662cbbe3d42UL, + /* 79 */ 0x2cf8ddddde6eab2aUL, 0x9bf80ad51435f231UL, + 0x5deadacec9f04973UL, 0x29275b5d41d29b27UL, + /* 80 */ 0xcfde0f0895ebf14fUL, 0xb9aab96b054905a7UL, + 0xcae80dd9a1c420fdUL, 0x0a63bf2f1673bbc7UL, + /* 81 */ 0x092f6e11958fbc8cUL, 0x672a81e804822fadUL, + 0xcac8351560d52517UL, 0x6f3f7722c8f192f8UL, + /* 82 */ 0xf8ba90ccc2e894b7UL, 0x2c7557a438ff9f0dUL, + 0x894d1d855ae52359UL, 0x68e122157b743d69UL, + /* 83 */ 0xd87e5570cfb919f3UL, 0x3f2cdecd95798db9UL, + 0x2121154710c0a2ceUL, 0x3c66a115246dc5b2UL, + /* 84 */ 0xcbedc562294ecb72UL, 0xba7143c36a280b16UL, + 0x9610c2efd4078b67UL, 0x6144735d946a4b1eUL, + /* 85 */ 0x536f111ed75b3350UL, 0x0211db8c2041d81bUL, + 0xf93cb1000e10413cUL, 0x149dfd3c039e8876UL, + /* 86 */ 0xd479dde46b63155bUL, 0xb66e15e93c837976UL, + 0xdafde43b1f13e038UL, 0x5fafda1a2e4b0b35UL, + /* 87 */ 0x3600bbdf17197581UL, 0x3972050bbe3cd2c2UL, + 0x5938906dbdd5be86UL, 0x34fce5e43f9b860fUL, + /* 88 */ 0x75a8a4cd42d14d02UL, 0x828dabc53441df65UL, + 0x33dcabedd2e131d3UL, 0x3ebad76fb814d25fUL, + /* 89 */ 0xd4906f566f70e10fUL, 0x5d12f7aa51690f5aUL, + 0x45adb16e76cefcf2UL, 0x01f768aead232999UL, + /* 90 */ 0x2b6cc77b6248febdUL, 0x3cd30628ec3aaffdUL, + 0xce1c0b80d4ef486aUL, 0x4c3bff2ea6f66c23UL, + /* 91 */ 0x3f2ec4094aeaeb5fUL, 0x61b19b286e372ca7UL, + 0x5eefa966de2a701dUL, 0x23b20565de55e3efUL, + /* 92 */ 0xe301ca5279d58557UL, 0x07b2d4ce27c2874fUL, + 0xa532cd8a9dcf1d67UL, 0x2a52fee23f2bff56UL, + /* 93 */ 0x8624efb37cd8663dUL, 0xbbc7ac20ffbd7594UL, + 0x57b85e9c82d37445UL, 0x7b3052cb86a6ec66UL, + /* 94 */ 0x3482f0ad2525e91eUL, 0x2cb68043d28edca0UL, + 0xaf4f6d052e1b003aUL, 0x185f8c2529781b0aUL, + /* 95 */ 0xaa41de5bd80ce0d6UL, 0x9407b2416853e9d6UL, + 0x563ec36e357f4c3aUL, 0x4cc4b8dd0e297bceUL, + /* 96 */ 0xa2fc1a52ffb8730eUL, 0x1811f16e67058e37UL, + 0x10f9a366cddf4ee1UL, 0x72f4a0c4a0b9f099UL, + /* 97 */ 0x8c16c06f663f4ea7UL, 0x693b3af74e970fbaUL, + 0x2102e7f1d69ec345UL, 0x0ba53cbc968a8089UL, + /* 98 */ 0xca3d9dc7fea15537UL, 0x4c6824bb51536493UL, + 0xb9886314844006b1UL, 0x40d2a72ab454cc60UL, + /* 99 */ 0x5936a1b712570975UL, 0x91b9d648debda657UL, + 0x3344094bb64330eaUL, 0x006ba10d12ee51d0UL, + /* 100 */ 0x19228468f5de5d58UL, 0x0eb12f4c38cc05b0UL, + 0xa1039f9dd5601990UL, 0x4502d4ce4fff0e0bUL, + /* 101 */ 0xeb2054106837c189UL, 0xd0f6544c6dd3b93cUL, + 0x40727064c416d74fUL, 0x6e15c6114b502ef0UL, + /* 102 */ 0x4df2a398cfb1a76bUL, 0x11256c7419f2f6b1UL, + 0x4a497962066e6043UL, 0x705b3aab41355b44UL, + /* 103 */ 0x365ef536d797b1d8UL, 0x00076bd622ddf0dbUL, + 0x3bbf33b0e0575a88UL, 0x3777aa05c8e4ca4dUL, + /* 104 */ 0x392745c85578db5fUL, 0x6fda4149dbae5ae2UL, + 0xb1f0b00b8adc9867UL, 0x09963437d36f1da3UL, + /* 105 */ 0x7e824e90a5dc3853UL, 0xccb5f6641f135cbdUL, + 0x6736d86c87ce8fccUL, 0x625f3ce26604249fUL, + /* 106 */ 0xaf8ac8059502f63fUL, 0x0c05e70a2e351469UL, + 0x35292e9c764b6305UL, 0x1a394360c7e23ac3UL, + /* 107 */ 0xd5c6d53251183264UL, 0x62065abd43c2b74fUL, + 0xb5fbf5d03b973f9bUL, 0x13a3da3661206e5eUL, + /* 108 */ 0xc6bd5837725d94e5UL, 0x18e30912205016c5UL, + 0x2088ce1570033c68UL, 0x7fba1f495c837987UL, + /* 109 */ 0x5a8c7423f2f9079dUL, 0x1735157b34023fc5UL, + 0xe4f9b49ad2fab351UL, 0x6691ff72c878e33cUL, + /* 110 */ 0x122c2adedc5eff3eUL, 0xf8dd4bf1d8956cf4UL, + 0xeb86205d9e9e5bdaUL, 0x049b92b9d975c743UL, + /* 111 */ 0xa5379730b0f6c05aUL, 0x72a0ffacc6f3a553UL, + 0xb0032c34b20dcd6dUL, 0x470e9dbc88d5164aUL, + /* 112 */ 0xb19cf10ca237c047UL, 0xb65466711f6c81a2UL, + 0xb3321bd16dd80b43UL, 0x48c14f600c5fbe8eUL, + /* 113 */ 0x66451c264aa6c803UL, 0xb66e3904a4fa7da6UL, + 0xd45f19b0b3128395UL, 0x31602627c3c9bc10UL, + /* 114 */ 0x3120dc4832e4e10dUL, 0xeb20c46756c717f7UL, + 0x00f52e3f67280294UL, 0x566d4fc14730c509UL, + /* 115 */ 0x7e3a5d40fd837206UL, 0xc1e926dc7159547aUL, + 0x216730fba68d6095UL, 0x22e8c3843f69cea7UL, + /* 116 */ 0x33d074e8930e4b2bUL, 0xb6e4350e84d15816UL, + 0x5534c26ad6ba2365UL, 0x7773c12f89f1f3f3UL, + /* 117 */ 0x8cba404da57962aaUL, 0x5b9897a81999ce56UL, + 0x508e862f121692fcUL, 0x3a81907fa093c291UL, + /* 118 */ 0x0dded0ff4725a510UL, 0x10d8cc10673fc503UL, + 0x5b9d151c9f1f4e89UL, 0x32a5c1d5cb09a44cUL, + /* 119 */ 0x1e0aa442b90541fbUL, 0x5f85eb7cc1b485dbUL, + 0xbee595ce8a9df2e5UL, 0x25e496c722422236UL, + /* 120 */ 0x5edf3c46cd0fe5b9UL, 0x34e75a7ed2a43388UL, + 0xe488de11d761e352UL, 0x0e878a01a085545cUL, + /* 121 */ 0xba493c77e021bb04UL, 0x2b4d1843c7df899aUL, + 0x9ea37a487ae80d67UL, 0x67a9958011e41794UL, + /* 122 */ 0x4b58051a6697b065UL, 0x47e33f7d8d6ba6d4UL, + 0xbb4da8d483ca46c1UL, 0x68becaa181c2db0dUL, + /* 123 */ 0x8d8980e90b989aa5UL, 0xf95eb14a2c93c99bUL, + 0x51c6c7c4796e73a2UL, 0x6e228363b5efb569UL, + /* 124 */ 0xc6bbc0b02dd624c8UL, 0x777eb47dec8170eeUL, + 0x3cde15a004cfafa9UL, 0x1dc6bc087160bf9bUL, + /* 125 */ 0x2e07e043eec34002UL, 0x18e9fc677a68dc7fUL, + 0xd8da03188bd15b9aUL, 0x48fbc3bb00568253UL, + /* 126 */ 0x57547d4cfb654ce1UL, 0xd3565b82a058e2adUL, + 0xf63eaf0bbf154478UL, 0x47531ef114dfbb18UL, + /* 127 */ 0xe1ec630a4278c587UL, 0x5507d546ca8e83f3UL, + 0x85e135c63adc0c2bUL, 0x0aa7efa85682844eUL, + /* 128 */ 0x72691ba8b3e1f615UL, 0x32b4e9701fbe3ffaUL, + 0x97b6d92e39bb7868UL, 0x2cfe53dea02e39e8UL, + /* 129 */ 0x687392cd85cd52b0UL, 0x27ff66c910e29831UL, + 0x97134556a9832d06UL, 0x269bb0360a84f8a0UL, + /* 130 */ 0x706e55457643f85cUL, 0x3734a48c9b597d1bUL, + 0x7aee91e8c6efa472UL, 0x5cd6abc198a9d9e0UL, + /* 131 */ 0x0e04de06cb3ce41aUL, 0xd8c6eb893402e138UL, + 0x904659bb686e3772UL, 0x7215c371746ba8c8UL, + /* 132 */ 0xfd12a97eeae4a2d9UL, 0x9514b7516394f2c5UL, + 0x266fd5809208f294UL, 0x5c847085619a26b9UL, + /* 133 */ 0x52985410fed694eaUL, 0x3c905b934a2ed254UL, + 0x10bb47692d3be467UL, 0x063b3d2d69e5e9e1UL, + /* 134 */ 0x472726eedda57debUL, 0xefb6c4ae10f41891UL, + 0x2b1641917b307614UL, 0x117c554fc4f45b7cUL, + /* 135 */ 0xc07cf3118f9d8812UL, 0x01dbd82050017939UL, + 0xd7e803f4171b2827UL, 0x1015e87487d225eaUL, + /* 136 */ 0xc58de3fed23acc4dUL, 0x50db91c294a7be2dUL, + 0x0b94d43d1c9cf457UL, 0x6b1640fa6e37524aUL, + /* 137 */ 0x692f346c5fda0d09UL, 0x200b1c59fa4d3151UL, + 0xb8c46f760777a296UL, 0x4b38395f3ffdfbcfUL, + /* 138 */ 0x18d25e00be54d671UL, 0x60d50582bec8aba6UL, + 0x87ad8f263b78b982UL, 0x50fdf64e9cda0432UL, + /* 139 */ 0x90f567aac578dcf0UL, 0xef1e9b0ef2a3133bUL, + 0x0eebba9242d9de71UL, 0x15473c9bf03101c7UL, + /* 140 */ 0x7c77e8ae56b78095UL, 0xb678e7666e6f078eUL, + 0x2da0b9615348ba1fUL, 0x7cf931c1ff733f0bUL, + /* 141 */ 0x26b357f50a0a366cUL, 0xe9708cf42b87d732UL, + 0xc13aeea5f91cb2c0UL, 0x35d90c991143bb4cUL, + /* 142 */ 0x47c1c404a9a0d9dcUL, 0x659e58451972d251UL, + 0x3875a8c473b38c31UL, 0x1fbd9ed379561f24UL, + /* 143 */ 0x11fabc6fd41ec28dUL, 0x7ef8dfe3cd2a2dcaUL, + 0x72e73b5d8c404595UL, 0x6135fa4954b72f27UL, + /* 144 */ 0xccfc32a2de24b69cUL, 0x3f55698c1f095d88UL, + 0xbe3350ed5ac3f929UL, 0x5e9bf806ca477eebUL, + /* 145 */ 0xe9ce8fb63c309f68UL, 0x5376f63565e1f9f4UL, + 0xd1afcfb35a6393f1UL, 0x6632a1ede5623506UL, + /* 146 */ 0x0b7d6c390c2ded4cUL, 0x56cb3281df04cb1fUL, + 0x66305a1249ecc3c7UL, 0x5d588b60a38ca72aUL, + /* 147 */ 0xa6ecbf78e8e5f42dUL, 0x86eeb44b3c8a3eecUL, + 0xec219c48fbd21604UL, 0x1aaf1af517c36731UL, + /* 148 */ 0xc306a2836769bde7UL, 0x208280622b1e2adbUL, + 0x8027f51ffbff94a6UL, 0x76cfa1ce1124f26bUL, + /* 149 */ 0x18eb00562422abb6UL, 0xf377c4d58f8c29c3UL, + 0x4dbbc207f531561aUL, 0x0253b7f082128a27UL, + /* 150 */ 0x3d1f091cb62c17e0UL, 0x4860e1abd64628a9UL, + 0x52d17436309d4253UL, 0x356f97e13efae576UL, + /* 151 */ 0xd351e11aa150535bUL, 0x3e6b45bb1dd878ccUL, + 0x0c776128bed92c98UL, 0x1d34ae93032885b8UL, + /* 152 */ 0x4ba0488ca85ba4c3UL, 0x985348c33c9ce6ceUL, + 0x66124c6f97bda770UL, 0x0f81a0290654124aUL, + /* 153 */ 0x9ed09ca6569b86fdUL, 0x811009fd18af9a2dUL, + 0xff08d03f93d8c20aUL, 0x52a148199faef26bUL, + /* 154 */ 0x3e03f9dc2d8d1b73UL, 0x4205801873961a70UL, + 0xc0d987f041a35970UL, 0x07aa1f15a1c0d549UL, + /* 155 */ 0xdfd46ce08cd27224UL, 0x6d0a024f934e4239UL, + 0x808a7a6399897b59UL, 0x0a4556e9e13d95a2UL, + /* 156 */ 0xd21a991fe9c13045UL, 0x9b0e8548fe7751b8UL, + 0x5da643cb4bf30035UL, 0x77db28d63940f721UL, + /* 157 */ 0xfc5eeb614adc9011UL, 0x5229419ae8c411ebUL, + 0x9ec3e7787d1dcf74UL, 0x340d053e216e4cb5UL, + /* 158 */ 0xcac7af39b48df2b4UL, 0xc0faec2871a10a94UL, + 0x140a69245ca575edUL, 0x0cf1c37134273a4cUL, + /* 159 */ 0xc8ee306ac224b8a5UL, 0x57eaee7ccb4930b0UL, + 0xa1e806bdaacbe74fUL, 0x7d9a62742eeb657dUL, + /* 160 */ 0x9eb6b6ef546c4830UL, 0x885cca1fddb36e2eUL, + 0xe6b9f383ef0d7105UL, 0x58654fef9d2e0412UL, + /* 161 */ 0xa905c4ffbe0e8e26UL, 0x942de5df9b31816eUL, + 0x497d723f802e88e1UL, 0x30684dea602f408dUL, + /* 162 */ 0x21e5a278a3e6cb34UL, 0xaefb6e6f5b151dc4UL, + 0xb30b8e049d77ca15UL, 0x28c3c9cf53b98981UL, + /* 163 */ 0x287fb721556cdd2aUL, 0x0d317ca897022274UL, + 0x7468c7423a543258UL, 0x4a7f11464eb5642fUL, + /* 164 */ 0xa237a4774d193aa6UL, 0xd865986ea92129a1UL, + 0x24c515ecf87c1a88UL, 0x604003575f39f5ebUL, + /* 165 */ 0x47b9f189570a9b27UL, 0x2b98cede465e4b78UL, + 0x026df551dbb85c20UL, 0x74fcd91047e21901UL, + /* 166 */ 0x13e2a90a23c1bfa3UL, 0x0cb0074e478519f6UL, + 0x5ff1cbbe3af6cf44UL, 0x67fe5438be812dbeUL, + /* 167 */ 0xd13cf64fa40f05b0UL, 0x054dfb2f32283787UL, + 0x4173915b7f0d2aeaUL, 0x482f144f1f610d4eUL, + /* 168 */ 0xf6210201b47f8234UL, 0x5d0ae1929e70b990UL, + 0xdcd7f455b049567cUL, 0x7e93d0f1f0916f01UL, + /* 169 */ 0xdd79cbf18a7db4faUL, 0xbe8391bf6f74c62fUL, + 0x027145d14b8291bdUL, 0x585a73ea2cbf1705UL, + /* 170 */ 0x485ca03e928a0db2UL, 0x10fc01a5742857e7UL, + 0x2f482edbd6d551a7UL, 0x0f0433b5048fdb8aUL, + /* 171 */ 0x60da2e8dd7dc6247UL, 0x88b4c9d38cd4819aUL, + 0x13033ac001f66697UL, 0x273b24fe3b367d75UL, + /* 172 */ 0xc6e8f66a31b3b9d4UL, 0x281514a494df49d5UL, + 0xd1726fdfc8b23da7UL, 0x4b3ae7d103dee548UL, + /* 173 */ 0xc6256e19ce4b9d7eUL, 0xff5c5cf186e3c61cUL, + 0xacc63ca34b8ec145UL, 0x74621888fee66574UL, + /* 174 */ 0x956f409645290a1eUL, 0xef0bf8e3263a962eUL, + 0xed6a50eb5ec2647bUL, 0x0694283a9dca7502UL, + /* 175 */ 0x769b963643a2dcd1UL, 0x42b7c8ea09fc5353UL, + 0x4f002aee13397eabUL, 0x63005e2c19b7d63aUL, + /* 176 */ 0xca6736da63023beaUL, 0x966c7f6db12a99b7UL, + 0xace09390c537c5e1UL, 0x0b696063a1aa89eeUL, + /* 177 */ 0xebb03e97288c56e5UL, 0x432a9f9f938c8be8UL, + 0xa6a5a93d5b717f71UL, 0x1a5fb4c3e18f9d97UL, + /* 178 */ 0x1c94e7ad1c60cdceUL, 0xee202a43fc02c4a0UL, + 0x8dafe4d867c46a20UL, 0x0a10263c8ac27b58UL, + /* 179 */ 0xd0dea9dfe4432a4aUL, 0x856af87bbe9277c5UL, + 0xce8472acc212c71aUL, 0x6f151b6d9bbb1e91UL, + /* 180 */ 0x26776c527ceed56aUL, 0x7d211cb7fbf8faecUL, + 0x37ae66a6fd4609ccUL, 0x1f81b702d2770c42UL, + /* 181 */ 0x2fb0b057eac58392UL, 0xe1dd89fe29744e9dUL, + 0xc964f8eb17beb4f8UL, 0x29571073c9a2d41eUL, + /* 182 */ 0xa948a18981c0e254UL, 0x2df6369b65b22830UL, + 0xa33eb2d75fcfd3c6UL, 0x078cd6ec4199a01fUL, + /* 183 */ 0x4a584a41ad900d2fUL, 0x32142b78e2c74c52UL, + 0x68c4e8338431c978UL, 0x7f69ea9008689fc2UL, + /* 184 */ 0x52f2c81e46a38265UL, 0xfd78072d04a832fdUL, + 0x8cd7d5fa25359e94UL, 0x4de71b7454cc29d2UL, + /* 185 */ 0x42eb60ad1eda6ac9UL, 0x0aad37dfdbc09c3aUL, + 0x81004b71e33cc191UL, 0x44e6be345122803cUL, + /* 186 */ 0x03fe8388ba1920dbUL, 0xf5d57c32150db008UL, + 0x49c8c4281af60c29UL, 0x21edb518de701aeeUL, + /* 187 */ 0x7fb63e418f06dc99UL, 0xa4460d99c166d7b8UL, + 0x24dd5248ce520a83UL, 0x5ec3ad712b928358UL, + /* 188 */ 0x15022a5fbd17930fUL, 0xa4f64a77d82570e3UL, + 0x12bc8d6915783712UL, 0x498194c0fc620abbUL, + /* 189 */ 0x38a2d9d255686c82UL, 0x785c6bd9193e21f0UL, + 0xe4d5c81ab24a5484UL, 0x56307860b2e20989UL, + /* 190 */ 0x429d55f78b4d74c4UL, 0x22f1834643350131UL, + 0x1e60c24598c71fffUL, 0x59f2f014979983efUL, + /* 191 */ 0x46a47d56eb494a44UL, 0x3e22a854d636a18eUL, + 0xb346e15274491c3bUL, 0x2ceafd4e5390cde7UL, + /* 192 */ 0xba8a8538be0d6675UL, 0x4b9074bb50818e23UL, + 0xcbdab89085d304c3UL, 0x61a24fe0e56192c4UL, + /* 193 */ 0xcb7615e6db525bcbUL, 0xdd7d8c35a567e4caUL, + 0xe6b4153acafcdd69UL, 0x2d668e097f3c9766UL, + /* 194 */ 0xa57e7e265ce55ef0UL, 0x5d9f4e527cd4b967UL, + 0xfbc83606492fd1e5UL, 0x090d52beb7c3f7aeUL, + /* 195 */ 0x09b9515a1e7b4d7cUL, 0x1f266a2599da44c0UL, + 0xa1c49548e2c55504UL, 0x7ef04287126f15ccUL, + /* 196 */ 0xfed1659dbd30ef15UL, 0x8b4ab9eec4e0277bUL, + 0x884d6236a5df3291UL, 0x1fd96ea6bf5cf788UL, + /* 197 */ 0x42a161981f190d9aUL, 0x61d849507e6052c1UL, + 0x9fe113bf285a2cd5UL, 0x7c22d676dbad85d8UL, + /* 198 */ 0x82e770ed2bfbd27dUL, 0x4c05b2ece996f5a5UL, + 0xcd40a9c2b0900150UL, 0x5895319213d9bf64UL, + /* 199 */ 0xe7cc5d703fea2e08UL, 0xb50c491258e2188cUL, + 0xcce30baa48205bf0UL, 0x537c659ccfa32d62UL, + /* 200 */ 0x37b6623a98cfc088UL, 0xfe9bed1fa4d6aca4UL, + 0x04d29b8e56a8d1b0UL, 0x725f71c40b519575UL, + /* 201 */ 0x28c7f89cd0339ce6UL, 0x8367b14469ddc18bUL, + 0x883ada83a6a1652cUL, 0x585f1974034d6c17UL, + /* 202 */ 0x89cfb266f1b19188UL, 0xe63b4863e7c35217UL, + 0xd88c9da6b4c0526aUL, 0x3e035c9df0954635UL, + /* 203 */ 0xdd9d5412fb45de9dUL, 0xdd684532e4cff40dUL, + 0x4b5c999b151d671cUL, 0x2d8c2cc811e7f690UL, + /* 204 */ 0x7f54be1d90055d40UL, 0xa464c5df464aaf40UL, + 0x33979624f0e917beUL, 0x2c018dc527356b30UL, + /* 205 */ 0xa5415024e330b3d4UL, 0x73ff3d96691652d3UL, + 0x94ec42c4ef9b59f1UL, 0x0747201618d08e5aUL, + /* 206 */ 0x4d6ca48aca411c53UL, 0x66415f2fcfa66119UL, + 0x9c4dd40051e227ffUL, 0x59810bc09a02f7ebUL, + /* 207 */ 0x2a7eb171b3dc101dUL, 0x441c5ab99ffef68eUL, + 0x32025c9b93b359eaUL, 0x5e8ce0a71e9d112fUL, + /* 208 */ 0xbfcccb92429503fdUL, 0xd271ba752f095d55UL, + 0x345ead5e972d091eUL, 0x18c8df11a83103baUL, + /* 209 */ 0x90cd949a9aed0f4cUL, 0xc5d1f4cb6660e37eUL, + 0xb8cac52d56c52e0bUL, 0x6e42e400c5808e0dUL, + /* 210 */ 0xa3b46966eeaefd23UL, 0x0c4f1f0be39ecdcaUL, + 0x189dc8c9d683a51dUL, 0x51f27f054c09351bUL, + /* 211 */ 0x4c487ccd2a320682UL, 0x587ea95bb3df1c96UL, + 0xc8ccf79e555cb8e8UL, 0x547dc829a206d73dUL, + /* 212 */ 0xb822a6cd80c39b06UL, 0xe96d54732000d4c6UL, + 0x28535b6f91463b4dUL, 0x228f4660e2486e1dUL, + /* 213 */ 0x98799538de8d3abfUL, 0x8cd8330045ebca6eUL, + 0x79952a008221e738UL, 0x4322e1a7535cd2bbUL, + /* 214 */ 0xb114c11819d1801cUL, 0x2016e4d84f3f5ec7UL, + 0xdd0e2df409260f4cUL, 0x5ec362c0ae5f7266UL, + /* 215 */ 0xc0462b18b8b2b4eeUL, 0x7cc8d950274d1afbUL, + 0xf25f7105436b02d2UL, 0x43bbf8dcbff9ccd3UL, + /* 216 */ 0xb6ad1767a039e9dfUL, 0xb0714da8f69d3583UL, + 0x5e55fa18b42931f5UL, 0x4ed5558f33c60961UL, + /* 217 */ 0x1fe37901c647a5ddUL, 0x593ddf1f8081d357UL, + 0x0249a4fd813fd7a6UL, 0x69acca274e9caf61UL, + /* 218 */ 0x047ba3ea330721c9UL, 0x83423fc20e7e1ea0UL, + 0x1df4c0af01314a60UL, 0x09a62dab89289527UL, + /* 219 */ 0xa5b325a49cc6cb00UL, 0xe94b5dc654b56cb6UL, + 0x3be28779adc994a0UL, 0x4296e8f8ba3a4aadUL, + /* 220 */ 0x328689761e451eabUL, 0x2e4d598bff59594aUL, + 0x49b96853d7a7084aUL, 0x4980a319601420a8UL, + /* 221 */ 0x9565b9e12f552c42UL, 0x8a5318db7100fe96UL, + 0x05c90b4d43add0d7UL, 0x538b4cd66a5d4edaUL, + /* 222 */ 0xf4e94fc3e89f039fUL, 0x592c9af26f618045UL, + 0x08a36eb5fd4b9550UL, 0x25fffaf6c2ed1419UL, + /* 223 */ 0x34434459cc79d354UL, 0xeeecbfb4b1d5476bUL, + 0xddeb34a061615d99UL, 0x5129cecceb64b773UL, + /* 224 */ 0xee43215894993520UL, 0x772f9c7cf14c0b3bUL, + 0xd2e2fce306bedad5UL, 0x715f42b546f06a97UL, + /* 225 */ 0x434ecdceda5b5f1aUL, 0x0da17115a49741a9UL, + 0x680bd77c73edad2eUL, 0x487c02354edd9041UL, + /* 226 */ 0xb8efeff3a70ed9c4UL, 0x56a32aa3e857e302UL, + 0xdf3a68bd48a2a5a0UL, 0x07f650b73176c444UL, + /* 227 */ 0xe38b9b1626e0ccb1UL, 0x79e053c18b09fb36UL, + 0x56d90319c9f94964UL, 0x1ca941e7ac9ff5c4UL, + /* 228 */ 0x49c4df29162fa0bbUL, 0x8488cf3282b33305UL, + 0x95dfda14cabb437dUL, 0x3391f78264d5ad86UL, + /* 229 */ 0x729ae06ae2b5095dUL, 0xd58a58d73259a946UL, + 0xe9834262d13921edUL, 0x27fedafaa54bb592UL, + /* 230 */ 0xa99dc5b829ad48bbUL, 0x5f025742499ee260UL, + 0x802c8ecd5d7513fdUL, 0x78ceb3ef3f6dd938UL, + /* 231 */ 0xc342f44f8a135d94UL, 0x7b9edb44828cdda3UL, + 0x9436d11a0537cfe7UL, 0x5064b164ec1ab4c8UL, + /* 232 */ 0x7020eccfd37eb2fcUL, 0x1f31ea3ed90d25fcUL, + 0x1b930d7bdfa1bb34UL, 0x5344467a48113044UL, + /* 233 */ 0x70073170f25e6dfbUL, 0xe385dc1a50114cc8UL, + 0x2348698ac8fc4f00UL, 0x2a77a55284dd40d8UL, + /* 234 */ 0xfe06afe0c98c6ce4UL, 0xc235df96dddfd6e4UL, + 0x1428d01e33bf1ed3UL, 0x785768ec9300bdafUL, + /* 235 */ 0x9702e57a91deb63bUL, 0x61bdb8bfe5ce8b80UL, + 0x645b426f3d1d58acUL, 0x4804a82227a557bcUL, + /* 236 */ 0x8e57048ab44d2601UL, 0x68d6501a4b3a6935UL, + 0xc39c9ec3f9e1c293UL, 0x4172f257d4de63e2UL, + /* 237 */ 0xd368b450330c6401UL, 0x040d3017418f2391UL, + 0x2c34bb6090b7d90dUL, 0x16f649228fdfd51fUL, + /* 238 */ 0xbea6818e2b928ef5UL, 0xe28ccf91cdc11e72UL, + 0x594aaa68e77a36cdUL, 0x313034806c7ffd0fUL, + /* 239 */ 0x8a9d27ac2249bd65UL, 0x19a3b464018e9512UL, + 0xc26ccff352b37ec7UL, 0x056f68341d797b21UL, + /* 240 */ 0x5e79d6757efd2327UL, 0xfabdbcb6553afe15UL, + 0xd3e7222c6eaf5a60UL, 0x7046c76d4dae743bUL, + /* 241 */ 0x660be872b18d4a55UL, 0x19992518574e1496UL, + 0xc103053a302bdcbbUL, 0x3ed8e9800b218e8eUL, + /* 242 */ 0x7b0b9239fa75e03eUL, 0xefe9fb684633c083UL, + 0x98a35fbe391a7793UL, 0x6065510fe2d0fe34UL, + /* 243 */ 0x55cb668548abad0cUL, 0xb4584548da87e527UL, + 0x2c43ecea0107c1ddUL, 0x526028809372de35UL, + /* 244 */ 0x3415c56af9213b1fUL, 0x5bee1a4d017e98dbUL, + 0x13f6b105b5cf709bUL, 0x5ff20e3482b29ab6UL, + /* 245 */ 0x0aa29c75cc2e6c90UL, 0xfc7d73ca3a70e206UL, + 0x899fc38fc4b5c515UL, 0x250386b124ffc207UL, + /* 246 */ 0x54ea28d5ae3d2b56UL, 0x9913149dd6de60ceUL, + 0x16694fc58f06d6c1UL, 0x46b23975eb018fc7UL, + /* 247 */ 0x470a6a0fb4b7b4e2UL, 0x5d92475a8f7253deUL, + 0xabeee5b52fbd3adbUL, 0x7fa20801a0806968UL, + /* 248 */ 0x76f3faf19f7714d2UL, 0xb3e840c12f4660c3UL, + 0x0fb4cd8df212744eUL, 0x4b065a251d3a2dd2UL, + /* 249 */ 0x5cebde383d77cd4aUL, 0x6adf39df882c9cb1UL, + 0xa2dd242eb09af759UL, 0x3147c0e50e5f6422UL, + /* 250 */ 0x164ca5101d1350dbUL, 0xf8d13479c33fc962UL, + 0xe640ce4d13e5da08UL, 0x4bdee0c45061f8baUL, + /* 251 */ 0xd7c46dc1a4edb1c9UL, 0x5514d7b6437fd98aUL, + 0x58942f6bb2a1c00bUL, 0x2dffb2ab1d70710eUL, + /* 252 */ 0xccdfcf2fc18b6d68UL, 0xa8ebcba8b7806167UL, + 0x980697f95e2937e3UL, 0x02fbba1cd0126e8cUL +}; + +/* c is two 512-bit products: c0[0:7]=a0[0:3]*b0[0:3] and c1[8:15]=a1[4:7]*b1[4:7] + * a is two 256-bit integers: a0[0:3] and a1[4:7] + * b is two 256-bit integers: b0[0:3] and b1[4:7] + */ +static void mul2_256x256_integer_adx(u64 *const c, const u64 *const a, + const u64 *const b) { - u64 x = a ^ b; - u64 minus_x = ~x + (u64)1U; - u64 x_or_minus_x = x | minus_x; - u64 xnx = x_or_minus_x >> (u32)63U; - return xnx - (u64)1U; + asm volatile( + "xorl %%r14d, %%r14d ;" + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ + "xorl %%r10d, %%r10d ;" + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ + "adox %%r10, %%r15 ;" + "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ + "adox %%r8, %%rax ;" + "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ + "adox %%r10, %%rbx ;" + /******************************************/ + "adox %%r14, %%rcx ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "adox %%r15, %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rax ;" + "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rbx ;" + "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rcx ;" + /******************************************/ + "adox %%r14, %%r15 ;" + "adcx %%r14, %%r15 ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "xorl %%r10d, %%r10d ;" + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "adox %%rax, %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rbx ;" + "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rcx ;" + "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%r15 ;" + /******************************************/ + "adox %%r14, %%rax ;" + "adcx %%r14, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "xorl %%r10d, %%r10d ;" + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "adox %%rbx, %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rcx ;" + "movq %%rcx, 32(%0) ;" + "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%r15 ;" + "movq %%r15, 40(%0) ;" + "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rax ;" + "movq %%rax, 48(%0) ;" + /******************************************/ + "adox %%r14, %%rbx ;" + "adcx %%r14, %%rbx ;" + "movq %%rbx, 56(%0) ;" + + "movq 32(%1), %%rdx; " /* C[0] */ + "mulx 32(%2), %%r8, %%r15; " /* C[0]*D[0] */ + "xorl %%r10d, %%r10d ;" + "movq %%r8, 64(%0);" + "mulx 40(%2), %%r10, %%rax; " /* C[0]*D[1] */ + "adox %%r10, %%r15 ;" + "mulx 48(%2), %%r8, %%rbx; " /* C[0]*D[2] */ + "adox %%r8, %%rax ;" + "mulx 56(%2), %%r10, %%rcx; " /* C[0]*D[3] */ + "adox %%r10, %%rbx ;" + /******************************************/ + "adox %%r14, %%rcx ;" + + "movq 40(%1), %%rdx; " /* C[1] */ + "xorl %%r10d, %%r10d ;" + "mulx 32(%2), %%r8, %%r9; " /* C[1]*D[0] */ + "adox %%r15, %%r8 ;" + "movq %%r8, 72(%0);" + "mulx 40(%2), %%r10, %%r11; " /* C[1]*D[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rax ;" + "mulx 48(%2), %%r8, %%r13; " /* C[1]*D[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rbx ;" + "mulx 56(%2), %%r10, %%r15; " /* C[1]*D[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rcx ;" + /******************************************/ + "adox %%r14, %%r15 ;" + "adcx %%r14, %%r15 ;" + + "movq 48(%1), %%rdx; " /* C[2] */ + "xorl %%r10d, %%r10d ;" + "mulx 32(%2), %%r8, %%r9; " /* C[2]*D[0] */ + "adox %%rax, %%r8 ;" + "movq %%r8, 80(%0);" + "mulx 40(%2), %%r10, %%r11; " /* C[2]*D[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rbx ;" + "mulx 48(%2), %%r8, %%r13; " /* C[2]*D[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%rcx ;" + "mulx 56(%2), %%r10, %%rax; " /* C[2]*D[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%r15 ;" + /******************************************/ + "adox %%r14, %%rax ;" + "adcx %%r14, %%rax ;" + + "movq 56(%1), %%rdx; " /* C[3] */ + "xorl %%r10d, %%r10d ;" + "mulx 32(%2), %%r8, %%r9; " /* C[3]*D[0] */ + "adox %%rbx, %%r8 ;" + "movq %%r8, 88(%0);" + "mulx 40(%2), %%r10, %%r11; " /* C[3]*D[1] */ + "adox %%r10, %%r9 ;" + "adcx %%r9, %%rcx ;" + "movq %%rcx, 96(%0) ;" + "mulx 48(%2), %%r8, %%r13; " /* C[3]*D[2] */ + "adox %%r8, %%r11 ;" + "adcx %%r11, %%r15 ;" + "movq %%r15, 104(%0) ;" + "mulx 56(%2), %%r10, %%rbx; " /* C[3]*D[3] */ + "adox %%r10, %%r13 ;" + "adcx %%r13, %%rax ;" + "movq %%rax, 112(%0) ;" + /******************************************/ + "adox %%r14, %%rbx ;" + "adcx %%r14, %%rbx ;" + "movq %%rbx, 120(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r14", "%r15"); } -static __always_inline u64 gte_mask(u64 a, u64 b) +static void mul2_256x256_integer_bmi2(u64 *const c, const u64 *const a, + const u64 *const b) { - u64 x = a; - u64 y = b; - u64 x_xor_y = x ^ y; - u64 x_sub_y = x - y; - u64 x_sub_y_xor_y = x_sub_y ^ y; - u64 q = x_xor_y | x_sub_y_xor_y; - u64 x_xor_q = x ^ q; - u64 x_xor_q_ = x_xor_q >> (u32)63U; - return x_xor_q_ - (u64)1U; + asm volatile( + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ + "addq %%r10, %%r15 ;" + "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ + "adcq %%r8, %%rax ;" + "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ + "adcq %%r10, %%rbx ;" + /******************************************/ + "adcq $0, %%rcx ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "addq %%r15, %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%r15 ;" + + "addq %%r9, %%rax ;" + "adcq %%r11, %%rbx ;" + "adcq %%r13, %%rcx ;" + "adcq $0, %%r15 ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "addq %%rax, %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rax ;" + + "addq %%r9, %%rbx ;" + "adcq %%r11, %%rcx ;" + "adcq %%r13, %%r15 ;" + "adcq $0, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "addq %%rbx, %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rbx ;" + + "addq %%r9, %%rcx ;" + "movq %%rcx, 32(%0) ;" + "adcq %%r11, %%r15 ;" + "movq %%r15, 40(%0) ;" + "adcq %%r13, %%rax ;" + "movq %%rax, 48(%0) ;" + "adcq $0, %%rbx ;" + "movq %%rbx, 56(%0) ;" + + "movq 32(%1), %%rdx; " /* C[0] */ + "mulx 32(%2), %%r8, %%r15; " /* C[0]*D[0] */ + "movq %%r8, 64(%0) ;" + "mulx 40(%2), %%r10, %%rax; " /* C[0]*D[1] */ + "addq %%r10, %%r15 ;" + "mulx 48(%2), %%r8, %%rbx; " /* C[0]*D[2] */ + "adcq %%r8, %%rax ;" + "mulx 56(%2), %%r10, %%rcx; " /* C[0]*D[3] */ + "adcq %%r10, %%rbx ;" + /******************************************/ + "adcq $0, %%rcx ;" + + "movq 40(%1), %%rdx; " /* C[1] */ + "mulx 32(%2), %%r8, %%r9; " /* C[1]*D[0] */ + "addq %%r15, %%r8 ;" + "movq %%r8, 72(%0) ;" + "mulx 40(%2), %%r10, %%r11; " /* C[1]*D[1] */ + "adcq %%r10, %%r9 ;" + "mulx 48(%2), %%r8, %%r13; " /* C[1]*D[2] */ + "adcq %%r8, %%r11 ;" + "mulx 56(%2), %%r10, %%r15; " /* C[1]*D[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%r15 ;" + + "addq %%r9, %%rax ;" + "adcq %%r11, %%rbx ;" + "adcq %%r13, %%rcx ;" + "adcq $0, %%r15 ;" + + "movq 48(%1), %%rdx; " /* C[2] */ + "mulx 32(%2), %%r8, %%r9; " /* C[2]*D[0] */ + "addq %%rax, %%r8 ;" + "movq %%r8, 80(%0) ;" + "mulx 40(%2), %%r10, %%r11; " /* C[2]*D[1] */ + "adcq %%r10, %%r9 ;" + "mulx 48(%2), %%r8, %%r13; " /* C[2]*D[2] */ + "adcq %%r8, %%r11 ;" + "mulx 56(%2), %%r10, %%rax; " /* C[2]*D[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rax ;" + + "addq %%r9, %%rbx ;" + "adcq %%r11, %%rcx ;" + "adcq %%r13, %%r15 ;" + "adcq $0, %%rax ;" + + "movq 56(%1), %%rdx; " /* C[3] */ + "mulx 32(%2), %%r8, %%r9; " /* C[3]*D[0] */ + "addq %%rbx, %%r8 ;" + "movq %%r8, 88(%0) ;" + "mulx 40(%2), %%r10, %%r11; " /* C[3]*D[1] */ + "adcq %%r10, %%r9 ;" + "mulx 48(%2), %%r8, %%r13; " /* C[3]*D[2] */ + "adcq %%r8, %%r11 ;" + "mulx 56(%2), %%r10, %%rbx; " /* C[3]*D[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rbx ;" + + "addq %%r9, %%rcx ;" + "movq %%rcx, 96(%0) ;" + "adcq %%r11, %%r15 ;" + "movq %%r15, 104(%0) ;" + "adcq %%r13, %%rax ;" + "movq %%rax, 112(%0) ;" + "adcq $0, %%rbx ;" + "movq %%rbx, 120(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r15"); } -/* Computes the addition of four-element f1 with value in f2 - * and returns the carry (if any) */ -static inline u64 add_scalar(u64 *out, const u64 *f1, u64 f2) +static void sqr2_256x256_integer_adx(u64 *const c, const u64 *const a) { - u64 carry_r; + asm volatile( + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 8(%1), %%r8, %%r14 ;" /* A[1]*A[0] */ + "xorl %%r15d, %%r15d;" + "mulx 16(%1), %%r9, %%r10 ;" /* A[2]*A[0] */ + "adcx %%r14, %%r9 ;" + "mulx 24(%1), %%rax, %%rcx ;" /* A[3]*A[0] */ + "adcx %%rax, %%r10 ;" + "movq 24(%1), %%rdx ;" /* A[3] */ + "mulx 8(%1), %%r11, %%rbx ;" /* A[1]*A[3] */ + "adcx %%rcx, %%r11 ;" + "mulx 16(%1), %%rax, %%r13 ;" /* A[2]*A[3] */ + "adcx %%rax, %%rbx ;" + "movq 8(%1), %%rdx ;" /* A[1] */ + "adcx %%r15, %%r13 ;" + "mulx 16(%1), %%rax, %%rcx ;" /* A[2]*A[1] */ + "movq $0, %%r14 ;" + /******************************************/ + "adcx %%r15, %%r14 ;" + + "xorl %%r15d, %%r15d;" + "adox %%rax, %%r10 ;" + "adcx %%r8, %%r8 ;" + "adox %%rcx, %%r11 ;" + "adcx %%r9, %%r9 ;" + "adox %%r15, %%rbx ;" + "adcx %%r10, %%r10 ;" + "adox %%r15, %%r13 ;" + "adcx %%r11, %%r11 ;" + "adox %%r15, %%r14 ;" + "adcx %%rbx, %%rbx ;" + "adcx %%r13, %%r13 ;" + "adcx %%r14, %%r14 ;" + + "movq (%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%rbx ;" + "movq %%rbx, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + + + "movq 32(%1), %%rdx ;" /* B[0] */ + "mulx 40(%1), %%r8, %%r14 ;" /* B[1]*B[0] */ + "xorl %%r15d, %%r15d;" + "mulx 48(%1), %%r9, %%r10 ;" /* B[2]*B[0] */ + "adcx %%r14, %%r9 ;" + "mulx 56(%1), %%rax, %%rcx ;" /* B[3]*B[0] */ + "adcx %%rax, %%r10 ;" + "movq 56(%1), %%rdx ;" /* B[3] */ + "mulx 40(%1), %%r11, %%rbx ;" /* B[1]*B[3] */ + "adcx %%rcx, %%r11 ;" + "mulx 48(%1), %%rax, %%r13 ;" /* B[2]*B[3] */ + "adcx %%rax, %%rbx ;" + "movq 40(%1), %%rdx ;" /* B[1] */ + "adcx %%r15, %%r13 ;" + "mulx 48(%1), %%rax, %%rcx ;" /* B[2]*B[1] */ + "movq $0, %%r14 ;" + /******************************************/ + "adcx %%r15, %%r14 ;" + + "xorl %%r15d, %%r15d;" + "adox %%rax, %%r10 ;" + "adcx %%r8, %%r8 ;" + "adox %%rcx, %%r11 ;" + "adcx %%r9, %%r9 ;" + "adox %%r15, %%rbx ;" + "adcx %%r10, %%r10 ;" + "adox %%r15, %%r13 ;" + "adcx %%r11, %%r11 ;" + "adox %%r15, %%r14 ;" + "adcx %%rbx, %%rbx ;" + "adcx %%r13, %%r13 ;" + "adcx %%r14, %%r14 ;" + + "movq 32(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[0]^2 */ + /*******************/ + "movq %%rax, 64(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 72(%0) ;" + "movq 40(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 80(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 88(%0) ;" + "movq 48(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 96(%0) ;" + "adcq %%rcx, %%rbx ;" + "movq %%rbx, 104(%0) ;" + "movq 56(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* B[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 112(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 120(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r14", "%r15"); +} +static void sqr2_256x256_integer_bmi2(u64 *const c, const u64 *const a) +{ asm volatile( - /* Clear registers to propagate the carry bit */ - " xor %%r8, %%r8;" - " xor %%r9, %%r9;" - " xor %%r10, %%r10;" - " xor %%r11, %%r11;" - " xor %1, %1;" - - /* Begin addition chain */ - " addq 0(%3), %0;" - " movq %0, 0(%2);" - " adcxq 8(%3), %%r8;" - " movq %%r8, 8(%2);" - " adcxq 16(%3), %%r9;" - " movq %%r9, 16(%2);" - " adcxq 24(%3), %%r10;" - " movq %%r10, 24(%2);" - - /* Return the carry bit in a register */ - " adcx %%r11, %1;" - : "+&r" (f2), "=&r" (carry_r) - : "r" (out), "r" (f1) - : "%r8", "%r9", "%r10", "%r11", "memory", "cc" - ); + "movq 8(%1), %%rdx ;" /* A[1] */ + "mulx (%1), %%r8, %%r9 ;" /* A[0]*A[1] */ + "mulx 16(%1), %%r10, %%r11 ;" /* A[2]*A[1] */ + "mulx 24(%1), %%rcx, %%r14 ;" /* A[3]*A[1] */ + + "movq 16(%1), %%rdx ;" /* A[2] */ + "mulx 24(%1), %%r15, %%r13 ;" /* A[3]*A[2] */ + "mulx (%1), %%rax, %%rdx ;" /* A[0]*A[2] */ + + "addq %%rax, %%r9 ;" + "adcq %%rdx, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq %%r14, %%r15 ;" + "adcq $0, %%r13 ;" + "movq $0, %%r14 ;" + "adcq $0, %%r14 ;" + + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 24(%1), %%rax, %%rcx ;" /* A[0]*A[3] */ + + "addq %%rax, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq $0, %%r15 ;" + "adcq $0, %%r13 ;" + "adcq $0, %%r14 ;" + + "shldq $1, %%r13, %%r14 ;" + "shldq $1, %%r15, %%r13 ;" + "shldq $1, %%r11, %%r15 ;" + "shldq $1, %%r10, %%r11 ;" + "shldq $1, %%r9, %%r10 ;" + "shldq $1, %%r8, %%r9 ;" + "shlq $1, %%r8 ;" + + /*******************/ + "mulx %%rdx, %%rax, %%rcx ; " /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%r15 ;" + "movq %%r15, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + + "movq 40(%1), %%rdx ;" /* B[1] */ + "mulx 32(%1), %%r8, %%r9 ;" /* B[0]*B[1] */ + "mulx 48(%1), %%r10, %%r11 ;" /* B[2]*B[1] */ + "mulx 56(%1), %%rcx, %%r14 ;" /* B[3]*B[1] */ + + "movq 48(%1), %%rdx ;" /* B[2] */ + "mulx 56(%1), %%r15, %%r13 ;" /* B[3]*B[2] */ + "mulx 32(%1), %%rax, %%rdx ;" /* B[0]*B[2] */ + + "addq %%rax, %%r9 ;" + "adcq %%rdx, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq %%r14, %%r15 ;" + "adcq $0, %%r13 ;" + "movq $0, %%r14 ;" + "adcq $0, %%r14 ;" + + "movq 32(%1), %%rdx ;" /* B[0] */ + "mulx 56(%1), %%rax, %%rcx ;" /* B[0]*B[3] */ + + "addq %%rax, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq $0, %%r15 ;" + "adcq $0, %%r13 ;" + "adcq $0, %%r14 ;" + + "shldq $1, %%r13, %%r14 ;" + "shldq $1, %%r15, %%r13 ;" + "shldq $1, %%r11, %%r15 ;" + "shldq $1, %%r10, %%r11 ;" + "shldq $1, %%r9, %%r10 ;" + "shldq $1, %%r8, %%r9 ;" + "shlq $1, %%r8 ;" + + /*******************/ + "mulx %%rdx, %%rax, %%rcx ; " /* B[0]^2 */ + /*******************/ + "movq %%rax, 64(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 72(%0) ;" + "movq 40(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* B[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 80(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 88(%0) ;" + "movq 48(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* B[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 96(%0) ;" + "adcq %%rcx, %%r15 ;" + "movq %%r15, 104(%0) ;" + "movq 56(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ; " /* B[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 112(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 120(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11", "%r13", "%r14", "%r15"); +} - return carry_r; +static void red_eltfp25519_2w_adx(u64 *const c, const u64 *const a) +{ + asm volatile( + "movl $38, %%edx; " /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10; " /* c*C[4] */ + "xorl %%ebx, %%ebx ;" + "adox (%1), %%r8 ;" + "mulx 40(%1), %%r9, %%r11; " /* c*C[5] */ + "adcx %%r10, %%r9 ;" + "adox 8(%1), %%r9 ;" + "mulx 48(%1), %%r10, %%rax; " /* c*C[6] */ + "adcx %%r11, %%r10 ;" + "adox 16(%1), %%r10 ;" + "mulx 56(%1), %%r11, %%rcx; " /* c*C[7] */ + "adcx %%rax, %%r11 ;" + "adox 24(%1), %%r11 ;" + /***************************************/ + "adcx %%rbx, %%rcx ;" + "adox %%rbx, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ + "adcx %%rcx, %%r8 ;" + "adcx %%rbx, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcx %%rbx, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcx %%rbx, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + + "mulx 96(%1), %%r8, %%r10; " /* c*C[4] */ + "xorl %%ebx, %%ebx ;" + "adox 64(%1), %%r8 ;" + "mulx 104(%1), %%r9, %%r11; " /* c*C[5] */ + "adcx %%r10, %%r9 ;" + "adox 72(%1), %%r9 ;" + "mulx 112(%1), %%r10, %%rax; " /* c*C[6] */ + "adcx %%r11, %%r10 ;" + "adox 80(%1), %%r10 ;" + "mulx 120(%1), %%r11, %%rcx; " /* c*C[7] */ + "adcx %%rax, %%r11 ;" + "adox 88(%1), %%r11 ;" + /****************************************/ + "adcx %%rbx, %%rcx ;" + "adox %%rbx, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ + "adcx %%rcx, %%r8 ;" + "adcx %%rbx, %%r9 ;" + "movq %%r9, 40(%0) ;" + "adcx %%rbx, %%r10 ;" + "movq %%r10, 48(%0) ;" + "adcx %%rbx, %%r11 ;" + "movq %%r11, 56(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 32(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11"); } -/* Computes the field addition of two field elements */ -static inline void fadd(u64 *out, const u64 *f1, const u64 *f2) +static void red_eltfp25519_2w_bmi2(u64 *const c, const u64 *const a) { asm volatile( - /* Compute the raw addition of f1 + f2 */ - " movq 0(%0), %%r8;" - " addq 0(%2), %%r8;" - " movq 8(%0), %%r9;" - " adcxq 8(%2), %%r9;" - " movq 16(%0), %%r10;" - " adcxq 16(%2), %%r10;" - " movq 24(%0), %%r11;" - " adcxq 24(%2), %%r11;" - - /* Wrap the result back into the field */ - - /* Step 1: Compute carry*38 */ - " mov $0, %%rax;" - " mov $38, %0;" - " cmovc %0, %%rax;" - - /* Step 2: Add carry*38 to the original sum */ - " xor %%rcx, %%rcx;" - " add %%rax, %%r8;" - " adcx %%rcx, %%r9;" - " movq %%r9, 8(%1);" - " adcx %%rcx, %%r10;" - " movq %%r10, 16(%1);" - " adcx %%rcx, %%r11;" - " movq %%r11, 24(%1);" - - /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ - " mov $0, %%rax;" - " cmovc %0, %%rax;" - " add %%rax, %%r8;" - " movq %%r8, 0(%1);" - : "+&r" (f2) - : "r" (out), "r" (f1) - : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc" - ); + "movl $38, %%edx ; " /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ + "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ + "addq %%r10, %%r9 ;" + "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcq %%r11, %%r10 ;" + "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcq %%rax, %%r11 ;" + /***************************************/ + "adcq $0, %%rcx ;" + "addq (%1), %%r8 ;" + "adcq 8(%1), %%r9 ;" + "adcq 16(%1), %%r10 ;" + "adcq 24(%1), %%r11 ;" + "adcq $0, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + + "mulx 96(%1), %%r8, %%r10 ;" /* c*C[4] */ + "mulx 104(%1), %%r9, %%r11 ;" /* c*C[5] */ + "addq %%r10, %%r9 ;" + "mulx 112(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcq %%r11, %%r10 ;" + "mulx 120(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcq %%rax, %%r11 ;" + /****************************************/ + "adcq $0, %%rcx ;" + "addq 64(%1), %%r8 ;" + "adcq 72(%1), %%r9 ;" + "adcq 80(%1), %%r10 ;" + "adcq 88(%1), %%r11 ;" + "adcq $0, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 40(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 48(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 56(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 32(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11"); } -/* Computes the field substraction of two field elements */ -static inline void fsub(u64 *out, const u64 *f1, const u64 *f2) +static void mul_256x256_integer_adx(u64 *const c, const u64 *const a, + const u64 *const b) { asm volatile( - /* Compute the raw substraction of f1-f2 */ - " movq 0(%1), %%r8;" - " subq 0(%2), %%r8;" - " movq 8(%1), %%r9;" - " sbbq 8(%2), %%r9;" - " movq 16(%1), %%r10;" - " sbbq 16(%2), %%r10;" - " movq 24(%1), %%r11;" - " sbbq 24(%2), %%r11;" - - /* Wrap the result back into the field */ - - /* Step 1: Compute carry*38 */ - " mov $0, %%rax;" - " mov $38, %%rcx;" - " cmovc %%rcx, %%rax;" - - /* Step 2: Substract carry*38 from the original difference */ - " sub %%rax, %%r8;" - " sbb $0, %%r9;" - " sbb $0, %%r10;" - " sbb $0, %%r11;" - - /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ - " mov $0, %%rax;" - " cmovc %%rcx, %%rax;" - " sub %%rax, %%r8;" - - /* Store the result */ - " movq %%r8, 0(%0);" - " movq %%r9, 8(%0);" - " movq %%r10, 16(%0);" - " movq %%r11, 24(%0);" - : - : "r" (out), "r" (f1), "r" (f2) - : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc" - ); + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r9; " /* A[0]*B[0] */ + "xorl %%r10d, %%r10d ;" + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[0]*B[1] */ + "adox %%r9, %%r10 ;" + "movq %%r10, 8(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[0]*B[2] */ + "adox %%r11, %%r15 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[0]*B[3] */ + "adox %%r13, %%r14 ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "xorl %%r10d, %%r10d ;" + "adcx 8(%0), %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adox %%r9, %%r10 ;" + "adcx %%r15, %%r10 ;" + "movq %%r10, 16(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[1]*B[2] */ + "adox %%r11, %%r15 ;" + "adcx %%r14, %%r15 ;" + "movq $0, %%r8 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[1]*B[3] */ + "adox %%r13, %%r14 ;" + "adcx %%rax, %%r14 ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + "adcx %%r8, %%rax ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "xorl %%r10d, %%r10d ;" + "adcx 16(%0), %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adox %%r9, %%r10 ;" + "adcx %%r15, %%r10 ;" + "movq %%r10, 24(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[2]*B[2] */ + "adox %%r11, %%r15 ;" + "adcx %%r14, %%r15 ;" + "movq $0, %%r8 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[2]*B[3] */ + "adox %%r13, %%r14 ;" + "adcx %%rax, %%r14 ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + "adcx %%r8, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "xorl %%r10d, %%r10d ;" + "adcx 24(%0), %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adox %%r9, %%r10 ;" + "adcx %%r15, %%r10 ;" + "movq %%r10, 32(%0) ;" + "mulx 16(%2), %%r15, %%r13; " /* A[3]*B[2] */ + "adox %%r11, %%r15 ;" + "adcx %%r14, %%r15 ;" + "movq %%r15, 40(%0) ;" + "movq $0, %%r8 ;" + "mulx 24(%2), %%r14, %%rdx; " /* A[3]*B[3] */ + "adox %%r13, %%r14 ;" + "adcx %%rax, %%r14 ;" + "movq %%r14, 48(%0) ;" + "movq $0, %%rax ;" + /******************************************/ + "adox %%rdx, %%rax ;" + "adcx %%r8, %%rax ;" + "movq %%rax, 56(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", + "%r13", "%r14", "%r15"); } -/* Computes a field multiplication: out <- f1 * f2 - * Uses the 8-element buffer tmp for intermediate results */ -static inline void fmul(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp) +static void mul_256x256_integer_bmi2(u64 *const c, const u64 *const a, + const u64 *const b) { asm volatile( - /* Compute the raw multiplication: tmp <- src1 * src2 */ - - /* Compute src1[0] * src2 */ - " movq 0(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " movq %%r8, 0(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" - /* Compute src1[1] * src2 */ - " movq 8(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" - /* Compute src1[2] * src2 */ - " movq 16(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" - /* Compute src1[3] * src2 */ - " movq 24(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 56(%0);" - /* Line up pointers */ - " mov %0, %1;" - " mov %2, %0;" - - /* Wrap the result back into the field */ - - /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ - " mov $38, %%rdx;" - " mulxq 32(%1), %%r8, %%r13;" - " xor %3, %3;" - " adoxq 0(%1), %%r8;" - " mulxq 40(%1), %%r9, %%rbx;" - " adcx %%r13, %%r9;" - " adoxq 8(%1), %%r9;" - " mulxq 48(%1), %%r10, %%r13;" - " adcx %%rbx, %%r10;" - " adoxq 16(%1), %%r10;" - " mulxq 56(%1), %%r11, %%rax;" - " adcx %%r13, %%r11;" - " adoxq 24(%1), %%r11;" - " adcx %3, %%rax;" - " adox %3, %%rax;" - " imul %%rdx, %%rax;" - - /* Step 2: Fold the carry back into dst */ - " add %%rax, %%r8;" - " adcx %3, %%r9;" - " movq %%r9, 8(%0);" - " adcx %3, %%r10;" - " movq %%r10, 16(%0);" - " adcx %3, %%r11;" - " movq %%r11, 24(%0);" - - /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ - " mov $0, %%rax;" - " cmovc %%rdx, %%rax;" - " add %%rax, %%r8;" - " movq %%r8, 0(%0);" - : "+&r" (tmp), "+&r" (f1), "+&r" (out), "+&r" (f2) - : - : "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "memory", "cc" - ); + "movq (%1), %%rdx; " /* A[0] */ + "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ + "movq %%r8, (%0) ;" + "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ + "addq %%r10, %%r15 ;" + "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ + "adcq %%r8, %%rax ;" + "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ + "adcq %%r10, %%rbx ;" + /******************************************/ + "adcq $0, %%rcx ;" + + "movq 8(%1), %%rdx; " /* A[1] */ + "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ + "addq %%r15, %%r8 ;" + "movq %%r8, 8(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%r15 ;" + + "addq %%r9, %%rax ;" + "adcq %%r11, %%rbx ;" + "adcq %%r13, %%rcx ;" + "adcq $0, %%r15 ;" + + "movq 16(%1), %%rdx; " /* A[2] */ + "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ + "addq %%rax, %%r8 ;" + "movq %%r8, 16(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rax ;" + + "addq %%r9, %%rbx ;" + "adcq %%r11, %%rcx ;" + "adcq %%r13, %%r15 ;" + "adcq $0, %%rax ;" + + "movq 24(%1), %%rdx; " /* A[3] */ + "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ + "addq %%rbx, %%r8 ;" + "movq %%r8, 24(%0) ;" + "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ + "adcq %%r10, %%r9 ;" + "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ + "adcq %%r8, %%r11 ;" + "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ + "adcq %%r10, %%r13 ;" + /******************************************/ + "adcq $0, %%rbx ;" + + "addq %%r9, %%rcx ;" + "movq %%rcx, 32(%0) ;" + "adcq %%r11, %%r15 ;" + "movq %%r15, 40(%0) ;" + "adcq %%r13, %%rax ;" + "movq %%rax, 48(%0) ;" + "adcq $0, %%rbx ;" + "movq %%rbx, 56(%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r15"); } -/* Computes two field multiplications: - * out[0] <- f1[0] * f2[0] - * out[1] <- f1[1] * f2[1] - * Uses the 16-element buffer tmp for intermediate results. */ -static inline void fmul2(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp) +static void sqr_256x256_integer_adx(u64 *const c, const u64 *const a) { asm volatile( - /* Compute the raw multiplication tmp[0] <- f1[0] * f2[0] */ - - /* Compute src1[0] * src2 */ - " movq 0(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " movq %%r8, 0(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" - /* Compute src1[1] * src2 */ - " movq 8(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" - /* Compute src1[2] * src2 */ - " movq 16(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" - /* Compute src1[3] * src2 */ - " movq 24(%1), %%rdx;" - " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);" - " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);" - " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;" - " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 56(%0);" - - /* Compute the raw multiplication tmp[1] <- f1[1] * f2[1] */ - - /* Compute src1[0] * src2 */ - " movq 32(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " movq %%r8, 64(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 72(%0);" - " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" - " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" - /* Compute src1[1] * src2 */ - " movq 40(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 72(%0), %%r8;" " movq %%r8, 72(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 80(%0);" - " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" - /* Compute src1[2] * src2 */ - " movq 48(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 80(%0), %%r8;" " movq %%r8, 80(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 88(%0);" - " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" - " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" - /* Compute src1[3] * src2 */ - " movq 56(%1), %%rdx;" - " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 88(%0), %%r8;" " movq %%r8, 88(%0);" - " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 96(%0);" - " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 104(%0);" " mov $0, %%r8;" - " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 112(%0);" " mov $0, %%rax;" - " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 120(%0);" - /* Line up pointers */ - " mov %0, %1;" - " mov %2, %0;" - - /* Wrap the results back into the field */ - - /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ - " mov $38, %%rdx;" - " mulxq 32(%1), %%r8, %%r13;" - " xor %3, %3;" - " adoxq 0(%1), %%r8;" - " mulxq 40(%1), %%r9, %%rbx;" - " adcx %%r13, %%r9;" - " adoxq 8(%1), %%r9;" - " mulxq 48(%1), %%r10, %%r13;" - " adcx %%rbx, %%r10;" - " adoxq 16(%1), %%r10;" - " mulxq 56(%1), %%r11, %%rax;" - " adcx %%r13, %%r11;" - " adoxq 24(%1), %%r11;" - " adcx %3, %%rax;" - " adox %3, %%rax;" - " imul %%rdx, %%rax;" - - /* Step 2: Fold the carry back into dst */ - " add %%rax, %%r8;" - " adcx %3, %%r9;" - " movq %%r9, 8(%0);" - " adcx %3, %%r10;" - " movq %%r10, 16(%0);" - " adcx %3, %%r11;" - " movq %%r11, 24(%0);" - - /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ - " mov $0, %%rax;" - " cmovc %%rdx, %%rax;" - " add %%rax, %%r8;" - " movq %%r8, 0(%0);" - - /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ - " mov $38, %%rdx;" - " mulxq 96(%1), %%r8, %%r13;" - " xor %3, %3;" - " adoxq 64(%1), %%r8;" - " mulxq 104(%1), %%r9, %%rbx;" - " adcx %%r13, %%r9;" - " adoxq 72(%1), %%r9;" - " mulxq 112(%1), %%r10, %%r13;" - " adcx %%rbx, %%r10;" - " adoxq 80(%1), %%r10;" - " mulxq 120(%1), %%r11, %%rax;" - " adcx %%r13, %%r11;" - " adoxq 88(%1), %%r11;" - " adcx %3, %%rax;" - " adox %3, %%rax;" - " imul %%rdx, %%rax;" - - /* Step 2: Fold the carry back into dst */ - " add %%rax, %%r8;" - " adcx %3, %%r9;" - " movq %%r9, 40(%0);" - " adcx %3, %%r10;" - " movq %%r10, 48(%0);" - " adcx %3, %%r11;" - " movq %%r11, 56(%0);" - - /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ - " mov $0, %%rax;" - " cmovc %%rdx, %%rax;" - " add %%rax, %%r8;" - " movq %%r8, 32(%0);" - : "+&r" (tmp), "+&r" (f1), "+&r" (out), "+&r" (f2) - : - : "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "memory", "cc" - ); + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 8(%1), %%r8, %%r14 ;" /* A[1]*A[0] */ + "xorl %%r15d, %%r15d;" + "mulx 16(%1), %%r9, %%r10 ;" /* A[2]*A[0] */ + "adcx %%r14, %%r9 ;" + "mulx 24(%1), %%rax, %%rcx ;" /* A[3]*A[0] */ + "adcx %%rax, %%r10 ;" + "movq 24(%1), %%rdx ;" /* A[3] */ + "mulx 8(%1), %%r11, %%rbx ;" /* A[1]*A[3] */ + "adcx %%rcx, %%r11 ;" + "mulx 16(%1), %%rax, %%r13 ;" /* A[2]*A[3] */ + "adcx %%rax, %%rbx ;" + "movq 8(%1), %%rdx ;" /* A[1] */ + "adcx %%r15, %%r13 ;" + "mulx 16(%1), %%rax, %%rcx ;" /* A[2]*A[1] */ + "movq $0, %%r14 ;" + /******************************************/ + "adcx %%r15, %%r14 ;" + + "xorl %%r15d, %%r15d;" + "adox %%rax, %%r10 ;" + "adcx %%r8, %%r8 ;" + "adox %%rcx, %%r11 ;" + "adcx %%r9, %%r9 ;" + "adox %%r15, %%rbx ;" + "adcx %%r10, %%r10 ;" + "adox %%r15, %%r13 ;" + "adcx %%r11, %%r11 ;" + "adox %%r15, %%r14 ;" + "adcx %%rbx, %%rbx ;" + "adcx %%r13, %%r13 ;" + "adcx %%r14, %%r14 ;" + + "movq (%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%rbx ;" + "movq %%rbx, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11", "%r13", "%r14", "%r15"); } -/* Computes the field multiplication of four-element f1 with value in f2 */ -static inline void fmul_scalar(u64 *out, const u64 *f1, u64 f2) +static void sqr_256x256_integer_bmi2(u64 *const c, const u64 *const a) { - register u64 f2_r asm("rdx") = f2; + asm volatile( + "movq 8(%1), %%rdx ;" /* A[1] */ + "mulx (%1), %%r8, %%r9 ;" /* A[0]*A[1] */ + "mulx 16(%1), %%r10, %%r11 ;" /* A[2]*A[1] */ + "mulx 24(%1), %%rcx, %%r14 ;" /* A[3]*A[1] */ + + "movq 16(%1), %%rdx ;" /* A[2] */ + "mulx 24(%1), %%r15, %%r13 ;" /* A[3]*A[2] */ + "mulx (%1), %%rax, %%rdx ;" /* A[0]*A[2] */ + + "addq %%rax, %%r9 ;" + "adcq %%rdx, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq %%r14, %%r15 ;" + "adcq $0, %%r13 ;" + "movq $0, %%r14 ;" + "adcq $0, %%r14 ;" + + "movq (%1), %%rdx ;" /* A[0] */ + "mulx 24(%1), %%rax, %%rcx ;" /* A[0]*A[3] */ + + "addq %%rax, %%r10 ;" + "adcq %%rcx, %%r11 ;" + "adcq $0, %%r15 ;" + "adcq $0, %%r13 ;" + "adcq $0, %%r14 ;" + + "shldq $1, %%r13, %%r14 ;" + "shldq $1, %%r15, %%r13 ;" + "shldq $1, %%r11, %%r15 ;" + "shldq $1, %%r10, %%r11 ;" + "shldq $1, %%r9, %%r10 ;" + "shldq $1, %%r8, %%r9 ;" + "shlq $1, %%r8 ;" + + /*******************/ + "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ + /*******************/ + "movq %%rax, 0(%0) ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, 8(%0) ;" + "movq 8(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ + "adcq %%rax, %%r9 ;" + "movq %%r9, 16(%0) ;" + "adcq %%rcx, %%r10 ;" + "movq %%r10, 24(%0) ;" + "movq 16(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ + "adcq %%rax, %%r11 ;" + "movq %%r11, 32(%0) ;" + "adcq %%rcx, %%r15 ;" + "movq %%r15, 40(%0) ;" + "movq 24(%1), %%rdx ;" + "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ + "adcq %%rax, %%r13 ;" + "movq %%r13, 48(%0) ;" + "adcq %%rcx, %%r14 ;" + "movq %%r14, 56(%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11", "%r13", "%r14", "%r15"); +} +static void red_eltfp25519_1w_adx(u64 *const c, const u64 *const a) +{ asm volatile( - /* Compute the raw multiplication of f1*f2 */ - " mulxq 0(%2), %%r8, %%rcx;" /* f1[0]*f2 */ - " mulxq 8(%2), %%r9, %%rbx;" /* f1[1]*f2 */ - " add %%rcx, %%r9;" - " mov $0, %%rcx;" - " mulxq 16(%2), %%r10, %%r13;" /* f1[2]*f2 */ - " adcx %%rbx, %%r10;" - " mulxq 24(%2), %%r11, %%rax;" /* f1[3]*f2 */ - " adcx %%r13, %%r11;" - " adcx %%rcx, %%rax;" - - /* Wrap the result back into the field */ - - /* Step 1: Compute carry*38 */ - " mov $38, %%rdx;" - " imul %%rdx, %%rax;" - - /* Step 2: Fold the carry back into dst */ - " add %%rax, %%r8;" - " adcx %%rcx, %%r9;" - " movq %%r9, 8(%1);" - " adcx %%rcx, %%r10;" - " movq %%r10, 16(%1);" - " adcx %%rcx, %%r11;" - " movq %%r11, 24(%1);" - - /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ - " mov $0, %%rax;" - " cmovc %%rdx, %%rax;" - " add %%rax, %%r8;" - " movq %%r8, 0(%1);" - : "+&r" (f2_r) - : "r" (out), "r" (f1) - : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "memory", "cc" - ); + "movl $38, %%edx ;" /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ + "xorl %%ebx, %%ebx ;" + "adox (%1), %%r8 ;" + "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ + "adcx %%r10, %%r9 ;" + "adox 8(%1), %%r9 ;" + "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcx %%r11, %%r10 ;" + "adox 16(%1), %%r10 ;" + "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcx %%rax, %%r11 ;" + "adox 24(%1), %%r11 ;" + /***************************************/ + "adcx %%rbx, %%rcx ;" + "adox %%rbx, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ + "adcx %%rcx, %%r8 ;" + "adcx %%rbx, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcx %%rbx, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcx %%rbx, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", + "%r10", "%r11"); } -/* Computes p1 <- bit ? p2 : p1 in constant time */ -static inline void cswap2(u64 bit, const u64 *p1, const u64 *p2) +static void red_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a) { asm volatile( - /* Invert the polarity of bit to match cmov expectations */ - " add $18446744073709551615, %0;" - - /* cswap p1[0], p2[0] */ - " movq 0(%1), %%r8;" - " movq 0(%2), %%r9;" - " mov %%r8, %%r10;" - " cmovc %%r9, %%r8;" - " cmovc %%r10, %%r9;" - " movq %%r8, 0(%1);" - " movq %%r9, 0(%2);" - - /* cswap p1[1], p2[1] */ - " movq 8(%1), %%r8;" - " movq 8(%2), %%r9;" - " mov %%r8, %%r10;" - " cmovc %%r9, %%r8;" - " cmovc %%r10, %%r9;" - " movq %%r8, 8(%1);" - " movq %%r9, 8(%2);" - - /* cswap p1[2], p2[2] */ - " movq 16(%1), %%r8;" - " movq 16(%2), %%r9;" - " mov %%r8, %%r10;" - " cmovc %%r9, %%r8;" - " cmovc %%r10, %%r9;" - " movq %%r8, 16(%1);" - " movq %%r9, 16(%2);" - - /* cswap p1[3], p2[3] */ - " movq 24(%1), %%r8;" - " movq 24(%2), %%r9;" - " mov %%r8, %%r10;" - " cmovc %%r9, %%r8;" - " cmovc %%r10, %%r9;" - " movq %%r8, 24(%1);" - " movq %%r9, 24(%2);" - - /* cswap p1[4], p2[4] */ - " movq 32(%1), %%r8;" - " movq 32(%2), %%r9;" - " mov %%r8, %%r10;" - " cmovc %%r9, %%r8;" - " cmovc %%r10, %%r9;" - " movq %%r8, 32(%1);" - " movq %%r9, 32(%2);" - - /* cswap p1[5], p2[5] */ - " movq 40(%1), %%r8;" - " movq 40(%2), %%r9;" - " mov %%r8, %%r10;" - " cmovc %%r9, %%r8;" - " cmovc %%r10, %%r9;" - " movq %%r8, 40(%1);" - " movq %%r9, 40(%2);" - - /* cswap p1[6], p2[6] */ - " movq 48(%1), %%r8;" - " movq 48(%2), %%r9;" - " mov %%r8, %%r10;" - " cmovc %%r9, %%r8;" - " cmovc %%r10, %%r9;" - " movq %%r8, 48(%1);" - " movq %%r9, 48(%2);" - - /* cswap p1[7], p2[7] */ - " movq 56(%1), %%r8;" - " movq 56(%2), %%r9;" - " mov %%r8, %%r10;" - " cmovc %%r9, %%r8;" - " cmovc %%r10, %%r9;" - " movq %%r8, 56(%1);" - " movq %%r9, 56(%2);" - : "+&r" (bit) - : "r" (p1), "r" (p2) - : "%r8", "%r9", "%r10", "memory", "cc" - ); + "movl $38, %%edx ;" /* 2*c = 38 = 2^256 */ + "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ + "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ + "addq %%r10, %%r9 ;" + "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ + "adcq %%r11, %%r10 ;" + "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ + "adcq %%rax, %%r11 ;" + /***************************************/ + "adcq $0, %%rcx ;" + "addq (%1), %%r8 ;" + "adcq 8(%1), %%r9 ;" + "adcq 16(%1), %%r10 ;" + "adcq 24(%1), %%r11 ;" + "adcq $0, %%rcx ;" + "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11"); } -/* Computes the square of a field element: out <- f * f - * Uses the 8-element buffer tmp for intermediate results */ -static inline void fsqr(u64 *out, const u64 *f, u64 *tmp) +static __always_inline void +add_eltfp25519_1w_adx(u64 *const c, const u64 *const a, const u64 *const b) { asm volatile( - /* Compute the raw multiplication: tmp <- f * f */ - - /* Step 1: Compute all partial products */ - " movq 0(%1), %%rdx;" /* f[0] */ - " mulxq 8(%1), %%r8, %%r14;" " xor %%r15, %%r15;" /* f[1]*f[0] */ - " mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ - " mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ - " movq 24(%1), %%rdx;" /* f[3] */ - " mulxq 8(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */ - " mulxq 16(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */ - " movq 8(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */ - " mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ - - /* Step 2: Compute two parallel carry chains */ - " xor %%r15, %%r15;" - " adox %%rax, %%r10;" - " adcx %%r8, %%r8;" - " adox %%rcx, %%r11;" - " adcx %%r9, %%r9;" - " adox %%r15, %%rbx;" - " adcx %%r10, %%r10;" - " adox %%r15, %%r13;" - " adcx %%r11, %%r11;" - " adox %%r15, %%r14;" - " adcx %%rbx, %%rbx;" - " adcx %%r13, %%r13;" - " adcx %%r14, %%r14;" - - /* Step 3: Compute intermediate squares */ - " movq 0(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ - " movq %%rax, 0(%0);" - " add %%rcx, %%r8;" " movq %%r8, 8(%0);" - " movq 8(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ - " adcx %%rax, %%r9;" " movq %%r9, 16(%0);" - " adcx %%rcx, %%r10;" " movq %%r10, 24(%0);" - " movq 16(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ - " adcx %%rax, %%r11;" " movq %%r11, 32(%0);" - " adcx %%rcx, %%rbx;" " movq %%rbx, 40(%0);" - " movq 24(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ - " adcx %%rax, %%r13;" " movq %%r13, 48(%0);" - " adcx %%rcx, %%r14;" " movq %%r14, 56(%0);" - - /* Line up pointers */ - " mov %0, %1;" - " mov %2, %0;" - - /* Wrap the result back into the field */ - - /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ - " mov $38, %%rdx;" - " mulxq 32(%1), %%r8, %%r13;" - " xor %%rcx, %%rcx;" - " adoxq 0(%1), %%r8;" - " mulxq 40(%1), %%r9, %%rbx;" - " adcx %%r13, %%r9;" - " adoxq 8(%1), %%r9;" - " mulxq 48(%1), %%r10, %%r13;" - " adcx %%rbx, %%r10;" - " adoxq 16(%1), %%r10;" - " mulxq 56(%1), %%r11, %%rax;" - " adcx %%r13, %%r11;" - " adoxq 24(%1), %%r11;" - " adcx %%rcx, %%rax;" - " adox %%rcx, %%rax;" - " imul %%rdx, %%rax;" - - /* Step 2: Fold the carry back into dst */ - " add %%rax, %%r8;" - " adcx %%rcx, %%r9;" - " movq %%r9, 8(%0);" - " adcx %%rcx, %%r10;" - " movq %%r10, 16(%0);" - " adcx %%rcx, %%r11;" - " movq %%r11, 24(%0);" - - /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ - " mov $0, %%rax;" - " cmovc %%rdx, %%rax;" - " add %%rax, %%r8;" - " movq %%r8, 0(%0);" - : "+&r" (tmp), "+&r" (f), "+&r" (out) - : - : "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "%r15", "memory", "cc" - ); + "mov $38, %%eax ;" + "xorl %%ecx, %%ecx ;" + "movq (%2), %%r8 ;" + "adcx (%1), %%r8 ;" + "movq 8(%2), %%r9 ;" + "adcx 8(%1), %%r9 ;" + "movq 16(%2), %%r10 ;" + "adcx 16(%1), %%r10 ;" + "movq 24(%2), %%r11 ;" + "adcx 24(%1), %%r11 ;" + "cmovc %%eax, %%ecx ;" + "xorl %%eax, %%eax ;" + "adcx %%rcx, %%r8 ;" + "adcx %%rax, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcx %%rax, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcx %%rax, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $38, %%ecx ;" + "cmovc %%ecx, %%eax ;" + "addq %%rax, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); } -/* Computes two field squarings: - * out[0] <- f[0] * f[0] - * out[1] <- f[1] * f[1] - * Uses the 16-element buffer tmp for intermediate results */ -static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) +static __always_inline void +add_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a, const u64 *const b) { asm volatile( - /* Step 1: Compute all partial products */ - " movq 0(%1), %%rdx;" /* f[0] */ - " mulxq 8(%1), %%r8, %%r14;" " xor %%r15, %%r15;" /* f[1]*f[0] */ - " mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ - " mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ - " movq 24(%1), %%rdx;" /* f[3] */ - " mulxq 8(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */ - " mulxq 16(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */ - " movq 8(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */ - " mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ - - /* Step 2: Compute two parallel carry chains */ - " xor %%r15, %%r15;" - " adox %%rax, %%r10;" - " adcx %%r8, %%r8;" - " adox %%rcx, %%r11;" - " adcx %%r9, %%r9;" - " adox %%r15, %%rbx;" - " adcx %%r10, %%r10;" - " adox %%r15, %%r13;" - " adcx %%r11, %%r11;" - " adox %%r15, %%r14;" - " adcx %%rbx, %%rbx;" - " adcx %%r13, %%r13;" - " adcx %%r14, %%r14;" - - /* Step 3: Compute intermediate squares */ - " movq 0(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ - " movq %%rax, 0(%0);" - " add %%rcx, %%r8;" " movq %%r8, 8(%0);" - " movq 8(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ - " adcx %%rax, %%r9;" " movq %%r9, 16(%0);" - " adcx %%rcx, %%r10;" " movq %%r10, 24(%0);" - " movq 16(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ - " adcx %%rax, %%r11;" " movq %%r11, 32(%0);" - " adcx %%rcx, %%rbx;" " movq %%rbx, 40(%0);" - " movq 24(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ - " adcx %%rax, %%r13;" " movq %%r13, 48(%0);" - " adcx %%rcx, %%r14;" " movq %%r14, 56(%0);" - - /* Step 1: Compute all partial products */ - " movq 32(%1), %%rdx;" /* f[0] */ - " mulxq 40(%1), %%r8, %%r14;" " xor %%r15, %%r15;" /* f[1]*f[0] */ - " mulxq 48(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ - " mulxq 56(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ - " movq 56(%1), %%rdx;" /* f[3] */ - " mulxq 40(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */ - " mulxq 48(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */ - " movq 40(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */ - " mulxq 48(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ - - /* Step 2: Compute two parallel carry chains */ - " xor %%r15, %%r15;" - " adox %%rax, %%r10;" - " adcx %%r8, %%r8;" - " adox %%rcx, %%r11;" - " adcx %%r9, %%r9;" - " adox %%r15, %%rbx;" - " adcx %%r10, %%r10;" - " adox %%r15, %%r13;" - " adcx %%r11, %%r11;" - " adox %%r15, %%r14;" - " adcx %%rbx, %%rbx;" - " adcx %%r13, %%r13;" - " adcx %%r14, %%r14;" - - /* Step 3: Compute intermediate squares */ - " movq 32(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ - " movq %%rax, 64(%0);" - " add %%rcx, %%r8;" " movq %%r8, 72(%0);" - " movq 40(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ - " adcx %%rax, %%r9;" " movq %%r9, 80(%0);" - " adcx %%rcx, %%r10;" " movq %%r10, 88(%0);" - " movq 48(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ - " adcx %%rax, %%r11;" " movq %%r11, 96(%0);" - " adcx %%rcx, %%rbx;" " movq %%rbx, 104(%0);" - " movq 56(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ - " adcx %%rax, %%r13;" " movq %%r13, 112(%0);" - " adcx %%rcx, %%r14;" " movq %%r14, 120(%0);" - - /* Line up pointers */ - " mov %0, %1;" - " mov %2, %0;" - - /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ - " mov $38, %%rdx;" - " mulxq 32(%1), %%r8, %%r13;" - " xor %%rcx, %%rcx;" - " adoxq 0(%1), %%r8;" - " mulxq 40(%1), %%r9, %%rbx;" - " adcx %%r13, %%r9;" - " adoxq 8(%1), %%r9;" - " mulxq 48(%1), %%r10, %%r13;" - " adcx %%rbx, %%r10;" - " adoxq 16(%1), %%r10;" - " mulxq 56(%1), %%r11, %%rax;" - " adcx %%r13, %%r11;" - " adoxq 24(%1), %%r11;" - " adcx %%rcx, %%rax;" - " adox %%rcx, %%rax;" - " imul %%rdx, %%rax;" - - /* Step 2: Fold the carry back into dst */ - " add %%rax, %%r8;" - " adcx %%rcx, %%r9;" - " movq %%r9, 8(%0);" - " adcx %%rcx, %%r10;" - " movq %%r10, 16(%0);" - " adcx %%rcx, %%r11;" - " movq %%r11, 24(%0);" - - /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ - " mov $0, %%rax;" - " cmovc %%rdx, %%rax;" - " add %%rax, %%r8;" - " movq %%r8, 0(%0);" - - /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ - " mov $38, %%rdx;" - " mulxq 96(%1), %%r8, %%r13;" - " xor %%rcx, %%rcx;" - " adoxq 64(%1), %%r8;" - " mulxq 104(%1), %%r9, %%rbx;" - " adcx %%r13, %%r9;" - " adoxq 72(%1), %%r9;" - " mulxq 112(%1), %%r10, %%r13;" - " adcx %%rbx, %%r10;" - " adoxq 80(%1), %%r10;" - " mulxq 120(%1), %%r11, %%rax;" - " adcx %%r13, %%r11;" - " adoxq 88(%1), %%r11;" - " adcx %%rcx, %%rax;" - " adox %%rcx, %%rax;" - " imul %%rdx, %%rax;" - - /* Step 2: Fold the carry back into dst */ - " add %%rax, %%r8;" - " adcx %%rcx, %%r9;" - " movq %%r9, 40(%0);" - " adcx %%rcx, %%r10;" - " movq %%r10, 48(%0);" - " adcx %%rcx, %%r11;" - " movq %%r11, 56(%0);" - - /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ - " mov $0, %%rax;" - " cmovc %%rdx, %%rax;" - " add %%rax, %%r8;" - " movq %%r8, 32(%0);" - : "+&r" (tmp), "+&r" (f), "+&r" (out) - : - : "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "%r15", "memory", "cc" - ); + "mov $38, %%eax ;" + "movq (%2), %%r8 ;" + "addq (%1), %%r8 ;" + "movq 8(%2), %%r9 ;" + "adcq 8(%1), %%r9 ;" + "movq 16(%2), %%r10 ;" + "adcq 16(%1), %%r10 ;" + "movq 24(%2), %%r11 ;" + "adcq 24(%1), %%r11 ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); } -static void point_add_and_double(u64 *q, u64 *p01_tmp1, u64 *tmp2) +static __always_inline void +sub_eltfp25519_1w(u64 *const c, const u64 *const a, const u64 *const b) { - u64 *nq = p01_tmp1; - u64 *nq_p1 = p01_tmp1 + (u32)8U; - u64 *tmp1 = p01_tmp1 + (u32)16U; - u64 *x1 = q; - u64 *x2 = nq; - u64 *z2 = nq + (u32)4U; - u64 *z3 = nq_p1 + (u32)4U; - u64 *a = tmp1; - u64 *b = tmp1 + (u32)4U; - u64 *ab = tmp1; - u64 *dc = tmp1 + (u32)8U; - u64 *x3; - u64 *z31; - u64 *d0; - u64 *c0; - u64 *a1; - u64 *b1; - u64 *d; - u64 *c; - u64 *ab1; - u64 *dc1; - fadd(a, x2, z2); - fsub(b, x2, z2); - x3 = nq_p1; - z31 = nq_p1 + (u32)4U; - d0 = dc; - c0 = dc + (u32)4U; - fadd(c0, x3, z31); - fsub(d0, x3, z31); - fmul2(dc, dc, ab, tmp2); - fadd(x3, d0, c0); - fsub(z31, d0, c0); - a1 = tmp1; - b1 = tmp1 + (u32)4U; - d = tmp1 + (u32)8U; - c = tmp1 + (u32)12U; - ab1 = tmp1; - dc1 = tmp1 + (u32)8U; - fsqr2(dc1, ab1, tmp2); - fsqr2(nq_p1, nq_p1, tmp2); - a1[0U] = c[0U]; - a1[1U] = c[1U]; - a1[2U] = c[2U]; - a1[3U] = c[3U]; - fsub(c, d, c); - fmul_scalar(b1, c, (u64)121665U); - fadd(b1, b1, d); - fmul2(nq, dc1, ab1, tmp2); - fmul(z3, z3, x1, tmp2); + asm volatile( + "mov $38, %%eax ;" + "movq (%1), %%r8 ;" + "subq (%2), %%r8 ;" + "movq 8(%1), %%r9 ;" + "sbbq 8(%2), %%r9 ;" + "movq 16(%1), %%r10 ;" + "sbbq 16(%2), %%r10 ;" + "movq 24(%1), %%r11 ;" + "sbbq 24(%2), %%r11 ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "subq %%rcx, %%r8 ;" + "sbbq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "sbbq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "sbbq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%eax, %%ecx ;" + "subq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(b) + : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); } -static void point_double(u64 *nq, u64 *tmp1, u64 *tmp2) +/* Multiplication by a24 = (A+2)/4 = (486662+2)/4 = 121666 */ +static __always_inline void +mul_a24_eltfp25519_1w(u64 *const c, const u64 *const a) { - u64 *x2 = nq; - u64 *z2 = nq + (u32)4U; - u64 *a = tmp1; - u64 *b = tmp1 + (u32)4U; - u64 *d = tmp1 + (u32)8U; - u64 *c = tmp1 + (u32)12U; - u64 *ab = tmp1; - u64 *dc = tmp1 + (u32)8U; - fadd(a, x2, z2); - fsub(b, x2, z2); - fsqr2(dc, ab, tmp2); - a[0U] = c[0U]; - a[1U] = c[1U]; - a[2U] = c[2U]; - a[3U] = c[3U]; - fsub(c, d, c); - fmul_scalar(b, c, (u64)121665U); - fadd(b, b, d); - fmul2(nq, dc, ab, tmp2); + const u64 a24 = 121666; + asm volatile( + "movq %2, %%rdx ;" + "mulx (%1), %%r8, %%r10 ;" + "mulx 8(%1), %%r9, %%r11 ;" + "addq %%r10, %%r9 ;" + "mulx 16(%1), %%r10, %%rax ;" + "adcq %%r11, %%r10 ;" + "mulx 24(%1), %%r11, %%rcx ;" + "adcq %%rax, %%r11 ;" + /**************************/ + "adcq $0, %%rcx ;" + "movl $38, %%edx ;" /* 2*c = 38 = 2^256 mod 2^255-19*/ + "imul %%rdx, %%rcx ;" + "addq %%rcx, %%r8 ;" + "adcq $0, %%r9 ;" + "movq %%r9, 8(%0) ;" + "adcq $0, %%r10 ;" + "movq %%r10, 16(%0) ;" + "adcq $0, %%r11 ;" + "movq %%r11, 24(%0) ;" + "mov $0, %%ecx ;" + "cmovc %%edx, %%ecx ;" + "addq %%rcx, %%r8 ;" + "movq %%r8, (%0) ;" + : + : "r"(c), "r"(a), "r"(a24) + : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", + "%r11"); } -static void montgomery_ladder(u64 *out, const u8 *key, u64 *init1) +static void inv_eltfp25519_1w_adx(u64 *const c, const u64 *const a) { - u64 tmp2[16U] = { 0U }; - u64 p01_tmp1_swap[33U] = { 0U }; - u64 *p0 = p01_tmp1_swap; - u64 *p01 = p01_tmp1_swap; - u64 *p03 = p01; - u64 *p11 = p01 + (u32)8U; - u64 *x0; - u64 *z0; - u64 *p01_tmp1; - u64 *p01_tmp11; - u64 *nq10; - u64 *nq_p11; - u64 *swap1; - u64 sw0; - u64 *nq1; - u64 *tmp1; - memcpy(p11, init1, (u32)8U * sizeof(init1[0U])); - x0 = p03; - z0 = p03 + (u32)4U; - x0[0U] = (u64)1U; - x0[1U] = (u64)0U; - x0[2U] = (u64)0U; - x0[3U] = (u64)0U; - z0[0U] = (u64)0U; - z0[1U] = (u64)0U; - z0[2U] = (u64)0U; - z0[3U] = (u64)0U; - p01_tmp1 = p01_tmp1_swap; - p01_tmp11 = p01_tmp1_swap; - nq10 = p01_tmp1_swap; - nq_p11 = p01_tmp1_swap + (u32)8U; - swap1 = p01_tmp1_swap + (u32)32U; - cswap2((u64)1U, nq10, nq_p11); - point_add_and_double(init1, p01_tmp11, tmp2); - swap1[0U] = (u64)1U; - { - u32 i; - for (i = (u32)0U; i < (u32)251U; i = i + (u32)1U) { - u64 *p01_tmp12 = p01_tmp1_swap; - u64 *swap2 = p01_tmp1_swap + (u32)32U; - u64 *nq2 = p01_tmp12; - u64 *nq_p12 = p01_tmp12 + (u32)8U; - u64 bit = (u64)(key[((u32)253U - i) / (u32)8U] >> ((u32)253U - i) % (u32)8U & (u8)1U); - u64 sw = swap2[0U] ^ bit; - cswap2(sw, nq2, nq_p12); - point_add_and_double(init1, p01_tmp12, tmp2); - swap2[0U] = bit; - } - } - sw0 = swap1[0U]; - cswap2(sw0, nq10, nq_p11); - nq1 = p01_tmp1; - tmp1 = p01_tmp1 + (u32)16U; - point_double(nq1, tmp1, tmp2); - point_double(nq1, tmp1, tmp2); - point_double(nq1, tmp1, tmp2); - memcpy(out, p0, (u32)8U * sizeof(p0[0U])); - - memzero_explicit(tmp2, sizeof(tmp2)); - memzero_explicit(p01_tmp1_swap, sizeof(p01_tmp1_swap)); + struct { + eltfp25519_1w_buffer buffer; + eltfp25519_1w x0, x1, x2; + } __aligned(32) m; + u64 *T[4]; + + T[0] = m.x0; + T[1] = c; /* x^(-1) */ + T[2] = m.x1; + T[3] = m.x2; + + copy_eltfp25519_1w(T[1], a); + sqrn_eltfp25519_1w_adx(T[1], 1); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_adx(T[2], 2); + mul_eltfp25519_1w_adx(T[0], a, T[2]); + mul_eltfp25519_1w_adx(T[1], T[1], T[0]); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_adx(T[2], 1); + mul_eltfp25519_1w_adx(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 5); + mul_eltfp25519_1w_adx(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 10); + mul_eltfp25519_1w_adx(T[2], T[2], T[0]); + copy_eltfp25519_1w(T[3], T[2]); + sqrn_eltfp25519_1w_adx(T[3], 20); + mul_eltfp25519_1w_adx(T[3], T[3], T[2]); + sqrn_eltfp25519_1w_adx(T[3], 10); + mul_eltfp25519_1w_adx(T[3], T[3], T[0]); + copy_eltfp25519_1w(T[0], T[3]); + sqrn_eltfp25519_1w_adx(T[0], 50); + mul_eltfp25519_1w_adx(T[0], T[0], T[3]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 100); + mul_eltfp25519_1w_adx(T[2], T[2], T[0]); + sqrn_eltfp25519_1w_adx(T[2], 50); + mul_eltfp25519_1w_adx(T[2], T[2], T[3]); + sqrn_eltfp25519_1w_adx(T[2], 5); + mul_eltfp25519_1w_adx(T[1], T[1], T[2]); + + memzero_explicit(&m, sizeof(m)); } -static void fsquare_times(u64 *o, const u64 *inp, u64 *tmp, u32 n1) +static void inv_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a) +{ + struct { + eltfp25519_1w_buffer buffer; + eltfp25519_1w x0, x1, x2; + } __aligned(32) m; + u64 *T[5]; + + T[0] = m.x0; + T[1] = c; /* x^(-1) */ + T[2] = m.x1; + T[3] = m.x2; + + copy_eltfp25519_1w(T[1], a); + sqrn_eltfp25519_1w_bmi2(T[1], 1); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_bmi2(T[2], 2); + mul_eltfp25519_1w_bmi2(T[0], a, T[2]); + mul_eltfp25519_1w_bmi2(T[1], T[1], T[0]); + copy_eltfp25519_1w(T[2], T[1]); + sqrn_eltfp25519_1w_bmi2(T[2], 1); + mul_eltfp25519_1w_bmi2(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 5); + mul_eltfp25519_1w_bmi2(T[0], T[0], T[2]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 10); + mul_eltfp25519_1w_bmi2(T[2], T[2], T[0]); + copy_eltfp25519_1w(T[3], T[2]); + sqrn_eltfp25519_1w_bmi2(T[3], 20); + mul_eltfp25519_1w_bmi2(T[3], T[3], T[2]); + sqrn_eltfp25519_1w_bmi2(T[3], 10); + mul_eltfp25519_1w_bmi2(T[3], T[3], T[0]); + copy_eltfp25519_1w(T[0], T[3]); + sqrn_eltfp25519_1w_bmi2(T[0], 50); + mul_eltfp25519_1w_bmi2(T[0], T[0], T[3]); + copy_eltfp25519_1w(T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 100); + mul_eltfp25519_1w_bmi2(T[2], T[2], T[0]); + sqrn_eltfp25519_1w_bmi2(T[2], 50); + mul_eltfp25519_1w_bmi2(T[2], T[2], T[3]); + sqrn_eltfp25519_1w_bmi2(T[2], 5); + mul_eltfp25519_1w_bmi2(T[1], T[1], T[2]); + + memzero_explicit(&m, sizeof(m)); +} + +/* Given c, a 256-bit number, fred_eltfp25519_1w updates c + * with a number such that 0 <= C < 2**255-19. + */ +static __always_inline void fred_eltfp25519_1w(u64 *const c) { - u32 i; - fsqr(o, inp, tmp); - for (i = (u32)0U; i < n1 - (u32)1U; i = i + (u32)1U) - fsqr(o, o, tmp); + u64 tmp0 = 38, tmp1 = 19; + asm volatile( + "btrq $63, %3 ;" /* Put bit 255 in carry flag and clear */ + "cmovncl %k5, %k4 ;" /* c[255] ? 38 : 19 */ + + /* Add either 19 or 38 to c */ + "addq %4, %0 ;" + "adcq $0, %1 ;" + "adcq $0, %2 ;" + "adcq $0, %3 ;" + + /* Test for bit 255 again; only triggered on overflow modulo 2^255-19 */ + "movl $0, %k4 ;" + "cmovnsl %k5, %k4 ;" /* c[255] ? 0 : 19 */ + "btrq $63, %3 ;" /* Clear bit 255 */ + + /* Subtract 19 if necessary */ + "subq %4, %0 ;" + "sbbq $0, %1 ;" + "sbbq $0, %2 ;" + "sbbq $0, %3 ;" + + : "+r"(c[0]), "+r"(c[1]), "+r"(c[2]), "+r"(c[3]), "+r"(tmp0), + "+r"(tmp1) + : + : "memory", "cc"); } -static void finv(u64 *o, const u64 *i, u64 *tmp) +static __always_inline void cswap(u8 bit, u64 *const px, u64 *const py) { - u64 t1[16U] = { 0U }; - u64 *a0 = t1; - u64 *b = t1 + (u32)4U; - u64 *c = t1 + (u32)8U; - u64 *t00 = t1 + (u32)12U; - u64 *tmp1 = tmp; - u64 *a; - u64 *t0; - fsquare_times(a0, i, tmp1, (u32)1U); - fsquare_times(t00, a0, tmp1, (u32)2U); - fmul(b, t00, i, tmp); - fmul(a0, b, a0, tmp); - fsquare_times(t00, a0, tmp1, (u32)1U); - fmul(b, t00, b, tmp); - fsquare_times(t00, b, tmp1, (u32)5U); - fmul(b, t00, b, tmp); - fsquare_times(t00, b, tmp1, (u32)10U); - fmul(c, t00, b, tmp); - fsquare_times(t00, c, tmp1, (u32)20U); - fmul(t00, t00, c, tmp); - fsquare_times(t00, t00, tmp1, (u32)10U); - fmul(b, t00, b, tmp); - fsquare_times(t00, b, tmp1, (u32)50U); - fmul(c, t00, b, tmp); - fsquare_times(t00, c, tmp1, (u32)100U); - fmul(t00, t00, c, tmp); - fsquare_times(t00, t00, tmp1, (u32)50U); - fmul(t00, t00, b, tmp); - fsquare_times(t00, t00, tmp1, (u32)5U); - a = t1; - t0 = t1 + (u32)12U; - fmul(o, t0, a, tmp); + u64 temp; + asm volatile( + "test %9, %9 ;" + "movq %0, %8 ;" + "cmovnzq %4, %0 ;" + "cmovnzq %8, %4 ;" + "movq %1, %8 ;" + "cmovnzq %5, %1 ;" + "cmovnzq %8, %5 ;" + "movq %2, %8 ;" + "cmovnzq %6, %2 ;" + "cmovnzq %8, %6 ;" + "movq %3, %8 ;" + "cmovnzq %7, %3 ;" + "cmovnzq %8, %7 ;" + : "+r"(px[0]), "+r"(px[1]), "+r"(px[2]), "+r"(px[3]), + "+r"(py[0]), "+r"(py[1]), "+r"(py[2]), "+r"(py[3]), + "=r"(temp) + : "r"(bit) + : "cc" + ); } -static void store_felem(u64 *b, u64 *f) +static __always_inline void cselect(u8 bit, u64 *const px, const u64 *const py) { - u64 f30 = f[3U]; - u64 top_bit0 = f30 >> (u32)63U; - u64 carry0; - u64 f31; - u64 top_bit; - u64 carry; - u64 f0; - u64 f1; - u64 f2; - u64 f3; - u64 m0; - u64 m1; - u64 m2; - u64 m3; - u64 mask; - u64 f0_; - u64 f1_; - u64 f2_; - u64 f3_; - u64 o0; - u64 o1; - u64 o2; - u64 o3; - f[3U] = f30 & (u64)0x7fffffffffffffffU; - carry0 = add_scalar(f, f, (u64)19U * top_bit0); - f31 = f[3U]; - top_bit = f31 >> (u32)63U; - f[3U] = f31 & (u64)0x7fffffffffffffffU; - carry = add_scalar(f, f, (u64)19U * top_bit); - f0 = f[0U]; - f1 = f[1U]; - f2 = f[2U]; - f3 = f[3U]; - m0 = gte_mask(f0, (u64)0xffffffffffffffedU); - m1 = eq_mask(f1, (u64)0xffffffffffffffffU); - m2 = eq_mask(f2, (u64)0xffffffffffffffffU); - m3 = eq_mask(f3, (u64)0x7fffffffffffffffU); - mask = ((m0 & m1) & m2) & m3; - f0_ = f0 - (mask & (u64)0xffffffffffffffedU); - f1_ = f1 - (mask & (u64)0xffffffffffffffffU); - f2_ = f2 - (mask & (u64)0xffffffffffffffffU); - f3_ = f3 - (mask & (u64)0x7fffffffffffffffU); - o0 = f0_; - o1 = f1_; - o2 = f2_; - o3 = f3_; - b[0U] = o0; - b[1U] = o1; - b[2U] = o2; - b[3U] = o3; + asm volatile( + "test %4, %4 ;" + "cmovnzq %5, %0 ;" + "cmovnzq %6, %1 ;" + "cmovnzq %7, %2 ;" + "cmovnzq %8, %3 ;" + : "+r"(px[0]), "+r"(px[1]), "+r"(px[2]), "+r"(px[3]) + : "r"(bit), "rm"(py[0]), "rm"(py[1]), "rm"(py[2]), "rm"(py[3]) + : "cc" + ); } -static void encode_point(u8 *o, const u64 *i) +static void curve25519_adx(u8 shared[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE], + const u8 session_key[CURVE25519_KEY_SIZE]) { - const u64 *x = i; - const u64 *z = i + (u32)4U; - u64 tmp[4U] = { 0U }; - u64 tmp_w[16U] = { 0U }; - finv(tmp, z, tmp_w); - fmul(tmp, tmp, x, tmp_w); - store_felem((u64 *)o, tmp); + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[6 * NUM_WORDS_ELTFP25519]; + u8 session[CURVE25519_KEY_SIZE]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + int i = 0, j = 0; + u64 prev = 0; + u64 *const X1 = (u64 *)m.session; + u64 *const key = (u64 *)m.private; + u64 *const Px = m.coordinates + 0; + u64 *const Pz = m.coordinates + 4; + u64 *const Qx = m.coordinates + 8; + u64 *const Qz = m.coordinates + 12; + u64 *const X2 = Qx; + u64 *const Z2 = Qz; + u64 *const X3 = Px; + u64 *const Z3 = Pz; + u64 *const X2Z2 = Qx; + u64 *const X3Z3 = Px; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const D = m.workspace + 8; + u64 *const C = m.workspace + 12; + u64 *const DA = m.workspace + 16; + u64 *const CB = m.workspace + 20; + u64 *const AB = A; + u64 *const DC = D; + u64 *const DACB = DA; + + memcpy(m.private, private_key, sizeof(m.private)); + memcpy(m.session, session_key, sizeof(m.session)); + + curve25519_clamp_secret(m.private); + + /* As in the draft: + * When receiving such an array, implementations of curve25519 + * MUST mask the most-significant bit in the final byte. This + * is done to preserve compatibility with point formats which + * reserve the sign bit for use in other protocols and to + * increase resistance to implementation fingerprinting + */ + m.session[CURVE25519_KEY_SIZE - 1] &= (1 << (255 % 8)) - 1; + + copy_eltfp25519_1w(Px, X1); + setzero_eltfp25519_1w(Pz); + setzero_eltfp25519_1w(Qx); + setzero_eltfp25519_1w(Qz); + + Pz[0] = 1; + Qx[0] = 1; + + /* main-loop */ + prev = 0; + j = 62; + for (i = 3; i >= 0; --i) { + while (j >= 0) { + u64 bit = (key[i] >> j) & 0x1; + u64 swap = bit ^ prev; + prev = bit; + + add_eltfp25519_1w_adx(A, X2, Z2); /* A = (X2+Z2) */ + sub_eltfp25519_1w(B, X2, Z2); /* B = (X2-Z2) */ + add_eltfp25519_1w_adx(C, X3, Z3); /* C = (X3+Z3) */ + sub_eltfp25519_1w(D, X3, Z3); /* D = (X3-Z3) */ + mul_eltfp25519_2w_adx(DACB, AB, DC); /* [DA|CB] = [A|B]*[D|C] */ + + cselect(swap, A, C); + cselect(swap, B, D); + + sqr_eltfp25519_2w_adx(AB); /* [AA|BB] = [A^2|B^2] */ + add_eltfp25519_1w_adx(X3, DA, CB); /* X3 = (DA+CB) */ + sub_eltfp25519_1w(Z3, DA, CB); /* Z3 = (DA-CB) */ + sqr_eltfp25519_2w_adx(X3Z3); /* [X3|Z3] = [(DA+CB)|(DA+CB)]^2 */ + + copy_eltfp25519_1w(X2, B); /* X2 = B^2 */ + sub_eltfp25519_1w(Z2, A, B); /* Z2 = E = AA-BB */ + + mul_a24_eltfp25519_1w(B, Z2); /* B = a24*E */ + add_eltfp25519_1w_adx(B, B, X2); /* B = a24*E+B */ + mul_eltfp25519_2w_adx(X2Z2, X2Z2, AB); /* [X2|Z2] = [B|E]*[A|a24*E+B] */ + mul_eltfp25519_1w_adx(Z3, Z3, X1); /* Z3 = Z3*X1 */ + --j; + } + j = 63; + } + + inv_eltfp25519_1w_adx(A, Qz); + mul_eltfp25519_1w_adx((u64 *)shared, Qx, A); + fred_eltfp25519_1w((u64 *)shared); + + memzero_explicit(&m, sizeof(m)); } -static void curve25519_ever64(u8 *out, const u8 *priv, const u8 *pub) +static void curve25519_adx_base(u8 session_key[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE]) { - u64 init1[8U] = { 0U }; - u64 tmp[4U] = { 0U }; - u64 tmp3; - u64 *x; - u64 *z; - { - u32 i; - for (i = (u32)0U; i < (u32)4U; i = i + (u32)1U) { - u64 *os = tmp; - const u8 *bj = pub + i * (u32)8U; - u64 u = *(u64 *)bj; - u64 r = u; - u64 x0 = r; - os[i] = x0; + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[4 * NUM_WORDS_ELTFP25519]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + const int ite[4] = { 64, 64, 64, 63 }; + const int q = 3; + u64 swap = 1; + + int i = 0, j = 0, k = 0; + u64 *const key = (u64 *)m.private; + u64 *const Ur1 = m.coordinates + 0; + u64 *const Zr1 = m.coordinates + 4; + u64 *const Ur2 = m.coordinates + 8; + u64 *const Zr2 = m.coordinates + 12; + + u64 *const UZr1 = m.coordinates + 0; + u64 *const ZUr2 = m.coordinates + 8; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const C = m.workspace + 8; + u64 *const D = m.workspace + 12; + + u64 *const AB = m.workspace + 0; + u64 *const CD = m.workspace + 8; + + const u64 *const P = table_ladder_8k; + + memcpy(m.private, private_key, sizeof(m.private)); + + curve25519_clamp_secret(m.private); + + setzero_eltfp25519_1w(Ur1); + setzero_eltfp25519_1w(Zr1); + setzero_eltfp25519_1w(Zr2); + Ur1[0] = 1; + Zr1[0] = 1; + Zr2[0] = 1; + + /* G-S */ + Ur2[3] = 0x1eaecdeee27cab34UL; + Ur2[2] = 0xadc7a0b9235d48e2UL; + Ur2[1] = 0xbbf095ae14b2edf8UL; + Ur2[0] = 0x7e94e1fec82faabdUL; + + /* main-loop */ + j = q; + for (i = 0; i < NUM_WORDS_ELTFP25519; ++i) { + while (j < ite[i]) { + u64 bit = (key[i] >> j) & 0x1; + k = (64 * i + j - q); + swap = swap ^ bit; + cswap(swap, Ur1, Ur2); + cswap(swap, Zr1, Zr2); + swap = bit; + /* Addition */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + add_eltfp25519_1w_adx(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + mul_eltfp25519_1w_adx(C, &P[4 * k], B); /* C = M0-B */ + sub_eltfp25519_1w(B, A, C); /* B = (Ur1+Zr1) - M*(Ur1-Zr1) */ + add_eltfp25519_1w_adx(A, A, C); /* A = (Ur1+Zr1) + M*(Ur1-Zr1) */ + sqr_eltfp25519_2w_adx(AB); /* A = A^2 | B = B^2 */ + mul_eltfp25519_2w_adx(UZr1, ZUr2, AB); /* Ur1 = Zr2*A | Zr1 = Ur2*B */ + ++j; } + j = 0; } - tmp3 = tmp[3U]; - tmp[3U] = tmp3 & (u64)0x7fffffffffffffffU; - x = init1; - z = init1 + (u32)4U; - z[0U] = (u64)1U; - z[1U] = (u64)0U; - z[2U] = (u64)0U; - z[3U] = (u64)0U; - x[0U] = tmp[0U]; - x[1U] = tmp[1U]; - x[2U] = tmp[2U]; - x[3U] = tmp[3U]; - montgomery_ladder(init1, priv, init1); - encode_point(out, init1); + + /* Doubling */ + for (i = 0; i < q; ++i) { + add_eltfp25519_1w_adx(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + sqr_eltfp25519_2w_adx(AB); /* A = A**2 B = B**2 */ + copy_eltfp25519_1w(C, B); /* C = B */ + sub_eltfp25519_1w(B, A, B); /* B = A-B */ + mul_a24_eltfp25519_1w(D, B); /* D = my_a24*B */ + add_eltfp25519_1w_adx(D, D, C); /* D = D+C */ + mul_eltfp25519_2w_adx(UZr1, AB, CD); /* Ur1 = A*B Zr1 = Zr1*A */ + } + + /* Convert to affine coordinates */ + inv_eltfp25519_1w_adx(A, Zr1); + mul_eltfp25519_1w_adx((u64 *)session_key, Ur1, A); + fred_eltfp25519_1w((u64 *)session_key); + + memzero_explicit(&m, sizeof(m)); } -/* The below constants were generated using this sage script: - * - * #!/usr/bin/env sage - * import sys - * from sage.all import * - * def limbs(n): - * n = int(n) - * l = ((n >> 0) % 2^64, (n >> 64) % 2^64, (n >> 128) % 2^64, (n >> 192) % 2^64) - * return "0x%016xULL, 0x%016xULL, 0x%016xULL, 0x%016xULL" % l - * ec = EllipticCurve(GF(2^255 - 19), [0, 486662, 0, 1, 0]) - * p_minus_s = (ec.lift_x(9) - ec.lift_x(1))[0] - * print("static const u64 p_minus_s[] = { %s };\n" % limbs(p_minus_s)) - * print("static const u64 table_ladder[] = {") - * p = ec.lift_x(9) - * for i in range(252): - * l = (p[0] + p[2]) / (p[0] - p[2]) - * print(("\t%s" + ("," if i != 251 else "")) % limbs(l)) - * p = p * 2 - * print("};") - * - */ +static void curve25519_bmi2(u8 shared[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE], + const u8 session_key[CURVE25519_KEY_SIZE]) +{ + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[6 * NUM_WORDS_ELTFP25519]; + u8 session[CURVE25519_KEY_SIZE]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + int i = 0, j = 0; + u64 prev = 0; + u64 *const X1 = (u64 *)m.session; + u64 *const key = (u64 *)m.private; + u64 *const Px = m.coordinates + 0; + u64 *const Pz = m.coordinates + 4; + u64 *const Qx = m.coordinates + 8; + u64 *const Qz = m.coordinates + 12; + u64 *const X2 = Qx; + u64 *const Z2 = Qz; + u64 *const X3 = Px; + u64 *const Z3 = Pz; + u64 *const X2Z2 = Qx; + u64 *const X3Z3 = Px; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const D = m.workspace + 8; + u64 *const C = m.workspace + 12; + u64 *const DA = m.workspace + 16; + u64 *const CB = m.workspace + 20; + u64 *const AB = A; + u64 *const DC = D; + u64 *const DACB = DA; + + memcpy(m.private, private_key, sizeof(m.private)); + memcpy(m.session, session_key, sizeof(m.session)); + + curve25519_clamp_secret(m.private); + + /* As in the draft: + * When receiving such an array, implementations of curve25519 + * MUST mask the most-significant bit in the final byte. This + * is done to preserve compatibility with point formats which + * reserve the sign bit for use in other protocols and to + * increase resistance to implementation fingerprinting + */ + m.session[CURVE25519_KEY_SIZE - 1] &= (1 << (255 % 8)) - 1; + + copy_eltfp25519_1w(Px, X1); + setzero_eltfp25519_1w(Pz); + setzero_eltfp25519_1w(Qx); + setzero_eltfp25519_1w(Qz); + + Pz[0] = 1; + Qx[0] = 1; + + /* main-loop */ + prev = 0; + j = 62; + for (i = 3; i >= 0; --i) { + while (j >= 0) { + u64 bit = (key[i] >> j) & 0x1; + u64 swap = bit ^ prev; + prev = bit; + + add_eltfp25519_1w_bmi2(A, X2, Z2); /* A = (X2+Z2) */ + sub_eltfp25519_1w(B, X2, Z2); /* B = (X2-Z2) */ + add_eltfp25519_1w_bmi2(C, X3, Z3); /* C = (X3+Z3) */ + sub_eltfp25519_1w(D, X3, Z3); /* D = (X3-Z3) */ + mul_eltfp25519_2w_bmi2(DACB, AB, DC); /* [DA|CB] = [A|B]*[D|C] */ + + cselect(swap, A, C); + cselect(swap, B, D); + + sqr_eltfp25519_2w_bmi2(AB); /* [AA|BB] = [A^2|B^2] */ + add_eltfp25519_1w_bmi2(X3, DA, CB); /* X3 = (DA+CB) */ + sub_eltfp25519_1w(Z3, DA, CB); /* Z3 = (DA-CB) */ + sqr_eltfp25519_2w_bmi2(X3Z3); /* [X3|Z3] = [(DA+CB)|(DA+CB)]^2 */ + + copy_eltfp25519_1w(X2, B); /* X2 = B^2 */ + sub_eltfp25519_1w(Z2, A, B); /* Z2 = E = AA-BB */ + + mul_a24_eltfp25519_1w(B, Z2); /* B = a24*E */ + add_eltfp25519_1w_bmi2(B, B, X2); /* B = a24*E+B */ + mul_eltfp25519_2w_bmi2(X2Z2, X2Z2, AB); /* [X2|Z2] = [B|E]*[A|a24*E+B] */ + mul_eltfp25519_1w_bmi2(Z3, Z3, X1); /* Z3 = Z3*X1 */ + --j; + } + j = 63; + } -static const u64 p_minus_s[] = { 0x816b1e0137d48290ULL, 0x440f6a51eb4d1207ULL, 0x52385f46dca2b71dULL, 0x215132111d8354cbULL }; - -static const u64 table_ladder[] = { - 0xfffffffffffffff3ULL, 0xffffffffffffffffULL, 0xffffffffffffffffULL, 0x5fffffffffffffffULL, - 0x6b8220f416aafe96ULL, 0x82ebeb2b4f566a34ULL, 0xd5a9a5b075a5950fULL, 0x5142b2cf4b2488f4ULL, - 0x6aaebc750069680cULL, 0x89cf7820a0f99c41ULL, 0x2a58d9183b56d0f4ULL, 0x4b5aca80e36011a4ULL, - 0x329132348c29745dULL, 0xf4a2e616e1642fd7ULL, 0x1e45bb03ff67bc34ULL, 0x306912d0f42a9b4aULL, - 0xff886507e6af7154ULL, 0x04f50e13dfeec82fULL, 0xaa512fe82abab5ceULL, 0x174e251a68d5f222ULL, - 0xcf96700d82028898ULL, 0x1743e3370a2c02c5ULL, 0x379eec98b4e86eaaULL, 0x0c59888a51e0482eULL, - 0xfbcbf1d699b5d189ULL, 0xacaef0d58e9fdc84ULL, 0xc1c20d06231f7614ULL, 0x2938218da274f972ULL, - 0xf6af49beff1d7f18ULL, 0xcc541c22387ac9c2ULL, 0x96fcc9ef4015c56bULL, 0x69c1627c690913a9ULL, - 0x7a86fd2f4733db0eULL, 0xfdb8c4f29e087de9ULL, 0x095e4b1a8ea2a229ULL, 0x1ad7a7c829b37a79ULL, - 0x342d89cad17ea0c0ULL, 0x67bedda6cced2051ULL, 0x19ca31bf2bb42f74ULL, 0x3df7b4c84980acbbULL, - 0xa8c6444dc80ad883ULL, 0xb91e440366e3ab85ULL, 0xc215cda00164f6d8ULL, 0x3d867c6ef247e668ULL, - 0xc7dd582bcc3e658cULL, 0xfd2c4748ee0e5528ULL, 0xa0fd9b95cc9f4f71ULL, 0x7529d871b0675ddfULL, - 0xb8f568b42d3cbd78ULL, 0x1233011b91f3da82ULL, 0x2dce6ccd4a7c3b62ULL, 0x75e7fc8e9e498603ULL, - 0x2f4f13f1fcd0b6ecULL, 0xf1a8ca1f29ff7a45ULL, 0xc249c1a72981e29bULL, 0x6ebe0dbb8c83b56aULL, - 0x7114fa8d170bb222ULL, 0x65a2dcd5bf93935fULL, 0xbdc41f68b59c979aULL, 0x2f0eef79a2ce9289ULL, - 0x42ecbf0c083c37ceULL, 0x2930bc09ec496322ULL, 0xf294b0c19cfeac0dULL, 0x3780aa4bedfabb80ULL, - 0x56c17d3e7cead929ULL, 0xe7cb4beb2e5722c5ULL, 0x0ce931732dbfe15aULL, 0x41b883c7621052f8ULL, - 0xdbf75ca0c3d25350ULL, 0x2936be086eb1e351ULL, 0xc936e03cb4a9b212ULL, 0x1d45bf82322225aaULL, - 0xe81ab1036a024cc5ULL, 0xe212201c304c9a72ULL, 0xc5d73fba6832b1fcULL, 0x20ffdb5a4d839581ULL, - 0xa283d367be5d0fadULL, 0x6c2b25ca8b164475ULL, 0x9d4935467caaf22eULL, 0x5166408eee85ff49ULL, - 0x3c67baa2fab4e361ULL, 0xb3e433c67ef35cefULL, 0x5259729241159b1cULL, 0x6a621892d5b0ab33ULL, - 0x20b74a387555cdcbULL, 0x532aa10e1208923fULL, 0xeaa17b7762281dd1ULL, 0x61ab3443f05c44bfULL, - 0x257a6c422324def8ULL, 0x131c6c1017e3cf7fULL, 0x23758739f630a257ULL, 0x295a407a01a78580ULL, - 0xf8c443246d5da8d9ULL, 0x19d775450c52fa5dULL, 0x2afcfc92731bf83dULL, 0x7d10c8e81b2b4700ULL, - 0xc8e0271f70baa20bULL, 0x993748867ca63957ULL, 0x5412efb3cb7ed4bbULL, 0x3196d36173e62975ULL, - 0xde5bcad141c7dffcULL, 0x47cc8cd2b395c848ULL, 0xa34cd942e11af3cbULL, 0x0256dbf2d04ecec2ULL, - 0x875ab7e94b0e667fULL, 0xcad4dd83c0850d10ULL, 0x47f12e8f4e72c79fULL, 0x5f1a87bb8c85b19bULL, - 0x7ae9d0b6437f51b8ULL, 0x12c7ce5518879065ULL, 0x2ade09fe5cf77aeeULL, 0x23a05a2f7d2c5627ULL, - 0x5908e128f17c169aULL, 0xf77498dd8ad0852dULL, 0x74b4c4ceab102f64ULL, 0x183abadd10139845ULL, - 0xb165ba8daa92aaacULL, 0xd5c5ef9599386705ULL, 0xbe2f8f0cf8fc40d1ULL, 0x2701e635ee204514ULL, - 0x629fa80020156514ULL, 0xf223868764a8c1ceULL, 0x5b894fff0b3f060eULL, 0x60d9944cf708a3faULL, - 0xaeea001a1c7a201fULL, 0xebf16a633ee2ce63ULL, 0x6f7709594c7a07e1ULL, 0x79b958150d0208cbULL, - 0x24b55e5301d410e7ULL, 0xe3a34edff3fdc84dULL, 0xd88768e4904032d8ULL, 0x131384427b3aaeecULL, - 0x8405e51286234f14ULL, 0x14dc4739adb4c529ULL, 0xb8a2b5b250634ffdULL, 0x2fe2a94ad8a7ff93ULL, - 0xec5c57efe843faddULL, 0x2843ce40f0bb9918ULL, 0xa4b561d6cf3d6305ULL, 0x743629bde8fb777eULL, - 0x343edd46bbaf738fULL, 0xed981828b101a651ULL, 0xa401760b882c797aULL, 0x1fc223e28dc88730ULL, - 0x48604e91fc0fba0eULL, 0xb637f78f052c6fa4ULL, 0x91ccac3d09e9239cULL, 0x23f7eed4437a687cULL, - 0x5173b1118d9bd800ULL, 0x29d641b63189d4a7ULL, 0xfdbf177988bbc586ULL, 0x2959894fcad81df5ULL, - 0xaebc8ef3b4bbc899ULL, 0x4148995ab26992b9ULL, 0x24e20b0134f92cfbULL, 0x40d158894a05dee8ULL, - 0x46b00b1185af76f6ULL, 0x26bac77873187a79ULL, 0x3dc0bf95ab8fff5fULL, 0x2a608bd8945524d7ULL, - 0x26449588bd446302ULL, 0x7c4bc21c0388439cULL, 0x8e98a4f383bd11b2ULL, 0x26218d7bc9d876b9ULL, - 0xe3081542997c178aULL, 0x3c2d29a86fb6606fULL, 0x5c217736fa279374ULL, 0x7dde05734afeb1faULL, - 0x3bf10e3906d42babULL, 0xe4f7803e1980649cULL, 0xe6053bf89595bf7aULL, 0x394faf38da245530ULL, - 0x7a8efb58896928f4ULL, 0xfbc778e9cc6a113cULL, 0x72670ce330af596fULL, 0x48f222a81d3d6cf7ULL, - 0xf01fce410d72caa7ULL, 0x5a20ecc7213b5595ULL, 0x7bc21165c1fa1483ULL, 0x07f89ae31da8a741ULL, - 0x05d2c2b4c6830ff9ULL, 0xd43e330fc6316293ULL, 0xa5a5590a96d3a904ULL, 0x705edb91a65333b6ULL, - 0x048ee15e0bb9a5f7ULL, 0x3240cfca9e0aaf5dULL, 0x8f4b71ceedc4a40bULL, 0x621c0da3de544a6dULL, - 0x92872836a08c4091ULL, 0xce8375b010c91445ULL, 0x8a72eb524f276394ULL, 0x2667fcfa7ec83635ULL, - 0x7f4c173345e8752aULL, 0x061b47feee7079a5ULL, 0x25dd9afa9f86ff34ULL, 0x3780cef5425dc89cULL, - 0x1a46035a513bb4e9ULL, 0x3e1ef379ac575adaULL, 0xc78c5f1c5fa24b50ULL, 0x321a967634fd9f22ULL, - 0x946707b8826e27faULL, 0x3dca84d64c506fd0ULL, 0xc189218075e91436ULL, 0x6d9284169b3b8484ULL, - 0x3a67e840383f2ddfULL, 0x33eec9a30c4f9b75ULL, 0x3ec7c86fa783ef47ULL, 0x26ec449fbac9fbc4ULL, - 0x5c0f38cba09b9e7dULL, 0x81168cc762a3478cULL, 0x3e23b0d306fc121cULL, 0x5a238aa0a5efdcddULL, - 0x1ba26121c4ea43ffULL, 0x36f8c77f7c8832b5ULL, 0x88fbea0b0adcf99aULL, 0x5ca9938ec25bebf9ULL, - 0xd5436a5e51fccda0ULL, 0x1dbc4797c2cd893bULL, 0x19346a65d3224a08ULL, 0x0f5034e49b9af466ULL, - 0xf23c3967a1e0b96eULL, 0xe58b08fa867a4d88ULL, 0xfb2fabc6a7341679ULL, 0x2a75381eb6026946ULL, - 0xc80a3be4c19420acULL, 0x66b1f6c681f2b6dcULL, 0x7cf7036761e93388ULL, 0x25abbbd8a660a4c4ULL, - 0x91ea12ba14fd5198ULL, 0x684950fc4a3cffa9ULL, 0xf826842130f5ad28ULL, 0x3ea988f75301a441ULL, - 0xc978109a695f8c6fULL, 0x1746eb4a0530c3f3ULL, 0x444d6d77b4459995ULL, 0x75952b8c054e5cc7ULL, - 0xa3703f7915f4d6aaULL, 0x66c346202f2647d8ULL, 0xd01469df811d644bULL, 0x77fea47d81a5d71fULL, - 0xc5e9529ef57ca381ULL, 0x6eeeb4b9ce2f881aULL, 0xb6e91a28e8009bd6ULL, 0x4b80be3e9afc3fecULL, - 0x7e3773c526aed2c5ULL, 0x1b4afcb453c9a49dULL, 0xa920bdd7baffb24dULL, 0x7c54699f122d400eULL, - 0xef46c8e14fa94bc8ULL, 0xe0b074ce2952ed5eULL, 0xbea450e1dbd885d5ULL, 0x61b68649320f712cULL, - 0x8a485f7309ccbdd1ULL, 0xbd06320d7d4d1a2dULL, 0x25232973322dbef4ULL, 0x445dc4758c17f770ULL, - 0xdb0434177cc8933cULL, 0xed6fe82175ea059fULL, 0x1efebefdc053db34ULL, 0x4adbe867c65daf99ULL, - 0x3acd71a2a90609dfULL, 0xe5e991856dd04050ULL, 0x1ec69b688157c23cULL, 0x697427f6885cfe4dULL, - 0xd7be7b9b65e1a851ULL, 0xa03d28d522c536ddULL, 0x28399d658fd2b645ULL, 0x49e5b7e17c2641e1ULL, - 0x6f8c3a98700457a4ULL, 0x5078f0a25ebb6778ULL, 0xd13c3ccbc382960fULL, 0x2e003258a7df84b1ULL, - 0x8ad1f39be6296a1cULL, 0xc1eeaa652a5fbfb2ULL, 0x33ee0673fd26f3cbULL, 0x59256173a69d2cccULL, - 0x41ea07aa4e18fc41ULL, 0xd9fc19527c87a51eULL, 0xbdaacb805831ca6fULL, 0x445b652dc916694fULL, - 0xce92a3a7f2172315ULL, 0x1edc282de11b9964ULL, 0xa1823aafe04c314aULL, 0x790a2d94437cf586ULL, - 0x71c447fb93f6e009ULL, 0x8922a56722845276ULL, 0xbf70903b204f5169ULL, 0x2f7a89891ba319feULL, - 0x02a08eb577e2140cULL, 0xed9a4ed4427bdcf4ULL, 0x5253ec44e4323cd1ULL, 0x3e88363c14e9355bULL, - 0xaa66c14277110b8cULL, 0x1ae0391610a23390ULL, 0x2030bd12c93fc2a2ULL, 0x3ee141579555c7abULL, - 0x9214de3a6d6e7d41ULL, 0x3ccdd88607f17efeULL, 0x674f1288f8e11217ULL, 0x5682250f329f93d0ULL, - 0x6cf00b136d2e396eULL, 0x6e4cf86f1014debfULL, 0x5930b1b5bfcc4e83ULL, 0x047069b48aba16b6ULL, - 0x0d4ce4ab69b20793ULL, 0xb24db91a97d0fb9eULL, 0xcdfa50f54e00d01dULL, 0x221b1085368bddb5ULL, - 0xe7e59468b1e3d8d2ULL, 0x53c56563bd122f93ULL, 0xeee8a903e0663f09ULL, 0x61efa662cbbe3d42ULL, - 0x2cf8ddddde6eab2aULL, 0x9bf80ad51435f231ULL, 0x5deadacec9f04973ULL, 0x29275b5d41d29b27ULL, - 0xcfde0f0895ebf14fULL, 0xb9aab96b054905a7ULL, 0xcae80dd9a1c420fdULL, 0x0a63bf2f1673bbc7ULL, - 0x092f6e11958fbc8cULL, 0x672a81e804822fadULL, 0xcac8351560d52517ULL, 0x6f3f7722c8f192f8ULL, - 0xf8ba90ccc2e894b7ULL, 0x2c7557a438ff9f0dULL, 0x894d1d855ae52359ULL, 0x68e122157b743d69ULL, - 0xd87e5570cfb919f3ULL, 0x3f2cdecd95798db9ULL, 0x2121154710c0a2ceULL, 0x3c66a115246dc5b2ULL, - 0xcbedc562294ecb72ULL, 0xba7143c36a280b16ULL, 0x9610c2efd4078b67ULL, 0x6144735d946a4b1eULL, - 0x536f111ed75b3350ULL, 0x0211db8c2041d81bULL, 0xf93cb1000e10413cULL, 0x149dfd3c039e8876ULL, - 0xd479dde46b63155bULL, 0xb66e15e93c837976ULL, 0xdafde43b1f13e038ULL, 0x5fafda1a2e4b0b35ULL, - 0x3600bbdf17197581ULL, 0x3972050bbe3cd2c2ULL, 0x5938906dbdd5be86ULL, 0x34fce5e43f9b860fULL, - 0x75a8a4cd42d14d02ULL, 0x828dabc53441df65ULL, 0x33dcabedd2e131d3ULL, 0x3ebad76fb814d25fULL, - 0xd4906f566f70e10fULL, 0x5d12f7aa51690f5aULL, 0x45adb16e76cefcf2ULL, 0x01f768aead232999ULL, - 0x2b6cc77b6248febdULL, 0x3cd30628ec3aaffdULL, 0xce1c0b80d4ef486aULL, 0x4c3bff2ea6f66c23ULL, - 0x3f2ec4094aeaeb5fULL, 0x61b19b286e372ca7ULL, 0x5eefa966de2a701dULL, 0x23b20565de55e3efULL, - 0xe301ca5279d58557ULL, 0x07b2d4ce27c2874fULL, 0xa532cd8a9dcf1d67ULL, 0x2a52fee23f2bff56ULL, - 0x8624efb37cd8663dULL, 0xbbc7ac20ffbd7594ULL, 0x57b85e9c82d37445ULL, 0x7b3052cb86a6ec66ULL, - 0x3482f0ad2525e91eULL, 0x2cb68043d28edca0ULL, 0xaf4f6d052e1b003aULL, 0x185f8c2529781b0aULL, - 0xaa41de5bd80ce0d6ULL, 0x9407b2416853e9d6ULL, 0x563ec36e357f4c3aULL, 0x4cc4b8dd0e297bceULL, - 0xa2fc1a52ffb8730eULL, 0x1811f16e67058e37ULL, 0x10f9a366cddf4ee1ULL, 0x72f4a0c4a0b9f099ULL, - 0x8c16c06f663f4ea7ULL, 0x693b3af74e970fbaULL, 0x2102e7f1d69ec345ULL, 0x0ba53cbc968a8089ULL, - 0xca3d9dc7fea15537ULL, 0x4c6824bb51536493ULL, 0xb9886314844006b1ULL, 0x40d2a72ab454cc60ULL, - 0x5936a1b712570975ULL, 0x91b9d648debda657ULL, 0x3344094bb64330eaULL, 0x006ba10d12ee51d0ULL, - 0x19228468f5de5d58ULL, 0x0eb12f4c38cc05b0ULL, 0xa1039f9dd5601990ULL, 0x4502d4ce4fff0e0bULL, - 0xeb2054106837c189ULL, 0xd0f6544c6dd3b93cULL, 0x40727064c416d74fULL, 0x6e15c6114b502ef0ULL, - 0x4df2a398cfb1a76bULL, 0x11256c7419f2f6b1ULL, 0x4a497962066e6043ULL, 0x705b3aab41355b44ULL, - 0x365ef536d797b1d8ULL, 0x00076bd622ddf0dbULL, 0x3bbf33b0e0575a88ULL, 0x3777aa05c8e4ca4dULL, - 0x392745c85578db5fULL, 0x6fda4149dbae5ae2ULL, 0xb1f0b00b8adc9867ULL, 0x09963437d36f1da3ULL, - 0x7e824e90a5dc3853ULL, 0xccb5f6641f135cbdULL, 0x6736d86c87ce8fccULL, 0x625f3ce26604249fULL, - 0xaf8ac8059502f63fULL, 0x0c05e70a2e351469ULL, 0x35292e9c764b6305ULL, 0x1a394360c7e23ac3ULL, - 0xd5c6d53251183264ULL, 0x62065abd43c2b74fULL, 0xb5fbf5d03b973f9bULL, 0x13a3da3661206e5eULL, - 0xc6bd5837725d94e5ULL, 0x18e30912205016c5ULL, 0x2088ce1570033c68ULL, 0x7fba1f495c837987ULL, - 0x5a8c7423f2f9079dULL, 0x1735157b34023fc5ULL, 0xe4f9b49ad2fab351ULL, 0x6691ff72c878e33cULL, - 0x122c2adedc5eff3eULL, 0xf8dd4bf1d8956cf4ULL, 0xeb86205d9e9e5bdaULL, 0x049b92b9d975c743ULL, - 0xa5379730b0f6c05aULL, 0x72a0ffacc6f3a553ULL, 0xb0032c34b20dcd6dULL, 0x470e9dbc88d5164aULL, - 0xb19cf10ca237c047ULL, 0xb65466711f6c81a2ULL, 0xb3321bd16dd80b43ULL, 0x48c14f600c5fbe8eULL, - 0x66451c264aa6c803ULL, 0xb66e3904a4fa7da6ULL, 0xd45f19b0b3128395ULL, 0x31602627c3c9bc10ULL, - 0x3120dc4832e4e10dULL, 0xeb20c46756c717f7ULL, 0x00f52e3f67280294ULL, 0x566d4fc14730c509ULL, - 0x7e3a5d40fd837206ULL, 0xc1e926dc7159547aULL, 0x216730fba68d6095ULL, 0x22e8c3843f69cea7ULL, - 0x33d074e8930e4b2bULL, 0xb6e4350e84d15816ULL, 0x5534c26ad6ba2365ULL, 0x7773c12f89f1f3f3ULL, - 0x8cba404da57962aaULL, 0x5b9897a81999ce56ULL, 0x508e862f121692fcULL, 0x3a81907fa093c291ULL, - 0x0dded0ff4725a510ULL, 0x10d8cc10673fc503ULL, 0x5b9d151c9f1f4e89ULL, 0x32a5c1d5cb09a44cULL, - 0x1e0aa442b90541fbULL, 0x5f85eb7cc1b485dbULL, 0xbee595ce8a9df2e5ULL, 0x25e496c722422236ULL, - 0x5edf3c46cd0fe5b9ULL, 0x34e75a7ed2a43388ULL, 0xe488de11d761e352ULL, 0x0e878a01a085545cULL, - 0xba493c77e021bb04ULL, 0x2b4d1843c7df899aULL, 0x9ea37a487ae80d67ULL, 0x67a9958011e41794ULL, - 0x4b58051a6697b065ULL, 0x47e33f7d8d6ba6d4ULL, 0xbb4da8d483ca46c1ULL, 0x68becaa181c2db0dULL, - 0x8d8980e90b989aa5ULL, 0xf95eb14a2c93c99bULL, 0x51c6c7c4796e73a2ULL, 0x6e228363b5efb569ULL, - 0xc6bbc0b02dd624c8ULL, 0x777eb47dec8170eeULL, 0x3cde15a004cfafa9ULL, 0x1dc6bc087160bf9bULL, - 0x2e07e043eec34002ULL, 0x18e9fc677a68dc7fULL, 0xd8da03188bd15b9aULL, 0x48fbc3bb00568253ULL, - 0x57547d4cfb654ce1ULL, 0xd3565b82a058e2adULL, 0xf63eaf0bbf154478ULL, 0x47531ef114dfbb18ULL, - 0xe1ec630a4278c587ULL, 0x5507d546ca8e83f3ULL, 0x85e135c63adc0c2bULL, 0x0aa7efa85682844eULL, - 0x72691ba8b3e1f615ULL, 0x32b4e9701fbe3ffaULL, 0x97b6d92e39bb7868ULL, 0x2cfe53dea02e39e8ULL, - 0x687392cd85cd52b0ULL, 0x27ff66c910e29831ULL, 0x97134556a9832d06ULL, 0x269bb0360a84f8a0ULL, - 0x706e55457643f85cULL, 0x3734a48c9b597d1bULL, 0x7aee91e8c6efa472ULL, 0x5cd6abc198a9d9e0ULL, - 0x0e04de06cb3ce41aULL, 0xd8c6eb893402e138ULL, 0x904659bb686e3772ULL, 0x7215c371746ba8c8ULL, - 0xfd12a97eeae4a2d9ULL, 0x9514b7516394f2c5ULL, 0x266fd5809208f294ULL, 0x5c847085619a26b9ULL, - 0x52985410fed694eaULL, 0x3c905b934a2ed254ULL, 0x10bb47692d3be467ULL, 0x063b3d2d69e5e9e1ULL, - 0x472726eedda57debULL, 0xefb6c4ae10f41891ULL, 0x2b1641917b307614ULL, 0x117c554fc4f45b7cULL, - 0xc07cf3118f9d8812ULL, 0x01dbd82050017939ULL, 0xd7e803f4171b2827ULL, 0x1015e87487d225eaULL, - 0xc58de3fed23acc4dULL, 0x50db91c294a7be2dULL, 0x0b94d43d1c9cf457ULL, 0x6b1640fa6e37524aULL, - 0x692f346c5fda0d09ULL, 0x200b1c59fa4d3151ULL, 0xb8c46f760777a296ULL, 0x4b38395f3ffdfbcfULL, - 0x18d25e00be54d671ULL, 0x60d50582bec8aba6ULL, 0x87ad8f263b78b982ULL, 0x50fdf64e9cda0432ULL, - 0x90f567aac578dcf0ULL, 0xef1e9b0ef2a3133bULL, 0x0eebba9242d9de71ULL, 0x15473c9bf03101c7ULL, - 0x7c77e8ae56b78095ULL, 0xb678e7666e6f078eULL, 0x2da0b9615348ba1fULL, 0x7cf931c1ff733f0bULL, - 0x26b357f50a0a366cULL, 0xe9708cf42b87d732ULL, 0xc13aeea5f91cb2c0ULL, 0x35d90c991143bb4cULL, - 0x47c1c404a9a0d9dcULL, 0x659e58451972d251ULL, 0x3875a8c473b38c31ULL, 0x1fbd9ed379561f24ULL, - 0x11fabc6fd41ec28dULL, 0x7ef8dfe3cd2a2dcaULL, 0x72e73b5d8c404595ULL, 0x6135fa4954b72f27ULL, - 0xccfc32a2de24b69cULL, 0x3f55698c1f095d88ULL, 0xbe3350ed5ac3f929ULL, 0x5e9bf806ca477eebULL, - 0xe9ce8fb63c309f68ULL, 0x5376f63565e1f9f4ULL, 0xd1afcfb35a6393f1ULL, 0x6632a1ede5623506ULL, - 0x0b7d6c390c2ded4cULL, 0x56cb3281df04cb1fULL, 0x66305a1249ecc3c7ULL, 0x5d588b60a38ca72aULL, - 0xa6ecbf78e8e5f42dULL, 0x86eeb44b3c8a3eecULL, 0xec219c48fbd21604ULL, 0x1aaf1af517c36731ULL, - 0xc306a2836769bde7ULL, 0x208280622b1e2adbULL, 0x8027f51ffbff94a6ULL, 0x76cfa1ce1124f26bULL, - 0x18eb00562422abb6ULL, 0xf377c4d58f8c29c3ULL, 0x4dbbc207f531561aULL, 0x0253b7f082128a27ULL, - 0x3d1f091cb62c17e0ULL, 0x4860e1abd64628a9ULL, 0x52d17436309d4253ULL, 0x356f97e13efae576ULL, - 0xd351e11aa150535bULL, 0x3e6b45bb1dd878ccULL, 0x0c776128bed92c98ULL, 0x1d34ae93032885b8ULL, - 0x4ba0488ca85ba4c3ULL, 0x985348c33c9ce6ceULL, 0x66124c6f97bda770ULL, 0x0f81a0290654124aULL, - 0x9ed09ca6569b86fdULL, 0x811009fd18af9a2dULL, 0xff08d03f93d8c20aULL, 0x52a148199faef26bULL, - 0x3e03f9dc2d8d1b73ULL, 0x4205801873961a70ULL, 0xc0d987f041a35970ULL, 0x07aa1f15a1c0d549ULL, - 0xdfd46ce08cd27224ULL, 0x6d0a024f934e4239ULL, 0x808a7a6399897b59ULL, 0x0a4556e9e13d95a2ULL, - 0xd21a991fe9c13045ULL, 0x9b0e8548fe7751b8ULL, 0x5da643cb4bf30035ULL, 0x77db28d63940f721ULL, - 0xfc5eeb614adc9011ULL, 0x5229419ae8c411ebULL, 0x9ec3e7787d1dcf74ULL, 0x340d053e216e4cb5ULL, - 0xcac7af39b48df2b4ULL, 0xc0faec2871a10a94ULL, 0x140a69245ca575edULL, 0x0cf1c37134273a4cULL, - 0xc8ee306ac224b8a5ULL, 0x57eaee7ccb4930b0ULL, 0xa1e806bdaacbe74fULL, 0x7d9a62742eeb657dULL, - 0x9eb6b6ef546c4830ULL, 0x885cca1fddb36e2eULL, 0xe6b9f383ef0d7105ULL, 0x58654fef9d2e0412ULL, - 0xa905c4ffbe0e8e26ULL, 0x942de5df9b31816eULL, 0x497d723f802e88e1ULL, 0x30684dea602f408dULL, - 0x21e5a278a3e6cb34ULL, 0xaefb6e6f5b151dc4ULL, 0xb30b8e049d77ca15ULL, 0x28c3c9cf53b98981ULL, - 0x287fb721556cdd2aULL, 0x0d317ca897022274ULL, 0x7468c7423a543258ULL, 0x4a7f11464eb5642fULL, - 0xa237a4774d193aa6ULL, 0xd865986ea92129a1ULL, 0x24c515ecf87c1a88ULL, 0x604003575f39f5ebULL, - 0x47b9f189570a9b27ULL, 0x2b98cede465e4b78ULL, 0x026df551dbb85c20ULL, 0x74fcd91047e21901ULL, - 0x13e2a90a23c1bfa3ULL, 0x0cb0074e478519f6ULL, 0x5ff1cbbe3af6cf44ULL, 0x67fe5438be812dbeULL, - 0xd13cf64fa40f05b0ULL, 0x054dfb2f32283787ULL, 0x4173915b7f0d2aeaULL, 0x482f144f1f610d4eULL, - 0xf6210201b47f8234ULL, 0x5d0ae1929e70b990ULL, 0xdcd7f455b049567cULL, 0x7e93d0f1f0916f01ULL, - 0xdd79cbf18a7db4faULL, 0xbe8391bf6f74c62fULL, 0x027145d14b8291bdULL, 0x585a73ea2cbf1705ULL, - 0x485ca03e928a0db2ULL, 0x10fc01a5742857e7ULL, 0x2f482edbd6d551a7ULL, 0x0f0433b5048fdb8aULL, - 0x60da2e8dd7dc6247ULL, 0x88b4c9d38cd4819aULL, 0x13033ac001f66697ULL, 0x273b24fe3b367d75ULL, - 0xc6e8f66a31b3b9d4ULL, 0x281514a494df49d5ULL, 0xd1726fdfc8b23da7ULL, 0x4b3ae7d103dee548ULL, - 0xc6256e19ce4b9d7eULL, 0xff5c5cf186e3c61cULL, 0xacc63ca34b8ec145ULL, 0x74621888fee66574ULL, - 0x956f409645290a1eULL, 0xef0bf8e3263a962eULL, 0xed6a50eb5ec2647bULL, 0x0694283a9dca7502ULL, - 0x769b963643a2dcd1ULL, 0x42b7c8ea09fc5353ULL, 0x4f002aee13397eabULL, 0x63005e2c19b7d63aULL, - 0xca6736da63023beaULL, 0x966c7f6db12a99b7ULL, 0xace09390c537c5e1ULL, 0x0b696063a1aa89eeULL, - 0xebb03e97288c56e5ULL, 0x432a9f9f938c8be8ULL, 0xa6a5a93d5b717f71ULL, 0x1a5fb4c3e18f9d97ULL, - 0x1c94e7ad1c60cdceULL, 0xee202a43fc02c4a0ULL, 0x8dafe4d867c46a20ULL, 0x0a10263c8ac27b58ULL, - 0xd0dea9dfe4432a4aULL, 0x856af87bbe9277c5ULL, 0xce8472acc212c71aULL, 0x6f151b6d9bbb1e91ULL, - 0x26776c527ceed56aULL, 0x7d211cb7fbf8faecULL, 0x37ae66a6fd4609ccULL, 0x1f81b702d2770c42ULL, - 0x2fb0b057eac58392ULL, 0xe1dd89fe29744e9dULL, 0xc964f8eb17beb4f8ULL, 0x29571073c9a2d41eULL, - 0xa948a18981c0e254ULL, 0x2df6369b65b22830ULL, 0xa33eb2d75fcfd3c6ULL, 0x078cd6ec4199a01fULL, - 0x4a584a41ad900d2fULL, 0x32142b78e2c74c52ULL, 0x68c4e8338431c978ULL, 0x7f69ea9008689fc2ULL, - 0x52f2c81e46a38265ULL, 0xfd78072d04a832fdULL, 0x8cd7d5fa25359e94ULL, 0x4de71b7454cc29d2ULL, - 0x42eb60ad1eda6ac9ULL, 0x0aad37dfdbc09c3aULL, 0x81004b71e33cc191ULL, 0x44e6be345122803cULL, - 0x03fe8388ba1920dbULL, 0xf5d57c32150db008ULL, 0x49c8c4281af60c29ULL, 0x21edb518de701aeeULL, - 0x7fb63e418f06dc99ULL, 0xa4460d99c166d7b8ULL, 0x24dd5248ce520a83ULL, 0x5ec3ad712b928358ULL, - 0x15022a5fbd17930fULL, 0xa4f64a77d82570e3ULL, 0x12bc8d6915783712ULL, 0x498194c0fc620abbULL, - 0x38a2d9d255686c82ULL, 0x785c6bd9193e21f0ULL, 0xe4d5c81ab24a5484ULL, 0x56307860b2e20989ULL, - 0x429d55f78b4d74c4ULL, 0x22f1834643350131ULL, 0x1e60c24598c71fffULL, 0x59f2f014979983efULL, - 0x46a47d56eb494a44ULL, 0x3e22a854d636a18eULL, 0xb346e15274491c3bULL, 0x2ceafd4e5390cde7ULL, - 0xba8a8538be0d6675ULL, 0x4b9074bb50818e23ULL, 0xcbdab89085d304c3ULL, 0x61a24fe0e56192c4ULL, - 0xcb7615e6db525bcbULL, 0xdd7d8c35a567e4caULL, 0xe6b4153acafcdd69ULL, 0x2d668e097f3c9766ULL, - 0xa57e7e265ce55ef0ULL, 0x5d9f4e527cd4b967ULL, 0xfbc83606492fd1e5ULL, 0x090d52beb7c3f7aeULL, - 0x09b9515a1e7b4d7cULL, 0x1f266a2599da44c0ULL, 0xa1c49548e2c55504ULL, 0x7ef04287126f15ccULL, - 0xfed1659dbd30ef15ULL, 0x8b4ab9eec4e0277bULL, 0x884d6236a5df3291ULL, 0x1fd96ea6bf5cf788ULL, - 0x42a161981f190d9aULL, 0x61d849507e6052c1ULL, 0x9fe113bf285a2cd5ULL, 0x7c22d676dbad85d8ULL, - 0x82e770ed2bfbd27dULL, 0x4c05b2ece996f5a5ULL, 0xcd40a9c2b0900150ULL, 0x5895319213d9bf64ULL, - 0xe7cc5d703fea2e08ULL, 0xb50c491258e2188cULL, 0xcce30baa48205bf0ULL, 0x537c659ccfa32d62ULL, - 0x37b6623a98cfc088ULL, 0xfe9bed1fa4d6aca4ULL, 0x04d29b8e56a8d1b0ULL, 0x725f71c40b519575ULL, - 0x28c7f89cd0339ce6ULL, 0x8367b14469ddc18bULL, 0x883ada83a6a1652cULL, 0x585f1974034d6c17ULL, - 0x89cfb266f1b19188ULL, 0xe63b4863e7c35217ULL, 0xd88c9da6b4c0526aULL, 0x3e035c9df0954635ULL, - 0xdd9d5412fb45de9dULL, 0xdd684532e4cff40dULL, 0x4b5c999b151d671cULL, 0x2d8c2cc811e7f690ULL, - 0x7f54be1d90055d40ULL, 0xa464c5df464aaf40ULL, 0x33979624f0e917beULL, 0x2c018dc527356b30ULL, - 0xa5415024e330b3d4ULL, 0x73ff3d96691652d3ULL, 0x94ec42c4ef9b59f1ULL, 0x0747201618d08e5aULL, - 0x4d6ca48aca411c53ULL, 0x66415f2fcfa66119ULL, 0x9c4dd40051e227ffULL, 0x59810bc09a02f7ebULL, - 0x2a7eb171b3dc101dULL, 0x441c5ab99ffef68eULL, 0x32025c9b93b359eaULL, 0x5e8ce0a71e9d112fULL, - 0xbfcccb92429503fdULL, 0xd271ba752f095d55ULL, 0x345ead5e972d091eULL, 0x18c8df11a83103baULL, - 0x90cd949a9aed0f4cULL, 0xc5d1f4cb6660e37eULL, 0xb8cac52d56c52e0bULL, 0x6e42e400c5808e0dULL, - 0xa3b46966eeaefd23ULL, 0x0c4f1f0be39ecdcaULL, 0x189dc8c9d683a51dULL, 0x51f27f054c09351bULL, - 0x4c487ccd2a320682ULL, 0x587ea95bb3df1c96ULL, 0xc8ccf79e555cb8e8ULL, 0x547dc829a206d73dULL, - 0xb822a6cd80c39b06ULL, 0xe96d54732000d4c6ULL, 0x28535b6f91463b4dULL, 0x228f4660e2486e1dULL, - 0x98799538de8d3abfULL, 0x8cd8330045ebca6eULL, 0x79952a008221e738ULL, 0x4322e1a7535cd2bbULL, - 0xb114c11819d1801cULL, 0x2016e4d84f3f5ec7ULL, 0xdd0e2df409260f4cULL, 0x5ec362c0ae5f7266ULL, - 0xc0462b18b8b2b4eeULL, 0x7cc8d950274d1afbULL, 0xf25f7105436b02d2ULL, 0x43bbf8dcbff9ccd3ULL, - 0xb6ad1767a039e9dfULL, 0xb0714da8f69d3583ULL, 0x5e55fa18b42931f5ULL, 0x4ed5558f33c60961ULL, - 0x1fe37901c647a5ddULL, 0x593ddf1f8081d357ULL, 0x0249a4fd813fd7a6ULL, 0x69acca274e9caf61ULL, - 0x047ba3ea330721c9ULL, 0x83423fc20e7e1ea0ULL, 0x1df4c0af01314a60ULL, 0x09a62dab89289527ULL, - 0xa5b325a49cc6cb00ULL, 0xe94b5dc654b56cb6ULL, 0x3be28779adc994a0ULL, 0x4296e8f8ba3a4aadULL, - 0x328689761e451eabULL, 0x2e4d598bff59594aULL, 0x49b96853d7a7084aULL, 0x4980a319601420a8ULL, - 0x9565b9e12f552c42ULL, 0x8a5318db7100fe96ULL, 0x05c90b4d43add0d7ULL, 0x538b4cd66a5d4edaULL, - 0xf4e94fc3e89f039fULL, 0x592c9af26f618045ULL, 0x08a36eb5fd4b9550ULL, 0x25fffaf6c2ed1419ULL, - 0x34434459cc79d354ULL, 0xeeecbfb4b1d5476bULL, 0xddeb34a061615d99ULL, 0x5129cecceb64b773ULL, - 0xee43215894993520ULL, 0x772f9c7cf14c0b3bULL, 0xd2e2fce306bedad5ULL, 0x715f42b546f06a97ULL, - 0x434ecdceda5b5f1aULL, 0x0da17115a49741a9ULL, 0x680bd77c73edad2eULL, 0x487c02354edd9041ULL, - 0xb8efeff3a70ed9c4ULL, 0x56a32aa3e857e302ULL, 0xdf3a68bd48a2a5a0ULL, 0x07f650b73176c444ULL, - 0xe38b9b1626e0ccb1ULL, 0x79e053c18b09fb36ULL, 0x56d90319c9f94964ULL, 0x1ca941e7ac9ff5c4ULL, - 0x49c4df29162fa0bbULL, 0x8488cf3282b33305ULL, 0x95dfda14cabb437dULL, 0x3391f78264d5ad86ULL, - 0x729ae06ae2b5095dULL, 0xd58a58d73259a946ULL, 0xe9834262d13921edULL, 0x27fedafaa54bb592ULL, - 0xa99dc5b829ad48bbULL, 0x5f025742499ee260ULL, 0x802c8ecd5d7513fdULL, 0x78ceb3ef3f6dd938ULL, - 0xc342f44f8a135d94ULL, 0x7b9edb44828cdda3ULL, 0x9436d11a0537cfe7ULL, 0x5064b164ec1ab4c8ULL, - 0x7020eccfd37eb2fcULL, 0x1f31ea3ed90d25fcULL, 0x1b930d7bdfa1bb34ULL, 0x5344467a48113044ULL, - 0x70073170f25e6dfbULL, 0xe385dc1a50114cc8ULL, 0x2348698ac8fc4f00ULL, 0x2a77a55284dd40d8ULL, - 0xfe06afe0c98c6ce4ULL, 0xc235df96dddfd6e4ULL, 0x1428d01e33bf1ed3ULL, 0x785768ec9300bdafULL, - 0x9702e57a91deb63bULL, 0x61bdb8bfe5ce8b80ULL, 0x645b426f3d1d58acULL, 0x4804a82227a557bcULL, - 0x8e57048ab44d2601ULL, 0x68d6501a4b3a6935ULL, 0xc39c9ec3f9e1c293ULL, 0x4172f257d4de63e2ULL, - 0xd368b450330c6401ULL, 0x040d3017418f2391ULL, 0x2c34bb6090b7d90dULL, 0x16f649228fdfd51fULL, - 0xbea6818e2b928ef5ULL, 0xe28ccf91cdc11e72ULL, 0x594aaa68e77a36cdULL, 0x313034806c7ffd0fULL, - 0x8a9d27ac2249bd65ULL, 0x19a3b464018e9512ULL, 0xc26ccff352b37ec7ULL, 0x056f68341d797b21ULL, - 0x5e79d6757efd2327ULL, 0xfabdbcb6553afe15ULL, 0xd3e7222c6eaf5a60ULL, 0x7046c76d4dae743bULL, - 0x660be872b18d4a55ULL, 0x19992518574e1496ULL, 0xc103053a302bdcbbULL, 0x3ed8e9800b218e8eULL, - 0x7b0b9239fa75e03eULL, 0xefe9fb684633c083ULL, 0x98a35fbe391a7793ULL, 0x6065510fe2d0fe34ULL, - 0x55cb668548abad0cULL, 0xb4584548da87e527ULL, 0x2c43ecea0107c1ddULL, 0x526028809372de35ULL, - 0x3415c56af9213b1fULL, 0x5bee1a4d017e98dbULL, 0x13f6b105b5cf709bULL, 0x5ff20e3482b29ab6ULL, - 0x0aa29c75cc2e6c90ULL, 0xfc7d73ca3a70e206ULL, 0x899fc38fc4b5c515ULL, 0x250386b124ffc207ULL, - 0x54ea28d5ae3d2b56ULL, 0x9913149dd6de60ceULL, 0x16694fc58f06d6c1ULL, 0x46b23975eb018fc7ULL, - 0x470a6a0fb4b7b4e2ULL, 0x5d92475a8f7253deULL, 0xabeee5b52fbd3adbULL, 0x7fa20801a0806968ULL, - 0x76f3faf19f7714d2ULL, 0xb3e840c12f4660c3ULL, 0x0fb4cd8df212744eULL, 0x4b065a251d3a2dd2ULL, - 0x5cebde383d77cd4aULL, 0x6adf39df882c9cb1ULL, 0xa2dd242eb09af759ULL, 0x3147c0e50e5f6422ULL, - 0x164ca5101d1350dbULL, 0xf8d13479c33fc962ULL, 0xe640ce4d13e5da08ULL, 0x4bdee0c45061f8baULL, - 0xd7c46dc1a4edb1c9ULL, 0x5514d7b6437fd98aULL, 0x58942f6bb2a1c00bULL, 0x2dffb2ab1d70710eULL, - 0xccdfcf2fc18b6d68ULL, 0xa8ebcba8b7806167ULL, 0x980697f95e2937e3ULL, 0x02fbba1cd0126e8cULL -}; + inv_eltfp25519_1w_bmi2(A, Qz); + mul_eltfp25519_1w_bmi2((u64 *)shared, Qx, A); + fred_eltfp25519_1w((u64 *)shared); -static void curve25519_ever64_base(u8 *out, const u8 *priv) + memzero_explicit(&m, sizeof(m)); +} + +static void curve25519_bmi2_base(u8 session_key[CURVE25519_KEY_SIZE], + const u8 private_key[CURVE25519_KEY_SIZE]) { + struct { + u64 buffer[4 * NUM_WORDS_ELTFP25519]; + u64 coordinates[4 * NUM_WORDS_ELTFP25519]; + u64 workspace[4 * NUM_WORDS_ELTFP25519]; + u8 private[CURVE25519_KEY_SIZE]; + } __aligned(32) m; + + const int ite[4] = { 64, 64, 64, 63 }; + const int q = 3; u64 swap = 1; - int i, j, k; - u64 tmp[16 + 32 + 4]; - u64 *x1 = &tmp[0]; - u64 *z1 = &tmp[4]; - u64 *x2 = &tmp[8]; - u64 *z2 = &tmp[12]; - u64 *xz1 = &tmp[0]; - u64 *xz2 = &tmp[8]; - u64 *a = &tmp[0 + 16]; - u64 *b = &tmp[4 + 16]; - u64 *c = &tmp[8 + 16]; - u64 *ab = &tmp[0 + 16]; - u64 *abcd = &tmp[0 + 16]; - u64 *ef = &tmp[16 + 16]; - u64 *efgh = &tmp[16 + 16]; - u64 *key = &tmp[0 + 16 + 32]; - - memcpy(key, priv, 32); - ((u8 *)key)[0] &= 248; - ((u8 *)key)[31] = (((u8 *)key)[31] & 127) | 64; - - x1[0] = 1, x1[1] = x1[2] = x1[3] = 0; - z1[0] = 1, z1[1] = z1[2] = z1[3] = 0; - z2[0] = 1, z2[1] = z2[2] = z2[3] = 0; - memcpy(x2, p_minus_s, sizeof(p_minus_s)); - - j = 3; - for (i = 0; i < 4; ++i) { - while (j < (const int[]){ 64, 64, 64, 63 }[i]) { - u64 bit = (key[i] >> j) & 1; - k = (64 * i + j - 3); + + int i = 0, j = 0, k = 0; + u64 *const key = (u64 *)m.private; + u64 *const Ur1 = m.coordinates + 0; + u64 *const Zr1 = m.coordinates + 4; + u64 *const Ur2 = m.coordinates + 8; + u64 *const Zr2 = m.coordinates + 12; + + u64 *const UZr1 = m.coordinates + 0; + u64 *const ZUr2 = m.coordinates + 8; + + u64 *const A = m.workspace + 0; + u64 *const B = m.workspace + 4; + u64 *const C = m.workspace + 8; + u64 *const D = m.workspace + 12; + + u64 *const AB = m.workspace + 0; + u64 *const CD = m.workspace + 8; + + const u64 *const P = table_ladder_8k; + + memcpy(m.private, private_key, sizeof(m.private)); + + curve25519_clamp_secret(m.private); + + setzero_eltfp25519_1w(Ur1); + setzero_eltfp25519_1w(Zr1); + setzero_eltfp25519_1w(Zr2); + Ur1[0] = 1; + Zr1[0] = 1; + Zr2[0] = 1; + + /* G-S */ + Ur2[3] = 0x1eaecdeee27cab34UL; + Ur2[2] = 0xadc7a0b9235d48e2UL; + Ur2[1] = 0xbbf095ae14b2edf8UL; + Ur2[0] = 0x7e94e1fec82faabdUL; + + /* main-loop */ + j = q; + for (i = 0; i < NUM_WORDS_ELTFP25519; ++i) { + while (j < ite[i]) { + u64 bit = (key[i] >> j) & 0x1; + k = (64 * i + j - q); swap = swap ^ bit; - cswap2(swap, xz1, xz2); + cswap(swap, Ur1, Ur2); + cswap(swap, Zr1, Zr2); swap = bit; - fsub(b, x1, z1); - fadd(a, x1, z1); - fmul(c, &table_ladder[4 * k], b, ef); - fsub(b, a, c); - fadd(a, a, c); - fsqr2(ab, ab, efgh); - fmul2(xz1, xz2, ab, efgh); + /* Addition */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + add_eltfp25519_1w_bmi2(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + mul_eltfp25519_1w_bmi2(C, &P[4 * k], B);/* C = M0-B */ + sub_eltfp25519_1w(B, A, C); /* B = (Ur1+Zr1) - M*(Ur1-Zr1) */ + add_eltfp25519_1w_bmi2(A, A, C); /* A = (Ur1+Zr1) + M*(Ur1-Zr1) */ + sqr_eltfp25519_2w_bmi2(AB); /* A = A^2 | B = B^2 */ + mul_eltfp25519_2w_bmi2(UZr1, ZUr2, AB); /* Ur1 = Zr2*A | Zr1 = Ur2*B */ ++j; } j = 0; } - point_double(xz1, abcd, efgh); - point_double(xz1, abcd, efgh); - point_double(xz1, abcd, efgh); - encode_point(out, xz1); + /* Doubling */ + for (i = 0; i < q; ++i) { + add_eltfp25519_1w_bmi2(A, Ur1, Zr1); /* A = Ur1+Zr1 */ + sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ + sqr_eltfp25519_2w_bmi2(AB); /* A = A**2 B = B**2 */ + copy_eltfp25519_1w(C, B); /* C = B */ + sub_eltfp25519_1w(B, A, B); /* B = A-B */ + mul_a24_eltfp25519_1w(D, B); /* D = my_a24*B */ + add_eltfp25519_1w_bmi2(D, D, C); /* D = D+C */ + mul_eltfp25519_2w_bmi2(UZr1, AB, CD); /* Ur1 = A*B Zr1 = Zr1*A */ + } - memzero_explicit(tmp, sizeof(tmp)); -} + /* Convert to affine coordinates */ + inv_eltfp25519_1w_bmi2(A, Zr1); + mul_eltfp25519_1w_bmi2((u64 *)session_key, Ur1, A); + fred_eltfp25519_1w((u64 *)session_key); -static __ro_after_init DEFINE_STATIC_KEY_FALSE(curve25519_use_bmi2_adx); + memzero_explicit(&m, sizeof(m)); +} void curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE], const u8 basepoint[CURVE25519_KEY_SIZE]) { - if (static_branch_likely(&curve25519_use_bmi2_adx)) - curve25519_ever64(mypublic, secret, basepoint); + if (static_branch_likely(&curve25519_use_adx)) + curve25519_adx(mypublic, secret, basepoint); + else if (static_branch_likely(&curve25519_use_bmi2)) + curve25519_bmi2(mypublic, secret, basepoint); else curve25519_generic(mypublic, secret, basepoint); } @@ -1395,8 +2355,10 @@ EXPORT_SYMBOL(curve25519_arch); void curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE]) { - if (static_branch_likely(&curve25519_use_bmi2_adx)) - curve25519_ever64_base(pub, secret); + if (static_branch_likely(&curve25519_use_adx)) + curve25519_adx_base(pub, secret); + else if (static_branch_likely(&curve25519_use_bmi2)) + curve25519_bmi2_base(pub, secret); else curve25519_generic(pub, secret, curve25519_base_point); } @@ -1487,11 +2449,12 @@ static struct kpp_alg curve25519_alg = { .max_size = curve25519_max_size, }; - static int __init curve25519_mod_init(void) { - if (boot_cpu_has(X86_FEATURE_BMI2) && boot_cpu_has(X86_FEATURE_ADX)) - static_branch_enable(&curve25519_use_bmi2_adx); + if (boot_cpu_has(X86_FEATURE_BMI2)) + static_branch_enable(&curve25519_use_bmi2); + else if (boot_cpu_has(X86_FEATURE_ADX)) + static_branch_enable(&curve25519_use_adx); else return 0; return IS_REACHABLE(CONFIG_CRYPTO_KPP) ? @@ -1511,4 +2474,3 @@ module_exit(curve25519_mod_exit); MODULE_ALIAS_CRYPTO("curve25519"); MODULE_ALIAS_CRYPTO("curve25519-x86"); MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Jason A. Donenfeld "); diff --git a/arch/x86/crypto/poly1305-avx2-x86_64.S b/arch/x86/crypto/poly1305-avx2-x86_64.S new file mode 100644 index 0000000000000..d6063feda9daf --- /dev/null +++ b/arch/x86/crypto/poly1305-avx2-x86_64.S @@ -0,0 +1,390 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Poly1305 authenticator algorithm, RFC7539, x64 AVX2 functions + * + * Copyright (C) 2015 Martin Willi + */ + +#include + +.section .rodata.cst32.ANMASK, "aM", @progbits, 32 +.align 32 +ANMASK: .octa 0x0000000003ffffff0000000003ffffff + .octa 0x0000000003ffffff0000000003ffffff + +.section .rodata.cst32.ORMASK, "aM", @progbits, 32 +.align 32 +ORMASK: .octa 0x00000000010000000000000001000000 + .octa 0x00000000010000000000000001000000 + +.text + +#define h0 0x00(%rdi) +#define h1 0x04(%rdi) +#define h2 0x08(%rdi) +#define h3 0x0c(%rdi) +#define h4 0x10(%rdi) +#define r0 0x00(%rdx) +#define r1 0x04(%rdx) +#define r2 0x08(%rdx) +#define r3 0x0c(%rdx) +#define r4 0x10(%rdx) +#define u0 0x00(%r8) +#define u1 0x04(%r8) +#define u2 0x08(%r8) +#define u3 0x0c(%r8) +#define u4 0x10(%r8) +#define w0 0x14(%r8) +#define w1 0x18(%r8) +#define w2 0x1c(%r8) +#define w3 0x20(%r8) +#define w4 0x24(%r8) +#define y0 0x28(%r8) +#define y1 0x2c(%r8) +#define y2 0x30(%r8) +#define y3 0x34(%r8) +#define y4 0x38(%r8) +#define m %rsi +#define hc0 %ymm0 +#define hc1 %ymm1 +#define hc2 %ymm2 +#define hc3 %ymm3 +#define hc4 %ymm4 +#define hc0x %xmm0 +#define hc1x %xmm1 +#define hc2x %xmm2 +#define hc3x %xmm3 +#define hc4x %xmm4 +#define t1 %ymm5 +#define t2 %ymm6 +#define t1x %xmm5 +#define t2x %xmm6 +#define ruwy0 %ymm7 +#define ruwy1 %ymm8 +#define ruwy2 %ymm9 +#define ruwy3 %ymm10 +#define ruwy4 %ymm11 +#define ruwy0x %xmm7 +#define ruwy1x %xmm8 +#define ruwy2x %xmm9 +#define ruwy3x %xmm10 +#define ruwy4x %xmm11 +#define svxz1 %ymm12 +#define svxz2 %ymm13 +#define svxz3 %ymm14 +#define svxz4 %ymm15 +#define d0 %r9 +#define d1 %r10 +#define d2 %r11 +#define d3 %r12 +#define d4 %r13 + +SYM_FUNC_START(poly1305_4block_avx2) + # %rdi: Accumulator h[5] + # %rsi: 64 byte input block m + # %rdx: Poly1305 key r[5] + # %rcx: Quadblock count + # %r8: Poly1305 derived key r^2 u[5], r^3 w[5], r^4 y[5], + + # This four-block variant uses loop unrolled block processing. It + # requires 4 Poly1305 keys: r, r^2, r^3 and r^4: + # h = (h + m) * r => h = (h + m1) * r^4 + m2 * r^3 + m3 * r^2 + m4 * r + + vzeroupper + push %rbx + push %r12 + push %r13 + + # combine r0,u0,w0,y0 + vmovd y0,ruwy0x + vmovd w0,t1x + vpunpcklqdq t1,ruwy0,ruwy0 + vmovd u0,t1x + vmovd r0,t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,ruwy0,ruwy0 + + # combine r1,u1,w1,y1 and s1=r1*5,v1=u1*5,x1=w1*5,z1=y1*5 + vmovd y1,ruwy1x + vmovd w1,t1x + vpunpcklqdq t1,ruwy1,ruwy1 + vmovd u1,t1x + vmovd r1,t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,ruwy1,ruwy1 + vpslld $2,ruwy1,svxz1 + vpaddd ruwy1,svxz1,svxz1 + + # combine r2,u2,w2,y2 and s2=r2*5,v2=u2*5,x2=w2*5,z2=y2*5 + vmovd y2,ruwy2x + vmovd w2,t1x + vpunpcklqdq t1,ruwy2,ruwy2 + vmovd u2,t1x + vmovd r2,t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,ruwy2,ruwy2 + vpslld $2,ruwy2,svxz2 + vpaddd ruwy2,svxz2,svxz2 + + # combine r3,u3,w3,y3 and s3=r3*5,v3=u3*5,x3=w3*5,z3=y3*5 + vmovd y3,ruwy3x + vmovd w3,t1x + vpunpcklqdq t1,ruwy3,ruwy3 + vmovd u3,t1x + vmovd r3,t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,ruwy3,ruwy3 + vpslld $2,ruwy3,svxz3 + vpaddd ruwy3,svxz3,svxz3 + + # combine r4,u4,w4,y4 and s4=r4*5,v4=u4*5,x4=w4*5,z4=y4*5 + vmovd y4,ruwy4x + vmovd w4,t1x + vpunpcklqdq t1,ruwy4,ruwy4 + vmovd u4,t1x + vmovd r4,t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,ruwy4,ruwy4 + vpslld $2,ruwy4,svxz4 + vpaddd ruwy4,svxz4,svxz4 + +.Ldoblock4: + # hc0 = [m[48-51] & 0x3ffffff, m[32-35] & 0x3ffffff, + # m[16-19] & 0x3ffffff, m[ 0- 3] & 0x3ffffff + h0] + vmovd 0x00(m),hc0x + vmovd 0x10(m),t1x + vpunpcklqdq t1,hc0,hc0 + vmovd 0x20(m),t1x + vmovd 0x30(m),t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,hc0,hc0 + vpand ANMASK(%rip),hc0,hc0 + vmovd h0,t1x + vpaddd t1,hc0,hc0 + # hc1 = [(m[51-54] >> 2) & 0x3ffffff, (m[35-38] >> 2) & 0x3ffffff, + # (m[19-22] >> 2) & 0x3ffffff, (m[ 3- 6] >> 2) & 0x3ffffff + h1] + vmovd 0x03(m),hc1x + vmovd 0x13(m),t1x + vpunpcklqdq t1,hc1,hc1 + vmovd 0x23(m),t1x + vmovd 0x33(m),t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,hc1,hc1 + vpsrld $2,hc1,hc1 + vpand ANMASK(%rip),hc1,hc1 + vmovd h1,t1x + vpaddd t1,hc1,hc1 + # hc2 = [(m[54-57] >> 4) & 0x3ffffff, (m[38-41] >> 4) & 0x3ffffff, + # (m[22-25] >> 4) & 0x3ffffff, (m[ 6- 9] >> 4) & 0x3ffffff + h2] + vmovd 0x06(m),hc2x + vmovd 0x16(m),t1x + vpunpcklqdq t1,hc2,hc2 + vmovd 0x26(m),t1x + vmovd 0x36(m),t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,hc2,hc2 + vpsrld $4,hc2,hc2 + vpand ANMASK(%rip),hc2,hc2 + vmovd h2,t1x + vpaddd t1,hc2,hc2 + # hc3 = [(m[57-60] >> 6) & 0x3ffffff, (m[41-44] >> 6) & 0x3ffffff, + # (m[25-28] >> 6) & 0x3ffffff, (m[ 9-12] >> 6) & 0x3ffffff + h3] + vmovd 0x09(m),hc3x + vmovd 0x19(m),t1x + vpunpcklqdq t1,hc3,hc3 + vmovd 0x29(m),t1x + vmovd 0x39(m),t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,hc3,hc3 + vpsrld $6,hc3,hc3 + vpand ANMASK(%rip),hc3,hc3 + vmovd h3,t1x + vpaddd t1,hc3,hc3 + # hc4 = [(m[60-63] >> 8) | (1<<24), (m[44-47] >> 8) | (1<<24), + # (m[28-31] >> 8) | (1<<24), (m[12-15] >> 8) | (1<<24) + h4] + vmovd 0x0c(m),hc4x + vmovd 0x1c(m),t1x + vpunpcklqdq t1,hc4,hc4 + vmovd 0x2c(m),t1x + vmovd 0x3c(m),t2x + vpunpcklqdq t2,t1,t1 + vperm2i128 $0x20,t1,hc4,hc4 + vpsrld $8,hc4,hc4 + vpor ORMASK(%rip),hc4,hc4 + vmovd h4,t1x + vpaddd t1,hc4,hc4 + + # t1 = [ hc0[3] * r0, hc0[2] * u0, hc0[1] * w0, hc0[0] * y0 ] + vpmuludq hc0,ruwy0,t1 + # t1 += [ hc1[3] * s4, hc1[2] * v4, hc1[1] * x4, hc1[0] * z4 ] + vpmuludq hc1,svxz4,t2 + vpaddq t2,t1,t1 + # t1 += [ hc2[3] * s3, hc2[2] * v3, hc2[1] * x3, hc2[0] * z3 ] + vpmuludq hc2,svxz3,t2 + vpaddq t2,t1,t1 + # t1 += [ hc3[3] * s2, hc3[2] * v2, hc3[1] * x2, hc3[0] * z2 ] + vpmuludq hc3,svxz2,t2 + vpaddq t2,t1,t1 + # t1 += [ hc4[3] * s1, hc4[2] * v1, hc4[1] * x1, hc4[0] * z1 ] + vpmuludq hc4,svxz1,t2 + vpaddq t2,t1,t1 + # d0 = t1[0] + t1[1] + t[2] + t[3] + vpermq $0xee,t1,t2 + vpaddq t2,t1,t1 + vpsrldq $8,t1,t2 + vpaddq t2,t1,t1 + vmovq t1x,d0 + + # t1 = [ hc0[3] * r1, hc0[2] * u1,hc0[1] * w1, hc0[0] * y1 ] + vpmuludq hc0,ruwy1,t1 + # t1 += [ hc1[3] * r0, hc1[2] * u0, hc1[1] * w0, hc1[0] * y0 ] + vpmuludq hc1,ruwy0,t2 + vpaddq t2,t1,t1 + # t1 += [ hc2[3] * s4, hc2[2] * v4, hc2[1] * x4, hc2[0] * z4 ] + vpmuludq hc2,svxz4,t2 + vpaddq t2,t1,t1 + # t1 += [ hc3[3] * s3, hc3[2] * v3, hc3[1] * x3, hc3[0] * z3 ] + vpmuludq hc3,svxz3,t2 + vpaddq t2,t1,t1 + # t1 += [ hc4[3] * s2, hc4[2] * v2, hc4[1] * x2, hc4[0] * z2 ] + vpmuludq hc4,svxz2,t2 + vpaddq t2,t1,t1 + # d1 = t1[0] + t1[1] + t1[3] + t1[4] + vpermq $0xee,t1,t2 + vpaddq t2,t1,t1 + vpsrldq $8,t1,t2 + vpaddq t2,t1,t1 + vmovq t1x,d1 + + # t1 = [ hc0[3] * r2, hc0[2] * u2, hc0[1] * w2, hc0[0] * y2 ] + vpmuludq hc0,ruwy2,t1 + # t1 += [ hc1[3] * r1, hc1[2] * u1, hc1[1] * w1, hc1[0] * y1 ] + vpmuludq hc1,ruwy1,t2 + vpaddq t2,t1,t1 + # t1 += [ hc2[3] * r0, hc2[2] * u0, hc2[1] * w0, hc2[0] * y0 ] + vpmuludq hc2,ruwy0,t2 + vpaddq t2,t1,t1 + # t1 += [ hc3[3] * s4, hc3[2] * v4, hc3[1] * x4, hc3[0] * z4 ] + vpmuludq hc3,svxz4,t2 + vpaddq t2,t1,t1 + # t1 += [ hc4[3] * s3, hc4[2] * v3, hc4[1] * x3, hc4[0] * z3 ] + vpmuludq hc4,svxz3,t2 + vpaddq t2,t1,t1 + # d2 = t1[0] + t1[1] + t1[2] + t1[3] + vpermq $0xee,t1,t2 + vpaddq t2,t1,t1 + vpsrldq $8,t1,t2 + vpaddq t2,t1,t1 + vmovq t1x,d2 + + # t1 = [ hc0[3] * r3, hc0[2] * u3, hc0[1] * w3, hc0[0] * y3 ] + vpmuludq hc0,ruwy3,t1 + # t1 += [ hc1[3] * r2, hc1[2] * u2, hc1[1] * w2, hc1[0] * y2 ] + vpmuludq hc1,ruwy2,t2 + vpaddq t2,t1,t1 + # t1 += [ hc2[3] * r1, hc2[2] * u1, hc2[1] * w1, hc2[0] * y1 ] + vpmuludq hc2,ruwy1,t2 + vpaddq t2,t1,t1 + # t1 += [ hc3[3] * r0, hc3[2] * u0, hc3[1] * w0, hc3[0] * y0 ] + vpmuludq hc3,ruwy0,t2 + vpaddq t2,t1,t1 + # t1 += [ hc4[3] * s4, hc4[2] * v4, hc4[1] * x4, hc4[0] * z4 ] + vpmuludq hc4,svxz4,t2 + vpaddq t2,t1,t1 + # d3 = t1[0] + t1[1] + t1[2] + t1[3] + vpermq $0xee,t1,t2 + vpaddq t2,t1,t1 + vpsrldq $8,t1,t2 + vpaddq t2,t1,t1 + vmovq t1x,d3 + + # t1 = [ hc0[3] * r4, hc0[2] * u4, hc0[1] * w4, hc0[0] * y4 ] + vpmuludq hc0,ruwy4,t1 + # t1 += [ hc1[3] * r3, hc1[2] * u3, hc1[1] * w3, hc1[0] * y3 ] + vpmuludq hc1,ruwy3,t2 + vpaddq t2,t1,t1 + # t1 += [ hc2[3] * r2, hc2[2] * u2, hc2[1] * w2, hc2[0] * y2 ] + vpmuludq hc2,ruwy2,t2 + vpaddq t2,t1,t1 + # t1 += [ hc3[3] * r1, hc3[2] * u1, hc3[1] * w1, hc3[0] * y1 ] + vpmuludq hc3,ruwy1,t2 + vpaddq t2,t1,t1 + # t1 += [ hc4[3] * r0, hc4[2] * u0, hc4[1] * w0, hc4[0] * y0 ] + vpmuludq hc4,ruwy0,t2 + vpaddq t2,t1,t1 + # d4 = t1[0] + t1[1] + t1[2] + t1[3] + vpermq $0xee,t1,t2 + vpaddq t2,t1,t1 + vpsrldq $8,t1,t2 + vpaddq t2,t1,t1 + vmovq t1x,d4 + + # Now do a partial reduction mod (2^130)-5, carrying h0 -> h1 -> h2 -> + # h3 -> h4 -> h0 -> h1 to get h0,h2,h3,h4 < 2^26 and h1 < 2^26 + a small + # amount. Careful: we must not assume the carry bits 'd0 >> 26', + # 'd1 >> 26', 'd2 >> 26', 'd3 >> 26', and '(d4 >> 26) * 5' fit in 32-bit + # integers. It's true in a single-block implementation, but not here. + + # d1 += d0 >> 26 + mov d0,%rax + shr $26,%rax + add %rax,d1 + # h0 = d0 & 0x3ffffff + mov d0,%rbx + and $0x3ffffff,%ebx + + # d2 += d1 >> 26 + mov d1,%rax + shr $26,%rax + add %rax,d2 + # h1 = d1 & 0x3ffffff + mov d1,%rax + and $0x3ffffff,%eax + mov %eax,h1 + + # d3 += d2 >> 26 + mov d2,%rax + shr $26,%rax + add %rax,d3 + # h2 = d2 & 0x3ffffff + mov d2,%rax + and $0x3ffffff,%eax + mov %eax,h2 + + # d4 += d3 >> 26 + mov d3,%rax + shr $26,%rax + add %rax,d4 + # h3 = d3 & 0x3ffffff + mov d3,%rax + and $0x3ffffff,%eax + mov %eax,h3 + + # h0 += (d4 >> 26) * 5 + mov d4,%rax + shr $26,%rax + lea (%rax,%rax,4),%rax + add %rax,%rbx + # h4 = d4 & 0x3ffffff + mov d4,%rax + and $0x3ffffff,%eax + mov %eax,h4 + + # h1 += h0 >> 26 + mov %rbx,%rax + shr $26,%rax + add %eax,h1 + # h0 = h0 & 0x3ffffff + andl $0x3ffffff,%ebx + mov %ebx,h0 + + add $0x40,m + dec %rcx + jnz .Ldoblock4 + + vzeroupper + pop %r13 + pop %r12 + pop %rbx + ret +SYM_FUNC_END(poly1305_4block_avx2) diff --git a/arch/x86/crypto/poly1305-sse2-x86_64.S b/arch/x86/crypto/poly1305-sse2-x86_64.S new file mode 100644 index 0000000000000..d8ea29b96640b --- /dev/null +++ b/arch/x86/crypto/poly1305-sse2-x86_64.S @@ -0,0 +1,590 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Poly1305 authenticator algorithm, RFC7539, x64 SSE2 functions + * + * Copyright (C) 2015 Martin Willi + */ + +#include + +.section .rodata.cst16.ANMASK, "aM", @progbits, 16 +.align 16 +ANMASK: .octa 0x0000000003ffffff0000000003ffffff + +.section .rodata.cst16.ORMASK, "aM", @progbits, 16 +.align 16 +ORMASK: .octa 0x00000000010000000000000001000000 + +.text + +#define h0 0x00(%rdi) +#define h1 0x04(%rdi) +#define h2 0x08(%rdi) +#define h3 0x0c(%rdi) +#define h4 0x10(%rdi) +#define r0 0x00(%rdx) +#define r1 0x04(%rdx) +#define r2 0x08(%rdx) +#define r3 0x0c(%rdx) +#define r4 0x10(%rdx) +#define s1 0x00(%rsp) +#define s2 0x04(%rsp) +#define s3 0x08(%rsp) +#define s4 0x0c(%rsp) +#define m %rsi +#define h01 %xmm0 +#define h23 %xmm1 +#define h44 %xmm2 +#define t1 %xmm3 +#define t2 %xmm4 +#define t3 %xmm5 +#define t4 %xmm6 +#define mask %xmm7 +#define d0 %r8 +#define d1 %r9 +#define d2 %r10 +#define d3 %r11 +#define d4 %r12 + +SYM_FUNC_START(poly1305_block_sse2) + # %rdi: Accumulator h[5] + # %rsi: 16 byte input block m + # %rdx: Poly1305 key r[5] + # %rcx: Block count + + # This single block variant tries to improve performance by doing two + # multiplications in parallel using SSE instructions. There is quite + # some quardword packing involved, hence the speedup is marginal. + + push %rbx + push %r12 + sub $0x10,%rsp + + # s1..s4 = r1..r4 * 5 + mov r1,%eax + lea (%eax,%eax,4),%eax + mov %eax,s1 + mov r2,%eax + lea (%eax,%eax,4),%eax + mov %eax,s2 + mov r3,%eax + lea (%eax,%eax,4),%eax + mov %eax,s3 + mov r4,%eax + lea (%eax,%eax,4),%eax + mov %eax,s4 + + movdqa ANMASK(%rip),mask + +.Ldoblock: + # h01 = [0, h1, 0, h0] + # h23 = [0, h3, 0, h2] + # h44 = [0, h4, 0, h4] + movd h0,h01 + movd h1,t1 + movd h2,h23 + movd h3,t2 + movd h4,h44 + punpcklqdq t1,h01 + punpcklqdq t2,h23 + punpcklqdq h44,h44 + + # h01 += [ (m[3-6] >> 2) & 0x3ffffff, m[0-3] & 0x3ffffff ] + movd 0x00(m),t1 + movd 0x03(m),t2 + psrld $2,t2 + punpcklqdq t2,t1 + pand mask,t1 + paddd t1,h01 + # h23 += [ (m[9-12] >> 6) & 0x3ffffff, (m[6-9] >> 4) & 0x3ffffff ] + movd 0x06(m),t1 + movd 0x09(m),t2 + psrld $4,t1 + psrld $6,t2 + punpcklqdq t2,t1 + pand mask,t1 + paddd t1,h23 + # h44 += [ (m[12-15] >> 8) | (1 << 24), (m[12-15] >> 8) | (1 << 24) ] + mov 0x0c(m),%eax + shr $8,%eax + or $0x01000000,%eax + movd %eax,t1 + pshufd $0xc4,t1,t1 + paddd t1,h44 + + # t1[0] = h0 * r0 + h2 * s3 + # t1[1] = h1 * s4 + h3 * s2 + movd r0,t1 + movd s4,t2 + punpcklqdq t2,t1 + pmuludq h01,t1 + movd s3,t2 + movd s2,t3 + punpcklqdq t3,t2 + pmuludq h23,t2 + paddq t2,t1 + # t2[0] = h0 * r1 + h2 * s4 + # t2[1] = h1 * r0 + h3 * s3 + movd r1,t2 + movd r0,t3 + punpcklqdq t3,t2 + pmuludq h01,t2 + movd s4,t3 + movd s3,t4 + punpcklqdq t4,t3 + pmuludq h23,t3 + paddq t3,t2 + # t3[0] = h4 * s1 + # t3[1] = h4 * s2 + movd s1,t3 + movd s2,t4 + punpcklqdq t4,t3 + pmuludq h44,t3 + # d0 = t1[0] + t1[1] + t3[0] + # d1 = t2[0] + t2[1] + t3[1] + movdqa t1,t4 + punpcklqdq t2,t4 + punpckhqdq t2,t1 + paddq t4,t1 + paddq t3,t1 + movq t1,d0 + psrldq $8,t1 + movq t1,d1 + + # t1[0] = h0 * r2 + h2 * r0 + # t1[1] = h1 * r1 + h3 * s4 + movd r2,t1 + movd r1,t2 + punpcklqdq t2,t1 + pmuludq h01,t1 + movd r0,t2 + movd s4,t3 + punpcklqdq t3,t2 + pmuludq h23,t2 + paddq t2,t1 + # t2[0] = h0 * r3 + h2 * r1 + # t2[1] = h1 * r2 + h3 * r0 + movd r3,t2 + movd r2,t3 + punpcklqdq t3,t2 + pmuludq h01,t2 + movd r1,t3 + movd r0,t4 + punpcklqdq t4,t3 + pmuludq h23,t3 + paddq t3,t2 + # t3[0] = h4 * s3 + # t3[1] = h4 * s4 + movd s3,t3 + movd s4,t4 + punpcklqdq t4,t3 + pmuludq h44,t3 + # d2 = t1[0] + t1[1] + t3[0] + # d3 = t2[0] + t2[1] + t3[1] + movdqa t1,t4 + punpcklqdq t2,t4 + punpckhqdq t2,t1 + paddq t4,t1 + paddq t3,t1 + movq t1,d2 + psrldq $8,t1 + movq t1,d3 + + # t1[0] = h0 * r4 + h2 * r2 + # t1[1] = h1 * r3 + h3 * r1 + movd r4,t1 + movd r3,t2 + punpcklqdq t2,t1 + pmuludq h01,t1 + movd r2,t2 + movd r1,t3 + punpcklqdq t3,t2 + pmuludq h23,t2 + paddq t2,t1 + # t3[0] = h4 * r0 + movd r0,t3 + pmuludq h44,t3 + # d4 = t1[0] + t1[1] + t3[0] + movdqa t1,t4 + psrldq $8,t4 + paddq t4,t1 + paddq t3,t1 + movq t1,d4 + + # d1 += d0 >> 26 + mov d0,%rax + shr $26,%rax + add %rax,d1 + # h0 = d0 & 0x3ffffff + mov d0,%rbx + and $0x3ffffff,%ebx + + # d2 += d1 >> 26 + mov d1,%rax + shr $26,%rax + add %rax,d2 + # h1 = d1 & 0x3ffffff + mov d1,%rax + and $0x3ffffff,%eax + mov %eax,h1 + + # d3 += d2 >> 26 + mov d2,%rax + shr $26,%rax + add %rax,d3 + # h2 = d2 & 0x3ffffff + mov d2,%rax + and $0x3ffffff,%eax + mov %eax,h2 + + # d4 += d3 >> 26 + mov d3,%rax + shr $26,%rax + add %rax,d4 + # h3 = d3 & 0x3ffffff + mov d3,%rax + and $0x3ffffff,%eax + mov %eax,h3 + + # h0 += (d4 >> 26) * 5 + mov d4,%rax + shr $26,%rax + lea (%rax,%rax,4),%rax + add %rax,%rbx + # h4 = d4 & 0x3ffffff + mov d4,%rax + and $0x3ffffff,%eax + mov %eax,h4 + + # h1 += h0 >> 26 + mov %rbx,%rax + shr $26,%rax + add %eax,h1 + # h0 = h0 & 0x3ffffff + andl $0x3ffffff,%ebx + mov %ebx,h0 + + add $0x10,m + dec %rcx + jnz .Ldoblock + + # Zeroing of key material + mov %rcx,0x00(%rsp) + mov %rcx,0x08(%rsp) + + add $0x10,%rsp + pop %r12 + pop %rbx + ret +SYM_FUNC_END(poly1305_block_sse2) + + +#define u0 0x00(%r8) +#define u1 0x04(%r8) +#define u2 0x08(%r8) +#define u3 0x0c(%r8) +#define u4 0x10(%r8) +#define hc0 %xmm0 +#define hc1 %xmm1 +#define hc2 %xmm2 +#define hc3 %xmm5 +#define hc4 %xmm6 +#define ru0 %xmm7 +#define ru1 %xmm8 +#define ru2 %xmm9 +#define ru3 %xmm10 +#define ru4 %xmm11 +#define sv1 %xmm12 +#define sv2 %xmm13 +#define sv3 %xmm14 +#define sv4 %xmm15 +#undef d0 +#define d0 %r13 + +SYM_FUNC_START(poly1305_2block_sse2) + # %rdi: Accumulator h[5] + # %rsi: 16 byte input block m + # %rdx: Poly1305 key r[5] + # %rcx: Doubleblock count + # %r8: Poly1305 derived key r^2 u[5] + + # This two-block variant further improves performance by using loop + # unrolled block processing. This is more straight forward and does + # less byte shuffling, but requires a second Poly1305 key r^2: + # h = (h + m) * r => h = (h + m1) * r^2 + m2 * r + + push %rbx + push %r12 + push %r13 + + # combine r0,u0 + movd u0,ru0 + movd r0,t1 + punpcklqdq t1,ru0 + + # combine r1,u1 and s1=r1*5,v1=u1*5 + movd u1,ru1 + movd r1,t1 + punpcklqdq t1,ru1 + movdqa ru1,sv1 + pslld $2,sv1 + paddd ru1,sv1 + + # combine r2,u2 and s2=r2*5,v2=u2*5 + movd u2,ru2 + movd r2,t1 + punpcklqdq t1,ru2 + movdqa ru2,sv2 + pslld $2,sv2 + paddd ru2,sv2 + + # combine r3,u3 and s3=r3*5,v3=u3*5 + movd u3,ru3 + movd r3,t1 + punpcklqdq t1,ru3 + movdqa ru3,sv3 + pslld $2,sv3 + paddd ru3,sv3 + + # combine r4,u4 and s4=r4*5,v4=u4*5 + movd u4,ru4 + movd r4,t1 + punpcklqdq t1,ru4 + movdqa ru4,sv4 + pslld $2,sv4 + paddd ru4,sv4 + +.Ldoblock2: + # hc0 = [ m[16-19] & 0x3ffffff, h0 + m[0-3] & 0x3ffffff ] + movd 0x00(m),hc0 + movd 0x10(m),t1 + punpcklqdq t1,hc0 + pand ANMASK(%rip),hc0 + movd h0,t1 + paddd t1,hc0 + # hc1 = [ (m[19-22] >> 2) & 0x3ffffff, h1 + (m[3-6] >> 2) & 0x3ffffff ] + movd 0x03(m),hc1 + movd 0x13(m),t1 + punpcklqdq t1,hc1 + psrld $2,hc1 + pand ANMASK(%rip),hc1 + movd h1,t1 + paddd t1,hc1 + # hc2 = [ (m[22-25] >> 4) & 0x3ffffff, h2 + (m[6-9] >> 4) & 0x3ffffff ] + movd 0x06(m),hc2 + movd 0x16(m),t1 + punpcklqdq t1,hc2 + psrld $4,hc2 + pand ANMASK(%rip),hc2 + movd h2,t1 + paddd t1,hc2 + # hc3 = [ (m[25-28] >> 6) & 0x3ffffff, h3 + (m[9-12] >> 6) & 0x3ffffff ] + movd 0x09(m),hc3 + movd 0x19(m),t1 + punpcklqdq t1,hc3 + psrld $6,hc3 + pand ANMASK(%rip),hc3 + movd h3,t1 + paddd t1,hc3 + # hc4 = [ (m[28-31] >> 8) | (1<<24), h4 + (m[12-15] >> 8) | (1<<24) ] + movd 0x0c(m),hc4 + movd 0x1c(m),t1 + punpcklqdq t1,hc4 + psrld $8,hc4 + por ORMASK(%rip),hc4 + movd h4,t1 + paddd t1,hc4 + + # t1 = [ hc0[1] * r0, hc0[0] * u0 ] + movdqa ru0,t1 + pmuludq hc0,t1 + # t1 += [ hc1[1] * s4, hc1[0] * v4 ] + movdqa sv4,t2 + pmuludq hc1,t2 + paddq t2,t1 + # t1 += [ hc2[1] * s3, hc2[0] * v3 ] + movdqa sv3,t2 + pmuludq hc2,t2 + paddq t2,t1 + # t1 += [ hc3[1] * s2, hc3[0] * v2 ] + movdqa sv2,t2 + pmuludq hc3,t2 + paddq t2,t1 + # t1 += [ hc4[1] * s1, hc4[0] * v1 ] + movdqa sv1,t2 + pmuludq hc4,t2 + paddq t2,t1 + # d0 = t1[0] + t1[1] + movdqa t1,t2 + psrldq $8,t2 + paddq t2,t1 + movq t1,d0 + + # t1 = [ hc0[1] * r1, hc0[0] * u1 ] + movdqa ru1,t1 + pmuludq hc0,t1 + # t1 += [ hc1[1] * r0, hc1[0] * u0 ] + movdqa ru0,t2 + pmuludq hc1,t2 + paddq t2,t1 + # t1 += [ hc2[1] * s4, hc2[0] * v4 ] + movdqa sv4,t2 + pmuludq hc2,t2 + paddq t2,t1 + # t1 += [ hc3[1] * s3, hc3[0] * v3 ] + movdqa sv3,t2 + pmuludq hc3,t2 + paddq t2,t1 + # t1 += [ hc4[1] * s2, hc4[0] * v2 ] + movdqa sv2,t2 + pmuludq hc4,t2 + paddq t2,t1 + # d1 = t1[0] + t1[1] + movdqa t1,t2 + psrldq $8,t2 + paddq t2,t1 + movq t1,d1 + + # t1 = [ hc0[1] * r2, hc0[0] * u2 ] + movdqa ru2,t1 + pmuludq hc0,t1 + # t1 += [ hc1[1] * r1, hc1[0] * u1 ] + movdqa ru1,t2 + pmuludq hc1,t2 + paddq t2,t1 + # t1 += [ hc2[1] * r0, hc2[0] * u0 ] + movdqa ru0,t2 + pmuludq hc2,t2 + paddq t2,t1 + # t1 += [ hc3[1] * s4, hc3[0] * v4 ] + movdqa sv4,t2 + pmuludq hc3,t2 + paddq t2,t1 + # t1 += [ hc4[1] * s3, hc4[0] * v3 ] + movdqa sv3,t2 + pmuludq hc4,t2 + paddq t2,t1 + # d2 = t1[0] + t1[1] + movdqa t1,t2 + psrldq $8,t2 + paddq t2,t1 + movq t1,d2 + + # t1 = [ hc0[1] * r3, hc0[0] * u3 ] + movdqa ru3,t1 + pmuludq hc0,t1 + # t1 += [ hc1[1] * r2, hc1[0] * u2 ] + movdqa ru2,t2 + pmuludq hc1,t2 + paddq t2,t1 + # t1 += [ hc2[1] * r1, hc2[0] * u1 ] + movdqa ru1,t2 + pmuludq hc2,t2 + paddq t2,t1 + # t1 += [ hc3[1] * r0, hc3[0] * u0 ] + movdqa ru0,t2 + pmuludq hc3,t2 + paddq t2,t1 + # t1 += [ hc4[1] * s4, hc4[0] * v4 ] + movdqa sv4,t2 + pmuludq hc4,t2 + paddq t2,t1 + # d3 = t1[0] + t1[1] + movdqa t1,t2 + psrldq $8,t2 + paddq t2,t1 + movq t1,d3 + + # t1 = [ hc0[1] * r4, hc0[0] * u4 ] + movdqa ru4,t1 + pmuludq hc0,t1 + # t1 += [ hc1[1] * r3, hc1[0] * u3 ] + movdqa ru3,t2 + pmuludq hc1,t2 + paddq t2,t1 + # t1 += [ hc2[1] * r2, hc2[0] * u2 ] + movdqa ru2,t2 + pmuludq hc2,t2 + paddq t2,t1 + # t1 += [ hc3[1] * r1, hc3[0] * u1 ] + movdqa ru1,t2 + pmuludq hc3,t2 + paddq t2,t1 + # t1 += [ hc4[1] * r0, hc4[0] * u0 ] + movdqa ru0,t2 + pmuludq hc4,t2 + paddq t2,t1 + # d4 = t1[0] + t1[1] + movdqa t1,t2 + psrldq $8,t2 + paddq t2,t1 + movq t1,d4 + + # Now do a partial reduction mod (2^130)-5, carrying h0 -> h1 -> h2 -> + # h3 -> h4 -> h0 -> h1 to get h0,h2,h3,h4 < 2^26 and h1 < 2^26 + a small + # amount. Careful: we must not assume the carry bits 'd0 >> 26', + # 'd1 >> 26', 'd2 >> 26', 'd3 >> 26', and '(d4 >> 26) * 5' fit in 32-bit + # integers. It's true in a single-block implementation, but not here. + + # d1 += d0 >> 26 + mov d0,%rax + shr $26,%rax + add %rax,d1 + # h0 = d0 & 0x3ffffff + mov d0,%rbx + and $0x3ffffff,%ebx + + # d2 += d1 >> 26 + mov d1,%rax + shr $26,%rax + add %rax,d2 + # h1 = d1 & 0x3ffffff + mov d1,%rax + and $0x3ffffff,%eax + mov %eax,h1 + + # d3 += d2 >> 26 + mov d2,%rax + shr $26,%rax + add %rax,d3 + # h2 = d2 & 0x3ffffff + mov d2,%rax + and $0x3ffffff,%eax + mov %eax,h2 + + # d4 += d3 >> 26 + mov d3,%rax + shr $26,%rax + add %rax,d4 + # h3 = d3 & 0x3ffffff + mov d3,%rax + and $0x3ffffff,%eax + mov %eax,h3 + + # h0 += (d4 >> 26) * 5 + mov d4,%rax + shr $26,%rax + lea (%rax,%rax,4),%rax + add %rax,%rbx + # h4 = d4 & 0x3ffffff + mov d4,%rax + and $0x3ffffff,%eax + mov %eax,h4 + + # h1 += h0 >> 26 + mov %rbx,%rax + shr $26,%rax + add %eax,h1 + # h0 = h0 & 0x3ffffff + andl $0x3ffffff,%ebx + mov %ebx,h0 + + add $0x20,m + dec %rcx + jnz .Ldoblock2 + + pop %r13 + pop %r12 + pop %rbx + ret +SYM_FUNC_END(poly1305_2block_sse2) diff --git a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl deleted file mode 100644 index 7a6b5380a46fc..0000000000000 --- a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl +++ /dev/null @@ -1,4265 +0,0 @@ -#!/usr/bin/env perl -# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause -# -# Copyright (C) 2017-2018 Samuel Neves . All Rights Reserved. -# Copyright (C) 2017-2019 Jason A. Donenfeld . All Rights Reserved. -# Copyright (C) 2006-2017 CRYPTOGAMS by . All Rights Reserved. -# -# This code is taken from the OpenSSL project but the author, Andy Polyakov, -# has relicensed it under the licenses specified in the SPDX header above. -# The original headers, including the original license headers, are -# included below for completeness. -# -# ==================================================================== -# Written by Andy Polyakov for the OpenSSL -# project. The module is, however, dual licensed under OpenSSL and -# CRYPTOGAMS licenses depending on where you obtain it. For further -# details see http://www.openssl.org/~appro/cryptogams/. -# ==================================================================== -# -# This module implements Poly1305 hash for x86_64. -# -# March 2015 -# -# Initial release. -# -# December 2016 -# -# Add AVX512F+VL+BW code path. -# -# November 2017 -# -# Convert AVX512F+VL+BW code path to pure AVX512F, so that it can be -# executed even on Knights Landing. Trigger for modification was -# observation that AVX512 code paths can negatively affect overall -# Skylake-X system performance. Since we are likely to suppress -# AVX512F capability flag [at least on Skylake-X], conversion serves -# as kind of "investment protection". Note that next *lake processor, -# Cannonlake, has AVX512IFMA code path to execute... -# -# Numbers are cycles per processed byte with poly1305_blocks alone, -# measured with rdtsc at fixed clock frequency. -# -# IALU/gcc-4.8(*) AVX(**) AVX2 AVX-512 -# P4 4.46/+120% - -# Core 2 2.41/+90% - -# Westmere 1.88/+120% - -# Sandy Bridge 1.39/+140% 1.10 -# Haswell 1.14/+175% 1.11 0.65 -# Skylake[-X] 1.13/+120% 0.96 0.51 [0.35] -# Silvermont 2.83/+95% - -# Knights L 3.60/? 1.65 1.10 0.41(***) -# Goldmont 1.70/+180% - -# VIA Nano 1.82/+150% - -# Sledgehammer 1.38/+160% - -# Bulldozer 2.30/+130% 0.97 -# Ryzen 1.15/+200% 1.08 1.18 -# -# (*) improvement coefficients relative to clang are more modest and -# are ~50% on most processors, in both cases we are comparing to -# __int128 code; -# (**) SSE2 implementation was attempted, but among non-AVX processors -# it was faster than integer-only code only on older Intel P4 and -# Core processors, 50-30%, less newer processor is, but slower on -# contemporary ones, for example almost 2x slower on Atom, and as -# former are naturally disappearing, SSE2 is deemed unnecessary; -# (***) strangely enough performance seems to vary from core to core, -# listed result is best case; - -$flavour = shift; -$output = shift; -if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } - -$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); -$kernel=0; $kernel=1 if (!$flavour && !$output); - -if (!$kernel) { - $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; - ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or - ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or - die "can't locate x86_64-xlate.pl"; - - open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; - *STDOUT=*OUT; - - if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` - =~ /GNU assembler version ([2-9]\.[0-9]+)/) { - $avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25); - } - - if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && - `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { - $avx = ($1>=2.09) + ($1>=2.10) + ($1>=2.12); - $avx += 1 if ($1==2.11 && $2>=8); - } - - if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && - `ml64 2>&1` =~ /Version ([0-9]+)\./) { - $avx = ($1>=10) + ($1>=11); - } - - if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) { - $avx = ($2>=3.0) + ($2>3.0); - } -} else { - $avx = 4; # The kernel uses ifdefs for this. -} - -sub declare_function() { - my ($name, $align, $nargs) = @_; - if($kernel) { - $code .= ".align $align\n"; - $code .= "SYM_FUNC_START($name)\n"; - $code .= ".L$name:\n"; - } else { - $code .= ".globl $name\n"; - $code .= ".type $name,\@function,$nargs\n"; - $code .= ".align $align\n"; - $code .= "$name:\n"; - } -} - -sub end_function() { - my ($name) = @_; - if($kernel) { - $code .= "SYM_FUNC_END($name)\n"; - } else { - $code .= ".size $name,.-$name\n"; - } -} - -$code.=<<___ if $kernel; -#include -___ - -if ($avx) { -$code.=<<___ if $kernel; -.section .rodata -___ -$code.=<<___; -.align 64 -.Lconst: -.Lmask24: -.long 0x0ffffff,0,0x0ffffff,0,0x0ffffff,0,0x0ffffff,0 -.L129: -.long `1<<24`,0,`1<<24`,0,`1<<24`,0,`1<<24`,0 -.Lmask26: -.long 0x3ffffff,0,0x3ffffff,0,0x3ffffff,0,0x3ffffff,0 -.Lpermd_avx2: -.long 2,2,2,3,2,0,2,1 -.Lpermd_avx512: -.long 0,0,0,1, 0,2,0,3, 0,4,0,5, 0,6,0,7 - -.L2_44_inp_permd: -.long 0,1,1,2,2,3,7,7 -.L2_44_inp_shift: -.quad 0,12,24,64 -.L2_44_mask: -.quad 0xfffffffffff,0xfffffffffff,0x3ffffffffff,0xffffffffffffffff -.L2_44_shift_rgt: -.quad 44,44,42,64 -.L2_44_shift_lft: -.quad 8,8,10,64 - -.align 64 -.Lx_mask44: -.quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff -.quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff -.Lx_mask42: -.quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff -.quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff -___ -} -$code.=<<___ if (!$kernel); -.asciz "Poly1305 for x86_64, CRYPTOGAMS by " -.align 16 -___ - -my ($ctx,$inp,$len,$padbit)=("%rdi","%rsi","%rdx","%rcx"); -my ($mac,$nonce)=($inp,$len); # *_emit arguments -my ($d1,$d2,$d3, $r0,$r1,$s1)=("%r8","%r9","%rdi","%r11","%r12","%r13"); -my ($h0,$h1,$h2)=("%r14","%rbx","%r10"); - -sub poly1305_iteration { -# input: copy of $r1 in %rax, $h0-$h2, $r0-$r1 -# output: $h0-$h2 *= $r0-$r1 -$code.=<<___; - mulq $h0 # h0*r1 - mov %rax,$d2 - mov $r0,%rax - mov %rdx,$d3 - - mulq $h0 # h0*r0 - mov %rax,$h0 # future $h0 - mov $r0,%rax - mov %rdx,$d1 - - mulq $h1 # h1*r0 - add %rax,$d2 - mov $s1,%rax - adc %rdx,$d3 - - mulq $h1 # h1*s1 - mov $h2,$h1 # borrow $h1 - add %rax,$h0 - adc %rdx,$d1 - - imulq $s1,$h1 # h2*s1 - add $h1,$d2 - mov $d1,$h1 - adc \$0,$d3 - - imulq $r0,$h2 # h2*r0 - add $d2,$h1 - mov \$-4,%rax # mask value - adc $h2,$d3 - - and $d3,%rax # last reduction step - mov $d3,$h2 - shr \$2,$d3 - and \$3,$h2 - add $d3,%rax - add %rax,$h0 - adc \$0,$h1 - adc \$0,$h2 -___ -} - -######################################################################## -# Layout of opaque area is following. -# -# unsigned __int64 h[3]; # current hash value base 2^64 -# unsigned __int64 r[2]; # key value base 2^64 - -$code.=<<___; -.text -___ -$code.=<<___ if (!$kernel); -.extern OPENSSL_ia32cap_P - -.globl poly1305_init_x86_64 -.hidden poly1305_init_x86_64 -.globl poly1305_blocks_x86_64 -.hidden poly1305_blocks_x86_64 -.globl poly1305_emit_x86_64 -.hidden poly1305_emit_x86_64 -___ -&declare_function("poly1305_init_x86_64", 32, 3); -$code.=<<___; - xor %rax,%rax - mov %rax,0($ctx) # initialize hash value - mov %rax,8($ctx) - mov %rax,16($ctx) - - cmp \$0,$inp - je .Lno_key -___ -$code.=<<___ if (!$kernel); - lea poly1305_blocks_x86_64(%rip),%r10 - lea poly1305_emit_x86_64(%rip),%r11 -___ -$code.=<<___ if (!$kernel && $avx); - mov OPENSSL_ia32cap_P+4(%rip),%r9 - lea poly1305_blocks_avx(%rip),%rax - lea poly1305_emit_avx(%rip),%rcx - bt \$`60-32`,%r9 # AVX? - cmovc %rax,%r10 - cmovc %rcx,%r11 -___ -$code.=<<___ if (!$kernel && $avx>1); - lea poly1305_blocks_avx2(%rip),%rax - bt \$`5+32`,%r9 # AVX2? - cmovc %rax,%r10 -___ -$code.=<<___ if (!$kernel && $avx>3); - mov \$`(1<<31|1<<21|1<<16)`,%rax - shr \$32,%r9 - and %rax,%r9 - cmp %rax,%r9 - je .Linit_base2_44 -___ -$code.=<<___; - mov \$0x0ffffffc0fffffff,%rax - mov \$0x0ffffffc0ffffffc,%rcx - and 0($inp),%rax - and 8($inp),%rcx - mov %rax,24($ctx) - mov %rcx,32($ctx) -___ -$code.=<<___ if (!$kernel && $flavour !~ /elf32/); - mov %r10,0(%rdx) - mov %r11,8(%rdx) -___ -$code.=<<___ if (!$kernel && $flavour =~ /elf32/); - mov %r10d,0(%rdx) - mov %r11d,4(%rdx) -___ -$code.=<<___; - mov \$1,%eax -.Lno_key: - ret -___ -&end_function("poly1305_init_x86_64"); - -&declare_function("poly1305_blocks_x86_64", 32, 4); -$code.=<<___; -.cfi_startproc -.Lblocks: - shr \$4,$len - jz .Lno_data # too short - - push %rbx -.cfi_push %rbx - push %r12 -.cfi_push %r12 - push %r13 -.cfi_push %r13 - push %r14 -.cfi_push %r14 - push %r15 -.cfi_push %r15 - push $ctx -.cfi_push $ctx -.Lblocks_body: - - mov $len,%r15 # reassign $len - - mov 24($ctx),$r0 # load r - mov 32($ctx),$s1 - - mov 0($ctx),$h0 # load hash value - mov 8($ctx),$h1 - mov 16($ctx),$h2 - - mov $s1,$r1 - shr \$2,$s1 - mov $r1,%rax - add $r1,$s1 # s1 = r1 + (r1 >> 2) - jmp .Loop - -.align 32 -.Loop: - add 0($inp),$h0 # accumulate input - adc 8($inp),$h1 - lea 16($inp),$inp - adc $padbit,$h2 -___ - - &poly1305_iteration(); - -$code.=<<___; - mov $r1,%rax - dec %r15 # len-=16 - jnz .Loop - - mov 0(%rsp),$ctx -.cfi_restore $ctx - - mov $h0,0($ctx) # store hash value - mov $h1,8($ctx) - mov $h2,16($ctx) - - mov 8(%rsp),%r15 -.cfi_restore %r15 - mov 16(%rsp),%r14 -.cfi_restore %r14 - mov 24(%rsp),%r13 -.cfi_restore %r13 - mov 32(%rsp),%r12 -.cfi_restore %r12 - mov 40(%rsp),%rbx -.cfi_restore %rbx - lea 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 -.Lno_data: -.Lblocks_epilogue: - ret -.cfi_endproc -___ -&end_function("poly1305_blocks_x86_64"); - -&declare_function("poly1305_emit_x86_64", 32, 3); -$code.=<<___; -.Lemit: - mov 0($ctx),%r8 # load hash value - mov 8($ctx),%r9 - mov 16($ctx),%r10 - - mov %r8,%rax - add \$5,%r8 # compare to modulus - mov %r9,%rcx - adc \$0,%r9 - adc \$0,%r10 - shr \$2,%r10 # did 130-bit value overflow? - cmovnz %r8,%rax - cmovnz %r9,%rcx - - add 0($nonce),%rax # accumulate nonce - adc 8($nonce),%rcx - mov %rax,0($mac) # write result - mov %rcx,8($mac) - - ret -___ -&end_function("poly1305_emit_x86_64"); -if ($avx) { - -if($kernel) { - $code .= "#ifdef CONFIG_AS_AVX\n"; -} - -######################################################################## -# Layout of opaque area is following. -# -# unsigned __int32 h[5]; # current hash value base 2^26 -# unsigned __int32 is_base2_26; -# unsigned __int64 r[2]; # key value base 2^64 -# unsigned __int64 pad; -# struct { unsigned __int32 r^2, r^1, r^4, r^3; } r[9]; -# -# where r^n are base 2^26 digits of degrees of multiplier key. There are -# 5 digits, but last four are interleaved with multiples of 5, totalling -# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4. - -my ($H0,$H1,$H2,$H3,$H4, $T0,$T1,$T2,$T3,$T4, $D0,$D1,$D2,$D3,$D4, $MASK) = - map("%xmm$_",(0..15)); - -$code.=<<___; -.type __poly1305_block,\@abi-omnipotent -.align 32 -__poly1305_block: - push $ctx -___ - &poly1305_iteration(); -$code.=<<___; - pop $ctx - ret -.size __poly1305_block,.-__poly1305_block - -.type __poly1305_init_avx,\@abi-omnipotent -.align 32 -__poly1305_init_avx: - push %rbp - mov %rsp,%rbp - mov $r0,$h0 - mov $r1,$h1 - xor $h2,$h2 - - lea 48+64($ctx),$ctx # size optimization - - mov $r1,%rax - call __poly1305_block # r^2 - - mov \$0x3ffffff,%eax # save interleaved r^2 and r base 2^26 - mov \$0x3ffffff,%edx - mov $h0,$d1 - and $h0#d,%eax - mov $r0,$d2 - and $r0#d,%edx - mov %eax,`16*0+0-64`($ctx) - shr \$26,$d1 - mov %edx,`16*0+4-64`($ctx) - shr \$26,$d2 - - mov \$0x3ffffff,%eax - mov \$0x3ffffff,%edx - and $d1#d,%eax - and $d2#d,%edx - mov %eax,`16*1+0-64`($ctx) - lea (%rax,%rax,4),%eax # *5 - mov %edx,`16*1+4-64`($ctx) - lea (%rdx,%rdx,4),%edx # *5 - mov %eax,`16*2+0-64`($ctx) - shr \$26,$d1 - mov %edx,`16*2+4-64`($ctx) - shr \$26,$d2 - - mov $h1,%rax - mov $r1,%rdx - shl \$12,%rax - shl \$12,%rdx - or $d1,%rax - or $d2,%rdx - and \$0x3ffffff,%eax - and \$0x3ffffff,%edx - mov %eax,`16*3+0-64`($ctx) - lea (%rax,%rax,4),%eax # *5 - mov %edx,`16*3+4-64`($ctx) - lea (%rdx,%rdx,4),%edx # *5 - mov %eax,`16*4+0-64`($ctx) - mov $h1,$d1 - mov %edx,`16*4+4-64`($ctx) - mov $r1,$d2 - - mov \$0x3ffffff,%eax - mov \$0x3ffffff,%edx - shr \$14,$d1 - shr \$14,$d2 - and $d1#d,%eax - and $d2#d,%edx - mov %eax,`16*5+0-64`($ctx) - lea (%rax,%rax,4),%eax # *5 - mov %edx,`16*5+4-64`($ctx) - lea (%rdx,%rdx,4),%edx # *5 - mov %eax,`16*6+0-64`($ctx) - shr \$26,$d1 - mov %edx,`16*6+4-64`($ctx) - shr \$26,$d2 - - mov $h2,%rax - shl \$24,%rax - or %rax,$d1 - mov $d1#d,`16*7+0-64`($ctx) - lea ($d1,$d1,4),$d1 # *5 - mov $d2#d,`16*7+4-64`($ctx) - lea ($d2,$d2,4),$d2 # *5 - mov $d1#d,`16*8+0-64`($ctx) - mov $d2#d,`16*8+4-64`($ctx) - - mov $r1,%rax - call __poly1305_block # r^3 - - mov \$0x3ffffff,%eax # save r^3 base 2^26 - mov $h0,$d1 - and $h0#d,%eax - shr \$26,$d1 - mov %eax,`16*0+12-64`($ctx) - - mov \$0x3ffffff,%edx - and $d1#d,%edx - mov %edx,`16*1+12-64`($ctx) - lea (%rdx,%rdx,4),%edx # *5 - shr \$26,$d1 - mov %edx,`16*2+12-64`($ctx) - - mov $h1,%rax - shl \$12,%rax - or $d1,%rax - and \$0x3ffffff,%eax - mov %eax,`16*3+12-64`($ctx) - lea (%rax,%rax,4),%eax # *5 - mov $h1,$d1 - mov %eax,`16*4+12-64`($ctx) - - mov \$0x3ffffff,%edx - shr \$14,$d1 - and $d1#d,%edx - mov %edx,`16*5+12-64`($ctx) - lea (%rdx,%rdx,4),%edx # *5 - shr \$26,$d1 - mov %edx,`16*6+12-64`($ctx) - - mov $h2,%rax - shl \$24,%rax - or %rax,$d1 - mov $d1#d,`16*7+12-64`($ctx) - lea ($d1,$d1,4),$d1 # *5 - mov $d1#d,`16*8+12-64`($ctx) - - mov $r1,%rax - call __poly1305_block # r^4 - - mov \$0x3ffffff,%eax # save r^4 base 2^26 - mov $h0,$d1 - and $h0#d,%eax - shr \$26,$d1 - mov %eax,`16*0+8-64`($ctx) - - mov \$0x3ffffff,%edx - and $d1#d,%edx - mov %edx,`16*1+8-64`($ctx) - lea (%rdx,%rdx,4),%edx # *5 - shr \$26,$d1 - mov %edx,`16*2+8-64`($ctx) - - mov $h1,%rax - shl \$12,%rax - or $d1,%rax - and \$0x3ffffff,%eax - mov %eax,`16*3+8-64`($ctx) - lea (%rax,%rax,4),%eax # *5 - mov $h1,$d1 - mov %eax,`16*4+8-64`($ctx) - - mov \$0x3ffffff,%edx - shr \$14,$d1 - and $d1#d,%edx - mov %edx,`16*5+8-64`($ctx) - lea (%rdx,%rdx,4),%edx # *5 - shr \$26,$d1 - mov %edx,`16*6+8-64`($ctx) - - mov $h2,%rax - shl \$24,%rax - or %rax,$d1 - mov $d1#d,`16*7+8-64`($ctx) - lea ($d1,$d1,4),$d1 # *5 - mov $d1#d,`16*8+8-64`($ctx) - - lea -48-64($ctx),$ctx # size [de-]optimization - pop %rbp - ret -.size __poly1305_init_avx,.-__poly1305_init_avx -___ - -&declare_function("poly1305_blocks_avx", 32, 4); -$code.=<<___; -.cfi_startproc - mov 20($ctx),%r8d # is_base2_26 - cmp \$128,$len - jae .Lblocks_avx - test %r8d,%r8d - jz .Lblocks - -.Lblocks_avx: - and \$-16,$len - jz .Lno_data_avx - - vzeroupper - - test %r8d,%r8d - jz .Lbase2_64_avx - - test \$31,$len - jz .Leven_avx - - push %rbp -.cfi_push %rbp - mov %rsp,%rbp - push %rbx -.cfi_push %rbx - push %r12 -.cfi_push %r12 - push %r13 -.cfi_push %r13 - push %r14 -.cfi_push %r14 - push %r15 -.cfi_push %r15 -.Lblocks_avx_body: - - mov $len,%r15 # reassign $len - - mov 0($ctx),$d1 # load hash value - mov 8($ctx),$d2 - mov 16($ctx),$h2#d - - mov 24($ctx),$r0 # load r - mov 32($ctx),$s1 - - ################################# base 2^26 -> base 2^64 - mov $d1#d,$h0#d - and \$`-1*(1<<31)`,$d1 - mov $d2,$r1 # borrow $r1 - mov $d2#d,$h1#d - and \$`-1*(1<<31)`,$d2 - - shr \$6,$d1 - shl \$52,$r1 - add $d1,$h0 - shr \$12,$h1 - shr \$18,$d2 - add $r1,$h0 - adc $d2,$h1 - - mov $h2,$d1 - shl \$40,$d1 - shr \$24,$h2 - add $d1,$h1 - adc \$0,$h2 # can be partially reduced... - - mov \$-4,$d2 # ... so reduce - mov $h2,$d1 - and $h2,$d2 - shr \$2,$d1 - and \$3,$h2 - add $d2,$d1 # =*5 - add $d1,$h0 - adc \$0,$h1 - adc \$0,$h2 - - mov $s1,$r1 - mov $s1,%rax - shr \$2,$s1 - add $r1,$s1 # s1 = r1 + (r1 >> 2) - - add 0($inp),$h0 # accumulate input - adc 8($inp),$h1 - lea 16($inp),$inp - adc $padbit,$h2 - - call __poly1305_block - - test $padbit,$padbit # if $padbit is zero, - jz .Lstore_base2_64_avx # store hash in base 2^64 format - - ################################# base 2^64 -> base 2^26 - mov $h0,%rax - mov $h0,%rdx - shr \$52,$h0 - mov $h1,$r0 - mov $h1,$r1 - shr \$26,%rdx - and \$0x3ffffff,%rax # h[0] - shl \$12,$r0 - and \$0x3ffffff,%rdx # h[1] - shr \$14,$h1 - or $r0,$h0 - shl \$24,$h2 - and \$0x3ffffff,$h0 # h[2] - shr \$40,$r1 - and \$0x3ffffff,$h1 # h[3] - or $r1,$h2 # h[4] - - sub \$16,%r15 - jz .Lstore_base2_26_avx - - vmovd %rax#d,$H0 - vmovd %rdx#d,$H1 - vmovd $h0#d,$H2 - vmovd $h1#d,$H3 - vmovd $h2#d,$H4 - jmp .Lproceed_avx - -.align 32 -.Lstore_base2_64_avx: - mov $h0,0($ctx) - mov $h1,8($ctx) - mov $h2,16($ctx) # note that is_base2_26 is zeroed - jmp .Ldone_avx - -.align 16 -.Lstore_base2_26_avx: - mov %rax#d,0($ctx) # store hash value base 2^26 - mov %rdx#d,4($ctx) - mov $h0#d,8($ctx) - mov $h1#d,12($ctx) - mov $h2#d,16($ctx) -.align 16 -.Ldone_avx: - pop %r15 -.cfi_restore %r15 - pop %r14 -.cfi_restore %r14 - pop %r13 -.cfi_restore %r13 - pop %r12 -.cfi_restore %r12 - pop %rbx -.cfi_restore %rbx - pop %rbp -.cfi_restore %rbp -.Lno_data_avx: -.Lblocks_avx_epilogue: - ret -.cfi_endproc - -.align 32 -.Lbase2_64_avx: -.cfi_startproc - push %rbp -.cfi_push %rbp - mov %rsp,%rbp - push %rbx -.cfi_push %rbx - push %r12 -.cfi_push %r12 - push %r13 -.cfi_push %r13 - push %r14 -.cfi_push %r14 - push %r15 -.cfi_push %r15 -.Lbase2_64_avx_body: - - mov $len,%r15 # reassign $len - - mov 24($ctx),$r0 # load r - mov 32($ctx),$s1 - - mov 0($ctx),$h0 # load hash value - mov 8($ctx),$h1 - mov 16($ctx),$h2#d - - mov $s1,$r1 - mov $s1,%rax - shr \$2,$s1 - add $r1,$s1 # s1 = r1 + (r1 >> 2) - - test \$31,$len - jz .Linit_avx - - add 0($inp),$h0 # accumulate input - adc 8($inp),$h1 - lea 16($inp),$inp - adc $padbit,$h2 - sub \$16,%r15 - - call __poly1305_block - -.Linit_avx: - ################################# base 2^64 -> base 2^26 - mov $h0,%rax - mov $h0,%rdx - shr \$52,$h0 - mov $h1,$d1 - mov $h1,$d2 - shr \$26,%rdx - and \$0x3ffffff,%rax # h[0] - shl \$12,$d1 - and \$0x3ffffff,%rdx # h[1] - shr \$14,$h1 - or $d1,$h0 - shl \$24,$h2 - and \$0x3ffffff,$h0 # h[2] - shr \$40,$d2 - and \$0x3ffffff,$h1 # h[3] - or $d2,$h2 # h[4] - - vmovd %rax#d,$H0 - vmovd %rdx#d,$H1 - vmovd $h0#d,$H2 - vmovd $h1#d,$H3 - vmovd $h2#d,$H4 - movl \$1,20($ctx) # set is_base2_26 - - call __poly1305_init_avx - -.Lproceed_avx: - mov %r15,$len - pop %r15 -.cfi_restore %r15 - pop %r14 -.cfi_restore %r14 - pop %r13 -.cfi_restore %r13 - pop %r12 -.cfi_restore %r12 - pop %rbx -.cfi_restore %rbx - pop %rbp -.cfi_restore %rbp -.Lbase2_64_avx_epilogue: - jmp .Ldo_avx -.cfi_endproc - -.align 32 -.Leven_avx: -.cfi_startproc - vmovd 4*0($ctx),$H0 # load hash value - vmovd 4*1($ctx),$H1 - vmovd 4*2($ctx),$H2 - vmovd 4*3($ctx),$H3 - vmovd 4*4($ctx),$H4 - -.Ldo_avx: -___ -$code.=<<___ if (!$win64); - lea 8(%rsp),%r10 -.cfi_def_cfa_register %r10 - and \$-32,%rsp - sub \$-8,%rsp - lea -0x58(%rsp),%r11 - sub \$0x178,%rsp -___ -$code.=<<___ if ($win64); - lea -0xf8(%rsp),%r11 - sub \$0x218,%rsp - vmovdqa %xmm6,0x50(%r11) - vmovdqa %xmm7,0x60(%r11) - vmovdqa %xmm8,0x70(%r11) - vmovdqa %xmm9,0x80(%r11) - vmovdqa %xmm10,0x90(%r11) - vmovdqa %xmm11,0xa0(%r11) - vmovdqa %xmm12,0xb0(%r11) - vmovdqa %xmm13,0xc0(%r11) - vmovdqa %xmm14,0xd0(%r11) - vmovdqa %xmm15,0xe0(%r11) -.Ldo_avx_body: -___ -$code.=<<___; - sub \$64,$len - lea -32($inp),%rax - cmovc %rax,$inp - - vmovdqu `16*3`($ctx),$D4 # preload r0^2 - lea `16*3+64`($ctx),$ctx # size optimization - lea .Lconst(%rip),%rcx - - ################################################################ - # load input - vmovdqu 16*2($inp),$T0 - vmovdqu 16*3($inp),$T1 - vmovdqa 64(%rcx),$MASK # .Lmask26 - - vpsrldq \$6,$T0,$T2 # splat input - vpsrldq \$6,$T1,$T3 - vpunpckhqdq $T1,$T0,$T4 # 4 - vpunpcklqdq $T1,$T0,$T0 # 0:1 - vpunpcklqdq $T3,$T2,$T3 # 2:3 - - vpsrlq \$40,$T4,$T4 # 4 - vpsrlq \$26,$T0,$T1 - vpand $MASK,$T0,$T0 # 0 - vpsrlq \$4,$T3,$T2 - vpand $MASK,$T1,$T1 # 1 - vpsrlq \$30,$T3,$T3 - vpand $MASK,$T2,$T2 # 2 - vpand $MASK,$T3,$T3 # 3 - vpor 32(%rcx),$T4,$T4 # padbit, yes, always - - jbe .Lskip_loop_avx - - # expand and copy pre-calculated table to stack - vmovdqu `16*1-64`($ctx),$D1 - vmovdqu `16*2-64`($ctx),$D2 - vpshufd \$0xEE,$D4,$D3 # 34xx -> 3434 - vpshufd \$0x44,$D4,$D0 # xx12 -> 1212 - vmovdqa $D3,-0x90(%r11) - vmovdqa $D0,0x00(%rsp) - vpshufd \$0xEE,$D1,$D4 - vmovdqu `16*3-64`($ctx),$D0 - vpshufd \$0x44,$D1,$D1 - vmovdqa $D4,-0x80(%r11) - vmovdqa $D1,0x10(%rsp) - vpshufd \$0xEE,$D2,$D3 - vmovdqu `16*4-64`($ctx),$D1 - vpshufd \$0x44,$D2,$D2 - vmovdqa $D3,-0x70(%r11) - vmovdqa $D2,0x20(%rsp) - vpshufd \$0xEE,$D0,$D4 - vmovdqu `16*5-64`($ctx),$D2 - vpshufd \$0x44,$D0,$D0 - vmovdqa $D4,-0x60(%r11) - vmovdqa $D0,0x30(%rsp) - vpshufd \$0xEE,$D1,$D3 - vmovdqu `16*6-64`($ctx),$D0 - vpshufd \$0x44,$D1,$D1 - vmovdqa $D3,-0x50(%r11) - vmovdqa $D1,0x40(%rsp) - vpshufd \$0xEE,$D2,$D4 - vmovdqu `16*7-64`($ctx),$D1 - vpshufd \$0x44,$D2,$D2 - vmovdqa $D4,-0x40(%r11) - vmovdqa $D2,0x50(%rsp) - vpshufd \$0xEE,$D0,$D3 - vmovdqu `16*8-64`($ctx),$D2 - vpshufd \$0x44,$D0,$D0 - vmovdqa $D3,-0x30(%r11) - vmovdqa $D0,0x60(%rsp) - vpshufd \$0xEE,$D1,$D4 - vpshufd \$0x44,$D1,$D1 - vmovdqa $D4,-0x20(%r11) - vmovdqa $D1,0x70(%rsp) - vpshufd \$0xEE,$D2,$D3 - vmovdqa 0x00(%rsp),$D4 # preload r0^2 - vpshufd \$0x44,$D2,$D2 - vmovdqa $D3,-0x10(%r11) - vmovdqa $D2,0x80(%rsp) - - jmp .Loop_avx - -.align 32 -.Loop_avx: - ################################################################ - # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 - # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r - # \___________________/ - # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 - # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r - # \___________________/ \____________________/ - # - # Note that we start with inp[2:3]*r^2. This is because it - # doesn't depend on reduction in previous iteration. - ################################################################ - # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 - # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 - # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 - # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 - # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 - # - # though note that $Tx and $Hx are "reversed" in this section, - # and $D4 is preloaded with r0^2... - - vpmuludq $T0,$D4,$D0 # d0 = h0*r0 - vpmuludq $T1,$D4,$D1 # d1 = h1*r0 - vmovdqa $H2,0x20(%r11) # offload hash - vpmuludq $T2,$D4,$D2 # d3 = h2*r0 - vmovdqa 0x10(%rsp),$H2 # r1^2 - vpmuludq $T3,$D4,$D3 # d3 = h3*r0 - vpmuludq $T4,$D4,$D4 # d4 = h4*r0 - - vmovdqa $H0,0x00(%r11) # - vpmuludq 0x20(%rsp),$T4,$H0 # h4*s1 - vmovdqa $H1,0x10(%r11) # - vpmuludq $T3,$H2,$H1 # h3*r1 - vpaddq $H0,$D0,$D0 # d0 += h4*s1 - vpaddq $H1,$D4,$D4 # d4 += h3*r1 - vmovdqa $H3,0x30(%r11) # - vpmuludq $T2,$H2,$H0 # h2*r1 - vpmuludq $T1,$H2,$H1 # h1*r1 - vpaddq $H0,$D3,$D3 # d3 += h2*r1 - vmovdqa 0x30(%rsp),$H3 # r2^2 - vpaddq $H1,$D2,$D2 # d2 += h1*r1 - vmovdqa $H4,0x40(%r11) # - vpmuludq $T0,$H2,$H2 # h0*r1 - vpmuludq $T2,$H3,$H0 # h2*r2 - vpaddq $H2,$D1,$D1 # d1 += h0*r1 - - vmovdqa 0x40(%rsp),$H4 # s2^2 - vpaddq $H0,$D4,$D4 # d4 += h2*r2 - vpmuludq $T1,$H3,$H1 # h1*r2 - vpmuludq $T0,$H3,$H3 # h0*r2 - vpaddq $H1,$D3,$D3 # d3 += h1*r2 - vmovdqa 0x50(%rsp),$H2 # r3^2 - vpaddq $H3,$D2,$D2 # d2 += h0*r2 - vpmuludq $T4,$H4,$H0 # h4*s2 - vpmuludq $T3,$H4,$H4 # h3*s2 - vpaddq $H0,$D1,$D1 # d1 += h4*s2 - vmovdqa 0x60(%rsp),$H3 # s3^2 - vpaddq $H4,$D0,$D0 # d0 += h3*s2 - - vmovdqa 0x80(%rsp),$H4 # s4^2 - vpmuludq $T1,$H2,$H1 # h1*r3 - vpmuludq $T0,$H2,$H2 # h0*r3 - vpaddq $H1,$D4,$D4 # d4 += h1*r3 - vpaddq $H2,$D3,$D3 # d3 += h0*r3 - vpmuludq $T4,$H3,$H0 # h4*s3 - vpmuludq $T3,$H3,$H1 # h3*s3 - vpaddq $H0,$D2,$D2 # d2 += h4*s3 - vmovdqu 16*0($inp),$H0 # load input - vpaddq $H1,$D1,$D1 # d1 += h3*s3 - vpmuludq $T2,$H3,$H3 # h2*s3 - vpmuludq $T2,$H4,$T2 # h2*s4 - vpaddq $H3,$D0,$D0 # d0 += h2*s3 - - vmovdqu 16*1($inp),$H1 # - vpaddq $T2,$D1,$D1 # d1 += h2*s4 - vpmuludq $T3,$H4,$T3 # h3*s4 - vpmuludq $T4,$H4,$T4 # h4*s4 - vpsrldq \$6,$H0,$H2 # splat input - vpaddq $T3,$D2,$D2 # d2 += h3*s4 - vpaddq $T4,$D3,$D3 # d3 += h4*s4 - vpsrldq \$6,$H1,$H3 # - vpmuludq 0x70(%rsp),$T0,$T4 # h0*r4 - vpmuludq $T1,$H4,$T0 # h1*s4 - vpunpckhqdq $H1,$H0,$H4 # 4 - vpaddq $T4,$D4,$D4 # d4 += h0*r4 - vmovdqa -0x90(%r11),$T4 # r0^4 - vpaddq $T0,$D0,$D0 # d0 += h1*s4 - - vpunpcklqdq $H1,$H0,$H0 # 0:1 - vpunpcklqdq $H3,$H2,$H3 # 2:3 - - #vpsrlq \$40,$H4,$H4 # 4 - vpsrldq \$`40/8`,$H4,$H4 # 4 - vpsrlq \$26,$H0,$H1 - vpand $MASK,$H0,$H0 # 0 - vpsrlq \$4,$H3,$H2 - vpand $MASK,$H1,$H1 # 1 - vpand 0(%rcx),$H4,$H4 # .Lmask24 - vpsrlq \$30,$H3,$H3 - vpand $MASK,$H2,$H2 # 2 - vpand $MASK,$H3,$H3 # 3 - vpor 32(%rcx),$H4,$H4 # padbit, yes, always - - vpaddq 0x00(%r11),$H0,$H0 # add hash value - vpaddq 0x10(%r11),$H1,$H1 - vpaddq 0x20(%r11),$H2,$H2 - vpaddq 0x30(%r11),$H3,$H3 - vpaddq 0x40(%r11),$H4,$H4 - - lea 16*2($inp),%rax - lea 16*4($inp),$inp - sub \$64,$len - cmovc %rax,$inp - - ################################################################ - # Now we accumulate (inp[0:1]+hash)*r^4 - ################################################################ - # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 - # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 - # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 - # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 - # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 - - vpmuludq $H0,$T4,$T0 # h0*r0 - vpmuludq $H1,$T4,$T1 # h1*r0 - vpaddq $T0,$D0,$D0 - vpaddq $T1,$D1,$D1 - vmovdqa -0x80(%r11),$T2 # r1^4 - vpmuludq $H2,$T4,$T0 # h2*r0 - vpmuludq $H3,$T4,$T1 # h3*r0 - vpaddq $T0,$D2,$D2 - vpaddq $T1,$D3,$D3 - vpmuludq $H4,$T4,$T4 # h4*r0 - vpmuludq -0x70(%r11),$H4,$T0 # h4*s1 - vpaddq $T4,$D4,$D4 - - vpaddq $T0,$D0,$D0 # d0 += h4*s1 - vpmuludq $H2,$T2,$T1 # h2*r1 - vpmuludq $H3,$T2,$T0 # h3*r1 - vpaddq $T1,$D3,$D3 # d3 += h2*r1 - vmovdqa -0x60(%r11),$T3 # r2^4 - vpaddq $T0,$D4,$D4 # d4 += h3*r1 - vpmuludq $H1,$T2,$T1 # h1*r1 - vpmuludq $H0,$T2,$T2 # h0*r1 - vpaddq $T1,$D2,$D2 # d2 += h1*r1 - vpaddq $T2,$D1,$D1 # d1 += h0*r1 - - vmovdqa -0x50(%r11),$T4 # s2^4 - vpmuludq $H2,$T3,$T0 # h2*r2 - vpmuludq $H1,$T3,$T1 # h1*r2 - vpaddq $T0,$D4,$D4 # d4 += h2*r2 - vpaddq $T1,$D3,$D3 # d3 += h1*r2 - vmovdqa -0x40(%r11),$T2 # r3^4 - vpmuludq $H0,$T3,$T3 # h0*r2 - vpmuludq $H4,$T4,$T0 # h4*s2 - vpaddq $T3,$D2,$D2 # d2 += h0*r2 - vpaddq $T0,$D1,$D1 # d1 += h4*s2 - vmovdqa -0x30(%r11),$T3 # s3^4 - vpmuludq $H3,$T4,$T4 # h3*s2 - vpmuludq $H1,$T2,$T1 # h1*r3 - vpaddq $T4,$D0,$D0 # d0 += h3*s2 - - vmovdqa -0x10(%r11),$T4 # s4^4 - vpaddq $T1,$D4,$D4 # d4 += h1*r3 - vpmuludq $H0,$T2,$T2 # h0*r3 - vpmuludq $H4,$T3,$T0 # h4*s3 - vpaddq $T2,$D3,$D3 # d3 += h0*r3 - vpaddq $T0,$D2,$D2 # d2 += h4*s3 - vmovdqu 16*2($inp),$T0 # load input - vpmuludq $H3,$T3,$T2 # h3*s3 - vpmuludq $H2,$T3,$T3 # h2*s3 - vpaddq $T2,$D1,$D1 # d1 += h3*s3 - vmovdqu 16*3($inp),$T1 # - vpaddq $T3,$D0,$D0 # d0 += h2*s3 - - vpmuludq $H2,$T4,$H2 # h2*s4 - vpmuludq $H3,$T4,$H3 # h3*s4 - vpsrldq \$6,$T0,$T2 # splat input - vpaddq $H2,$D1,$D1 # d1 += h2*s4 - vpmuludq $H4,$T4,$H4 # h4*s4 - vpsrldq \$6,$T1,$T3 # - vpaddq $H3,$D2,$H2 # h2 = d2 + h3*s4 - vpaddq $H4,$D3,$H3 # h3 = d3 + h4*s4 - vpmuludq -0x20(%r11),$H0,$H4 # h0*r4 - vpmuludq $H1,$T4,$H0 - vpunpckhqdq $T1,$T0,$T4 # 4 - vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 - vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 - - vpunpcklqdq $T1,$T0,$T0 # 0:1 - vpunpcklqdq $T3,$T2,$T3 # 2:3 - - #vpsrlq \$40,$T4,$T4 # 4 - vpsrldq \$`40/8`,$T4,$T4 # 4 - vpsrlq \$26,$T0,$T1 - vmovdqa 0x00(%rsp),$D4 # preload r0^2 - vpand $MASK,$T0,$T0 # 0 - vpsrlq \$4,$T3,$T2 - vpand $MASK,$T1,$T1 # 1 - vpand 0(%rcx),$T4,$T4 # .Lmask24 - vpsrlq \$30,$T3,$T3 - vpand $MASK,$T2,$T2 # 2 - vpand $MASK,$T3,$T3 # 3 - vpor 32(%rcx),$T4,$T4 # padbit, yes, always - - ################################################################ - # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein - # and P. Schwabe - - vpsrlq \$26,$H3,$D3 - vpand $MASK,$H3,$H3 - vpaddq $D3,$H4,$H4 # h3 -> h4 - - vpsrlq \$26,$H0,$D0 - vpand $MASK,$H0,$H0 - vpaddq $D0,$D1,$H1 # h0 -> h1 - - vpsrlq \$26,$H4,$D0 - vpand $MASK,$H4,$H4 - - vpsrlq \$26,$H1,$D1 - vpand $MASK,$H1,$H1 - vpaddq $D1,$H2,$H2 # h1 -> h2 - - vpaddq $D0,$H0,$H0 - vpsllq \$2,$D0,$D0 - vpaddq $D0,$H0,$H0 # h4 -> h0 - - vpsrlq \$26,$H2,$D2 - vpand $MASK,$H2,$H2 - vpaddq $D2,$H3,$H3 # h2 -> h3 - - vpsrlq \$26,$H0,$D0 - vpand $MASK,$H0,$H0 - vpaddq $D0,$H1,$H1 # h0 -> h1 - - vpsrlq \$26,$H3,$D3 - vpand $MASK,$H3,$H3 - vpaddq $D3,$H4,$H4 # h3 -> h4 - - ja .Loop_avx - -.Lskip_loop_avx: - ################################################################ - # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 - - vpshufd \$0x10,$D4,$D4 # r0^n, xx12 -> x1x2 - add \$32,$len - jnz .Long_tail_avx - - vpaddq $H2,$T2,$T2 - vpaddq $H0,$T0,$T0 - vpaddq $H1,$T1,$T1 - vpaddq $H3,$T3,$T3 - vpaddq $H4,$T4,$T4 - -.Long_tail_avx: - vmovdqa $H2,0x20(%r11) - vmovdqa $H0,0x00(%r11) - vmovdqa $H1,0x10(%r11) - vmovdqa $H3,0x30(%r11) - vmovdqa $H4,0x40(%r11) - - # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 - # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 - # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 - # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 - # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 - - vpmuludq $T2,$D4,$D2 # d2 = h2*r0 - vpmuludq $T0,$D4,$D0 # d0 = h0*r0 - vpshufd \$0x10,`16*1-64`($ctx),$H2 # r1^n - vpmuludq $T1,$D4,$D1 # d1 = h1*r0 - vpmuludq $T3,$D4,$D3 # d3 = h3*r0 - vpmuludq $T4,$D4,$D4 # d4 = h4*r0 - - vpmuludq $T3,$H2,$H0 # h3*r1 - vpaddq $H0,$D4,$D4 # d4 += h3*r1 - vpshufd \$0x10,`16*2-64`($ctx),$H3 # s1^n - vpmuludq $T2,$H2,$H1 # h2*r1 - vpaddq $H1,$D3,$D3 # d3 += h2*r1 - vpshufd \$0x10,`16*3-64`($ctx),$H4 # r2^n - vpmuludq $T1,$H2,$H0 # h1*r1 - vpaddq $H0,$D2,$D2 # d2 += h1*r1 - vpmuludq $T0,$H2,$H2 # h0*r1 - vpaddq $H2,$D1,$D1 # d1 += h0*r1 - vpmuludq $T4,$H3,$H3 # h4*s1 - vpaddq $H3,$D0,$D0 # d0 += h4*s1 - - vpshufd \$0x10,`16*4-64`($ctx),$H2 # s2^n - vpmuludq $T2,$H4,$H1 # h2*r2 - vpaddq $H1,$D4,$D4 # d4 += h2*r2 - vpmuludq $T1,$H4,$H0 # h1*r2 - vpaddq $H0,$D3,$D3 # d3 += h1*r2 - vpshufd \$0x10,`16*5-64`($ctx),$H3 # r3^n - vpmuludq $T0,$H4,$H4 # h0*r2 - vpaddq $H4,$D2,$D2 # d2 += h0*r2 - vpmuludq $T4,$H2,$H1 # h4*s2 - vpaddq $H1,$D1,$D1 # d1 += h4*s2 - vpshufd \$0x10,`16*6-64`($ctx),$H4 # s3^n - vpmuludq $T3,$H2,$H2 # h3*s2 - vpaddq $H2,$D0,$D0 # d0 += h3*s2 - - vpmuludq $T1,$H3,$H0 # h1*r3 - vpaddq $H0,$D4,$D4 # d4 += h1*r3 - vpmuludq $T0,$H3,$H3 # h0*r3 - vpaddq $H3,$D3,$D3 # d3 += h0*r3 - vpshufd \$0x10,`16*7-64`($ctx),$H2 # r4^n - vpmuludq $T4,$H4,$H1 # h4*s3 - vpaddq $H1,$D2,$D2 # d2 += h4*s3 - vpshufd \$0x10,`16*8-64`($ctx),$H3 # s4^n - vpmuludq $T3,$H4,$H0 # h3*s3 - vpaddq $H0,$D1,$D1 # d1 += h3*s3 - vpmuludq $T2,$H4,$H4 # h2*s3 - vpaddq $H4,$D0,$D0 # d0 += h2*s3 - - vpmuludq $T0,$H2,$H2 # h0*r4 - vpaddq $H2,$D4,$D4 # h4 = d4 + h0*r4 - vpmuludq $T4,$H3,$H1 # h4*s4 - vpaddq $H1,$D3,$D3 # h3 = d3 + h4*s4 - vpmuludq $T3,$H3,$H0 # h3*s4 - vpaddq $H0,$D2,$D2 # h2 = d2 + h3*s4 - vpmuludq $T2,$H3,$H1 # h2*s4 - vpaddq $H1,$D1,$D1 # h1 = d1 + h2*s4 - vpmuludq $T1,$H3,$H3 # h1*s4 - vpaddq $H3,$D0,$D0 # h0 = d0 + h1*s4 - - jz .Lshort_tail_avx - - vmovdqu 16*0($inp),$H0 # load input - vmovdqu 16*1($inp),$H1 - - vpsrldq \$6,$H0,$H2 # splat input - vpsrldq \$6,$H1,$H3 - vpunpckhqdq $H1,$H0,$H4 # 4 - vpunpcklqdq $H1,$H0,$H0 # 0:1 - vpunpcklqdq $H3,$H2,$H3 # 2:3 - - vpsrlq \$40,$H4,$H4 # 4 - vpsrlq \$26,$H0,$H1 - vpand $MASK,$H0,$H0 # 0 - vpsrlq \$4,$H3,$H2 - vpand $MASK,$H1,$H1 # 1 - vpsrlq \$30,$H3,$H3 - vpand $MASK,$H2,$H2 # 2 - vpand $MASK,$H3,$H3 # 3 - vpor 32(%rcx),$H4,$H4 # padbit, yes, always - - vpshufd \$0x32,`16*0-64`($ctx),$T4 # r0^n, 34xx -> x3x4 - vpaddq 0x00(%r11),$H0,$H0 - vpaddq 0x10(%r11),$H1,$H1 - vpaddq 0x20(%r11),$H2,$H2 - vpaddq 0x30(%r11),$H3,$H3 - vpaddq 0x40(%r11),$H4,$H4 - - ################################################################ - # multiply (inp[0:1]+hash) by r^4:r^3 and accumulate - - vpmuludq $H0,$T4,$T0 # h0*r0 - vpaddq $T0,$D0,$D0 # d0 += h0*r0 - vpmuludq $H1,$T4,$T1 # h1*r0 - vpaddq $T1,$D1,$D1 # d1 += h1*r0 - vpmuludq $H2,$T4,$T0 # h2*r0 - vpaddq $T0,$D2,$D2 # d2 += h2*r0 - vpshufd \$0x32,`16*1-64`($ctx),$T2 # r1^n - vpmuludq $H3,$T4,$T1 # h3*r0 - vpaddq $T1,$D3,$D3 # d3 += h3*r0 - vpmuludq $H4,$T4,$T4 # h4*r0 - vpaddq $T4,$D4,$D4 # d4 += h4*r0 - - vpmuludq $H3,$T2,$T0 # h3*r1 - vpaddq $T0,$D4,$D4 # d4 += h3*r1 - vpshufd \$0x32,`16*2-64`($ctx),$T3 # s1 - vpmuludq $H2,$T2,$T1 # h2*r1 - vpaddq $T1,$D3,$D3 # d3 += h2*r1 - vpshufd \$0x32,`16*3-64`($ctx),$T4 # r2 - vpmuludq $H1,$T2,$T0 # h1*r1 - vpaddq $T0,$D2,$D2 # d2 += h1*r1 - vpmuludq $H0,$T2,$T2 # h0*r1 - vpaddq $T2,$D1,$D1 # d1 += h0*r1 - vpmuludq $H4,$T3,$T3 # h4*s1 - vpaddq $T3,$D0,$D0 # d0 += h4*s1 - - vpshufd \$0x32,`16*4-64`($ctx),$T2 # s2 - vpmuludq $H2,$T4,$T1 # h2*r2 - vpaddq $T1,$D4,$D4 # d4 += h2*r2 - vpmuludq $H1,$T4,$T0 # h1*r2 - vpaddq $T0,$D3,$D3 # d3 += h1*r2 - vpshufd \$0x32,`16*5-64`($ctx),$T3 # r3 - vpmuludq $H0,$T4,$T4 # h0*r2 - vpaddq $T4,$D2,$D2 # d2 += h0*r2 - vpmuludq $H4,$T2,$T1 # h4*s2 - vpaddq $T1,$D1,$D1 # d1 += h4*s2 - vpshufd \$0x32,`16*6-64`($ctx),$T4 # s3 - vpmuludq $H3,$T2,$T2 # h3*s2 - vpaddq $T2,$D0,$D0 # d0 += h3*s2 - - vpmuludq $H1,$T3,$T0 # h1*r3 - vpaddq $T0,$D4,$D4 # d4 += h1*r3 - vpmuludq $H0,$T3,$T3 # h0*r3 - vpaddq $T3,$D3,$D3 # d3 += h0*r3 - vpshufd \$0x32,`16*7-64`($ctx),$T2 # r4 - vpmuludq $H4,$T4,$T1 # h4*s3 - vpaddq $T1,$D2,$D2 # d2 += h4*s3 - vpshufd \$0x32,`16*8-64`($ctx),$T3 # s4 - vpmuludq $H3,$T4,$T0 # h3*s3 - vpaddq $T0,$D1,$D1 # d1 += h3*s3 - vpmuludq $H2,$T4,$T4 # h2*s3 - vpaddq $T4,$D0,$D0 # d0 += h2*s3 - - vpmuludq $H0,$T2,$T2 # h0*r4 - vpaddq $T2,$D4,$D4 # d4 += h0*r4 - vpmuludq $H4,$T3,$T1 # h4*s4 - vpaddq $T1,$D3,$D3 # d3 += h4*s4 - vpmuludq $H3,$T3,$T0 # h3*s4 - vpaddq $T0,$D2,$D2 # d2 += h3*s4 - vpmuludq $H2,$T3,$T1 # h2*s4 - vpaddq $T1,$D1,$D1 # d1 += h2*s4 - vpmuludq $H1,$T3,$T3 # h1*s4 - vpaddq $T3,$D0,$D0 # d0 += h1*s4 - -.Lshort_tail_avx: - ################################################################ - # horizontal addition - - vpsrldq \$8,$D4,$T4 - vpsrldq \$8,$D3,$T3 - vpsrldq \$8,$D1,$T1 - vpsrldq \$8,$D0,$T0 - vpsrldq \$8,$D2,$T2 - vpaddq $T3,$D3,$D3 - vpaddq $T4,$D4,$D4 - vpaddq $T0,$D0,$D0 - vpaddq $T1,$D1,$D1 - vpaddq $T2,$D2,$D2 - - ################################################################ - # lazy reduction - - vpsrlq \$26,$D3,$H3 - vpand $MASK,$D3,$D3 - vpaddq $H3,$D4,$D4 # h3 -> h4 - - vpsrlq \$26,$D0,$H0 - vpand $MASK,$D0,$D0 - vpaddq $H0,$D1,$D1 # h0 -> h1 - - vpsrlq \$26,$D4,$H4 - vpand $MASK,$D4,$D4 - - vpsrlq \$26,$D1,$H1 - vpand $MASK,$D1,$D1 - vpaddq $H1,$D2,$D2 # h1 -> h2 - - vpaddq $H4,$D0,$D0 - vpsllq \$2,$H4,$H4 - vpaddq $H4,$D0,$D0 # h4 -> h0 - - vpsrlq \$26,$D2,$H2 - vpand $MASK,$D2,$D2 - vpaddq $H2,$D3,$D3 # h2 -> h3 - - vpsrlq \$26,$D0,$H0 - vpand $MASK,$D0,$D0 - vpaddq $H0,$D1,$D1 # h0 -> h1 - - vpsrlq \$26,$D3,$H3 - vpand $MASK,$D3,$D3 - vpaddq $H3,$D4,$D4 # h3 -> h4 - - vmovd $D0,`4*0-48-64`($ctx) # save partially reduced - vmovd $D1,`4*1-48-64`($ctx) - vmovd $D2,`4*2-48-64`($ctx) - vmovd $D3,`4*3-48-64`($ctx) - vmovd $D4,`4*4-48-64`($ctx) -___ -$code.=<<___ if ($win64); - vmovdqa 0x50(%r11),%xmm6 - vmovdqa 0x60(%r11),%xmm7 - vmovdqa 0x70(%r11),%xmm8 - vmovdqa 0x80(%r11),%xmm9 - vmovdqa 0x90(%r11),%xmm10 - vmovdqa 0xa0(%r11),%xmm11 - vmovdqa 0xb0(%r11),%xmm12 - vmovdqa 0xc0(%r11),%xmm13 - vmovdqa 0xd0(%r11),%xmm14 - vmovdqa 0xe0(%r11),%xmm15 - lea 0xf8(%r11),%rsp -.Ldo_avx_epilogue: -___ -$code.=<<___ if (!$win64); - lea -8(%r10),%rsp -.cfi_def_cfa_register %rsp -___ -$code.=<<___; - vzeroupper - ret -.cfi_endproc -___ -&end_function("poly1305_blocks_avx"); - -&declare_function("poly1305_emit_avx", 32, 3); -$code.=<<___; - cmpl \$0,20($ctx) # is_base2_26? - je .Lemit - - mov 0($ctx),%eax # load hash value base 2^26 - mov 4($ctx),%ecx - mov 8($ctx),%r8d - mov 12($ctx),%r11d - mov 16($ctx),%r10d - - shl \$26,%rcx # base 2^26 -> base 2^64 - mov %r8,%r9 - shl \$52,%r8 - add %rcx,%rax - shr \$12,%r9 - add %rax,%r8 # h0 - adc \$0,%r9 - - shl \$14,%r11 - mov %r10,%rax - shr \$24,%r10 - add %r11,%r9 - shl \$40,%rax - add %rax,%r9 # h1 - adc \$0,%r10 # h2 - - mov %r10,%rax # could be partially reduced, so reduce - mov %r10,%rcx - and \$3,%r10 - shr \$2,%rax - and \$-4,%rcx - add %rcx,%rax - add %rax,%r8 - adc \$0,%r9 - adc \$0,%r10 - - mov %r8,%rax - add \$5,%r8 # compare to modulus - mov %r9,%rcx - adc \$0,%r9 - adc \$0,%r10 - shr \$2,%r10 # did 130-bit value overflow? - cmovnz %r8,%rax - cmovnz %r9,%rcx - - add 0($nonce),%rax # accumulate nonce - adc 8($nonce),%rcx - mov %rax,0($mac) # write result - mov %rcx,8($mac) - - ret -___ -&end_function("poly1305_emit_avx"); - -if ($kernel) { - $code .= "#endif\n"; -} - -if ($avx>1) { - -if ($kernel) { - $code .= "#ifdef CONFIG_AS_AVX2\n"; -} - -my ($H0,$H1,$H2,$H3,$H4, $MASK, $T4,$T0,$T1,$T2,$T3, $D0,$D1,$D2,$D3,$D4) = - map("%ymm$_",(0..15)); -my $S4=$MASK; - -sub poly1305_blocks_avxN { - my ($avx512) = @_; - my $suffix = $avx512 ? "_avx512" : ""; -$code.=<<___; -.cfi_startproc - mov 20($ctx),%r8d # is_base2_26 - cmp \$128,$len - jae .Lblocks_avx2$suffix - test %r8d,%r8d - jz .Lblocks - -.Lblocks_avx2$suffix: - and \$-16,$len - jz .Lno_data_avx2$suffix - - vzeroupper - - test %r8d,%r8d - jz .Lbase2_64_avx2$suffix - - test \$63,$len - jz .Leven_avx2$suffix - - push %rbp -.cfi_push %rbp - mov %rsp,%rbp - push %rbx -.cfi_push %rbx - push %r12 -.cfi_push %r12 - push %r13 -.cfi_push %r13 - push %r14 -.cfi_push %r14 - push %r15 -.cfi_push %r15 -.Lblocks_avx2_body$suffix: - - mov $len,%r15 # reassign $len - - mov 0($ctx),$d1 # load hash value - mov 8($ctx),$d2 - mov 16($ctx),$h2#d - - mov 24($ctx),$r0 # load r - mov 32($ctx),$s1 - - ################################# base 2^26 -> base 2^64 - mov $d1#d,$h0#d - and \$`-1*(1<<31)`,$d1 - mov $d2,$r1 # borrow $r1 - mov $d2#d,$h1#d - and \$`-1*(1<<31)`,$d2 - - shr \$6,$d1 - shl \$52,$r1 - add $d1,$h0 - shr \$12,$h1 - shr \$18,$d2 - add $r1,$h0 - adc $d2,$h1 - - mov $h2,$d1 - shl \$40,$d1 - shr \$24,$h2 - add $d1,$h1 - adc \$0,$h2 # can be partially reduced... - - mov \$-4,$d2 # ... so reduce - mov $h2,$d1 - and $h2,$d2 - shr \$2,$d1 - and \$3,$h2 - add $d2,$d1 # =*5 - add $d1,$h0 - adc \$0,$h1 - adc \$0,$h2 - - mov $s1,$r1 - mov $s1,%rax - shr \$2,$s1 - add $r1,$s1 # s1 = r1 + (r1 >> 2) - -.Lbase2_26_pre_avx2$suffix: - add 0($inp),$h0 # accumulate input - adc 8($inp),$h1 - lea 16($inp),$inp - adc $padbit,$h2 - sub \$16,%r15 - - call __poly1305_block - mov $r1,%rax - - test \$63,%r15 - jnz .Lbase2_26_pre_avx2$suffix - - test $padbit,$padbit # if $padbit is zero, - jz .Lstore_base2_64_avx2$suffix # store hash in base 2^64 format - - ################################# base 2^64 -> base 2^26 - mov $h0,%rax - mov $h0,%rdx - shr \$52,$h0 - mov $h1,$r0 - mov $h1,$r1 - shr \$26,%rdx - and \$0x3ffffff,%rax # h[0] - shl \$12,$r0 - and \$0x3ffffff,%rdx # h[1] - shr \$14,$h1 - or $r0,$h0 - shl \$24,$h2 - and \$0x3ffffff,$h0 # h[2] - shr \$40,$r1 - and \$0x3ffffff,$h1 # h[3] - or $r1,$h2 # h[4] - - test %r15,%r15 - jz .Lstore_base2_26_avx2$suffix - - vmovd %rax#d,%x#$H0 - vmovd %rdx#d,%x#$H1 - vmovd $h0#d,%x#$H2 - vmovd $h1#d,%x#$H3 - vmovd $h2#d,%x#$H4 - jmp .Lproceed_avx2$suffix - -.align 32 -.Lstore_base2_64_avx2$suffix: - mov $h0,0($ctx) - mov $h1,8($ctx) - mov $h2,16($ctx) # note that is_base2_26 is zeroed - jmp .Ldone_avx2$suffix - -.align 16 -.Lstore_base2_26_avx2$suffix: - mov %rax#d,0($ctx) # store hash value base 2^26 - mov %rdx#d,4($ctx) - mov $h0#d,8($ctx) - mov $h1#d,12($ctx) - mov $h2#d,16($ctx) -.align 16 -.Ldone_avx2$suffix: - pop %r15 -.cfi_restore %r15 - pop %r14 -.cfi_restore %r14 - pop %r13 -.cfi_restore %r13 - pop %r12 -.cfi_restore %r12 - pop %rbx -.cfi_restore %rbx - pop %rbp -.cfi_restore %rbp -.Lno_data_avx2$suffix: -.Lblocks_avx2_epilogue$suffix: - ret -.cfi_endproc - -.align 32 -.Lbase2_64_avx2$suffix: -.cfi_startproc - push %rbp -.cfi_push %rbp - mov %rsp,%rbp - push %rbx -.cfi_push %rbx - push %r12 -.cfi_push %r12 - push %r13 -.cfi_push %r13 - push %r14 -.cfi_push %r14 - push %r15 -.cfi_push %r15 -.Lbase2_64_avx2_body$suffix: - - mov $len,%r15 # reassign $len - - mov 24($ctx),$r0 # load r - mov 32($ctx),$s1 - - mov 0($ctx),$h0 # load hash value - mov 8($ctx),$h1 - mov 16($ctx),$h2#d - - mov $s1,$r1 - mov $s1,%rax - shr \$2,$s1 - add $r1,$s1 # s1 = r1 + (r1 >> 2) - - test \$63,$len - jz .Linit_avx2$suffix - -.Lbase2_64_pre_avx2$suffix: - add 0($inp),$h0 # accumulate input - adc 8($inp),$h1 - lea 16($inp),$inp - adc $padbit,$h2 - sub \$16,%r15 - - call __poly1305_block - mov $r1,%rax - - test \$63,%r15 - jnz .Lbase2_64_pre_avx2$suffix - -.Linit_avx2$suffix: - ################################# base 2^64 -> base 2^26 - mov $h0,%rax - mov $h0,%rdx - shr \$52,$h0 - mov $h1,$d1 - mov $h1,$d2 - shr \$26,%rdx - and \$0x3ffffff,%rax # h[0] - shl \$12,$d1 - and \$0x3ffffff,%rdx # h[1] - shr \$14,$h1 - or $d1,$h0 - shl \$24,$h2 - and \$0x3ffffff,$h0 # h[2] - shr \$40,$d2 - and \$0x3ffffff,$h1 # h[3] - or $d2,$h2 # h[4] - - vmovd %rax#d,%x#$H0 - vmovd %rdx#d,%x#$H1 - vmovd $h0#d,%x#$H2 - vmovd $h1#d,%x#$H3 - vmovd $h2#d,%x#$H4 - movl \$1,20($ctx) # set is_base2_26 - - call __poly1305_init_avx - -.Lproceed_avx2$suffix: - mov %r15,$len # restore $len -___ -$code.=<<___ if (!$kernel); - mov OPENSSL_ia32cap_P+8(%rip),%r9d - mov \$`(1<<31|1<<30|1<<16)`,%r11d -___ -$code.=<<___; - pop %r15 -.cfi_restore %r15 - pop %r14 -.cfi_restore %r14 - pop %r13 -.cfi_restore %r13 - pop %r12 -.cfi_restore %r12 - pop %rbx -.cfi_restore %rbx - pop %rbp -.cfi_restore %rbp -.Lbase2_64_avx2_epilogue$suffix: - jmp .Ldo_avx2$suffix -.cfi_endproc - -.align 32 -.Leven_avx2$suffix: -.cfi_startproc -___ -$code.=<<___ if (!$kernel); - mov OPENSSL_ia32cap_P+8(%rip),%r9d -___ -$code.=<<___; - vmovd 4*0($ctx),%x#$H0 # load hash value base 2^26 - vmovd 4*1($ctx),%x#$H1 - vmovd 4*2($ctx),%x#$H2 - vmovd 4*3($ctx),%x#$H3 - vmovd 4*4($ctx),%x#$H4 - -.Ldo_avx2$suffix: -___ -$code.=<<___ if (!$kernel && $avx>2); - cmp \$512,$len - jb .Lskip_avx512 - and %r11d,%r9d - test \$`1<<16`,%r9d # check for AVX512F - jnz .Lblocks_avx512 -.Lskip_avx512$suffix: -___ -$code.=<<___ if ($avx > 2 && $avx512 && $kernel); - cmp \$512,$len - jae .Lblocks_avx512 -___ -$code.=<<___ if (!$win64); - lea 8(%rsp),%r10 -.cfi_def_cfa_register %r10 - sub \$0x128,%rsp -___ -$code.=<<___ if ($win64); - lea 8(%rsp),%r10 - sub \$0x1c8,%rsp - vmovdqa %xmm6,-0xb0(%r10) - vmovdqa %xmm7,-0xa0(%r10) - vmovdqa %xmm8,-0x90(%r10) - vmovdqa %xmm9,-0x80(%r10) - vmovdqa %xmm10,-0x70(%r10) - vmovdqa %xmm11,-0x60(%r10) - vmovdqa %xmm12,-0x50(%r10) - vmovdqa %xmm13,-0x40(%r10) - vmovdqa %xmm14,-0x30(%r10) - vmovdqa %xmm15,-0x20(%r10) -.Ldo_avx2_body$suffix: -___ -$code.=<<___; - lea .Lconst(%rip),%rcx - lea 48+64($ctx),$ctx # size optimization - vmovdqa 96(%rcx),$T0 # .Lpermd_avx2 - - # expand and copy pre-calculated table to stack - vmovdqu `16*0-64`($ctx),%x#$T2 - and \$-512,%rsp - vmovdqu `16*1-64`($ctx),%x#$T3 - vmovdqu `16*2-64`($ctx),%x#$T4 - vmovdqu `16*3-64`($ctx),%x#$D0 - vmovdqu `16*4-64`($ctx),%x#$D1 - vmovdqu `16*5-64`($ctx),%x#$D2 - lea 0x90(%rsp),%rax # size optimization - vmovdqu `16*6-64`($ctx),%x#$D3 - vpermd $T2,$T0,$T2 # 00003412 -> 14243444 - vmovdqu `16*7-64`($ctx),%x#$D4 - vpermd $T3,$T0,$T3 - vmovdqu `16*8-64`($ctx),%x#$MASK - vpermd $T4,$T0,$T4 - vmovdqa $T2,0x00(%rsp) - vpermd $D0,$T0,$D0 - vmovdqa $T3,0x20-0x90(%rax) - vpermd $D1,$T0,$D1 - vmovdqa $T4,0x40-0x90(%rax) - vpermd $D2,$T0,$D2 - vmovdqa $D0,0x60-0x90(%rax) - vpermd $D3,$T0,$D3 - vmovdqa $D1,0x80-0x90(%rax) - vpermd $D4,$T0,$D4 - vmovdqa $D2,0xa0-0x90(%rax) - vpermd $MASK,$T0,$MASK - vmovdqa $D3,0xc0-0x90(%rax) - vmovdqa $D4,0xe0-0x90(%rax) - vmovdqa $MASK,0x100-0x90(%rax) - vmovdqa 64(%rcx),$MASK # .Lmask26 - - ################################################################ - # load input - vmovdqu 16*0($inp),%x#$T0 - vmovdqu 16*1($inp),%x#$T1 - vinserti128 \$1,16*2($inp),$T0,$T0 - vinserti128 \$1,16*3($inp),$T1,$T1 - lea 16*4($inp),$inp - - vpsrldq \$6,$T0,$T2 # splat input - vpsrldq \$6,$T1,$T3 - vpunpckhqdq $T1,$T0,$T4 # 4 - vpunpcklqdq $T3,$T2,$T2 # 2:3 - vpunpcklqdq $T1,$T0,$T0 # 0:1 - - vpsrlq \$30,$T2,$T3 - vpsrlq \$4,$T2,$T2 - vpsrlq \$26,$T0,$T1 - vpsrlq \$40,$T4,$T4 # 4 - vpand $MASK,$T2,$T2 # 2 - vpand $MASK,$T0,$T0 # 0 - vpand $MASK,$T1,$T1 # 1 - vpand $MASK,$T3,$T3 # 3 - vpor 32(%rcx),$T4,$T4 # padbit, yes, always - - vpaddq $H2,$T2,$H2 # accumulate input - sub \$64,$len - jz .Ltail_avx2$suffix - jmp .Loop_avx2$suffix - -.align 32 -.Loop_avx2$suffix: - ################################################################ - # ((inp[0]*r^4+inp[4])*r^4+inp[ 8])*r^4 - # ((inp[1]*r^4+inp[5])*r^4+inp[ 9])*r^3 - # ((inp[2]*r^4+inp[6])*r^4+inp[10])*r^2 - # ((inp[3]*r^4+inp[7])*r^4+inp[11])*r^1 - # \________/\__________/ - ################################################################ - #vpaddq $H2,$T2,$H2 # accumulate input - vpaddq $H0,$T0,$H0 - vmovdqa `32*0`(%rsp),$T0 # r0^4 - vpaddq $H1,$T1,$H1 - vmovdqa `32*1`(%rsp),$T1 # r1^4 - vpaddq $H3,$T3,$H3 - vmovdqa `32*3`(%rsp),$T2 # r2^4 - vpaddq $H4,$T4,$H4 - vmovdqa `32*6-0x90`(%rax),$T3 # s3^4 - vmovdqa `32*8-0x90`(%rax),$S4 # s4^4 - - # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 - # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 - # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 - # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 - # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 - # - # however, as h2 is "chronologically" first one available pull - # corresponding operations up, so it's - # - # d4 = h2*r2 + h4*r0 + h3*r1 + h1*r3 + h0*r4 - # d3 = h2*r1 + h3*r0 + h1*r2 + h0*r3 + h4*5*r4 - # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 - # d1 = h2*5*r4 + h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 - # d0 = h2*5*r3 + h0*r0 + h4*5*r1 + h3*5*r2 + h1*5*r4 - - vpmuludq $H2,$T0,$D2 # d2 = h2*r0 - vpmuludq $H2,$T1,$D3 # d3 = h2*r1 - vpmuludq $H2,$T2,$D4 # d4 = h2*r2 - vpmuludq $H2,$T3,$D0 # d0 = h2*s3 - vpmuludq $H2,$S4,$D1 # d1 = h2*s4 - - vpmuludq $H0,$T1,$T4 # h0*r1 - vpmuludq $H1,$T1,$H2 # h1*r1, borrow $H2 as temp - vpaddq $T4,$D1,$D1 # d1 += h0*r1 - vpaddq $H2,$D2,$D2 # d2 += h1*r1 - vpmuludq $H3,$T1,$T4 # h3*r1 - vpmuludq `32*2`(%rsp),$H4,$H2 # h4*s1 - vpaddq $T4,$D4,$D4 # d4 += h3*r1 - vpaddq $H2,$D0,$D0 # d0 += h4*s1 - vmovdqa `32*4-0x90`(%rax),$T1 # s2 - - vpmuludq $H0,$T0,$T4 # h0*r0 - vpmuludq $H1,$T0,$H2 # h1*r0 - vpaddq $T4,$D0,$D0 # d0 += h0*r0 - vpaddq $H2,$D1,$D1 # d1 += h1*r0 - vpmuludq $H3,$T0,$T4 # h3*r0 - vpmuludq $H4,$T0,$H2 # h4*r0 - vmovdqu 16*0($inp),%x#$T0 # load input - vpaddq $T4,$D3,$D3 # d3 += h3*r0 - vpaddq $H2,$D4,$D4 # d4 += h4*r0 - vinserti128 \$1,16*2($inp),$T0,$T0 - - vpmuludq $H3,$T1,$T4 # h3*s2 - vpmuludq $H4,$T1,$H2 # h4*s2 - vmovdqu 16*1($inp),%x#$T1 - vpaddq $T4,$D0,$D0 # d0 += h3*s2 - vpaddq $H2,$D1,$D1 # d1 += h4*s2 - vmovdqa `32*5-0x90`(%rax),$H2 # r3 - vpmuludq $H1,$T2,$T4 # h1*r2 - vpmuludq $H0,$T2,$T2 # h0*r2 - vpaddq $T4,$D3,$D3 # d3 += h1*r2 - vpaddq $T2,$D2,$D2 # d2 += h0*r2 - vinserti128 \$1,16*3($inp),$T1,$T1 - lea 16*4($inp),$inp - - vpmuludq $H1,$H2,$T4 # h1*r3 - vpmuludq $H0,$H2,$H2 # h0*r3 - vpsrldq \$6,$T0,$T2 # splat input - vpaddq $T4,$D4,$D4 # d4 += h1*r3 - vpaddq $H2,$D3,$D3 # d3 += h0*r3 - vpmuludq $H3,$T3,$T4 # h3*s3 - vpmuludq $H4,$T3,$H2 # h4*s3 - vpsrldq \$6,$T1,$T3 - vpaddq $T4,$D1,$D1 # d1 += h3*s3 - vpaddq $H2,$D2,$D2 # d2 += h4*s3 - vpunpckhqdq $T1,$T0,$T4 # 4 - - vpmuludq $H3,$S4,$H3 # h3*s4 - vpmuludq $H4,$S4,$H4 # h4*s4 - vpunpcklqdq $T1,$T0,$T0 # 0:1 - vpaddq $H3,$D2,$H2 # h2 = d2 + h3*r4 - vpaddq $H4,$D3,$H3 # h3 = d3 + h4*r4 - vpunpcklqdq $T3,$T2,$T3 # 2:3 - vpmuludq `32*7-0x90`(%rax),$H0,$H4 # h0*r4 - vpmuludq $H1,$S4,$H0 # h1*s4 - vmovdqa 64(%rcx),$MASK # .Lmask26 - vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 - vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 - - ################################################################ - # lazy reduction (interleaved with tail of input splat) - - vpsrlq \$26,$H3,$D3 - vpand $MASK,$H3,$H3 - vpaddq $D3,$H4,$H4 # h3 -> h4 - - vpsrlq \$26,$H0,$D0 - vpand $MASK,$H0,$H0 - vpaddq $D0,$D1,$H1 # h0 -> h1 - - vpsrlq \$26,$H4,$D4 - vpand $MASK,$H4,$H4 - - vpsrlq \$4,$T3,$T2 - - vpsrlq \$26,$H1,$D1 - vpand $MASK,$H1,$H1 - vpaddq $D1,$H2,$H2 # h1 -> h2 - - vpaddq $D4,$H0,$H0 - vpsllq \$2,$D4,$D4 - vpaddq $D4,$H0,$H0 # h4 -> h0 - - vpand $MASK,$T2,$T2 # 2 - vpsrlq \$26,$T0,$T1 - - vpsrlq \$26,$H2,$D2 - vpand $MASK,$H2,$H2 - vpaddq $D2,$H3,$H3 # h2 -> h3 - - vpaddq $T2,$H2,$H2 # modulo-scheduled - vpsrlq \$30,$T3,$T3 - - vpsrlq \$26,$H0,$D0 - vpand $MASK,$H0,$H0 - vpaddq $D0,$H1,$H1 # h0 -> h1 - - vpsrlq \$40,$T4,$T4 # 4 - - vpsrlq \$26,$H3,$D3 - vpand $MASK,$H3,$H3 - vpaddq $D3,$H4,$H4 # h3 -> h4 - - vpand $MASK,$T0,$T0 # 0 - vpand $MASK,$T1,$T1 # 1 - vpand $MASK,$T3,$T3 # 3 - vpor 32(%rcx),$T4,$T4 # padbit, yes, always - - sub \$64,$len - jnz .Loop_avx2$suffix - - .byte 0x66,0x90 -.Ltail_avx2$suffix: - ################################################################ - # while above multiplications were by r^4 in all lanes, in last - # iteration we multiply least significant lane by r^4 and most - # significant one by r, so copy of above except that references - # to the precomputed table are displaced by 4... - - #vpaddq $H2,$T2,$H2 # accumulate input - vpaddq $H0,$T0,$H0 - vmovdqu `32*0+4`(%rsp),$T0 # r0^4 - vpaddq $H1,$T1,$H1 - vmovdqu `32*1+4`(%rsp),$T1 # r1^4 - vpaddq $H3,$T3,$H3 - vmovdqu `32*3+4`(%rsp),$T2 # r2^4 - vpaddq $H4,$T4,$H4 - vmovdqu `32*6+4-0x90`(%rax),$T3 # s3^4 - vmovdqu `32*8+4-0x90`(%rax),$S4 # s4^4 - - vpmuludq $H2,$T0,$D2 # d2 = h2*r0 - vpmuludq $H2,$T1,$D3 # d3 = h2*r1 - vpmuludq $H2,$T2,$D4 # d4 = h2*r2 - vpmuludq $H2,$T3,$D0 # d0 = h2*s3 - vpmuludq $H2,$S4,$D1 # d1 = h2*s4 - - vpmuludq $H0,$T1,$T4 # h0*r1 - vpmuludq $H1,$T1,$H2 # h1*r1 - vpaddq $T4,$D1,$D1 # d1 += h0*r1 - vpaddq $H2,$D2,$D2 # d2 += h1*r1 - vpmuludq $H3,$T1,$T4 # h3*r1 - vpmuludq `32*2+4`(%rsp),$H4,$H2 # h4*s1 - vpaddq $T4,$D4,$D4 # d4 += h3*r1 - vpaddq $H2,$D0,$D0 # d0 += h4*s1 - - vpmuludq $H0,$T0,$T4 # h0*r0 - vpmuludq $H1,$T0,$H2 # h1*r0 - vpaddq $T4,$D0,$D0 # d0 += h0*r0 - vmovdqu `32*4+4-0x90`(%rax),$T1 # s2 - vpaddq $H2,$D1,$D1 # d1 += h1*r0 - vpmuludq $H3,$T0,$T4 # h3*r0 - vpmuludq $H4,$T0,$H2 # h4*r0 - vpaddq $T4,$D3,$D3 # d3 += h3*r0 - vpaddq $H2,$D4,$D4 # d4 += h4*r0 - - vpmuludq $H3,$T1,$T4 # h3*s2 - vpmuludq $H4,$T1,$H2 # h4*s2 - vpaddq $T4,$D0,$D0 # d0 += h3*s2 - vpaddq $H2,$D1,$D1 # d1 += h4*s2 - vmovdqu `32*5+4-0x90`(%rax),$H2 # r3 - vpmuludq $H1,$T2,$T4 # h1*r2 - vpmuludq $H0,$T2,$T2 # h0*r2 - vpaddq $T4,$D3,$D3 # d3 += h1*r2 - vpaddq $T2,$D2,$D2 # d2 += h0*r2 - - vpmuludq $H1,$H2,$T4 # h1*r3 - vpmuludq $H0,$H2,$H2 # h0*r3 - vpaddq $T4,$D4,$D4 # d4 += h1*r3 - vpaddq $H2,$D3,$D3 # d3 += h0*r3 - vpmuludq $H3,$T3,$T4 # h3*s3 - vpmuludq $H4,$T3,$H2 # h4*s3 - vpaddq $T4,$D1,$D1 # d1 += h3*s3 - vpaddq $H2,$D2,$D2 # d2 += h4*s3 - - vpmuludq $H3,$S4,$H3 # h3*s4 - vpmuludq $H4,$S4,$H4 # h4*s4 - vpaddq $H3,$D2,$H2 # h2 = d2 + h3*r4 - vpaddq $H4,$D3,$H3 # h3 = d3 + h4*r4 - vpmuludq `32*7+4-0x90`(%rax),$H0,$H4 # h0*r4 - vpmuludq $H1,$S4,$H0 # h1*s4 - vmovdqa 64(%rcx),$MASK # .Lmask26 - vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 - vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 - - ################################################################ - # horizontal addition - - vpsrldq \$8,$D1,$T1 - vpsrldq \$8,$H2,$T2 - vpsrldq \$8,$H3,$T3 - vpsrldq \$8,$H4,$T4 - vpsrldq \$8,$H0,$T0 - vpaddq $T1,$D1,$D1 - vpaddq $T2,$H2,$H2 - vpaddq $T3,$H3,$H3 - vpaddq $T4,$H4,$H4 - vpaddq $T0,$H0,$H0 - - vpermq \$0x2,$H3,$T3 - vpermq \$0x2,$H4,$T4 - vpermq \$0x2,$H0,$T0 - vpermq \$0x2,$D1,$T1 - vpermq \$0x2,$H2,$T2 - vpaddq $T3,$H3,$H3 - vpaddq $T4,$H4,$H4 - vpaddq $T0,$H0,$H0 - vpaddq $T1,$D1,$D1 - vpaddq $T2,$H2,$H2 - - ################################################################ - # lazy reduction - - vpsrlq \$26,$H3,$D3 - vpand $MASK,$H3,$H3 - vpaddq $D3,$H4,$H4 # h3 -> h4 - - vpsrlq \$26,$H0,$D0 - vpand $MASK,$H0,$H0 - vpaddq $D0,$D1,$H1 # h0 -> h1 - - vpsrlq \$26,$H4,$D4 - vpand $MASK,$H4,$H4 - - vpsrlq \$26,$H1,$D1 - vpand $MASK,$H1,$H1 - vpaddq $D1,$H2,$H2 # h1 -> h2 - - vpaddq $D4,$H0,$H0 - vpsllq \$2,$D4,$D4 - vpaddq $D4,$H0,$H0 # h4 -> h0 - - vpsrlq \$26,$H2,$D2 - vpand $MASK,$H2,$H2 - vpaddq $D2,$H3,$H3 # h2 -> h3 - - vpsrlq \$26,$H0,$D0 - vpand $MASK,$H0,$H0 - vpaddq $D0,$H1,$H1 # h0 -> h1 - - vpsrlq \$26,$H3,$D3 - vpand $MASK,$H3,$H3 - vpaddq $D3,$H4,$H4 # h3 -> h4 - - vmovd %x#$H0,`4*0-48-64`($ctx)# save partially reduced - vmovd %x#$H1,`4*1-48-64`($ctx) - vmovd %x#$H2,`4*2-48-64`($ctx) - vmovd %x#$H3,`4*3-48-64`($ctx) - vmovd %x#$H4,`4*4-48-64`($ctx) -___ -$code.=<<___ if ($win64); - vmovdqa -0xb0(%r10),%xmm6 - vmovdqa -0xa0(%r10),%xmm7 - vmovdqa -0x90(%r10),%xmm8 - vmovdqa -0x80(%r10),%xmm9 - vmovdqa -0x70(%r10),%xmm10 - vmovdqa -0x60(%r10),%xmm11 - vmovdqa -0x50(%r10),%xmm12 - vmovdqa -0x40(%r10),%xmm13 - vmovdqa -0x30(%r10),%xmm14 - vmovdqa -0x20(%r10),%xmm15 - lea -8(%r10),%rsp -.Ldo_avx2_epilogue$suffix: -___ -$code.=<<___ if (!$win64); - lea -8(%r10),%rsp -.cfi_def_cfa_register %rsp -___ -$code.=<<___; - vzeroupper - ret -.cfi_endproc -___ -if($avx > 2 && $avx512) { -my ($R0,$R1,$R2,$R3,$R4, $S1,$S2,$S3,$S4) = map("%zmm$_",(16..24)); -my ($M0,$M1,$M2,$M3,$M4) = map("%zmm$_",(25..29)); -my $PADBIT="%zmm30"; - -map(s/%y/%z/,($T4,$T0,$T1,$T2,$T3)); # switch to %zmm domain -map(s/%y/%z/,($D0,$D1,$D2,$D3,$D4)); -map(s/%y/%z/,($H0,$H1,$H2,$H3,$H4)); -map(s/%y/%z/,($MASK)); - -$code.=<<___; -.cfi_startproc -.Lblocks_avx512: - mov \$15,%eax - kmovw %eax,%k2 -___ -$code.=<<___ if (!$win64); - lea 8(%rsp),%r10 -.cfi_def_cfa_register %r10 - sub \$0x128,%rsp -___ -$code.=<<___ if ($win64); - lea 8(%rsp),%r10 - sub \$0x1c8,%rsp - vmovdqa %xmm6,-0xb0(%r10) - vmovdqa %xmm7,-0xa0(%r10) - vmovdqa %xmm8,-0x90(%r10) - vmovdqa %xmm9,-0x80(%r10) - vmovdqa %xmm10,-0x70(%r10) - vmovdqa %xmm11,-0x60(%r10) - vmovdqa %xmm12,-0x50(%r10) - vmovdqa %xmm13,-0x40(%r10) - vmovdqa %xmm14,-0x30(%r10) - vmovdqa %xmm15,-0x20(%r10) -.Ldo_avx512_body: -___ -$code.=<<___; - lea .Lconst(%rip),%rcx - lea 48+64($ctx),$ctx # size optimization - vmovdqa 96(%rcx),%y#$T2 # .Lpermd_avx2 - - # expand pre-calculated table - vmovdqu `16*0-64`($ctx),%x#$D0 # will become expanded ${R0} - and \$-512,%rsp - vmovdqu `16*1-64`($ctx),%x#$D1 # will become ... ${R1} - mov \$0x20,%rax - vmovdqu `16*2-64`($ctx),%x#$T0 # ... ${S1} - vmovdqu `16*3-64`($ctx),%x#$D2 # ... ${R2} - vmovdqu `16*4-64`($ctx),%x#$T1 # ... ${S2} - vmovdqu `16*5-64`($ctx),%x#$D3 # ... ${R3} - vmovdqu `16*6-64`($ctx),%x#$T3 # ... ${S3} - vmovdqu `16*7-64`($ctx),%x#$D4 # ... ${R4} - vmovdqu `16*8-64`($ctx),%x#$T4 # ... ${S4} - vpermd $D0,$T2,$R0 # 00003412 -> 14243444 - vpbroadcastq 64(%rcx),$MASK # .Lmask26 - vpermd $D1,$T2,$R1 - vpermd $T0,$T2,$S1 - vpermd $D2,$T2,$R2 - vmovdqa64 $R0,0x00(%rsp){%k2} # save in case $len%128 != 0 - vpsrlq \$32,$R0,$T0 # 14243444 -> 01020304 - vpermd $T1,$T2,$S2 - vmovdqu64 $R1,0x00(%rsp,%rax){%k2} - vpsrlq \$32,$R1,$T1 - vpermd $D3,$T2,$R3 - vmovdqa64 $S1,0x40(%rsp){%k2} - vpermd $T3,$T2,$S3 - vpermd $D4,$T2,$R4 - vmovdqu64 $R2,0x40(%rsp,%rax){%k2} - vpermd $T4,$T2,$S4 - vmovdqa64 $S2,0x80(%rsp){%k2} - vmovdqu64 $R3,0x80(%rsp,%rax){%k2} - vmovdqa64 $S3,0xc0(%rsp){%k2} - vmovdqu64 $R4,0xc0(%rsp,%rax){%k2} - vmovdqa64 $S4,0x100(%rsp){%k2} - - ################################################################ - # calculate 5th through 8th powers of the key - # - # d0 = r0'*r0 + r1'*5*r4 + r2'*5*r3 + r3'*5*r2 + r4'*5*r1 - # d1 = r0'*r1 + r1'*r0 + r2'*5*r4 + r3'*5*r3 + r4'*5*r2 - # d2 = r0'*r2 + r1'*r1 + r2'*r0 + r3'*5*r4 + r4'*5*r3 - # d3 = r0'*r3 + r1'*r2 + r2'*r1 + r3'*r0 + r4'*5*r4 - # d4 = r0'*r4 + r1'*r3 + r2'*r2 + r3'*r1 + r4'*r0 - - vpmuludq $T0,$R0,$D0 # d0 = r0'*r0 - vpmuludq $T0,$R1,$D1 # d1 = r0'*r1 - vpmuludq $T0,$R2,$D2 # d2 = r0'*r2 - vpmuludq $T0,$R3,$D3 # d3 = r0'*r3 - vpmuludq $T0,$R4,$D4 # d4 = r0'*r4 - vpsrlq \$32,$R2,$T2 - - vpmuludq $T1,$S4,$M0 - vpmuludq $T1,$R0,$M1 - vpmuludq $T1,$R1,$M2 - vpmuludq $T1,$R2,$M3 - vpmuludq $T1,$R3,$M4 - vpsrlq \$32,$R3,$T3 - vpaddq $M0,$D0,$D0 # d0 += r1'*5*r4 - vpaddq $M1,$D1,$D1 # d1 += r1'*r0 - vpaddq $M2,$D2,$D2 # d2 += r1'*r1 - vpaddq $M3,$D3,$D3 # d3 += r1'*r2 - vpaddq $M4,$D4,$D4 # d4 += r1'*r3 - - vpmuludq $T2,$S3,$M0 - vpmuludq $T2,$S4,$M1 - vpmuludq $T2,$R1,$M3 - vpmuludq $T2,$R2,$M4 - vpmuludq $T2,$R0,$M2 - vpsrlq \$32,$R4,$T4 - vpaddq $M0,$D0,$D0 # d0 += r2'*5*r3 - vpaddq $M1,$D1,$D1 # d1 += r2'*5*r4 - vpaddq $M3,$D3,$D3 # d3 += r2'*r1 - vpaddq $M4,$D4,$D4 # d4 += r2'*r2 - vpaddq $M2,$D2,$D2 # d2 += r2'*r0 - - vpmuludq $T3,$S2,$M0 - vpmuludq $T3,$R0,$M3 - vpmuludq $T3,$R1,$M4 - vpmuludq $T3,$S3,$M1 - vpmuludq $T3,$S4,$M2 - vpaddq $M0,$D0,$D0 # d0 += r3'*5*r2 - vpaddq $M3,$D3,$D3 # d3 += r3'*r0 - vpaddq $M4,$D4,$D4 # d4 += r3'*r1 - vpaddq $M1,$D1,$D1 # d1 += r3'*5*r3 - vpaddq $M2,$D2,$D2 # d2 += r3'*5*r4 - - vpmuludq $T4,$S4,$M3 - vpmuludq $T4,$R0,$M4 - vpmuludq $T4,$S1,$M0 - vpmuludq $T4,$S2,$M1 - vpmuludq $T4,$S3,$M2 - vpaddq $M3,$D3,$D3 # d3 += r2'*5*r4 - vpaddq $M4,$D4,$D4 # d4 += r2'*r0 - vpaddq $M0,$D0,$D0 # d0 += r2'*5*r1 - vpaddq $M1,$D1,$D1 # d1 += r2'*5*r2 - vpaddq $M2,$D2,$D2 # d2 += r2'*5*r3 - - ################################################################ - # load input - vmovdqu64 16*0($inp),%z#$T3 - vmovdqu64 16*4($inp),%z#$T4 - lea 16*8($inp),$inp - - ################################################################ - # lazy reduction - - vpsrlq \$26,$D3,$M3 - vpandq $MASK,$D3,$D3 - vpaddq $M3,$D4,$D4 # d3 -> d4 - - vpsrlq \$26,$D0,$M0 - vpandq $MASK,$D0,$D0 - vpaddq $M0,$D1,$D1 # d0 -> d1 - - vpsrlq \$26,$D4,$M4 - vpandq $MASK,$D4,$D4 - - vpsrlq \$26,$D1,$M1 - vpandq $MASK,$D1,$D1 - vpaddq $M1,$D2,$D2 # d1 -> d2 - - vpaddq $M4,$D0,$D0 - vpsllq \$2,$M4,$M4 - vpaddq $M4,$D0,$D0 # d4 -> d0 - - vpsrlq \$26,$D2,$M2 - vpandq $MASK,$D2,$D2 - vpaddq $M2,$D3,$D3 # d2 -> d3 - - vpsrlq \$26,$D0,$M0 - vpandq $MASK,$D0,$D0 - vpaddq $M0,$D1,$D1 # d0 -> d1 - - vpsrlq \$26,$D3,$M3 - vpandq $MASK,$D3,$D3 - vpaddq $M3,$D4,$D4 # d3 -> d4 - - ################################################################ - # at this point we have 14243444 in $R0-$S4 and 05060708 in - # $D0-$D4, ... - - vpunpcklqdq $T4,$T3,$T0 # transpose input - vpunpckhqdq $T4,$T3,$T4 - - # ... since input 64-bit lanes are ordered as 73625140, we could - # "vperm" it to 76543210 (here and in each loop iteration), *or* - # we could just flow along, hence the goal for $R0-$S4 is - # 1858286838784888 ... - - vmovdqa32 128(%rcx),$M0 # .Lpermd_avx512: - mov \$0x7777,%eax - kmovw %eax,%k1 - - vpermd $R0,$M0,$R0 # 14243444 -> 1---2---3---4--- - vpermd $R1,$M0,$R1 - vpermd $R2,$M0,$R2 - vpermd $R3,$M0,$R3 - vpermd $R4,$M0,$R4 - - vpermd $D0,$M0,${R0}{%k1} # 05060708 -> 1858286838784888 - vpermd $D1,$M0,${R1}{%k1} - vpermd $D2,$M0,${R2}{%k1} - vpermd $D3,$M0,${R3}{%k1} - vpermd $D4,$M0,${R4}{%k1} - - vpslld \$2,$R1,$S1 # *5 - vpslld \$2,$R2,$S2 - vpslld \$2,$R3,$S3 - vpslld \$2,$R4,$S4 - vpaddd $R1,$S1,$S1 - vpaddd $R2,$S2,$S2 - vpaddd $R3,$S3,$S3 - vpaddd $R4,$S4,$S4 - - vpbroadcastq 32(%rcx),$PADBIT # .L129 - - vpsrlq \$52,$T0,$T2 # splat input - vpsllq \$12,$T4,$T3 - vporq $T3,$T2,$T2 - vpsrlq \$26,$T0,$T1 - vpsrlq \$14,$T4,$T3 - vpsrlq \$40,$T4,$T4 # 4 - vpandq $MASK,$T2,$T2 # 2 - vpandq $MASK,$T0,$T0 # 0 - #vpandq $MASK,$T1,$T1 # 1 - #vpandq $MASK,$T3,$T3 # 3 - #vporq $PADBIT,$T4,$T4 # padbit, yes, always - - vpaddq $H2,$T2,$H2 # accumulate input - sub \$192,$len - jbe .Ltail_avx512 - jmp .Loop_avx512 - -.align 32 -.Loop_avx512: - ################################################################ - # ((inp[0]*r^8+inp[ 8])*r^8+inp[16])*r^8 - # ((inp[1]*r^8+inp[ 9])*r^8+inp[17])*r^7 - # ((inp[2]*r^8+inp[10])*r^8+inp[18])*r^6 - # ((inp[3]*r^8+inp[11])*r^8+inp[19])*r^5 - # ((inp[4]*r^8+inp[12])*r^8+inp[20])*r^4 - # ((inp[5]*r^8+inp[13])*r^8+inp[21])*r^3 - # ((inp[6]*r^8+inp[14])*r^8+inp[22])*r^2 - # ((inp[7]*r^8+inp[15])*r^8+inp[23])*r^1 - # \________/\___________/ - ################################################################ - #vpaddq $H2,$T2,$H2 # accumulate input - - # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 - # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 - # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 - # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 - # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 - # - # however, as h2 is "chronologically" first one available pull - # corresponding operations up, so it's - # - # d3 = h2*r1 + h0*r3 + h1*r2 + h3*r0 + h4*5*r4 - # d4 = h2*r2 + h0*r4 + h1*r3 + h3*r1 + h4*r0 - # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1 - # d1 = h2*5*r4 + h0*r1 + h1*r0 + h3*5*r3 + h4*5*r2 - # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3 - - vpmuludq $H2,$R1,$D3 # d3 = h2*r1 - vpaddq $H0,$T0,$H0 - vpmuludq $H2,$R2,$D4 # d4 = h2*r2 - vpandq $MASK,$T1,$T1 # 1 - vpmuludq $H2,$S3,$D0 # d0 = h2*s3 - vpandq $MASK,$T3,$T3 # 3 - vpmuludq $H2,$S4,$D1 # d1 = h2*s4 - vporq $PADBIT,$T4,$T4 # padbit, yes, always - vpmuludq $H2,$R0,$D2 # d2 = h2*r0 - vpaddq $H1,$T1,$H1 # accumulate input - vpaddq $H3,$T3,$H3 - vpaddq $H4,$T4,$H4 - - vmovdqu64 16*0($inp),$T3 # load input - vmovdqu64 16*4($inp),$T4 - lea 16*8($inp),$inp - vpmuludq $H0,$R3,$M3 - vpmuludq $H0,$R4,$M4 - vpmuludq $H0,$R0,$M0 - vpmuludq $H0,$R1,$M1 - vpaddq $M3,$D3,$D3 # d3 += h0*r3 - vpaddq $M4,$D4,$D4 # d4 += h0*r4 - vpaddq $M0,$D0,$D0 # d0 += h0*r0 - vpaddq $M1,$D1,$D1 # d1 += h0*r1 - - vpmuludq $H1,$R2,$M3 - vpmuludq $H1,$R3,$M4 - vpmuludq $H1,$S4,$M0 - vpmuludq $H0,$R2,$M2 - vpaddq $M3,$D3,$D3 # d3 += h1*r2 - vpaddq $M4,$D4,$D4 # d4 += h1*r3 - vpaddq $M0,$D0,$D0 # d0 += h1*s4 - vpaddq $M2,$D2,$D2 # d2 += h0*r2 - - vpunpcklqdq $T4,$T3,$T0 # transpose input - vpunpckhqdq $T4,$T3,$T4 - - vpmuludq $H3,$R0,$M3 - vpmuludq $H3,$R1,$M4 - vpmuludq $H1,$R0,$M1 - vpmuludq $H1,$R1,$M2 - vpaddq $M3,$D3,$D3 # d3 += h3*r0 - vpaddq $M4,$D4,$D4 # d4 += h3*r1 - vpaddq $M1,$D1,$D1 # d1 += h1*r0 - vpaddq $M2,$D2,$D2 # d2 += h1*r1 - - vpmuludq $H4,$S4,$M3 - vpmuludq $H4,$R0,$M4 - vpmuludq $H3,$S2,$M0 - vpmuludq $H3,$S3,$M1 - vpaddq $M3,$D3,$D3 # d3 += h4*s4 - vpmuludq $H3,$S4,$M2 - vpaddq $M4,$D4,$D4 # d4 += h4*r0 - vpaddq $M0,$D0,$D0 # d0 += h3*s2 - vpaddq $M1,$D1,$D1 # d1 += h3*s3 - vpaddq $M2,$D2,$D2 # d2 += h3*s4 - - vpmuludq $H4,$S1,$M0 - vpmuludq $H4,$S2,$M1 - vpmuludq $H4,$S3,$M2 - vpaddq $M0,$D0,$H0 # h0 = d0 + h4*s1 - vpaddq $M1,$D1,$H1 # h1 = d2 + h4*s2 - vpaddq $M2,$D2,$H2 # h2 = d3 + h4*s3 - - ################################################################ - # lazy reduction (interleaved with input splat) - - vpsrlq \$52,$T0,$T2 # splat input - vpsllq \$12,$T4,$T3 - - vpsrlq \$26,$D3,$H3 - vpandq $MASK,$D3,$D3 - vpaddq $H3,$D4,$H4 # h3 -> h4 - - vporq $T3,$T2,$T2 - - vpsrlq \$26,$H0,$D0 - vpandq $MASK,$H0,$H0 - vpaddq $D0,$H1,$H1 # h0 -> h1 - - vpandq $MASK,$T2,$T2 # 2 - - vpsrlq \$26,$H4,$D4 - vpandq $MASK,$H4,$H4 - - vpsrlq \$26,$H1,$D1 - vpandq $MASK,$H1,$H1 - vpaddq $D1,$H2,$H2 # h1 -> h2 - - vpaddq $D4,$H0,$H0 - vpsllq \$2,$D4,$D4 - vpaddq $D4,$H0,$H0 # h4 -> h0 - - vpaddq $T2,$H2,$H2 # modulo-scheduled - vpsrlq \$26,$T0,$T1 - - vpsrlq \$26,$H2,$D2 - vpandq $MASK,$H2,$H2 - vpaddq $D2,$D3,$H3 # h2 -> h3 - - vpsrlq \$14,$T4,$T3 - - vpsrlq \$26,$H0,$D0 - vpandq $MASK,$H0,$H0 - vpaddq $D0,$H1,$H1 # h0 -> h1 - - vpsrlq \$40,$T4,$T4 # 4 - - vpsrlq \$26,$H3,$D3 - vpandq $MASK,$H3,$H3 - vpaddq $D3,$H4,$H4 # h3 -> h4 - - vpandq $MASK,$T0,$T0 # 0 - #vpandq $MASK,$T1,$T1 # 1 - #vpandq $MASK,$T3,$T3 # 3 - #vporq $PADBIT,$T4,$T4 # padbit, yes, always - - sub \$128,$len - ja .Loop_avx512 - -.Ltail_avx512: - ################################################################ - # while above multiplications were by r^8 in all lanes, in last - # iteration we multiply least significant lane by r^8 and most - # significant one by r, that's why table gets shifted... - - vpsrlq \$32,$R0,$R0 # 0105020603070408 - vpsrlq \$32,$R1,$R1 - vpsrlq \$32,$R2,$R2 - vpsrlq \$32,$S3,$S3 - vpsrlq \$32,$S4,$S4 - vpsrlq \$32,$R3,$R3 - vpsrlq \$32,$R4,$R4 - vpsrlq \$32,$S1,$S1 - vpsrlq \$32,$S2,$S2 - - ################################################################ - # load either next or last 64 byte of input - lea ($inp,$len),$inp - - #vpaddq $H2,$T2,$H2 # accumulate input - vpaddq $H0,$T0,$H0 - - vpmuludq $H2,$R1,$D3 # d3 = h2*r1 - vpmuludq $H2,$R2,$D4 # d4 = h2*r2 - vpmuludq $H2,$S3,$D0 # d0 = h2*s3 - vpandq $MASK,$T1,$T1 # 1 - vpmuludq $H2,$S4,$D1 # d1 = h2*s4 - vpandq $MASK,$T3,$T3 # 3 - vpmuludq $H2,$R0,$D2 # d2 = h2*r0 - vporq $PADBIT,$T4,$T4 # padbit, yes, always - vpaddq $H1,$T1,$H1 # accumulate input - vpaddq $H3,$T3,$H3 - vpaddq $H4,$T4,$H4 - - vmovdqu 16*0($inp),%x#$T0 - vpmuludq $H0,$R3,$M3 - vpmuludq $H0,$R4,$M4 - vpmuludq $H0,$R0,$M0 - vpmuludq $H0,$R1,$M1 - vpaddq $M3,$D3,$D3 # d3 += h0*r3 - vpaddq $M4,$D4,$D4 # d4 += h0*r4 - vpaddq $M0,$D0,$D0 # d0 += h0*r0 - vpaddq $M1,$D1,$D1 # d1 += h0*r1 - - vmovdqu 16*1($inp),%x#$T1 - vpmuludq $H1,$R2,$M3 - vpmuludq $H1,$R3,$M4 - vpmuludq $H1,$S4,$M0 - vpmuludq $H0,$R2,$M2 - vpaddq $M3,$D3,$D3 # d3 += h1*r2 - vpaddq $M4,$D4,$D4 # d4 += h1*r3 - vpaddq $M0,$D0,$D0 # d0 += h1*s4 - vpaddq $M2,$D2,$D2 # d2 += h0*r2 - - vinserti128 \$1,16*2($inp),%y#$T0,%y#$T0 - vpmuludq $H3,$R0,$M3 - vpmuludq $H3,$R1,$M4 - vpmuludq $H1,$R0,$M1 - vpmuludq $H1,$R1,$M2 - vpaddq $M3,$D3,$D3 # d3 += h3*r0 - vpaddq $M4,$D4,$D4 # d4 += h3*r1 - vpaddq $M1,$D1,$D1 # d1 += h1*r0 - vpaddq $M2,$D2,$D2 # d2 += h1*r1 - - vinserti128 \$1,16*3($inp),%y#$T1,%y#$T1 - vpmuludq $H4,$S4,$M3 - vpmuludq $H4,$R0,$M4 - vpmuludq $H3,$S2,$M0 - vpmuludq $H3,$S3,$M1 - vpmuludq $H3,$S4,$M2 - vpaddq $M3,$D3,$H3 # h3 = d3 + h4*s4 - vpaddq $M4,$D4,$D4 # d4 += h4*r0 - vpaddq $M0,$D0,$D0 # d0 += h3*s2 - vpaddq $M1,$D1,$D1 # d1 += h3*s3 - vpaddq $M2,$D2,$D2 # d2 += h3*s4 - - vpmuludq $H4,$S1,$M0 - vpmuludq $H4,$S2,$M1 - vpmuludq $H4,$S3,$M2 - vpaddq $M0,$D0,$H0 # h0 = d0 + h4*s1 - vpaddq $M1,$D1,$H1 # h1 = d2 + h4*s2 - vpaddq $M2,$D2,$H2 # h2 = d3 + h4*s3 - - ################################################################ - # horizontal addition - - mov \$1,%eax - vpermq \$0xb1,$H3,$D3 - vpermq \$0xb1,$D4,$H4 - vpermq \$0xb1,$H0,$D0 - vpermq \$0xb1,$H1,$D1 - vpermq \$0xb1,$H2,$D2 - vpaddq $D3,$H3,$H3 - vpaddq $D4,$H4,$H4 - vpaddq $D0,$H0,$H0 - vpaddq $D1,$H1,$H1 - vpaddq $D2,$H2,$H2 - - kmovw %eax,%k3 - vpermq \$0x2,$H3,$D3 - vpermq \$0x2,$H4,$D4 - vpermq \$0x2,$H0,$D0 - vpermq \$0x2,$H1,$D1 - vpermq \$0x2,$H2,$D2 - vpaddq $D3,$H3,$H3 - vpaddq $D4,$H4,$H4 - vpaddq $D0,$H0,$H0 - vpaddq $D1,$H1,$H1 - vpaddq $D2,$H2,$H2 - - vextracti64x4 \$0x1,$H3,%y#$D3 - vextracti64x4 \$0x1,$H4,%y#$D4 - vextracti64x4 \$0x1,$H0,%y#$D0 - vextracti64x4 \$0x1,$H1,%y#$D1 - vextracti64x4 \$0x1,$H2,%y#$D2 - vpaddq $D3,$H3,${H3}{%k3}{z} # keep single qword in case - vpaddq $D4,$H4,${H4}{%k3}{z} # it's passed to .Ltail_avx2 - vpaddq $D0,$H0,${H0}{%k3}{z} - vpaddq $D1,$H1,${H1}{%k3}{z} - vpaddq $D2,$H2,${H2}{%k3}{z} -___ -map(s/%z/%y/,($T0,$T1,$T2,$T3,$T4, $PADBIT)); -map(s/%z/%y/,($H0,$H1,$H2,$H3,$H4, $D0,$D1,$D2,$D3,$D4, $MASK)); -$code.=<<___; - ################################################################ - # lazy reduction (interleaved with input splat) - - vpsrlq \$26,$H3,$D3 - vpand $MASK,$H3,$H3 - vpsrldq \$6,$T0,$T2 # splat input - vpsrldq \$6,$T1,$T3 - vpunpckhqdq $T1,$T0,$T4 # 4 - vpaddq $D3,$H4,$H4 # h3 -> h4 - - vpsrlq \$26,$H0,$D0 - vpand $MASK,$H0,$H0 - vpunpcklqdq $T3,$T2,$T2 # 2:3 - vpunpcklqdq $T1,$T0,$T0 # 0:1 - vpaddq $D0,$H1,$H1 # h0 -> h1 - - vpsrlq \$26,$H4,$D4 - vpand $MASK,$H4,$H4 - - vpsrlq \$26,$H1,$D1 - vpand $MASK,$H1,$H1 - vpsrlq \$30,$T2,$T3 - vpsrlq \$4,$T2,$T2 - vpaddq $D1,$H2,$H2 # h1 -> h2 - - vpaddq $D4,$H0,$H0 - vpsllq \$2,$D4,$D4 - vpsrlq \$26,$T0,$T1 - vpsrlq \$40,$T4,$T4 # 4 - vpaddq $D4,$H0,$H0 # h4 -> h0 - - vpsrlq \$26,$H2,$D2 - vpand $MASK,$H2,$H2 - vpand $MASK,$T2,$T2 # 2 - vpand $MASK,$T0,$T0 # 0 - vpaddq $D2,$H3,$H3 # h2 -> h3 - - vpsrlq \$26,$H0,$D0 - vpand $MASK,$H0,$H0 - vpaddq $H2,$T2,$H2 # accumulate input for .Ltail_avx2 - vpand $MASK,$T1,$T1 # 1 - vpaddq $D0,$H1,$H1 # h0 -> h1 - - vpsrlq \$26,$H3,$D3 - vpand $MASK,$H3,$H3 - vpand $MASK,$T3,$T3 # 3 - vpor 32(%rcx),$T4,$T4 # padbit, yes, always - vpaddq $D3,$H4,$H4 # h3 -> h4 - - lea 0x90(%rsp),%rax # size optimization for .Ltail_avx2 - add \$64,$len - jnz .Ltail_avx2$suffix - - vpsubq $T2,$H2,$H2 # undo input accumulation - vmovd %x#$H0,`4*0-48-64`($ctx)# save partially reduced - vmovd %x#$H1,`4*1-48-64`($ctx) - vmovd %x#$H2,`4*2-48-64`($ctx) - vmovd %x#$H3,`4*3-48-64`($ctx) - vmovd %x#$H4,`4*4-48-64`($ctx) - vzeroall -___ -$code.=<<___ if ($win64); - movdqa -0xb0(%r10),%xmm6 - movdqa -0xa0(%r10),%xmm7 - movdqa -0x90(%r10),%xmm8 - movdqa -0x80(%r10),%xmm9 - movdqa -0x70(%r10),%xmm10 - movdqa -0x60(%r10),%xmm11 - movdqa -0x50(%r10),%xmm12 - movdqa -0x40(%r10),%xmm13 - movdqa -0x30(%r10),%xmm14 - movdqa -0x20(%r10),%xmm15 - lea -8(%r10),%rsp -.Ldo_avx512_epilogue: -___ -$code.=<<___ if (!$win64); - lea -8(%r10),%rsp -.cfi_def_cfa_register %rsp -___ -$code.=<<___; - ret -.cfi_endproc -___ - -} - -} - -&declare_function("poly1305_blocks_avx2", 32, 4); -poly1305_blocks_avxN(0); -&end_function("poly1305_blocks_avx2"); - -if($kernel) { - $code .= "#endif\n"; -} - -####################################################################### -if ($avx>2) { -# On entry we have input length divisible by 64. But since inner loop -# processes 128 bytes per iteration, cases when length is not divisible -# by 128 are handled by passing tail 64 bytes to .Ltail_avx2. For this -# reason stack layout is kept identical to poly1305_blocks_avx2. If not -# for this tail, we wouldn't have to even allocate stack frame... - -if($kernel) { - $code .= "#ifdef CONFIG_AS_AVX512\n"; -} - -&declare_function("poly1305_blocks_avx512", 32, 4); -poly1305_blocks_avxN(1); -&end_function("poly1305_blocks_avx512"); - -if ($kernel) { - $code .= "#endif\n"; -} - -if (!$kernel && $avx>3) { -######################################################################## -# VPMADD52 version using 2^44 radix. -# -# One can argue that base 2^52 would be more natural. Well, even though -# some operations would be more natural, one has to recognize couple of -# things. Base 2^52 doesn't provide advantage over base 2^44 if you look -# at amount of multiply-n-accumulate operations. Secondly, it makes it -# impossible to pre-compute multiples of 5 [referred to as s[]/sN in -# reference implementations], which means that more such operations -# would have to be performed in inner loop, which in turn makes critical -# path longer. In other words, even though base 2^44 reduction might -# look less elegant, overall critical path is actually shorter... - -######################################################################## -# Layout of opaque area is following. -# -# unsigned __int64 h[3]; # current hash value base 2^44 -# unsigned __int64 s[2]; # key value*20 base 2^44 -# unsigned __int64 r[3]; # key value base 2^44 -# struct { unsigned __int64 r^1, r^3, r^2, r^4; } R[4]; -# # r^n positions reflect -# # placement in register, not -# # memory, R[3] is R[1]*20 - -$code.=<<___; -.type poly1305_init_base2_44,\@function,3 -.align 32 -poly1305_init_base2_44: - xor %rax,%rax - mov %rax,0($ctx) # initialize hash value - mov %rax,8($ctx) - mov %rax,16($ctx) - -.Linit_base2_44: - lea poly1305_blocks_vpmadd52(%rip),%r10 - lea poly1305_emit_base2_44(%rip),%r11 - - mov \$0x0ffffffc0fffffff,%rax - mov \$0x0ffffffc0ffffffc,%rcx - and 0($inp),%rax - mov \$0x00000fffffffffff,%r8 - and 8($inp),%rcx - mov \$0x00000fffffffffff,%r9 - and %rax,%r8 - shrd \$44,%rcx,%rax - mov %r8,40($ctx) # r0 - and %r9,%rax - shr \$24,%rcx - mov %rax,48($ctx) # r1 - lea (%rax,%rax,4),%rax # *5 - mov %rcx,56($ctx) # r2 - shl \$2,%rax # magic <<2 - lea (%rcx,%rcx,4),%rcx # *5 - shl \$2,%rcx # magic <<2 - mov %rax,24($ctx) # s1 - mov %rcx,32($ctx) # s2 - movq \$-1,64($ctx) # write impossible value -___ -$code.=<<___ if ($flavour !~ /elf32/); - mov %r10,0(%rdx) - mov %r11,8(%rdx) -___ -$code.=<<___ if ($flavour =~ /elf32/); - mov %r10d,0(%rdx) - mov %r11d,4(%rdx) -___ -$code.=<<___; - mov \$1,%eax - ret -.size poly1305_init_base2_44,.-poly1305_init_base2_44 -___ -{ -my ($H0,$H1,$H2,$r2r1r0,$r1r0s2,$r0s2s1,$Dlo,$Dhi) = map("%ymm$_",(0..5,16,17)); -my ($T0,$inp_permd,$inp_shift,$PAD) = map("%ymm$_",(18..21)); -my ($reduc_mask,$reduc_rght,$reduc_left) = map("%ymm$_",(22..25)); - -$code.=<<___; -.type poly1305_blocks_vpmadd52,\@function,4 -.align 32 -poly1305_blocks_vpmadd52: - shr \$4,$len - jz .Lno_data_vpmadd52 # too short - - shl \$40,$padbit - mov 64($ctx),%r8 # peek on power of the key - - # if powers of the key are not calculated yet, process up to 3 - # blocks with this single-block subroutine, otherwise ensure that - # length is divisible by 2 blocks and pass the rest down to next - # subroutine... - - mov \$3,%rax - mov \$1,%r10 - cmp \$4,$len # is input long - cmovae %r10,%rax - test %r8,%r8 # is power value impossible? - cmovns %r10,%rax - - and $len,%rax # is input of favourable length? - jz .Lblocks_vpmadd52_4x - - sub %rax,$len - mov \$7,%r10d - mov \$1,%r11d - kmovw %r10d,%k7 - lea .L2_44_inp_permd(%rip),%r10 - kmovw %r11d,%k1 - - vmovq $padbit,%x#$PAD - vmovdqa64 0(%r10),$inp_permd # .L2_44_inp_permd - vmovdqa64 32(%r10),$inp_shift # .L2_44_inp_shift - vpermq \$0xcf,$PAD,$PAD - vmovdqa64 64(%r10),$reduc_mask # .L2_44_mask - - vmovdqu64 0($ctx),${Dlo}{%k7}{z} # load hash value - vmovdqu64 40($ctx),${r2r1r0}{%k7}{z} # load keys - vmovdqu64 32($ctx),${r1r0s2}{%k7}{z} - vmovdqu64 24($ctx),${r0s2s1}{%k7}{z} - - vmovdqa64 96(%r10),$reduc_rght # .L2_44_shift_rgt - vmovdqa64 128(%r10),$reduc_left # .L2_44_shift_lft - - jmp .Loop_vpmadd52 - -.align 32 -.Loop_vpmadd52: - vmovdqu32 0($inp),%x#$T0 # load input as ----3210 - lea 16($inp),$inp - - vpermd $T0,$inp_permd,$T0 # ----3210 -> --322110 - vpsrlvq $inp_shift,$T0,$T0 - vpandq $reduc_mask,$T0,$T0 - vporq $PAD,$T0,$T0 - - vpaddq $T0,$Dlo,$Dlo # accumulate input - - vpermq \$0,$Dlo,${H0}{%k7}{z} # smash hash value - vpermq \$0b01010101,$Dlo,${H1}{%k7}{z} - vpermq \$0b10101010,$Dlo,${H2}{%k7}{z} - - vpxord $Dlo,$Dlo,$Dlo - vpxord $Dhi,$Dhi,$Dhi - - vpmadd52luq $r2r1r0,$H0,$Dlo - vpmadd52huq $r2r1r0,$H0,$Dhi - - vpmadd52luq $r1r0s2,$H1,$Dlo - vpmadd52huq $r1r0s2,$H1,$Dhi - - vpmadd52luq $r0s2s1,$H2,$Dlo - vpmadd52huq $r0s2s1,$H2,$Dhi - - vpsrlvq $reduc_rght,$Dlo,$T0 # 0 in topmost qword - vpsllvq $reduc_left,$Dhi,$Dhi # 0 in topmost qword - vpandq $reduc_mask,$Dlo,$Dlo - - vpaddq $T0,$Dhi,$Dhi - - vpermq \$0b10010011,$Dhi,$Dhi # 0 in lowest qword - - vpaddq $Dhi,$Dlo,$Dlo # note topmost qword :-) - - vpsrlvq $reduc_rght,$Dlo,$T0 # 0 in topmost word - vpandq $reduc_mask,$Dlo,$Dlo - - vpermq \$0b10010011,$T0,$T0 - - vpaddq $T0,$Dlo,$Dlo - - vpermq \$0b10010011,$Dlo,${T0}{%k1}{z} - - vpaddq $T0,$Dlo,$Dlo - vpsllq \$2,$T0,$T0 - - vpaddq $T0,$Dlo,$Dlo - - dec %rax # len-=16 - jnz .Loop_vpmadd52 - - vmovdqu64 $Dlo,0($ctx){%k7} # store hash value - - test $len,$len - jnz .Lblocks_vpmadd52_4x - -.Lno_data_vpmadd52: - ret -.size poly1305_blocks_vpmadd52,.-poly1305_blocks_vpmadd52 -___ -} -{ -######################################################################## -# As implied by its name 4x subroutine processes 4 blocks in parallel -# (but handles even 4*n+2 blocks lengths). It takes up to 4th key power -# and is handled in 256-bit %ymm registers. - -my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17)); -my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23)); -my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31)); - -$code.=<<___; -.type poly1305_blocks_vpmadd52_4x,\@function,4 -.align 32 -poly1305_blocks_vpmadd52_4x: - shr \$4,$len - jz .Lno_data_vpmadd52_4x # too short - - shl \$40,$padbit - mov 64($ctx),%r8 # peek on power of the key - -.Lblocks_vpmadd52_4x: - vpbroadcastq $padbit,$PAD - - vmovdqa64 .Lx_mask44(%rip),$mask44 - mov \$5,%eax - vmovdqa64 .Lx_mask42(%rip),$mask42 - kmovw %eax,%k1 # used in 2x path - - test %r8,%r8 # is power value impossible? - js .Linit_vpmadd52 # if it is, then init R[4] - - vmovq 0($ctx),%x#$H0 # load current hash value - vmovq 8($ctx),%x#$H1 - vmovq 16($ctx),%x#$H2 - - test \$3,$len # is length 4*n+2? - jnz .Lblocks_vpmadd52_2x_do - -.Lblocks_vpmadd52_4x_do: - vpbroadcastq 64($ctx),$R0 # load 4th power of the key - vpbroadcastq 96($ctx),$R1 - vpbroadcastq 128($ctx),$R2 - vpbroadcastq 160($ctx),$S1 - -.Lblocks_vpmadd52_4x_key_loaded: - vpsllq \$2,$R2,$S2 # S2 = R2*5*4 - vpaddq $R2,$S2,$S2 - vpsllq \$2,$S2,$S2 - - test \$7,$len # is len 8*n? - jz .Lblocks_vpmadd52_8x - - vmovdqu64 16*0($inp),$T2 # load data - vmovdqu64 16*2($inp),$T3 - lea 16*4($inp),$inp - - vpunpcklqdq $T3,$T2,$T1 # transpose data - vpunpckhqdq $T3,$T2,$T3 - - # at this point 64-bit lanes are ordered as 3-1-2-0 - - vpsrlq \$24,$T3,$T2 # splat the data - vporq $PAD,$T2,$T2 - vpaddq $T2,$H2,$H2 # accumulate input - vpandq $mask44,$T1,$T0 - vpsrlq \$44,$T1,$T1 - vpsllq \$20,$T3,$T3 - vporq $T3,$T1,$T1 - vpandq $mask44,$T1,$T1 - - sub \$4,$len - jz .Ltail_vpmadd52_4x - jmp .Loop_vpmadd52_4x - ud2 - -.align 32 -.Linit_vpmadd52: - vmovq 24($ctx),%x#$S1 # load key - vmovq 56($ctx),%x#$H2 - vmovq 32($ctx),%x#$S2 - vmovq 40($ctx),%x#$R0 - vmovq 48($ctx),%x#$R1 - - vmovdqa $R0,$H0 - vmovdqa $R1,$H1 - vmovdqa $H2,$R2 - - mov \$2,%eax - -.Lmul_init_vpmadd52: - vpxorq $D0lo,$D0lo,$D0lo - vpmadd52luq $H2,$S1,$D0lo - vpxorq $D0hi,$D0hi,$D0hi - vpmadd52huq $H2,$S1,$D0hi - vpxorq $D1lo,$D1lo,$D1lo - vpmadd52luq $H2,$S2,$D1lo - vpxorq $D1hi,$D1hi,$D1hi - vpmadd52huq $H2,$S2,$D1hi - vpxorq $D2lo,$D2lo,$D2lo - vpmadd52luq $H2,$R0,$D2lo - vpxorq $D2hi,$D2hi,$D2hi - vpmadd52huq $H2,$R0,$D2hi - - vpmadd52luq $H0,$R0,$D0lo - vpmadd52huq $H0,$R0,$D0hi - vpmadd52luq $H0,$R1,$D1lo - vpmadd52huq $H0,$R1,$D1hi - vpmadd52luq $H0,$R2,$D2lo - vpmadd52huq $H0,$R2,$D2hi - - vpmadd52luq $H1,$S2,$D0lo - vpmadd52huq $H1,$S2,$D0hi - vpmadd52luq $H1,$R0,$D1lo - vpmadd52huq $H1,$R0,$D1hi - vpmadd52luq $H1,$R1,$D2lo - vpmadd52huq $H1,$R1,$D2hi - - ################################################################ - # partial reduction - vpsrlq \$44,$D0lo,$tmp - vpsllq \$8,$D0hi,$D0hi - vpandq $mask44,$D0lo,$H0 - vpaddq $tmp,$D0hi,$D0hi - - vpaddq $D0hi,$D1lo,$D1lo - - vpsrlq \$44,$D1lo,$tmp - vpsllq \$8,$D1hi,$D1hi - vpandq $mask44,$D1lo,$H1 - vpaddq $tmp,$D1hi,$D1hi - - vpaddq $D1hi,$D2lo,$D2lo - - vpsrlq \$42,$D2lo,$tmp - vpsllq \$10,$D2hi,$D2hi - vpandq $mask42,$D2lo,$H2 - vpaddq $tmp,$D2hi,$D2hi - - vpaddq $D2hi,$H0,$H0 - vpsllq \$2,$D2hi,$D2hi - - vpaddq $D2hi,$H0,$H0 - - vpsrlq \$44,$H0,$tmp # additional step - vpandq $mask44,$H0,$H0 - - vpaddq $tmp,$H1,$H1 - - dec %eax - jz .Ldone_init_vpmadd52 - - vpunpcklqdq $R1,$H1,$R1 # 1,2 - vpbroadcastq %x#$H1,%x#$H1 # 2,2 - vpunpcklqdq $R2,$H2,$R2 - vpbroadcastq %x#$H2,%x#$H2 - vpunpcklqdq $R0,$H0,$R0 - vpbroadcastq %x#$H0,%x#$H0 - - vpsllq \$2,$R1,$S1 # S1 = R1*5*4 - vpsllq \$2,$R2,$S2 # S2 = R2*5*4 - vpaddq $R1,$S1,$S1 - vpaddq $R2,$S2,$S2 - vpsllq \$2,$S1,$S1 - vpsllq \$2,$S2,$S2 - - jmp .Lmul_init_vpmadd52 - ud2 - -.align 32 -.Ldone_init_vpmadd52: - vinserti128 \$1,%x#$R1,$H1,$R1 # 1,2,3,4 - vinserti128 \$1,%x#$R2,$H2,$R2 - vinserti128 \$1,%x#$R0,$H0,$R0 - - vpermq \$0b11011000,$R1,$R1 # 1,3,2,4 - vpermq \$0b11011000,$R2,$R2 - vpermq \$0b11011000,$R0,$R0 - - vpsllq \$2,$R1,$S1 # S1 = R1*5*4 - vpaddq $R1,$S1,$S1 - vpsllq \$2,$S1,$S1 - - vmovq 0($ctx),%x#$H0 # load current hash value - vmovq 8($ctx),%x#$H1 - vmovq 16($ctx),%x#$H2 - - test \$3,$len # is length 4*n+2? - jnz .Ldone_init_vpmadd52_2x - - vmovdqu64 $R0,64($ctx) # save key powers - vpbroadcastq %x#$R0,$R0 # broadcast 4th power - vmovdqu64 $R1,96($ctx) - vpbroadcastq %x#$R1,$R1 - vmovdqu64 $R2,128($ctx) - vpbroadcastq %x#$R2,$R2 - vmovdqu64 $S1,160($ctx) - vpbroadcastq %x#$S1,$S1 - - jmp .Lblocks_vpmadd52_4x_key_loaded - ud2 - -.align 32 -.Ldone_init_vpmadd52_2x: - vmovdqu64 $R0,64($ctx) # save key powers - vpsrldq \$8,$R0,$R0 # 0-1-0-2 - vmovdqu64 $R1,96($ctx) - vpsrldq \$8,$R1,$R1 - vmovdqu64 $R2,128($ctx) - vpsrldq \$8,$R2,$R2 - vmovdqu64 $S1,160($ctx) - vpsrldq \$8,$S1,$S1 - jmp .Lblocks_vpmadd52_2x_key_loaded - ud2 - -.align 32 -.Lblocks_vpmadd52_2x_do: - vmovdqu64 128+8($ctx),${R2}{%k1}{z}# load 2nd and 1st key powers - vmovdqu64 160+8($ctx),${S1}{%k1}{z} - vmovdqu64 64+8($ctx),${R0}{%k1}{z} - vmovdqu64 96+8($ctx),${R1}{%k1}{z} - -.Lblocks_vpmadd52_2x_key_loaded: - vmovdqu64 16*0($inp),$T2 # load data - vpxorq $T3,$T3,$T3 - lea 16*2($inp),$inp - - vpunpcklqdq $T3,$T2,$T1 # transpose data - vpunpckhqdq $T3,$T2,$T3 - - # at this point 64-bit lanes are ordered as x-1-x-0 - - vpsrlq \$24,$T3,$T2 # splat the data - vporq $PAD,$T2,$T2 - vpaddq $T2,$H2,$H2 # accumulate input - vpandq $mask44,$T1,$T0 - vpsrlq \$44,$T1,$T1 - vpsllq \$20,$T3,$T3 - vporq $T3,$T1,$T1 - vpandq $mask44,$T1,$T1 - - jmp .Ltail_vpmadd52_2x - ud2 - -.align 32 -.Loop_vpmadd52_4x: - #vpaddq $T2,$H2,$H2 # accumulate input - vpaddq $T0,$H0,$H0 - vpaddq $T1,$H1,$H1 - - vpxorq $D0lo,$D0lo,$D0lo - vpmadd52luq $H2,$S1,$D0lo - vpxorq $D0hi,$D0hi,$D0hi - vpmadd52huq $H2,$S1,$D0hi - vpxorq $D1lo,$D1lo,$D1lo - vpmadd52luq $H2,$S2,$D1lo - vpxorq $D1hi,$D1hi,$D1hi - vpmadd52huq $H2,$S2,$D1hi - vpxorq $D2lo,$D2lo,$D2lo - vpmadd52luq $H2,$R0,$D2lo - vpxorq $D2hi,$D2hi,$D2hi - vpmadd52huq $H2,$R0,$D2hi - - vmovdqu64 16*0($inp),$T2 # load data - vmovdqu64 16*2($inp),$T3 - lea 16*4($inp),$inp - vpmadd52luq $H0,$R0,$D0lo - vpmadd52huq $H0,$R0,$D0hi - vpmadd52luq $H0,$R1,$D1lo - vpmadd52huq $H0,$R1,$D1hi - vpmadd52luq $H0,$R2,$D2lo - vpmadd52huq $H0,$R2,$D2hi - - vpunpcklqdq $T3,$T2,$T1 # transpose data - vpunpckhqdq $T3,$T2,$T3 - vpmadd52luq $H1,$S2,$D0lo - vpmadd52huq $H1,$S2,$D0hi - vpmadd52luq $H1,$R0,$D1lo - vpmadd52huq $H1,$R0,$D1hi - vpmadd52luq $H1,$R1,$D2lo - vpmadd52huq $H1,$R1,$D2hi - - ################################################################ - # partial reduction (interleaved with data splat) - vpsrlq \$44,$D0lo,$tmp - vpsllq \$8,$D0hi,$D0hi - vpandq $mask44,$D0lo,$H0 - vpaddq $tmp,$D0hi,$D0hi - - vpsrlq \$24,$T3,$T2 - vporq $PAD,$T2,$T2 - vpaddq $D0hi,$D1lo,$D1lo - - vpsrlq \$44,$D1lo,$tmp - vpsllq \$8,$D1hi,$D1hi - vpandq $mask44,$D1lo,$H1 - vpaddq $tmp,$D1hi,$D1hi - - vpandq $mask44,$T1,$T0 - vpsrlq \$44,$T1,$T1 - vpsllq \$20,$T3,$T3 - vpaddq $D1hi,$D2lo,$D2lo - - vpsrlq \$42,$D2lo,$tmp - vpsllq \$10,$D2hi,$D2hi - vpandq $mask42,$D2lo,$H2 - vpaddq $tmp,$D2hi,$D2hi - - vpaddq $T2,$H2,$H2 # accumulate input - vpaddq $D2hi,$H0,$H0 - vpsllq \$2,$D2hi,$D2hi - - vpaddq $D2hi,$H0,$H0 - vporq $T3,$T1,$T1 - vpandq $mask44,$T1,$T1 - - vpsrlq \$44,$H0,$tmp # additional step - vpandq $mask44,$H0,$H0 - - vpaddq $tmp,$H1,$H1 - - sub \$4,$len # len-=64 - jnz .Loop_vpmadd52_4x - -.Ltail_vpmadd52_4x: - vmovdqu64 128($ctx),$R2 # load all key powers - vmovdqu64 160($ctx),$S1 - vmovdqu64 64($ctx),$R0 - vmovdqu64 96($ctx),$R1 - -.Ltail_vpmadd52_2x: - vpsllq \$2,$R2,$S2 # S2 = R2*5*4 - vpaddq $R2,$S2,$S2 - vpsllq \$2,$S2,$S2 - - #vpaddq $T2,$H2,$H2 # accumulate input - vpaddq $T0,$H0,$H0 - vpaddq $T1,$H1,$H1 - - vpxorq $D0lo,$D0lo,$D0lo - vpmadd52luq $H2,$S1,$D0lo - vpxorq $D0hi,$D0hi,$D0hi - vpmadd52huq $H2,$S1,$D0hi - vpxorq $D1lo,$D1lo,$D1lo - vpmadd52luq $H2,$S2,$D1lo - vpxorq $D1hi,$D1hi,$D1hi - vpmadd52huq $H2,$S2,$D1hi - vpxorq $D2lo,$D2lo,$D2lo - vpmadd52luq $H2,$R0,$D2lo - vpxorq $D2hi,$D2hi,$D2hi - vpmadd52huq $H2,$R0,$D2hi - - vpmadd52luq $H0,$R0,$D0lo - vpmadd52huq $H0,$R0,$D0hi - vpmadd52luq $H0,$R1,$D1lo - vpmadd52huq $H0,$R1,$D1hi - vpmadd52luq $H0,$R2,$D2lo - vpmadd52huq $H0,$R2,$D2hi - - vpmadd52luq $H1,$S2,$D0lo - vpmadd52huq $H1,$S2,$D0hi - vpmadd52luq $H1,$R0,$D1lo - vpmadd52huq $H1,$R0,$D1hi - vpmadd52luq $H1,$R1,$D2lo - vpmadd52huq $H1,$R1,$D2hi - - ################################################################ - # horizontal addition - - mov \$1,%eax - kmovw %eax,%k1 - vpsrldq \$8,$D0lo,$T0 - vpsrldq \$8,$D0hi,$H0 - vpsrldq \$8,$D1lo,$T1 - vpsrldq \$8,$D1hi,$H1 - vpaddq $T0,$D0lo,$D0lo - vpaddq $H0,$D0hi,$D0hi - vpsrldq \$8,$D2lo,$T2 - vpsrldq \$8,$D2hi,$H2 - vpaddq $T1,$D1lo,$D1lo - vpaddq $H1,$D1hi,$D1hi - vpermq \$0x2,$D0lo,$T0 - vpermq \$0x2,$D0hi,$H0 - vpaddq $T2,$D2lo,$D2lo - vpaddq $H2,$D2hi,$D2hi - - vpermq \$0x2,$D1lo,$T1 - vpermq \$0x2,$D1hi,$H1 - vpaddq $T0,$D0lo,${D0lo}{%k1}{z} - vpaddq $H0,$D0hi,${D0hi}{%k1}{z} - vpermq \$0x2,$D2lo,$T2 - vpermq \$0x2,$D2hi,$H2 - vpaddq $T1,$D1lo,${D1lo}{%k1}{z} - vpaddq $H1,$D1hi,${D1hi}{%k1}{z} - vpaddq $T2,$D2lo,${D2lo}{%k1}{z} - vpaddq $H2,$D2hi,${D2hi}{%k1}{z} - - ################################################################ - # partial reduction - vpsrlq \$44,$D0lo,$tmp - vpsllq \$8,$D0hi,$D0hi - vpandq $mask44,$D0lo,$H0 - vpaddq $tmp,$D0hi,$D0hi - - vpaddq $D0hi,$D1lo,$D1lo - - vpsrlq \$44,$D1lo,$tmp - vpsllq \$8,$D1hi,$D1hi - vpandq $mask44,$D1lo,$H1 - vpaddq $tmp,$D1hi,$D1hi - - vpaddq $D1hi,$D2lo,$D2lo - - vpsrlq \$42,$D2lo,$tmp - vpsllq \$10,$D2hi,$D2hi - vpandq $mask42,$D2lo,$H2 - vpaddq $tmp,$D2hi,$D2hi - - vpaddq $D2hi,$H0,$H0 - vpsllq \$2,$D2hi,$D2hi - - vpaddq $D2hi,$H0,$H0 - - vpsrlq \$44,$H0,$tmp # additional step - vpandq $mask44,$H0,$H0 - - vpaddq $tmp,$H1,$H1 - # at this point $len is - # either 4*n+2 or 0... - sub \$2,$len # len-=32 - ja .Lblocks_vpmadd52_4x_do - - vmovq %x#$H0,0($ctx) - vmovq %x#$H1,8($ctx) - vmovq %x#$H2,16($ctx) - vzeroall - -.Lno_data_vpmadd52_4x: - ret -.size poly1305_blocks_vpmadd52_4x,.-poly1305_blocks_vpmadd52_4x -___ -} -{ -######################################################################## -# As implied by its name 8x subroutine processes 8 blocks in parallel... -# This is intermediate version, as it's used only in cases when input -# length is either 8*n, 8*n+1 or 8*n+2... - -my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17)); -my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23)); -my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31)); -my ($RR0,$RR1,$RR2,$SS1,$SS2) = map("%ymm$_",(6..10)); - -$code.=<<___; -.type poly1305_blocks_vpmadd52_8x,\@function,4 -.align 32 -poly1305_blocks_vpmadd52_8x: - shr \$4,$len - jz .Lno_data_vpmadd52_8x # too short - - shl \$40,$padbit - mov 64($ctx),%r8 # peek on power of the key - - vmovdqa64 .Lx_mask44(%rip),$mask44 - vmovdqa64 .Lx_mask42(%rip),$mask42 - - test %r8,%r8 # is power value impossible? - js .Linit_vpmadd52 # if it is, then init R[4] - - vmovq 0($ctx),%x#$H0 # load current hash value - vmovq 8($ctx),%x#$H1 - vmovq 16($ctx),%x#$H2 - -.Lblocks_vpmadd52_8x: - ################################################################ - # fist we calculate more key powers - - vmovdqu64 128($ctx),$R2 # load 1-3-2-4 powers - vmovdqu64 160($ctx),$S1 - vmovdqu64 64($ctx),$R0 - vmovdqu64 96($ctx),$R1 - - vpsllq \$2,$R2,$S2 # S2 = R2*5*4 - vpaddq $R2,$S2,$S2 - vpsllq \$2,$S2,$S2 - - vpbroadcastq %x#$R2,$RR2 # broadcast 4th power - vpbroadcastq %x#$R0,$RR0 - vpbroadcastq %x#$R1,$RR1 - - vpxorq $D0lo,$D0lo,$D0lo - vpmadd52luq $RR2,$S1,$D0lo - vpxorq $D0hi,$D0hi,$D0hi - vpmadd52huq $RR2,$S1,$D0hi - vpxorq $D1lo,$D1lo,$D1lo - vpmadd52luq $RR2,$S2,$D1lo - vpxorq $D1hi,$D1hi,$D1hi - vpmadd52huq $RR2,$S2,$D1hi - vpxorq $D2lo,$D2lo,$D2lo - vpmadd52luq $RR2,$R0,$D2lo - vpxorq $D2hi,$D2hi,$D2hi - vpmadd52huq $RR2,$R0,$D2hi - - vpmadd52luq $RR0,$R0,$D0lo - vpmadd52huq $RR0,$R0,$D0hi - vpmadd52luq $RR0,$R1,$D1lo - vpmadd52huq $RR0,$R1,$D1hi - vpmadd52luq $RR0,$R2,$D2lo - vpmadd52huq $RR0,$R2,$D2hi - - vpmadd52luq $RR1,$S2,$D0lo - vpmadd52huq $RR1,$S2,$D0hi - vpmadd52luq $RR1,$R0,$D1lo - vpmadd52huq $RR1,$R0,$D1hi - vpmadd52luq $RR1,$R1,$D2lo - vpmadd52huq $RR1,$R1,$D2hi - - ################################################################ - # partial reduction - vpsrlq \$44,$D0lo,$tmp - vpsllq \$8,$D0hi,$D0hi - vpandq $mask44,$D0lo,$RR0 - vpaddq $tmp,$D0hi,$D0hi - - vpaddq $D0hi,$D1lo,$D1lo - - vpsrlq \$44,$D1lo,$tmp - vpsllq \$8,$D1hi,$D1hi - vpandq $mask44,$D1lo,$RR1 - vpaddq $tmp,$D1hi,$D1hi - - vpaddq $D1hi,$D2lo,$D2lo - - vpsrlq \$42,$D2lo,$tmp - vpsllq \$10,$D2hi,$D2hi - vpandq $mask42,$D2lo,$RR2 - vpaddq $tmp,$D2hi,$D2hi - - vpaddq $D2hi,$RR0,$RR0 - vpsllq \$2,$D2hi,$D2hi - - vpaddq $D2hi,$RR0,$RR0 - - vpsrlq \$44,$RR0,$tmp # additional step - vpandq $mask44,$RR0,$RR0 - - vpaddq $tmp,$RR1,$RR1 - - ################################################################ - # At this point Rx holds 1324 powers, RRx - 5768, and the goal - # is 15263748, which reflects how data is loaded... - - vpunpcklqdq $R2,$RR2,$T2 # 3748 - vpunpckhqdq $R2,$RR2,$R2 # 1526 - vpunpcklqdq $R0,$RR0,$T0 - vpunpckhqdq $R0,$RR0,$R0 - vpunpcklqdq $R1,$RR1,$T1 - vpunpckhqdq $R1,$RR1,$R1 -___ -######## switch to %zmm -map(s/%y/%z/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2); -map(s/%y/%z/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi); -map(s/%y/%z/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD); -map(s/%y/%z/, $RR0,$RR1,$RR2,$SS1,$SS2); - -$code.=<<___; - vshufi64x2 \$0x44,$R2,$T2,$RR2 # 15263748 - vshufi64x2 \$0x44,$R0,$T0,$RR0 - vshufi64x2 \$0x44,$R1,$T1,$RR1 - - vmovdqu64 16*0($inp),$T2 # load data - vmovdqu64 16*4($inp),$T3 - lea 16*8($inp),$inp - - vpsllq \$2,$RR2,$SS2 # S2 = R2*5*4 - vpsllq \$2,$RR1,$SS1 # S1 = R1*5*4 - vpaddq $RR2,$SS2,$SS2 - vpaddq $RR1,$SS1,$SS1 - vpsllq \$2,$SS2,$SS2 - vpsllq \$2,$SS1,$SS1 - - vpbroadcastq $padbit,$PAD - vpbroadcastq %x#$mask44,$mask44 - vpbroadcastq %x#$mask42,$mask42 - - vpbroadcastq %x#$SS1,$S1 # broadcast 8th power - vpbroadcastq %x#$SS2,$S2 - vpbroadcastq %x#$RR0,$R0 - vpbroadcastq %x#$RR1,$R1 - vpbroadcastq %x#$RR2,$R2 - - vpunpcklqdq $T3,$T2,$T1 # transpose data - vpunpckhqdq $T3,$T2,$T3 - - # at this point 64-bit lanes are ordered as 73625140 - - vpsrlq \$24,$T3,$T2 # splat the data - vporq $PAD,$T2,$T2 - vpaddq $T2,$H2,$H2 # accumulate input - vpandq $mask44,$T1,$T0 - vpsrlq \$44,$T1,$T1 - vpsllq \$20,$T3,$T3 - vporq $T3,$T1,$T1 - vpandq $mask44,$T1,$T1 - - sub \$8,$len - jz .Ltail_vpmadd52_8x - jmp .Loop_vpmadd52_8x - -.align 32 -.Loop_vpmadd52_8x: - #vpaddq $T2,$H2,$H2 # accumulate input - vpaddq $T0,$H0,$H0 - vpaddq $T1,$H1,$H1 - - vpxorq $D0lo,$D0lo,$D0lo - vpmadd52luq $H2,$S1,$D0lo - vpxorq $D0hi,$D0hi,$D0hi - vpmadd52huq $H2,$S1,$D0hi - vpxorq $D1lo,$D1lo,$D1lo - vpmadd52luq $H2,$S2,$D1lo - vpxorq $D1hi,$D1hi,$D1hi - vpmadd52huq $H2,$S2,$D1hi - vpxorq $D2lo,$D2lo,$D2lo - vpmadd52luq $H2,$R0,$D2lo - vpxorq $D2hi,$D2hi,$D2hi - vpmadd52huq $H2,$R0,$D2hi - - vmovdqu64 16*0($inp),$T2 # load data - vmovdqu64 16*4($inp),$T3 - lea 16*8($inp),$inp - vpmadd52luq $H0,$R0,$D0lo - vpmadd52huq $H0,$R0,$D0hi - vpmadd52luq $H0,$R1,$D1lo - vpmadd52huq $H0,$R1,$D1hi - vpmadd52luq $H0,$R2,$D2lo - vpmadd52huq $H0,$R2,$D2hi - - vpunpcklqdq $T3,$T2,$T1 # transpose data - vpunpckhqdq $T3,$T2,$T3 - vpmadd52luq $H1,$S2,$D0lo - vpmadd52huq $H1,$S2,$D0hi - vpmadd52luq $H1,$R0,$D1lo - vpmadd52huq $H1,$R0,$D1hi - vpmadd52luq $H1,$R1,$D2lo - vpmadd52huq $H1,$R1,$D2hi - - ################################################################ - # partial reduction (interleaved with data splat) - vpsrlq \$44,$D0lo,$tmp - vpsllq \$8,$D0hi,$D0hi - vpandq $mask44,$D0lo,$H0 - vpaddq $tmp,$D0hi,$D0hi - - vpsrlq \$24,$T3,$T2 - vporq $PAD,$T2,$T2 - vpaddq $D0hi,$D1lo,$D1lo - - vpsrlq \$44,$D1lo,$tmp - vpsllq \$8,$D1hi,$D1hi - vpandq $mask44,$D1lo,$H1 - vpaddq $tmp,$D1hi,$D1hi - - vpandq $mask44,$T1,$T0 - vpsrlq \$44,$T1,$T1 - vpsllq \$20,$T3,$T3 - vpaddq $D1hi,$D2lo,$D2lo - - vpsrlq \$42,$D2lo,$tmp - vpsllq \$10,$D2hi,$D2hi - vpandq $mask42,$D2lo,$H2 - vpaddq $tmp,$D2hi,$D2hi - - vpaddq $T2,$H2,$H2 # accumulate input - vpaddq $D2hi,$H0,$H0 - vpsllq \$2,$D2hi,$D2hi - - vpaddq $D2hi,$H0,$H0 - vporq $T3,$T1,$T1 - vpandq $mask44,$T1,$T1 - - vpsrlq \$44,$H0,$tmp # additional step - vpandq $mask44,$H0,$H0 - - vpaddq $tmp,$H1,$H1 - - sub \$8,$len # len-=128 - jnz .Loop_vpmadd52_8x - -.Ltail_vpmadd52_8x: - #vpaddq $T2,$H2,$H2 # accumulate input - vpaddq $T0,$H0,$H0 - vpaddq $T1,$H1,$H1 - - vpxorq $D0lo,$D0lo,$D0lo - vpmadd52luq $H2,$SS1,$D0lo - vpxorq $D0hi,$D0hi,$D0hi - vpmadd52huq $H2,$SS1,$D0hi - vpxorq $D1lo,$D1lo,$D1lo - vpmadd52luq $H2,$SS2,$D1lo - vpxorq $D1hi,$D1hi,$D1hi - vpmadd52huq $H2,$SS2,$D1hi - vpxorq $D2lo,$D2lo,$D2lo - vpmadd52luq $H2,$RR0,$D2lo - vpxorq $D2hi,$D2hi,$D2hi - vpmadd52huq $H2,$RR0,$D2hi - - vpmadd52luq $H0,$RR0,$D0lo - vpmadd52huq $H0,$RR0,$D0hi - vpmadd52luq $H0,$RR1,$D1lo - vpmadd52huq $H0,$RR1,$D1hi - vpmadd52luq $H0,$RR2,$D2lo - vpmadd52huq $H0,$RR2,$D2hi - - vpmadd52luq $H1,$SS2,$D0lo - vpmadd52huq $H1,$SS2,$D0hi - vpmadd52luq $H1,$RR0,$D1lo - vpmadd52huq $H1,$RR0,$D1hi - vpmadd52luq $H1,$RR1,$D2lo - vpmadd52huq $H1,$RR1,$D2hi - - ################################################################ - # horizontal addition - - mov \$1,%eax - kmovw %eax,%k1 - vpsrldq \$8,$D0lo,$T0 - vpsrldq \$8,$D0hi,$H0 - vpsrldq \$8,$D1lo,$T1 - vpsrldq \$8,$D1hi,$H1 - vpaddq $T0,$D0lo,$D0lo - vpaddq $H0,$D0hi,$D0hi - vpsrldq \$8,$D2lo,$T2 - vpsrldq \$8,$D2hi,$H2 - vpaddq $T1,$D1lo,$D1lo - vpaddq $H1,$D1hi,$D1hi - vpermq \$0x2,$D0lo,$T0 - vpermq \$0x2,$D0hi,$H0 - vpaddq $T2,$D2lo,$D2lo - vpaddq $H2,$D2hi,$D2hi - - vpermq \$0x2,$D1lo,$T1 - vpermq \$0x2,$D1hi,$H1 - vpaddq $T0,$D0lo,$D0lo - vpaddq $H0,$D0hi,$D0hi - vpermq \$0x2,$D2lo,$T2 - vpermq \$0x2,$D2hi,$H2 - vpaddq $T1,$D1lo,$D1lo - vpaddq $H1,$D1hi,$D1hi - vextracti64x4 \$1,$D0lo,%y#$T0 - vextracti64x4 \$1,$D0hi,%y#$H0 - vpaddq $T2,$D2lo,$D2lo - vpaddq $H2,$D2hi,$D2hi - - vextracti64x4 \$1,$D1lo,%y#$T1 - vextracti64x4 \$1,$D1hi,%y#$H1 - vextracti64x4 \$1,$D2lo,%y#$T2 - vextracti64x4 \$1,$D2hi,%y#$H2 -___ -######## switch back to %ymm -map(s/%z/%y/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2); -map(s/%z/%y/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi); -map(s/%z/%y/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD); - -$code.=<<___; - vpaddq $T0,$D0lo,${D0lo}{%k1}{z} - vpaddq $H0,$D0hi,${D0hi}{%k1}{z} - vpaddq $T1,$D1lo,${D1lo}{%k1}{z} - vpaddq $H1,$D1hi,${D1hi}{%k1}{z} - vpaddq $T2,$D2lo,${D2lo}{%k1}{z} - vpaddq $H2,$D2hi,${D2hi}{%k1}{z} - - ################################################################ - # partial reduction - vpsrlq \$44,$D0lo,$tmp - vpsllq \$8,$D0hi,$D0hi - vpandq $mask44,$D0lo,$H0 - vpaddq $tmp,$D0hi,$D0hi - - vpaddq $D0hi,$D1lo,$D1lo - - vpsrlq \$44,$D1lo,$tmp - vpsllq \$8,$D1hi,$D1hi - vpandq $mask44,$D1lo,$H1 - vpaddq $tmp,$D1hi,$D1hi - - vpaddq $D1hi,$D2lo,$D2lo - - vpsrlq \$42,$D2lo,$tmp - vpsllq \$10,$D2hi,$D2hi - vpandq $mask42,$D2lo,$H2 - vpaddq $tmp,$D2hi,$D2hi - - vpaddq $D2hi,$H0,$H0 - vpsllq \$2,$D2hi,$D2hi - - vpaddq $D2hi,$H0,$H0 - - vpsrlq \$44,$H0,$tmp # additional step - vpandq $mask44,$H0,$H0 - - vpaddq $tmp,$H1,$H1 - - ################################################################ - - vmovq %x#$H0,0($ctx) - vmovq %x#$H1,8($ctx) - vmovq %x#$H2,16($ctx) - vzeroall - -.Lno_data_vpmadd52_8x: - ret -.size poly1305_blocks_vpmadd52_8x,.-poly1305_blocks_vpmadd52_8x -___ -} -$code.=<<___; -.type poly1305_emit_base2_44,\@function,3 -.align 32 -poly1305_emit_base2_44: - mov 0($ctx),%r8 # load hash value - mov 8($ctx),%r9 - mov 16($ctx),%r10 - - mov %r9,%rax - shr \$20,%r9 - shl \$44,%rax - mov %r10,%rcx - shr \$40,%r10 - shl \$24,%rcx - - add %rax,%r8 - adc %rcx,%r9 - adc \$0,%r10 - - mov %r8,%rax - add \$5,%r8 # compare to modulus - mov %r9,%rcx - adc \$0,%r9 - adc \$0,%r10 - shr \$2,%r10 # did 130-bit value overflow? - cmovnz %r8,%rax - cmovnz %r9,%rcx - - add 0($nonce),%rax # accumulate nonce - adc 8($nonce),%rcx - mov %rax,0($mac) # write result - mov %rcx,8($mac) - - ret -.size poly1305_emit_base2_44,.-poly1305_emit_base2_44 -___ -} } } -} - -if (!$kernel) -{ # chacha20-poly1305 helpers -my ($out,$inp,$otp,$len)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order - ("%rdi","%rsi","%rdx","%rcx"); # Unix order -$code.=<<___; -.globl xor128_encrypt_n_pad -.type xor128_encrypt_n_pad,\@abi-omnipotent -.align 16 -xor128_encrypt_n_pad: - sub $otp,$inp - sub $otp,$out - mov $len,%r10 # put len aside - shr \$4,$len # len / 16 - jz .Ltail_enc - nop -.Loop_enc_xmm: - movdqu ($inp,$otp),%xmm0 - pxor ($otp),%xmm0 - movdqu %xmm0,($out,$otp) - movdqa %xmm0,($otp) - lea 16($otp),$otp - dec $len - jnz .Loop_enc_xmm - - and \$15,%r10 # len % 16 - jz .Ldone_enc - -.Ltail_enc: - mov \$16,$len - sub %r10,$len - xor %eax,%eax -.Loop_enc_byte: - mov ($inp,$otp),%al - xor ($otp),%al - mov %al,($out,$otp) - mov %al,($otp) - lea 1($otp),$otp - dec %r10 - jnz .Loop_enc_byte - - xor %eax,%eax -.Loop_enc_pad: - mov %al,($otp) - lea 1($otp),$otp - dec $len - jnz .Loop_enc_pad - -.Ldone_enc: - mov $otp,%rax - ret -.size xor128_encrypt_n_pad,.-xor128_encrypt_n_pad - -.globl xor128_decrypt_n_pad -.type xor128_decrypt_n_pad,\@abi-omnipotent -.align 16 -xor128_decrypt_n_pad: - sub $otp,$inp - sub $otp,$out - mov $len,%r10 # put len aside - shr \$4,$len # len / 16 - jz .Ltail_dec - nop -.Loop_dec_xmm: - movdqu ($inp,$otp),%xmm0 - movdqa ($otp),%xmm1 - pxor %xmm0,%xmm1 - movdqu %xmm1,($out,$otp) - movdqa %xmm0,($otp) - lea 16($otp),$otp - dec $len - jnz .Loop_dec_xmm - - pxor %xmm1,%xmm1 - and \$15,%r10 # len % 16 - jz .Ldone_dec - -.Ltail_dec: - mov \$16,$len - sub %r10,$len - xor %eax,%eax - xor %r11,%r11 -.Loop_dec_byte: - mov ($inp,$otp),%r11b - mov ($otp),%al - xor %r11b,%al - mov %al,($out,$otp) - mov %r11b,($otp) - lea 1($otp),$otp - dec %r10 - jnz .Loop_dec_byte - - xor %eax,%eax -.Loop_dec_pad: - mov %al,($otp) - lea 1($otp),$otp - dec $len - jnz .Loop_dec_pad - -.Ldone_dec: - mov $otp,%rax - ret -.size xor128_decrypt_n_pad,.-xor128_decrypt_n_pad -___ -} - -# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, -# CONTEXT *context,DISPATCHER_CONTEXT *disp) -if ($win64) { -$rec="%rcx"; -$frame="%rdx"; -$context="%r8"; -$disp="%r9"; - -$code.=<<___; -.extern __imp_RtlVirtualUnwind -.type se_handler,\@abi-omnipotent -.align 16 -se_handler: - push %rsi - push %rdi - push %rbx - push %rbp - push %r12 - push %r13 - push %r14 - push %r15 - pushfq - sub \$64,%rsp - - mov 120($context),%rax # pull context->Rax - mov 248($context),%rbx # pull context->Rip - - mov 8($disp),%rsi # disp->ImageBase - mov 56($disp),%r11 # disp->HandlerData - - mov 0(%r11),%r10d # HandlerData[0] - lea (%rsi,%r10),%r10 # prologue label - cmp %r10,%rbx # context->Rip<.Lprologue - jb .Lcommon_seh_tail - - mov 152($context),%rax # pull context->Rsp - - mov 4(%r11),%r10d # HandlerData[1] - lea (%rsi,%r10),%r10 # epilogue label - cmp %r10,%rbx # context->Rip>=.Lepilogue - jae .Lcommon_seh_tail - - lea 48(%rax),%rax - - mov -8(%rax),%rbx - mov -16(%rax),%rbp - mov -24(%rax),%r12 - mov -32(%rax),%r13 - mov -40(%rax),%r14 - mov -48(%rax),%r15 - mov %rbx,144($context) # restore context->Rbx - mov %rbp,160($context) # restore context->Rbp - mov %r12,216($context) # restore context->R12 - mov %r13,224($context) # restore context->R13 - mov %r14,232($context) # restore context->R14 - mov %r15,240($context) # restore context->R14 - - jmp .Lcommon_seh_tail -.size se_handler,.-se_handler - -.type avx_handler,\@abi-omnipotent -.align 16 -avx_handler: - push %rsi - push %rdi - push %rbx - push %rbp - push %r12 - push %r13 - push %r14 - push %r15 - pushfq - sub \$64,%rsp - - mov 120($context),%rax # pull context->Rax - mov 248($context),%rbx # pull context->Rip - - mov 8($disp),%rsi # disp->ImageBase - mov 56($disp),%r11 # disp->HandlerData - - mov 0(%r11),%r10d # HandlerData[0] - lea (%rsi,%r10),%r10 # prologue label - cmp %r10,%rbx # context->RipRsp - - mov 4(%r11),%r10d # HandlerData[1] - lea (%rsi,%r10),%r10 # epilogue label - cmp %r10,%rbx # context->Rip>=epilogue label - jae .Lcommon_seh_tail - - mov 208($context),%rax # pull context->R11 - - lea 0x50(%rax),%rsi - lea 0xf8(%rax),%rax - lea 512($context),%rdi # &context.Xmm6 - mov \$20,%ecx - .long 0xa548f3fc # cld; rep movsq - -.Lcommon_seh_tail: - mov 8(%rax),%rdi - mov 16(%rax),%rsi - mov %rax,152($context) # restore context->Rsp - mov %rsi,168($context) # restore context->Rsi - mov %rdi,176($context) # restore context->Rdi - - mov 40($disp),%rdi # disp->ContextRecord - mov $context,%rsi # context - mov \$154,%ecx # sizeof(CONTEXT) - .long 0xa548f3fc # cld; rep movsq - - mov $disp,%rsi - xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER - mov 8(%rsi),%rdx # arg2, disp->ImageBase - mov 0(%rsi),%r8 # arg3, disp->ControlPc - mov 16(%rsi),%r9 # arg4, disp->FunctionEntry - mov 40(%rsi),%r10 # disp->ContextRecord - lea 56(%rsi),%r11 # &disp->HandlerData - lea 24(%rsi),%r12 # &disp->EstablisherFrame - mov %r10,32(%rsp) # arg5 - mov %r11,40(%rsp) # arg6 - mov %r12,48(%rsp) # arg7 - mov %rcx,56(%rsp) # arg8, (NULL) - call *__imp_RtlVirtualUnwind(%rip) - - mov \$1,%eax # ExceptionContinueSearch - add \$64,%rsp - popfq - pop %r15 - pop %r14 - pop %r13 - pop %r12 - pop %rbp - pop %rbx - pop %rdi - pop %rsi - ret -.size avx_handler,.-avx_handler - -.section .pdata -.align 4 - .rva .LSEH_begin_poly1305_init_x86_64 - .rva .LSEH_end_poly1305_init_x86_64 - .rva .LSEH_info_poly1305_init_x86_64 - - .rva .LSEH_begin_poly1305_blocks_x86_64 - .rva .LSEH_end_poly1305_blocks_x86_64 - .rva .LSEH_info_poly1305_blocks_x86_64 - - .rva .LSEH_begin_poly1305_emit_x86_64 - .rva .LSEH_end_poly1305_emit_x86_64 - .rva .LSEH_info_poly1305_emit_x86_64 -___ -$code.=<<___ if ($avx); - .rva .LSEH_begin_poly1305_blocks_avx - .rva .Lbase2_64_avx - .rva .LSEH_info_poly1305_blocks_avx_1 - - .rva .Lbase2_64_avx - .rva .Leven_avx - .rva .LSEH_info_poly1305_blocks_avx_2 - - .rva .Leven_avx - .rva .LSEH_end_poly1305_blocks_avx - .rva .LSEH_info_poly1305_blocks_avx_3 - - .rva .LSEH_begin_poly1305_emit_avx - .rva .LSEH_end_poly1305_emit_avx - .rva .LSEH_info_poly1305_emit_avx -___ -$code.=<<___ if ($avx>1); - .rva .LSEH_begin_poly1305_blocks_avx2 - .rva .Lbase2_64_avx2 - .rva .LSEH_info_poly1305_blocks_avx2_1 - - .rva .Lbase2_64_avx2 - .rva .Leven_avx2 - .rva .LSEH_info_poly1305_blocks_avx2_2 - - .rva .Leven_avx2 - .rva .LSEH_end_poly1305_blocks_avx2 - .rva .LSEH_info_poly1305_blocks_avx2_3 -___ -$code.=<<___ if ($avx>2); - .rva .LSEH_begin_poly1305_blocks_avx512 - .rva .LSEH_end_poly1305_blocks_avx512 - .rva .LSEH_info_poly1305_blocks_avx512 -___ -$code.=<<___; -.section .xdata -.align 8 -.LSEH_info_poly1305_init_x86_64: - .byte 9,0,0,0 - .rva se_handler - .rva .LSEH_begin_poly1305_init_x86_64,.LSEH_begin_poly1305_init_x86_64 - -.LSEH_info_poly1305_blocks_x86_64: - .byte 9,0,0,0 - .rva se_handler - .rva .Lblocks_body,.Lblocks_epilogue - -.LSEH_info_poly1305_emit_x86_64: - .byte 9,0,0,0 - .rva se_handler - .rva .LSEH_begin_poly1305_emit_x86_64,.LSEH_begin_poly1305_emit_x86_64 -___ -$code.=<<___ if ($avx); -.LSEH_info_poly1305_blocks_avx_1: - .byte 9,0,0,0 - .rva se_handler - .rva .Lblocks_avx_body,.Lblocks_avx_epilogue # HandlerData[] - -.LSEH_info_poly1305_blocks_avx_2: - .byte 9,0,0,0 - .rva se_handler - .rva .Lbase2_64_avx_body,.Lbase2_64_avx_epilogue # HandlerData[] - -.LSEH_info_poly1305_blocks_avx_3: - .byte 9,0,0,0 - .rva avx_handler - .rva .Ldo_avx_body,.Ldo_avx_epilogue # HandlerData[] - -.LSEH_info_poly1305_emit_avx: - .byte 9,0,0,0 - .rva se_handler - .rva .LSEH_begin_poly1305_emit_avx,.LSEH_begin_poly1305_emit_avx -___ -$code.=<<___ if ($avx>1); -.LSEH_info_poly1305_blocks_avx2_1: - .byte 9,0,0,0 - .rva se_handler - .rva .Lblocks_avx2_body,.Lblocks_avx2_epilogue # HandlerData[] - -.LSEH_info_poly1305_blocks_avx2_2: - .byte 9,0,0,0 - .rva se_handler - .rva .Lbase2_64_avx2_body,.Lbase2_64_avx2_epilogue # HandlerData[] - -.LSEH_info_poly1305_blocks_avx2_3: - .byte 9,0,0,0 - .rva avx_handler - .rva .Ldo_avx2_body,.Ldo_avx2_epilogue # HandlerData[] -___ -$code.=<<___ if ($avx>2); -.LSEH_info_poly1305_blocks_avx512: - .byte 9,0,0,0 - .rva avx_handler - .rva .Ldo_avx512_body,.Ldo_avx512_epilogue # HandlerData[] -___ -} - -open SELF,$0; -while() { - next if (/^#!/); - last if (!s/^#/\/\// and !/^$/); - print; -} -close SELF; - -foreach (split('\n',$code)) { - s/\`([^\`]*)\`/eval($1)/ge; - s/%r([a-z]+)#d/%e$1/g; - s/%r([0-9]+)#d/%r$1d/g; - s/%x#%[yz]/%x/g or s/%y#%z/%y/g or s/%z#%[yz]/%z/g; - - if ($kernel) { - s/(^\.type.*),[0-9]+$/\1/; - s/(^\.type.*),\@abi-omnipotent+$/\1,\@function/; - next if /^\.cfi.*/; - } - - print $_,"\n"; -} -close STDOUT; diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index 79bb58737d528..0cc4537e6617c 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -1,6 +1,8 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT +// SPDX-License-Identifier: GPL-2.0-or-later /* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * Poly1305 authenticator algorithm, RFC7539, SIMD glue code + * + * Copyright (C) 2015 Martin Willi */ #include @@ -11,166 +13,108 @@ #include #include #include -#include #include -asmlinkage void poly1305_init_x86_64(void *ctx, - const u8 key[POLY1305_KEY_SIZE]); -asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp, - const size_t len, const u32 padbit); -asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_DIGEST_SIZE], - const u32 nonce[4]); -asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_DIGEST_SIZE], - const u32 nonce[4]); -asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len, - const u32 padbit); -asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len, - const u32 padbit); -asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp, - const size_t len, const u32 padbit); - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx); +asmlinkage void poly1305_block_sse2(u32 *h, const u8 *src, + const u32 *r, unsigned int blocks); +asmlinkage void poly1305_2block_sse2(u32 *h, const u8 *src, const u32 *r, + unsigned int blocks, const u32 *u); +asmlinkage void poly1305_4block_avx2(u32 *h, const u8 *src, const u32 *r, + unsigned int blocks, const u32 *u); + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_simd); static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2); -static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx512); - -struct poly1305_arch_internal { - union { - struct { - u32 h[5]; - u32 is_base2_26; - }; - u64 hs[3]; - }; - u64 r[2]; - u64 pad; - struct { u32 r2, r1, r4, r3; } rn[9]; -}; -/* The AVX code uses base 2^26, while the scalar code uses base 2^64. If we hit - * the unfortunate situation of using AVX and then having to go back to scalar - * -- because the user is silly and has called the update function from two - * separate contexts -- then we need to convert back to the original base before - * proceeding. It is possible to reason that the initial reduction below is - * sufficient given the implementation invariants. However, for an avoidance of - * doubt and because this is not performance critical, we do the full reduction - * anyway. Z3 proof of below function: https://xn--4db.cc/ltPtHCKN/py - */ -static void convert_to_base2_64(void *ctx) +static void poly1305_simd_mult(u32 *a, const u32 *b) { - struct poly1305_arch_internal *state = ctx; - u32 cy; - - if (!state->is_base2_26) - return; - - cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy; - cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy; - cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy; - cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy; - state->hs[0] = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0]; - state->hs[1] = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12); - state->hs[2] = state->h[4] >> 24; -#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) - cy = (state->hs[2] >> 2) + (state->hs[2] & ~3ULL); - state->hs[2] &= 3; - state->hs[0] += cy; - state->hs[1] += (cy = ULT(state->hs[0], cy)); - state->hs[2] += ULT(state->hs[1], cy); -#undef ULT - state->is_base2_26 = 0; -} + u8 m[POLY1305_BLOCK_SIZE]; -static void poly1305_simd_init(void *ctx, const u8 key[POLY1305_KEY_SIZE]) -{ - poly1305_init_x86_64(ctx, key); + memset(m, 0, sizeof(m)); + /* The poly1305 block function adds a hi-bit to the accumulator which + * we don't need for key multiplication; compensate for it. */ + a[4] -= 1 << 24; + poly1305_block_sse2(a, m, b, 1); } -static void poly1305_simd_blocks(void *ctx, const u8 *inp, size_t len, - const u32 padbit) +static unsigned int poly1305_scalar_blocks(struct poly1305_desc_ctx *dctx, + const u8 *src, unsigned int srclen) { - struct poly1305_arch_internal *state = ctx; - - /* SIMD disables preemption, so relax after processing each page. */ - BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE || - PAGE_SIZE % POLY1305_BLOCK_SIZE); - - if (!IS_ENABLED(CONFIG_AS_AVX) || !static_branch_likely(&poly1305_use_avx) || - (len < (POLY1305_BLOCK_SIZE * 18) && !state->is_base2_26) || - !crypto_simd_usable()) { - convert_to_base2_64(ctx); - poly1305_blocks_x86_64(ctx, inp, len, padbit); - return; - } + unsigned int datalen; - for (;;) { - const size_t bytes = min_t(size_t, len, PAGE_SIZE); - - kernel_fpu_begin(); - if (IS_ENABLED(CONFIG_AS_AVX512) && static_branch_likely(&poly1305_use_avx512)) - poly1305_blocks_avx512(ctx, inp, bytes, padbit); - else if (IS_ENABLED(CONFIG_AS_AVX2) && static_branch_likely(&poly1305_use_avx2)) - poly1305_blocks_avx2(ctx, inp, bytes, padbit); - else - poly1305_blocks_avx(ctx, inp, bytes, padbit); - kernel_fpu_end(); - len -= bytes; - if (!len) - break; - inp += bytes; + if (unlikely(!dctx->sset)) { + datalen = crypto_poly1305_setdesckey(dctx, src, srclen); + src += srclen - datalen; + srclen = datalen; + } + if (srclen >= POLY1305_BLOCK_SIZE) { + poly1305_core_blocks(&dctx->h, dctx->r, src, + srclen / POLY1305_BLOCK_SIZE, 1); + srclen %= POLY1305_BLOCK_SIZE; } + return srclen; } -static void poly1305_simd_emit(void *ctx, u8 mac[POLY1305_DIGEST_SIZE], - const u32 nonce[4]) +static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx, + const u8 *src, unsigned int srclen) { - if (!IS_ENABLED(CONFIG_AS_AVX) || !static_branch_likely(&poly1305_use_avx)) - poly1305_emit_x86_64(ctx, mac, nonce); - else - poly1305_emit_avx(ctx, mac, nonce); -} + unsigned int blocks, datalen; -void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 *key) -{ - poly1305_simd_init(&dctx->h, key); - dctx->s[0] = get_unaligned_le32(&key[16]); - dctx->s[1] = get_unaligned_le32(&key[20]); - dctx->s[2] = get_unaligned_le32(&key[24]); - dctx->s[3] = get_unaligned_le32(&key[28]); - dctx->buflen = 0; - dctx->sset = true; -} -EXPORT_SYMBOL(poly1305_init_arch); - -static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx, - const u8 *inp, unsigned int len) -{ - unsigned int acc = 0; if (unlikely(!dctx->sset)) { - if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) { - poly1305_simd_init(&dctx->h, inp); - inp += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - acc += POLY1305_BLOCK_SIZE; - dctx->rset = 1; + datalen = crypto_poly1305_setdesckey(dctx, src, srclen); + src += srclen - datalen; + srclen = datalen; + } + + if (IS_ENABLED(CONFIG_AS_AVX2) && + static_branch_likely(&poly1305_use_avx2) && + srclen >= POLY1305_BLOCK_SIZE * 4) { + if (unlikely(dctx->rset < 4)) { + if (dctx->rset < 2) { + dctx->r[1] = dctx->r[0]; + poly1305_simd_mult(dctx->r[1].r, dctx->r[0].r); + } + dctx->r[2] = dctx->r[1]; + poly1305_simd_mult(dctx->r[2].r, dctx->r[0].r); + dctx->r[3] = dctx->r[2]; + poly1305_simd_mult(dctx->r[3].r, dctx->r[0].r); + dctx->rset = 4; } - if (len >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(&inp[0]); - dctx->s[1] = get_unaligned_le32(&inp[4]); - dctx->s[2] = get_unaligned_le32(&inp[8]); - dctx->s[3] = get_unaligned_le32(&inp[12]); - inp += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - acc += POLY1305_BLOCK_SIZE; - dctx->sset = true; + blocks = srclen / (POLY1305_BLOCK_SIZE * 4); + poly1305_4block_avx2(dctx->h.h, src, dctx->r[0].r, blocks, + dctx->r[1].r); + src += POLY1305_BLOCK_SIZE * 4 * blocks; + srclen -= POLY1305_BLOCK_SIZE * 4 * blocks; + } + + if (likely(srclen >= POLY1305_BLOCK_SIZE * 2)) { + if (unlikely(dctx->rset < 2)) { + dctx->r[1] = dctx->r[0]; + poly1305_simd_mult(dctx->r[1].r, dctx->r[0].r); + dctx->rset = 2; } + blocks = srclen / (POLY1305_BLOCK_SIZE * 2); + poly1305_2block_sse2(dctx->h.h, src, dctx->r[0].r, + blocks, dctx->r[1].r); + src += POLY1305_BLOCK_SIZE * 2 * blocks; + srclen -= POLY1305_BLOCK_SIZE * 2 * blocks; + } + if (srclen >= POLY1305_BLOCK_SIZE) { + poly1305_block_sse2(dctx->h.h, src, dctx->r[0].r, 1); + srclen -= POLY1305_BLOCK_SIZE; } - return acc; + return srclen; +} + +void poly1305_init_arch(struct poly1305_desc_ctx *desc, const u8 *key) +{ + poly1305_init_generic(desc, key); } +EXPORT_SYMBOL(poly1305_init_arch); void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int srclen) { - unsigned int bytes, used; + unsigned int bytes; if (unlikely(dctx->buflen)) { bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); @@ -180,19 +124,31 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, dctx->buflen += bytes; if (dctx->buflen == POLY1305_BLOCK_SIZE) { - if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf, POLY1305_BLOCK_SIZE))) - poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1); + if (static_branch_likely(&poly1305_use_simd) && + likely(crypto_simd_usable())) { + kernel_fpu_begin(); + poly1305_simd_blocks(dctx, dctx->buf, + POLY1305_BLOCK_SIZE); + kernel_fpu_end(); + } else { + poly1305_scalar_blocks(dctx, dctx->buf, + POLY1305_BLOCK_SIZE); + } dctx->buflen = 0; } } if (likely(srclen >= POLY1305_BLOCK_SIZE)) { - bytes = round_down(srclen, POLY1305_BLOCK_SIZE); - srclen -= bytes; - used = crypto_poly1305_setdctxkey(dctx, src, bytes); - if (likely(bytes - used)) - poly1305_simd_blocks(&dctx->h, src + used, bytes - used, 1); - src += bytes; + if (static_branch_likely(&poly1305_use_simd) && + likely(crypto_simd_usable())) { + kernel_fpu_begin(); + bytes = poly1305_simd_blocks(dctx, src, srclen); + kernel_fpu_end(); + } else { + bytes = poly1305_scalar_blocks(dctx, src, srclen); + } + src += srclen - bytes; + srclen = bytes; } if (unlikely(srclen)) { @@ -202,17 +158,9 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, } EXPORT_SYMBOL(poly1305_update_arch); -void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) +void poly1305_final_arch(struct poly1305_desc_ctx *desc, u8 *digest) { - if (unlikely(dctx->buflen)) { - dctx->buf[dctx->buflen++] = 1; - memset(dctx->buf + dctx->buflen, 0, - POLY1305_BLOCK_SIZE - dctx->buflen); - poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); - } - - poly1305_simd_emit(&dctx->h, dst, dctx->s); - *dctx = (struct poly1305_desc_ctx){}; + poly1305_final_generic(desc, digest); } EXPORT_SYMBOL(poly1305_final_arch); @@ -220,34 +168,38 @@ static int crypto_poly1305_init(struct shash_desc *desc) { struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - *dctx = (struct poly1305_desc_ctx){}; + poly1305_core_init(&dctx->h); + dctx->buflen = 0; + dctx->rset = 0; + dctx->sset = false; + return 0; } -static int crypto_poly1305_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) +static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) { struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - poly1305_update_arch(dctx, src, srclen); + if (unlikely(!dctx->sset)) + return -ENOKEY; + + poly1305_final_generic(dctx, dst); return 0; } -static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) +static int poly1305_simd_update(struct shash_desc *desc, + const u8 *src, unsigned int srclen) { struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - if (unlikely(!dctx->sset)) - return -ENOKEY; - - poly1305_final_arch(dctx, dst); + poly1305_update_arch(dctx, src, srclen); return 0; } static struct shash_alg alg = { .digestsize = POLY1305_DIGEST_SIZE, .init = crypto_poly1305_init, - .update = crypto_poly1305_update, + .update = poly1305_simd_update, .final = crypto_poly1305_final, .descsize = sizeof(struct poly1305_desc_ctx), .base = { @@ -261,19 +213,17 @@ static struct shash_alg alg = { static int __init poly1305_simd_mod_init(void) { - if (IS_ENABLED(CONFIG_AS_AVX) && boot_cpu_has(X86_FEATURE_AVX) && - cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) - static_branch_enable(&poly1305_use_avx); - if (IS_ENABLED(CONFIG_AS_AVX2) && boot_cpu_has(X86_FEATURE_AVX) && + if (!boot_cpu_has(X86_FEATURE_XMM2)) + return 0; + + static_branch_enable(&poly1305_use_simd); + + if (IS_ENABLED(CONFIG_AS_AVX2) && + boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) static_branch_enable(&poly1305_use_avx2); - if (IS_ENABLED(CONFIG_AS_AVX512) && boot_cpu_has(X86_FEATURE_AVX) && - boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX512F) && - cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | XFEATURE_MASK_AVX512, NULL) && - /* Skylake downclocks unacceptably much when using zmm, but later generations are fast. */ - boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X) - static_branch_enable(&poly1305_use_avx512); + return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? crypto_register_shash(&alg) : 0; } @@ -287,7 +237,7 @@ module_init(poly1305_simd_mod_init); module_exit(poly1305_simd_mod_exit); MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Jason A. Donenfeld "); +MODULE_AUTHOR("Martin Willi "); MODULE_DESCRIPTION("Poly1305 authenticator"); MODULE_ALIAS_CRYPTO("poly1305"); MODULE_ALIAS_CRYPTO("poly1305-simd"); diff --git a/crypto/Kconfig b/crypto/Kconfig index c24a47406f8f5..cdb51d4272d0c 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -136,6 +136,8 @@ config CRYPTO_USER Userspace configuration for cryptographic instantiations such as cbc(aes). +if CRYPTO_MANAGER2 + config CRYPTO_MANAGER_DISABLE_TESTS bool "Disable run-time self tests" default y @@ -153,6 +155,8 @@ config CRYPTO_MANAGER_EXTRA_TESTS This is intended for developer use only, as these tests take much longer to run than the normal self tests. +endif # if CRYPTO_MANAGER2 + config CRYPTO_GF128MUL tristate diff --git a/crypto/adiantum.c b/crypto/adiantum.c index 24440f6a69153..9dc53cf9b1f17 100644 --- a/crypto/adiantum.c +++ b/crypto/adiantum.c @@ -72,7 +72,7 @@ struct adiantum_tfm_ctx { struct crypto_skcipher *streamcipher; struct crypto_cipher *blockcipher; struct crypto_shash *hash; - struct poly1305_core_key header_hash_key; + struct poly1305_key header_hash_key; }; struct adiantum_request_ctx { @@ -249,7 +249,7 @@ static void adiantum_hash_header(struct skcipher_request *req) poly1305_core_blocks(&state, &tctx->header_hash_key, req->iv, TWEAK_SIZE / POLY1305_BLOCK_SIZE, 1); - poly1305_core_emit(&state, NULL, &rctx->header_hash); + poly1305_core_emit(&state, &rctx->header_hash); } /* Hash the left-hand part (the "bulk") of the message using NHPoly1305 */ diff --git a/crypto/nhpoly1305.c b/crypto/nhpoly1305.c index 8a3006c3b51b9..f6b6a52092b49 100644 --- a/crypto/nhpoly1305.c +++ b/crypto/nhpoly1305.c @@ -210,7 +210,7 @@ int crypto_nhpoly1305_final_helper(struct shash_desc *desc, u8 *dst, nh_t nh_fn) if (state->nh_remaining) process_nh_hash_value(state, key); - poly1305_core_emit(&state->poly_state, NULL, dst); + poly1305_core_emit(&state->poly_state, dst); return 0; } EXPORT_SYMBOL(crypto_nhpoly1305_final_helper); diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c index 94af47eb6fa69..21edbd8c99fbc 100644 --- a/crypto/poly1305_generic.c +++ b/crypto/poly1305_generic.c @@ -31,29 +31,6 @@ static int crypto_poly1305_init(struct shash_desc *desc) return 0; } -static unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, - const u8 *src, unsigned int srclen) -{ - if (!dctx->sset) { - if (!dctx->rset && srclen >= POLY1305_BLOCK_SIZE) { - poly1305_core_setkey(&dctx->core_r, src); - src += POLY1305_BLOCK_SIZE; - srclen -= POLY1305_BLOCK_SIZE; - dctx->rset = 2; - } - if (srclen >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(src + 0); - dctx->s[1] = get_unaligned_le32(src + 4); - dctx->s[2] = get_unaligned_le32(src + 8); - dctx->s[3] = get_unaligned_le32(src + 12); - src += POLY1305_BLOCK_SIZE; - srclen -= POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - } - return srclen; -} - static void poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int srclen) { @@ -65,7 +42,7 @@ static void poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, srclen = datalen; } - poly1305_core_blocks(&dctx->h, &dctx->core_r, src, + poly1305_core_blocks(&dctx->h, dctx->r, src, srclen / POLY1305_BLOCK_SIZE, 1); } diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig index 537b1af285253..e148bbd9e783e 100644 --- a/drivers/net/Kconfig +++ b/drivers/net/Kconfig @@ -71,49 +71,6 @@ config DUMMY To compile this driver as a module, choose M here: the module will be called dummy. -config WIREGUARD - tristate "WireGuard secure network tunnel" - depends on NET && INET - depends on IPV6 || !IPV6 - select NET_UDP_TUNNEL - select DST_CACHE - select CRYPTO - select CRYPTO_LIB_CURVE25519 - select CRYPTO_LIB_CHACHA20POLY1305 - select CRYPTO_LIB_BLAKE2S - select CRYPTO_CHACHA20_X86_64 if X86 && 64BIT - select CRYPTO_POLY1305_X86_64 if X86 && 64BIT - select CRYPTO_BLAKE2S_X86 if X86 && 64BIT - select CRYPTO_CURVE25519_X86 if X86 && 64BIT - select ARM_CRYPTO if ARM - select ARM64_CRYPTO if ARM64 - select CRYPTO_CHACHA20_NEON if (ARM || ARM64) && KERNEL_MODE_NEON - select CRYPTO_POLY1305_NEON if ARM64 && KERNEL_MODE_NEON - select CRYPTO_POLY1305_ARM if ARM - select CRYPTO_CURVE25519_NEON if ARM && KERNEL_MODE_NEON - select CRYPTO_CHACHA_MIPS if CPU_MIPS32_R2 - select CRYPTO_POLY1305_MIPS if CPU_MIPS32 || (CPU_MIPS64 && 64BIT) - help - WireGuard is a secure, fast, and easy to use replacement for IPSec - that uses modern cryptography and clever networking tricks. It's - designed to be fairly general purpose and abstract enough to fit most - use cases, while at the same time remaining extremely simple to - configure. See www.wireguard.com for more info. - - It's safe to say Y or M here, as the driver is very lightweight and - is only in use when an administrator chooses to add an interface. - -config WIREGUARD_DEBUG - bool "Debugging checks and verbose messages" - depends on WIREGUARD - help - This will write log messages for handshake and other events - that occur for a WireGuard interface. It will also perform some - extra validation checks and unit tests at various points. This is - only useful for debugging. - - Say N here unless you know what you're doing. - config EQUALIZER tristate "EQL (serial line load balancing) support" ---help--- diff --git a/drivers/net/Makefile b/drivers/net/Makefile index 953b7c12f0b0c..0d3ba056cda3c 100644 --- a/drivers/net/Makefile +++ b/drivers/net/Makefile @@ -10,7 +10,6 @@ obj-$(CONFIG_BONDING) += bonding/ obj-$(CONFIG_IPVLAN) += ipvlan/ obj-$(CONFIG_IPVTAP) += ipvlan/ obj-$(CONFIG_DUMMY) += dummy.o -obj-$(CONFIG_WIREGUARD) += wireguard/ obj-$(CONFIG_EQUALIZER) += eql.o obj-$(CONFIG_IFB) += ifb.o obj-$(CONFIG_MACSEC) += macsec.o diff --git a/drivers/net/wireguard/Makefile b/drivers/net/wireguard/Makefile deleted file mode 100644 index fc52b2cb500b3..0000000000000 --- a/drivers/net/wireguard/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -ccflags-y := -O3 -ccflags-y += -D'pr_fmt(fmt)=KBUILD_MODNAME ": " fmt' -ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DDEBUG -wireguard-y := main.o -wireguard-y += noise.o -wireguard-y += device.o -wireguard-y += peer.o -wireguard-y += timers.o -wireguard-y += queueing.o -wireguard-y += send.o -wireguard-y += receive.o -wireguard-y += socket.o -wireguard-y += peerlookup.o -wireguard-y += allowedips.o -wireguard-y += ratelimiter.o -wireguard-y += cookie.o -wireguard-y += netlink.o -obj-$(CONFIG_WIREGUARD) := wireguard.o diff --git a/drivers/net/wireguard/allowedips.c b/drivers/net/wireguard/allowedips.c deleted file mode 100644 index 3725e9cd85f4f..0000000000000 --- a/drivers/net/wireguard/allowedips.c +++ /dev/null @@ -1,377 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include "allowedips.h" -#include "peer.h" - -static void swap_endian(u8 *dst, const u8 *src, u8 bits) -{ - if (bits == 32) { - *(u32 *)dst = be32_to_cpu(*(const __be32 *)src); - } else if (bits == 128) { - ((u64 *)dst)[0] = be64_to_cpu(((const __be64 *)src)[0]); - ((u64 *)dst)[1] = be64_to_cpu(((const __be64 *)src)[1]); - } -} - -static void copy_and_assign_cidr(struct allowedips_node *node, const u8 *src, - u8 cidr, u8 bits) -{ - node->cidr = cidr; - node->bit_at_a = cidr / 8U; -#ifdef __LITTLE_ENDIAN - node->bit_at_a ^= (bits / 8U - 1U) % 8U; -#endif - node->bit_at_b = 7U - (cidr % 8U); - node->bitlen = bits; - memcpy(node->bits, src, bits / 8U); -} -#define CHOOSE_NODE(parent, key) \ - parent->bit[(key[parent->bit_at_a] >> parent->bit_at_b) & 1] - -static void push_rcu(struct allowedips_node **stack, - struct allowedips_node __rcu *p, unsigned int *len) -{ - if (rcu_access_pointer(p)) { - WARN_ON(IS_ENABLED(DEBUG) && *len >= 128); - stack[(*len)++] = rcu_dereference_raw(p); - } -} - -static void root_free_rcu(struct rcu_head *rcu) -{ - struct allowedips_node *node, *stack[128] = { - container_of(rcu, struct allowedips_node, rcu) }; - unsigned int len = 1; - - while (len > 0 && (node = stack[--len])) { - push_rcu(stack, node->bit[0], &len); - push_rcu(stack, node->bit[1], &len); - kfree(node); - } -} - -static void root_remove_peer_lists(struct allowedips_node *root) -{ - struct allowedips_node *node, *stack[128] = { root }; - unsigned int len = 1; - - while (len > 0 && (node = stack[--len])) { - push_rcu(stack, node->bit[0], &len); - push_rcu(stack, node->bit[1], &len); - if (rcu_access_pointer(node->peer)) - list_del(&node->peer_list); - } -} - -static void walk_remove_by_peer(struct allowedips_node __rcu **top, - struct wg_peer *peer, struct mutex *lock) -{ -#define REF(p) rcu_access_pointer(p) -#define DEREF(p) rcu_dereference_protected(*(p), lockdep_is_held(lock)) -#define PUSH(p) ({ \ - WARN_ON(IS_ENABLED(DEBUG) && len >= 128); \ - stack[len++] = p; \ - }) - - struct allowedips_node __rcu **stack[128], **nptr; - struct allowedips_node *node, *prev; - unsigned int len; - - if (unlikely(!peer || !REF(*top))) - return; - - for (prev = NULL, len = 0, PUSH(top); len > 0; prev = node) { - nptr = stack[len - 1]; - node = DEREF(nptr); - if (!node) { - --len; - continue; - } - if (!prev || REF(prev->bit[0]) == node || - REF(prev->bit[1]) == node) { - if (REF(node->bit[0])) - PUSH(&node->bit[0]); - else if (REF(node->bit[1])) - PUSH(&node->bit[1]); - } else if (REF(node->bit[0]) == prev) { - if (REF(node->bit[1])) - PUSH(&node->bit[1]); - } else { - if (rcu_dereference_protected(node->peer, - lockdep_is_held(lock)) == peer) { - RCU_INIT_POINTER(node->peer, NULL); - list_del_init(&node->peer_list); - if (!node->bit[0] || !node->bit[1]) { - rcu_assign_pointer(*nptr, DEREF( - &node->bit[!REF(node->bit[0])])); - kfree_rcu(node, rcu); - node = DEREF(nptr); - } - } - --len; - } - } - -#undef REF -#undef DEREF -#undef PUSH -} - -static unsigned int fls128(u64 a, u64 b) -{ - return a ? fls64(a) + 64U : fls64(b); -} - -static u8 common_bits(const struct allowedips_node *node, const u8 *key, - u8 bits) -{ - if (bits == 32) - return 32U - fls(*(const u32 *)node->bits ^ *(const u32 *)key); - else if (bits == 128) - return 128U - fls128( - *(const u64 *)&node->bits[0] ^ *(const u64 *)&key[0], - *(const u64 *)&node->bits[8] ^ *(const u64 *)&key[8]); - return 0; -} - -static bool prefix_matches(const struct allowedips_node *node, const u8 *key, - u8 bits) -{ - /* This could be much faster if it actually just compared the common - * bits properly, by precomputing a mask bswap(~0 << (32 - cidr)), and - * the rest, but it turns out that common_bits is already super fast on - * modern processors, even taking into account the unfortunate bswap. - * So, we just inline it like this instead. - */ - return common_bits(node, key, bits) >= node->cidr; -} - -static struct allowedips_node *find_node(struct allowedips_node *trie, u8 bits, - const u8 *key) -{ - struct allowedips_node *node = trie, *found = NULL; - - while (node && prefix_matches(node, key, bits)) { - if (rcu_access_pointer(node->peer)) - found = node; - if (node->cidr == bits) - break; - node = rcu_dereference_bh(CHOOSE_NODE(node, key)); - } - return found; -} - -/* Returns a strong reference to a peer */ -static struct wg_peer *lookup(struct allowedips_node __rcu *root, u8 bits, - const void *be_ip) -{ - /* Aligned so it can be passed to fls/fls64 */ - u8 ip[16] __aligned(__alignof(u64)); - struct allowedips_node *node; - struct wg_peer *peer = NULL; - - swap_endian(ip, be_ip, bits); - - rcu_read_lock_bh(); -retry: - node = find_node(rcu_dereference_bh(root), bits, ip); - if (node) { - peer = wg_peer_get_maybe_zero(rcu_dereference_bh(node->peer)); - if (!peer) - goto retry; - } - rcu_read_unlock_bh(); - return peer; -} - -static bool node_placement(struct allowedips_node __rcu *trie, const u8 *key, - u8 cidr, u8 bits, struct allowedips_node **rnode, - struct mutex *lock) -{ - struct allowedips_node *node = rcu_dereference_protected(trie, - lockdep_is_held(lock)); - struct allowedips_node *parent = NULL; - bool exact = false; - - while (node && node->cidr <= cidr && prefix_matches(node, key, bits)) { - parent = node; - if (parent->cidr == cidr) { - exact = true; - break; - } - node = rcu_dereference_protected(CHOOSE_NODE(parent, key), - lockdep_is_held(lock)); - } - *rnode = parent; - return exact; -} - -static int add(struct allowedips_node __rcu **trie, u8 bits, const u8 *key, - u8 cidr, struct wg_peer *peer, struct mutex *lock) -{ - struct allowedips_node *node, *parent, *down, *newnode; - - if (unlikely(cidr > bits || !peer)) - return -EINVAL; - - if (!rcu_access_pointer(*trie)) { - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (unlikely(!node)) - return -ENOMEM; - RCU_INIT_POINTER(node->peer, peer); - list_add_tail(&node->peer_list, &peer->allowedips_list); - copy_and_assign_cidr(node, key, cidr, bits); - rcu_assign_pointer(*trie, node); - return 0; - } - if (node_placement(*trie, key, cidr, bits, &node, lock)) { - rcu_assign_pointer(node->peer, peer); - list_move_tail(&node->peer_list, &peer->allowedips_list); - return 0; - } - - newnode = kzalloc(sizeof(*newnode), GFP_KERNEL); - if (unlikely(!newnode)) - return -ENOMEM; - RCU_INIT_POINTER(newnode->peer, peer); - list_add_tail(&newnode->peer_list, &peer->allowedips_list); - copy_and_assign_cidr(newnode, key, cidr, bits); - - if (!node) { - down = rcu_dereference_protected(*trie, lockdep_is_held(lock)); - } else { - down = rcu_dereference_protected(CHOOSE_NODE(node, key), - lockdep_is_held(lock)); - if (!down) { - rcu_assign_pointer(CHOOSE_NODE(node, key), newnode); - return 0; - } - } - cidr = min(cidr, common_bits(down, key, bits)); - parent = node; - - if (newnode->cidr == cidr) { - rcu_assign_pointer(CHOOSE_NODE(newnode, down->bits), down); - if (!parent) - rcu_assign_pointer(*trie, newnode); - else - rcu_assign_pointer(CHOOSE_NODE(parent, newnode->bits), - newnode); - } else { - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (unlikely(!node)) { - list_del(&newnode->peer_list); - kfree(newnode); - return -ENOMEM; - } - INIT_LIST_HEAD(&node->peer_list); - copy_and_assign_cidr(node, newnode->bits, cidr, bits); - - rcu_assign_pointer(CHOOSE_NODE(node, down->bits), down); - rcu_assign_pointer(CHOOSE_NODE(node, newnode->bits), newnode); - if (!parent) - rcu_assign_pointer(*trie, node); - else - rcu_assign_pointer(CHOOSE_NODE(parent, node->bits), - node); - } - return 0; -} - -void wg_allowedips_init(struct allowedips *table) -{ - table->root4 = table->root6 = NULL; - table->seq = 1; -} - -void wg_allowedips_free(struct allowedips *table, struct mutex *lock) -{ - struct allowedips_node __rcu *old4 = table->root4, *old6 = table->root6; - - ++table->seq; - RCU_INIT_POINTER(table->root4, NULL); - RCU_INIT_POINTER(table->root6, NULL); - if (rcu_access_pointer(old4)) { - struct allowedips_node *node = rcu_dereference_protected(old4, - lockdep_is_held(lock)); - - root_remove_peer_lists(node); - call_rcu(&node->rcu, root_free_rcu); - } - if (rcu_access_pointer(old6)) { - struct allowedips_node *node = rcu_dereference_protected(old6, - lockdep_is_held(lock)); - - root_remove_peer_lists(node); - call_rcu(&node->rcu, root_free_rcu); - } -} - -int wg_allowedips_insert_v4(struct allowedips *table, const struct in_addr *ip, - u8 cidr, struct wg_peer *peer, struct mutex *lock) -{ - /* Aligned so it can be passed to fls */ - u8 key[4] __aligned(__alignof(u32)); - - ++table->seq; - swap_endian(key, (const u8 *)ip, 32); - return add(&table->root4, 32, key, cidr, peer, lock); -} - -int wg_allowedips_insert_v6(struct allowedips *table, const struct in6_addr *ip, - u8 cidr, struct wg_peer *peer, struct mutex *lock) -{ - /* Aligned so it can be passed to fls64 */ - u8 key[16] __aligned(__alignof(u64)); - - ++table->seq; - swap_endian(key, (const u8 *)ip, 128); - return add(&table->root6, 128, key, cidr, peer, lock); -} - -void wg_allowedips_remove_by_peer(struct allowedips *table, - struct wg_peer *peer, struct mutex *lock) -{ - ++table->seq; - walk_remove_by_peer(&table->root4, peer, lock); - walk_remove_by_peer(&table->root6, peer, lock); -} - -int wg_allowedips_read_node(struct allowedips_node *node, u8 ip[16], u8 *cidr) -{ - const unsigned int cidr_bytes = DIV_ROUND_UP(node->cidr, 8U); - swap_endian(ip, node->bits, node->bitlen); - memset(ip + cidr_bytes, 0, node->bitlen / 8U - cidr_bytes); - if (node->cidr) - ip[cidr_bytes - 1U] &= ~0U << (-node->cidr % 8U); - - *cidr = node->cidr; - return node->bitlen == 32 ? AF_INET : AF_INET6; -} - -/* Returns a strong reference to a peer */ -struct wg_peer *wg_allowedips_lookup_dst(struct allowedips *table, - struct sk_buff *skb) -{ - if (skb->protocol == htons(ETH_P_IP)) - return lookup(table->root4, 32, &ip_hdr(skb)->daddr); - else if (skb->protocol == htons(ETH_P_IPV6)) - return lookup(table->root6, 128, &ipv6_hdr(skb)->daddr); - return NULL; -} - -/* Returns a strong reference to a peer */ -struct wg_peer *wg_allowedips_lookup_src(struct allowedips *table, - struct sk_buff *skb) -{ - if (skb->protocol == htons(ETH_P_IP)) - return lookup(table->root4, 32, &ip_hdr(skb)->saddr); - else if (skb->protocol == htons(ETH_P_IPV6)) - return lookup(table->root6, 128, &ipv6_hdr(skb)->saddr); - return NULL; -} - -#include "selftest/allowedips.c" diff --git a/drivers/net/wireguard/allowedips.h b/drivers/net/wireguard/allowedips.h deleted file mode 100644 index e5c83cafcef4c..0000000000000 --- a/drivers/net/wireguard/allowedips.h +++ /dev/null @@ -1,59 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#ifndef _WG_ALLOWEDIPS_H -#define _WG_ALLOWEDIPS_H - -#include -#include -#include - -struct wg_peer; - -struct allowedips_node { - struct wg_peer __rcu *peer; - struct allowedips_node __rcu *bit[2]; - /* While it may seem scandalous that we waste space for v4, - * we're alloc'ing to the nearest power of 2 anyway, so this - * doesn't actually make a difference. - */ - u8 bits[16] __aligned(__alignof(u64)); - u8 cidr, bit_at_a, bit_at_b, bitlen; - - /* Keep rarely used list at bottom to be beyond cache line. */ - union { - struct list_head peer_list; - struct rcu_head rcu; - }; -}; - -struct allowedips { - struct allowedips_node __rcu *root4; - struct allowedips_node __rcu *root6; - u64 seq; -}; - -void wg_allowedips_init(struct allowedips *table); -void wg_allowedips_free(struct allowedips *table, struct mutex *mutex); -int wg_allowedips_insert_v4(struct allowedips *table, const struct in_addr *ip, - u8 cidr, struct wg_peer *peer, struct mutex *lock); -int wg_allowedips_insert_v6(struct allowedips *table, const struct in6_addr *ip, - u8 cidr, struct wg_peer *peer, struct mutex *lock); -void wg_allowedips_remove_by_peer(struct allowedips *table, - struct wg_peer *peer, struct mutex *lock); -/* The ip input pointer should be __aligned(__alignof(u64))) */ -int wg_allowedips_read_node(struct allowedips_node *node, u8 ip[16], u8 *cidr); - -/* These return a strong reference to a peer: */ -struct wg_peer *wg_allowedips_lookup_dst(struct allowedips *table, - struct sk_buff *skb); -struct wg_peer *wg_allowedips_lookup_src(struct allowedips *table, - struct sk_buff *skb); - -#ifdef DEBUG -bool wg_allowedips_selftest(void); -#endif - -#endif /* _WG_ALLOWEDIPS_H */ diff --git a/drivers/net/wireguard/cookie.c b/drivers/net/wireguard/cookie.c deleted file mode 100644 index 4956f0499c198..0000000000000 --- a/drivers/net/wireguard/cookie.c +++ /dev/null @@ -1,236 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include "cookie.h" -#include "peer.h" -#include "device.h" -#include "messages.h" -#include "ratelimiter.h" -#include "timers.h" - -#include -#include - -#include -#include - -void wg_cookie_checker_init(struct cookie_checker *checker, - struct wg_device *wg) -{ - init_rwsem(&checker->secret_lock); - checker->secret_birthdate = ktime_get_coarse_boottime_ns(); - get_random_bytes(checker->secret, NOISE_HASH_LEN); - checker->device = wg; -} - -enum { COOKIE_KEY_LABEL_LEN = 8 }; -static const u8 mac1_key_label[COOKIE_KEY_LABEL_LEN] = "mac1----"; -static const u8 cookie_key_label[COOKIE_KEY_LABEL_LEN] = "cookie--"; - -static void precompute_key(u8 key[NOISE_SYMMETRIC_KEY_LEN], - const u8 pubkey[NOISE_PUBLIC_KEY_LEN], - const u8 label[COOKIE_KEY_LABEL_LEN]) -{ - struct blake2s_state blake; - - blake2s_init(&blake, NOISE_SYMMETRIC_KEY_LEN); - blake2s_update(&blake, label, COOKIE_KEY_LABEL_LEN); - blake2s_update(&blake, pubkey, NOISE_PUBLIC_KEY_LEN); - blake2s_final(&blake, key); -} - -/* Must hold peer->handshake.static_identity->lock */ -void wg_cookie_checker_precompute_device_keys(struct cookie_checker *checker) -{ - if (likely(checker->device->static_identity.has_identity)) { - precompute_key(checker->cookie_encryption_key, - checker->device->static_identity.static_public, - cookie_key_label); - precompute_key(checker->message_mac1_key, - checker->device->static_identity.static_public, - mac1_key_label); - } else { - memset(checker->cookie_encryption_key, 0, - NOISE_SYMMETRIC_KEY_LEN); - memset(checker->message_mac1_key, 0, NOISE_SYMMETRIC_KEY_LEN); - } -} - -void wg_cookie_checker_precompute_peer_keys(struct wg_peer *peer) -{ - precompute_key(peer->latest_cookie.cookie_decryption_key, - peer->handshake.remote_static, cookie_key_label); - precompute_key(peer->latest_cookie.message_mac1_key, - peer->handshake.remote_static, mac1_key_label); -} - -void wg_cookie_init(struct cookie *cookie) -{ - memset(cookie, 0, sizeof(*cookie)); - init_rwsem(&cookie->lock); -} - -static void compute_mac1(u8 mac1[COOKIE_LEN], const void *message, size_t len, - const u8 key[NOISE_SYMMETRIC_KEY_LEN]) -{ - len = len - sizeof(struct message_macs) + - offsetof(struct message_macs, mac1); - blake2s(mac1, message, key, COOKIE_LEN, len, NOISE_SYMMETRIC_KEY_LEN); -} - -static void compute_mac2(u8 mac2[COOKIE_LEN], const void *message, size_t len, - const u8 cookie[COOKIE_LEN]) -{ - len = len - sizeof(struct message_macs) + - offsetof(struct message_macs, mac2); - blake2s(mac2, message, cookie, COOKIE_LEN, len, COOKIE_LEN); -} - -static void make_cookie(u8 cookie[COOKIE_LEN], struct sk_buff *skb, - struct cookie_checker *checker) -{ - struct blake2s_state state; - - if (wg_birthdate_has_expired(checker->secret_birthdate, - COOKIE_SECRET_MAX_AGE)) { - down_write(&checker->secret_lock); - checker->secret_birthdate = ktime_get_coarse_boottime_ns(); - get_random_bytes(checker->secret, NOISE_HASH_LEN); - up_write(&checker->secret_lock); - } - - down_read(&checker->secret_lock); - - blake2s_init_key(&state, COOKIE_LEN, checker->secret, NOISE_HASH_LEN); - if (skb->protocol == htons(ETH_P_IP)) - blake2s_update(&state, (u8 *)&ip_hdr(skb)->saddr, - sizeof(struct in_addr)); - else if (skb->protocol == htons(ETH_P_IPV6)) - blake2s_update(&state, (u8 *)&ipv6_hdr(skb)->saddr, - sizeof(struct in6_addr)); - blake2s_update(&state, (u8 *)&udp_hdr(skb)->source, sizeof(__be16)); - blake2s_final(&state, cookie); - - up_read(&checker->secret_lock); -} - -enum cookie_mac_state wg_cookie_validate_packet(struct cookie_checker *checker, - struct sk_buff *skb, - bool check_cookie) -{ - struct message_macs *macs = (struct message_macs *) - (skb->data + skb->len - sizeof(*macs)); - enum cookie_mac_state ret; - u8 computed_mac[COOKIE_LEN]; - u8 cookie[COOKIE_LEN]; - - ret = INVALID_MAC; - compute_mac1(computed_mac, skb->data, skb->len, - checker->message_mac1_key); - if (crypto_memneq(computed_mac, macs->mac1, COOKIE_LEN)) - goto out; - - ret = VALID_MAC_BUT_NO_COOKIE; - - if (!check_cookie) - goto out; - - make_cookie(cookie, skb, checker); - - compute_mac2(computed_mac, skb->data, skb->len, cookie); - if (crypto_memneq(computed_mac, macs->mac2, COOKIE_LEN)) - goto out; - - ret = VALID_MAC_WITH_COOKIE_BUT_RATELIMITED; - if (!wg_ratelimiter_allow(skb, dev_net(checker->device->dev))) - goto out; - - ret = VALID_MAC_WITH_COOKIE; - -out: - return ret; -} - -void wg_cookie_add_mac_to_packet(void *message, size_t len, - struct wg_peer *peer) -{ - struct message_macs *macs = (struct message_macs *) - ((u8 *)message + len - sizeof(*macs)); - - down_write(&peer->latest_cookie.lock); - compute_mac1(macs->mac1, message, len, - peer->latest_cookie.message_mac1_key); - memcpy(peer->latest_cookie.last_mac1_sent, macs->mac1, COOKIE_LEN); - peer->latest_cookie.have_sent_mac1 = true; - up_write(&peer->latest_cookie.lock); - - down_read(&peer->latest_cookie.lock); - if (peer->latest_cookie.is_valid && - !wg_birthdate_has_expired(peer->latest_cookie.birthdate, - COOKIE_SECRET_MAX_AGE - COOKIE_SECRET_LATENCY)) - compute_mac2(macs->mac2, message, len, - peer->latest_cookie.cookie); - else - memset(macs->mac2, 0, COOKIE_LEN); - up_read(&peer->latest_cookie.lock); -} - -void wg_cookie_message_create(struct message_handshake_cookie *dst, - struct sk_buff *skb, __le32 index, - struct cookie_checker *checker) -{ - struct message_macs *macs = (struct message_macs *) - ((u8 *)skb->data + skb->len - sizeof(*macs)); - u8 cookie[COOKIE_LEN]; - - dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE); - dst->receiver_index = index; - get_random_bytes_wait(dst->nonce, COOKIE_NONCE_LEN); - - make_cookie(cookie, skb, checker); - xchacha20poly1305_encrypt(dst->encrypted_cookie, cookie, COOKIE_LEN, - macs->mac1, COOKIE_LEN, dst->nonce, - checker->cookie_encryption_key); -} - -void wg_cookie_message_consume(struct message_handshake_cookie *src, - struct wg_device *wg) -{ - struct wg_peer *peer = NULL; - u8 cookie[COOKIE_LEN]; - bool ret; - - if (unlikely(!wg_index_hashtable_lookup(wg->index_hashtable, - INDEX_HASHTABLE_HANDSHAKE | - INDEX_HASHTABLE_KEYPAIR, - src->receiver_index, &peer))) - return; - - down_read(&peer->latest_cookie.lock); - if (unlikely(!peer->latest_cookie.have_sent_mac1)) { - up_read(&peer->latest_cookie.lock); - goto out; - } - ret = xchacha20poly1305_decrypt( - cookie, src->encrypted_cookie, sizeof(src->encrypted_cookie), - peer->latest_cookie.last_mac1_sent, COOKIE_LEN, src->nonce, - peer->latest_cookie.cookie_decryption_key); - up_read(&peer->latest_cookie.lock); - - if (ret) { - down_write(&peer->latest_cookie.lock); - memcpy(peer->latest_cookie.cookie, cookie, COOKIE_LEN); - peer->latest_cookie.birthdate = ktime_get_coarse_boottime_ns(); - peer->latest_cookie.is_valid = true; - peer->latest_cookie.have_sent_mac1 = false; - up_write(&peer->latest_cookie.lock); - } else { - net_dbg_ratelimited("%s: Could not decrypt invalid cookie response\n", - wg->dev->name); - } - -out: - wg_peer_put(peer); -} diff --git a/drivers/net/wireguard/cookie.h b/drivers/net/wireguard/cookie.h deleted file mode 100644 index c4bd61ca03f24..0000000000000 --- a/drivers/net/wireguard/cookie.h +++ /dev/null @@ -1,59 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#ifndef _WG_COOKIE_H -#define _WG_COOKIE_H - -#include "messages.h" -#include - -struct wg_peer; - -struct cookie_checker { - u8 secret[NOISE_HASH_LEN]; - u8 cookie_encryption_key[NOISE_SYMMETRIC_KEY_LEN]; - u8 message_mac1_key[NOISE_SYMMETRIC_KEY_LEN]; - u64 secret_birthdate; - struct rw_semaphore secret_lock; - struct wg_device *device; -}; - -struct cookie { - u64 birthdate; - bool is_valid; - u8 cookie[COOKIE_LEN]; - bool have_sent_mac1; - u8 last_mac1_sent[COOKIE_LEN]; - u8 cookie_decryption_key[NOISE_SYMMETRIC_KEY_LEN]; - u8 message_mac1_key[NOISE_SYMMETRIC_KEY_LEN]; - struct rw_semaphore lock; -}; - -enum cookie_mac_state { - INVALID_MAC, - VALID_MAC_BUT_NO_COOKIE, - VALID_MAC_WITH_COOKIE_BUT_RATELIMITED, - VALID_MAC_WITH_COOKIE -}; - -void wg_cookie_checker_init(struct cookie_checker *checker, - struct wg_device *wg); -void wg_cookie_checker_precompute_device_keys(struct cookie_checker *checker); -void wg_cookie_checker_precompute_peer_keys(struct wg_peer *peer); -void wg_cookie_init(struct cookie *cookie); - -enum cookie_mac_state wg_cookie_validate_packet(struct cookie_checker *checker, - struct sk_buff *skb, - bool check_cookie); -void wg_cookie_add_mac_to_packet(void *message, size_t len, - struct wg_peer *peer); - -void wg_cookie_message_create(struct message_handshake_cookie *src, - struct sk_buff *skb, __le32 index, - struct cookie_checker *checker); -void wg_cookie_message_consume(struct message_handshake_cookie *src, - struct wg_device *wg); - -#endif /* _WG_COOKIE_H */ diff --git a/drivers/net/wireguard/device.c b/drivers/net/wireguard/device.c deleted file mode 100644 index 3ac3f8570ca1b..0000000000000 --- a/drivers/net/wireguard/device.c +++ /dev/null @@ -1,459 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include "queueing.h" -#include "socket.h" -#include "timers.h" -#include "device.h" -#include "ratelimiter.h" -#include "peer.h" -#include "messages.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static LIST_HEAD(device_list); - -static int wg_open(struct net_device *dev) -{ - struct in_device *dev_v4 = __in_dev_get_rtnl(dev); - struct inet6_dev *dev_v6 = __in6_dev_get(dev); - struct wg_device *wg = netdev_priv(dev); - struct wg_peer *peer; - int ret; - - if (dev_v4) { - /* At some point we might put this check near the ip_rt_send_ - * redirect call of ip_forward in net/ipv4/ip_forward.c, similar - * to the current secpath check. - */ - IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); - IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; - } - if (dev_v6) - dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE; - - ret = wg_socket_init(wg, wg->incoming_port); - if (ret < 0) - return ret; - mutex_lock(&wg->device_update_lock); - list_for_each_entry(peer, &wg->peer_list, peer_list) { - wg_packet_send_staged_packets(peer); - if (peer->persistent_keepalive_interval) - wg_packet_send_keepalive(peer); - } - mutex_unlock(&wg->device_update_lock); - return 0; -} - -#ifdef CONFIG_PM_SLEEP -static int wg_pm_notification(struct notifier_block *nb, unsigned long action, - void *data) -{ - struct wg_device *wg; - struct wg_peer *peer; - - /* If the machine is constantly suspending and resuming, as part of - * its normal operation rather than as a somewhat rare event, then we - * don't actually want to clear keys. - */ - if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) || IS_ENABLED(CONFIG_ANDROID)) - return 0; - - if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE) - return 0; - - rtnl_lock(); - list_for_each_entry(wg, &device_list, device_list) { - mutex_lock(&wg->device_update_lock); - list_for_each_entry(peer, &wg->peer_list, peer_list) { - del_timer(&peer->timer_zero_key_material); - wg_noise_handshake_clear(&peer->handshake); - wg_noise_keypairs_clear(&peer->keypairs); - } - mutex_unlock(&wg->device_update_lock); - } - rtnl_unlock(); - rcu_barrier(); - return 0; -} - -static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification }; -#endif - -static int wg_stop(struct net_device *dev) -{ - struct wg_device *wg = netdev_priv(dev); - struct wg_peer *peer; - - mutex_lock(&wg->device_update_lock); - list_for_each_entry(peer, &wg->peer_list, peer_list) { - wg_packet_purge_staged_packets(peer); - wg_timers_stop(peer); - wg_noise_handshake_clear(&peer->handshake); - wg_noise_keypairs_clear(&peer->keypairs); - wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); - } - mutex_unlock(&wg->device_update_lock); - skb_queue_purge(&wg->incoming_handshakes); - wg_socket_reinit(wg, NULL, NULL); - return 0; -} - -static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev) -{ - struct wg_device *wg = netdev_priv(dev); - struct sk_buff_head packets; - struct wg_peer *peer; - struct sk_buff *next; - sa_family_t family; - u32 mtu; - int ret; - - if (unlikely(!wg_check_packet_protocol(skb))) { - ret = -EPROTONOSUPPORT; - net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name); - goto err; - } - - peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb); - if (unlikely(!peer)) { - ret = -ENOKEY; - if (skb->protocol == htons(ETH_P_IP)) - net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n", - dev->name, &ip_hdr(skb)->daddr); - else if (skb->protocol == htons(ETH_P_IPV6)) - net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n", - dev->name, &ipv6_hdr(skb)->daddr); - goto err; - } - - family = READ_ONCE(peer->endpoint.addr.sa_family); - if (unlikely(family != AF_INET && family != AF_INET6)) { - ret = -EDESTADDRREQ; - net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n", - dev->name, peer->internal_id); - goto err_peer; - } - - mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu; - - __skb_queue_head_init(&packets); - if (!skb_is_gso(skb)) { - skb_mark_not_on_list(skb); - } else { - struct sk_buff *segs = skb_gso_segment(skb, 0); - - if (unlikely(IS_ERR(segs))) { - ret = PTR_ERR(segs); - goto err_peer; - } - dev_kfree_skb(skb); - skb = segs; - } - - skb_list_walk_safe(skb, skb, next) { - skb_mark_not_on_list(skb); - - skb = skb_share_check(skb, GFP_ATOMIC); - if (unlikely(!skb)) - continue; - - /* We only need to keep the original dst around for icmp, - * so at this point we're in a position to drop it. - */ - skb_dst_drop(skb); - - PACKET_CB(skb)->mtu = mtu; - - __skb_queue_tail(&packets, skb); - } - - spin_lock_bh(&peer->staged_packet_queue.lock); - /* If the queue is getting too big, we start removing the oldest packets - * until it's small again. We do this before adding the new packet, so - * we don't remove GSO segments that are in excess. - */ - while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) { - dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue)); - ++dev->stats.tx_dropped; - } - skb_queue_splice_tail(&packets, &peer->staged_packet_queue); - spin_unlock_bh(&peer->staged_packet_queue.lock); - - wg_packet_send_staged_packets(peer); - - wg_peer_put(peer); - return NETDEV_TX_OK; - -err_peer: - wg_peer_put(peer); -err: - ++dev->stats.tx_errors; - if (skb->protocol == htons(ETH_P_IP)) - icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0); - else if (skb->protocol == htons(ETH_P_IPV6)) - icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0); - kfree_skb(skb); - return ret; -} - -static const struct net_device_ops netdev_ops = { - .ndo_open = wg_open, - .ndo_stop = wg_stop, - .ndo_start_xmit = wg_xmit, - .ndo_get_stats64 = ip_tunnel_get_stats64 -}; - -static void wg_destruct(struct net_device *dev) -{ - struct wg_device *wg = netdev_priv(dev); - - rtnl_lock(); - list_del(&wg->device_list); - rtnl_unlock(); - mutex_lock(&wg->device_update_lock); - wg->incoming_port = 0; - wg_socket_reinit(wg, NULL, NULL); - /* The final references are cleared in the below calls to destroy_workqueue. */ - wg_peer_remove_all(wg); - destroy_workqueue(wg->handshake_receive_wq); - destroy_workqueue(wg->handshake_send_wq); - destroy_workqueue(wg->packet_crypt_wq); - wg_packet_queue_free(&wg->decrypt_queue, true); - wg_packet_queue_free(&wg->encrypt_queue, true); - rcu_barrier(); /* Wait for all the peers to be actually freed. */ - wg_ratelimiter_uninit(); - memzero_explicit(&wg->static_identity, sizeof(wg->static_identity)); - skb_queue_purge(&wg->incoming_handshakes); - free_percpu(dev->tstats); - free_percpu(wg->incoming_handshakes_worker); - if (wg->have_creating_net_ref) - put_net(wg->creating_net); - kvfree(wg->index_hashtable); - kvfree(wg->peer_hashtable); - mutex_unlock(&wg->device_update_lock); - - pr_debug("%s: Interface deleted\n", dev->name); - free_netdev(dev); -} - -static const struct device_type device_type = { .name = KBUILD_MODNAME }; - -static void wg_setup(struct net_device *dev) -{ - struct wg_device *wg = netdev_priv(dev); - enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM | - NETIF_F_SG | NETIF_F_GSO | - NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA }; - const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) + - max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); - - dev->netdev_ops = &netdev_ops; - dev->hard_header_len = 0; - dev->addr_len = 0; - dev->needed_headroom = DATA_PACKET_HEAD_ROOM; - dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE); - dev->type = ARPHRD_NONE; - dev->flags = IFF_POINTOPOINT | IFF_NOARP; - dev->priv_flags |= IFF_NO_QUEUE; - dev->features |= NETIF_F_LLTX; - dev->features |= WG_NETDEV_FEATURES; - dev->hw_features |= WG_NETDEV_FEATURES; - dev->hw_enc_features |= WG_NETDEV_FEATURES; - dev->mtu = ETH_DATA_LEN - overhead; - dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead; - - SET_NETDEV_DEVTYPE(dev, &device_type); - - /* We need to keep the dst around in case of icmp replies. */ - netif_keep_dst(dev); - - memset(wg, 0, sizeof(*wg)); - wg->dev = dev; -} - -static int wg_newlink(struct net *src_net, struct net_device *dev, - struct nlattr *tb[], struct nlattr *data[], - struct netlink_ext_ack *extack) -{ - struct wg_device *wg = netdev_priv(dev); - int ret = -ENOMEM; - - wg->creating_net = src_net; - init_rwsem(&wg->static_identity.lock); - mutex_init(&wg->socket_update_lock); - mutex_init(&wg->device_update_lock); - skb_queue_head_init(&wg->incoming_handshakes); - wg_allowedips_init(&wg->peer_allowedips); - wg_cookie_checker_init(&wg->cookie_checker, wg); - INIT_LIST_HEAD(&wg->peer_list); - wg->device_update_gen = 1; - - wg->peer_hashtable = wg_pubkey_hashtable_alloc(); - if (!wg->peer_hashtable) - return ret; - - wg->index_hashtable = wg_index_hashtable_alloc(); - if (!wg->index_hashtable) - goto err_free_peer_hashtable; - - dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); - if (!dev->tstats) - goto err_free_index_hashtable; - - wg->incoming_handshakes_worker = - wg_packet_percpu_multicore_worker_alloc( - wg_packet_handshake_receive_worker, wg); - if (!wg->incoming_handshakes_worker) - goto err_free_tstats; - - wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s", - WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name); - if (!wg->handshake_receive_wq) - goto err_free_incoming_handshakes; - - wg->handshake_send_wq = alloc_workqueue("wg-kex-%s", - WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name); - if (!wg->handshake_send_wq) - goto err_destroy_handshake_receive; - - wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s", - WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name); - if (!wg->packet_crypt_wq) - goto err_destroy_handshake_send; - - ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker, - true, MAX_QUEUED_PACKETS); - if (ret < 0) - goto err_destroy_packet_crypt; - - ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker, - true, MAX_QUEUED_PACKETS); - if (ret < 0) - goto err_free_encrypt_queue; - - ret = wg_ratelimiter_init(); - if (ret < 0) - goto err_free_decrypt_queue; - - ret = register_netdevice(dev); - if (ret < 0) - goto err_uninit_ratelimiter; - - list_add(&wg->device_list, &device_list); - - /* We wait until the end to assign priv_destructor, so that - * register_netdevice doesn't call it for us if it fails. - */ - dev->priv_destructor = wg_destruct; - - pr_debug("%s: Interface created\n", dev->name); - return ret; - -err_uninit_ratelimiter: - wg_ratelimiter_uninit(); -err_free_decrypt_queue: - wg_packet_queue_free(&wg->decrypt_queue, true); -err_free_encrypt_queue: - wg_packet_queue_free(&wg->encrypt_queue, true); -err_destroy_packet_crypt: - destroy_workqueue(wg->packet_crypt_wq); -err_destroy_handshake_send: - destroy_workqueue(wg->handshake_send_wq); -err_destroy_handshake_receive: - destroy_workqueue(wg->handshake_receive_wq); -err_free_incoming_handshakes: - free_percpu(wg->incoming_handshakes_worker); -err_free_tstats: - free_percpu(dev->tstats); -err_free_index_hashtable: - kvfree(wg->index_hashtable); -err_free_peer_hashtable: - kvfree(wg->peer_hashtable); - return ret; -} - -static struct rtnl_link_ops link_ops __read_mostly = { - .kind = KBUILD_MODNAME, - .priv_size = sizeof(struct wg_device), - .setup = wg_setup, - .newlink = wg_newlink, -}; - -static int wg_netdevice_notification(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct net_device *dev = ((struct netdev_notifier_info *)data)->dev; - struct wg_device *wg = netdev_priv(dev); - - ASSERT_RTNL(); - - if (action != NETDEV_REGISTER || dev->netdev_ops != &netdev_ops) - return 0; - - if (dev_net(dev) == wg->creating_net && wg->have_creating_net_ref) { - put_net(wg->creating_net); - wg->have_creating_net_ref = false; - } else if (dev_net(dev) != wg->creating_net && - !wg->have_creating_net_ref) { - wg->have_creating_net_ref = true; - get_net(wg->creating_net); - } - return 0; -} - -static struct notifier_block netdevice_notifier = { - .notifier_call = wg_netdevice_notification -}; - -int __init wg_device_init(void) -{ - int ret; - -#ifdef CONFIG_PM_SLEEP - ret = register_pm_notifier(&pm_notifier); - if (ret) - return ret; -#endif - - ret = register_netdevice_notifier(&netdevice_notifier); - if (ret) - goto error_pm; - - ret = rtnl_link_register(&link_ops); - if (ret) - goto error_netdevice; - - return 0; - -error_netdevice: - unregister_netdevice_notifier(&netdevice_notifier); -error_pm: -#ifdef CONFIG_PM_SLEEP - unregister_pm_notifier(&pm_notifier); -#endif - return ret; -} - -void wg_device_uninit(void) -{ - rtnl_link_unregister(&link_ops); - unregister_netdevice_notifier(&netdevice_notifier); -#ifdef CONFIG_PM_SLEEP - unregister_pm_notifier(&pm_notifier); -#endif - rcu_barrier(); -} diff --git a/drivers/net/wireguard/device.h b/drivers/net/wireguard/device.h deleted file mode 100644 index b15a8be9d8169..0000000000000 --- a/drivers/net/wireguard/device.h +++ /dev/null @@ -1,65 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#ifndef _WG_DEVICE_H -#define _WG_DEVICE_H - -#include "noise.h" -#include "allowedips.h" -#include "peerlookup.h" -#include "cookie.h" - -#include -#include -#include -#include -#include -#include - -struct wg_device; - -struct multicore_worker { - void *ptr; - struct work_struct work; -}; - -struct crypt_queue { - struct ptr_ring ring; - union { - struct { - struct multicore_worker __percpu *worker; - int last_cpu; - }; - struct work_struct work; - }; -}; - -struct wg_device { - struct net_device *dev; - struct crypt_queue encrypt_queue, decrypt_queue; - struct sock __rcu *sock4, *sock6; - struct net *creating_net; - struct noise_static_identity static_identity; - struct workqueue_struct *handshake_receive_wq, *handshake_send_wq; - struct workqueue_struct *packet_crypt_wq; - struct sk_buff_head incoming_handshakes; - int incoming_handshake_cpu; - struct multicore_worker __percpu *incoming_handshakes_worker; - struct cookie_checker cookie_checker; - struct pubkey_hashtable *peer_hashtable; - struct index_hashtable *index_hashtable; - struct allowedips peer_allowedips; - struct mutex device_update_lock, socket_update_lock; - struct list_head device_list, peer_list; - unsigned int num_peers, device_update_gen; - u32 fwmark; - u16 incoming_port; - bool have_creating_net_ref; -}; - -int wg_device_init(void); -void wg_device_uninit(void); - -#endif /* _WG_DEVICE_H */ diff --git a/drivers/net/wireguard/main.c b/drivers/net/wireguard/main.c deleted file mode 100644 index 7a7d5f1a80fc7..0000000000000 --- a/drivers/net/wireguard/main.c +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include "version.h" -#include "device.h" -#include "noise.h" -#include "queueing.h" -#include "ratelimiter.h" -#include "netlink.h" - -#include - -#include -#include -#include -#include - -static int __init mod_init(void) -{ - int ret; - -#ifdef DEBUG - if (!wg_allowedips_selftest() || !wg_packet_counter_selftest() || - !wg_ratelimiter_selftest()) - return -ENOTRECOVERABLE; -#endif - wg_noise_init(); - - ret = wg_device_init(); - if (ret < 0) - goto err_device; - - ret = wg_genetlink_init(); - if (ret < 0) - goto err_netlink; - - pr_info("WireGuard " WIREGUARD_VERSION " loaded. See www.wireguard.com for information.\n"); - pr_info("Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved.\n"); - - return 0; - -err_netlink: - wg_device_uninit(); -err_device: - return ret; -} - -static void __exit mod_exit(void) -{ - wg_genetlink_uninit(); - wg_device_uninit(); -} - -module_init(mod_init); -module_exit(mod_exit); -MODULE_LICENSE("GPL v2"); -MODULE_DESCRIPTION("WireGuard secure network tunnel"); -MODULE_AUTHOR("Jason A. Donenfeld "); -MODULE_VERSION(WIREGUARD_VERSION); -MODULE_ALIAS_RTNL_LINK(KBUILD_MODNAME); -MODULE_ALIAS_GENL_FAMILY(WG_GENL_NAME); diff --git a/drivers/net/wireguard/messages.h b/drivers/net/wireguard/messages.h deleted file mode 100644 index b8a7b9ce32ba7..0000000000000 --- a/drivers/net/wireguard/messages.h +++ /dev/null @@ -1,128 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#ifndef _WG_MESSAGES_H -#define _WG_MESSAGES_H - -#include -#include -#include - -#include -#include -#include - -enum noise_lengths { - NOISE_PUBLIC_KEY_LEN = CURVE25519_KEY_SIZE, - NOISE_SYMMETRIC_KEY_LEN = CHACHA20POLY1305_KEY_SIZE, - NOISE_TIMESTAMP_LEN = sizeof(u64) + sizeof(u32), - NOISE_AUTHTAG_LEN = CHACHA20POLY1305_AUTHTAG_SIZE, - NOISE_HASH_LEN = BLAKE2S_HASH_SIZE -}; - -#define noise_encrypted_len(plain_len) ((plain_len) + NOISE_AUTHTAG_LEN) - -enum cookie_values { - COOKIE_SECRET_MAX_AGE = 2 * 60, - COOKIE_SECRET_LATENCY = 5, - COOKIE_NONCE_LEN = XCHACHA20POLY1305_NONCE_SIZE, - COOKIE_LEN = 16 -}; - -enum counter_values { - COUNTER_BITS_TOTAL = 2048, - COUNTER_REDUNDANT_BITS = BITS_PER_LONG, - COUNTER_WINDOW_SIZE = COUNTER_BITS_TOTAL - COUNTER_REDUNDANT_BITS -}; - -enum limits { - REKEY_AFTER_MESSAGES = 1ULL << 60, - REJECT_AFTER_MESSAGES = U64_MAX - COUNTER_WINDOW_SIZE - 1, - REKEY_TIMEOUT = 5, - REKEY_TIMEOUT_JITTER_MAX_JIFFIES = HZ / 3, - REKEY_AFTER_TIME = 120, - REJECT_AFTER_TIME = 180, - INITIATIONS_PER_SECOND = 50, - MAX_PEERS_PER_DEVICE = 1U << 20, - KEEPALIVE_TIMEOUT = 10, - MAX_TIMER_HANDSHAKES = 90 / REKEY_TIMEOUT, - MAX_QUEUED_INCOMING_HANDSHAKES = 4096, /* TODO: replace this with DQL */ - MAX_STAGED_PACKETS = 128, - MAX_QUEUED_PACKETS = 1024 /* TODO: replace this with DQL */ -}; - -enum message_type { - MESSAGE_INVALID = 0, - MESSAGE_HANDSHAKE_INITIATION = 1, - MESSAGE_HANDSHAKE_RESPONSE = 2, - MESSAGE_HANDSHAKE_COOKIE = 3, - MESSAGE_DATA = 4 -}; - -struct message_header { - /* The actual layout of this that we want is: - * u8 type - * u8 reserved_zero[3] - * - * But it turns out that by encoding this as little endian, - * we achieve the same thing, and it makes checking faster. - */ - __le32 type; -}; - -struct message_macs { - u8 mac1[COOKIE_LEN]; - u8 mac2[COOKIE_LEN]; -}; - -struct message_handshake_initiation { - struct message_header header; - __le32 sender_index; - u8 unencrypted_ephemeral[NOISE_PUBLIC_KEY_LEN]; - u8 encrypted_static[noise_encrypted_len(NOISE_PUBLIC_KEY_LEN)]; - u8 encrypted_timestamp[noise_encrypted_len(NOISE_TIMESTAMP_LEN)]; - struct message_macs macs; -}; - -struct message_handshake_response { - struct message_header header; - __le32 sender_index; - __le32 receiver_index; - u8 unencrypted_ephemeral[NOISE_PUBLIC_KEY_LEN]; - u8 encrypted_nothing[noise_encrypted_len(0)]; - struct message_macs macs; -}; - -struct message_handshake_cookie { - struct message_header header; - __le32 receiver_index; - u8 nonce[COOKIE_NONCE_LEN]; - u8 encrypted_cookie[noise_encrypted_len(COOKIE_LEN)]; -}; - -struct message_data { - struct message_header header; - __le32 key_idx; - __le64 counter; - u8 encrypted_data[]; -}; - -#define message_data_len(plain_len) \ - (noise_encrypted_len(plain_len) + sizeof(struct message_data)) - -enum message_alignments { - MESSAGE_PADDING_MULTIPLE = 16, - MESSAGE_MINIMUM_LENGTH = message_data_len(0) -}; - -#define SKB_HEADER_LEN \ - (max(sizeof(struct iphdr), sizeof(struct ipv6hdr)) + \ - sizeof(struct udphdr) + NET_SKB_PAD) -#define DATA_PACKET_HEAD_ROOM \ - ALIGN(sizeof(struct message_data) + SKB_HEADER_LEN, 4) - -enum { HANDSHAKE_DSCP = 0x88 /* AF41, plus 00 ECN */ }; - -#endif /* _WG_MESSAGES_H */ diff --git a/drivers/net/wireguard/netlink.c b/drivers/net/wireguard/netlink.c deleted file mode 100644 index 802099c8828a6..0000000000000 --- a/drivers/net/wireguard/netlink.c +++ /dev/null @@ -1,636 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include "netlink.h" -#include "device.h" -#include "peer.h" -#include "socket.h" -#include "queueing.h" -#include "messages.h" - -#include - -#include -#include -#include -#include - -static struct genl_family genl_family; - -static const struct nla_policy device_policy[WGDEVICE_A_MAX + 1] = { - [WGDEVICE_A_IFINDEX] = { .type = NLA_U32 }, - [WGDEVICE_A_IFNAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 }, - [WGDEVICE_A_PRIVATE_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, - [WGDEVICE_A_PUBLIC_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, - [WGDEVICE_A_FLAGS] = { .type = NLA_U32 }, - [WGDEVICE_A_LISTEN_PORT] = { .type = NLA_U16 }, - [WGDEVICE_A_FWMARK] = { .type = NLA_U32 }, - [WGDEVICE_A_PEERS] = { .type = NLA_NESTED } -}; - -static const struct nla_policy peer_policy[WGPEER_A_MAX + 1] = { - [WGPEER_A_PUBLIC_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, - [WGPEER_A_PRESHARED_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_SYMMETRIC_KEY_LEN }, - [WGPEER_A_FLAGS] = { .type = NLA_U32 }, - [WGPEER_A_ENDPOINT] = { .type = NLA_MIN_LEN, .len = sizeof(struct sockaddr) }, - [WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL] = { .type = NLA_U16 }, - [WGPEER_A_LAST_HANDSHAKE_TIME] = { .type = NLA_EXACT_LEN, .len = sizeof(struct __kernel_timespec) }, - [WGPEER_A_RX_BYTES] = { .type = NLA_U64 }, - [WGPEER_A_TX_BYTES] = { .type = NLA_U64 }, - [WGPEER_A_ALLOWEDIPS] = { .type = NLA_NESTED }, - [WGPEER_A_PROTOCOL_VERSION] = { .type = NLA_U32 } -}; - -static const struct nla_policy allowedip_policy[WGALLOWEDIP_A_MAX + 1] = { - [WGALLOWEDIP_A_FAMILY] = { .type = NLA_U16 }, - [WGALLOWEDIP_A_IPADDR] = { .type = NLA_MIN_LEN, .len = sizeof(struct in_addr) }, - [WGALLOWEDIP_A_CIDR_MASK] = { .type = NLA_U8 } -}; - -static struct wg_device *lookup_interface(struct nlattr **attrs, - struct sk_buff *skb) -{ - struct net_device *dev = NULL; - - if (!attrs[WGDEVICE_A_IFINDEX] == !attrs[WGDEVICE_A_IFNAME]) - return ERR_PTR(-EBADR); - if (attrs[WGDEVICE_A_IFINDEX]) - dev = dev_get_by_index(sock_net(skb->sk), - nla_get_u32(attrs[WGDEVICE_A_IFINDEX])); - else if (attrs[WGDEVICE_A_IFNAME]) - dev = dev_get_by_name(sock_net(skb->sk), - nla_data(attrs[WGDEVICE_A_IFNAME])); - if (!dev) - return ERR_PTR(-ENODEV); - if (!dev->rtnl_link_ops || !dev->rtnl_link_ops->kind || - strcmp(dev->rtnl_link_ops->kind, KBUILD_MODNAME)) { - dev_put(dev); - return ERR_PTR(-EOPNOTSUPP); - } - return netdev_priv(dev); -} - -static int get_allowedips(struct sk_buff *skb, const u8 *ip, u8 cidr, - int family) -{ - struct nlattr *allowedip_nest; - - allowedip_nest = nla_nest_start(skb, 0); - if (!allowedip_nest) - return -EMSGSIZE; - - if (nla_put_u8(skb, WGALLOWEDIP_A_CIDR_MASK, cidr) || - nla_put_u16(skb, WGALLOWEDIP_A_FAMILY, family) || - nla_put(skb, WGALLOWEDIP_A_IPADDR, family == AF_INET6 ? - sizeof(struct in6_addr) : sizeof(struct in_addr), ip)) { - nla_nest_cancel(skb, allowedip_nest); - return -EMSGSIZE; - } - - nla_nest_end(skb, allowedip_nest); - return 0; -} - -struct dump_ctx { - struct wg_device *wg; - struct wg_peer *next_peer; - u64 allowedips_seq; - struct allowedips_node *next_allowedip; -}; - -#define DUMP_CTX(cb) ((struct dump_ctx *)(cb)->args) - -static int -get_peer(struct wg_peer *peer, struct sk_buff *skb, struct dump_ctx *ctx) -{ - - struct nlattr *allowedips_nest, *peer_nest = nla_nest_start(skb, 0); - struct allowedips_node *allowedips_node = ctx->next_allowedip; - bool fail; - - if (!peer_nest) - return -EMSGSIZE; - - down_read(&peer->handshake.lock); - fail = nla_put(skb, WGPEER_A_PUBLIC_KEY, NOISE_PUBLIC_KEY_LEN, - peer->handshake.remote_static); - up_read(&peer->handshake.lock); - if (fail) - goto err; - - if (!allowedips_node) { - const struct __kernel_timespec last_handshake = { - .tv_sec = peer->walltime_last_handshake.tv_sec, - .tv_nsec = peer->walltime_last_handshake.tv_nsec - }; - - down_read(&peer->handshake.lock); - fail = nla_put(skb, WGPEER_A_PRESHARED_KEY, - NOISE_SYMMETRIC_KEY_LEN, - peer->handshake.preshared_key); - up_read(&peer->handshake.lock); - if (fail) - goto err; - - if (nla_put(skb, WGPEER_A_LAST_HANDSHAKE_TIME, - sizeof(last_handshake), &last_handshake) || - nla_put_u16(skb, WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL, - peer->persistent_keepalive_interval) || - nla_put_u64_64bit(skb, WGPEER_A_TX_BYTES, peer->tx_bytes, - WGPEER_A_UNSPEC) || - nla_put_u64_64bit(skb, WGPEER_A_RX_BYTES, peer->rx_bytes, - WGPEER_A_UNSPEC) || - nla_put_u32(skb, WGPEER_A_PROTOCOL_VERSION, 1)) - goto err; - - read_lock_bh(&peer->endpoint_lock); - if (peer->endpoint.addr.sa_family == AF_INET) - fail = nla_put(skb, WGPEER_A_ENDPOINT, - sizeof(peer->endpoint.addr4), - &peer->endpoint.addr4); - else if (peer->endpoint.addr.sa_family == AF_INET6) - fail = nla_put(skb, WGPEER_A_ENDPOINT, - sizeof(peer->endpoint.addr6), - &peer->endpoint.addr6); - read_unlock_bh(&peer->endpoint_lock); - if (fail) - goto err; - allowedips_node = - list_first_entry_or_null(&peer->allowedips_list, - struct allowedips_node, peer_list); - } - if (!allowedips_node) - goto no_allowedips; - if (!ctx->allowedips_seq) - ctx->allowedips_seq = peer->device->peer_allowedips.seq; - else if (ctx->allowedips_seq != peer->device->peer_allowedips.seq) - goto no_allowedips; - - allowedips_nest = nla_nest_start(skb, WGPEER_A_ALLOWEDIPS); - if (!allowedips_nest) - goto err; - - list_for_each_entry_from(allowedips_node, &peer->allowedips_list, - peer_list) { - u8 cidr, ip[16] __aligned(__alignof(u64)); - int family; - - family = wg_allowedips_read_node(allowedips_node, ip, &cidr); - if (get_allowedips(skb, ip, cidr, family)) { - nla_nest_end(skb, allowedips_nest); - nla_nest_end(skb, peer_nest); - ctx->next_allowedip = allowedips_node; - return -EMSGSIZE; - } - } - nla_nest_end(skb, allowedips_nest); -no_allowedips: - nla_nest_end(skb, peer_nest); - ctx->next_allowedip = NULL; - ctx->allowedips_seq = 0; - return 0; -err: - nla_nest_cancel(skb, peer_nest); - return -EMSGSIZE; -} - -static int wg_get_device_start(struct netlink_callback *cb) -{ - struct wg_device *wg; - - wg = lookup_interface(genl_dumpit_info(cb)->attrs, cb->skb); - if (IS_ERR(wg)) - return PTR_ERR(wg); - DUMP_CTX(cb)->wg = wg; - return 0; -} - -static int wg_get_device_dump(struct sk_buff *skb, struct netlink_callback *cb) -{ - struct wg_peer *peer, *next_peer_cursor; - struct dump_ctx *ctx = DUMP_CTX(cb); - struct wg_device *wg = ctx->wg; - struct nlattr *peers_nest; - int ret = -EMSGSIZE; - bool done = true; - void *hdr; - - rtnl_lock(); - mutex_lock(&wg->device_update_lock); - cb->seq = wg->device_update_gen; - next_peer_cursor = ctx->next_peer; - - hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, - &genl_family, NLM_F_MULTI, WG_CMD_GET_DEVICE); - if (!hdr) - goto out; - genl_dump_check_consistent(cb, hdr); - - if (!ctx->next_peer) { - if (nla_put_u16(skb, WGDEVICE_A_LISTEN_PORT, - wg->incoming_port) || - nla_put_u32(skb, WGDEVICE_A_FWMARK, wg->fwmark) || - nla_put_u32(skb, WGDEVICE_A_IFINDEX, wg->dev->ifindex) || - nla_put_string(skb, WGDEVICE_A_IFNAME, wg->dev->name)) - goto out; - - down_read(&wg->static_identity.lock); - if (wg->static_identity.has_identity) { - if (nla_put(skb, WGDEVICE_A_PRIVATE_KEY, - NOISE_PUBLIC_KEY_LEN, - wg->static_identity.static_private) || - nla_put(skb, WGDEVICE_A_PUBLIC_KEY, - NOISE_PUBLIC_KEY_LEN, - wg->static_identity.static_public)) { - up_read(&wg->static_identity.lock); - goto out; - } - } - up_read(&wg->static_identity.lock); - } - - peers_nest = nla_nest_start(skb, WGDEVICE_A_PEERS); - if (!peers_nest) - goto out; - ret = 0; - /* If the last cursor was removed via list_del_init in peer_remove, then - * we just treat this the same as there being no more peers left. The - * reason is that seq_nr should indicate to userspace that this isn't a - * coherent dump anyway, so they'll try again. - */ - if (list_empty(&wg->peer_list) || - (ctx->next_peer && list_empty(&ctx->next_peer->peer_list))) { - nla_nest_cancel(skb, peers_nest); - goto out; - } - lockdep_assert_held(&wg->device_update_lock); - peer = list_prepare_entry(ctx->next_peer, &wg->peer_list, peer_list); - list_for_each_entry_continue(peer, &wg->peer_list, peer_list) { - if (get_peer(peer, skb, ctx)) { - done = false; - break; - } - next_peer_cursor = peer; - } - nla_nest_end(skb, peers_nest); - -out: - if (!ret && !done && next_peer_cursor) - wg_peer_get(next_peer_cursor); - wg_peer_put(ctx->next_peer); - mutex_unlock(&wg->device_update_lock); - rtnl_unlock(); - - if (ret) { - genlmsg_cancel(skb, hdr); - return ret; - } - genlmsg_end(skb, hdr); - if (done) { - ctx->next_peer = NULL; - return 0; - } - ctx->next_peer = next_peer_cursor; - return skb->len; - - /* At this point, we can't really deal ourselves with safely zeroing out - * the private key material after usage. This will need an additional API - * in the kernel for marking skbs as zero_on_free. - */ -} - -static int wg_get_device_done(struct netlink_callback *cb) -{ - struct dump_ctx *ctx = DUMP_CTX(cb); - - if (ctx->wg) - dev_put(ctx->wg->dev); - wg_peer_put(ctx->next_peer); - return 0; -} - -static int set_port(struct wg_device *wg, u16 port) -{ - struct wg_peer *peer; - - if (wg->incoming_port == port) - return 0; - list_for_each_entry(peer, &wg->peer_list, peer_list) - wg_socket_clear_peer_endpoint_src(peer); - if (!netif_running(wg->dev)) { - wg->incoming_port = port; - return 0; - } - return wg_socket_init(wg, port); -} - -static int set_allowedip(struct wg_peer *peer, struct nlattr **attrs) -{ - int ret = -EINVAL; - u16 family; - u8 cidr; - - if (!attrs[WGALLOWEDIP_A_FAMILY] || !attrs[WGALLOWEDIP_A_IPADDR] || - !attrs[WGALLOWEDIP_A_CIDR_MASK]) - return ret; - family = nla_get_u16(attrs[WGALLOWEDIP_A_FAMILY]); - cidr = nla_get_u8(attrs[WGALLOWEDIP_A_CIDR_MASK]); - - if (family == AF_INET && cidr <= 32 && - nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in_addr)) - ret = wg_allowedips_insert_v4( - &peer->device->peer_allowedips, - nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, - &peer->device->device_update_lock); - else if (family == AF_INET6 && cidr <= 128 && - nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in6_addr)) - ret = wg_allowedips_insert_v6( - &peer->device->peer_allowedips, - nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, - &peer->device->device_update_lock); - - return ret; -} - -static int set_peer(struct wg_device *wg, struct nlattr **attrs) -{ - u8 *public_key = NULL, *preshared_key = NULL; - struct wg_peer *peer = NULL; - u32 flags = 0; - int ret; - - ret = -EINVAL; - if (attrs[WGPEER_A_PUBLIC_KEY] && - nla_len(attrs[WGPEER_A_PUBLIC_KEY]) == NOISE_PUBLIC_KEY_LEN) - public_key = nla_data(attrs[WGPEER_A_PUBLIC_KEY]); - else - goto out; - if (attrs[WGPEER_A_PRESHARED_KEY] && - nla_len(attrs[WGPEER_A_PRESHARED_KEY]) == NOISE_SYMMETRIC_KEY_LEN) - preshared_key = nla_data(attrs[WGPEER_A_PRESHARED_KEY]); - - if (attrs[WGPEER_A_FLAGS]) - flags = nla_get_u32(attrs[WGPEER_A_FLAGS]); - ret = -EOPNOTSUPP; - if (flags & ~__WGPEER_F_ALL) - goto out; - - ret = -EPFNOSUPPORT; - if (attrs[WGPEER_A_PROTOCOL_VERSION]) { - if (nla_get_u32(attrs[WGPEER_A_PROTOCOL_VERSION]) != 1) - goto out; - } - - peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, - nla_data(attrs[WGPEER_A_PUBLIC_KEY])); - ret = 0; - if (!peer) { /* Peer doesn't exist yet. Add a new one. */ - if (flags & (WGPEER_F_REMOVE_ME | WGPEER_F_UPDATE_ONLY)) - goto out; - - /* The peer is new, so there aren't allowed IPs to remove. */ - flags &= ~WGPEER_F_REPLACE_ALLOWEDIPS; - - down_read(&wg->static_identity.lock); - if (wg->static_identity.has_identity && - !memcmp(nla_data(attrs[WGPEER_A_PUBLIC_KEY]), - wg->static_identity.static_public, - NOISE_PUBLIC_KEY_LEN)) { - /* We silently ignore peers that have the same public - * key as the device. The reason we do it silently is - * that we'd like for people to be able to reuse the - * same set of API calls across peers. - */ - up_read(&wg->static_identity.lock); - ret = 0; - goto out; - } - up_read(&wg->static_identity.lock); - - peer = wg_peer_create(wg, public_key, preshared_key); - if (IS_ERR(peer)) { - ret = PTR_ERR(peer); - peer = NULL; - goto out; - } - /* Take additional reference, as though we've just been - * looked up. - */ - wg_peer_get(peer); - } - - if (flags & WGPEER_F_REMOVE_ME) { - wg_peer_remove(peer); - goto out; - } - - if (preshared_key) { - down_write(&peer->handshake.lock); - memcpy(&peer->handshake.preshared_key, preshared_key, - NOISE_SYMMETRIC_KEY_LEN); - up_write(&peer->handshake.lock); - } - - if (attrs[WGPEER_A_ENDPOINT]) { - struct sockaddr *addr = nla_data(attrs[WGPEER_A_ENDPOINT]); - size_t len = nla_len(attrs[WGPEER_A_ENDPOINT]); - - if ((len == sizeof(struct sockaddr_in) && - addr->sa_family == AF_INET) || - (len == sizeof(struct sockaddr_in6) && - addr->sa_family == AF_INET6)) { - struct endpoint endpoint = { { { 0 } } }; - - memcpy(&endpoint.addr, addr, len); - wg_socket_set_peer_endpoint(peer, &endpoint); - } - } - - if (flags & WGPEER_F_REPLACE_ALLOWEDIPS) - wg_allowedips_remove_by_peer(&wg->peer_allowedips, peer, - &wg->device_update_lock); - - if (attrs[WGPEER_A_ALLOWEDIPS]) { - struct nlattr *attr, *allowedip[WGALLOWEDIP_A_MAX + 1]; - int rem; - - nla_for_each_nested(attr, attrs[WGPEER_A_ALLOWEDIPS], rem) { - ret = nla_parse_nested(allowedip, WGALLOWEDIP_A_MAX, - attr, allowedip_policy, NULL); - if (ret < 0) - goto out; - ret = set_allowedip(peer, allowedip); - if (ret < 0) - goto out; - } - } - - if (attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]) { - const u16 persistent_keepalive_interval = nla_get_u16( - attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]); - const bool send_keepalive = - !peer->persistent_keepalive_interval && - persistent_keepalive_interval && - netif_running(wg->dev); - - peer->persistent_keepalive_interval = persistent_keepalive_interval; - if (send_keepalive) - wg_packet_send_keepalive(peer); - } - - if (netif_running(wg->dev)) - wg_packet_send_staged_packets(peer); - -out: - wg_peer_put(peer); - if (attrs[WGPEER_A_PRESHARED_KEY]) - memzero_explicit(nla_data(attrs[WGPEER_A_PRESHARED_KEY]), - nla_len(attrs[WGPEER_A_PRESHARED_KEY])); - return ret; -} - -static int wg_set_device(struct sk_buff *skb, struct genl_info *info) -{ - struct wg_device *wg = lookup_interface(info->attrs, skb); - u32 flags = 0; - int ret; - - if (IS_ERR(wg)) { - ret = PTR_ERR(wg); - goto out_nodev; - } - - rtnl_lock(); - mutex_lock(&wg->device_update_lock); - - if (info->attrs[WGDEVICE_A_FLAGS]) - flags = nla_get_u32(info->attrs[WGDEVICE_A_FLAGS]); - ret = -EOPNOTSUPP; - if (flags & ~__WGDEVICE_F_ALL) - goto out; - - ret = -EPERM; - if ((info->attrs[WGDEVICE_A_LISTEN_PORT] || - info->attrs[WGDEVICE_A_FWMARK]) && - !ns_capable(wg->creating_net->user_ns, CAP_NET_ADMIN)) - goto out; - - ++wg->device_update_gen; - - if (info->attrs[WGDEVICE_A_FWMARK]) { - struct wg_peer *peer; - - wg->fwmark = nla_get_u32(info->attrs[WGDEVICE_A_FWMARK]); - list_for_each_entry(peer, &wg->peer_list, peer_list) - wg_socket_clear_peer_endpoint_src(peer); - } - - if (info->attrs[WGDEVICE_A_LISTEN_PORT]) { - ret = set_port(wg, - nla_get_u16(info->attrs[WGDEVICE_A_LISTEN_PORT])); - if (ret) - goto out; - } - - if (flags & WGDEVICE_F_REPLACE_PEERS) - wg_peer_remove_all(wg); - - if (info->attrs[WGDEVICE_A_PRIVATE_KEY] && - nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY]) == - NOISE_PUBLIC_KEY_LEN) { - u8 *private_key = nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]); - u8 public_key[NOISE_PUBLIC_KEY_LEN]; - struct wg_peer *peer, *temp; - - if (!crypto_memneq(wg->static_identity.static_private, - private_key, NOISE_PUBLIC_KEY_LEN)) - goto skip_set_private_key; - - /* We remove before setting, to prevent race, which means doing - * two 25519-genpub ops. - */ - if (curve25519_generate_public(public_key, private_key)) { - peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, - public_key); - if (peer) { - wg_peer_put(peer); - wg_peer_remove(peer); - } - } - - down_write(&wg->static_identity.lock); - wg_noise_set_static_identity_private_key(&wg->static_identity, - private_key); - list_for_each_entry_safe(peer, temp, &wg->peer_list, - peer_list) { - wg_noise_precompute_static_static(peer); - wg_noise_expire_current_peer_keypairs(peer); - } - wg_cookie_checker_precompute_device_keys(&wg->cookie_checker); - up_write(&wg->static_identity.lock); - } -skip_set_private_key: - - if (info->attrs[WGDEVICE_A_PEERS]) { - struct nlattr *attr, *peer[WGPEER_A_MAX + 1]; - int rem; - - nla_for_each_nested(attr, info->attrs[WGDEVICE_A_PEERS], rem) { - ret = nla_parse_nested(peer, WGPEER_A_MAX, attr, - peer_policy, NULL); - if (ret < 0) - goto out; - ret = set_peer(wg, peer); - if (ret < 0) - goto out; - } - } - ret = 0; - -out: - mutex_unlock(&wg->device_update_lock); - rtnl_unlock(); - dev_put(wg->dev); -out_nodev: - if (info->attrs[WGDEVICE_A_PRIVATE_KEY]) - memzero_explicit(nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]), - nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY])); - return ret; -} - -static const struct genl_ops genl_ops[] = { - { - .cmd = WG_CMD_GET_DEVICE, - .start = wg_get_device_start, - .dumpit = wg_get_device_dump, - .done = wg_get_device_done, - .flags = GENL_UNS_ADMIN_PERM - }, { - .cmd = WG_CMD_SET_DEVICE, - .doit = wg_set_device, - .flags = GENL_UNS_ADMIN_PERM - } -}; - -static struct genl_family genl_family __ro_after_init = { - .ops = genl_ops, - .n_ops = ARRAY_SIZE(genl_ops), - .name = WG_GENL_NAME, - .version = WG_GENL_VERSION, - .maxattr = WGDEVICE_A_MAX, - .module = THIS_MODULE, - .policy = device_policy, - .netnsok = true -}; - -int __init wg_genetlink_init(void) -{ - return genl_register_family(&genl_family); -} - -void __exit wg_genetlink_uninit(void) -{ - genl_unregister_family(&genl_family); -} diff --git a/drivers/net/wireguard/netlink.h b/drivers/net/wireguard/netlink.h deleted file mode 100644 index 15100d92e2e32..0000000000000 --- a/drivers/net/wireguard/netlink.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#ifndef _WG_NETLINK_H -#define _WG_NETLINK_H - -int wg_genetlink_init(void); -void wg_genetlink_uninit(void); - -#endif /* _WG_NETLINK_H */ diff --git a/drivers/net/wireguard/noise.c b/drivers/net/wireguard/noise.c deleted file mode 100644 index 708dc61c974f7..0000000000000 --- a/drivers/net/wireguard/noise.c +++ /dev/null @@ -1,835 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include "noise.h" -#include "device.h" -#include "peer.h" -#include "messages.h" -#include "queueing.h" -#include "peerlookup.h" - -#include -#include -#include -#include -#include -#include - -/* This implements Noise_IKpsk2: - * - * <- s - * ****** - * -> e, es, s, ss, {t} - * <- e, ee, se, psk, {} - */ - -static const u8 handshake_name[37] = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; -static const u8 identifier_name[34] = "WireGuard v1 zx2c4 Jason@zx2c4.com"; -static u8 handshake_init_hash[NOISE_HASH_LEN] __ro_after_init; -static u8 handshake_init_chaining_key[NOISE_HASH_LEN] __ro_after_init; -static atomic64_t keypair_counter = ATOMIC64_INIT(0); - -void __init wg_noise_init(void) -{ - struct blake2s_state blake; - - blake2s(handshake_init_chaining_key, handshake_name, NULL, - NOISE_HASH_LEN, sizeof(handshake_name), 0); - blake2s_init(&blake, NOISE_HASH_LEN); - blake2s_update(&blake, handshake_init_chaining_key, NOISE_HASH_LEN); - blake2s_update(&blake, identifier_name, sizeof(identifier_name)); - blake2s_final(&blake, handshake_init_hash); -} - -/* Must hold peer->handshake.static_identity->lock */ -void wg_noise_precompute_static_static(struct wg_peer *peer) -{ - down_write(&peer->handshake.lock); - if (!peer->handshake.static_identity->has_identity || - !curve25519(peer->handshake.precomputed_static_static, - peer->handshake.static_identity->static_private, - peer->handshake.remote_static)) - memset(peer->handshake.precomputed_static_static, 0, - NOISE_PUBLIC_KEY_LEN); - up_write(&peer->handshake.lock); -} - -void wg_noise_handshake_init(struct noise_handshake *handshake, - struct noise_static_identity *static_identity, - const u8 peer_public_key[NOISE_PUBLIC_KEY_LEN], - const u8 peer_preshared_key[NOISE_SYMMETRIC_KEY_LEN], - struct wg_peer *peer) -{ - memset(handshake, 0, sizeof(*handshake)); - init_rwsem(&handshake->lock); - handshake->entry.type = INDEX_HASHTABLE_HANDSHAKE; - handshake->entry.peer = peer; - memcpy(handshake->remote_static, peer_public_key, NOISE_PUBLIC_KEY_LEN); - if (peer_preshared_key) - memcpy(handshake->preshared_key, peer_preshared_key, - NOISE_SYMMETRIC_KEY_LEN); - handshake->static_identity = static_identity; - handshake->state = HANDSHAKE_ZEROED; - wg_noise_precompute_static_static(peer); -} - -static void handshake_zero(struct noise_handshake *handshake) -{ - memset(&handshake->ephemeral_private, 0, NOISE_PUBLIC_KEY_LEN); - memset(&handshake->remote_ephemeral, 0, NOISE_PUBLIC_KEY_LEN); - memset(&handshake->hash, 0, NOISE_HASH_LEN); - memset(&handshake->chaining_key, 0, NOISE_HASH_LEN); - handshake->remote_index = 0; - handshake->state = HANDSHAKE_ZEROED; -} - -void wg_noise_handshake_clear(struct noise_handshake *handshake) -{ - wg_index_hashtable_remove( - handshake->entry.peer->device->index_hashtable, - &handshake->entry); - down_write(&handshake->lock); - handshake_zero(handshake); - up_write(&handshake->lock); - wg_index_hashtable_remove( - handshake->entry.peer->device->index_hashtable, - &handshake->entry); -} - -static struct noise_keypair *keypair_create(struct wg_peer *peer) -{ - struct noise_keypair *keypair = kzalloc(sizeof(*keypair), GFP_KERNEL); - - if (unlikely(!keypair)) - return NULL; - keypair->internal_id = atomic64_inc_return(&keypair_counter); - keypair->entry.type = INDEX_HASHTABLE_KEYPAIR; - keypair->entry.peer = peer; - kref_init(&keypair->refcount); - return keypair; -} - -static void keypair_free_rcu(struct rcu_head *rcu) -{ - kzfree(container_of(rcu, struct noise_keypair, rcu)); -} - -static void keypair_free_kref(struct kref *kref) -{ - struct noise_keypair *keypair = - container_of(kref, struct noise_keypair, refcount); - - net_dbg_ratelimited("%s: Keypair %llu destroyed for peer %llu\n", - keypair->entry.peer->device->dev->name, - keypair->internal_id, - keypair->entry.peer->internal_id); - wg_index_hashtable_remove(keypair->entry.peer->device->index_hashtable, - &keypair->entry); - call_rcu(&keypair->rcu, keypair_free_rcu); -} - -void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now) -{ - if (unlikely(!keypair)) - return; - if (unlikely(unreference_now)) - wg_index_hashtable_remove( - keypair->entry.peer->device->index_hashtable, - &keypair->entry); - kref_put(&keypair->refcount, keypair_free_kref); -} - -struct noise_keypair *wg_noise_keypair_get(struct noise_keypair *keypair) -{ - RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(), - "Taking noise keypair reference without holding the RCU BH read lock"); - if (unlikely(!keypair || !kref_get_unless_zero(&keypair->refcount))) - return NULL; - return keypair; -} - -void wg_noise_keypairs_clear(struct noise_keypairs *keypairs) -{ - struct noise_keypair *old; - - spin_lock_bh(&keypairs->keypair_update_lock); - - /* We zero the next_keypair before zeroing the others, so that - * wg_noise_received_with_keypair returns early before subsequent ones - * are zeroed. - */ - old = rcu_dereference_protected(keypairs->next_keypair, - lockdep_is_held(&keypairs->keypair_update_lock)); - RCU_INIT_POINTER(keypairs->next_keypair, NULL); - wg_noise_keypair_put(old, true); - - old = rcu_dereference_protected(keypairs->previous_keypair, - lockdep_is_held(&keypairs->keypair_update_lock)); - RCU_INIT_POINTER(keypairs->previous_keypair, NULL); - wg_noise_keypair_put(old, true); - - old = rcu_dereference_protected(keypairs->current_keypair, - lockdep_is_held(&keypairs->keypair_update_lock)); - RCU_INIT_POINTER(keypairs->current_keypair, NULL); - wg_noise_keypair_put(old, true); - - spin_unlock_bh(&keypairs->keypair_update_lock); -} - -void wg_noise_expire_current_peer_keypairs(struct wg_peer *peer) -{ - struct noise_keypair *keypair; - - wg_noise_handshake_clear(&peer->handshake); - wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); - - spin_lock_bh(&peer->keypairs.keypair_update_lock); - keypair = rcu_dereference_protected(peer->keypairs.next_keypair, - lockdep_is_held(&peer->keypairs.keypair_update_lock)); - if (keypair) - keypair->sending.is_valid = false; - keypair = rcu_dereference_protected(peer->keypairs.current_keypair, - lockdep_is_held(&peer->keypairs.keypair_update_lock)); - if (keypair) - keypair->sending.is_valid = false; - spin_unlock_bh(&peer->keypairs.keypair_update_lock); -} - -static void add_new_keypair(struct noise_keypairs *keypairs, - struct noise_keypair *new_keypair) -{ - struct noise_keypair *previous_keypair, *next_keypair, *current_keypair; - - spin_lock_bh(&keypairs->keypair_update_lock); - previous_keypair = rcu_dereference_protected(keypairs->previous_keypair, - lockdep_is_held(&keypairs->keypair_update_lock)); - next_keypair = rcu_dereference_protected(keypairs->next_keypair, - lockdep_is_held(&keypairs->keypair_update_lock)); - current_keypair = rcu_dereference_protected(keypairs->current_keypair, - lockdep_is_held(&keypairs->keypair_update_lock)); - if (new_keypair->i_am_the_initiator) { - /* If we're the initiator, it means we've sent a handshake, and - * received a confirmation response, which means this new - * keypair can now be used. - */ - if (next_keypair) { - /* If there already was a next keypair pending, we - * demote it to be the previous keypair, and free the - * existing current. Note that this means KCI can result - * in this transition. It would perhaps be more sound to - * always just get rid of the unused next keypair - * instead of putting it in the previous slot, but this - * might be a bit less robust. Something to think about - * for the future. - */ - RCU_INIT_POINTER(keypairs->next_keypair, NULL); - rcu_assign_pointer(keypairs->previous_keypair, - next_keypair); - wg_noise_keypair_put(current_keypair, true); - } else /* If there wasn't an existing next keypair, we replace - * the previous with the current one. - */ - rcu_assign_pointer(keypairs->previous_keypair, - current_keypair); - /* At this point we can get rid of the old previous keypair, and - * set up the new keypair. - */ - wg_noise_keypair_put(previous_keypair, true); - rcu_assign_pointer(keypairs->current_keypair, new_keypair); - } else { - /* If we're the responder, it means we can't use the new keypair - * until we receive confirmation via the first data packet, so - * we get rid of the existing previous one, the possibly - * existing next one, and slide in the new next one. - */ - rcu_assign_pointer(keypairs->next_keypair, new_keypair); - wg_noise_keypair_put(next_keypair, true); - RCU_INIT_POINTER(keypairs->previous_keypair, NULL); - wg_noise_keypair_put(previous_keypair, true); - } - spin_unlock_bh(&keypairs->keypair_update_lock); -} - -bool wg_noise_received_with_keypair(struct noise_keypairs *keypairs, - struct noise_keypair *received_keypair) -{ - struct noise_keypair *old_keypair; - bool key_is_new; - - /* We first check without taking the spinlock. */ - key_is_new = received_keypair == - rcu_access_pointer(keypairs->next_keypair); - if (likely(!key_is_new)) - return false; - - spin_lock_bh(&keypairs->keypair_update_lock); - /* After locking, we double check that things didn't change from - * beneath us. - */ - if (unlikely(received_keypair != - rcu_dereference_protected(keypairs->next_keypair, - lockdep_is_held(&keypairs->keypair_update_lock)))) { - spin_unlock_bh(&keypairs->keypair_update_lock); - return false; - } - - /* When we've finally received the confirmation, we slide the next - * into the current, the current into the previous, and get rid of - * the old previous. - */ - old_keypair = rcu_dereference_protected(keypairs->previous_keypair, - lockdep_is_held(&keypairs->keypair_update_lock)); - rcu_assign_pointer(keypairs->previous_keypair, - rcu_dereference_protected(keypairs->current_keypair, - lockdep_is_held(&keypairs->keypair_update_lock))); - wg_noise_keypair_put(old_keypair, true); - rcu_assign_pointer(keypairs->current_keypair, received_keypair); - RCU_INIT_POINTER(keypairs->next_keypair, NULL); - - spin_unlock_bh(&keypairs->keypair_update_lock); - return true; -} - -/* Must hold static_identity->lock */ -void wg_noise_set_static_identity_private_key( - struct noise_static_identity *static_identity, - const u8 private_key[NOISE_PUBLIC_KEY_LEN]) -{ - memcpy(static_identity->static_private, private_key, - NOISE_PUBLIC_KEY_LEN); - curve25519_clamp_secret(static_identity->static_private); - static_identity->has_identity = curve25519_generate_public( - static_identity->static_public, private_key); -} - -/* This is Hugo Krawczyk's HKDF: - * - https://eprint.iacr.org/2010/264.pdf - * - https://tools.ietf.org/html/rfc5869 - */ -static void kdf(u8 *first_dst, u8 *second_dst, u8 *third_dst, const u8 *data, - size_t first_len, size_t second_len, size_t third_len, - size_t data_len, const u8 chaining_key[NOISE_HASH_LEN]) -{ - u8 output[BLAKE2S_HASH_SIZE + 1]; - u8 secret[BLAKE2S_HASH_SIZE]; - - WARN_ON(IS_ENABLED(DEBUG) && - (first_len > BLAKE2S_HASH_SIZE || - second_len > BLAKE2S_HASH_SIZE || - third_len > BLAKE2S_HASH_SIZE || - ((second_len || second_dst || third_len || third_dst) && - (!first_len || !first_dst)) || - ((third_len || third_dst) && (!second_len || !second_dst)))); - - /* Extract entropy from data into secret */ - blake2s256_hmac(secret, data, chaining_key, data_len, NOISE_HASH_LEN); - - if (!first_dst || !first_len) - goto out; - - /* Expand first key: key = secret, data = 0x1 */ - output[0] = 1; - blake2s256_hmac(output, output, secret, 1, BLAKE2S_HASH_SIZE); - memcpy(first_dst, output, first_len); - - if (!second_dst || !second_len) - goto out; - - /* Expand second key: key = secret, data = first-key || 0x2 */ - output[BLAKE2S_HASH_SIZE] = 2; - blake2s256_hmac(output, output, secret, BLAKE2S_HASH_SIZE + 1, - BLAKE2S_HASH_SIZE); - memcpy(second_dst, output, second_len); - - if (!third_dst || !third_len) - goto out; - - /* Expand third key: key = secret, data = second-key || 0x3 */ - output[BLAKE2S_HASH_SIZE] = 3; - blake2s256_hmac(output, output, secret, BLAKE2S_HASH_SIZE + 1, - BLAKE2S_HASH_SIZE); - memcpy(third_dst, output, third_len); - -out: - /* Clear sensitive data from stack */ - memzero_explicit(secret, BLAKE2S_HASH_SIZE); - memzero_explicit(output, BLAKE2S_HASH_SIZE + 1); -} - -static void symmetric_key_init(struct noise_symmetric_key *key) -{ - spin_lock_init(&key->counter.receive.lock); - atomic64_set(&key->counter.counter, 0); - memset(key->counter.receive.backtrack, 0, - sizeof(key->counter.receive.backtrack)); - key->birthdate = ktime_get_coarse_boottime_ns(); - key->is_valid = true; -} - -static void derive_keys(struct noise_symmetric_key *first_dst, - struct noise_symmetric_key *second_dst, - const u8 chaining_key[NOISE_HASH_LEN]) -{ - kdf(first_dst->key, second_dst->key, NULL, NULL, - NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, 0, - chaining_key); - symmetric_key_init(first_dst); - symmetric_key_init(second_dst); -} - -static bool __must_check mix_dh(u8 chaining_key[NOISE_HASH_LEN], - u8 key[NOISE_SYMMETRIC_KEY_LEN], - const u8 private[NOISE_PUBLIC_KEY_LEN], - const u8 public[NOISE_PUBLIC_KEY_LEN]) -{ - u8 dh_calculation[NOISE_PUBLIC_KEY_LEN]; - - if (unlikely(!curve25519(dh_calculation, private, public))) - return false; - kdf(chaining_key, key, NULL, dh_calculation, NOISE_HASH_LEN, - NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, chaining_key); - memzero_explicit(dh_calculation, NOISE_PUBLIC_KEY_LEN); - return true; -} - -static bool __must_check mix_precomputed_dh(u8 chaining_key[NOISE_HASH_LEN], - u8 key[NOISE_SYMMETRIC_KEY_LEN], - const u8 precomputed[NOISE_PUBLIC_KEY_LEN]) -{ - static u8 zero_point[NOISE_PUBLIC_KEY_LEN]; - if (unlikely(!crypto_memneq(precomputed, zero_point, NOISE_PUBLIC_KEY_LEN))) - return false; - kdf(chaining_key, key, NULL, precomputed, NOISE_HASH_LEN, - NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, - chaining_key); - return true; -} - -static void mix_hash(u8 hash[NOISE_HASH_LEN], const u8 *src, size_t src_len) -{ - struct blake2s_state blake; - - blake2s_init(&blake, NOISE_HASH_LEN); - blake2s_update(&blake, hash, NOISE_HASH_LEN); - blake2s_update(&blake, src, src_len); - blake2s_final(&blake, hash); -} - -static void mix_psk(u8 chaining_key[NOISE_HASH_LEN], u8 hash[NOISE_HASH_LEN], - u8 key[NOISE_SYMMETRIC_KEY_LEN], - const u8 psk[NOISE_SYMMETRIC_KEY_LEN]) -{ - u8 temp_hash[NOISE_HASH_LEN]; - - kdf(chaining_key, temp_hash, key, psk, NOISE_HASH_LEN, NOISE_HASH_LEN, - NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, chaining_key); - mix_hash(hash, temp_hash, NOISE_HASH_LEN); - memzero_explicit(temp_hash, NOISE_HASH_LEN); -} - -static void handshake_init(u8 chaining_key[NOISE_HASH_LEN], - u8 hash[NOISE_HASH_LEN], - const u8 remote_static[NOISE_PUBLIC_KEY_LEN]) -{ - memcpy(hash, handshake_init_hash, NOISE_HASH_LEN); - memcpy(chaining_key, handshake_init_chaining_key, NOISE_HASH_LEN); - mix_hash(hash, remote_static, NOISE_PUBLIC_KEY_LEN); -} - -static void message_encrypt(u8 *dst_ciphertext, const u8 *src_plaintext, - size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN], - u8 hash[NOISE_HASH_LEN]) -{ - chacha20poly1305_encrypt(dst_ciphertext, src_plaintext, src_len, hash, - NOISE_HASH_LEN, - 0 /* Always zero for Noise_IK */, key); - mix_hash(hash, dst_ciphertext, noise_encrypted_len(src_len)); -} - -static bool message_decrypt(u8 *dst_plaintext, const u8 *src_ciphertext, - size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN], - u8 hash[NOISE_HASH_LEN]) -{ - if (!chacha20poly1305_decrypt(dst_plaintext, src_ciphertext, src_len, - hash, NOISE_HASH_LEN, - 0 /* Always zero for Noise_IK */, key)) - return false; - mix_hash(hash, src_ciphertext, src_len); - return true; -} - -static void message_ephemeral(u8 ephemeral_dst[NOISE_PUBLIC_KEY_LEN], - const u8 ephemeral_src[NOISE_PUBLIC_KEY_LEN], - u8 chaining_key[NOISE_HASH_LEN], - u8 hash[NOISE_HASH_LEN]) -{ - if (ephemeral_dst != ephemeral_src) - memcpy(ephemeral_dst, ephemeral_src, NOISE_PUBLIC_KEY_LEN); - mix_hash(hash, ephemeral_src, NOISE_PUBLIC_KEY_LEN); - kdf(chaining_key, NULL, NULL, ephemeral_src, NOISE_HASH_LEN, 0, 0, - NOISE_PUBLIC_KEY_LEN, chaining_key); -} - -static void tai64n_now(u8 output[NOISE_TIMESTAMP_LEN]) -{ - struct timespec64 now; - - ktime_get_real_ts64(&now); - - /* In order to prevent some sort of infoleak from precise timers, we - * round down the nanoseconds part to the closest rounded-down power of - * two to the maximum initiations per second allowed anyway by the - * implementation. - */ - now.tv_nsec = ALIGN_DOWN(now.tv_nsec, - rounddown_pow_of_two(NSEC_PER_SEC / INITIATIONS_PER_SECOND)); - - /* https://cr.yp.to/libtai/tai64.html */ - *(__be64 *)output = cpu_to_be64(0x400000000000000aULL + now.tv_sec); - *(__be32 *)(output + sizeof(__be64)) = cpu_to_be32(now.tv_nsec); -} - -bool -wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, - struct noise_handshake *handshake) -{ - u8 timestamp[NOISE_TIMESTAMP_LEN]; - u8 key[NOISE_SYMMETRIC_KEY_LEN]; - bool ret = false; - - /* We need to wait for crng _before_ taking any locks, since - * curve25519_generate_secret uses get_random_bytes_wait. - */ - wait_for_random_bytes(); - - down_read(&handshake->static_identity->lock); - down_write(&handshake->lock); - - if (unlikely(!handshake->static_identity->has_identity)) - goto out; - - dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION); - - handshake_init(handshake->chaining_key, handshake->hash, - handshake->remote_static); - - /* e */ - curve25519_generate_secret(handshake->ephemeral_private); - if (!curve25519_generate_public(dst->unencrypted_ephemeral, - handshake->ephemeral_private)) - goto out; - message_ephemeral(dst->unencrypted_ephemeral, - dst->unencrypted_ephemeral, handshake->chaining_key, - handshake->hash); - - /* es */ - if (!mix_dh(handshake->chaining_key, key, handshake->ephemeral_private, - handshake->remote_static)) - goto out; - - /* s */ - message_encrypt(dst->encrypted_static, - handshake->static_identity->static_public, - NOISE_PUBLIC_KEY_LEN, key, handshake->hash); - - /* ss */ - if (!mix_precomputed_dh(handshake->chaining_key, key, - handshake->precomputed_static_static)) - goto out; - - /* {t} */ - tai64n_now(timestamp); - message_encrypt(dst->encrypted_timestamp, timestamp, - NOISE_TIMESTAMP_LEN, key, handshake->hash); - - dst->sender_index = wg_index_hashtable_insert( - handshake->entry.peer->device->index_hashtable, - &handshake->entry); - - handshake->state = HANDSHAKE_CREATED_INITIATION; - ret = true; - -out: - up_write(&handshake->lock); - up_read(&handshake->static_identity->lock); - memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); - return ret; -} - -struct wg_peer * -wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, - struct wg_device *wg) -{ - struct wg_peer *peer = NULL, *ret_peer = NULL; - struct noise_handshake *handshake; - bool replay_attack, flood_attack; - u8 key[NOISE_SYMMETRIC_KEY_LEN]; - u8 chaining_key[NOISE_HASH_LEN]; - u8 hash[NOISE_HASH_LEN]; - u8 s[NOISE_PUBLIC_KEY_LEN]; - u8 e[NOISE_PUBLIC_KEY_LEN]; - u8 t[NOISE_TIMESTAMP_LEN]; - u64 initiation_consumption; - - down_read(&wg->static_identity.lock); - if (unlikely(!wg->static_identity.has_identity)) - goto out; - - handshake_init(chaining_key, hash, wg->static_identity.static_public); - - /* e */ - message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash); - - /* es */ - if (!mix_dh(chaining_key, key, wg->static_identity.static_private, e)) - goto out; - - /* s */ - if (!message_decrypt(s, src->encrypted_static, - sizeof(src->encrypted_static), key, hash)) - goto out; - - /* Lookup which peer we're actually talking to */ - peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, s); - if (!peer) - goto out; - handshake = &peer->handshake; - - /* ss */ - if (!mix_precomputed_dh(chaining_key, key, - handshake->precomputed_static_static)) - goto out; - - /* {t} */ - if (!message_decrypt(t, src->encrypted_timestamp, - sizeof(src->encrypted_timestamp), key, hash)) - goto out; - - down_read(&handshake->lock); - replay_attack = memcmp(t, handshake->latest_timestamp, - NOISE_TIMESTAMP_LEN) <= 0; - flood_attack = (s64)handshake->last_initiation_consumption + - NSEC_PER_SEC / INITIATIONS_PER_SECOND > - (s64)ktime_get_coarse_boottime_ns(); - up_read(&handshake->lock); - if (replay_attack || flood_attack) - goto out; - - /* Success! Copy everything to peer */ - down_write(&handshake->lock); - memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN); - if (memcmp(t, handshake->latest_timestamp, NOISE_TIMESTAMP_LEN) > 0) - memcpy(handshake->latest_timestamp, t, NOISE_TIMESTAMP_LEN); - memcpy(handshake->hash, hash, NOISE_HASH_LEN); - memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN); - handshake->remote_index = src->sender_index; - if ((s64)(handshake->last_initiation_consumption - - (initiation_consumption = ktime_get_coarse_boottime_ns())) < 0) - handshake->last_initiation_consumption = initiation_consumption; - handshake->state = HANDSHAKE_CONSUMED_INITIATION; - up_write(&handshake->lock); - ret_peer = peer; - -out: - memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); - memzero_explicit(hash, NOISE_HASH_LEN); - memzero_explicit(chaining_key, NOISE_HASH_LEN); - up_read(&wg->static_identity.lock); - if (!ret_peer) - wg_peer_put(peer); - return ret_peer; -} - -bool wg_noise_handshake_create_response(struct message_handshake_response *dst, - struct noise_handshake *handshake) -{ - u8 key[NOISE_SYMMETRIC_KEY_LEN]; - bool ret = false; - - /* We need to wait for crng _before_ taking any locks, since - * curve25519_generate_secret uses get_random_bytes_wait. - */ - wait_for_random_bytes(); - - down_read(&handshake->static_identity->lock); - down_write(&handshake->lock); - - if (handshake->state != HANDSHAKE_CONSUMED_INITIATION) - goto out; - - dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE); - dst->receiver_index = handshake->remote_index; - - /* e */ - curve25519_generate_secret(handshake->ephemeral_private); - if (!curve25519_generate_public(dst->unencrypted_ephemeral, - handshake->ephemeral_private)) - goto out; - message_ephemeral(dst->unencrypted_ephemeral, - dst->unencrypted_ephemeral, handshake->chaining_key, - handshake->hash); - - /* ee */ - if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, - handshake->remote_ephemeral)) - goto out; - - /* se */ - if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, - handshake->remote_static)) - goto out; - - /* psk */ - mix_psk(handshake->chaining_key, handshake->hash, key, - handshake->preshared_key); - - /* {} */ - message_encrypt(dst->encrypted_nothing, NULL, 0, key, handshake->hash); - - dst->sender_index = wg_index_hashtable_insert( - handshake->entry.peer->device->index_hashtable, - &handshake->entry); - - handshake->state = HANDSHAKE_CREATED_RESPONSE; - ret = true; - -out: - up_write(&handshake->lock); - up_read(&handshake->static_identity->lock); - memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); - return ret; -} - -struct wg_peer * -wg_noise_handshake_consume_response(struct message_handshake_response *src, - struct wg_device *wg) -{ - enum noise_handshake_state state = HANDSHAKE_ZEROED; - struct wg_peer *peer = NULL, *ret_peer = NULL; - struct noise_handshake *handshake; - u8 key[NOISE_SYMMETRIC_KEY_LEN]; - u8 hash[NOISE_HASH_LEN]; - u8 chaining_key[NOISE_HASH_LEN]; - u8 e[NOISE_PUBLIC_KEY_LEN]; - u8 ephemeral_private[NOISE_PUBLIC_KEY_LEN]; - u8 static_private[NOISE_PUBLIC_KEY_LEN]; - - down_read(&wg->static_identity.lock); - - if (unlikely(!wg->static_identity.has_identity)) - goto out; - - handshake = (struct noise_handshake *)wg_index_hashtable_lookup( - wg->index_hashtable, INDEX_HASHTABLE_HANDSHAKE, - src->receiver_index, &peer); - if (unlikely(!handshake)) - goto out; - - down_read(&handshake->lock); - state = handshake->state; - memcpy(hash, handshake->hash, NOISE_HASH_LEN); - memcpy(chaining_key, handshake->chaining_key, NOISE_HASH_LEN); - memcpy(ephemeral_private, handshake->ephemeral_private, - NOISE_PUBLIC_KEY_LEN); - up_read(&handshake->lock); - - if (state != HANDSHAKE_CREATED_INITIATION) - goto fail; - - /* e */ - message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash); - - /* ee */ - if (!mix_dh(chaining_key, NULL, ephemeral_private, e)) - goto fail; - - /* se */ - if (!mix_dh(chaining_key, NULL, wg->static_identity.static_private, e)) - goto fail; - - /* psk */ - mix_psk(chaining_key, hash, key, handshake->preshared_key); - - /* {} */ - if (!message_decrypt(NULL, src->encrypted_nothing, - sizeof(src->encrypted_nothing), key, hash)) - goto fail; - - /* Success! Copy everything to peer */ - down_write(&handshake->lock); - /* It's important to check that the state is still the same, while we - * have an exclusive lock. - */ - if (handshake->state != state) { - up_write(&handshake->lock); - goto fail; - } - memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN); - memcpy(handshake->hash, hash, NOISE_HASH_LEN); - memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN); - handshake->remote_index = src->sender_index; - handshake->state = HANDSHAKE_CONSUMED_RESPONSE; - up_write(&handshake->lock); - ret_peer = peer; - goto out; - -fail: - wg_peer_put(peer); -out: - memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); - memzero_explicit(hash, NOISE_HASH_LEN); - memzero_explicit(chaining_key, NOISE_HASH_LEN); - memzero_explicit(ephemeral_private, NOISE_PUBLIC_KEY_LEN); - memzero_explicit(static_private, NOISE_PUBLIC_KEY_LEN); - up_read(&wg->static_identity.lock); - return ret_peer; -} - -bool wg_noise_handshake_begin_session(struct noise_handshake *handshake, - struct noise_keypairs *keypairs) -{ - struct noise_keypair *new_keypair; - bool ret = false; - - down_write(&handshake->lock); - if (handshake->state != HANDSHAKE_CREATED_RESPONSE && - handshake->state != HANDSHAKE_CONSUMED_RESPONSE) - goto out; - - new_keypair = keypair_create(handshake->entry.peer); - if (!new_keypair) - goto out; - new_keypair->i_am_the_initiator = handshake->state == - HANDSHAKE_CONSUMED_RESPONSE; - new_keypair->remote_index = handshake->remote_index; - - if (new_keypair->i_am_the_initiator) - derive_keys(&new_keypair->sending, &new_keypair->receiving, - handshake->chaining_key); - else - derive_keys(&new_keypair->receiving, &new_keypair->sending, - handshake->chaining_key); - - handshake_zero(handshake); - rcu_read_lock_bh(); - if (likely(!READ_ONCE(container_of(handshake, struct wg_peer, - handshake)->is_dead))) { - add_new_keypair(keypairs, new_keypair); - net_dbg_ratelimited("%s: Keypair %llu created for peer %llu\n", - handshake->entry.peer->device->dev->name, - new_keypair->internal_id, - handshake->entry.peer->internal_id); - ret = wg_index_hashtable_replace( - handshake->entry.peer->device->index_hashtable, - &handshake->entry, &new_keypair->entry); - } else { - kzfree(new_keypair); - } - rcu_read_unlock_bh(); - -out: - up_write(&handshake->lock); - return ret; -} diff --git a/drivers/net/wireguard/noise.h b/drivers/net/wireguard/noise.h deleted file mode 100644 index f532d59d3f19a..0000000000000 --- a/drivers/net/wireguard/noise.h +++ /dev/null @@ -1,137 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ -#ifndef _WG_NOISE_H -#define _WG_NOISE_H - -#include "messages.h" -#include "peerlookup.h" - -#include -#include -#include -#include -#include -#include - -union noise_counter { - struct { - u64 counter; - unsigned long backtrack[COUNTER_BITS_TOTAL / BITS_PER_LONG]; - spinlock_t lock; - } receive; - atomic64_t counter; -}; - -struct noise_symmetric_key { - u8 key[NOISE_SYMMETRIC_KEY_LEN]; - union noise_counter counter; - u64 birthdate; - bool is_valid; -}; - -struct noise_keypair { - struct index_hashtable_entry entry; - struct noise_symmetric_key sending; - struct noise_symmetric_key receiving; - __le32 remote_index; - bool i_am_the_initiator; - struct kref refcount; - struct rcu_head rcu; - u64 internal_id; -}; - -struct noise_keypairs { - struct noise_keypair __rcu *current_keypair; - struct noise_keypair __rcu *previous_keypair; - struct noise_keypair __rcu *next_keypair; - spinlock_t keypair_update_lock; -}; - -struct noise_static_identity { - u8 static_public[NOISE_PUBLIC_KEY_LEN]; - u8 static_private[NOISE_PUBLIC_KEY_LEN]; - struct rw_semaphore lock; - bool has_identity; -}; - -enum noise_handshake_state { - HANDSHAKE_ZEROED, - HANDSHAKE_CREATED_INITIATION, - HANDSHAKE_CONSUMED_INITIATION, - HANDSHAKE_CREATED_RESPONSE, - HANDSHAKE_CONSUMED_RESPONSE -}; - -struct noise_handshake { - struct index_hashtable_entry entry; - - enum noise_handshake_state state; - u64 last_initiation_consumption; - - struct noise_static_identity *static_identity; - - u8 ephemeral_private[NOISE_PUBLIC_KEY_LEN]; - u8 remote_static[NOISE_PUBLIC_KEY_LEN]; - u8 remote_ephemeral[NOISE_PUBLIC_KEY_LEN]; - u8 precomputed_static_static[NOISE_PUBLIC_KEY_LEN]; - - u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]; - - u8 hash[NOISE_HASH_LEN]; - u8 chaining_key[NOISE_HASH_LEN]; - - u8 latest_timestamp[NOISE_TIMESTAMP_LEN]; - __le32 remote_index; - - /* Protects all members except the immutable (after noise_handshake_ - * init): remote_static, precomputed_static_static, static_identity. - */ - struct rw_semaphore lock; -}; - -struct wg_device; - -void wg_noise_init(void); -void wg_noise_handshake_init(struct noise_handshake *handshake, - struct noise_static_identity *static_identity, - const u8 peer_public_key[NOISE_PUBLIC_KEY_LEN], - const u8 peer_preshared_key[NOISE_SYMMETRIC_KEY_LEN], - struct wg_peer *peer); -void wg_noise_handshake_clear(struct noise_handshake *handshake); -static inline void wg_noise_reset_last_sent_handshake(atomic64_t *handshake_ns) -{ - atomic64_set(handshake_ns, ktime_get_coarse_boottime_ns() - - (u64)(REKEY_TIMEOUT + 1) * NSEC_PER_SEC); -} - -void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now); -struct noise_keypair *wg_noise_keypair_get(struct noise_keypair *keypair); -void wg_noise_keypairs_clear(struct noise_keypairs *keypairs); -bool wg_noise_received_with_keypair(struct noise_keypairs *keypairs, - struct noise_keypair *received_keypair); -void wg_noise_expire_current_peer_keypairs(struct wg_peer *peer); - -void wg_noise_set_static_identity_private_key( - struct noise_static_identity *static_identity, - const u8 private_key[NOISE_PUBLIC_KEY_LEN]); -void wg_noise_precompute_static_static(struct wg_peer *peer); - -bool -wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, - struct noise_handshake *handshake); -struct wg_peer * -wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, - struct wg_device *wg); - -bool wg_noise_handshake_create_response(struct message_handshake_response *dst, - struct noise_handshake *handshake); -struct wg_peer * -wg_noise_handshake_consume_response(struct message_handshake_response *src, - struct wg_device *wg); - -bool wg_noise_handshake_begin_session(struct noise_handshake *handshake, - struct noise_keypairs *keypairs); - -#endif /* _WG_NOISE_H */ diff --git a/drivers/net/wireguard/peer.c b/drivers/net/wireguard/peer.c deleted file mode 100644 index 1d634bd3038f0..0000000000000 --- a/drivers/net/wireguard/peer.c +++ /dev/null @@ -1,237 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include "peer.h" -#include "device.h" -#include "queueing.h" -#include "timers.h" -#include "peerlookup.h" -#include "noise.h" - -#include -#include -#include -#include - -static atomic64_t peer_counter = ATOMIC64_INIT(0); - -struct wg_peer *wg_peer_create(struct wg_device *wg, - const u8 public_key[NOISE_PUBLIC_KEY_LEN], - const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]) -{ - struct wg_peer *peer; - int ret = -ENOMEM; - - lockdep_assert_held(&wg->device_update_lock); - - if (wg->num_peers >= MAX_PEERS_PER_DEVICE) - return ERR_PTR(ret); - - peer = kzalloc(sizeof(*peer), GFP_KERNEL); - if (unlikely(!peer)) - return ERR_PTR(ret); - peer->device = wg; - - wg_noise_handshake_init(&peer->handshake, &wg->static_identity, - public_key, preshared_key, peer); - if (dst_cache_init(&peer->endpoint_cache, GFP_KERNEL)) - goto err_1; - if (wg_packet_queue_init(&peer->tx_queue, wg_packet_tx_worker, false, - MAX_QUEUED_PACKETS)) - goto err_2; - if (wg_packet_queue_init(&peer->rx_queue, NULL, false, - MAX_QUEUED_PACKETS)) - goto err_3; - - peer->internal_id = atomic64_inc_return(&peer_counter); - peer->serial_work_cpu = nr_cpumask_bits; - wg_cookie_init(&peer->latest_cookie); - wg_timers_init(peer); - wg_cookie_checker_precompute_peer_keys(peer); - spin_lock_init(&peer->keypairs.keypair_update_lock); - INIT_WORK(&peer->transmit_handshake_work, - wg_packet_handshake_send_worker); - rwlock_init(&peer->endpoint_lock); - kref_init(&peer->refcount); - skb_queue_head_init(&peer->staged_packet_queue); - wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); - set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state); - netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll, - NAPI_POLL_WEIGHT); - napi_enable(&peer->napi); - list_add_tail(&peer->peer_list, &wg->peer_list); - INIT_LIST_HEAD(&peer->allowedips_list); - wg_pubkey_hashtable_add(wg->peer_hashtable, peer); - ++wg->num_peers; - pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id); - return peer; - -err_3: - wg_packet_queue_free(&peer->tx_queue, false); -err_2: - dst_cache_destroy(&peer->endpoint_cache); -err_1: - kfree(peer); - return ERR_PTR(ret); -} - -struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer) -{ - RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(), - "Taking peer reference without holding the RCU read lock"); - if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount))) - return NULL; - return peer; -} - -static void peer_make_dead(struct wg_peer *peer) -{ - /* Remove from configuration-time lookup structures. */ - list_del_init(&peer->peer_list); - wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer, - &peer->device->device_update_lock); - wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer); - - /* Mark as dead, so that we don't allow jumping contexts after. */ - WRITE_ONCE(peer->is_dead, true); - - /* The caller must now synchronize_rcu() for this to take effect. */ -} - -static void peer_remove_after_dead(struct wg_peer *peer) -{ - WARN_ON(!peer->is_dead); - - /* No more keypairs can be created for this peer, since is_dead protects - * add_new_keypair, so we can now destroy existing ones. - */ - wg_noise_keypairs_clear(&peer->keypairs); - - /* Destroy all ongoing timers that were in-flight at the beginning of - * this function. - */ - wg_timers_stop(peer); - - /* The transition between packet encryption/decryption queues isn't - * guarded by is_dead, but each reference's life is strictly bounded by - * two generations: once for parallel crypto and once for serial - * ingestion, so we can simply flush twice, and be sure that we no - * longer have references inside these queues. - */ - - /* a) For encrypt/decrypt. */ - flush_workqueue(peer->device->packet_crypt_wq); - /* b.1) For send (but not receive, since that's napi). */ - flush_workqueue(peer->device->packet_crypt_wq); - /* b.2.1) For receive (but not send, since that's wq). */ - napi_disable(&peer->napi); - /* b.2.1) It's now safe to remove the napi struct, which must be done - * here from process context. - */ - netif_napi_del(&peer->napi); - - /* Ensure any workstructs we own (like transmit_handshake_work or - * clear_peer_work) no longer are in use. - */ - flush_workqueue(peer->device->handshake_send_wq); - - /* After the above flushes, a peer might still be active in a few - * different contexts: 1) from xmit(), before hitting is_dead and - * returning, 2) from wg_packet_consume_data(), before hitting is_dead - * and returning, 3) from wg_receive_handshake_packet() after a point - * where it has processed an incoming handshake packet, but where - * all calls to pass it off to timers fails because of is_dead. We won't - * have new references in (1) eventually, because we're removed from - * allowedips; we won't have new references in (2) eventually, because - * wg_index_hashtable_lookup will always return NULL, since we removed - * all existing keypairs and no more can be created; we won't have new - * references in (3) eventually, because we're removed from the pubkey - * hash table, which allows for a maximum of one handshake response, - * via the still-uncleared index hashtable entry, but not more than one, - * and in wg_cookie_message_consume, the lookup eventually gets a peer - * with a refcount of zero, so no new reference is taken. - */ - - --peer->device->num_peers; - wg_peer_put(peer); -} - -/* We have a separate "remove" function make sure that all active places where - * a peer is currently operating will eventually come to an end and not pass - * their reference onto another context. - */ -void wg_peer_remove(struct wg_peer *peer) -{ - if (unlikely(!peer)) - return; - lockdep_assert_held(&peer->device->device_update_lock); - - peer_make_dead(peer); - synchronize_rcu(); - peer_remove_after_dead(peer); -} - -void wg_peer_remove_all(struct wg_device *wg) -{ - struct wg_peer *peer, *temp; - LIST_HEAD(dead_peers); - - lockdep_assert_held(&wg->device_update_lock); - - /* Avoid having to traverse individually for each one. */ - wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock); - - list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) { - peer_make_dead(peer); - list_add_tail(&peer->peer_list, &dead_peers); - } - synchronize_rcu(); - list_for_each_entry_safe(peer, temp, &dead_peers, peer_list) - peer_remove_after_dead(peer); -} - -static void rcu_release(struct rcu_head *rcu) -{ - struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu); - - dst_cache_destroy(&peer->endpoint_cache); - wg_packet_queue_free(&peer->rx_queue, false); - wg_packet_queue_free(&peer->tx_queue, false); - - /* The final zeroing takes care of clearing any remaining handshake key - * material and other potentially sensitive information. - */ - kzfree(peer); -} - -static void kref_release(struct kref *refcount) -{ - struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount); - - pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n", - peer->device->dev->name, peer->internal_id, - &peer->endpoint.addr); - - /* Remove ourself from dynamic runtime lookup structures, now that the - * last reference is gone. - */ - wg_index_hashtable_remove(peer->device->index_hashtable, - &peer->handshake.entry); - - /* Remove any lingering packets that didn't have a chance to be - * transmitted. - */ - wg_packet_purge_staged_packets(peer); - - /* Free the memory used. */ - call_rcu(&peer->rcu, rcu_release); -} - -void wg_peer_put(struct wg_peer *peer) -{ - if (unlikely(!peer)) - return; - kref_put(&peer->refcount, kref_release); -} diff --git a/drivers/net/wireguard/peer.h b/drivers/net/wireguard/peer.h deleted file mode 100644 index 23af409229972..0000000000000 --- a/drivers/net/wireguard/peer.h +++ /dev/null @@ -1,83 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#ifndef _WG_PEER_H -#define _WG_PEER_H - -#include "device.h" -#include "noise.h" -#include "cookie.h" - -#include -#include -#include -#include -#include - -struct wg_device; - -struct endpoint { - union { - struct sockaddr addr; - struct sockaddr_in addr4; - struct sockaddr_in6 addr6; - }; - union { - struct { - struct in_addr src4; - /* Essentially the same as addr6->scope_id */ - int src_if4; - }; - struct in6_addr src6; - }; -}; - -struct wg_peer { - struct wg_device *device; - struct crypt_queue tx_queue, rx_queue; - struct sk_buff_head staged_packet_queue; - int serial_work_cpu; - struct noise_keypairs keypairs; - struct endpoint endpoint; - struct dst_cache endpoint_cache; - rwlock_t endpoint_lock; - struct noise_handshake handshake; - atomic64_t last_sent_handshake; - struct work_struct transmit_handshake_work, clear_peer_work; - struct cookie latest_cookie; - struct hlist_node pubkey_hash; - u64 rx_bytes, tx_bytes; - struct timer_list timer_retransmit_handshake, timer_send_keepalive; - struct timer_list timer_new_handshake, timer_zero_key_material; - struct timer_list timer_persistent_keepalive; - unsigned int timer_handshake_attempts; - u16 persistent_keepalive_interval; - bool timer_need_another_keepalive; - bool sent_lastminute_handshake; - struct timespec64 walltime_last_handshake; - struct kref refcount; - struct rcu_head rcu; - struct list_head peer_list; - struct list_head allowedips_list; - u64 internal_id; - struct napi_struct napi; - bool is_dead; -}; - -struct wg_peer *wg_peer_create(struct wg_device *wg, - const u8 public_key[NOISE_PUBLIC_KEY_LEN], - const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]); - -struct wg_peer *__must_check wg_peer_get_maybe_zero(struct wg_peer *peer); -static inline struct wg_peer *wg_peer_get(struct wg_peer *peer) -{ - kref_get(&peer->refcount); - return peer; -} -void wg_peer_put(struct wg_peer *peer); -void wg_peer_remove(struct wg_peer *peer); -void wg_peer_remove_all(struct wg_device *wg); - -#endif /* _WG_PEER_H */ diff --git a/drivers/net/wireguard/peerlookup.c b/drivers/net/wireguard/peerlookup.c deleted file mode 100644 index e4deb331476b3..0000000000000 --- a/drivers/net/wireguard/peerlookup.c +++ /dev/null @@ -1,221 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include "peerlookup.h" -#include "peer.h" -#include "noise.h" - -static struct hlist_head *pubkey_bucket(struct pubkey_hashtable *table, - const u8 pubkey[NOISE_PUBLIC_KEY_LEN]) -{ - /* siphash gives us a secure 64bit number based on a random key. Since - * the bits are uniformly distributed, we can then mask off to get the - * bits we need. - */ - const u64 hash = siphash(pubkey, NOISE_PUBLIC_KEY_LEN, &table->key); - - return &table->hashtable[hash & (HASH_SIZE(table->hashtable) - 1)]; -} - -struct pubkey_hashtable *wg_pubkey_hashtable_alloc(void) -{ - struct pubkey_hashtable *table = kvmalloc(sizeof(*table), GFP_KERNEL); - - if (!table) - return NULL; - - get_random_bytes(&table->key, sizeof(table->key)); - hash_init(table->hashtable); - mutex_init(&table->lock); - return table; -} - -void wg_pubkey_hashtable_add(struct pubkey_hashtable *table, - struct wg_peer *peer) -{ - mutex_lock(&table->lock); - hlist_add_head_rcu(&peer->pubkey_hash, - pubkey_bucket(table, peer->handshake.remote_static)); - mutex_unlock(&table->lock); -} - -void wg_pubkey_hashtable_remove(struct pubkey_hashtable *table, - struct wg_peer *peer) -{ - mutex_lock(&table->lock); - hlist_del_init_rcu(&peer->pubkey_hash); - mutex_unlock(&table->lock); -} - -/* Returns a strong reference to a peer */ -struct wg_peer * -wg_pubkey_hashtable_lookup(struct pubkey_hashtable *table, - const u8 pubkey[NOISE_PUBLIC_KEY_LEN]) -{ - struct wg_peer *iter_peer, *peer = NULL; - - rcu_read_lock_bh(); - hlist_for_each_entry_rcu_bh(iter_peer, pubkey_bucket(table, pubkey), - pubkey_hash) { - if (!memcmp(pubkey, iter_peer->handshake.remote_static, - NOISE_PUBLIC_KEY_LEN)) { - peer = iter_peer; - break; - } - } - peer = wg_peer_get_maybe_zero(peer); - rcu_read_unlock_bh(); - return peer; -} - -static struct hlist_head *index_bucket(struct index_hashtable *table, - const __le32 index) -{ - /* Since the indices are random and thus all bits are uniformly - * distributed, we can find its bucket simply by masking. - */ - return &table->hashtable[(__force u32)index & - (HASH_SIZE(table->hashtable) - 1)]; -} - -struct index_hashtable *wg_index_hashtable_alloc(void) -{ - struct index_hashtable *table = kvmalloc(sizeof(*table), GFP_KERNEL); - - if (!table) - return NULL; - - hash_init(table->hashtable); - spin_lock_init(&table->lock); - return table; -} - -/* At the moment, we limit ourselves to 2^20 total peers, which generally might - * amount to 2^20*3 items in this hashtable. The algorithm below works by - * picking a random number and testing it. We can see that these limits mean we - * usually succeed pretty quickly: - * - * >>> def calculation(tries, size): - * ... return (size / 2**32)**(tries - 1) * (1 - (size / 2**32)) - * ... - * >>> calculation(1, 2**20 * 3) - * 0.999267578125 - * >>> calculation(2, 2**20 * 3) - * 0.0007318854331970215 - * >>> calculation(3, 2**20 * 3) - * 5.360489012673497e-07 - * >>> calculation(4, 2**20 * 3) - * 3.9261394135792216e-10 - * - * At the moment, we don't do any masking, so this algorithm isn't exactly - * constant time in either the random guessing or in the hash list lookup. We - * could require a minimum of 3 tries, which would successfully mask the - * guessing. this would not, however, help with the growing hash lengths, which - * is another thing to consider moving forward. - */ - -__le32 wg_index_hashtable_insert(struct index_hashtable *table, - struct index_hashtable_entry *entry) -{ - struct index_hashtable_entry *existing_entry; - - spin_lock_bh(&table->lock); - hlist_del_init_rcu(&entry->index_hash); - spin_unlock_bh(&table->lock); - - rcu_read_lock_bh(); - -search_unused_slot: - /* First we try to find an unused slot, randomly, while unlocked. */ - entry->index = (__force __le32)get_random_u32(); - hlist_for_each_entry_rcu_bh(existing_entry, - index_bucket(table, entry->index), - index_hash) { - if (existing_entry->index == entry->index) - /* If it's already in use, we continue searching. */ - goto search_unused_slot; - } - - /* Once we've found an unused slot, we lock it, and then double-check - * that nobody else stole it from us. - */ - spin_lock_bh(&table->lock); - hlist_for_each_entry_rcu_bh(existing_entry, - index_bucket(table, entry->index), - index_hash) { - if (existing_entry->index == entry->index) { - spin_unlock_bh(&table->lock); - /* If it was stolen, we start over. */ - goto search_unused_slot; - } - } - /* Otherwise, we know we have it exclusively (since we're locked), - * so we insert. - */ - hlist_add_head_rcu(&entry->index_hash, - index_bucket(table, entry->index)); - spin_unlock_bh(&table->lock); - - rcu_read_unlock_bh(); - - return entry->index; -} - -bool wg_index_hashtable_replace(struct index_hashtable *table, - struct index_hashtable_entry *old, - struct index_hashtable_entry *new) -{ - if (unlikely(hlist_unhashed(&old->index_hash))) - return false; - spin_lock_bh(&table->lock); - new->index = old->index; - hlist_replace_rcu(&old->index_hash, &new->index_hash); - - /* Calling init here NULLs out index_hash, and in fact after this - * function returns, it's theoretically possible for this to get - * reinserted elsewhere. That means the RCU lookup below might either - * terminate early or jump between buckets, in which case the packet - * simply gets dropped, which isn't terrible. - */ - INIT_HLIST_NODE(&old->index_hash); - spin_unlock_bh(&table->lock); - return true; -} - -void wg_index_hashtable_remove(struct index_hashtable *table, - struct index_hashtable_entry *entry) -{ - spin_lock_bh(&table->lock); - hlist_del_init_rcu(&entry->index_hash); - spin_unlock_bh(&table->lock); -} - -/* Returns a strong reference to a entry->peer */ -struct index_hashtable_entry * -wg_index_hashtable_lookup(struct index_hashtable *table, - const enum index_hashtable_type type_mask, - const __le32 index, struct wg_peer **peer) -{ - struct index_hashtable_entry *iter_entry, *entry = NULL; - - rcu_read_lock_bh(); - hlist_for_each_entry_rcu_bh(iter_entry, index_bucket(table, index), - index_hash) { - if (iter_entry->index == index) { - if (likely(iter_entry->type & type_mask)) - entry = iter_entry; - break; - } - } - if (likely(entry)) { - entry->peer = wg_peer_get_maybe_zero(entry->peer); - if (likely(entry->peer)) - *peer = entry->peer; - else - entry = NULL; - } - rcu_read_unlock_bh(); - return entry; -} diff --git a/drivers/net/wireguard/peerlookup.h b/drivers/net/wireguard/peerlookup.h deleted file mode 100644 index ced8117976800..0000000000000 --- a/drivers/net/wireguard/peerlookup.h +++ /dev/null @@ -1,64 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#ifndef _WG_PEERLOOKUP_H -#define _WG_PEERLOOKUP_H - -#include "messages.h" - -#include -#include -#include - -struct wg_peer; - -struct pubkey_hashtable { - /* TODO: move to rhashtable */ - DECLARE_HASHTABLE(hashtable, 11); - siphash_key_t key; - struct mutex lock; -}; - -struct pubkey_hashtable *wg_pubkey_hashtable_alloc(void); -void wg_pubkey_hashtable_add(struct pubkey_hashtable *table, - struct wg_peer *peer); -void wg_pubkey_hashtable_remove(struct pubkey_hashtable *table, - struct wg_peer *peer); -struct wg_peer * -wg_pubkey_hashtable_lookup(struct pubkey_hashtable *table, - const u8 pubkey[NOISE_PUBLIC_KEY_LEN]); - -struct index_hashtable { - /* TODO: move to rhashtable */ - DECLARE_HASHTABLE(hashtable, 13); - spinlock_t lock; -}; - -enum index_hashtable_type { - INDEX_HASHTABLE_HANDSHAKE = 1U << 0, - INDEX_HASHTABLE_KEYPAIR = 1U << 1 -}; - -struct index_hashtable_entry { - struct wg_peer *peer; - struct hlist_node index_hash; - enum index_hashtable_type type; - __le32 index; -}; - -struct index_hashtable *wg_index_hashtable_alloc(void); -__le32 wg_index_hashtable_insert(struct index_hashtable *table, - struct index_hashtable_entry *entry); -bool wg_index_hashtable_replace(struct index_hashtable *table, - struct index_hashtable_entry *old, - struct index_hashtable_entry *new); -void wg_index_hashtable_remove(struct index_hashtable *table, - struct index_hashtable_entry *entry); -struct index_hashtable_entry * -wg_index_hashtable_lookup(struct index_hashtable *table, - const enum index_hashtable_type type_mask, - const __le32 index, struct wg_peer **peer); - -#endif /* _WG_PEERLOOKUP_H */ diff --git a/drivers/net/wireguard/queueing.c b/drivers/net/wireguard/queueing.c deleted file mode 100644 index 5c964fcb994ec..0000000000000 --- a/drivers/net/wireguard/queueing.c +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include "queueing.h" - -struct multicore_worker __percpu * -wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr) -{ - int cpu; - struct multicore_worker __percpu *worker = - alloc_percpu(struct multicore_worker); - - if (!worker) - return NULL; - - for_each_possible_cpu(cpu) { - per_cpu_ptr(worker, cpu)->ptr = ptr; - INIT_WORK(&per_cpu_ptr(worker, cpu)->work, function); - } - return worker; -} - -int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, - bool multicore, unsigned int len) -{ - int ret; - - memset(queue, 0, sizeof(*queue)); - ret = ptr_ring_init(&queue->ring, len, GFP_KERNEL); - if (ret) - return ret; - if (function) { - if (multicore) { - queue->worker = wg_packet_percpu_multicore_worker_alloc( - function, queue); - if (!queue->worker) - return -ENOMEM; - } else { - INIT_WORK(&queue->work, function); - } - } - return 0; -} - -void wg_packet_queue_free(struct crypt_queue *queue, bool multicore) -{ - if (multicore) - free_percpu(queue->worker); - WARN_ON(!__ptr_ring_empty(&queue->ring)); - ptr_ring_cleanup(&queue->ring, NULL); -} diff --git a/drivers/net/wireguard/queueing.h b/drivers/net/wireguard/queueing.h deleted file mode 100644 index cf1e0e2376d81..0000000000000 --- a/drivers/net/wireguard/queueing.h +++ /dev/null @@ -1,200 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#ifndef _WG_QUEUEING_H -#define _WG_QUEUEING_H - -#include "peer.h" -#include -#include -#include -#include - -struct wg_device; -struct wg_peer; -struct multicore_worker; -struct crypt_queue; -struct sk_buff; - -/* queueing.c APIs: */ -int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, - bool multicore, unsigned int len); -void wg_packet_queue_free(struct crypt_queue *queue, bool multicore); -struct multicore_worker __percpu * -wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr); - -/* receive.c APIs: */ -void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb); -void wg_packet_handshake_receive_worker(struct work_struct *work); -/* NAPI poll function: */ -int wg_packet_rx_poll(struct napi_struct *napi, int budget); -/* Workqueue worker: */ -void wg_packet_decrypt_worker(struct work_struct *work); - -/* send.c APIs: */ -void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, - bool is_retry); -void wg_packet_send_handshake_response(struct wg_peer *peer); -void wg_packet_send_handshake_cookie(struct wg_device *wg, - struct sk_buff *initiating_skb, - __le32 sender_index); -void wg_packet_send_keepalive(struct wg_peer *peer); -void wg_packet_purge_staged_packets(struct wg_peer *peer); -void wg_packet_send_staged_packets(struct wg_peer *peer); -/* Workqueue workers: */ -void wg_packet_handshake_send_worker(struct work_struct *work); -void wg_packet_tx_worker(struct work_struct *work); -void wg_packet_encrypt_worker(struct work_struct *work); - -enum packet_state { - PACKET_STATE_UNCRYPTED, - PACKET_STATE_CRYPTED, - PACKET_STATE_DEAD -}; - -struct packet_cb { - u64 nonce; - struct noise_keypair *keypair; - atomic_t state; - u32 mtu; - u8 ds; -}; - -#define PACKET_CB(skb) ((struct packet_cb *)((skb)->cb)) -#define PACKET_PEER(skb) (PACKET_CB(skb)->keypair->entry.peer) - -/* Returns either the correct skb->protocol value, or 0 if invalid. */ -static inline __be16 wg_examine_packet_protocol(struct sk_buff *skb) -{ - if (skb_network_header(skb) >= skb->head && - (skb_network_header(skb) + sizeof(struct iphdr)) <= - skb_tail_pointer(skb) && - ip_hdr(skb)->version == 4) - return htons(ETH_P_IP); - if (skb_network_header(skb) >= skb->head && - (skb_network_header(skb) + sizeof(struct ipv6hdr)) <= - skb_tail_pointer(skb) && - ipv6_hdr(skb)->version == 6) - return htons(ETH_P_IPV6); - return 0; -} - -static inline bool wg_check_packet_protocol(struct sk_buff *skb) -{ - __be16 real_protocol = wg_examine_packet_protocol(skb); - return real_protocol && skb->protocol == real_protocol; -} - -static inline void wg_reset_packet(struct sk_buff *skb) -{ - skb_scrub_packet(skb, true); - memset(&skb->headers_start, 0, - offsetof(struct sk_buff, headers_end) - - offsetof(struct sk_buff, headers_start)); - skb->queue_mapping = 0; - skb->nohdr = 0; - skb->peeked = 0; - skb->mac_len = 0; - skb->dev = NULL; -#ifdef CONFIG_NET_SCHED - skb->tc_index = 0; - skb_reset_tc(skb); -#endif - skb->hdr_len = skb_headroom(skb); - skb_reset_mac_header(skb); - skb_reset_network_header(skb); - skb_reset_transport_header(skb); - skb_probe_transport_header(skb); - skb_reset_inner_headers(skb); -} - -static inline int wg_cpumask_choose_online(int *stored_cpu, unsigned int id) -{ - unsigned int cpu = *stored_cpu, cpu_index, i; - - if (unlikely(cpu == nr_cpumask_bits || - !cpumask_test_cpu(cpu, cpu_online_mask))) { - cpu_index = id % cpumask_weight(cpu_online_mask); - cpu = cpumask_first(cpu_online_mask); - for (i = 0; i < cpu_index; ++i) - cpu = cpumask_next(cpu, cpu_online_mask); - *stored_cpu = cpu; - } - return cpu; -} - -/* This function is racy, in the sense that next is unlocked, so it could return - * the same CPU twice. A race-free version of this would be to instead store an - * atomic sequence number, do an increment-and-return, and then iterate through - * every possible CPU until we get to that index -- choose_cpu. However that's - * a bit slower, and it doesn't seem like this potential race actually - * introduces any performance loss, so we live with it. - */ -static inline int wg_cpumask_next_online(int *next) -{ - int cpu = *next; - - while (unlikely(!cpumask_test_cpu(cpu, cpu_online_mask))) - cpu = cpumask_next(cpu, cpu_online_mask) % nr_cpumask_bits; - *next = cpumask_next(cpu, cpu_online_mask) % nr_cpumask_bits; - return cpu; -} - -static inline int wg_queue_enqueue_per_device_and_peer( - struct crypt_queue *device_queue, struct crypt_queue *peer_queue, - struct sk_buff *skb, struct workqueue_struct *wq, int *next_cpu) -{ - int cpu; - - atomic_set_release(&PACKET_CB(skb)->state, PACKET_STATE_UNCRYPTED); - /* We first queue this up for the peer ingestion, but the consumer - * will wait for the state to change to CRYPTED or DEAD before. - */ - if (unlikely(ptr_ring_produce_bh(&peer_queue->ring, skb))) - return -ENOSPC; - /* Then we queue it up in the device queue, which consumes the - * packet as soon as it can. - */ - cpu = wg_cpumask_next_online(next_cpu); - if (unlikely(ptr_ring_produce_bh(&device_queue->ring, skb))) - return -EPIPE; - queue_work_on(cpu, wq, &per_cpu_ptr(device_queue->worker, cpu)->work); - return 0; -} - -static inline void wg_queue_enqueue_per_peer(struct crypt_queue *queue, - struct sk_buff *skb, - enum packet_state state) -{ - /* We take a reference, because as soon as we call atomic_set, the - * peer can be freed from below us. - */ - struct wg_peer *peer = wg_peer_get(PACKET_PEER(skb)); - - atomic_set_release(&PACKET_CB(skb)->state, state); - queue_work_on(wg_cpumask_choose_online(&peer->serial_work_cpu, - peer->internal_id), - peer->device->packet_crypt_wq, &queue->work); - wg_peer_put(peer); -} - -static inline void wg_queue_enqueue_per_peer_napi(struct sk_buff *skb, - enum packet_state state) -{ - /* We take a reference, because as soon as we call atomic_set, the - * peer can be freed from below us. - */ - struct wg_peer *peer = wg_peer_get(PACKET_PEER(skb)); - - atomic_set_release(&PACKET_CB(skb)->state, state); - napi_schedule(&peer->napi); - wg_peer_put(peer); -} - -#ifdef DEBUG -bool wg_packet_counter_selftest(void); -#endif - -#endif /* _WG_QUEUEING_H */ diff --git a/drivers/net/wireguard/ratelimiter.c b/drivers/net/wireguard/ratelimiter.c deleted file mode 100644 index 3fedd1d21f5ee..0000000000000 --- a/drivers/net/wireguard/ratelimiter.c +++ /dev/null @@ -1,223 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include "ratelimiter.h" -#include -#include -#include -#include - -static struct kmem_cache *entry_cache; -static hsiphash_key_t key; -static spinlock_t table_lock = __SPIN_LOCK_UNLOCKED("ratelimiter_table_lock"); -static DEFINE_MUTEX(init_lock); -static u64 init_refcnt; /* Protected by init_lock, hence not atomic. */ -static atomic_t total_entries = ATOMIC_INIT(0); -static unsigned int max_entries, table_size; -static void wg_ratelimiter_gc_entries(struct work_struct *); -static DECLARE_DEFERRABLE_WORK(gc_work, wg_ratelimiter_gc_entries); -static struct hlist_head *table_v4; -#if IS_ENABLED(CONFIG_IPV6) -static struct hlist_head *table_v6; -#endif - -struct ratelimiter_entry { - u64 last_time_ns, tokens, ip; - void *net; - spinlock_t lock; - struct hlist_node hash; - struct rcu_head rcu; -}; - -enum { - PACKETS_PER_SECOND = 20, - PACKETS_BURSTABLE = 5, - PACKET_COST = NSEC_PER_SEC / PACKETS_PER_SECOND, - TOKEN_MAX = PACKET_COST * PACKETS_BURSTABLE -}; - -static void entry_free(struct rcu_head *rcu) -{ - kmem_cache_free(entry_cache, - container_of(rcu, struct ratelimiter_entry, rcu)); - atomic_dec(&total_entries); -} - -static void entry_uninit(struct ratelimiter_entry *entry) -{ - hlist_del_rcu(&entry->hash); - call_rcu(&entry->rcu, entry_free); -} - -/* Calling this function with a NULL work uninits all entries. */ -static void wg_ratelimiter_gc_entries(struct work_struct *work) -{ - const u64 now = ktime_get_coarse_boottime_ns(); - struct ratelimiter_entry *entry; - struct hlist_node *temp; - unsigned int i; - - for (i = 0; i < table_size; ++i) { - spin_lock(&table_lock); - hlist_for_each_entry_safe(entry, temp, &table_v4[i], hash) { - if (unlikely(!work) || - now - entry->last_time_ns > NSEC_PER_SEC) - entry_uninit(entry); - } -#if IS_ENABLED(CONFIG_IPV6) - hlist_for_each_entry_safe(entry, temp, &table_v6[i], hash) { - if (unlikely(!work) || - now - entry->last_time_ns > NSEC_PER_SEC) - entry_uninit(entry); - } -#endif - spin_unlock(&table_lock); - if (likely(work)) - cond_resched(); - } - if (likely(work)) - queue_delayed_work(system_power_efficient_wq, &gc_work, HZ); -} - -bool wg_ratelimiter_allow(struct sk_buff *skb, struct net *net) -{ - /* We only take the bottom half of the net pointer, so that we can hash - * 3 words in the end. This way, siphash's len param fits into the final - * u32, and we don't incur an extra round. - */ - const u32 net_word = (unsigned long)net; - struct ratelimiter_entry *entry; - struct hlist_head *bucket; - u64 ip; - - if (skb->protocol == htons(ETH_P_IP)) { - ip = (u64 __force)ip_hdr(skb)->saddr; - bucket = &table_v4[hsiphash_2u32(net_word, ip, &key) & - (table_size - 1)]; - } -#if IS_ENABLED(CONFIG_IPV6) - else if (skb->protocol == htons(ETH_P_IPV6)) { - /* Only use 64 bits, so as to ratelimit the whole /64. */ - memcpy(&ip, &ipv6_hdr(skb)->saddr, sizeof(ip)); - bucket = &table_v6[hsiphash_3u32(net_word, ip >> 32, ip, &key) & - (table_size - 1)]; - } -#endif - else - return false; - rcu_read_lock(); - hlist_for_each_entry_rcu(entry, bucket, hash) { - if (entry->net == net && entry->ip == ip) { - u64 now, tokens; - bool ret; - /* Quasi-inspired by nft_limit.c, but this is actually a - * slightly different algorithm. Namely, we incorporate - * the burst as part of the maximum tokens, rather than - * as part of the rate. - */ - spin_lock(&entry->lock); - now = ktime_get_coarse_boottime_ns(); - tokens = min_t(u64, TOKEN_MAX, - entry->tokens + now - - entry->last_time_ns); - entry->last_time_ns = now; - ret = tokens >= PACKET_COST; - entry->tokens = ret ? tokens - PACKET_COST : tokens; - spin_unlock(&entry->lock); - rcu_read_unlock(); - return ret; - } - } - rcu_read_unlock(); - - if (atomic_inc_return(&total_entries) > max_entries) - goto err_oom; - - entry = kmem_cache_alloc(entry_cache, GFP_KERNEL); - if (unlikely(!entry)) - goto err_oom; - - entry->net = net; - entry->ip = ip; - INIT_HLIST_NODE(&entry->hash); - spin_lock_init(&entry->lock); - entry->last_time_ns = ktime_get_coarse_boottime_ns(); - entry->tokens = TOKEN_MAX - PACKET_COST; - spin_lock(&table_lock); - hlist_add_head_rcu(&entry->hash, bucket); - spin_unlock(&table_lock); - return true; - -err_oom: - atomic_dec(&total_entries); - return false; -} - -int wg_ratelimiter_init(void) -{ - mutex_lock(&init_lock); - if (++init_refcnt != 1) - goto out; - - entry_cache = KMEM_CACHE(ratelimiter_entry, 0); - if (!entry_cache) - goto err; - - /* xt_hashlimit.c uses a slightly different algorithm for ratelimiting, - * but what it shares in common is that it uses a massive hashtable. So, - * we borrow their wisdom about good table sizes on different systems - * dependent on RAM. This calculation here comes from there. - */ - table_size = (totalram_pages() > (1U << 30) / PAGE_SIZE) ? 8192 : - max_t(unsigned long, 16, roundup_pow_of_two( - (totalram_pages() << PAGE_SHIFT) / - (1U << 14) / sizeof(struct hlist_head))); - max_entries = table_size * 8; - - table_v4 = kvzalloc(table_size * sizeof(*table_v4), GFP_KERNEL); - if (unlikely(!table_v4)) - goto err_kmemcache; - -#if IS_ENABLED(CONFIG_IPV6) - table_v6 = kvzalloc(table_size * sizeof(*table_v6), GFP_KERNEL); - if (unlikely(!table_v6)) { - kvfree(table_v4); - goto err_kmemcache; - } -#endif - - queue_delayed_work(system_power_efficient_wq, &gc_work, HZ); - get_random_bytes(&key, sizeof(key)); -out: - mutex_unlock(&init_lock); - return 0; - -err_kmemcache: - kmem_cache_destroy(entry_cache); -err: - --init_refcnt; - mutex_unlock(&init_lock); - return -ENOMEM; -} - -void wg_ratelimiter_uninit(void) -{ - mutex_lock(&init_lock); - if (!init_refcnt || --init_refcnt) - goto out; - - cancel_delayed_work_sync(&gc_work); - wg_ratelimiter_gc_entries(NULL); - rcu_barrier(); - kvfree(table_v4); -#if IS_ENABLED(CONFIG_IPV6) - kvfree(table_v6); -#endif - kmem_cache_destroy(entry_cache); -out: - mutex_unlock(&init_lock); -} - -#include "selftest/ratelimiter.c" diff --git a/drivers/net/wireguard/ratelimiter.h b/drivers/net/wireguard/ratelimiter.h deleted file mode 100644 index 83067f71ea99e..0000000000000 --- a/drivers/net/wireguard/ratelimiter.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#ifndef _WG_RATELIMITER_H -#define _WG_RATELIMITER_H - -#include - -int wg_ratelimiter_init(void); -void wg_ratelimiter_uninit(void); -bool wg_ratelimiter_allow(struct sk_buff *skb, struct net *net); - -#ifdef DEBUG -bool wg_ratelimiter_selftest(void); -#endif - -#endif /* _WG_RATELIMITER_H */ diff --git a/drivers/net/wireguard/receive.c b/drivers/net/wireguard/receive.c deleted file mode 100644 index da3b782ab7d31..0000000000000 --- a/drivers/net/wireguard/receive.c +++ /dev/null @@ -1,597 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include "queueing.h" -#include "device.h" -#include "peer.h" -#include "timers.h" -#include "messages.h" -#include "cookie.h" -#include "socket.h" - -#include -#include -#include -#include - -/* Must be called with bh disabled. */ -static void update_rx_stats(struct wg_peer *peer, size_t len) -{ - struct pcpu_sw_netstats *tstats = - get_cpu_ptr(peer->device->dev->tstats); - - u64_stats_update_begin(&tstats->syncp); - ++tstats->rx_packets; - tstats->rx_bytes += len; - peer->rx_bytes += len; - u64_stats_update_end(&tstats->syncp); - put_cpu_ptr(tstats); -} - -#define SKB_TYPE_LE32(skb) (((struct message_header *)(skb)->data)->type) - -static size_t validate_header_len(struct sk_buff *skb) -{ - if (unlikely(skb->len < sizeof(struct message_header))) - return 0; - if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_DATA) && - skb->len >= MESSAGE_MINIMUM_LENGTH) - return sizeof(struct message_data); - if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION) && - skb->len == sizeof(struct message_handshake_initiation)) - return sizeof(struct message_handshake_initiation); - if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) && - skb->len == sizeof(struct message_handshake_response)) - return sizeof(struct message_handshake_response); - if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE) && - skb->len == sizeof(struct message_handshake_cookie)) - return sizeof(struct message_handshake_cookie); - return 0; -} - -static int prepare_skb_header(struct sk_buff *skb, struct wg_device *wg) -{ - size_t data_offset, data_len, header_len; - struct udphdr *udp; - - if (unlikely(!wg_check_packet_protocol(skb) || - skb_transport_header(skb) < skb->head || - (skb_transport_header(skb) + sizeof(struct udphdr)) > - skb_tail_pointer(skb))) - return -EINVAL; /* Bogus IP header */ - udp = udp_hdr(skb); - data_offset = (u8 *)udp - skb->data; - if (unlikely(data_offset > U16_MAX || - data_offset + sizeof(struct udphdr) > skb->len)) - /* Packet has offset at impossible location or isn't big enough - * to have UDP fields. - */ - return -EINVAL; - data_len = ntohs(udp->len); - if (unlikely(data_len < sizeof(struct udphdr) || - data_len > skb->len - data_offset)) - /* UDP packet is reporting too small of a size or lying about - * its size. - */ - return -EINVAL; - data_len -= sizeof(struct udphdr); - data_offset = (u8 *)udp + sizeof(struct udphdr) - skb->data; - if (unlikely(!pskb_may_pull(skb, - data_offset + sizeof(struct message_header)) || - pskb_trim(skb, data_len + data_offset) < 0)) - return -EINVAL; - skb_pull(skb, data_offset); - if (unlikely(skb->len != data_len)) - /* Final len does not agree with calculated len */ - return -EINVAL; - header_len = validate_header_len(skb); - if (unlikely(!header_len)) - return -EINVAL; - __skb_push(skb, data_offset); - if (unlikely(!pskb_may_pull(skb, data_offset + header_len))) - return -EINVAL; - __skb_pull(skb, data_offset); - return 0; -} - -static void wg_receive_handshake_packet(struct wg_device *wg, - struct sk_buff *skb) -{ - enum cookie_mac_state mac_state; - struct wg_peer *peer = NULL; - /* This is global, so that our load calculation applies to the whole - * system. We don't care about races with it at all. - */ - static u64 last_under_load; - bool packet_needs_cookie; - bool under_load; - - if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE)) { - net_dbg_skb_ratelimited("%s: Receiving cookie response from %pISpfsc\n", - wg->dev->name, skb); - wg_cookie_message_consume( - (struct message_handshake_cookie *)skb->data, wg); - return; - } - - under_load = skb_queue_len(&wg->incoming_handshakes) >= - MAX_QUEUED_INCOMING_HANDSHAKES / 8; - if (under_load) { - last_under_load = ktime_get_coarse_boottime_ns(); - } else if (last_under_load) { - under_load = !wg_birthdate_has_expired(last_under_load, 1); - if (!under_load) - last_under_load = 0; - } - mac_state = wg_cookie_validate_packet(&wg->cookie_checker, skb, - under_load); - if ((under_load && mac_state == VALID_MAC_WITH_COOKIE) || - (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE)) { - packet_needs_cookie = false; - } else if (under_load && mac_state == VALID_MAC_BUT_NO_COOKIE) { - packet_needs_cookie = true; - } else { - net_dbg_skb_ratelimited("%s: Invalid MAC of handshake, dropping packet from %pISpfsc\n", - wg->dev->name, skb); - return; - } - - switch (SKB_TYPE_LE32(skb)) { - case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): { - struct message_handshake_initiation *message = - (struct message_handshake_initiation *)skb->data; - - if (packet_needs_cookie) { - wg_packet_send_handshake_cookie(wg, skb, - message->sender_index); - return; - } - peer = wg_noise_handshake_consume_initiation(message, wg); - if (unlikely(!peer)) { - net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n", - wg->dev->name, skb); - return; - } - wg_socket_set_peer_endpoint_from_skb(peer, skb); - net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n", - wg->dev->name, peer->internal_id, - &peer->endpoint.addr); - wg_packet_send_handshake_response(peer); - break; - } - case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): { - struct message_handshake_response *message = - (struct message_handshake_response *)skb->data; - - if (packet_needs_cookie) { - wg_packet_send_handshake_cookie(wg, skb, - message->sender_index); - return; - } - peer = wg_noise_handshake_consume_response(message, wg); - if (unlikely(!peer)) { - net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n", - wg->dev->name, skb); - return; - } - wg_socket_set_peer_endpoint_from_skb(peer, skb); - net_dbg_ratelimited("%s: Receiving handshake response from peer %llu (%pISpfsc)\n", - wg->dev->name, peer->internal_id, - &peer->endpoint.addr); - if (wg_noise_handshake_begin_session(&peer->handshake, - &peer->keypairs)) { - wg_timers_session_derived(peer); - wg_timers_handshake_complete(peer); - /* Calling this function will either send any existing - * packets in the queue and not send a keepalive, which - * is the best case, Or, if there's nothing in the - * queue, it will send a keepalive, in order to give - * immediate confirmation of the session. - */ - wg_packet_send_keepalive(peer); - } - break; - } - } - - if (unlikely(!peer)) { - WARN(1, "Somehow a wrong type of packet wound up in the handshake queue!\n"); - return; - } - - local_bh_disable(); - update_rx_stats(peer, skb->len); - local_bh_enable(); - - wg_timers_any_authenticated_packet_received(peer); - wg_timers_any_authenticated_packet_traversal(peer); - wg_peer_put(peer); -} - -void wg_packet_handshake_receive_worker(struct work_struct *work) -{ - struct wg_device *wg = container_of(work, struct multicore_worker, - work)->ptr; - struct sk_buff *skb; - - while ((skb = skb_dequeue(&wg->incoming_handshakes)) != NULL) { - wg_receive_handshake_packet(wg, skb); - dev_kfree_skb(skb); - cond_resched(); - } -} - -static void keep_key_fresh(struct wg_peer *peer) -{ - struct noise_keypair *keypair; - bool send = false; - - if (peer->sent_lastminute_handshake) - return; - - rcu_read_lock_bh(); - keypair = rcu_dereference_bh(peer->keypairs.current_keypair); - if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) && - keypair->i_am_the_initiator && - unlikely(wg_birthdate_has_expired(keypair->sending.birthdate, - REJECT_AFTER_TIME - KEEPALIVE_TIMEOUT - REKEY_TIMEOUT))) - send = true; - rcu_read_unlock_bh(); - - if (send) { - peer->sent_lastminute_handshake = true; - wg_packet_send_queued_handshake_initiation(peer, false); - } -} - -static bool decrypt_packet(struct sk_buff *skb, struct noise_symmetric_key *key) -{ - struct scatterlist sg[MAX_SKB_FRAGS + 8]; - struct sk_buff *trailer; - unsigned int offset; - int num_frags; - - if (unlikely(!key)) - return false; - - if (unlikely(!READ_ONCE(key->is_valid) || - wg_birthdate_has_expired(key->birthdate, REJECT_AFTER_TIME) || - key->counter.receive.counter >= REJECT_AFTER_MESSAGES)) { - WRITE_ONCE(key->is_valid, false); - return false; - } - - PACKET_CB(skb)->nonce = - le64_to_cpu(((struct message_data *)skb->data)->counter); - - /* We ensure that the network header is part of the packet before we - * call skb_cow_data, so that there's no chance that data is removed - * from the skb, so that later we can extract the original endpoint. - */ - offset = skb->data - skb_network_header(skb); - skb_push(skb, offset); - num_frags = skb_cow_data(skb, 0, &trailer); - offset += sizeof(struct message_data); - skb_pull(skb, offset); - if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) - return false; - - sg_init_table(sg, num_frags); - if (skb_to_sgvec(skb, sg, 0, skb->len) <= 0) - return false; - - if (!chacha20poly1305_decrypt_sg_inplace(sg, skb->len, NULL, 0, - PACKET_CB(skb)->nonce, - key->key)) - return false; - - /* Another ugly situation of pushing and pulling the header so as to - * keep endpoint information intact. - */ - skb_push(skb, offset); - if (pskb_trim(skb, skb->len - noise_encrypted_len(0))) - return false; - skb_pull(skb, offset); - - return true; -} - -/* This is RFC6479, a replay detection bitmap algorithm that avoids bitshifts */ -static bool counter_validate(union noise_counter *counter, u64 their_counter) -{ - unsigned long index, index_current, top, i; - bool ret = false; - - spin_lock_bh(&counter->receive.lock); - - if (unlikely(counter->receive.counter >= REJECT_AFTER_MESSAGES + 1 || - their_counter >= REJECT_AFTER_MESSAGES)) - goto out; - - ++their_counter; - - if (unlikely((COUNTER_WINDOW_SIZE + their_counter) < - counter->receive.counter)) - goto out; - - index = their_counter >> ilog2(BITS_PER_LONG); - - if (likely(their_counter > counter->receive.counter)) { - index_current = counter->receive.counter >> ilog2(BITS_PER_LONG); - top = min_t(unsigned long, index - index_current, - COUNTER_BITS_TOTAL / BITS_PER_LONG); - for (i = 1; i <= top; ++i) - counter->receive.backtrack[(i + index_current) & - ((COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1)] = 0; - counter->receive.counter = their_counter; - } - - index &= (COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1; - ret = !test_and_set_bit(their_counter & (BITS_PER_LONG - 1), - &counter->receive.backtrack[index]); - -out: - spin_unlock_bh(&counter->receive.lock); - return ret; -} - -#include "selftest/counter.c" - -static void wg_packet_consume_data_done(struct wg_peer *peer, - struct sk_buff *skb, - struct endpoint *endpoint) -{ - struct net_device *dev = peer->device->dev; - unsigned int len, len_before_trim; - struct wg_peer *routed_peer; - - wg_socket_set_peer_endpoint(peer, endpoint); - - if (unlikely(wg_noise_received_with_keypair(&peer->keypairs, - PACKET_CB(skb)->keypair))) { - wg_timers_handshake_complete(peer); - wg_packet_send_staged_packets(peer); - } - - keep_key_fresh(peer); - - wg_timers_any_authenticated_packet_received(peer); - wg_timers_any_authenticated_packet_traversal(peer); - - /* A packet with length 0 is a keepalive packet */ - if (unlikely(!skb->len)) { - update_rx_stats(peer, message_data_len(0)); - net_dbg_ratelimited("%s: Receiving keepalive packet from peer %llu (%pISpfsc)\n", - dev->name, peer->internal_id, - &peer->endpoint.addr); - goto packet_processed; - } - - wg_timers_data_received(peer); - - if (unlikely(skb_network_header(skb) < skb->head)) - goto dishonest_packet_size; - if (unlikely(!(pskb_network_may_pull(skb, sizeof(struct iphdr)) && - (ip_hdr(skb)->version == 4 || - (ip_hdr(skb)->version == 6 && - pskb_network_may_pull(skb, sizeof(struct ipv6hdr))))))) - goto dishonest_packet_type; - - skb->dev = dev; - /* We've already verified the Poly1305 auth tag, which means this packet - * was not modified in transit. We can therefore tell the networking - * stack that all checksums of every layer of encapsulation have already - * been checked "by the hardware" and therefore is unnecessary to check - * again in software. - */ - skb->ip_summed = CHECKSUM_UNNECESSARY; - skb->csum_level = ~0; /* All levels */ - skb->protocol = wg_examine_packet_protocol(skb); - if (skb->protocol == htons(ETH_P_IP)) { - len = ntohs(ip_hdr(skb)->tot_len); - if (unlikely(len < sizeof(struct iphdr))) - goto dishonest_packet_size; - if (INET_ECN_is_ce(PACKET_CB(skb)->ds)) - IP_ECN_set_ce(ip_hdr(skb)); - } else if (skb->protocol == htons(ETH_P_IPV6)) { - len = ntohs(ipv6_hdr(skb)->payload_len) + - sizeof(struct ipv6hdr); - if (INET_ECN_is_ce(PACKET_CB(skb)->ds)) - IP6_ECN_set_ce(skb, ipv6_hdr(skb)); - } else { - goto dishonest_packet_type; - } - - if (unlikely(len > skb->len)) - goto dishonest_packet_size; - len_before_trim = skb->len; - if (unlikely(pskb_trim(skb, len))) - goto packet_processed; - - routed_peer = wg_allowedips_lookup_src(&peer->device->peer_allowedips, - skb); - wg_peer_put(routed_peer); /* We don't need the extra reference. */ - - if (unlikely(routed_peer != peer)) - goto dishonest_packet_peer; - - if (unlikely(napi_gro_receive(&peer->napi, skb) == GRO_DROP)) { - ++dev->stats.rx_dropped; - net_dbg_ratelimited("%s: Failed to give packet to userspace from peer %llu (%pISpfsc)\n", - dev->name, peer->internal_id, - &peer->endpoint.addr); - } else { - update_rx_stats(peer, message_data_len(len_before_trim)); - } - return; - -dishonest_packet_peer: - net_dbg_skb_ratelimited("%s: Packet has unallowed src IP (%pISc) from peer %llu (%pISpfsc)\n", - dev->name, skb, peer->internal_id, - &peer->endpoint.addr); - ++dev->stats.rx_errors; - ++dev->stats.rx_frame_errors; - goto packet_processed; -dishonest_packet_type: - net_dbg_ratelimited("%s: Packet is neither ipv4 nor ipv6 from peer %llu (%pISpfsc)\n", - dev->name, peer->internal_id, &peer->endpoint.addr); - ++dev->stats.rx_errors; - ++dev->stats.rx_frame_errors; - goto packet_processed; -dishonest_packet_size: - net_dbg_ratelimited("%s: Packet has incorrect size from peer %llu (%pISpfsc)\n", - dev->name, peer->internal_id, &peer->endpoint.addr); - ++dev->stats.rx_errors; - ++dev->stats.rx_length_errors; - goto packet_processed; -packet_processed: - dev_kfree_skb(skb); -} - -int wg_packet_rx_poll(struct napi_struct *napi, int budget) -{ - struct wg_peer *peer = container_of(napi, struct wg_peer, napi); - struct crypt_queue *queue = &peer->rx_queue; - struct noise_keypair *keypair; - struct endpoint endpoint; - enum packet_state state; - struct sk_buff *skb; - int work_done = 0; - bool free; - - if (unlikely(budget <= 0)) - return 0; - - while ((skb = __ptr_ring_peek(&queue->ring)) != NULL && - (state = atomic_read_acquire(&PACKET_CB(skb)->state)) != - PACKET_STATE_UNCRYPTED) { - __ptr_ring_discard_one(&queue->ring); - peer = PACKET_PEER(skb); - keypair = PACKET_CB(skb)->keypair; - free = true; - - if (unlikely(state != PACKET_STATE_CRYPTED)) - goto next; - - if (unlikely(!counter_validate(&keypair->receiving.counter, - PACKET_CB(skb)->nonce))) { - net_dbg_ratelimited("%s: Packet has invalid nonce %llu (max %llu)\n", - peer->device->dev->name, - PACKET_CB(skb)->nonce, - keypair->receiving.counter.receive.counter); - goto next; - } - - if (unlikely(wg_socket_endpoint_from_skb(&endpoint, skb))) - goto next; - - wg_reset_packet(skb); - wg_packet_consume_data_done(peer, skb, &endpoint); - free = false; - -next: - wg_noise_keypair_put(keypair, false); - wg_peer_put(peer); - if (unlikely(free)) - dev_kfree_skb(skb); - - if (++work_done >= budget) - break; - } - - if (work_done < budget) - napi_complete_done(napi, work_done); - - return work_done; -} - -void wg_packet_decrypt_worker(struct work_struct *work) -{ - struct crypt_queue *queue = container_of(work, struct multicore_worker, - work)->ptr; - struct sk_buff *skb; - - while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) { - enum packet_state state = likely(decrypt_packet(skb, - &PACKET_CB(skb)->keypair->receiving)) ? - PACKET_STATE_CRYPTED : PACKET_STATE_DEAD; - wg_queue_enqueue_per_peer_napi(skb, state); - } -} - -static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb) -{ - __le32 idx = ((struct message_data *)skb->data)->key_idx; - struct wg_peer *peer = NULL; - int ret; - - rcu_read_lock_bh(); - PACKET_CB(skb)->keypair = - (struct noise_keypair *)wg_index_hashtable_lookup( - wg->index_hashtable, INDEX_HASHTABLE_KEYPAIR, idx, - &peer); - if (unlikely(!wg_noise_keypair_get(PACKET_CB(skb)->keypair))) - goto err_keypair; - - if (unlikely(READ_ONCE(peer->is_dead))) - goto err; - - ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, - &peer->rx_queue, skb, - wg->packet_crypt_wq, - &wg->decrypt_queue.last_cpu); - if (unlikely(ret == -EPIPE)) - wg_queue_enqueue_per_peer_napi(skb, PACKET_STATE_DEAD); - if (likely(!ret || ret == -EPIPE)) { - rcu_read_unlock_bh(); - return; - } -err: - wg_noise_keypair_put(PACKET_CB(skb)->keypair, false); -err_keypair: - rcu_read_unlock_bh(); - wg_peer_put(peer); - dev_kfree_skb(skb); -} - -void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb) -{ - if (unlikely(prepare_skb_header(skb, wg) < 0)) - goto err; - switch (SKB_TYPE_LE32(skb)) { - case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): - case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): - case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): { - int cpu; - - if (skb_queue_len(&wg->incoming_handshakes) > - MAX_QUEUED_INCOMING_HANDSHAKES || - unlikely(!rng_is_initialized())) { - net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n", - wg->dev->name, skb); - goto err; - } - skb_queue_tail(&wg->incoming_handshakes, skb); - /* Queues up a call to packet_process_queued_handshake_ - * packets(skb): - */ - cpu = wg_cpumask_next_online(&wg->incoming_handshake_cpu); - queue_work_on(cpu, wg->handshake_receive_wq, - &per_cpu_ptr(wg->incoming_handshakes_worker, cpu)->work); - break; - } - case cpu_to_le32(MESSAGE_DATA): - PACKET_CB(skb)->ds = ip_tunnel_get_dsfield(ip_hdr(skb), skb); - wg_packet_consume_data(wg, skb); - break; - default: - WARN(1, "Non-exhaustive parsing of packet header lead to unknown packet type!\n"); - goto err; - } - return; - -err: - dev_kfree_skb(skb); -} diff --git a/drivers/net/wireguard/selftest/allowedips.c b/drivers/net/wireguard/selftest/allowedips.c deleted file mode 100644 index 846db14cb046b..0000000000000 --- a/drivers/net/wireguard/selftest/allowedips.c +++ /dev/null @@ -1,683 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - * - * This contains some basic static unit tests for the allowedips data structure. - * It also has two additional modes that are disabled and meant to be used by - * folks directly playing with this file. If you define the macro - * DEBUG_PRINT_TRIE_GRAPHVIZ to be 1, then every time there's a full tree in - * memory, it will be printed out as KERN_DEBUG in a format that can be passed - * to graphviz (the dot command) to visualize it. If you define the macro - * DEBUG_RANDOM_TRIE to be 1, then there will be an extremely costly set of - * randomized tests done against a trivial implementation, which may take - * upwards of a half-hour to complete. There's no set of users who should be - * enabling these, and the only developers that should go anywhere near these - * nobs are the ones who are reading this comment. - */ - -#ifdef DEBUG - -#include - -static __init void swap_endian_and_apply_cidr(u8 *dst, const u8 *src, u8 bits, - u8 cidr) -{ - swap_endian(dst, src, bits); - memset(dst + (cidr + 7) / 8, 0, bits / 8 - (cidr + 7) / 8); - if (cidr) - dst[(cidr + 7) / 8 - 1] &= ~0U << ((8 - (cidr % 8)) % 8); -} - -static __init void print_node(struct allowedips_node *node, u8 bits) -{ - char *fmt_connection = KERN_DEBUG "\t\"%p/%d\" -> \"%p/%d\";\n"; - char *fmt_declaration = KERN_DEBUG - "\t\"%p/%d\"[style=%s, color=\"#%06x\"];\n"; - char *style = "dotted"; - u8 ip1[16], ip2[16]; - u32 color = 0; - - if (bits == 32) { - fmt_connection = KERN_DEBUG "\t\"%pI4/%d\" -> \"%pI4/%d\";\n"; - fmt_declaration = KERN_DEBUG - "\t\"%pI4/%d\"[style=%s, color=\"#%06x\"];\n"; - } else if (bits == 128) { - fmt_connection = KERN_DEBUG "\t\"%pI6/%d\" -> \"%pI6/%d\";\n"; - fmt_declaration = KERN_DEBUG - "\t\"%pI6/%d\"[style=%s, color=\"#%06x\"];\n"; - } - if (node->peer) { - hsiphash_key_t key = { { 0 } }; - - memcpy(&key, &node->peer, sizeof(node->peer)); - color = hsiphash_1u32(0xdeadbeef, &key) % 200 << 16 | - hsiphash_1u32(0xbabecafe, &key) % 200 << 8 | - hsiphash_1u32(0xabad1dea, &key) % 200; - style = "bold"; - } - swap_endian_and_apply_cidr(ip1, node->bits, bits, node->cidr); - printk(fmt_declaration, ip1, node->cidr, style, color); - if (node->bit[0]) { - swap_endian_and_apply_cidr(ip2, - rcu_dereference_raw(node->bit[0])->bits, bits, - node->cidr); - printk(fmt_connection, ip1, node->cidr, ip2, - rcu_dereference_raw(node->bit[0])->cidr); - print_node(rcu_dereference_raw(node->bit[0]), bits); - } - if (node->bit[1]) { - swap_endian_and_apply_cidr(ip2, - rcu_dereference_raw(node->bit[1])->bits, - bits, node->cidr); - printk(fmt_connection, ip1, node->cidr, ip2, - rcu_dereference_raw(node->bit[1])->cidr); - print_node(rcu_dereference_raw(node->bit[1]), bits); - } -} - -static __init void print_tree(struct allowedips_node __rcu *top, u8 bits) -{ - printk(KERN_DEBUG "digraph trie {\n"); - print_node(rcu_dereference_raw(top), bits); - printk(KERN_DEBUG "}\n"); -} - -enum { - NUM_PEERS = 2000, - NUM_RAND_ROUTES = 400, - NUM_MUTATED_ROUTES = 100, - NUM_QUERIES = NUM_RAND_ROUTES * NUM_MUTATED_ROUTES * 30 -}; - -struct horrible_allowedips { - struct hlist_head head; -}; - -struct horrible_allowedips_node { - struct hlist_node table; - union nf_inet_addr ip; - union nf_inet_addr mask; - u8 ip_version; - void *value; -}; - -static __init void horrible_allowedips_init(struct horrible_allowedips *table) -{ - INIT_HLIST_HEAD(&table->head); -} - -static __init void horrible_allowedips_free(struct horrible_allowedips *table) -{ - struct horrible_allowedips_node *node; - struct hlist_node *h; - - hlist_for_each_entry_safe(node, h, &table->head, table) { - hlist_del(&node->table); - kfree(node); - } -} - -static __init inline union nf_inet_addr horrible_cidr_to_mask(u8 cidr) -{ - union nf_inet_addr mask; - - memset(&mask, 0x00, 128 / 8); - memset(&mask, 0xff, cidr / 8); - if (cidr % 32) - mask.all[cidr / 32] = (__force u32)htonl( - (0xFFFFFFFFUL << (32 - (cidr % 32))) & 0xFFFFFFFFUL); - return mask; -} - -static __init inline u8 horrible_mask_to_cidr(union nf_inet_addr subnet) -{ - return hweight32(subnet.all[0]) + hweight32(subnet.all[1]) + - hweight32(subnet.all[2]) + hweight32(subnet.all[3]); -} - -static __init inline void -horrible_mask_self(struct horrible_allowedips_node *node) -{ - if (node->ip_version == 4) { - node->ip.ip &= node->mask.ip; - } else if (node->ip_version == 6) { - node->ip.ip6[0] &= node->mask.ip6[0]; - node->ip.ip6[1] &= node->mask.ip6[1]; - node->ip.ip6[2] &= node->mask.ip6[2]; - node->ip.ip6[3] &= node->mask.ip6[3]; - } -} - -static __init inline bool -horrible_match_v4(const struct horrible_allowedips_node *node, - struct in_addr *ip) -{ - return (ip->s_addr & node->mask.ip) == node->ip.ip; -} - -static __init inline bool -horrible_match_v6(const struct horrible_allowedips_node *node, - struct in6_addr *ip) -{ - return (ip->in6_u.u6_addr32[0] & node->mask.ip6[0]) == - node->ip.ip6[0] && - (ip->in6_u.u6_addr32[1] & node->mask.ip6[1]) == - node->ip.ip6[1] && - (ip->in6_u.u6_addr32[2] & node->mask.ip6[2]) == - node->ip.ip6[2] && - (ip->in6_u.u6_addr32[3] & node->mask.ip6[3]) == node->ip.ip6[3]; -} - -static __init void -horrible_insert_ordered(struct horrible_allowedips *table, - struct horrible_allowedips_node *node) -{ - struct horrible_allowedips_node *other = NULL, *where = NULL; - u8 my_cidr = horrible_mask_to_cidr(node->mask); - - hlist_for_each_entry(other, &table->head, table) { - if (!memcmp(&other->mask, &node->mask, - sizeof(union nf_inet_addr)) && - !memcmp(&other->ip, &node->ip, - sizeof(union nf_inet_addr)) && - other->ip_version == node->ip_version) { - other->value = node->value; - kfree(node); - return; - } - where = other; - if (horrible_mask_to_cidr(other->mask) <= my_cidr) - break; - } - if (!other && !where) - hlist_add_head(&node->table, &table->head); - else if (!other) - hlist_add_behind(&node->table, &where->table); - else - hlist_add_before(&node->table, &where->table); -} - -static __init int -horrible_allowedips_insert_v4(struct horrible_allowedips *table, - struct in_addr *ip, u8 cidr, void *value) -{ - struct horrible_allowedips_node *node = kzalloc(sizeof(*node), - GFP_KERNEL); - - if (unlikely(!node)) - return -ENOMEM; - node->ip.in = *ip; - node->mask = horrible_cidr_to_mask(cidr); - node->ip_version = 4; - node->value = value; - horrible_mask_self(node); - horrible_insert_ordered(table, node); - return 0; -} - -static __init int -horrible_allowedips_insert_v6(struct horrible_allowedips *table, - struct in6_addr *ip, u8 cidr, void *value) -{ - struct horrible_allowedips_node *node = kzalloc(sizeof(*node), - GFP_KERNEL); - - if (unlikely(!node)) - return -ENOMEM; - node->ip.in6 = *ip; - node->mask = horrible_cidr_to_mask(cidr); - node->ip_version = 6; - node->value = value; - horrible_mask_self(node); - horrible_insert_ordered(table, node); - return 0; -} - -static __init void * -horrible_allowedips_lookup_v4(struct horrible_allowedips *table, - struct in_addr *ip) -{ - struct horrible_allowedips_node *node; - void *ret = NULL; - - hlist_for_each_entry(node, &table->head, table) { - if (node->ip_version != 4) - continue; - if (horrible_match_v4(node, ip)) { - ret = node->value; - break; - } - } - return ret; -} - -static __init void * -horrible_allowedips_lookup_v6(struct horrible_allowedips *table, - struct in6_addr *ip) -{ - struct horrible_allowedips_node *node; - void *ret = NULL; - - hlist_for_each_entry(node, &table->head, table) { - if (node->ip_version != 6) - continue; - if (horrible_match_v6(node, ip)) { - ret = node->value; - break; - } - } - return ret; -} - -static __init bool randomized_test(void) -{ - unsigned int i, j, k, mutate_amount, cidr; - u8 ip[16], mutate_mask[16], mutated[16]; - struct wg_peer **peers, *peer; - struct horrible_allowedips h; - DEFINE_MUTEX(mutex); - struct allowedips t; - bool ret = false; - - mutex_init(&mutex); - - wg_allowedips_init(&t); - horrible_allowedips_init(&h); - - peers = kcalloc(NUM_PEERS, sizeof(*peers), GFP_KERNEL); - if (unlikely(!peers)) { - pr_err("allowedips random self-test malloc: FAIL\n"); - goto free; - } - for (i = 0; i < NUM_PEERS; ++i) { - peers[i] = kzalloc(sizeof(*peers[i]), GFP_KERNEL); - if (unlikely(!peers[i])) { - pr_err("allowedips random self-test malloc: FAIL\n"); - goto free; - } - kref_init(&peers[i]->refcount); - } - - mutex_lock(&mutex); - - for (i = 0; i < NUM_RAND_ROUTES; ++i) { - prandom_bytes(ip, 4); - cidr = prandom_u32_max(32) + 1; - peer = peers[prandom_u32_max(NUM_PEERS)]; - if (wg_allowedips_insert_v4(&t, (struct in_addr *)ip, cidr, - peer, &mutex) < 0) { - pr_err("allowedips random self-test malloc: FAIL\n"); - goto free_locked; - } - if (horrible_allowedips_insert_v4(&h, (struct in_addr *)ip, - cidr, peer) < 0) { - pr_err("allowedips random self-test malloc: FAIL\n"); - goto free_locked; - } - for (j = 0; j < NUM_MUTATED_ROUTES; ++j) { - memcpy(mutated, ip, 4); - prandom_bytes(mutate_mask, 4); - mutate_amount = prandom_u32_max(32); - for (k = 0; k < mutate_amount / 8; ++k) - mutate_mask[k] = 0xff; - mutate_mask[k] = 0xff - << ((8 - (mutate_amount % 8)) % 8); - for (; k < 4; ++k) - mutate_mask[k] = 0; - for (k = 0; k < 4; ++k) - mutated[k] = (mutated[k] & mutate_mask[k]) | - (~mutate_mask[k] & - prandom_u32_max(256)); - cidr = prandom_u32_max(32) + 1; - peer = peers[prandom_u32_max(NUM_PEERS)]; - if (wg_allowedips_insert_v4(&t, - (struct in_addr *)mutated, - cidr, peer, &mutex) < 0) { - pr_err("allowedips random malloc: FAIL\n"); - goto free_locked; - } - if (horrible_allowedips_insert_v4(&h, - (struct in_addr *)mutated, cidr, peer)) { - pr_err("allowedips random self-test malloc: FAIL\n"); - goto free_locked; - } - } - } - - for (i = 0; i < NUM_RAND_ROUTES; ++i) { - prandom_bytes(ip, 16); - cidr = prandom_u32_max(128) + 1; - peer = peers[prandom_u32_max(NUM_PEERS)]; - if (wg_allowedips_insert_v6(&t, (struct in6_addr *)ip, cidr, - peer, &mutex) < 0) { - pr_err("allowedips random self-test malloc: FAIL\n"); - goto free_locked; - } - if (horrible_allowedips_insert_v6(&h, (struct in6_addr *)ip, - cidr, peer) < 0) { - pr_err("allowedips random self-test malloc: FAIL\n"); - goto free_locked; - } - for (j = 0; j < NUM_MUTATED_ROUTES; ++j) { - memcpy(mutated, ip, 16); - prandom_bytes(mutate_mask, 16); - mutate_amount = prandom_u32_max(128); - for (k = 0; k < mutate_amount / 8; ++k) - mutate_mask[k] = 0xff; - mutate_mask[k] = 0xff - << ((8 - (mutate_amount % 8)) % 8); - for (; k < 4; ++k) - mutate_mask[k] = 0; - for (k = 0; k < 4; ++k) - mutated[k] = (mutated[k] & mutate_mask[k]) | - (~mutate_mask[k] & - prandom_u32_max(256)); - cidr = prandom_u32_max(128) + 1; - peer = peers[prandom_u32_max(NUM_PEERS)]; - if (wg_allowedips_insert_v6(&t, - (struct in6_addr *)mutated, - cidr, peer, &mutex) < 0) { - pr_err("allowedips random self-test malloc: FAIL\n"); - goto free_locked; - } - if (horrible_allowedips_insert_v6( - &h, (struct in6_addr *)mutated, cidr, - peer)) { - pr_err("allowedips random self-test malloc: FAIL\n"); - goto free_locked; - } - } - } - - mutex_unlock(&mutex); - - if (IS_ENABLED(DEBUG_PRINT_TRIE_GRAPHVIZ)) { - print_tree(t.root4, 32); - print_tree(t.root6, 128); - } - - for (i = 0; i < NUM_QUERIES; ++i) { - prandom_bytes(ip, 4); - if (lookup(t.root4, 32, ip) != - horrible_allowedips_lookup_v4(&h, (struct in_addr *)ip)) { - pr_err("allowedips random self-test: FAIL\n"); - goto free; - } - } - - for (i = 0; i < NUM_QUERIES; ++i) { - prandom_bytes(ip, 16); - if (lookup(t.root6, 128, ip) != - horrible_allowedips_lookup_v6(&h, (struct in6_addr *)ip)) { - pr_err("allowedips random self-test: FAIL\n"); - goto free; - } - } - ret = true; - -free: - mutex_lock(&mutex); -free_locked: - wg_allowedips_free(&t, &mutex); - mutex_unlock(&mutex); - horrible_allowedips_free(&h); - if (peers) { - for (i = 0; i < NUM_PEERS; ++i) - kfree(peers[i]); - } - kfree(peers); - return ret; -} - -static __init inline struct in_addr *ip4(u8 a, u8 b, u8 c, u8 d) -{ - static struct in_addr ip; - u8 *split = (u8 *)&ip; - - split[0] = a; - split[1] = b; - split[2] = c; - split[3] = d; - return &ip; -} - -static __init inline struct in6_addr *ip6(u32 a, u32 b, u32 c, u32 d) -{ - static struct in6_addr ip; - __be32 *split = (__be32 *)&ip; - - split[0] = cpu_to_be32(a); - split[1] = cpu_to_be32(b); - split[2] = cpu_to_be32(c); - split[3] = cpu_to_be32(d); - return &ip; -} - -static __init struct wg_peer *init_peer(void) -{ - struct wg_peer *peer = kzalloc(sizeof(*peer), GFP_KERNEL); - - if (!peer) - return NULL; - kref_init(&peer->refcount); - INIT_LIST_HEAD(&peer->allowedips_list); - return peer; -} - -#define insert(version, mem, ipa, ipb, ipc, ipd, cidr) \ - wg_allowedips_insert_v##version(&t, ip##version(ipa, ipb, ipc, ipd), \ - cidr, mem, &mutex) - -#define maybe_fail() do { \ - ++i; \ - if (!_s) { \ - pr_info("allowedips self-test %zu: FAIL\n", i); \ - success = false; \ - } \ - } while (0) - -#define test(version, mem, ipa, ipb, ipc, ipd) do { \ - bool _s = lookup(t.root##version, (version) == 4 ? 32 : 128, \ - ip##version(ipa, ipb, ipc, ipd)) == (mem); \ - maybe_fail(); \ - } while (0) - -#define test_negative(version, mem, ipa, ipb, ipc, ipd) do { \ - bool _s = lookup(t.root##version, (version) == 4 ? 32 : 128, \ - ip##version(ipa, ipb, ipc, ipd)) != (mem); \ - maybe_fail(); \ - } while (0) - -#define test_boolean(cond) do { \ - bool _s = (cond); \ - maybe_fail(); \ - } while (0) - -bool __init wg_allowedips_selftest(void) -{ - bool found_a = false, found_b = false, found_c = false, found_d = false, - found_e = false, found_other = false; - struct wg_peer *a = init_peer(), *b = init_peer(), *c = init_peer(), - *d = init_peer(), *e = init_peer(), *f = init_peer(), - *g = init_peer(), *h = init_peer(); - struct allowedips_node *iter_node; - bool success = false; - struct allowedips t; - DEFINE_MUTEX(mutex); - struct in6_addr ip; - size_t i = 0, count = 0; - __be64 part; - - mutex_init(&mutex); - mutex_lock(&mutex); - wg_allowedips_init(&t); - - if (!a || !b || !c || !d || !e || !f || !g || !h) { - pr_err("allowedips self-test malloc: FAIL\n"); - goto free; - } - - insert(4, a, 192, 168, 4, 0, 24); - insert(4, b, 192, 168, 4, 4, 32); - insert(4, c, 192, 168, 0, 0, 16); - insert(4, d, 192, 95, 5, 64, 27); - /* replaces previous entry, and maskself is required */ - insert(4, c, 192, 95, 5, 65, 27); - insert(6, d, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128); - insert(6, c, 0x26075300, 0x60006b00, 0, 0, 64); - insert(4, e, 0, 0, 0, 0, 0); - insert(6, e, 0, 0, 0, 0, 0); - /* replaces previous entry */ - insert(6, f, 0, 0, 0, 0, 0); - insert(6, g, 0x24046800, 0, 0, 0, 32); - /* maskself is required */ - insert(6, h, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 64); - insert(6, a, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 128); - insert(6, c, 0x24446800, 0x40e40800, 0xdeaebeef, 0xdefbeef, 128); - insert(6, b, 0x24446800, 0xf0e40800, 0xeeaebeef, 0, 98); - insert(4, g, 64, 15, 112, 0, 20); - /* maskself is required */ - insert(4, h, 64, 15, 123, 211, 25); - insert(4, a, 10, 0, 0, 0, 25); - insert(4, b, 10, 0, 0, 128, 25); - insert(4, a, 10, 1, 0, 0, 30); - insert(4, b, 10, 1, 0, 4, 30); - insert(4, c, 10, 1, 0, 8, 29); - insert(4, d, 10, 1, 0, 16, 29); - - if (IS_ENABLED(DEBUG_PRINT_TRIE_GRAPHVIZ)) { - print_tree(t.root4, 32); - print_tree(t.root6, 128); - } - - success = true; - - test(4, a, 192, 168, 4, 20); - test(4, a, 192, 168, 4, 0); - test(4, b, 192, 168, 4, 4); - test(4, c, 192, 168, 200, 182); - test(4, c, 192, 95, 5, 68); - test(4, e, 192, 95, 5, 96); - test(6, d, 0x26075300, 0x60006b00, 0, 0xc05f0543); - test(6, c, 0x26075300, 0x60006b00, 0, 0xc02e01ee); - test(6, f, 0x26075300, 0x60006b01, 0, 0); - test(6, g, 0x24046800, 0x40040806, 0, 0x1006); - test(6, g, 0x24046800, 0x40040806, 0x1234, 0x5678); - test(6, f, 0x240467ff, 0x40040806, 0x1234, 0x5678); - test(6, f, 0x24046801, 0x40040806, 0x1234, 0x5678); - test(6, h, 0x24046800, 0x40040800, 0x1234, 0x5678); - test(6, h, 0x24046800, 0x40040800, 0, 0); - test(6, h, 0x24046800, 0x40040800, 0x10101010, 0x10101010); - test(6, a, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef); - test(4, g, 64, 15, 116, 26); - test(4, g, 64, 15, 127, 3); - test(4, g, 64, 15, 123, 1); - test(4, h, 64, 15, 123, 128); - test(4, h, 64, 15, 123, 129); - test(4, a, 10, 0, 0, 52); - test(4, b, 10, 0, 0, 220); - test(4, a, 10, 1, 0, 2); - test(4, b, 10, 1, 0, 6); - test(4, c, 10, 1, 0, 10); - test(4, d, 10, 1, 0, 20); - - insert(4, a, 1, 0, 0, 0, 32); - insert(4, a, 64, 0, 0, 0, 32); - insert(4, a, 128, 0, 0, 0, 32); - insert(4, a, 192, 0, 0, 0, 32); - insert(4, a, 255, 0, 0, 0, 32); - wg_allowedips_remove_by_peer(&t, a, &mutex); - test_negative(4, a, 1, 0, 0, 0); - test_negative(4, a, 64, 0, 0, 0); - test_negative(4, a, 128, 0, 0, 0); - test_negative(4, a, 192, 0, 0, 0); - test_negative(4, a, 255, 0, 0, 0); - - wg_allowedips_free(&t, &mutex); - wg_allowedips_init(&t); - insert(4, a, 192, 168, 0, 0, 16); - insert(4, a, 192, 168, 0, 0, 24); - wg_allowedips_remove_by_peer(&t, a, &mutex); - test_negative(4, a, 192, 168, 0, 1); - - /* These will hit the WARN_ON(len >= 128) in free_node if something - * goes wrong. - */ - for (i = 0; i < 128; ++i) { - part = cpu_to_be64(~(1LLU << (i % 64))); - memset(&ip, 0xff, 16); - memcpy((u8 *)&ip + (i < 64) * 8, &part, 8); - wg_allowedips_insert_v6(&t, &ip, 128, a, &mutex); - } - - wg_allowedips_free(&t, &mutex); - - wg_allowedips_init(&t); - insert(4, a, 192, 95, 5, 93, 27); - insert(6, a, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128); - insert(4, a, 10, 1, 0, 20, 29); - insert(6, a, 0x26075300, 0x6d8a6bf8, 0xdab1f1df, 0xc05f1523, 83); - insert(6, a, 0x26075300, 0x6d8a6bf8, 0xdab1f1df, 0xc05f1523, 21); - list_for_each_entry(iter_node, &a->allowedips_list, peer_list) { - u8 cidr, ip[16] __aligned(__alignof(u64)); - int family = wg_allowedips_read_node(iter_node, ip, &cidr); - - count++; - - if (cidr == 27 && family == AF_INET && - !memcmp(ip, ip4(192, 95, 5, 64), sizeof(struct in_addr))) - found_a = true; - else if (cidr == 128 && family == AF_INET6 && - !memcmp(ip, ip6(0x26075300, 0x60006b00, 0, 0xc05f0543), - sizeof(struct in6_addr))) - found_b = true; - else if (cidr == 29 && family == AF_INET && - !memcmp(ip, ip4(10, 1, 0, 16), sizeof(struct in_addr))) - found_c = true; - else if (cidr == 83 && family == AF_INET6 && - !memcmp(ip, ip6(0x26075300, 0x6d8a6bf8, 0xdab1e000, 0), - sizeof(struct in6_addr))) - found_d = true; - else if (cidr == 21 && family == AF_INET6 && - !memcmp(ip, ip6(0x26075000, 0, 0, 0), - sizeof(struct in6_addr))) - found_e = true; - else - found_other = true; - } - test_boolean(count == 5); - test_boolean(found_a); - test_boolean(found_b); - test_boolean(found_c); - test_boolean(found_d); - test_boolean(found_e); - test_boolean(!found_other); - - if (IS_ENABLED(DEBUG_RANDOM_TRIE) && success) - success = randomized_test(); - - if (success) - pr_info("allowedips self-tests: pass\n"); - -free: - wg_allowedips_free(&t, &mutex); - kfree(a); - kfree(b); - kfree(c); - kfree(d); - kfree(e); - kfree(f); - kfree(g); - kfree(h); - mutex_unlock(&mutex); - - return success; -} - -#undef test_negative -#undef test -#undef remove -#undef insert -#undef init_peer - -#endif diff --git a/drivers/net/wireguard/selftest/counter.c b/drivers/net/wireguard/selftest/counter.c deleted file mode 100644 index f4fbb9072ed73..0000000000000 --- a/drivers/net/wireguard/selftest/counter.c +++ /dev/null @@ -1,104 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#ifdef DEBUG -bool __init wg_packet_counter_selftest(void) -{ - unsigned int test_num = 0, i; - union noise_counter counter; - bool success = true; - -#define T_INIT do { \ - memset(&counter, 0, sizeof(union noise_counter)); \ - spin_lock_init(&counter.receive.lock); \ - } while (0) -#define T_LIM (COUNTER_WINDOW_SIZE + 1) -#define T(n, v) do { \ - ++test_num; \ - if (counter_validate(&counter, n) != (v)) { \ - pr_err("nonce counter self-test %u: FAIL\n", \ - test_num); \ - success = false; \ - } \ - } while (0) - - T_INIT; - /* 1 */ T(0, true); - /* 2 */ T(1, true); - /* 3 */ T(1, false); - /* 4 */ T(9, true); - /* 5 */ T(8, true); - /* 6 */ T(7, true); - /* 7 */ T(7, false); - /* 8 */ T(T_LIM, true); - /* 9 */ T(T_LIM - 1, true); - /* 10 */ T(T_LIM - 1, false); - /* 11 */ T(T_LIM - 2, true); - /* 12 */ T(2, true); - /* 13 */ T(2, false); - /* 14 */ T(T_LIM + 16, true); - /* 15 */ T(3, false); - /* 16 */ T(T_LIM + 16, false); - /* 17 */ T(T_LIM * 4, true); - /* 18 */ T(T_LIM * 4 - (T_LIM - 1), true); - /* 19 */ T(10, false); - /* 20 */ T(T_LIM * 4 - T_LIM, false); - /* 21 */ T(T_LIM * 4 - (T_LIM + 1), false); - /* 22 */ T(T_LIM * 4 - (T_LIM - 2), true); - /* 23 */ T(T_LIM * 4 + 1 - T_LIM, false); - /* 24 */ T(0, false); - /* 25 */ T(REJECT_AFTER_MESSAGES, false); - /* 26 */ T(REJECT_AFTER_MESSAGES - 1, true); - /* 27 */ T(REJECT_AFTER_MESSAGES, false); - /* 28 */ T(REJECT_AFTER_MESSAGES - 1, false); - /* 29 */ T(REJECT_AFTER_MESSAGES - 2, true); - /* 30 */ T(REJECT_AFTER_MESSAGES + 1, false); - /* 31 */ T(REJECT_AFTER_MESSAGES + 2, false); - /* 32 */ T(REJECT_AFTER_MESSAGES - 2, false); - /* 33 */ T(REJECT_AFTER_MESSAGES - 3, true); - /* 34 */ T(0, false); - - T_INIT; - for (i = 1; i <= COUNTER_WINDOW_SIZE; ++i) - T(i, true); - T(0, true); - T(0, false); - - T_INIT; - for (i = 2; i <= COUNTER_WINDOW_SIZE + 1; ++i) - T(i, true); - T(1, true); - T(0, false); - - T_INIT; - for (i = COUNTER_WINDOW_SIZE + 1; i-- > 0;) - T(i, true); - - T_INIT; - for (i = COUNTER_WINDOW_SIZE + 2; i-- > 1;) - T(i, true); - T(0, false); - - T_INIT; - for (i = COUNTER_WINDOW_SIZE + 1; i-- > 1;) - T(i, true); - T(COUNTER_WINDOW_SIZE + 1, true); - T(0, false); - - T_INIT; - for (i = COUNTER_WINDOW_SIZE + 1; i-- > 1;) - T(i, true); - T(0, true); - T(COUNTER_WINDOW_SIZE + 1, true); - -#undef T -#undef T_LIM -#undef T_INIT - - if (success) - pr_info("nonce counter self-tests: pass\n"); - return success; -} -#endif diff --git a/drivers/net/wireguard/selftest/ratelimiter.c b/drivers/net/wireguard/selftest/ratelimiter.c deleted file mode 100644 index bcd6462e45401..0000000000000 --- a/drivers/net/wireguard/selftest/ratelimiter.c +++ /dev/null @@ -1,226 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#ifdef DEBUG - -#include - -static const struct { - bool result; - unsigned int msec_to_sleep_before; -} expected_results[] __initconst = { - [0 ... PACKETS_BURSTABLE - 1] = { true, 0 }, - [PACKETS_BURSTABLE] = { false, 0 }, - [PACKETS_BURSTABLE + 1] = { true, MSEC_PER_SEC / PACKETS_PER_SECOND }, - [PACKETS_BURSTABLE + 2] = { false, 0 }, - [PACKETS_BURSTABLE + 3] = { true, (MSEC_PER_SEC / PACKETS_PER_SECOND) * 2 }, - [PACKETS_BURSTABLE + 4] = { true, 0 }, - [PACKETS_BURSTABLE + 5] = { false, 0 } -}; - -static __init unsigned int maximum_jiffies_at_index(int index) -{ - unsigned int total_msecs = 2 * MSEC_PER_SEC / PACKETS_PER_SECOND / 3; - int i; - - for (i = 0; i <= index; ++i) - total_msecs += expected_results[i].msec_to_sleep_before; - return msecs_to_jiffies(total_msecs); -} - -static __init int timings_test(struct sk_buff *skb4, struct iphdr *hdr4, - struct sk_buff *skb6, struct ipv6hdr *hdr6, - int *test) -{ - unsigned long loop_start_time; - int i; - - wg_ratelimiter_gc_entries(NULL); - rcu_barrier(); - loop_start_time = jiffies; - - for (i = 0; i < ARRAY_SIZE(expected_results); ++i) { - if (expected_results[i].msec_to_sleep_before) - msleep(expected_results[i].msec_to_sleep_before); - - if (time_is_before_jiffies(loop_start_time + - maximum_jiffies_at_index(i))) - return -ETIMEDOUT; - if (wg_ratelimiter_allow(skb4, &init_net) != - expected_results[i].result) - return -EXFULL; - ++(*test); - - hdr4->saddr = htonl(ntohl(hdr4->saddr) + i + 1); - if (time_is_before_jiffies(loop_start_time + - maximum_jiffies_at_index(i))) - return -ETIMEDOUT; - if (!wg_ratelimiter_allow(skb4, &init_net)) - return -EXFULL; - ++(*test); - - hdr4->saddr = htonl(ntohl(hdr4->saddr) - i - 1); - -#if IS_ENABLED(CONFIG_IPV6) - hdr6->saddr.in6_u.u6_addr32[2] = htonl(i); - hdr6->saddr.in6_u.u6_addr32[3] = htonl(i); - if (time_is_before_jiffies(loop_start_time + - maximum_jiffies_at_index(i))) - return -ETIMEDOUT; - if (wg_ratelimiter_allow(skb6, &init_net) != - expected_results[i].result) - return -EXFULL; - ++(*test); - - hdr6->saddr.in6_u.u6_addr32[0] = - htonl(ntohl(hdr6->saddr.in6_u.u6_addr32[0]) + i + 1); - if (time_is_before_jiffies(loop_start_time + - maximum_jiffies_at_index(i))) - return -ETIMEDOUT; - if (!wg_ratelimiter_allow(skb6, &init_net)) - return -EXFULL; - ++(*test); - - hdr6->saddr.in6_u.u6_addr32[0] = - htonl(ntohl(hdr6->saddr.in6_u.u6_addr32[0]) - i - 1); - - if (time_is_before_jiffies(loop_start_time + - maximum_jiffies_at_index(i))) - return -ETIMEDOUT; -#endif - } - return 0; -} - -static __init int capacity_test(struct sk_buff *skb4, struct iphdr *hdr4, - int *test) -{ - int i; - - wg_ratelimiter_gc_entries(NULL); - rcu_barrier(); - - if (atomic_read(&total_entries)) - return -EXFULL; - ++(*test); - - for (i = 0; i <= max_entries; ++i) { - hdr4->saddr = htonl(i); - if (wg_ratelimiter_allow(skb4, &init_net) != (i != max_entries)) - return -EXFULL; - ++(*test); - } - return 0; -} - -bool __init wg_ratelimiter_selftest(void) -{ - enum { TRIALS_BEFORE_GIVING_UP = 5000 }; - bool success = false; - int test = 0, trials; - struct sk_buff *skb4, *skb6; - struct iphdr *hdr4; - struct ipv6hdr *hdr6; - - if (IS_ENABLED(CONFIG_KASAN) || IS_ENABLED(CONFIG_UBSAN)) - return true; - - BUILD_BUG_ON(MSEC_PER_SEC % PACKETS_PER_SECOND != 0); - - if (wg_ratelimiter_init()) - goto out; - ++test; - if (wg_ratelimiter_init()) { - wg_ratelimiter_uninit(); - goto out; - } - ++test; - if (wg_ratelimiter_init()) { - wg_ratelimiter_uninit(); - wg_ratelimiter_uninit(); - goto out; - } - ++test; - - skb4 = alloc_skb(sizeof(struct iphdr), GFP_KERNEL); - if (unlikely(!skb4)) - goto err_nofree; - skb4->protocol = htons(ETH_P_IP); - hdr4 = (struct iphdr *)skb_put(skb4, sizeof(*hdr4)); - hdr4->saddr = htonl(8182); - skb_reset_network_header(skb4); - ++test; - -#if IS_ENABLED(CONFIG_IPV6) - skb6 = alloc_skb(sizeof(struct ipv6hdr), GFP_KERNEL); - if (unlikely(!skb6)) { - kfree_skb(skb4); - goto err_nofree; - } - skb6->protocol = htons(ETH_P_IPV6); - hdr6 = (struct ipv6hdr *)skb_put(skb6, sizeof(*hdr6)); - hdr6->saddr.in6_u.u6_addr32[0] = htonl(1212); - hdr6->saddr.in6_u.u6_addr32[1] = htonl(289188); - skb_reset_network_header(skb6); - ++test; -#endif - - for (trials = TRIALS_BEFORE_GIVING_UP;;) { - int test_count = 0, ret; - - ret = timings_test(skb4, hdr4, skb6, hdr6, &test_count); - if (ret == -ETIMEDOUT) { - if (!trials--) { - test += test_count; - goto err; - } - msleep(500); - continue; - } else if (ret < 0) { - test += test_count; - goto err; - } else { - test += test_count; - break; - } - } - - for (trials = TRIALS_BEFORE_GIVING_UP;;) { - int test_count = 0; - - if (capacity_test(skb4, hdr4, &test_count) < 0) { - if (!trials--) { - test += test_count; - goto err; - } - msleep(50); - continue; - } - test += test_count; - break; - } - - success = true; - -err: - kfree_skb(skb4); -#if IS_ENABLED(CONFIG_IPV6) - kfree_skb(skb6); -#endif -err_nofree: - wg_ratelimiter_uninit(); - wg_ratelimiter_uninit(); - wg_ratelimiter_uninit(); - /* Uninit one extra time to check underflow detection. */ - wg_ratelimiter_uninit(); -out: - if (success) - pr_info("ratelimiter self-tests: pass\n"); - else - pr_err("ratelimiter self-test %d: FAIL\n", test); - - return success; -} -#endif diff --git a/drivers/net/wireguard/send.c b/drivers/net/wireguard/send.c deleted file mode 100644 index 7348c10cbae3d..0000000000000 --- a/drivers/net/wireguard/send.c +++ /dev/null @@ -1,419 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include "queueing.h" -#include "timers.h" -#include "device.h" -#include "peer.h" -#include "socket.h" -#include "messages.h" -#include "cookie.h" - -#include -#include -#include -#include -#include -#include - -static void wg_packet_send_handshake_initiation(struct wg_peer *peer) -{ - struct message_handshake_initiation packet; - - if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), - REKEY_TIMEOUT)) - return; /* This function is rate limited. */ - - atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); - net_dbg_ratelimited("%s: Sending handshake initiation to peer %llu (%pISpfsc)\n", - peer->device->dev->name, peer->internal_id, - &peer->endpoint.addr); - - if (wg_noise_handshake_create_initiation(&packet, &peer->handshake)) { - wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); - wg_timers_any_authenticated_packet_traversal(peer); - wg_timers_any_authenticated_packet_sent(peer); - atomic64_set(&peer->last_sent_handshake, - ktime_get_coarse_boottime_ns()); - wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet), - HANDSHAKE_DSCP); - wg_timers_handshake_initiated(peer); - } -} - -void wg_packet_handshake_send_worker(struct work_struct *work) -{ - struct wg_peer *peer = container_of(work, struct wg_peer, - transmit_handshake_work); - - wg_packet_send_handshake_initiation(peer); - wg_peer_put(peer); -} - -void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, - bool is_retry) -{ - if (!is_retry) - peer->timer_handshake_attempts = 0; - - rcu_read_lock_bh(); - /* We check last_sent_handshake here in addition to the actual function - * we're queueing up, so that we don't queue things if not strictly - * necessary: - */ - if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), - REKEY_TIMEOUT) || - unlikely(READ_ONCE(peer->is_dead))) - goto out; - - wg_peer_get(peer); - /* Queues up calling packet_send_queued_handshakes(peer), where we do a - * peer_put(peer) after: - */ - if (!queue_work(peer->device->handshake_send_wq, - &peer->transmit_handshake_work)) - /* If the work was already queued, we want to drop the - * extra reference: - */ - wg_peer_put(peer); -out: - rcu_read_unlock_bh(); -} - -void wg_packet_send_handshake_response(struct wg_peer *peer) -{ - struct message_handshake_response packet; - - atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); - net_dbg_ratelimited("%s: Sending handshake response to peer %llu (%pISpfsc)\n", - peer->device->dev->name, peer->internal_id, - &peer->endpoint.addr); - - if (wg_noise_handshake_create_response(&packet, &peer->handshake)) { - wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); - if (wg_noise_handshake_begin_session(&peer->handshake, - &peer->keypairs)) { - wg_timers_session_derived(peer); - wg_timers_any_authenticated_packet_traversal(peer); - wg_timers_any_authenticated_packet_sent(peer); - atomic64_set(&peer->last_sent_handshake, - ktime_get_coarse_boottime_ns()); - wg_socket_send_buffer_to_peer(peer, &packet, - sizeof(packet), - HANDSHAKE_DSCP); - } - } -} - -void wg_packet_send_handshake_cookie(struct wg_device *wg, - struct sk_buff *initiating_skb, - __le32 sender_index) -{ - struct message_handshake_cookie packet; - - net_dbg_skb_ratelimited("%s: Sending cookie response for denied handshake message for %pISpfsc\n", - wg->dev->name, initiating_skb); - wg_cookie_message_create(&packet, initiating_skb, sender_index, - &wg->cookie_checker); - wg_socket_send_buffer_as_reply_to_skb(wg, initiating_skb, &packet, - sizeof(packet)); -} - -static void keep_key_fresh(struct wg_peer *peer) -{ - struct noise_keypair *keypair; - bool send = false; - - rcu_read_lock_bh(); - keypair = rcu_dereference_bh(peer->keypairs.current_keypair); - if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) && - (unlikely(atomic64_read(&keypair->sending.counter.counter) > - REKEY_AFTER_MESSAGES) || - (keypair->i_am_the_initiator && - unlikely(wg_birthdate_has_expired(keypair->sending.birthdate, - REKEY_AFTER_TIME))))) - send = true; - rcu_read_unlock_bh(); - - if (send) - wg_packet_send_queued_handshake_initiation(peer, false); -} - -static unsigned int calculate_skb_padding(struct sk_buff *skb) -{ - unsigned int padded_size, last_unit = skb->len; - - if (unlikely(!PACKET_CB(skb)->mtu)) - return ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE) - last_unit; - - /* We do this modulo business with the MTU, just in case the networking - * layer gives us a packet that's bigger than the MTU. In that case, we - * wouldn't want the final subtraction to overflow in the case of the - * padded_size being clamped. Fortunately, that's very rarely the case, - * so we optimize for that not happening. - */ - if (unlikely(last_unit > PACKET_CB(skb)->mtu)) - last_unit %= PACKET_CB(skb)->mtu; - - padded_size = min(PACKET_CB(skb)->mtu, - ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE)); - return padded_size - last_unit; -} - -static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair) -{ - unsigned int padding_len, plaintext_len, trailer_len; - struct scatterlist sg[MAX_SKB_FRAGS + 8]; - struct message_data *header; - struct sk_buff *trailer; - int num_frags; - - /* Calculate lengths. */ - padding_len = calculate_skb_padding(skb); - trailer_len = padding_len + noise_encrypted_len(0); - plaintext_len = skb->len + padding_len; - - /* Expand data section to have room for padding and auth tag. */ - num_frags = skb_cow_data(skb, trailer_len, &trailer); - if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) - return false; - - /* Set the padding to zeros, and make sure it and the auth tag are part - * of the skb. - */ - memset(skb_tail_pointer(trailer), 0, padding_len); - - /* Expand head section to have room for our header and the network - * stack's headers. - */ - if (unlikely(skb_cow_head(skb, DATA_PACKET_HEAD_ROOM) < 0)) - return false; - - /* Finalize checksum calculation for the inner packet, if required. */ - if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && - skb_checksum_help(skb))) - return false; - - /* Only after checksumming can we safely add on the padding at the end - * and the header. - */ - skb_set_inner_network_header(skb, 0); - header = (struct message_data *)skb_push(skb, sizeof(*header)); - header->header.type = cpu_to_le32(MESSAGE_DATA); - header->key_idx = keypair->remote_index; - header->counter = cpu_to_le64(PACKET_CB(skb)->nonce); - pskb_put(skb, trailer, trailer_len); - - /* Now we can encrypt the scattergather segments */ - sg_init_table(sg, num_frags); - if (skb_to_sgvec(skb, sg, sizeof(struct message_data), - noise_encrypted_len(plaintext_len)) <= 0) - return false; - return chacha20poly1305_encrypt_sg_inplace(sg, plaintext_len, NULL, 0, - PACKET_CB(skb)->nonce, - keypair->sending.key); -} - -void wg_packet_send_keepalive(struct wg_peer *peer) -{ - struct sk_buff *skb; - - if (skb_queue_empty(&peer->staged_packet_queue)) { - skb = alloc_skb(DATA_PACKET_HEAD_ROOM + MESSAGE_MINIMUM_LENGTH, - GFP_ATOMIC); - if (unlikely(!skb)) - return; - skb_reserve(skb, DATA_PACKET_HEAD_ROOM); - skb->dev = peer->device->dev; - PACKET_CB(skb)->mtu = skb->dev->mtu; - skb_queue_tail(&peer->staged_packet_queue, skb); - net_dbg_ratelimited("%s: Sending keepalive packet to peer %llu (%pISpfsc)\n", - peer->device->dev->name, peer->internal_id, - &peer->endpoint.addr); - } - - wg_packet_send_staged_packets(peer); -} - -static void wg_packet_create_data_done(struct sk_buff *first, - struct wg_peer *peer) -{ - struct sk_buff *skb, *next; - bool is_keepalive, data_sent = false; - - wg_timers_any_authenticated_packet_traversal(peer); - wg_timers_any_authenticated_packet_sent(peer); - skb_list_walk_safe(first, skb, next) { - is_keepalive = skb->len == message_data_len(0); - if (likely(!wg_socket_send_skb_to_peer(peer, skb, - PACKET_CB(skb)->ds) && !is_keepalive)) - data_sent = true; - } - - if (likely(data_sent)) - wg_timers_data_sent(peer); - - keep_key_fresh(peer); -} - -void wg_packet_tx_worker(struct work_struct *work) -{ - struct crypt_queue *queue = container_of(work, struct crypt_queue, - work); - struct noise_keypair *keypair; - enum packet_state state; - struct sk_buff *first; - struct wg_peer *peer; - - while ((first = __ptr_ring_peek(&queue->ring)) != NULL && - (state = atomic_read_acquire(&PACKET_CB(first)->state)) != - PACKET_STATE_UNCRYPTED) { - __ptr_ring_discard_one(&queue->ring); - peer = PACKET_PEER(first); - keypair = PACKET_CB(first)->keypair; - - if (likely(state == PACKET_STATE_CRYPTED)) - wg_packet_create_data_done(first, peer); - else - kfree_skb_list(first); - - wg_noise_keypair_put(keypair, false); - wg_peer_put(peer); - } -} - -void wg_packet_encrypt_worker(struct work_struct *work) -{ - struct crypt_queue *queue = container_of(work, struct multicore_worker, - work)->ptr; - struct sk_buff *first, *skb, *next; - - while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) { - enum packet_state state = PACKET_STATE_CRYPTED; - - skb_list_walk_safe(first, skb, next) { - if (likely(encrypt_packet(skb, - PACKET_CB(first)->keypair))) { - wg_reset_packet(skb); - } else { - state = PACKET_STATE_DEAD; - break; - } - } - wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, first, - state); - - } -} - -static void wg_packet_create_data(struct sk_buff *first) -{ - struct wg_peer *peer = PACKET_PEER(first); - struct wg_device *wg = peer->device; - int ret = -EINVAL; - - rcu_read_lock_bh(); - if (unlikely(READ_ONCE(peer->is_dead))) - goto err; - - ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, - &peer->tx_queue, first, - wg->packet_crypt_wq, - &wg->encrypt_queue.last_cpu); - if (unlikely(ret == -EPIPE)) - wg_queue_enqueue_per_peer(&peer->tx_queue, first, - PACKET_STATE_DEAD); -err: - rcu_read_unlock_bh(); - if (likely(!ret || ret == -EPIPE)) - return; - wg_noise_keypair_put(PACKET_CB(first)->keypair, false); - wg_peer_put(peer); - kfree_skb_list(first); -} - -void wg_packet_purge_staged_packets(struct wg_peer *peer) -{ - spin_lock_bh(&peer->staged_packet_queue.lock); - peer->device->dev->stats.tx_dropped += peer->staged_packet_queue.qlen; - __skb_queue_purge(&peer->staged_packet_queue); - spin_unlock_bh(&peer->staged_packet_queue.lock); -} - -void wg_packet_send_staged_packets(struct wg_peer *peer) -{ - struct noise_symmetric_key *key; - struct noise_keypair *keypair; - struct sk_buff_head packets; - struct sk_buff *skb; - - /* Steal the current queue into our local one. */ - __skb_queue_head_init(&packets); - spin_lock_bh(&peer->staged_packet_queue.lock); - skb_queue_splice_init(&peer->staged_packet_queue, &packets); - spin_unlock_bh(&peer->staged_packet_queue.lock); - if (unlikely(skb_queue_empty(&packets))) - return; - - /* First we make sure we have a valid reference to a valid key. */ - rcu_read_lock_bh(); - keypair = wg_noise_keypair_get( - rcu_dereference_bh(peer->keypairs.current_keypair)); - rcu_read_unlock_bh(); - if (unlikely(!keypair)) - goto out_nokey; - key = &keypair->sending; - if (unlikely(!READ_ONCE(key->is_valid))) - goto out_nokey; - if (unlikely(wg_birthdate_has_expired(key->birthdate, - REJECT_AFTER_TIME))) - goto out_invalid; - - /* After we know we have a somewhat valid key, we now try to assign - * nonces to all of the packets in the queue. If we can't assign nonces - * for all of them, we just consider it a failure and wait for the next - * handshake. - */ - skb_queue_walk(&packets, skb) { - /* 0 for no outer TOS: no leak. TODO: at some later point, we - * might consider using flowi->tos as outer instead. - */ - PACKET_CB(skb)->ds = ip_tunnel_ecn_encap(0, ip_hdr(skb), skb); - PACKET_CB(skb)->nonce = - atomic64_inc_return(&key->counter.counter) - 1; - if (unlikely(PACKET_CB(skb)->nonce >= REJECT_AFTER_MESSAGES)) - goto out_invalid; - } - - packets.prev->next = NULL; - wg_peer_get(keypair->entry.peer); - PACKET_CB(packets.next)->keypair = keypair; - wg_packet_create_data(packets.next); - return; - -out_invalid: - WRITE_ONCE(key->is_valid, false); -out_nokey: - wg_noise_keypair_put(keypair, false); - - /* We orphan the packets if we're waiting on a handshake, so that they - * don't block a socket's pool. - */ - skb_queue_walk(&packets, skb) - skb_orphan(skb); - /* Then we put them back on the top of the queue. We're not too - * concerned about accidentally getting things a little out of order if - * packets are being added really fast, because this queue is for before - * packets can even be sent and it's small anyway. - */ - spin_lock_bh(&peer->staged_packet_queue.lock); - skb_queue_splice(&packets, &peer->staged_packet_queue); - spin_unlock_bh(&peer->staged_packet_queue.lock); - - /* If we're exiting because there's something wrong with the key, it - * means we should initiate a new handshake. - */ - wg_packet_send_queued_handshake_initiation(peer, false); -} diff --git a/drivers/net/wireguard/socket.c b/drivers/net/wireguard/socket.c deleted file mode 100644 index b0d6541582d31..0000000000000 --- a/drivers/net/wireguard/socket.c +++ /dev/null @@ -1,437 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include "device.h" -#include "peer.h" -#include "socket.h" -#include "queueing.h" -#include "messages.h" - -#include -#include -#include -#include -#include -#include -#include - -static int send4(struct wg_device *wg, struct sk_buff *skb, - struct endpoint *endpoint, u8 ds, struct dst_cache *cache) -{ - struct flowi4 fl = { - .saddr = endpoint->src4.s_addr, - .daddr = endpoint->addr4.sin_addr.s_addr, - .fl4_dport = endpoint->addr4.sin_port, - .flowi4_mark = wg->fwmark, - .flowi4_proto = IPPROTO_UDP - }; - struct rtable *rt = NULL; - struct sock *sock; - int ret = 0; - - skb_mark_not_on_list(skb); - skb->dev = wg->dev; - skb->mark = wg->fwmark; - - rcu_read_lock_bh(); - sock = rcu_dereference_bh(wg->sock4); - - if (unlikely(!sock)) { - ret = -ENONET; - goto err; - } - - fl.fl4_sport = inet_sk(sock)->inet_sport; - - if (cache) - rt = dst_cache_get_ip4(cache, &fl.saddr); - - if (!rt) { - security_sk_classify_flow(sock, flowi4_to_flowi(&fl)); - if (unlikely(!inet_confirm_addr(sock_net(sock), NULL, 0, - fl.saddr, RT_SCOPE_HOST))) { - endpoint->src4.s_addr = 0; - *(__force __be32 *)&endpoint->src_if4 = 0; - fl.saddr = 0; - if (cache) - dst_cache_reset(cache); - } - rt = ip_route_output_flow(sock_net(sock), &fl, sock); - if (unlikely(endpoint->src_if4 && ((IS_ERR(rt) && - PTR_ERR(rt) == -EINVAL) || (!IS_ERR(rt) && - rt->dst.dev->ifindex != endpoint->src_if4)))) { - endpoint->src4.s_addr = 0; - *(__force __be32 *)&endpoint->src_if4 = 0; - fl.saddr = 0; - if (cache) - dst_cache_reset(cache); - if (!IS_ERR(rt)) - ip_rt_put(rt); - rt = ip_route_output_flow(sock_net(sock), &fl, sock); - } - if (unlikely(IS_ERR(rt))) { - ret = PTR_ERR(rt); - net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n", - wg->dev->name, &endpoint->addr, ret); - goto err; - } else if (unlikely(rt->dst.dev == skb->dev)) { - ip_rt_put(rt); - ret = -ELOOP; - net_dbg_ratelimited("%s: Avoiding routing loop to %pISpfsc\n", - wg->dev->name, &endpoint->addr); - goto err; - } - if (cache) - dst_cache_set_ip4(cache, &rt->dst, fl.saddr); - } - - skb->ignore_df = 1; - udp_tunnel_xmit_skb(rt, sock, skb, fl.saddr, fl.daddr, ds, - ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport, - fl.fl4_dport, false, false); - goto out; - -err: - kfree_skb(skb); -out: - rcu_read_unlock_bh(); - return ret; -} - -static int send6(struct wg_device *wg, struct sk_buff *skb, - struct endpoint *endpoint, u8 ds, struct dst_cache *cache) -{ -#if IS_ENABLED(CONFIG_IPV6) - struct flowi6 fl = { - .saddr = endpoint->src6, - .daddr = endpoint->addr6.sin6_addr, - .fl6_dport = endpoint->addr6.sin6_port, - .flowi6_mark = wg->fwmark, - .flowi6_oif = endpoint->addr6.sin6_scope_id, - .flowi6_proto = IPPROTO_UDP - /* TODO: addr->sin6_flowinfo */ - }; - struct dst_entry *dst = NULL; - struct sock *sock; - int ret = 0; - - skb_mark_not_on_list(skb); - skb->dev = wg->dev; - skb->mark = wg->fwmark; - - rcu_read_lock_bh(); - sock = rcu_dereference_bh(wg->sock6); - - if (unlikely(!sock)) { - ret = -ENONET; - goto err; - } - - fl.fl6_sport = inet_sk(sock)->inet_sport; - - if (cache) - dst = dst_cache_get_ip6(cache, &fl.saddr); - - if (!dst) { - security_sk_classify_flow(sock, flowi6_to_flowi(&fl)); - if (unlikely(!ipv6_addr_any(&fl.saddr) && - !ipv6_chk_addr(sock_net(sock), &fl.saddr, NULL, 0))) { - endpoint->src6 = fl.saddr = in6addr_any; - if (cache) - dst_cache_reset(cache); - } - dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sock), sock, &fl, - NULL); - if (unlikely(IS_ERR(dst))) { - ret = PTR_ERR(dst); - net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n", - wg->dev->name, &endpoint->addr, ret); - goto err; - } else if (unlikely(dst->dev == skb->dev)) { - dst_release(dst); - ret = -ELOOP; - net_dbg_ratelimited("%s: Avoiding routing loop to %pISpfsc\n", - wg->dev->name, &endpoint->addr); - goto err; - } - if (cache) - dst_cache_set_ip6(cache, dst, &fl.saddr); - } - - skb->ignore_df = 1; - udp_tunnel6_xmit_skb(dst, sock, skb, skb->dev, &fl.saddr, &fl.daddr, ds, - ip6_dst_hoplimit(dst), 0, fl.fl6_sport, - fl.fl6_dport, false); - goto out; - -err: - kfree_skb(skb); -out: - rcu_read_unlock_bh(); - return ret; -#else - return -EAFNOSUPPORT; -#endif -} - -int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, u8 ds) -{ - size_t skb_len = skb->len; - int ret = -EAFNOSUPPORT; - - read_lock_bh(&peer->endpoint_lock); - if (peer->endpoint.addr.sa_family == AF_INET) - ret = send4(peer->device, skb, &peer->endpoint, ds, - &peer->endpoint_cache); - else if (peer->endpoint.addr.sa_family == AF_INET6) - ret = send6(peer->device, skb, &peer->endpoint, ds, - &peer->endpoint_cache); - else - dev_kfree_skb(skb); - if (likely(!ret)) - peer->tx_bytes += skb_len; - read_unlock_bh(&peer->endpoint_lock); - - return ret; -} - -int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *buffer, - size_t len, u8 ds) -{ - struct sk_buff *skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC); - - if (unlikely(!skb)) - return -ENOMEM; - - skb_reserve(skb, SKB_HEADER_LEN); - skb_set_inner_network_header(skb, 0); - skb_put_data(skb, buffer, len); - return wg_socket_send_skb_to_peer(peer, skb, ds); -} - -int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg, - struct sk_buff *in_skb, void *buffer, - size_t len) -{ - int ret = 0; - struct sk_buff *skb; - struct endpoint endpoint; - - if (unlikely(!in_skb)) - return -EINVAL; - ret = wg_socket_endpoint_from_skb(&endpoint, in_skb); - if (unlikely(ret < 0)) - return ret; - - skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC); - if (unlikely(!skb)) - return -ENOMEM; - skb_reserve(skb, SKB_HEADER_LEN); - skb_set_inner_network_header(skb, 0); - skb_put_data(skb, buffer, len); - - if (endpoint.addr.sa_family == AF_INET) - ret = send4(wg, skb, &endpoint, 0, NULL); - else if (endpoint.addr.sa_family == AF_INET6) - ret = send6(wg, skb, &endpoint, 0, NULL); - /* No other possibilities if the endpoint is valid, which it is, - * as we checked above. - */ - - return ret; -} - -int wg_socket_endpoint_from_skb(struct endpoint *endpoint, - const struct sk_buff *skb) -{ - memset(endpoint, 0, sizeof(*endpoint)); - if (skb->protocol == htons(ETH_P_IP)) { - endpoint->addr4.sin_family = AF_INET; - endpoint->addr4.sin_port = udp_hdr(skb)->source; - endpoint->addr4.sin_addr.s_addr = ip_hdr(skb)->saddr; - endpoint->src4.s_addr = ip_hdr(skb)->daddr; - endpoint->src_if4 = skb->skb_iif; - } else if (skb->protocol == htons(ETH_P_IPV6)) { - endpoint->addr6.sin6_family = AF_INET6; - endpoint->addr6.sin6_port = udp_hdr(skb)->source; - endpoint->addr6.sin6_addr = ipv6_hdr(skb)->saddr; - endpoint->addr6.sin6_scope_id = ipv6_iface_scope_id( - &ipv6_hdr(skb)->saddr, skb->skb_iif); - endpoint->src6 = ipv6_hdr(skb)->daddr; - } else { - return -EINVAL; - } - return 0; -} - -static bool endpoint_eq(const struct endpoint *a, const struct endpoint *b) -{ - return (a->addr.sa_family == AF_INET && b->addr.sa_family == AF_INET && - a->addr4.sin_port == b->addr4.sin_port && - a->addr4.sin_addr.s_addr == b->addr4.sin_addr.s_addr && - a->src4.s_addr == b->src4.s_addr && a->src_if4 == b->src_if4) || - (a->addr.sa_family == AF_INET6 && - b->addr.sa_family == AF_INET6 && - a->addr6.sin6_port == b->addr6.sin6_port && - ipv6_addr_equal(&a->addr6.sin6_addr, &b->addr6.sin6_addr) && - a->addr6.sin6_scope_id == b->addr6.sin6_scope_id && - ipv6_addr_equal(&a->src6, &b->src6)) || - unlikely(!a->addr.sa_family && !b->addr.sa_family); -} - -void wg_socket_set_peer_endpoint(struct wg_peer *peer, - const struct endpoint *endpoint) -{ - /* First we check unlocked, in order to optimize, since it's pretty rare - * that an endpoint will change. If we happen to be mid-write, and two - * CPUs wind up writing the same thing or something slightly different, - * it doesn't really matter much either. - */ - if (endpoint_eq(endpoint, &peer->endpoint)) - return; - write_lock_bh(&peer->endpoint_lock); - if (endpoint->addr.sa_family == AF_INET) { - peer->endpoint.addr4 = endpoint->addr4; - peer->endpoint.src4 = endpoint->src4; - peer->endpoint.src_if4 = endpoint->src_if4; - } else if (endpoint->addr.sa_family == AF_INET6) { - peer->endpoint.addr6 = endpoint->addr6; - peer->endpoint.src6 = endpoint->src6; - } else { - goto out; - } - dst_cache_reset(&peer->endpoint_cache); -out: - write_unlock_bh(&peer->endpoint_lock); -} - -void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, - const struct sk_buff *skb) -{ - struct endpoint endpoint; - - if (!wg_socket_endpoint_from_skb(&endpoint, skb)) - wg_socket_set_peer_endpoint(peer, &endpoint); -} - -void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer) -{ - write_lock_bh(&peer->endpoint_lock); - memset(&peer->endpoint.src6, 0, sizeof(peer->endpoint.src6)); - dst_cache_reset(&peer->endpoint_cache); - write_unlock_bh(&peer->endpoint_lock); -} - -static int wg_receive(struct sock *sk, struct sk_buff *skb) -{ - struct wg_device *wg; - - if (unlikely(!sk)) - goto err; - wg = sk->sk_user_data; - if (unlikely(!wg)) - goto err; - skb_mark_not_on_list(skb); - wg_packet_receive(wg, skb); - return 0; - -err: - kfree_skb(skb); - return 0; -} - -static void sock_free(struct sock *sock) -{ - if (unlikely(!sock)) - return; - sk_clear_memalloc(sock); - udp_tunnel_sock_release(sock->sk_socket); -} - -static void set_sock_opts(struct socket *sock) -{ - sock->sk->sk_allocation = GFP_ATOMIC; - sock->sk->sk_sndbuf = INT_MAX; - sk_set_memalloc(sock->sk); -} - -int wg_socket_init(struct wg_device *wg, u16 port) -{ - int ret; - struct udp_tunnel_sock_cfg cfg = { - .sk_user_data = wg, - .encap_type = 1, - .encap_rcv = wg_receive - }; - struct socket *new4 = NULL, *new6 = NULL; - struct udp_port_cfg port4 = { - .family = AF_INET, - .local_ip.s_addr = htonl(INADDR_ANY), - .local_udp_port = htons(port), - .use_udp_checksums = true - }; -#if IS_ENABLED(CONFIG_IPV6) - int retries = 0; - struct udp_port_cfg port6 = { - .family = AF_INET6, - .local_ip6 = IN6ADDR_ANY_INIT, - .use_udp6_tx_checksums = true, - .use_udp6_rx_checksums = true, - .ipv6_v6only = true - }; -#endif - -#if IS_ENABLED(CONFIG_IPV6) -retry: -#endif - - ret = udp_sock_create(wg->creating_net, &port4, &new4); - if (ret < 0) { - pr_err("%s: Could not create IPv4 socket\n", wg->dev->name); - return ret; - } - set_sock_opts(new4); - setup_udp_tunnel_sock(wg->creating_net, new4, &cfg); - -#if IS_ENABLED(CONFIG_IPV6) - if (ipv6_mod_enabled()) { - port6.local_udp_port = inet_sk(new4->sk)->inet_sport; - ret = udp_sock_create(wg->creating_net, &port6, &new6); - if (ret < 0) { - udp_tunnel_sock_release(new4); - if (ret == -EADDRINUSE && !port && retries++ < 100) - goto retry; - pr_err("%s: Could not create IPv6 socket\n", - wg->dev->name); - return ret; - } - set_sock_opts(new6); - setup_udp_tunnel_sock(wg->creating_net, new6, &cfg); - } -#endif - - wg_socket_reinit(wg, new4->sk, new6 ? new6->sk : NULL); - return 0; -} - -void wg_socket_reinit(struct wg_device *wg, struct sock *new4, - struct sock *new6) -{ - struct sock *old4, *old6; - - mutex_lock(&wg->socket_update_lock); - old4 = rcu_dereference_protected(wg->sock4, - lockdep_is_held(&wg->socket_update_lock)); - old6 = rcu_dereference_protected(wg->sock6, - lockdep_is_held(&wg->socket_update_lock)); - rcu_assign_pointer(wg->sock4, new4); - rcu_assign_pointer(wg->sock6, new6); - if (new4) - wg->incoming_port = ntohs(inet_sk(new4)->inet_sport); - mutex_unlock(&wg->socket_update_lock); - synchronize_rcu(); - sock_free(old4); - sock_free(old6); -} diff --git a/drivers/net/wireguard/socket.h b/drivers/net/wireguard/socket.h deleted file mode 100644 index bab5848efbcdf..0000000000000 --- a/drivers/net/wireguard/socket.h +++ /dev/null @@ -1,44 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#ifndef _WG_SOCKET_H -#define _WG_SOCKET_H - -#include -#include -#include -#include - -int wg_socket_init(struct wg_device *wg, u16 port); -void wg_socket_reinit(struct wg_device *wg, struct sock *new4, - struct sock *new6); -int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *data, - size_t len, u8 ds); -int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, - u8 ds); -int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg, - struct sk_buff *in_skb, - void *out_buffer, size_t len); - -int wg_socket_endpoint_from_skb(struct endpoint *endpoint, - const struct sk_buff *skb); -void wg_socket_set_peer_endpoint(struct wg_peer *peer, - const struct endpoint *endpoint); -void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, - const struct sk_buff *skb); -void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer); - -#if defined(CONFIG_DYNAMIC_DEBUG) || defined(DEBUG) -#define net_dbg_skb_ratelimited(fmt, dev, skb, ...) do { \ - struct endpoint __endpoint; \ - wg_socket_endpoint_from_skb(&__endpoint, skb); \ - net_dbg_ratelimited(fmt, dev, &__endpoint.addr, \ - ##__VA_ARGS__); \ - } while (0) -#else -#define net_dbg_skb_ratelimited(fmt, skb, ...) -#endif - -#endif /* _WG_SOCKET_H */ diff --git a/drivers/net/wireguard/timers.c b/drivers/net/wireguard/timers.c deleted file mode 100644 index d54d32ac9bc41..0000000000000 --- a/drivers/net/wireguard/timers.c +++ /dev/null @@ -1,243 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#include "timers.h" -#include "device.h" -#include "peer.h" -#include "queueing.h" -#include "socket.h" - -/* - * - Timer for retransmitting the handshake if we don't hear back after - * `REKEY_TIMEOUT + jitter` ms. - * - * - Timer for sending empty packet if we have received a packet but after have - * not sent one for `KEEPALIVE_TIMEOUT` ms. - * - * - Timer for initiating new handshake if we have sent a packet but after have - * not received one (even empty) for `(KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) + - * jitter` ms. - * - * - Timer for zeroing out all ephemeral keys after `(REJECT_AFTER_TIME * 3)` ms - * if no new keys have been received. - * - * - Timer for, if enabled, sending an empty authenticated packet every user- - * specified seconds. - */ - -static inline void mod_peer_timer(struct wg_peer *peer, - struct timer_list *timer, - unsigned long expires) -{ - rcu_read_lock_bh(); - if (likely(netif_running(peer->device->dev) && - !READ_ONCE(peer->is_dead))) - mod_timer(timer, expires); - rcu_read_unlock_bh(); -} - -static void wg_expired_retransmit_handshake(struct timer_list *timer) -{ - struct wg_peer *peer = from_timer(peer, timer, - timer_retransmit_handshake); - - if (peer->timer_handshake_attempts > MAX_TIMER_HANDSHAKES) { - pr_debug("%s: Handshake for peer %llu (%pISpfsc) did not complete after %d attempts, giving up\n", - peer->device->dev->name, peer->internal_id, - &peer->endpoint.addr, MAX_TIMER_HANDSHAKES + 2); - - del_timer(&peer->timer_send_keepalive); - /* We drop all packets without a keypair and don't try again, - * if we try unsuccessfully for too long to make a handshake. - */ - wg_packet_purge_staged_packets(peer); - - /* We set a timer for destroying any residue that might be left - * of a partial exchange. - */ - if (!timer_pending(&peer->timer_zero_key_material)) - mod_peer_timer(peer, &peer->timer_zero_key_material, - jiffies + REJECT_AFTER_TIME * 3 * HZ); - } else { - ++peer->timer_handshake_attempts; - pr_debug("%s: Handshake for peer %llu (%pISpfsc) did not complete after %d seconds, retrying (try %d)\n", - peer->device->dev->name, peer->internal_id, - &peer->endpoint.addr, REKEY_TIMEOUT, - peer->timer_handshake_attempts + 1); - - /* We clear the endpoint address src address, in case this is - * the cause of trouble. - */ - wg_socket_clear_peer_endpoint_src(peer); - - wg_packet_send_queued_handshake_initiation(peer, true); - } -} - -static void wg_expired_send_keepalive(struct timer_list *timer) -{ - struct wg_peer *peer = from_timer(peer, timer, timer_send_keepalive); - - wg_packet_send_keepalive(peer); - if (peer->timer_need_another_keepalive) { - peer->timer_need_another_keepalive = false; - mod_peer_timer(peer, &peer->timer_send_keepalive, - jiffies + KEEPALIVE_TIMEOUT * HZ); - } -} - -static void wg_expired_new_handshake(struct timer_list *timer) -{ - struct wg_peer *peer = from_timer(peer, timer, timer_new_handshake); - - pr_debug("%s: Retrying handshake with peer %llu (%pISpfsc) because we stopped hearing back after %d seconds\n", - peer->device->dev->name, peer->internal_id, - &peer->endpoint.addr, KEEPALIVE_TIMEOUT + REKEY_TIMEOUT); - /* We clear the endpoint address src address, in case this is the cause - * of trouble. - */ - wg_socket_clear_peer_endpoint_src(peer); - wg_packet_send_queued_handshake_initiation(peer, false); -} - -static void wg_expired_zero_key_material(struct timer_list *timer) -{ - struct wg_peer *peer = from_timer(peer, timer, timer_zero_key_material); - - rcu_read_lock_bh(); - if (!READ_ONCE(peer->is_dead)) { - wg_peer_get(peer); - if (!queue_work(peer->device->handshake_send_wq, - &peer->clear_peer_work)) - /* If the work was already on the queue, we want to drop - * the extra reference. - */ - wg_peer_put(peer); - } - rcu_read_unlock_bh(); -} - -static void wg_queued_expired_zero_key_material(struct work_struct *work) -{ - struct wg_peer *peer = container_of(work, struct wg_peer, - clear_peer_work); - - pr_debug("%s: Zeroing out all keys for peer %llu (%pISpfsc), since we haven't received a new one in %d seconds\n", - peer->device->dev->name, peer->internal_id, - &peer->endpoint.addr, REJECT_AFTER_TIME * 3); - wg_noise_handshake_clear(&peer->handshake); - wg_noise_keypairs_clear(&peer->keypairs); - wg_peer_put(peer); -} - -static void wg_expired_send_persistent_keepalive(struct timer_list *timer) -{ - struct wg_peer *peer = from_timer(peer, timer, - timer_persistent_keepalive); - - if (likely(peer->persistent_keepalive_interval)) - wg_packet_send_keepalive(peer); -} - -/* Should be called after an authenticated data packet is sent. */ -void wg_timers_data_sent(struct wg_peer *peer) -{ - if (!timer_pending(&peer->timer_new_handshake)) - mod_peer_timer(peer, &peer->timer_new_handshake, - jiffies + (KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) * HZ + - prandom_u32_max(REKEY_TIMEOUT_JITTER_MAX_JIFFIES)); -} - -/* Should be called after an authenticated data packet is received. */ -void wg_timers_data_received(struct wg_peer *peer) -{ - if (likely(netif_running(peer->device->dev))) { - if (!timer_pending(&peer->timer_send_keepalive)) - mod_peer_timer(peer, &peer->timer_send_keepalive, - jiffies + KEEPALIVE_TIMEOUT * HZ); - else - peer->timer_need_another_keepalive = true; - } -} - -/* Should be called after any type of authenticated packet is sent, whether - * keepalive, data, or handshake. - */ -void wg_timers_any_authenticated_packet_sent(struct wg_peer *peer) -{ - del_timer(&peer->timer_send_keepalive); -} - -/* Should be called after any type of authenticated packet is received, whether - * keepalive, data, or handshake. - */ -void wg_timers_any_authenticated_packet_received(struct wg_peer *peer) -{ - del_timer(&peer->timer_new_handshake); -} - -/* Should be called after a handshake initiation message is sent. */ -void wg_timers_handshake_initiated(struct wg_peer *peer) -{ - mod_peer_timer(peer, &peer->timer_retransmit_handshake, - jiffies + REKEY_TIMEOUT * HZ + - prandom_u32_max(REKEY_TIMEOUT_JITTER_MAX_JIFFIES)); -} - -/* Should be called after a handshake response message is received and processed - * or when getting key confirmation via the first data message. - */ -void wg_timers_handshake_complete(struct wg_peer *peer) -{ - del_timer(&peer->timer_retransmit_handshake); - peer->timer_handshake_attempts = 0; - peer->sent_lastminute_handshake = false; - ktime_get_real_ts64(&peer->walltime_last_handshake); -} - -/* Should be called after an ephemeral key is created, which is before sending a - * handshake response or after receiving a handshake response. - */ -void wg_timers_session_derived(struct wg_peer *peer) -{ - mod_peer_timer(peer, &peer->timer_zero_key_material, - jiffies + REJECT_AFTER_TIME * 3 * HZ); -} - -/* Should be called before a packet with authentication, whether - * keepalive, data, or handshakem is sent, or after one is received. - */ -void wg_timers_any_authenticated_packet_traversal(struct wg_peer *peer) -{ - if (peer->persistent_keepalive_interval) - mod_peer_timer(peer, &peer->timer_persistent_keepalive, - jiffies + peer->persistent_keepalive_interval * HZ); -} - -void wg_timers_init(struct wg_peer *peer) -{ - timer_setup(&peer->timer_retransmit_handshake, - wg_expired_retransmit_handshake, 0); - timer_setup(&peer->timer_send_keepalive, wg_expired_send_keepalive, 0); - timer_setup(&peer->timer_new_handshake, wg_expired_new_handshake, 0); - timer_setup(&peer->timer_zero_key_material, - wg_expired_zero_key_material, 0); - timer_setup(&peer->timer_persistent_keepalive, - wg_expired_send_persistent_keepalive, 0); - INIT_WORK(&peer->clear_peer_work, wg_queued_expired_zero_key_material); - peer->timer_handshake_attempts = 0; - peer->sent_lastminute_handshake = false; - peer->timer_need_another_keepalive = false; -} - -void wg_timers_stop(struct wg_peer *peer) -{ - del_timer_sync(&peer->timer_retransmit_handshake); - del_timer_sync(&peer->timer_send_keepalive); - del_timer_sync(&peer->timer_new_handshake); - del_timer_sync(&peer->timer_zero_key_material); - del_timer_sync(&peer->timer_persistent_keepalive); - flush_work(&peer->clear_peer_work); -} diff --git a/drivers/net/wireguard/timers.h b/drivers/net/wireguard/timers.h deleted file mode 100644 index f0653dcb13266..0000000000000 --- a/drivers/net/wireguard/timers.h +++ /dev/null @@ -1,31 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#ifndef _WG_TIMERS_H -#define _WG_TIMERS_H - -#include - -struct wg_peer; - -void wg_timers_init(struct wg_peer *peer); -void wg_timers_stop(struct wg_peer *peer); -void wg_timers_data_sent(struct wg_peer *peer); -void wg_timers_data_received(struct wg_peer *peer); -void wg_timers_any_authenticated_packet_sent(struct wg_peer *peer); -void wg_timers_any_authenticated_packet_received(struct wg_peer *peer); -void wg_timers_handshake_initiated(struct wg_peer *peer); -void wg_timers_handshake_complete(struct wg_peer *peer); -void wg_timers_session_derived(struct wg_peer *peer); -void wg_timers_any_authenticated_packet_traversal(struct wg_peer *peer); - -static inline bool wg_birthdate_has_expired(u64 birthday_nanoseconds, - u64 expiration_seconds) -{ - return (s64)(birthday_nanoseconds + expiration_seconds * NSEC_PER_SEC) - <= (s64)ktime_get_coarse_boottime_ns(); -} - -#endif /* _WG_TIMERS_H */ diff --git a/drivers/net/wireguard/version.h b/drivers/net/wireguard/version.h deleted file mode 100644 index a1a269a116344..0000000000000 --- a/drivers/net/wireguard/version.h +++ /dev/null @@ -1 +0,0 @@ -#define WIREGUARD_VERSION "1.0.0" diff --git a/include/crypto/curve25519.h b/include/crypto/curve25519.h index 9ecb3c1f0f15d..4e6dc840b1592 100644 --- a/include/crypto/curve25519.h +++ b/include/crypto/curve25519.h @@ -33,8 +33,7 @@ bool __must_check curve25519(u8 mypublic[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE], const u8 basepoint[CURVE25519_KEY_SIZE]) { - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CURVE25519) && - (!IS_ENABLED(CONFIG_CRYPTO_CURVE25519_X86) || IS_ENABLED(CONFIG_AS_ADX))) + if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CURVE25519)) curve25519_arch(mypublic, secret, basepoint); else curve25519_generic(mypublic, secret, basepoint); @@ -50,8 +49,7 @@ __must_check curve25519_generate_public(u8 pub[CURVE25519_KEY_SIZE], CURVE25519_KEY_SIZE))) return false; - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CURVE25519) && - (!IS_ENABLED(CONFIG_CRYPTO_CURVE25519_X86) || IS_ENABLED(CONFIG_AS_ADX))) + if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CURVE25519)) curve25519_base_arch(pub, secret); else curve25519_generic(pub, secret, curve25519_base_point); diff --git a/include/crypto/internal/poly1305.h b/include/crypto/internal/poly1305.h index 064e52ca52480..479b0cab2a1ad 100644 --- a/include/crypto/internal/poly1305.h +++ b/include/crypto/internal/poly1305.h @@ -11,23 +11,48 @@ #include /* - * Poly1305 core functions. These only accept whole blocks; the caller must - * handle any needed block buffering and padding. 'hibit' must be 1 for any - * full blocks, or 0 for the final block if it had to be padded. If 'nonce' is - * non-NULL, then it's added at the end to compute the Poly1305 MAC. Otherwise, - * only the ε-almost-∆-universal hash function (not the full MAC) is computed. + * Poly1305 core functions. These implement the ε-almost-∆-universal hash + * function underlying the Poly1305 MAC, i.e. they don't add an encrypted nonce + * ("s key") at the end. They also only support block-aligned inputs. */ - -void poly1305_core_setkey(struct poly1305_core_key *key, const u8 *raw_key); +void poly1305_core_setkey(struct poly1305_key *key, const u8 *raw_key); static inline void poly1305_core_init(struct poly1305_state *state) { *state = (struct poly1305_state){}; } void poly1305_core_blocks(struct poly1305_state *state, - const struct poly1305_core_key *key, const void *src, + const struct poly1305_key *key, const void *src, unsigned int nblocks, u32 hibit); -void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], - void *dst); +void poly1305_core_emit(const struct poly1305_state *state, void *dst); + +/* + * Poly1305 requires a unique key for each tag, which implies that we can't set + * it on the tfm that gets accessed by multiple users simultaneously. Instead we + * expect the key as the first 32 bytes in the update() call. + */ +static inline +unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, + const u8 *src, unsigned int srclen) +{ + if (!dctx->sset) { + if (!dctx->rset && srclen >= POLY1305_BLOCK_SIZE) { + poly1305_core_setkey(dctx->r, src); + src += POLY1305_BLOCK_SIZE; + srclen -= POLY1305_BLOCK_SIZE; + dctx->rset = 1; + } + if (srclen >= POLY1305_BLOCK_SIZE) { + dctx->s[0] = get_unaligned_le32(src + 0); + dctx->s[1] = get_unaligned_le32(src + 4); + dctx->s[2] = get_unaligned_le32(src + 8); + dctx->s[3] = get_unaligned_le32(src + 12); + src += POLY1305_BLOCK_SIZE; + srclen -= POLY1305_BLOCK_SIZE; + dctx->sset = true; + } + } + return srclen; +} #endif diff --git a/include/crypto/nhpoly1305.h b/include/crypto/nhpoly1305.h index 306925fea1904..53c04423c582e 100644 --- a/include/crypto/nhpoly1305.h +++ b/include/crypto/nhpoly1305.h @@ -7,7 +7,7 @@ #define _NHPOLY1305_H #include -#include +#include /* NH parameterization: */ @@ -33,7 +33,7 @@ #define NHPOLY1305_KEY_SIZE (POLY1305_BLOCK_SIZE + NH_KEY_BYTES) struct nhpoly1305_key { - struct poly1305_core_key poly_key; + struct poly1305_key poly_key; u32 nh_key[NH_KEY_WORDS]; }; diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h index f1f67fc749cf4..74c6e1cd73eee 100644 --- a/include/crypto/poly1305.h +++ b/include/crypto/poly1305.h @@ -13,29 +13,12 @@ #define POLY1305_KEY_SIZE 32 #define POLY1305_DIGEST_SIZE 16 -/* The poly1305_key and poly1305_state types are mostly opaque and - * implementation-defined. Limbs might be in base 2^64 or base 2^26, or - * different yet. The union type provided keeps these 64-bit aligned for the - * case in which this is implemented using 64x64 multiplies. - */ - struct poly1305_key { - union { - u32 r[5]; - u64 r64[3]; - }; -}; - -struct poly1305_core_key { - struct poly1305_key key; - struct poly1305_key precomputed_s; + u32 r[5]; /* key, base 2^26 */ }; struct poly1305_state { - union { - u32 h[5]; - u64 h64[3]; - }; + u32 h[5]; /* accumulator, base 2^26 */ }; struct poly1305_desc_ctx { @@ -52,10 +35,7 @@ struct poly1305_desc_ctx { /* accumulator */ struct poly1305_state h; /* key */ - union { - struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE]; - struct poly1305_core_key core_r; - }; + struct poly1305_key r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE]; }; void poly1305_init_arch(struct poly1305_desc_ctx *desc, const u8 *key); diff --git a/include/linux/icmpv6.h b/include/linux/icmpv6.h index 33d3796023144..ef1cbb5f454f7 100644 --- a/include/linux/icmpv6.h +++ b/include/linux/icmpv6.h @@ -22,22 +22,12 @@ extern int inet6_unregister_icmp_sender(ip6_icmp_send_t *fn); int ip6_err_gen_icmpv6_unreach(struct sk_buff *skb, int nhs, int type, unsigned int data_len); -#if IS_ENABLED(CONFIG_NF_NAT) -void icmpv6_ndo_send(struct sk_buff *skb_in, u8 type, u8 code, __u32 info); -#else -#define icmpv6_ndo_send icmpv6_send -#endif - #else static inline void icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info) { -} -static inline void icmpv6_ndo_send(struct sk_buff *skb, - u8 type, u8 code, __u32 info) -{ } #endif diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index ca0df5f019dfd..f0553ed6f4c64 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1480,11 +1480,6 @@ static inline void skb_mark_not_on_list(struct sk_buff *skb) skb->next = NULL; } -/* Iterate through singly-linked GSO fragments of an skb. */ -#define skb_list_walk_safe(first, skb, next_skb) \ - for ((skb) = (first), (next_skb) = (skb) ? (skb)->next : NULL; (skb); \ - (skb) = (next_skb), (next_skb) = (skb) ? (skb)->next : NULL) - static inline void skb_list_del_init(struct sk_buff *skb) { __list_del_entry(&skb->list); diff --git a/include/net/icmp.h b/include/net/icmp.h index 9ac2d2672a938..5d4bfdba9adf0 100644 --- a/include/net/icmp.h +++ b/include/net/icmp.h @@ -43,12 +43,6 @@ static inline void icmp_send(struct sk_buff *skb_in, int type, int code, __be32 __icmp_send(skb_in, type, code, info, &IPCB(skb_in)->opt); } -#if IS_ENABLED(CONFIG_NF_NAT) -void icmp_ndo_send(struct sk_buff *skb_in, int type, int code, __be32 info); -#else -#define icmp_ndo_send icmp_send -#endif - int icmp_rcv(struct sk_buff *skb); int icmp_err(struct sk_buff *skb, u32 info); int icmp_init(void); diff --git a/include/uapi/linux/wireguard.h b/include/uapi/linux/wireguard.h deleted file mode 100644 index ae88be14c9478..0000000000000 --- a/include/uapi/linux/wireguard.h +++ /dev/null @@ -1,196 +0,0 @@ -/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */ -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - * - * Documentation - * ============= - * - * The below enums and macros are for interfacing with WireGuard, using generic - * netlink, with family WG_GENL_NAME and version WG_GENL_VERSION. It defines two - * methods: get and set. Note that while they share many common attributes, - * these two functions actually accept a slightly different set of inputs and - * outputs. - * - * WG_CMD_GET_DEVICE - * ----------------- - * - * May only be called via NLM_F_REQUEST | NLM_F_DUMP. The command should contain - * one but not both of: - * - * WGDEVICE_A_IFINDEX: NLA_U32 - * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 - * - * The kernel will then return several messages (NLM_F_MULTI) containing the - * following tree of nested items: - * - * WGDEVICE_A_IFINDEX: NLA_U32 - * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 - * WGDEVICE_A_PRIVATE_KEY: NLA_EXACT_LEN, len WG_KEY_LEN - * WGDEVICE_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN - * WGDEVICE_A_LISTEN_PORT: NLA_U16 - * WGDEVICE_A_FWMARK: NLA_U32 - * WGDEVICE_A_PEERS: NLA_NESTED - * 0: NLA_NESTED - * WGPEER_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN - * WGPEER_A_PRESHARED_KEY: NLA_EXACT_LEN, len WG_KEY_LEN - * WGPEER_A_ENDPOINT: NLA_MIN_LEN(struct sockaddr), struct sockaddr_in or struct sockaddr_in6 - * WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL: NLA_U16 - * WGPEER_A_LAST_HANDSHAKE_TIME: NLA_EXACT_LEN, struct __kernel_timespec - * WGPEER_A_RX_BYTES: NLA_U64 - * WGPEER_A_TX_BYTES: NLA_U64 - * WGPEER_A_ALLOWEDIPS: NLA_NESTED - * 0: NLA_NESTED - * WGALLOWEDIP_A_FAMILY: NLA_U16 - * WGALLOWEDIP_A_IPADDR: NLA_MIN_LEN(struct in_addr), struct in_addr or struct in6_addr - * WGALLOWEDIP_A_CIDR_MASK: NLA_U8 - * 0: NLA_NESTED - * ... - * 0: NLA_NESTED - * ... - * ... - * WGPEER_A_PROTOCOL_VERSION: NLA_U32 - * 0: NLA_NESTED - * ... - * ... - * - * It is possible that all of the allowed IPs of a single peer will not - * fit within a single netlink message. In that case, the same peer will - * be written in the following message, except it will only contain - * WGPEER_A_PUBLIC_KEY and WGPEER_A_ALLOWEDIPS. This may occur several - * times in a row for the same peer. It is then up to the receiver to - * coalesce adjacent peers. Likewise, it is possible that all peers will - * not fit within a single message. So, subsequent peers will be sent - * in following messages, except those will only contain WGDEVICE_A_IFNAME - * and WGDEVICE_A_PEERS. It is then up to the receiver to coalesce these - * messages to form the complete list of peers. - * - * Since this is an NLA_F_DUMP command, the final message will always be - * NLMSG_DONE, even if an error occurs. However, this NLMSG_DONE message - * contains an integer error code. It is either zero or a negative error - * code corresponding to the errno. - * - * WG_CMD_SET_DEVICE - * ----------------- - * - * May only be called via NLM_F_REQUEST. The command should contain the - * following tree of nested items, containing one but not both of - * WGDEVICE_A_IFINDEX and WGDEVICE_A_IFNAME: - * - * WGDEVICE_A_IFINDEX: NLA_U32 - * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 - * WGDEVICE_A_FLAGS: NLA_U32, 0 or WGDEVICE_F_REPLACE_PEERS if all current - * peers should be removed prior to adding the list below. - * WGDEVICE_A_PRIVATE_KEY: len WG_KEY_LEN, all zeros to remove - * WGDEVICE_A_LISTEN_PORT: NLA_U16, 0 to choose randomly - * WGDEVICE_A_FWMARK: NLA_U32, 0 to disable - * WGDEVICE_A_PEERS: NLA_NESTED - * 0: NLA_NESTED - * WGPEER_A_PUBLIC_KEY: len WG_KEY_LEN - * WGPEER_A_FLAGS: NLA_U32, 0 and/or WGPEER_F_REMOVE_ME if the - * specified peer should not exist at the end of the - * operation, rather than added/updated and/or - * WGPEER_F_REPLACE_ALLOWEDIPS if all current allowed - * IPs of this peer should be removed prior to adding - * the list below and/or WGPEER_F_UPDATE_ONLY if the - * peer should only be set if it already exists. - * WGPEER_A_PRESHARED_KEY: len WG_KEY_LEN, all zeros to remove - * WGPEER_A_ENDPOINT: struct sockaddr_in or struct sockaddr_in6 - * WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL: NLA_U16, 0 to disable - * WGPEER_A_ALLOWEDIPS: NLA_NESTED - * 0: NLA_NESTED - * WGALLOWEDIP_A_FAMILY: NLA_U16 - * WGALLOWEDIP_A_IPADDR: struct in_addr or struct in6_addr - * WGALLOWEDIP_A_CIDR_MASK: NLA_U8 - * 0: NLA_NESTED - * ... - * 0: NLA_NESTED - * ... - * ... - * WGPEER_A_PROTOCOL_VERSION: NLA_U32, should not be set or used at - * all by most users of this API, as the - * most recent protocol will be used when - * this is unset. Otherwise, must be set - * to 1. - * 0: NLA_NESTED - * ... - * ... - * - * It is possible that the amount of configuration data exceeds that of - * the maximum message length accepted by the kernel. In that case, several - * messages should be sent one after another, with each successive one - * filling in information not contained in the prior. Note that if - * WGDEVICE_F_REPLACE_PEERS is specified in the first message, it probably - * should not be specified in fragments that come after, so that the list - * of peers is only cleared the first time but appended after. Likewise for - * peers, if WGPEER_F_REPLACE_ALLOWEDIPS is specified in the first message - * of a peer, it likely should not be specified in subsequent fragments. - * - * If an error occurs, NLMSG_ERROR will reply containing an errno. - */ - -#ifndef _WG_UAPI_WIREGUARD_H -#define _WG_UAPI_WIREGUARD_H - -#define WG_GENL_NAME "wireguard" -#define WG_GENL_VERSION 1 - -#define WG_KEY_LEN 32 - -enum wg_cmd { - WG_CMD_GET_DEVICE, - WG_CMD_SET_DEVICE, - __WG_CMD_MAX -}; -#define WG_CMD_MAX (__WG_CMD_MAX - 1) - -enum wgdevice_flag { - WGDEVICE_F_REPLACE_PEERS = 1U << 0, - __WGDEVICE_F_ALL = WGDEVICE_F_REPLACE_PEERS -}; -enum wgdevice_attribute { - WGDEVICE_A_UNSPEC, - WGDEVICE_A_IFINDEX, - WGDEVICE_A_IFNAME, - WGDEVICE_A_PRIVATE_KEY, - WGDEVICE_A_PUBLIC_KEY, - WGDEVICE_A_FLAGS, - WGDEVICE_A_LISTEN_PORT, - WGDEVICE_A_FWMARK, - WGDEVICE_A_PEERS, - __WGDEVICE_A_LAST -}; -#define WGDEVICE_A_MAX (__WGDEVICE_A_LAST - 1) - -enum wgpeer_flag { - WGPEER_F_REMOVE_ME = 1U << 0, - WGPEER_F_REPLACE_ALLOWEDIPS = 1U << 1, - WGPEER_F_UPDATE_ONLY = 1U << 2, - __WGPEER_F_ALL = WGPEER_F_REMOVE_ME | WGPEER_F_REPLACE_ALLOWEDIPS | - WGPEER_F_UPDATE_ONLY -}; -enum wgpeer_attribute { - WGPEER_A_UNSPEC, - WGPEER_A_PUBLIC_KEY, - WGPEER_A_PRESHARED_KEY, - WGPEER_A_FLAGS, - WGPEER_A_ENDPOINT, - WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL, - WGPEER_A_LAST_HANDSHAKE_TIME, - WGPEER_A_RX_BYTES, - WGPEER_A_TX_BYTES, - WGPEER_A_ALLOWEDIPS, - WGPEER_A_PROTOCOL_VERSION, - __WGPEER_A_LAST -}; -#define WGPEER_A_MAX (__WGPEER_A_LAST - 1) - -enum wgallowedip_attribute { - WGALLOWEDIP_A_UNSPEC, - WGALLOWEDIP_A_FAMILY, - WGALLOWEDIP_A_IPADDR, - WGALLOWEDIP_A_CIDR_MASK, - __WGALLOWEDIP_A_LAST -}; -#define WGALLOWEDIP_A_MAX (__WGALLOWEDIP_A_LAST - 1) - -#endif /* _WG_UAPI_WIREGUARD_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 14c032de276e6..0b2c4fce26d95 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -90,7 +90,7 @@ config CRYPTO_LIB_DES config CRYPTO_LIB_POLY1305_RSIZE int default 2 if MIPS - default 11 if X86_64 + default 4 if X86_64 default 9 if ARM || ARM64 default 1 diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index d19e5195569ee..34a701ab8b92e 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -28,9 +28,7 @@ obj-$(CONFIG_CRYPTO_LIB_DES) += libdes.o libdes-y := des.o obj-$(CONFIG_CRYPTO_LIB_POLY1305_GENERIC) += libpoly1305.o -libpoly1305-y := poly1305-donna32.o -libpoly1305-$(CONFIG_ARCH_SUPPORTS_INT128) := poly1305-donna64.o -libpoly1305-y += poly1305.o +libpoly1305-y := poly1305.o obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o libsha256-y := sha256.o diff --git a/lib/crypto/chacha20poly1305-selftest.c b/lib/crypto/chacha20poly1305-selftest.c index fa43deda2660d..465de46dbdef2 100644 --- a/lib/crypto/chacha20poly1305-selftest.c +++ b/lib/crypto/chacha20poly1305-selftest.c @@ -4,7 +4,6 @@ */ #include -#include #include #include @@ -1927,1104 +1926,6 @@ static const u8 enc_key012[] __initconst = { 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 }; -/* wycheproof - rfc7539 */ -static const u8 enc_input013[] __initconst = { - 0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61, - 0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c, - 0x65, 0x6d, 0x65, 0x6e, 0x20, 0x6f, 0x66, 0x20, - 0x74, 0x68, 0x65, 0x20, 0x63, 0x6c, 0x61, 0x73, - 0x73, 0x20, 0x6f, 0x66, 0x20, 0x27, 0x39, 0x39, - 0x3a, 0x20, 0x49, 0x66, 0x20, 0x49, 0x20, 0x63, - 0x6f, 0x75, 0x6c, 0x64, 0x20, 0x6f, 0x66, 0x66, - 0x65, 0x72, 0x20, 0x79, 0x6f, 0x75, 0x20, 0x6f, - 0x6e, 0x6c, 0x79, 0x20, 0x6f, 0x6e, 0x65, 0x20, - 0x74, 0x69, 0x70, 0x20, 0x66, 0x6f, 0x72, 0x20, - 0x74, 0x68, 0x65, 0x20, 0x66, 0x75, 0x74, 0x75, - 0x72, 0x65, 0x2c, 0x20, 0x73, 0x75, 0x6e, 0x73, - 0x63, 0x72, 0x65, 0x65, 0x6e, 0x20, 0x77, 0x6f, - 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69, - 0x74, 0x2e -}; -static const u8 enc_output013[] __initconst = { - 0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb, - 0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2, - 0xa4, 0xad, 0xed, 0x51, 0x29, 0x6e, 0x08, 0xfe, - 0xa9, 0xe2, 0xb5, 0xa7, 0x36, 0xee, 0x62, 0xd6, - 0x3d, 0xbe, 0xa4, 0x5e, 0x8c, 0xa9, 0x67, 0x12, - 0x82, 0xfa, 0xfb, 0x69, 0xda, 0x92, 0x72, 0x8b, - 0x1a, 0x71, 0xde, 0x0a, 0x9e, 0x06, 0x0b, 0x29, - 0x05, 0xd6, 0xa5, 0xb6, 0x7e, 0xcd, 0x3b, 0x36, - 0x92, 0xdd, 0xbd, 0x7f, 0x2d, 0x77, 0x8b, 0x8c, - 0x98, 0x03, 0xae, 0xe3, 0x28, 0x09, 0x1b, 0x58, - 0xfa, 0xb3, 0x24, 0xe4, 0xfa, 0xd6, 0x75, 0x94, - 0x55, 0x85, 0x80, 0x8b, 0x48, 0x31, 0xd7, 0xbc, - 0x3f, 0xf4, 0xde, 0xf0, 0x8e, 0x4b, 0x7a, 0x9d, - 0xe5, 0x76, 0xd2, 0x65, 0x86, 0xce, 0xc6, 0x4b, - 0x61, 0x16, 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, - 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, - 0x06, 0x91 -}; -static const u8 enc_assoc013[] __initconst = { - 0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, - 0xc4, 0xc5, 0xc6, 0xc7 -}; -static const u8 enc_nonce013[] __initconst = { - 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, - 0x44, 0x45, 0x46, 0x47 -}; -static const u8 enc_key013[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - -/* wycheproof - misc */ -static const u8 enc_input014[] __initconst = { }; -static const u8 enc_output014[] __initconst = { - 0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5, - 0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d -}; -static const u8 enc_assoc014[] __initconst = { }; -static const u8 enc_nonce014[] __initconst = { - 0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1, - 0xea, 0x12, 0x37, 0x9d -}; -static const u8 enc_key014[] __initconst = { - 0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96, - 0x5e, 0xa3, 0x71, 0xd5, 0xff, 0x07, 0x3c, 0xf0, - 0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, 0xb2, 0x08, - 0x42, 0x6e, 0x11, 0x40, 0x9c, 0x09, 0xb9, 0xb0 -}; - -/* wycheproof - misc */ -static const u8 enc_input015[] __initconst = { }; -static const u8 enc_output015[] __initconst = { - 0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b, - 0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09 -}; -static const u8 enc_assoc015[] __initconst = { - 0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10 -}; -static const u8 enc_nonce015[] __initconst = { - 0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16, - 0xa3, 0xc6, 0xf6, 0x89 -}; -static const u8 enc_key015[] __initconst = { - 0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb, - 0x20, 0x4d, 0xb2, 0xc3, 0xf5, 0xc7, 0x46, 0x22, - 0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, 0x19, 0x63, - 0x91, 0xdb, 0xb9, 0x57, 0x7a, 0x25, 0x07, 0x42 -}; - -/* wycheproof - misc */ -static const u8 enc_input016[] __initconst = { - 0x2a -}; -static const u8 enc_output016[] __initconst = { - 0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80, - 0x1e, 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75, - 0x22 -}; -static const u8 enc_assoc016[] __initconst = { }; -static const u8 enc_nonce016[] __initconst = { - 0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd, - 0xee, 0xab, 0x60, 0xf1 -}; -static const u8 enc_key016[] __initconst = { - 0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50, - 0x08, 0xf5, 0x48, 0x4b, 0x4c, 0xb8, 0x03, 0xfa, - 0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, 0x1f, 0x6a, - 0xb6, 0xae, 0xf4, 0x91, 0x6a, 0x76, 0x62, 0x73 -}; - -/* wycheproof - misc */ -static const u8 enc_input017[] __initconst = { - 0x51 -}; -static const u8 enc_output017[] __initconst = { - 0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7, - 0xc6, 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72, - 0xb9 -}; -static const u8 enc_assoc017[] __initconst = { - 0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53 -}; -static const u8 enc_nonce017[] __initconst = { - 0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2, - 0x78, 0x2f, 0x44, 0x03 -}; -static const u8 enc_key017[] __initconst = { - 0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5, - 0x2b, 0xdb, 0x4a, 0x70, 0xb4, 0x43, 0x19, 0x9f, - 0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, 0x20, 0xc5, - 0x5e, 0x4b, 0x70, 0xf0, 0xfd, 0xa6, 0x20, 0xee -}; - -/* wycheproof - misc */ -static const u8 enc_input018[] __initconst = { - 0x5c, 0x60 -}; -static const u8 enc_output018[] __initconst = { - 0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39, - 0xc1, 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22, - 0xe5, 0xe2 -}; -static const u8 enc_assoc018[] __initconst = { }; -static const u8 enc_nonce018[] __initconst = { - 0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34, - 0x7e, 0x03, 0xf2, 0xdb -}; -static const u8 enc_key018[] __initconst = { - 0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89, - 0x19, 0x49, 0x89, 0x74, 0x35, 0x07, 0xbf, 0x3e, - 0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, 0x69, 0x5f, - 0xe6, 0xc0, 0x25, 0xfd, 0x9b, 0xa4, 0xc1, 0x00 -}; - -/* wycheproof - misc */ -static const u8 enc_input019[] __initconst = { - 0xdd, 0xf2 -}; -static const u8 enc_output019[] __initconst = { - 0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec, - 0x2e, 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24, - 0x37, 0xa2 -}; -static const u8 enc_assoc019[] __initconst = { - 0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf -}; -static const u8 enc_nonce019[] __initconst = { - 0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90, - 0xb6, 0x04, 0x0a, 0xc6 -}; -static const u8 enc_key019[] __initconst = { - 0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48, - 0xaa, 0x20, 0x30, 0xea, 0xcf, 0xe7, 0x2b, 0xff, - 0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, 0x33, 0x44, - 0xe5, 0x71, 0x8f, 0xe0, 0xd7, 0xab, 0x1a, 0x58 -}; - -/* wycheproof - misc */ -static const u8 enc_input020[] __initconst = { - 0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31 -}; -static const u8 enc_output020[] __initconst = { - 0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed, - 0x7a, 0x43, 0x4e, 0x26, 0x02, 0xd3, 0x94, 0x28, - 0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42 -}; -static const u8 enc_assoc020[] __initconst = { }; -static const u8 enc_nonce020[] __initconst = { - 0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59, - 0x6d, 0xc5, 0x5b, 0xb7 -}; -static const u8 enc_key020[] __initconst = { - 0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f, - 0x1f, 0x70, 0x21, 0xea, 0xb6, 0x9b, 0x70, 0x3f, - 0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, 0xf5, 0xd3, - 0x4a, 0x37, 0x4f, 0xdb, 0xfc, 0x5a, 0xda, 0xc7 -}; - -/* wycheproof - misc */ -static const u8 enc_input021[] __initconst = { - 0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90 -}; -static const u8 enc_output021[] __initconst = { - 0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18, - 0x5f, 0x14, 0x08, 0xee, 0x7f, 0xbf, 0x18, 0xf5, - 0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba -}; -static const u8 enc_assoc021[] __initconst = { - 0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53 -}; -static const u8 enc_nonce021[] __initconst = { - 0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98, - 0xde, 0x94, 0x83, 0x96 -}; -static const u8 enc_key021[] __initconst = { - 0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5, - 0x16, 0xda, 0x6a, 0x33, 0x02, 0x2a, 0x1d, 0xd4, - 0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, 0xc5, 0xda, - 0x52, 0xa7, 0x30, 0xf3, 0x4d, 0x84, 0x0d, 0x7f -}; - -/* wycheproof - misc */ -static const u8 enc_input022[] __initconst = { - 0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed -}; -static const u8 enc_output022[] __initconst = { - 0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17, - 0xfb, 0xe8, 0xa6, 0xed, 0x8f, 0xab, 0xb1, 0x93, - 0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21 -}; -static const u8 enc_assoc022[] __initconst = { }; -static const u8 enc_nonce022[] __initconst = { - 0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2, - 0x36, 0x18, 0x23, 0xd3 -}; -static const u8 enc_key022[] __initconst = { - 0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf, - 0xf1, 0xa4, 0xbc, 0x05, 0x55, 0x0b, 0xa8, 0x1d, - 0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, 0x12, 0xc7, - 0xb0, 0x0e, 0x60, 0xe4, 0x8d, 0xb7, 0xe1, 0x52 -}; - -/* wycheproof - misc */ -static const u8 enc_input023[] __initconst = { - 0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf -}; -static const u8 enc_output023[] __initconst = { - 0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0, - 0x45, 0x30, 0xf5, 0x1a, 0xee, 0xe0, 0x24, 0xe0, - 0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18 -}; -static const u8 enc_assoc023[] __initconst = { - 0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d -}; -static const u8 enc_nonce023[] __initconst = { - 0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33, - 0x66, 0x48, 0x4a, 0x78 -}; -static const u8 enc_key023[] __initconst = { - 0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54, - 0xde, 0x7a, 0x4a, 0xc5, 0x51, 0x95, 0x0f, 0x1a, - 0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, 0x75, 0xfe, - 0x4a, 0xbd, 0x56, 0x08, 0xd7, 0xcd, 0xda, 0xdd -}; - -/* wycheproof - misc */ -static const u8 enc_input024[] __initconst = { - 0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c, - 0x36, 0x8d, 0x14, 0xe0 -}; -static const u8 enc_output024[] __initconst = { - 0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a, - 0x6b, 0x50, 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27, - 0x58, 0xc2, 0xe1, 0x4e, 0x36, 0xd4, 0xfc, 0x10, - 0x6d, 0xcb, 0x29, 0xb4 -}; -static const u8 enc_assoc024[] __initconst = { }; -static const u8 enc_nonce024[] __initconst = { - 0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e, - 0x07, 0x53, 0x86, 0x56 -}; -static const u8 enc_key024[] __initconst = { - 0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0, - 0x15, 0xbd, 0xe9, 0x3f, 0x0a, 0xbf, 0x48, 0x39, - 0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, 0x9c, 0xb5, - 0x0b, 0x39, 0xf7, 0xf8, 0xa9, 0xcc, 0x51, 0xbe -}; - -/* wycheproof - misc */ -static const u8 enc_input025[] __initconst = { - 0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7, - 0x59, 0xb1, 0xa0, 0xda -}; -static const u8 enc_output025[] __initconst = { - 0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38, - 0xd9, 0xe5, 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c, - 0x9d, 0x8f, 0xa9, 0x15, 0xfd, 0x84, 0x23, 0xa7, - 0xcf, 0x05, 0x07, 0x2f -}; -static const u8 enc_assoc025[] __initconst = { - 0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9 -}; -static const u8 enc_nonce025[] __initconst = { - 0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d, - 0xd9, 0x06, 0xe9, 0xce -}; -static const u8 enc_key025[] __initconst = { - 0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40, - 0xbc, 0x10, 0x20, 0xc8, 0x13, 0x6f, 0xf2, 0x70, - 0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, 0x6f, 0x1e, - 0x82, 0x51, 0x18, 0xb0, 0x06, 0xb9, 0x12, 0x57 -}; - -/* wycheproof - misc */ -static const u8 enc_input026[] __initconst = { - 0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac, - 0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07 -}; -static const u8 enc_output026[] __initconst = { - 0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf, - 0x53, 0x97, 0x0b, 0xe2, 0x21, 0x00, 0x42, 0x2a, - 0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, 0x6e, 0x79, - 0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2 -}; -static const u8 enc_assoc026[] __initconst = { }; -static const u8 enc_nonce026[] __initconst = { - 0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda, - 0xd4, 0x61, 0xd2, 0x3c -}; -static const u8 enc_key026[] __initconst = { - 0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda, - 0x90, 0x62, 0x19, 0xe0, 0x8c, 0x0d, 0x0d, 0x77, - 0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, 0x4f, 0xe0, - 0x70, 0x9a, 0xf7, 0x55, 0xee, 0xec, 0x6d, 0xcb -}; - -/* wycheproof - misc */ -static const u8 enc_input027[] __initconst = { - 0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f, - 0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73 -}; -static const u8 enc_output027[] __initconst = { - 0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81, - 0x55, 0x74, 0x55, 0xe5, 0x98, 0x0f, 0xed, 0xfe, - 0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, 0xd3, 0xa9, - 0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17 -}; -static const u8 enc_assoc027[] __initconst = { - 0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12 -}; -static const u8 enc_nonce027[] __initconst = { - 0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14, - 0xc5, 0x03, 0x5b, 0x6a -}; -static const u8 enc_key027[] __initconst = { - 0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f, - 0xb5, 0x82, 0x21, 0xb7, 0xed, 0x79, 0x67, 0x38, - 0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, 0xa6, 0x0b, - 0xbc, 0x8b, 0xc4, 0x89, 0x3a, 0x58, 0x89, 0x4d -}; - -/* wycheproof - misc */ -static const u8 enc_input028[] __initconst = { - 0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0, - 0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88 -}; -static const u8 enc_output028[] __initconst = { - 0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4, - 0x83, 0x50, 0x0c, 0xea, 0xba, 0xb6, 0x7a, 0x13, - 0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, 0xa2, 0x2a, - 0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c -}; -static const u8 enc_assoc028[] __initconst = { }; -static const u8 enc_nonce028[] __initconst = { - 0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8, - 0x47, 0x40, 0xad, 0x9b -}; -static const u8 enc_key028[] __initconst = { - 0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7, - 0xd3, 0xdb, 0x99, 0xa8, 0xf5, 0x4b, 0x15, 0xd7, - 0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, 0x97, 0x63, - 0xb0, 0x19, 0xc1, 0x69, 0x9f, 0x87, 0x67, 0x4a -}; - -/* wycheproof - misc */ -static const u8 enc_input029[] __initconst = { - 0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09, - 0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6 -}; -static const u8 enc_output029[] __initconst = { - 0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3, - 0xd3, 0xa8, 0xae, 0x9e, 0xc1, 0x41, 0xc0, 0x0c, - 0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, 0xd0, 0xdc, - 0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d -}; -static const u8 enc_assoc029[] __initconst = { - 0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff -}; -static const u8 enc_nonce029[] __initconst = { - 0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80, - 0x07, 0x1f, 0x52, 0x66 -}; -static const u8 enc_key029[] __initconst = { - 0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8, - 0xa8, 0x01, 0x9e, 0xde, 0xe3, 0xf2, 0x59, 0x14, - 0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, 0x6e, 0x1d, - 0x77, 0x1c, 0x86, 0x25, 0x71, 0xd2, 0xea, 0x1e -}; - -/* wycheproof - misc */ -static const u8 enc_input030[] __initconst = { - 0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15, - 0x91, 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e, - 0x1f -}; -static const u8 enc_output030[] __initconst = { - 0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd, - 0x12, 0xa0, 0x3b, 0xb8, 0x57, 0x97, 0x64, 0x74, - 0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, 0xc3, 0x80, - 0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08, - 0x13 -}; -static const u8 enc_assoc030[] __initconst = { }; -static const u8 enc_nonce030[] __initconst = { - 0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42, - 0xdc, 0x03, 0x44, 0x5d -}; -static const u8 enc_key030[] __initconst = { - 0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21, - 0xc0, 0xcc, 0x24, 0xc0, 0xc0, 0xe2, 0x4c, 0x1e, - 0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, 0x9a, 0x4b, - 0x16, 0x61, 0x76, 0x62, 0x96, 0x16, 0xd0, 0x11 -}; - -/* wycheproof - misc */ -static const u8 enc_input031[] __initconst = { - 0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88, - 0x7b, 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c, - 0x9c -}; -static const u8 enc_output031[] __initconst = { - 0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b, - 0x8d, 0x17, 0xcb, 0xbb, 0x9a, 0xbf, 0xad, 0x9e, - 0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, 0x27, 0x38, - 0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c, - 0xeb -}; -static const u8 enc_assoc031[] __initconst = { - 0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79 -}; -static const u8 enc_nonce031[] __initconst = { - 0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10, - 0x63, 0x3d, 0x99, 0x3d -}; -static const u8 enc_key031[] __initconst = { - 0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7, - 0x6c, 0x70, 0x37, 0xb4, 0x8f, 0xe3, 0x57, 0x7f, - 0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, 0xa9, 0x82, - 0x17, 0x6d, 0x18, 0x23, 0x06, 0xff, 0xb8, 0x70 -}; - -/* wycheproof - misc */ -static const u8 enc_input032[] __initconst = { - 0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66, - 0x5d, 0x9a, 0xe5, 0xef, 0xe2, 0x79, 0xe7, 0xf7, - 0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11 -}; -static const u8 enc_output032[] __initconst = { - 0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d, - 0x9d, 0x8d, 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0, - 0x0d, 0x75, 0xf3, 0x80, 0x74, 0xbf, 0xe7, 0x64, - 0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, 0xff, 0x7d, - 0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a -}; -static const u8 enc_assoc032[] __initconst = { }; -static const u8 enc_nonce032[] __initconst = { - 0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76, - 0x2c, 0x65, 0xf3, 0x1b -}; -static const u8 enc_key032[] __initconst = { - 0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87, - 0xb3, 0x3f, 0x1a, 0x70, 0x90, 0xb6, 0x11, 0x4f, - 0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, 0xf2, 0xc2, - 0x76, 0xbd, 0xcf, 0xdb, 0xc5, 0xce, 0x9c, 0xd7 -}; - -/* wycheproof - misc */ -static const u8 enc_input033[] __initconst = { - 0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d, - 0xe1, 0x20, 0x08, 0x06, 0xa9, 0xee, 0x8e, 0x4c, - 0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93 -}; -static const u8 enc_output033[] __initconst = { - 0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69, - 0x20, 0xe8, 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4, - 0x88, 0x9e, 0xbe, 0x7a, 0x0a, 0xd0, 0xe9, 0x00, - 0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, 0xda, 0x63, - 0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65 -}; -static const u8 enc_assoc033[] __initconst = { - 0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08 -}; -static const u8 enc_nonce033[] __initconst = { - 0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd, - 0x78, 0x34, 0xed, 0x55 -}; -static const u8 enc_key033[] __initconst = { - 0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed, - 0xda, 0x95, 0x4f, 0x1f, 0x73, 0x92, 0x23, 0xda, - 0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, 0x03, 0x3d, - 0x0f, 0xae, 0x4b, 0x02, 0x83, 0xd1, 0x8b, 0xe3 -}; - -/* wycheproof - misc */ -static const u8 enc_input034[] __initconst = { - 0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f, - 0x42, 0xcc, 0x8c, 0x20, 0x3a, 0xb2, 0x4e, 0x2c, - 0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, 0xd3, 0x26, - 0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29 -}; -static const u8 enc_output034[] __initconst = { - 0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab, - 0xb6, 0x88, 0x76, 0xa6, 0xe9, 0x6a, 0x48, 0xfb, - 0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1, - 0x89, 0xc9, 0xfd, 0x2d, 0xb9, 0x17, 0x46, 0x56, - 0x6d, 0x3c, 0xa1, 0x0e, 0x31, 0x1b, 0x69, 0x5f, - 0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93 -}; -static const u8 enc_assoc034[] __initconst = { }; -static const u8 enc_nonce034[] __initconst = { - 0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31, - 0x37, 0x1a, 0x6f, 0xd2 -}; -static const u8 enc_key034[] __initconst = { - 0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e, - 0x6c, 0x54, 0xa8, 0x34, 0x9b, 0xae, 0x50, 0xa2, - 0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, 0xa1, 0x15, - 0x2c, 0x42, 0x4f, 0x03, 0xb6, 0x67, 0x1d, 0x71 -}; - -/* wycheproof - misc */ -static const u8 enc_input035[] __initconst = { - 0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2, - 0xab, 0xc9, 0xab, 0x7f, 0xa2, 0x43, 0x03, 0x5f, - 0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, 0x39, 0x56, - 0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb -}; -static const u8 enc_output035[] __initconst = { - 0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20, - 0x08, 0x46, 0x27, 0x58, 0x00, 0x80, 0xdd, 0xe5, - 0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e, - 0xd6, 0xf3, 0xd3, 0x26, 0xbc, 0x7b, 0x70, 0x53, - 0x4d, 0x4a, 0xa2, 0x83, 0x5a, 0x52, 0xe7, 0x2d, - 0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f -}; -static const u8 enc_assoc035[] __initconst = { - 0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48 -}; -static const u8 enc_nonce035[] __initconst = { - 0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8, - 0xb8, 0x1a, 0x1f, 0x8b -}; -static const u8 enc_key035[] __initconst = { - 0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f, - 0xd1, 0x43, 0x58, 0xfe, 0x2f, 0xf2, 0xa7, 0x40, - 0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, 0xe6, 0xa4, - 0x37, 0xf1, 0x9e, 0xa4, 0x99, 0x11, 0xce, 0xc4 -}; - -/* wycheproof - misc */ -static const u8 enc_input036[] __initconst = { - 0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a, - 0xe8, 0x97, 0xdf, 0x7b, 0xee, 0xca, 0xc1, 0xdb, - 0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, 0x60, 0xce, - 0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78 -}; -static const u8 enc_output036[] __initconst = { - 0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76, - 0x0f, 0x5f, 0xe1, 0x97, 0x23, 0xb9, 0x81, 0x1e, - 0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50, - 0x81, 0xac, 0x43, 0x0b, 0x95, 0x3b, 0x0e, 0x21, - 0x22, 0x25, 0x41, 0xaf, 0x46, 0xb8, 0x65, 0x33, - 0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea -}; -static const u8 enc_assoc036[] __initconst = { }; -static const u8 enc_nonce036[] __initconst = { - 0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2, - 0x2b, 0x7e, 0x6e, 0x6a -}; -static const u8 enc_key036[] __initconst = { - 0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c, - 0xb3, 0xff, 0xe8, 0xb7, 0x86, 0xa2, 0xb3, 0xeb, - 0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, 0x57, 0xb9, - 0xdc, 0x8c, 0x7b, 0xc5, 0x35, 0x46, 0x34, 0xa3 -}; - -/* wycheproof - misc */ -static const u8 enc_input037[] __initconst = { - 0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14, - 0xa1, 0x3a, 0x52, 0x57, 0xed, 0x6c, 0xff, 0xd3, - 0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, 0xb1, 0x79, - 0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e -}; -static const u8 enc_output037[] __initconst = { - 0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b, - 0xf6, 0xbe, 0x95, 0x07, 0xed, 0x4d, 0x1b, 0xb2, - 0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29, - 0xda, 0xf5, 0x07, 0x62, 0x84, 0x6f, 0xf2, 0xf4, - 0x7b, 0x99, 0x7d, 0x93, 0xc9, 0x82, 0x18, 0x9d, - 0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32 -}; -static const u8 enc_assoc037[] __initconst = { - 0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59 -}; -static const u8 enc_nonce037[] __initconst = { - 0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5, - 0x34, 0x0d, 0xd1, 0xb8 -}; -static const u8 enc_key037[] __initconst = { - 0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4, - 0x10, 0x70, 0x2f, 0xea, 0x61, 0xbc, 0x87, 0x3f, - 0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, 0x4a, 0xbd, - 0xe2, 0x5b, 0x78, 0x6a, 0x2d, 0x97, 0xf1, 0x45 -}; - -/* wycheproof - misc */ -static const u8 enc_input038[] __initconst = { - 0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4, - 0x66, 0xcf, 0x9a, 0xbb, 0x34, 0x93, 0x15, 0x2e, - 0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11, - 0x71, 0x34, 0x6b, 0x4d, 0x2c, 0xeb, 0xf8, 0x04, - 0x1a, 0xa3, 0xce, 0xdc, 0x0d, 0xfd, 0x7b, 0x46, - 0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a -}; -static const u8 enc_output038[] __initconst = { - 0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6, - 0x2e, 0x3d, 0xa4, 0x0b, 0xe2, 0xcb, 0x77, 0x00, - 0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6, - 0x52, 0xe7, 0x6a, 0x43, 0x2c, 0x65, 0x8d, 0x27, - 0x09, 0x5f, 0x0e, 0xb8, 0xf9, 0x40, 0xc3, 0x24, - 0x98, 0x1e, 0xa9, 0x35, 0xe5, 0x07, 0xf9, 0x8f, - 0x04, 0x69, 0x56, 0xdb, 0x3a, 0x51, 0x29, 0x08, - 0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9 -}; -static const u8 enc_assoc038[] __initconst = { }; -static const u8 enc_nonce038[] __initconst = { - 0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9, - 0x0b, 0xef, 0x55, 0xd2 -}; -static const u8 enc_key038[] __initconst = { - 0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51, - 0x81, 0x51, 0x65, 0xd5, 0x30, 0x1b, 0x22, 0x63, - 0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, 0x78, 0xe3, - 0xa0, 0xa5, 0xe8, 0xe1, 0xaa, 0xe0, 0xf2, 0x64 -}; - -/* wycheproof - misc */ -static const u8 enc_input039[] __initconst = { - 0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74, - 0x8b, 0xfb, 0x77, 0x85, 0xff, 0x91, 0xee, 0xb3, - 0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d, - 0x5e, 0x7b, 0x0e, 0x79, 0xef, 0x65, 0x36, 0x59, - 0xa9, 0x97, 0x8d, 0x75, 0x54, 0x2e, 0xf9, 0x1c, - 0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9 -}; -static const u8 enc_output039[] __initconst = { - 0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec, - 0xce, 0x24, 0x2e, 0x0e, 0x88, 0xb1, 0x38, 0x04, - 0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e, - 0x19, 0x98, 0x6b, 0x3e, 0x31, 0x8c, 0xae, 0x8d, - 0x80, 0x61, 0x98, 0xfb, 0x4c, 0x52, 0x7c, 0xc3, - 0x93, 0x50, 0xeb, 0xdd, 0xea, 0xc5, 0x73, 0xc4, - 0xcb, 0xf0, 0xbe, 0xfd, 0xa0, 0xb7, 0x02, 0x42, - 0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3 -}; -static const u8 enc_assoc039[] __initconst = { - 0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84 -}; -static const u8 enc_nonce039[] __initconst = { - 0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b, - 0x31, 0xcd, 0x4d, 0x95 -}; -static const u8 enc_key039[] __initconst = { - 0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c, - 0xf5, 0xd2, 0x3c, 0x40, 0x2b, 0xdb, 0x0d, 0x25, - 0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, 0x73, 0xd4, - 0x2e, 0xb5, 0xbe, 0x10, 0x9f, 0xdc, 0xb2, 0xac -}; - -/* wycheproof - misc */ -static const u8 enc_input040[] __initconst = { - 0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e, - 0x00, 0x8f, 0xf8, 0x5d, 0x5d, 0xdc, 0x38, 0xdd, - 0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f, - 0x1e, 0x40, 0x79, 0xcb, 0x64, 0xd0, 0xdc, 0x8f, - 0x5e, 0x67, 0x11, 0xcd, 0x49, 0x21, 0xa7, 0x88, - 0x7d, 0xe7, 0x6e, 0x26, 0x78, 0xfd, 0xc6, 0x76, - 0x18, 0xf1, 0x18, 0x55, 0x86, 0xbf, 0xea, 0x9d, - 0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82 -}; -static const u8 enc_output040[] __initconst = { - 0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5, - 0x75, 0x5c, 0x08, 0xf7, 0x47, 0xc0, 0xf8, 0xd8, - 0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c, - 0x12, 0xbb, 0x51, 0xbb, 0x18, 0x72, 0xc8, 0xe8, - 0xb8, 0x77, 0x67, 0x8b, 0xec, 0x44, 0x2c, 0xfc, - 0xbb, 0x0f, 0xf4, 0x64, 0xa6, 0x4b, 0x74, 0x33, - 0x2c, 0xf0, 0x72, 0x89, 0x8c, 0x7e, 0x0e, 0xdd, - 0xf6, 0x23, 0x2e, 0xa6, 0xe2, 0x7e, 0xfe, 0x50, - 0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56, - 0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13 -}; -static const u8 enc_assoc040[] __initconst = { }; -static const u8 enc_nonce040[] __initconst = { - 0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28, - 0x23, 0xcc, 0x06, 0x5b -}; -static const u8 enc_key040[] __initconst = { - 0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0, - 0xb0, 0x44, 0x47, 0x67, 0xf8, 0x0a, 0x25, 0xb8, - 0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, 0x4d, 0x30, - 0x52, 0x22, 0x6b, 0xaa, 0x1c, 0x6f, 0xb7, 0x01 -}; - -/* wycheproof - misc */ -static const u8 enc_input041[] __initconst = { - 0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88, - 0x32, 0x43, 0x71, 0x36, 0xb4, 0xae, 0x80, 0x5d, - 0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19, - 0x3a, 0x25, 0x01, 0x62, 0x93, 0xed, 0xba, 0x44, - 0x3c, 0x58, 0xe0, 0x7e, 0x7b, 0x71, 0x95, 0xec, - 0x5b, 0xd8, 0x45, 0x82, 0xa9, 0xd5, 0x6c, 0x8d, - 0x4a, 0x10, 0x8c, 0x7d, 0x7c, 0xe3, 0x4e, 0x6c, - 0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17 -}; -static const u8 enc_output041[] __initconst = { - 0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16, - 0x14, 0xf6, 0x36, 0x03, 0x1e, 0xeb, 0x42, 0xf1, - 0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8, - 0x71, 0xee, 0x8e, 0x73, 0x82, 0x0c, 0x82, 0x97, - 0x49, 0xf1, 0xab, 0xb4, 0x29, 0x43, 0x67, 0x84, - 0x9f, 0xb6, 0xc2, 0xaa, 0x56, 0xbd, 0xa8, 0xa3, - 0x07, 0x8f, 0x72, 0x3d, 0x7c, 0x1c, 0x85, 0x20, - 0x24, 0xb0, 0x17, 0xb5, 0x89, 0x73, 0xfb, 0x1e, - 0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14, - 0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec -}; -static const u8 enc_assoc041[] __initconst = { - 0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2 -}; -static const u8 enc_nonce041[] __initconst = { - 0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d, - 0xe4, 0xeb, 0xb1, 0x9c -}; -static const u8 enc_key041[] __initconst = { - 0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9, - 0x99, 0x8a, 0x90, 0x87, 0x5d, 0xa1, 0xd3, 0xa2, - 0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, 0x97, 0xa5, - 0x72, 0x52, 0x35, 0x19, 0x0e, 0x10, 0xa9, 0x7e -}; - -/* wycheproof - misc */ -static const u8 enc_input042[] __initconst = { - 0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba, - 0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, 0x34, 0x58, - 0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06, - 0x9d, 0x12, 0xd4, 0x5c, 0x22, 0x21, 0x75, 0x64, - 0x12, 0x15, 0x88, 0x03, 0x22, 0x97, 0xef, 0xf5, - 0x67, 0x83, 0x74, 0x2a, 0x5f, 0xc2, 0x2d, 0x74, - 0x10, 0xff, 0xb2, 0x9d, 0x66, 0x09, 0x86, 0x61, - 0xd7, 0x6f, 0x12, 0x6c, 0x3c, 0x27, 0x68, 0x9e, - 0x43, 0xb3, 0x72, 0x67, 0xca, 0xc5, 0xa3, 0xa6, - 0xd3, 0xab, 0x49, 0xe3, 0x91, 0xda, 0x29, 0xcd, - 0x30, 0x54, 0xa5, 0x69, 0x2e, 0x28, 0x07, 0xe4, - 0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5, - 0x92 -}; -static const u8 enc_output042[] __initconst = { - 0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97, - 0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf, - 0x23, 0xd1, 0x05, 0xfb, 0x4c, 0x84, 0xcf, 0x98, - 0x51, 0x5e, 0x1b, 0xc9, 0xa6, 0x4f, 0x8a, 0xd5, - 0xbe, 0x8f, 0x07, 0x21, 0xbd, 0xe5, 0x06, 0x45, - 0xd0, 0x00, 0x83, 0xc3, 0xa2, 0x63, 0xa3, 0x10, - 0x53, 0xb7, 0x60, 0x24, 0x5f, 0x52, 0xae, 0x28, - 0x66, 0xa5, 0xec, 0x83, 0xb1, 0x9f, 0x61, 0xbe, - 0x1d, 0x30, 0xd5, 0xc5, 0xd9, 0xfe, 0xcc, 0x4c, - 0xbb, 0xe0, 0x8f, 0xd3, 0x85, 0x81, 0x3a, 0x2a, - 0xa3, 0x9a, 0x00, 0xff, 0x9c, 0x10, 0xf7, 0xf2, - 0x37, 0x02, 0xad, 0xd1, 0xe4, 0xb2, 0xff, 0xa3, - 0x1c, 0x41, 0x86, 0x5f, 0xc7, 0x1d, 0xe1, 0x2b, - 0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b, - 0xb0 -}; -static const u8 enc_assoc042[] __initconst = { }; -static const u8 enc_nonce042[] __initconst = { - 0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03, - 0xac, 0xde, 0x27, 0x99 -}; -static const u8 enc_key042[] __initconst = { - 0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28, - 0x4c, 0xa5, 0xfc, 0x0f, 0x8c, 0x5f, 0x2b, 0x8d, - 0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, 0xa6, 0x6a, - 0x41, 0x99, 0x5e, 0x78, 0x33, 0x51, 0xf9, 0x01 -}; - -/* wycheproof - misc */ -static const u8 enc_input043[] __initconst = { - 0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff, - 0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, 0xb2, 0xcc, - 0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc, - 0x16, 0x5d, 0x06, 0xc2, 0xf5, 0x10, 0x0f, 0xf5, - 0xe8, 0xab, 0x28, 0x82, 0xc4, 0x75, 0xaf, 0xcd, - 0x05, 0xcc, 0xd4, 0x9f, 0x2e, 0x7d, 0x8f, 0x55, - 0xef, 0x3a, 0x72, 0xe3, 0xdc, 0x51, 0xd6, 0x85, - 0x2b, 0x8e, 0x6b, 0x9e, 0x7a, 0xec, 0xe5, 0x7b, - 0xe6, 0x55, 0x6b, 0x0b, 0x6d, 0x94, 0x13, 0xe3, - 0x3f, 0xc5, 0xfc, 0x24, 0xa9, 0xa2, 0x05, 0xad, - 0x59, 0x57, 0x4b, 0xb3, 0x9d, 0x94, 0x4a, 0x92, - 0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31, - 0x76 -}; -static const u8 enc_output043[] __initconst = { - 0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5, - 0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06, - 0x3e, 0x58, 0xed, 0xee, 0x4b, 0xb1, 0x22, 0x7e, - 0x71, 0x10, 0xac, 0x4d, 0x26, 0x20, 0xc2, 0xae, - 0xc2, 0xf8, 0x48, 0xf5, 0x6d, 0xee, 0xb0, 0x37, - 0xa8, 0xdc, 0xed, 0x75, 0xaf, 0xa8, 0xa6, 0xc8, - 0x90, 0xe2, 0xde, 0xe4, 0x2f, 0x95, 0x0b, 0xb3, - 0x3d, 0x9e, 0x24, 0x24, 0xd0, 0x8a, 0x50, 0x5d, - 0x89, 0x95, 0x63, 0x97, 0x3e, 0xd3, 0x88, 0x70, - 0xf3, 0xde, 0x6e, 0xe2, 0xad, 0xc7, 0xfe, 0x07, - 0x2c, 0x36, 0x6c, 0x14, 0xe2, 0xcf, 0x7c, 0xa6, - 0x2f, 0xb3, 0xd3, 0x6b, 0xee, 0x11, 0x68, 0x54, - 0x61, 0xb7, 0x0d, 0x44, 0xef, 0x8c, 0x66, 0xc5, - 0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac, - 0xf6 -}; -static const u8 enc_assoc043[] __initconst = { - 0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30 -}; -static const u8 enc_nonce043[] __initconst = { - 0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63, - 0xe5, 0x22, 0x94, 0x60 -}; -static const u8 enc_key043[] __initconst = { - 0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c, - 0x3e, 0x9b, 0xdc, 0x48, 0x0d, 0xea, 0x97, 0xb4, - 0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, 0xff, 0x6b, - 0x15, 0xbd, 0xd2, 0xa4, 0xa5, 0x68, 0x2c, 0x4e -}; - -/* wycheproof - misc */ -static const u8 enc_input044[] __initconst = { - 0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68, - 0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2 -}; -static const u8 enc_output044[] __initconst = { - 0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6, - 0x11, 0x92, 0x1b, 0x43, 0x8f, 0x74, 0x9f, 0x0b, - 0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, 0x34, 0x02, - 0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4 -}; -static const u8 enc_assoc044[] __initconst = { - 0x02 -}; -static const u8 enc_nonce044[] __initconst = { - 0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21, - 0x02, 0xd5, 0x06, 0x56 -}; -static const u8 enc_key044[] __initconst = { - 0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32, - 0x72, 0x05, 0x06, 0x07, 0xb2, 0x64, 0x18, 0x50, - 0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, 0x8b, 0xe0, - 0x22, 0x77, 0x0f, 0x2c, 0xe2, 0xf3, 0x14, 0x3c -}; - -/* wycheproof - misc */ -static const u8 enc_input045[] __initconst = { - 0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44, - 0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8 -}; -static const u8 enc_output045[] __initconst = { - 0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1, - 0x4d, 0x1f, 0x8f, 0xff, 0xab, 0x6d, 0x9e, 0x60, - 0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, 0x15, 0xa1, - 0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58 -}; -static const u8 enc_assoc045[] __initconst = { - 0xb6, 0x48 -}; -static const u8 enc_nonce045[] __initconst = { - 0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9, - 0x5b, 0x3a, 0xa7, 0x13 -}; -static const u8 enc_key045[] __initconst = { - 0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41, - 0xac, 0x78, 0x36, 0xbc, 0xe2, 0x51, 0x85, 0xa0, - 0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, 0x18, 0x44, - 0x4b, 0x6e, 0xc7, 0x2c, 0x3b, 0xd8, 0xe4, 0xdc -}; - -/* wycheproof - misc */ -static const u8 enc_input046[] __initconst = { - 0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23, - 0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47 -}; -static const u8 enc_output046[] __initconst = { - 0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda, - 0xc0, 0x3e, 0x5f, 0x74, 0x2b, 0xff, 0x73, 0xa1, - 0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, 0x38, 0x2b, - 0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8 -}; -static const u8 enc_assoc046[] __initconst = { - 0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd, - 0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0 -}; -static const u8 enc_nonce046[] __initconst = { - 0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b, - 0xa4, 0x65, 0x96, 0xdf -}; -static const u8 enc_key046[] __initconst = { - 0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08, - 0x2a, 0x92, 0x0b, 0x86, 0x36, 0x4e, 0xb8, 0x96, - 0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, 0xd5, 0x89, - 0x29, 0xfc, 0xb3, 0x66, 0x90, 0xe6, 0x39, 0x4f -}; - -/* wycheproof - misc */ -static const u8 enc_input047[] __initconst = { - 0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf, - 0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2 -}; -static const u8 enc_output047[] __initconst = { - 0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb, - 0x17, 0x77, 0x2a, 0x1c, 0x50, 0x0c, 0x50, 0x95, - 0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, 0xa0, 0x34, - 0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f -}; -static const u8 enc_assoc047[] __initconst = { - 0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07, - 0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b -}; -static const u8 enc_nonce047[] __initconst = { - 0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6, - 0x80, 0x92, 0x66, 0xd9 -}; -static const u8 enc_key047[] __initconst = { - 0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91, - 0xcf, 0x32, 0x0b, 0x13, 0x9f, 0xb5, 0x94, 0x23, - 0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, 0x65, 0xa6, - 0x7d, 0x4f, 0x63, 0x3f, 0x2c, 0x08, 0xf0, 0x16 -}; - -/* wycheproof - misc */ -static const u8 enc_input048[] __initconst = { - 0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32, - 0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3 -}; -static const u8 enc_output048[] __initconst = { - 0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b, - 0x1f, 0xd2, 0xb5, 0xc3, 0xa1, 0x5e, 0x35, 0x68, - 0x33, 0x35, 0x11, 0x86, 0x17, 0x96, 0x97, 0x84, - 0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3 -}; -static const u8 enc_assoc048[] __initconst = { - 0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab, - 0x03, 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c, - 0x0e -}; -static const u8 enc_nonce048[] __initconst = { - 0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40, - 0xfc, 0x10, 0x68, 0xc3 -}; -static const u8 enc_key048[] __initconst = { - 0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b, - 0xa6, 0xe0, 0xff, 0x15, 0xc4, 0x62, 0x78, 0x97, - 0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, 0xd0, 0x0b, - 0x2e, 0x93, 0xfd, 0xdc, 0x4b, 0xa7, 0x76, 0x57 -}; - -/* wycheproof - misc */ -static const u8 enc_input049[] __initconst = { - 0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2, - 0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6 -}; -static const u8 enc_output049[] __initconst = { - 0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87, - 0x20, 0x9a, 0x16, 0xe2, 0x85, 0x14, 0x31, 0x11, - 0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, 0x11, 0x7e, - 0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6 -}; -static const u8 enc_assoc049[] __initconst = { - 0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8, - 0x1e, 0x37, 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94, - 0x80, 0xa2, 0xa4, 0xd3, 0x35, 0x2a, 0x4f, 0xf5, - 0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda -}; -static const u8 enc_nonce049[] __initconst = { - 0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf, - 0xad, 0x14, 0xd5, 0x3e -}; -static const u8 enc_key049[] __initconst = { - 0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d, - 0x04, 0x63, 0x0f, 0x4a, 0x42, 0xf3, 0x5c, 0xfc, - 0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, 0x2b, 0x47, - 0x84, 0xf6, 0xf9, 0x29, 0x03, 0x30, 0xbe, 0xe0 -}; - -/* wycheproof - misc */ -static const u8 enc_input050[] __initconst = { - 0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18, - 0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c -}; -static const u8 enc_output050[] __initconst = { - 0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6, - 0x60, 0x2b, 0xc7, 0x78, 0x05, 0x6f, 0x0d, 0xca, - 0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, 0xb4, 0x6b, - 0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04 -}; -static const u8 enc_assoc050[] __initconst = { - 0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22, - 0x45, 0x0f, 0xeb, 0x10, 0x73, 0x7d, 0x90, 0x07, - 0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, 0x4d, 0x90, - 0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37 -}; -static const u8 enc_nonce050[] __initconst = { - 0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28, - 0x6a, 0x7b, 0x76, 0x51 -}; -static const u8 enc_key050[] __initconst = { - 0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1, - 0xba, 0x0c, 0x46, 0x72, 0xd8, 0xab, 0xc4, 0x6c, - 0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, 0x26, 0x4a, - 0xb0, 0x6d, 0x6c, 0x58, 0x07, 0x20, 0x51, 0x30 -}; - -/* wycheproof - misc */ -static const u8 enc_input051[] __initconst = { - 0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59, - 0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6 -}; -static const u8 enc_output051[] __initconst = { - 0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70, - 0xde, 0x32, 0x56, 0x2c, 0xcf, 0x2c, 0x2b, 0xbd, - 0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, 0x19, 0xb4, - 0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43 -}; -static const u8 enc_assoc051[] __initconst = { - 0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92, - 0x47, 0x02, 0x19, 0x05, 0x40, 0x2e, 0xa2, 0x57, - 0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, 0x56, 0x75, - 0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88, - 0x73 -}; -static const u8 enc_nonce051[] __initconst = { - 0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0, - 0xa8, 0xfa, 0x89, 0x49 -}; -static const u8 enc_key051[] __initconst = { - 0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27, - 0x96, 0x46, 0xac, 0x3d, 0x17, 0x6b, 0x96, 0x74, - 0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, 0x46, 0x9c, - 0x65, 0x1e, 0xc7, 0xe1, 0x6b, 0x4c, 0x11, 0x99 -}; - -/* wycheproof - misc */ -static const u8 enc_input052[] __initconst = { - 0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3, - 0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed -}; -static const u8 enc_output052[] __initconst = { - 0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc, - 0xc9, 0x76, 0x67, 0x2c, 0x44, 0x15, 0xe6, 0x9b, - 0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, 0x5d, 0xed, - 0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32 -}; -static const u8 enc_assoc052[] __initconst = { - 0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a, - 0x78, 0xa3, 0xd8, 0x0e, 0xa2, 0x56, 0x66, 0x14, - 0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae, - 0x84, 0x58, 0x3d, 0xa7, 0x3d, 0x7f, 0xf8, 0x5c, - 0x6f, 0x42, 0xca, 0x42, 0x05, 0x6a, 0x97, 0x92, - 0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61 -}; -static const u8 enc_nonce052[] __initconst = { - 0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73, - 0x0e, 0xc3, 0x5d, 0x12 -}; -static const u8 enc_key052[] __initconst = { - 0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5, - 0x29, 0xc1, 0x1f, 0x2f, 0x5a, 0xdd, 0x9a, 0xaf, - 0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, 0x7a, 0x2c, - 0xb7, 0xf7, 0x97, 0x49, 0xbd, 0x90, 0xaa, 0xf4 -}; - /* wycheproof - misc */ static const u8 enc_input053[] __initconst = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, @@ -3858,126 +2759,6 @@ static const u8 enc_key073[] __initconst = { 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; -/* wycheproof - checking for int overflows */ -static const u8 enc_input074[] __initconst = { - 0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51, - 0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69, - 0xdf, 0x7e, 0x8b, 0x76, 0x24, 0x63, 0x33, 0x0f, - 0xac, 0xc1, 0x6a, 0x57, 0x26, 0xbe, 0x71, 0x90, - 0xc6, 0x3c, 0x5a, 0x1c, 0x92, 0x65, 0x84, 0xa0, - 0x96, 0x75, 0x68, 0x28, 0xdc, 0xdc, 0x64, 0xac, - 0xdf, 0x96, 0x3d, 0x93, 0x1b, 0xf1, 0xda, 0xe2, - 0x38, 0xf3, 0xf1, 0x57, 0x22, 0x4a, 0xc4, 0xb5, - 0x42, 0xd7, 0x85, 0xb0, 0xdd, 0x84, 0xdb, 0x6b, - 0xe3, 0xbc, 0x5a, 0x36, 0x63, 0xe8, 0x41, 0x49, - 0xff, 0xbe, 0xd0, 0x9e, 0x54, 0xf7, 0x8f, 0x16, - 0xa8, 0x22, 0x3b, 0x24, 0xcb, 0x01, 0x9f, 0x58, - 0xb2, 0x1b, 0x0e, 0x55, 0x1e, 0x7a, 0xa0, 0x73, - 0x27, 0x62, 0x95, 0x51, 0x37, 0x6c, 0xcb, 0xc3, - 0x93, 0x76, 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c, - 0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a -}; -static const u8 enc_output074[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85, - 0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca -}; -static const u8 enc_assoc074[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff -}; -static const u8 enc_nonce074[] __initconst = { - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x00, 0x02, 0x50, 0x6e -}; -static const u8 enc_key074[] __initconst = { - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 -}; - -/* wycheproof - checking for int overflows */ -static const u8 enc_input075[] __initconst = { - 0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75, - 0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56, - 0x9e, 0x16, 0xf1, 0x11, 0xb5, 0xeb, 0x26, 0xc2, - 0x2d, 0x85, 0x9e, 0x3f, 0xf8, 0x22, 0xec, 0xed, - 0x3a, 0x6d, 0xd9, 0xa6, 0x0f, 0x22, 0x95, 0x7f, - 0x7b, 0x7c, 0x85, 0x7e, 0x88, 0x22, 0xeb, 0x9f, - 0xe0, 0xb8, 0xd7, 0x02, 0x21, 0x41, 0xf2, 0xd0, - 0xb4, 0x8f, 0x4b, 0x56, 0x12, 0xd3, 0x22, 0xa8, - 0x8d, 0xd0, 0xfe, 0x0b, 0x4d, 0x91, 0x79, 0x32, - 0x4f, 0x7c, 0x6c, 0x9e, 0x99, 0x0e, 0xfb, 0xd8, - 0x0e, 0x5e, 0xd6, 0x77, 0x58, 0x26, 0x49, 0x8b, - 0x1e, 0xfe, 0x0f, 0x71, 0xa0, 0xf3, 0xec, 0x5b, - 0x29, 0xcb, 0x28, 0xc2, 0x54, 0x0a, 0x7d, 0xcd, - 0x51, 0xb7, 0xda, 0xae, 0xe0, 0xff, 0x4a, 0x7f, - 0x3a, 0xc1, 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1, - 0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94 -}; -static const u8 enc_output075[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7, - 0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5 -}; -static const u8 enc_assoc075[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff -}; -static const u8 enc_nonce075[] __initconst = { - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x00, 0x03, 0x18, 0xa5 -}; -static const u8 enc_key075[] __initconst = { - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 -}; - /* wycheproof - checking for int overflows */ static const u8 enc_input076[] __initconst = { 0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85, @@ -4568,286 +3349,6 @@ static const u8 enc_key085[] __initconst = { 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; -/* wycheproof - special case tag */ -static const u8 enc_input086[] __initconst = { - 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, - 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, - 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, - 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, - 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, - 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, - 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, - 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d -}; -static const u8 enc_output086[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f -}; -static const u8 enc_assoc086[] __initconst = { - 0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, 0xbb, 0xc1, - 0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00 -}; -static const u8 enc_nonce086[] __initconst = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b -}; -static const u8 enc_key086[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - -/* wycheproof - special case tag */ -static const u8 enc_input087[] __initconst = { - 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, - 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, - 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, - 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, - 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, - 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, - 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, - 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d -}; -static const u8 enc_output087[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; -static const u8 enc_assoc087[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, 0x0a, 0x2f, - 0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58 -}; -static const u8 enc_nonce087[] __initconst = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b -}; -static const u8 enc_key087[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - -/* wycheproof - special case tag */ -static const u8 enc_input088[] __initconst = { - 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, - 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, - 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, - 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, - 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, - 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, - 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, - 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d -}; -static const u8 enc_output088[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff -}; -static const u8 enc_assoc088[] __initconst = { - 0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, 0x96, 0x1f, - 0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00 -}; -static const u8 enc_nonce088[] __initconst = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b -}; -static const u8 enc_key088[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - -/* wycheproof - special case tag */ -static const u8 enc_input089[] __initconst = { - 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, - 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, - 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, - 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, - 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, - 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, - 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, - 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d -}; -static const u8 enc_output089[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 -}; -static const u8 enc_assoc089[] __initconst = { - 0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, 0x6e, 0xae, - 0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02 -}; -static const u8 enc_nonce089[] __initconst = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b -}; -static const u8 enc_key089[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - -/* wycheproof - special case tag */ -static const u8 enc_input090[] __initconst = { - 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, - 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, - 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, - 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, - 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, - 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, - 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, - 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d -}; -static const u8 enc_output090[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, - 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f -}; -static const u8 enc_assoc090[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, 0x07, 0xbe, - 0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3 -}; -static const u8 enc_nonce090[] __initconst = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b -}; -static const u8 enc_key090[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - -/* wycheproof - special case tag */ -static const u8 enc_input091[] __initconst = { - 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, - 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, - 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, - 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, - 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, - 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, - 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, - 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d -}; -static const u8 enc_output091[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 -}; -static const u8 enc_assoc091[] __initconst = { - 0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, 0xd8, 0x30, - 0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01 -}; -static const u8 enc_nonce091[] __initconst = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b -}; -static const u8 enc_key091[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - -/* wycheproof - special case tag */ -static const u8 enc_input092[] __initconst = { - 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, - 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, - 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, - 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, - 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, - 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, - 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, - 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d -}; -static const u8 enc_output092[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; -static const u8 enc_assoc092[] __initconst = { - 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, 0x72, 0x11, - 0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01 -}; -static const u8 enc_nonce092[] __initconst = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b -}; -static const u8 enc_key092[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - /* wycheproof - edge case intermediate sums in poly1305 */ static const u8 enc_input093[] __initconst = { 0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d, @@ -5954,86 +4455,6 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input011), sizeof(enc_assoc011), sizeof(enc_nonce011) }, { enc_input012, enc_output012, enc_assoc012, enc_nonce012, enc_key012, sizeof(enc_input012), sizeof(enc_assoc012), sizeof(enc_nonce012) }, - { enc_input013, enc_output013, enc_assoc013, enc_nonce013, enc_key013, - sizeof(enc_input013), sizeof(enc_assoc013), sizeof(enc_nonce013) }, - { enc_input014, enc_output014, enc_assoc014, enc_nonce014, enc_key014, - sizeof(enc_input014), sizeof(enc_assoc014), sizeof(enc_nonce014) }, - { enc_input015, enc_output015, enc_assoc015, enc_nonce015, enc_key015, - sizeof(enc_input015), sizeof(enc_assoc015), sizeof(enc_nonce015) }, - { enc_input016, enc_output016, enc_assoc016, enc_nonce016, enc_key016, - sizeof(enc_input016), sizeof(enc_assoc016), sizeof(enc_nonce016) }, - { enc_input017, enc_output017, enc_assoc017, enc_nonce017, enc_key017, - sizeof(enc_input017), sizeof(enc_assoc017), sizeof(enc_nonce017) }, - { enc_input018, enc_output018, enc_assoc018, enc_nonce018, enc_key018, - sizeof(enc_input018), sizeof(enc_assoc018), sizeof(enc_nonce018) }, - { enc_input019, enc_output019, enc_assoc019, enc_nonce019, enc_key019, - sizeof(enc_input019), sizeof(enc_assoc019), sizeof(enc_nonce019) }, - { enc_input020, enc_output020, enc_assoc020, enc_nonce020, enc_key020, - sizeof(enc_input020), sizeof(enc_assoc020), sizeof(enc_nonce020) }, - { enc_input021, enc_output021, enc_assoc021, enc_nonce021, enc_key021, - sizeof(enc_input021), sizeof(enc_assoc021), sizeof(enc_nonce021) }, - { enc_input022, enc_output022, enc_assoc022, enc_nonce022, enc_key022, - sizeof(enc_input022), sizeof(enc_assoc022), sizeof(enc_nonce022) }, - { enc_input023, enc_output023, enc_assoc023, enc_nonce023, enc_key023, - sizeof(enc_input023), sizeof(enc_assoc023), sizeof(enc_nonce023) }, - { enc_input024, enc_output024, enc_assoc024, enc_nonce024, enc_key024, - sizeof(enc_input024), sizeof(enc_assoc024), sizeof(enc_nonce024) }, - { enc_input025, enc_output025, enc_assoc025, enc_nonce025, enc_key025, - sizeof(enc_input025), sizeof(enc_assoc025), sizeof(enc_nonce025) }, - { enc_input026, enc_output026, enc_assoc026, enc_nonce026, enc_key026, - sizeof(enc_input026), sizeof(enc_assoc026), sizeof(enc_nonce026) }, - { enc_input027, enc_output027, enc_assoc027, enc_nonce027, enc_key027, - sizeof(enc_input027), sizeof(enc_assoc027), sizeof(enc_nonce027) }, - { enc_input028, enc_output028, enc_assoc028, enc_nonce028, enc_key028, - sizeof(enc_input028), sizeof(enc_assoc028), sizeof(enc_nonce028) }, - { enc_input029, enc_output029, enc_assoc029, enc_nonce029, enc_key029, - sizeof(enc_input029), sizeof(enc_assoc029), sizeof(enc_nonce029) }, - { enc_input030, enc_output030, enc_assoc030, enc_nonce030, enc_key030, - sizeof(enc_input030), sizeof(enc_assoc030), sizeof(enc_nonce030) }, - { enc_input031, enc_output031, enc_assoc031, enc_nonce031, enc_key031, - sizeof(enc_input031), sizeof(enc_assoc031), sizeof(enc_nonce031) }, - { enc_input032, enc_output032, enc_assoc032, enc_nonce032, enc_key032, - sizeof(enc_input032), sizeof(enc_assoc032), sizeof(enc_nonce032) }, - { enc_input033, enc_output033, enc_assoc033, enc_nonce033, enc_key033, - sizeof(enc_input033), sizeof(enc_assoc033), sizeof(enc_nonce033) }, - { enc_input034, enc_output034, enc_assoc034, enc_nonce034, enc_key034, - sizeof(enc_input034), sizeof(enc_assoc034), sizeof(enc_nonce034) }, - { enc_input035, enc_output035, enc_assoc035, enc_nonce035, enc_key035, - sizeof(enc_input035), sizeof(enc_assoc035), sizeof(enc_nonce035) }, - { enc_input036, enc_output036, enc_assoc036, enc_nonce036, enc_key036, - sizeof(enc_input036), sizeof(enc_assoc036), sizeof(enc_nonce036) }, - { enc_input037, enc_output037, enc_assoc037, enc_nonce037, enc_key037, - sizeof(enc_input037), sizeof(enc_assoc037), sizeof(enc_nonce037) }, - { enc_input038, enc_output038, enc_assoc038, enc_nonce038, enc_key038, - sizeof(enc_input038), sizeof(enc_assoc038), sizeof(enc_nonce038) }, - { enc_input039, enc_output039, enc_assoc039, enc_nonce039, enc_key039, - sizeof(enc_input039), sizeof(enc_assoc039), sizeof(enc_nonce039) }, - { enc_input040, enc_output040, enc_assoc040, enc_nonce040, enc_key040, - sizeof(enc_input040), sizeof(enc_assoc040), sizeof(enc_nonce040) }, - { enc_input041, enc_output041, enc_assoc041, enc_nonce041, enc_key041, - sizeof(enc_input041), sizeof(enc_assoc041), sizeof(enc_nonce041) }, - { enc_input042, enc_output042, enc_assoc042, enc_nonce042, enc_key042, - sizeof(enc_input042), sizeof(enc_assoc042), sizeof(enc_nonce042) }, - { enc_input043, enc_output043, enc_assoc043, enc_nonce043, enc_key043, - sizeof(enc_input043), sizeof(enc_assoc043), sizeof(enc_nonce043) }, - { enc_input044, enc_output044, enc_assoc044, enc_nonce044, enc_key044, - sizeof(enc_input044), sizeof(enc_assoc044), sizeof(enc_nonce044) }, - { enc_input045, enc_output045, enc_assoc045, enc_nonce045, enc_key045, - sizeof(enc_input045), sizeof(enc_assoc045), sizeof(enc_nonce045) }, - { enc_input046, enc_output046, enc_assoc046, enc_nonce046, enc_key046, - sizeof(enc_input046), sizeof(enc_assoc046), sizeof(enc_nonce046) }, - { enc_input047, enc_output047, enc_assoc047, enc_nonce047, enc_key047, - sizeof(enc_input047), sizeof(enc_assoc047), sizeof(enc_nonce047) }, - { enc_input048, enc_output048, enc_assoc048, enc_nonce048, enc_key048, - sizeof(enc_input048), sizeof(enc_assoc048), sizeof(enc_nonce048) }, - { enc_input049, enc_output049, enc_assoc049, enc_nonce049, enc_key049, - sizeof(enc_input049), sizeof(enc_assoc049), sizeof(enc_nonce049) }, - { enc_input050, enc_output050, enc_assoc050, enc_nonce050, enc_key050, - sizeof(enc_input050), sizeof(enc_assoc050), sizeof(enc_nonce050) }, - { enc_input051, enc_output051, enc_assoc051, enc_nonce051, enc_key051, - sizeof(enc_input051), sizeof(enc_assoc051), sizeof(enc_nonce051) }, - { enc_input052, enc_output052, enc_assoc052, enc_nonce052, enc_key052, - sizeof(enc_input052), sizeof(enc_assoc052), sizeof(enc_nonce052) }, { enc_input053, enc_output053, enc_assoc053, enc_nonce053, enc_key053, sizeof(enc_input053), sizeof(enc_assoc053), sizeof(enc_nonce053) }, { enc_input054, enc_output054, enc_assoc054, enc_nonce054, enc_key054, @@ -6076,10 +4497,6 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input072), sizeof(enc_assoc072), sizeof(enc_nonce072) }, { enc_input073, enc_output073, enc_assoc073, enc_nonce073, enc_key073, sizeof(enc_input073), sizeof(enc_assoc073), sizeof(enc_nonce073) }, - { enc_input074, enc_output074, enc_assoc074, enc_nonce074, enc_key074, - sizeof(enc_input074), sizeof(enc_assoc074), sizeof(enc_nonce074) }, - { enc_input075, enc_output075, enc_assoc075, enc_nonce075, enc_key075, - sizeof(enc_input075), sizeof(enc_assoc075), sizeof(enc_nonce075) }, { enc_input076, enc_output076, enc_assoc076, enc_nonce076, enc_key076, sizeof(enc_input076), sizeof(enc_assoc076), sizeof(enc_nonce076) }, { enc_input077, enc_output077, enc_assoc077, enc_nonce077, enc_key077, @@ -6100,20 +4517,6 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input084), sizeof(enc_assoc084), sizeof(enc_nonce084) }, { enc_input085, enc_output085, enc_assoc085, enc_nonce085, enc_key085, sizeof(enc_input085), sizeof(enc_assoc085), sizeof(enc_nonce085) }, - { enc_input086, enc_output086, enc_assoc086, enc_nonce086, enc_key086, - sizeof(enc_input086), sizeof(enc_assoc086), sizeof(enc_nonce086) }, - { enc_input087, enc_output087, enc_assoc087, enc_nonce087, enc_key087, - sizeof(enc_input087), sizeof(enc_assoc087), sizeof(enc_nonce087) }, - { enc_input088, enc_output088, enc_assoc088, enc_nonce088, enc_key088, - sizeof(enc_input088), sizeof(enc_assoc088), sizeof(enc_nonce088) }, - { enc_input089, enc_output089, enc_assoc089, enc_nonce089, enc_key089, - sizeof(enc_input089), sizeof(enc_assoc089), sizeof(enc_nonce089) }, - { enc_input090, enc_output090, enc_assoc090, enc_nonce090, enc_key090, - sizeof(enc_input090), sizeof(enc_assoc090), sizeof(enc_nonce090) }, - { enc_input091, enc_output091, enc_assoc091, enc_nonce091, enc_key091, - sizeof(enc_input091), sizeof(enc_assoc091), sizeof(enc_nonce091) }, - { enc_input092, enc_output092, enc_assoc092, enc_nonce092, enc_key092, - sizeof(enc_input092), sizeof(enc_assoc092), sizeof(enc_nonce092) }, { enc_input093, enc_output093, enc_assoc093, enc_nonce093, enc_key093, sizeof(enc_input093), sizeof(enc_assoc093), sizeof(enc_nonce093) }, { enc_input094, enc_output094, enc_assoc094, enc_nonce094, enc_key094, @@ -8821,43 +7224,6 @@ xchacha20poly1305_dec_vectors[] __initconst = { sizeof(xdec_input001), sizeof(xdec_assoc001), sizeof(xdec_nonce001) } }; -/* This is for the selftests-only, since it is only useful for the purpose of - * testing the underlying primitives and interactions. - */ -static void __init -chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, - const u8 *ad, const size_t ad_len, - const u8 nonce[12], - const u8 key[CHACHA20POLY1305_KEY_SIZE]) -{ - const u8 *pad0 = page_address(ZERO_PAGE(0)); - struct poly1305_desc_ctx poly1305_state; - u32 chacha20_state[CHACHA_STATE_WORDS]; - union { - u8 block0[POLY1305_KEY_SIZE]; - __le64 lens[2]; - } b = {{ 0 }}; - u8 bottom_row[16] = { 0 }; - u32 le_key[8]; - int i; - - memcpy(&bottom_row[4], nonce, 12); - for (i = 0; i < 8; ++i) - le_key[i] = get_unaligned_le32(key + sizeof(le_key[i]) * i); - chacha_init(chacha20_state, le_key, bottom_row); - chacha20_crypt(chacha20_state, b.block0, b.block0, sizeof(b.block0)); - poly1305_init(&poly1305_state, b.block0); - poly1305_update(&poly1305_state, ad, ad_len); - poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf); - chacha20_crypt(chacha20_state, dst, src, src_len); - poly1305_update(&poly1305_state, dst, src_len); - poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf); - b.lens[0] = cpu_to_le64(ad_len); - b.lens[1] = cpu_to_le64(src_len); - poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); - poly1305_final(&poly1305_state, dst + src_len); -} - static void __init chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, @@ -8867,9 +7233,6 @@ chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, if (nonce_len == 8) chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, get_unaligned_le64(nonce), key); - else if (nonce_len == 12) - chacha20poly1305_encrypt_bignonce(dst, src, src_len, ad, - ad_len, nonce, key); else BUG(); } @@ -8885,14 +7248,14 @@ decryption_success(bool func_ret, bool expect_failure, int memcmp_result) bool __init chacha20poly1305_selftest(void) { enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 }; - size_t i, j, k, total_len; - u8 *computed_output = NULL, *input = NULL; + size_t i; + u8 *computed_output = NULL, *heap_src = NULL; + struct scatterlist sg_src; bool success = true, ret; - struct scatterlist sg_src[3]; + heap_src = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - input = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - if (!computed_output || !input) { + if (!heap_src || !computed_output) { pr_err("chacha20poly1305 self-test malloc: FAIL\n"); success = false; goto out; @@ -8921,17 +7284,17 @@ bool __init chacha20poly1305_selftest(void) for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { if (chacha20poly1305_enc_vectors[i].nlen != 8) continue; - memcpy(computed_output, chacha20poly1305_enc_vectors[i].input, + memcpy(heap_src, chacha20poly1305_enc_vectors[i].input, chacha20poly1305_enc_vectors[i].ilen); - sg_init_one(sg_src, computed_output, + sg_init_one(&sg_src, heap_src, chacha20poly1305_enc_vectors[i].ilen + POLY1305_DIGEST_SIZE); - ret = chacha20poly1305_encrypt_sg_inplace(sg_src, + chacha20poly1305_encrypt_sg_inplace(&sg_src, chacha20poly1305_enc_vectors[i].ilen, chacha20poly1305_enc_vectors[i].assoc, chacha20poly1305_enc_vectors[i].alen, get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce), chacha20poly1305_enc_vectors[i].key); - if (!ret || memcmp(computed_output, + if (memcmp(heap_src, chacha20poly1305_enc_vectors[i].output, chacha20poly1305_enc_vectors[i].ilen + POLY1305_DIGEST_SIZE)) { @@ -8963,11 +7326,11 @@ bool __init chacha20poly1305_selftest(void) } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { - memcpy(computed_output, chacha20poly1305_dec_vectors[i].input, + memcpy(heap_src, chacha20poly1305_dec_vectors[i].input, chacha20poly1305_dec_vectors[i].ilen); - sg_init_one(sg_src, computed_output, + sg_init_one(&sg_src, heap_src, chacha20poly1305_dec_vectors[i].ilen); - ret = chacha20poly1305_decrypt_sg_inplace(sg_src, + ret = chacha20poly1305_decrypt_sg_inplace(&sg_src, chacha20poly1305_dec_vectors[i].ilen, chacha20poly1305_dec_vectors[i].assoc, chacha20poly1305_dec_vectors[i].alen, @@ -8975,7 +7338,7 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_dec_vectors[i].key); if (!decryption_success(ret, chacha20poly1305_dec_vectors[i].failure, - memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, + memcmp(heap_src, chacha20poly1305_dec_vectors[i].output, chacha20poly1305_dec_vectors[i].ilen - POLY1305_DIGEST_SIZE))) { pr_err("chacha20poly1305 sg decryption self-test %zu: FAIL\n", @@ -9002,7 +7365,6 @@ bool __init chacha20poly1305_selftest(void) success = false; } } - for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_dec_vectors); ++i) { memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); ret = xchacha20poly1305_decrypt(computed_output, @@ -9024,59 +7386,8 @@ bool __init chacha20poly1305_selftest(void) } } - for (total_len = POLY1305_DIGEST_SIZE; IS_ENABLED(DEBUG_CHACHA20POLY1305_SLOW_CHUNK_TEST) - && total_len <= 1 << 10; ++total_len) { - for (i = 0; i <= total_len; ++i) { - for (j = i; j <= total_len; ++j) { - k = 0; - sg_init_table(sg_src, 3); - if (i) - sg_set_buf(&sg_src[k++], input, i); - if (j - i) - sg_set_buf(&sg_src[k++], input + i, j - i); - if (total_len - j) - sg_set_buf(&sg_src[k++], input + j, total_len - j); - sg_init_marker(sg_src, k); - memset(computed_output, 0, total_len); - memset(input, 0, total_len); - - if (!chacha20poly1305_encrypt_sg_inplace(sg_src, - total_len - POLY1305_DIGEST_SIZE, NULL, 0, - 0, enc_key001)) - goto chunkfail; - chacha20poly1305_encrypt(computed_output, - computed_output, - total_len - POLY1305_DIGEST_SIZE, NULL, 0, 0, - enc_key001); - if (memcmp(computed_output, input, total_len)) - goto chunkfail; - if (!chacha20poly1305_decrypt(computed_output, - input, total_len, NULL, 0, 0, enc_key001)) - goto chunkfail; - for (k = 0; k < total_len - POLY1305_DIGEST_SIZE; ++k) { - if (computed_output[k]) - goto chunkfail; - } - if (!chacha20poly1305_decrypt_sg_inplace(sg_src, - total_len, NULL, 0, 0, enc_key001)) - goto chunkfail; - for (k = 0; k < total_len - POLY1305_DIGEST_SIZE; ++k) { - if (input[k]) - goto chunkfail; - } - continue; - - chunkfail: - pr_err("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL\n", - total_len, i, j); - success = false; - } - - } - } - out: + kfree(heap_src); kfree(computed_output); - kfree(input); return success; } diff --git a/lib/crypto/poly1305-donna32.c b/lib/crypto/poly1305-donna32.c deleted file mode 100644 index 3cc77d94390b2..0000000000000 --- a/lib/crypto/poly1305-donna32.c +++ /dev/null @@ -1,204 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - * - * This is based in part on Andrew Moon's poly1305-donna, which is in the - * public domain. - */ - -#include -#include -#include - -void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[16]) -{ - /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ - key->key.r[0] = (get_unaligned_le32(&raw_key[0])) & 0x3ffffff; - key->key.r[1] = (get_unaligned_le32(&raw_key[3]) >> 2) & 0x3ffff03; - key->key.r[2] = (get_unaligned_le32(&raw_key[6]) >> 4) & 0x3ffc0ff; - key->key.r[3] = (get_unaligned_le32(&raw_key[9]) >> 6) & 0x3f03fff; - key->key.r[4] = (get_unaligned_le32(&raw_key[12]) >> 8) & 0x00fffff; - - /* s = 5*r */ - key->precomputed_s.r[0] = key->key.r[1] * 5; - key->precomputed_s.r[1] = key->key.r[2] * 5; - key->precomputed_s.r[2] = key->key.r[3] * 5; - key->precomputed_s.r[3] = key->key.r[4] * 5; -} -EXPORT_SYMBOL(poly1305_core_setkey); - -void poly1305_core_blocks(struct poly1305_state *state, - const struct poly1305_core_key *key, const void *src, - unsigned int nblocks, u32 hibit) -{ - const u8 *input = src; - u32 r0, r1, r2, r3, r4; - u32 s1, s2, s3, s4; - u32 h0, h1, h2, h3, h4; - u64 d0, d1, d2, d3, d4; - u32 c; - - if (!nblocks) - return; - - hibit <<= 24; - - r0 = key->key.r[0]; - r1 = key->key.r[1]; - r2 = key->key.r[2]; - r3 = key->key.r[3]; - r4 = key->key.r[4]; - - s1 = key->precomputed_s.r[0]; - s2 = key->precomputed_s.r[1]; - s3 = key->precomputed_s.r[2]; - s4 = key->precomputed_s.r[3]; - - h0 = state->h[0]; - h1 = state->h[1]; - h2 = state->h[2]; - h3 = state->h[3]; - h4 = state->h[4]; - - do { - /* h += m[i] */ - h0 += (get_unaligned_le32(&input[0])) & 0x3ffffff; - h1 += (get_unaligned_le32(&input[3]) >> 2) & 0x3ffffff; - h2 += (get_unaligned_le32(&input[6]) >> 4) & 0x3ffffff; - h3 += (get_unaligned_le32(&input[9]) >> 6) & 0x3ffffff; - h4 += (get_unaligned_le32(&input[12]) >> 8) | hibit; - - /* h *= r */ - d0 = ((u64)h0 * r0) + ((u64)h1 * s4) + - ((u64)h2 * s3) + ((u64)h3 * s2) + - ((u64)h4 * s1); - d1 = ((u64)h0 * r1) + ((u64)h1 * r0) + - ((u64)h2 * s4) + ((u64)h3 * s3) + - ((u64)h4 * s2); - d2 = ((u64)h0 * r2) + ((u64)h1 * r1) + - ((u64)h2 * r0) + ((u64)h3 * s4) + - ((u64)h4 * s3); - d3 = ((u64)h0 * r3) + ((u64)h1 * r2) + - ((u64)h2 * r1) + ((u64)h3 * r0) + - ((u64)h4 * s4); - d4 = ((u64)h0 * r4) + ((u64)h1 * r3) + - ((u64)h2 * r2) + ((u64)h3 * r1) + - ((u64)h4 * r0); - - /* (partial) h %= p */ - c = (u32)(d0 >> 26); - h0 = (u32)d0 & 0x3ffffff; - d1 += c; - c = (u32)(d1 >> 26); - h1 = (u32)d1 & 0x3ffffff; - d2 += c; - c = (u32)(d2 >> 26); - h2 = (u32)d2 & 0x3ffffff; - d3 += c; - c = (u32)(d3 >> 26); - h3 = (u32)d3 & 0x3ffffff; - d4 += c; - c = (u32)(d4 >> 26); - h4 = (u32)d4 & 0x3ffffff; - h0 += c * 5; - c = (h0 >> 26); - h0 = h0 & 0x3ffffff; - h1 += c; - - input += POLY1305_BLOCK_SIZE; - } while (--nblocks); - - state->h[0] = h0; - state->h[1] = h1; - state->h[2] = h2; - state->h[3] = h3; - state->h[4] = h4; -} -EXPORT_SYMBOL(poly1305_core_blocks); - -void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], - void *dst) -{ - u8 *mac = dst; - u32 h0, h1, h2, h3, h4, c; - u32 g0, g1, g2, g3, g4; - u64 f; - u32 mask; - - /* fully carry h */ - h0 = state->h[0]; - h1 = state->h[1]; - h2 = state->h[2]; - h3 = state->h[3]; - h4 = state->h[4]; - - c = h1 >> 26; - h1 = h1 & 0x3ffffff; - h2 += c; - c = h2 >> 26; - h2 = h2 & 0x3ffffff; - h3 += c; - c = h3 >> 26; - h3 = h3 & 0x3ffffff; - h4 += c; - c = h4 >> 26; - h4 = h4 & 0x3ffffff; - h0 += c * 5; - c = h0 >> 26; - h0 = h0 & 0x3ffffff; - h1 += c; - - /* compute h + -p */ - g0 = h0 + 5; - c = g0 >> 26; - g0 &= 0x3ffffff; - g1 = h1 + c; - c = g1 >> 26; - g1 &= 0x3ffffff; - g2 = h2 + c; - c = g2 >> 26; - g2 &= 0x3ffffff; - g3 = h3 + c; - c = g3 >> 26; - g3 &= 0x3ffffff; - g4 = h4 + c - (1UL << 26); - - /* select h if h < p, or h + -p if h >= p */ - mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1; - g0 &= mask; - g1 &= mask; - g2 &= mask; - g3 &= mask; - g4 &= mask; - mask = ~mask; - - h0 = (h0 & mask) | g0; - h1 = (h1 & mask) | g1; - h2 = (h2 & mask) | g2; - h3 = (h3 & mask) | g3; - h4 = (h4 & mask) | g4; - - /* h = h % (2^128) */ - h0 = ((h0) | (h1 << 26)) & 0xffffffff; - h1 = ((h1 >> 6) | (h2 << 20)) & 0xffffffff; - h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff; - h3 = ((h3 >> 18) | (h4 << 8)) & 0xffffffff; - - if (likely(nonce)) { - /* mac = (h + nonce) % (2^128) */ - f = (u64)h0 + nonce[0]; - h0 = (u32)f; - f = (u64)h1 + nonce[1] + (f >> 32); - h1 = (u32)f; - f = (u64)h2 + nonce[2] + (f >> 32); - h2 = (u32)f; - f = (u64)h3 + nonce[3] + (f >> 32); - h3 = (u32)f; - } - - put_unaligned_le32(h0, &mac[0]); - put_unaligned_le32(h1, &mac[4]); - put_unaligned_le32(h2, &mac[8]); - put_unaligned_le32(h3, &mac[12]); -} -EXPORT_SYMBOL(poly1305_core_emit); diff --git a/lib/crypto/poly1305-donna64.c b/lib/crypto/poly1305-donna64.c deleted file mode 100644 index 6ae181bb43450..0000000000000 --- a/lib/crypto/poly1305-donna64.c +++ /dev/null @@ -1,185 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - * - * This is based in part on Andrew Moon's poly1305-donna, which is in the - * public domain. - */ - -#include -#include -#include - -typedef __uint128_t u128; - -void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[16]) -{ - u64 t0, t1; - - /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ - t0 = get_unaligned_le64(&raw_key[0]); - t1 = get_unaligned_le64(&raw_key[8]); - - key->key.r64[0] = t0 & 0xffc0fffffffULL; - key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL; - key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL; - - /* s = 20*r */ - key->precomputed_s.r64[0] = key->key.r64[1] * 20; - key->precomputed_s.r64[1] = key->key.r64[2] * 20; -} -EXPORT_SYMBOL(poly1305_core_setkey); - -void poly1305_core_blocks(struct poly1305_state *state, - const struct poly1305_core_key *key, const void *src, - unsigned int nblocks, u32 hibit) -{ - const u8 *input = src; - u64 hibit64; - u64 r0, r1, r2; - u64 s1, s2; - u64 h0, h1, h2; - u64 c; - u128 d0, d1, d2, d; - - if (!nblocks) - return; - - hibit64 = ((u64)hibit) << 40; - - r0 = key->key.r64[0]; - r1 = key->key.r64[1]; - r2 = key->key.r64[2]; - - h0 = state->h64[0]; - h1 = state->h64[1]; - h2 = state->h64[2]; - - s1 = key->precomputed_s.r64[0]; - s2 = key->precomputed_s.r64[1]; - - do { - u64 t0, t1; - - /* h += m[i] */ - t0 = get_unaligned_le64(&input[0]); - t1 = get_unaligned_le64(&input[8]); - - h0 += t0 & 0xfffffffffffULL; - h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL; - h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64; - - /* h *= r */ - d0 = (u128)h0 * r0; - d = (u128)h1 * s2; - d0 += d; - d = (u128)h2 * s1; - d0 += d; - d1 = (u128)h0 * r1; - d = (u128)h1 * r0; - d1 += d; - d = (u128)h2 * s2; - d1 += d; - d2 = (u128)h0 * r2; - d = (u128)h1 * r1; - d2 += d; - d = (u128)h2 * r0; - d2 += d; - - /* (partial) h %= p */ - c = (u64)(d0 >> 44); - h0 = (u64)d0 & 0xfffffffffffULL; - d1 += c; - c = (u64)(d1 >> 44); - h1 = (u64)d1 & 0xfffffffffffULL; - d2 += c; - c = (u64)(d2 >> 42); - h2 = (u64)d2 & 0x3ffffffffffULL; - h0 += c * 5; - c = h0 >> 44; - h0 = h0 & 0xfffffffffffULL; - h1 += c; - - input += POLY1305_BLOCK_SIZE; - } while (--nblocks); - - state->h64[0] = h0; - state->h64[1] = h1; - state->h64[2] = h2; -} -EXPORT_SYMBOL(poly1305_core_blocks); - -void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], - void *dst) -{ - u8 *mac = dst; - u64 h0, h1, h2, c; - u64 g0, g1, g2; - u64 t0, t1; - - /* fully carry h */ - h0 = state->h64[0]; - h1 = state->h64[1]; - h2 = state->h64[2]; - - c = h1 >> 44; - h1 &= 0xfffffffffffULL; - h2 += c; - c = h2 >> 42; - h2 &= 0x3ffffffffffULL; - h0 += c * 5; - c = h0 >> 44; - h0 &= 0xfffffffffffULL; - h1 += c; - c = h1 >> 44; - h1 &= 0xfffffffffffULL; - h2 += c; - c = h2 >> 42; - h2 &= 0x3ffffffffffULL; - h0 += c * 5; - c = h0 >> 44; - h0 &= 0xfffffffffffULL; - h1 += c; - - /* compute h + -p */ - g0 = h0 + 5; - c = g0 >> 44; - g0 &= 0xfffffffffffULL; - g1 = h1 + c; - c = g1 >> 44; - g1 &= 0xfffffffffffULL; - g2 = h2 + c - (1ULL << 42); - - /* select h if h < p, or h + -p if h >= p */ - c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1; - g0 &= c; - g1 &= c; - g2 &= c; - c = ~c; - h0 = (h0 & c) | g0; - h1 = (h1 & c) | g1; - h2 = (h2 & c) | g2; - - if (likely(nonce)) { - /* h = (h + nonce) */ - t0 = ((u64)nonce[1] << 32) | nonce[0]; - t1 = ((u64)nonce[3] << 32) | nonce[2]; - - h0 += t0 & 0xfffffffffffULL; - c = h0 >> 44; - h0 &= 0xfffffffffffULL; - h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c; - c = h1 >> 44; - h1 &= 0xfffffffffffULL; - h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c; - h2 &= 0x3ffffffffffULL; - } - - /* mac = h % (2^128) */ - h0 = h0 | (h1 << 44); - h1 = (h1 >> 20) | (h2 << 24); - - put_unaligned_le64(h0, &mac[0]); - put_unaligned_le64(h1, &mac[8]); -} -EXPORT_SYMBOL(poly1305_core_emit); diff --git a/lib/crypto/poly1305.c b/lib/crypto/poly1305.c index 9d2d14df0fee5..32ec293c65ae7 100644 --- a/lib/crypto/poly1305.c +++ b/lib/crypto/poly1305.c @@ -12,9 +12,151 @@ #include #include +static inline u64 mlt(u64 a, u64 b) +{ + return a * b; +} + +static inline u32 sr(u64 v, u_char n) +{ + return v >> n; +} + +static inline u32 and(u32 v, u32 mask) +{ + return v & mask; +} + +void poly1305_core_setkey(struct poly1305_key *key, const u8 *raw_key) +{ + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + key->r[0] = (get_unaligned_le32(raw_key + 0) >> 0) & 0x3ffffff; + key->r[1] = (get_unaligned_le32(raw_key + 3) >> 2) & 0x3ffff03; + key->r[2] = (get_unaligned_le32(raw_key + 6) >> 4) & 0x3ffc0ff; + key->r[3] = (get_unaligned_le32(raw_key + 9) >> 6) & 0x3f03fff; + key->r[4] = (get_unaligned_le32(raw_key + 12) >> 8) & 0x00fffff; +} +EXPORT_SYMBOL_GPL(poly1305_core_setkey); + +void poly1305_core_blocks(struct poly1305_state *state, + const struct poly1305_key *key, const void *src, + unsigned int nblocks, u32 hibit) +{ + u32 r0, r1, r2, r3, r4; + u32 s1, s2, s3, s4; + u32 h0, h1, h2, h3, h4; + u64 d0, d1, d2, d3, d4; + + if (!nblocks) + return; + + r0 = key->r[0]; + r1 = key->r[1]; + r2 = key->r[2]; + r3 = key->r[3]; + r4 = key->r[4]; + + s1 = r1 * 5; + s2 = r2 * 5; + s3 = r3 * 5; + s4 = r4 * 5; + + h0 = state->h[0]; + h1 = state->h[1]; + h2 = state->h[2]; + h3 = state->h[3]; + h4 = state->h[4]; + + do { + /* h += m[i] */ + h0 += (get_unaligned_le32(src + 0) >> 0) & 0x3ffffff; + h1 += (get_unaligned_le32(src + 3) >> 2) & 0x3ffffff; + h2 += (get_unaligned_le32(src + 6) >> 4) & 0x3ffffff; + h3 += (get_unaligned_le32(src + 9) >> 6) & 0x3ffffff; + h4 += (get_unaligned_le32(src + 12) >> 8) | (hibit << 24); + + /* h *= r */ + d0 = mlt(h0, r0) + mlt(h1, s4) + mlt(h2, s3) + + mlt(h3, s2) + mlt(h4, s1); + d1 = mlt(h0, r1) + mlt(h1, r0) + mlt(h2, s4) + + mlt(h3, s3) + mlt(h4, s2); + d2 = mlt(h0, r2) + mlt(h1, r1) + mlt(h2, r0) + + mlt(h3, s4) + mlt(h4, s3); + d3 = mlt(h0, r3) + mlt(h1, r2) + mlt(h2, r1) + + mlt(h3, r0) + mlt(h4, s4); + d4 = mlt(h0, r4) + mlt(h1, r3) + mlt(h2, r2) + + mlt(h3, r1) + mlt(h4, r0); + + /* (partial) h %= p */ + d1 += sr(d0, 26); h0 = and(d0, 0x3ffffff); + d2 += sr(d1, 26); h1 = and(d1, 0x3ffffff); + d3 += sr(d2, 26); h2 = and(d2, 0x3ffffff); + d4 += sr(d3, 26); h3 = and(d3, 0x3ffffff); + h0 += sr(d4, 26) * 5; h4 = and(d4, 0x3ffffff); + h1 += h0 >> 26; h0 = h0 & 0x3ffffff; + + src += POLY1305_BLOCK_SIZE; + } while (--nblocks); + + state->h[0] = h0; + state->h[1] = h1; + state->h[2] = h2; + state->h[3] = h3; + state->h[4] = h4; +} +EXPORT_SYMBOL_GPL(poly1305_core_blocks); + +void poly1305_core_emit(const struct poly1305_state *state, void *dst) +{ + u32 h0, h1, h2, h3, h4; + u32 g0, g1, g2, g3, g4; + u32 mask; + + /* fully carry h */ + h0 = state->h[0]; + h1 = state->h[1]; + h2 = state->h[2]; + h3 = state->h[3]; + h4 = state->h[4]; + + h2 += (h1 >> 26); h1 = h1 & 0x3ffffff; + h3 += (h2 >> 26); h2 = h2 & 0x3ffffff; + h4 += (h3 >> 26); h3 = h3 & 0x3ffffff; + h0 += (h4 >> 26) * 5; h4 = h4 & 0x3ffffff; + h1 += (h0 >> 26); h0 = h0 & 0x3ffffff; + + /* compute h + -p */ + g0 = h0 + 5; + g1 = h1 + (g0 >> 26); g0 &= 0x3ffffff; + g2 = h2 + (g1 >> 26); g1 &= 0x3ffffff; + g3 = h3 + (g2 >> 26); g2 &= 0x3ffffff; + g4 = h4 + (g3 >> 26) - (1 << 26); g3 &= 0x3ffffff; + + /* select h if h < p, or h + -p if h >= p */ + mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1; + g0 &= mask; + g1 &= mask; + g2 &= mask; + g3 &= mask; + g4 &= mask; + mask = ~mask; + h0 = (h0 & mask) | g0; + h1 = (h1 & mask) | g1; + h2 = (h2 & mask) | g2; + h3 = (h3 & mask) | g3; + h4 = (h4 & mask) | g4; + + /* h = h % (2^128) */ + put_unaligned_le32((h0 >> 0) | (h1 << 26), dst + 0); + put_unaligned_le32((h1 >> 6) | (h2 << 20), dst + 4); + put_unaligned_le32((h2 >> 12) | (h3 << 14), dst + 8); + put_unaligned_le32((h3 >> 18) | (h4 << 8), dst + 12); +} +EXPORT_SYMBOL_GPL(poly1305_core_emit); + void poly1305_init_generic(struct poly1305_desc_ctx *desc, const u8 *key) { - poly1305_core_setkey(&desc->core_r, key); + poly1305_core_setkey(desc->r, key); desc->s[0] = get_unaligned_le32(key + 16); desc->s[1] = get_unaligned_le32(key + 20); desc->s[2] = get_unaligned_le32(key + 24); @@ -22,7 +164,7 @@ void poly1305_init_generic(struct poly1305_desc_ctx *desc, const u8 *key) poly1305_core_init(&desc->h); desc->buflen = 0; desc->sset = true; - desc->rset = 2; + desc->rset = 1; } EXPORT_SYMBOL_GPL(poly1305_init_generic); @@ -39,14 +181,13 @@ void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src, desc->buflen += bytes; if (desc->buflen == POLY1305_BLOCK_SIZE) { - poly1305_core_blocks(&desc->h, &desc->core_r, desc->buf, - 1, 1); + poly1305_core_blocks(&desc->h, desc->r, desc->buf, 1, 1); desc->buflen = 0; } } if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { - poly1305_core_blocks(&desc->h, &desc->core_r, src, + poly1305_core_blocks(&desc->h, desc->r, src, nbytes / POLY1305_BLOCK_SIZE, 1); src += nbytes - (nbytes % POLY1305_BLOCK_SIZE); nbytes %= POLY1305_BLOCK_SIZE; @@ -61,14 +202,28 @@ EXPORT_SYMBOL_GPL(poly1305_update_generic); void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *dst) { + __le32 digest[4]; + u64 f = 0; + if (unlikely(desc->buflen)) { desc->buf[desc->buflen++] = 1; memset(desc->buf + desc->buflen, 0, POLY1305_BLOCK_SIZE - desc->buflen); - poly1305_core_blocks(&desc->h, &desc->core_r, desc->buf, 1, 0); + poly1305_core_blocks(&desc->h, desc->r, desc->buf, 1, 0); } - poly1305_core_emit(&desc->h, desc->s, dst); + poly1305_core_emit(&desc->h, digest); + + /* mac = (h + s) % (2^128) */ + f = (f >> 32) + le32_to_cpu(digest[0]) + desc->s[0]; + put_unaligned_le32(f, dst + 0); + f = (f >> 32) + le32_to_cpu(digest[1]) + desc->s[1]; + put_unaligned_le32(f, dst + 4); + f = (f >> 32) + le32_to_cpu(digest[2]) + desc->s[2]; + put_unaligned_le32(f, dst + 8); + f = (f >> 32) + le32_to_cpu(digest[3]) + desc->s[3]; + put_unaligned_le32(f, dst + 12); + *desc = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL_GPL(poly1305_final_generic); diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index f369e7ce685b8..18068ed42f258 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -748,39 +748,6 @@ out:; } EXPORT_SYMBOL(__icmp_send); -#if IS_ENABLED(CONFIG_NF_NAT) -#include -void icmp_ndo_send(struct sk_buff *skb_in, int type, int code, __be32 info) -{ - struct sk_buff *cloned_skb = NULL; - enum ip_conntrack_info ctinfo; - struct nf_conn *ct; - __be32 orig_ip; - - ct = nf_ct_get(skb_in, &ctinfo); - if (!ct || !(ct->status & IPS_SRC_NAT)) { - icmp_send(skb_in, type, code, info); - return; - } - - if (skb_shared(skb_in)) - skb_in = cloned_skb = skb_clone(skb_in, GFP_ATOMIC); - - if (unlikely(!skb_in || skb_network_header(skb_in) < skb_in->head || - (skb_network_header(skb_in) + sizeof(struct iphdr)) > - skb_tail_pointer(skb_in) || skb_ensure_writable(skb_in, - skb_network_offset(skb_in) + sizeof(struct iphdr)))) - goto out; - - orig_ip = ip_hdr(skb_in)->saddr; - ip_hdr(skb_in)->saddr = ct->tuplehash[0].tuple.src.u3.ip; - icmp_send(skb_in, type, code, info); - ip_hdr(skb_in)->saddr = orig_ip; -out: - consume_skb(cloned_skb); -} -EXPORT_SYMBOL(icmp_ndo_send); -#endif static void icmp_socket_deliver(struct sk_buff *skb, u32 info) { diff --git a/net/ipv6/ip6_icmp.c b/net/ipv6/ip6_icmp.c index e0086758b6ee3..02045494c24cc 100644 --- a/net/ipv6/ip6_icmp.c +++ b/net/ipv6/ip6_icmp.c @@ -45,38 +45,4 @@ void icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info) rcu_read_unlock(); } EXPORT_SYMBOL(icmpv6_send); - -#if IS_ENABLED(CONFIG_NF_NAT) -#include -void icmpv6_ndo_send(struct sk_buff *skb_in, u8 type, u8 code, __u32 info) -{ - struct sk_buff *cloned_skb = NULL; - enum ip_conntrack_info ctinfo; - struct in6_addr orig_ip; - struct nf_conn *ct; - - ct = nf_ct_get(skb_in, &ctinfo); - if (!ct || !(ct->status & IPS_SRC_NAT)) { - icmpv6_send(skb_in, type, code, info); - return; - } - - if (skb_shared(skb_in)) - skb_in = cloned_skb = skb_clone(skb_in, GFP_ATOMIC); - - if (unlikely(!skb_in || skb_network_header(skb_in) < skb_in->head || - (skb_network_header(skb_in) + sizeof(struct ipv6hdr)) > - skb_tail_pointer(skb_in) || skb_ensure_writable(skb_in, - skb_network_offset(skb_in) + sizeof(struct ipv6hdr)))) - goto out; - - orig_ip = ipv6_hdr(skb_in)->saddr; - ipv6_hdr(skb_in)->saddr = ct->tuplehash[0].tuple.src.u3.in6; - icmpv6_send(skb_in, type, code, info); - ipv6_hdr(skb_in)->saddr = orig_ip; -out: - consume_skb(cloned_skb); -} -EXPORT_SYMBOL(icmpv6_ndo_send); -#endif #endif diff --git a/tools/testing/selftests/wireguard/netns.sh b/tools/testing/selftests/wireguard/netns.sh deleted file mode 100755 index 936e1ca9410ec..0000000000000 --- a/tools/testing/selftests/wireguard/netns.sh +++ /dev/null @@ -1,555 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-2.0 -# -# Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. -# -# This script tests the below topology: -# -# ┌─────────────────────┐ ┌──────────────────────────────────┐ ┌─────────────────────┐ -# │ $ns1 namespace │ │ $ns0 namespace │ │ $ns2 namespace │ -# │ │ │ │ │ │ -# │┌────────┐ │ │ ┌────────┐ │ │ ┌────────┐│ -# ││ wg0 │───────────┼───┼────────────│ lo │────────────┼───┼───────────│ wg0 ││ -# │├────────┴──────────┐│ │ ┌───────┴────────┴────────┐ │ │┌──────────┴────────┤│ -# ││192.168.241.1/24 ││ │ │(ns1) (ns2) │ │ ││192.168.241.2/24 ││ -# ││fd00::1/24 ││ │ │127.0.0.1:1 127.0.0.1:2│ │ ││fd00::2/24 ││ -# │└───────────────────┘│ │ │[::]:1 [::]:2 │ │ │└───────────────────┘│ -# └─────────────────────┘ │ └─────────────────────────┘ │ └─────────────────────┘ -# └──────────────────────────────────┘ -# -# After the topology is prepared we run a series of TCP/UDP iperf3 tests between the -# wireguard peers in $ns1 and $ns2. Note that $ns0 is the endpoint for the wg0 -# interfaces in $ns1 and $ns2. See https://www.wireguard.com/netns/ for further -# details on how this is accomplished. -set -e - -exec 3>&1 -export LANG=C -export WG_HIDE_KEYS=never -netns0="wg-test-$$-0" -netns1="wg-test-$$-1" -netns2="wg-test-$$-2" -pretty() { echo -e "\x1b[32m\x1b[1m[+] ${1:+NS$1: }${2}\x1b[0m" >&3; } -pp() { pretty "" "$*"; "$@"; } -maybe_exec() { if [[ $BASHPID -eq $$ ]]; then "$@"; else exec "$@"; fi; } -n0() { pretty 0 "$*"; maybe_exec ip netns exec $netns0 "$@"; } -n1() { pretty 1 "$*"; maybe_exec ip netns exec $netns1 "$@"; } -n2() { pretty 2 "$*"; maybe_exec ip netns exec $netns2 "$@"; } -ip0() { pretty 0 "ip $*"; ip -n $netns0 "$@"; } -ip1() { pretty 1 "ip $*"; ip -n $netns1 "$@"; } -ip2() { pretty 2 "ip $*"; ip -n $netns2 "$@"; } -sleep() { read -t "$1" -N 1 || true; } -waitiperf() { pretty "${1//*-}" "wait for iperf:5201 pid $2"; while [[ $(ss -N "$1" -tlpH 'sport = 5201') != *\"iperf3\",pid=$2,fd=* ]]; do sleep 0.1; done; } -waitncatudp() { pretty "${1//*-}" "wait for udp:1111 pid $2"; while [[ $(ss -N "$1" -ulpH 'sport = 1111') != *\"ncat\",pid=$2,fd=* ]]; do sleep 0.1; done; } -waitiface() { pretty "${1//*-}" "wait for $2 to come up"; ip netns exec "$1" bash -c "while [[ \$(< \"/sys/class/net/$2/operstate\") != up ]]; do read -t .1 -N 0 || true; done;"; } - -cleanup() { - set +e - exec 2>/dev/null - printf "$orig_message_cost" > /proc/sys/net/core/message_cost - ip0 link del dev wg0 - ip1 link del dev wg0 - ip2 link del dev wg0 - local to_kill="$(ip netns pids $netns0) $(ip netns pids $netns1) $(ip netns pids $netns2)" - [[ -n $to_kill ]] && kill $to_kill - pp ip netns del $netns1 - pp ip netns del $netns2 - pp ip netns del $netns0 - exit -} - -orig_message_cost="$(< /proc/sys/net/core/message_cost)" -trap cleanup EXIT -printf 0 > /proc/sys/net/core/message_cost - -ip netns del $netns0 2>/dev/null || true -ip netns del $netns1 2>/dev/null || true -ip netns del $netns2 2>/dev/null || true -pp ip netns add $netns0 -pp ip netns add $netns1 -pp ip netns add $netns2 -ip0 link set up dev lo - -ip0 link add dev wg0 type wireguard -ip0 link set wg0 netns $netns1 -ip0 link add dev wg0 type wireguard -ip0 link set wg0 netns $netns2 -key1="$(pp wg genkey)" -key2="$(pp wg genkey)" -key3="$(pp wg genkey)" -pub1="$(pp wg pubkey <<<"$key1")" -pub2="$(pp wg pubkey <<<"$key2")" -pub3="$(pp wg pubkey <<<"$key3")" -psk="$(pp wg genpsk)" -[[ -n $key1 && -n $key2 && -n $psk ]] - -configure_peers() { - ip1 addr add 192.168.241.1/24 dev wg0 - ip1 addr add fd00::1/24 dev wg0 - - ip2 addr add 192.168.241.2/24 dev wg0 - ip2 addr add fd00::2/24 dev wg0 - - n1 wg set wg0 \ - private-key <(echo "$key1") \ - listen-port 1 \ - peer "$pub2" \ - preshared-key <(echo "$psk") \ - allowed-ips 192.168.241.2/32,fd00::2/128 - n2 wg set wg0 \ - private-key <(echo "$key2") \ - listen-port 2 \ - peer "$pub1" \ - preshared-key <(echo "$psk") \ - allowed-ips 192.168.241.1/32,fd00::1/128 - - ip1 link set up dev wg0 - ip2 link set up dev wg0 -} -configure_peers - -tests() { - # Ping over IPv4 - n2 ping -c 10 -f -W 1 192.168.241.1 - n1 ping -c 10 -f -W 1 192.168.241.2 - - # Ping over IPv6 - n2 ping6 -c 10 -f -W 1 fd00::1 - n1 ping6 -c 10 -f -W 1 fd00::2 - - # TCP over IPv4 - n2 iperf3 -s -1 -B 192.168.241.2 & - waitiperf $netns2 $! - n1 iperf3 -Z -t 3 -c 192.168.241.2 - - # TCP over IPv6 - n1 iperf3 -s -1 -B fd00::1 & - waitiperf $netns1 $! - n2 iperf3 -Z -t 3 -c fd00::1 - - # UDP over IPv4 - n1 iperf3 -s -1 -B 192.168.241.1 & - waitiperf $netns1 $! - n2 iperf3 -Z -t 3 -b 0 -u -c 192.168.241.1 - - # UDP over IPv6 - n2 iperf3 -s -1 -B fd00::2 & - waitiperf $netns2 $! - n1 iperf3 -Z -t 3 -b 0 -u -c fd00::2 -} - -[[ $(ip1 link show dev wg0) =~ mtu\ ([0-9]+) ]] && orig_mtu="${BASH_REMATCH[1]}" -big_mtu=$(( 34816 - 1500 + $orig_mtu )) - -# Test using IPv4 as outer transport -n1 wg set wg0 peer "$pub2" endpoint 127.0.0.1:2 -n2 wg set wg0 peer "$pub1" endpoint 127.0.0.1:1 -# Before calling tests, we first make sure that the stats counters and timestamper are working -n2 ping -c 10 -f -W 1 192.168.241.1 -{ read _; read _; read _; read rx_bytes _; read _; read tx_bytes _; } < <(ip2 -stats link show dev wg0) -(( rx_bytes == 1372 && (tx_bytes == 1428 || tx_bytes == 1460) )) -{ read _; read _; read _; read rx_bytes _; read _; read tx_bytes _; } < <(ip1 -stats link show dev wg0) -(( tx_bytes == 1372 && (rx_bytes == 1428 || rx_bytes == 1460) )) -read _ rx_bytes tx_bytes < <(n2 wg show wg0 transfer) -(( rx_bytes == 1372 && (tx_bytes == 1428 || tx_bytes == 1460) )) -read _ rx_bytes tx_bytes < <(n1 wg show wg0 transfer) -(( tx_bytes == 1372 && (rx_bytes == 1428 || rx_bytes == 1460) )) -read _ timestamp < <(n1 wg show wg0 latest-handshakes) -(( timestamp != 0 )) - -tests -ip1 link set wg0 mtu $big_mtu -ip2 link set wg0 mtu $big_mtu -tests - -ip1 link set wg0 mtu $orig_mtu -ip2 link set wg0 mtu $orig_mtu - -# Test using IPv6 as outer transport -n1 wg set wg0 peer "$pub2" endpoint [::1]:2 -n2 wg set wg0 peer "$pub1" endpoint [::1]:1 -tests -ip1 link set wg0 mtu $big_mtu -ip2 link set wg0 mtu $big_mtu -tests - -# Test that route MTUs work with the padding -ip1 link set wg0 mtu 1300 -ip2 link set wg0 mtu 1300 -n1 wg set wg0 peer "$pub2" endpoint 127.0.0.1:2 -n2 wg set wg0 peer "$pub1" endpoint 127.0.0.1:1 -n0 iptables -A INPUT -m length --length 1360 -j DROP -n1 ip route add 192.168.241.2/32 dev wg0 mtu 1299 -n2 ip route add 192.168.241.1/32 dev wg0 mtu 1299 -n2 ping -c 1 -W 1 -s 1269 192.168.241.1 -n2 ip route delete 192.168.241.1/32 dev wg0 mtu 1299 -n1 ip route delete 192.168.241.2/32 dev wg0 mtu 1299 -n0 iptables -F INPUT - -ip1 link set wg0 mtu $orig_mtu -ip2 link set wg0 mtu $orig_mtu - -# Test using IPv4 that roaming works -ip0 -4 addr del 127.0.0.1/8 dev lo -ip0 -4 addr add 127.212.121.99/8 dev lo -n1 wg set wg0 listen-port 9999 -n1 wg set wg0 peer "$pub2" endpoint 127.0.0.1:2 -n1 ping6 -W 1 -c 1 fd00::2 -[[ $(n2 wg show wg0 endpoints) == "$pub1 127.212.121.99:9999" ]] - -# Test using IPv6 that roaming works -n1 wg set wg0 listen-port 9998 -n1 wg set wg0 peer "$pub2" endpoint [::1]:2 -n1 ping -W 1 -c 1 192.168.241.2 -[[ $(n2 wg show wg0 endpoints) == "$pub1 [::1]:9998" ]] - -# Test that crypto-RP filter works -n1 wg set wg0 peer "$pub2" allowed-ips 192.168.241.0/24 -exec 4< <(n1 ncat -l -u -p 1111) -ncat_pid=$! -waitncatudp $netns1 $ncat_pid -n2 ncat -u 192.168.241.1 1111 <<<"X" -read -r -N 1 -t 1 out <&4 && [[ $out == "X" ]] -kill $ncat_pid -more_specific_key="$(pp wg genkey | pp wg pubkey)" -n1 wg set wg0 peer "$more_specific_key" allowed-ips 192.168.241.2/32 -n2 wg set wg0 listen-port 9997 -exec 4< <(n1 ncat -l -u -p 1111) -ncat_pid=$! -waitncatudp $netns1 $ncat_pid -n2 ncat -u 192.168.241.1 1111 <<<"X" -! read -r -N 1 -t 1 out <&4 || false -kill $ncat_pid -n1 wg set wg0 peer "$more_specific_key" remove -[[ $(n1 wg show wg0 endpoints) == "$pub2 [::1]:9997" ]] - -# Test that we can change private keys keys and immediately handshake -n1 wg set wg0 private-key <(echo "$key1") peer "$pub2" preshared-key <(echo "$psk") allowed-ips 192.168.241.2/32 endpoint 127.0.0.1:2 -n2 wg set wg0 private-key <(echo "$key2") listen-port 2 peer "$pub1" preshared-key <(echo "$psk") allowed-ips 192.168.241.1/32 -n1 ping -W 1 -c 1 192.168.241.2 -n1 wg set wg0 private-key <(echo "$key3") -n2 wg set wg0 peer "$pub3" preshared-key <(echo "$psk") allowed-ips 192.168.241.1/32 peer "$pub1" remove -n1 ping -W 1 -c 1 192.168.241.2 - -ip1 link del wg0 -ip2 link del wg0 - -# Test using NAT. We now change the topology to this: -# ┌────────────────────────────────────────┐ ┌────────────────────────────────────────────────┐ ┌────────────────────────────────────────┐ -# │ $ns1 namespace │ │ $ns0 namespace │ │ $ns2 namespace │ -# │ │ │ │ │ │ -# │ ┌─────┐ ┌─────┐ │ │ ┌──────┐ ┌──────┐ │ │ ┌─────┐ ┌─────┐ │ -# │ │ wg0 │─────────────│vethc│───────────┼────┼────│vethrc│ │vethrs│──────────────┼─────┼──│veths│────────────│ wg0 │ │ -# │ ├─────┴──────────┐ ├─────┴──────────┐│ │ ├──────┴─────────┐ ├──────┴────────────┐ │ │ ├─────┴──────────┐ ├─────┴──────────┐ │ -# │ │192.168.241.1/24│ │192.168.1.100/24││ │ │192.168.1.1/24 │ │10.0.0.1/24 │ │ │ │10.0.0.100/24 │ │192.168.241.2/24│ │ -# │ │fd00::1/24 │ │ ││ │ │ │ │SNAT:192.168.1.0/24│ │ │ │ │ │fd00::2/24 │ │ -# │ └────────────────┘ └────────────────┘│ │ └────────────────┘ └───────────────────┘ │ │ └────────────────┘ └────────────────┘ │ -# └────────────────────────────────────────┘ └────────────────────────────────────────────────┘ └────────────────────────────────────────┘ - -ip1 link add dev wg0 type wireguard -ip2 link add dev wg0 type wireguard -configure_peers - -ip0 link add vethrc type veth peer name vethc -ip0 link add vethrs type veth peer name veths -ip0 link set vethc netns $netns1 -ip0 link set veths netns $netns2 -ip0 link set vethrc up -ip0 link set vethrs up -ip0 addr add 192.168.1.1/24 dev vethrc -ip0 addr add 10.0.0.1/24 dev vethrs -ip1 addr add 192.168.1.100/24 dev vethc -ip1 link set vethc up -ip1 route add default via 192.168.1.1 -ip2 addr add 10.0.0.100/24 dev veths -ip2 link set veths up -waitiface $netns0 vethrc -waitiface $netns0 vethrs -waitiface $netns1 vethc -waitiface $netns2 veths - -n0 bash -c 'printf 1 > /proc/sys/net/ipv4/ip_forward' -n0 bash -c 'printf 2 > /proc/sys/net/netfilter/nf_conntrack_udp_timeout' -n0 bash -c 'printf 2 > /proc/sys/net/netfilter/nf_conntrack_udp_timeout_stream' -n0 iptables -t nat -A POSTROUTING -s 192.168.1.0/24 -d 10.0.0.0/24 -j SNAT --to 10.0.0.1 - -n1 wg set wg0 peer "$pub2" endpoint 10.0.0.100:2 persistent-keepalive 1 -n1 ping -W 1 -c 1 192.168.241.2 -n2 ping -W 1 -c 1 192.168.241.1 -[[ $(n2 wg show wg0 endpoints) == "$pub1 10.0.0.1:1" ]] -# Demonstrate n2 can still send packets to n1, since persistent-keepalive will prevent connection tracking entry from expiring (to see entries: `n0 conntrack -L`). -pp sleep 3 -n2 ping -W 1 -c 1 192.168.241.1 -n1 wg set wg0 peer "$pub2" persistent-keepalive 0 - -# Do a wg-quick(8)-style policy routing for the default route, making sure vethc has a v6 address to tease out bugs. -ip1 -6 addr add fc00::9/96 dev vethc -ip1 -6 route add default via fc00::1 -ip2 -4 addr add 192.168.99.7/32 dev wg0 -ip2 -6 addr add abab::1111/128 dev wg0 -n1 wg set wg0 fwmark 51820 peer "$pub2" allowed-ips 192.168.99.7,abab::1111 -ip1 -6 route add default dev wg0 table 51820 -ip1 -6 rule add not fwmark 51820 table 51820 -ip1 -6 rule add table main suppress_prefixlength 0 -ip1 -4 route add default dev wg0 table 51820 -ip1 -4 rule add not fwmark 51820 table 51820 -ip1 -4 rule add table main suppress_prefixlength 0 -# Flood the pings instead of sending just one, to trigger routing table reference counting bugs. -n1 ping -W 1 -c 100 -f 192.168.99.7 -n1 ping -W 1 -c 100 -f abab::1111 - -# Have ns2 NAT into wg0 packets from ns0, but return an icmp error along the right route. -n2 iptables -t nat -A POSTROUTING -s 10.0.0.0/24 -d 192.168.241.0/24 -j SNAT --to 192.168.241.2 -n0 iptables -t filter -A INPUT \! -s 10.0.0.0/24 -i vethrs -j DROP # Manual rpfilter just to be explicit. -n2 bash -c 'printf 1 > /proc/sys/net/ipv4/ip_forward' -ip0 -4 route add 192.168.241.1 via 10.0.0.100 -n2 wg set wg0 peer "$pub1" remove -[[ $(! n0 ping -W 1 -c 1 192.168.241.1 || false) == *"From 10.0.0.100 icmp_seq=1 Destination Host Unreachable"* ]] - -n0 iptables -t nat -F -n0 iptables -t filter -F -n2 iptables -t nat -F -ip0 link del vethrc -ip0 link del vethrs -ip1 link del wg0 -ip2 link del wg0 - -# Test that saddr routing is sticky but not too sticky, changing to this topology: -# ┌────────────────────────────────────────┐ ┌────────────────────────────────────────┐ -# │ $ns1 namespace │ │ $ns2 namespace │ -# │ │ │ │ -# │ ┌─────┐ ┌─────┐ │ │ ┌─────┐ ┌─────┐ │ -# │ │ wg0 │─────────────│veth1│───────────┼────┼──│veth2│────────────│ wg0 │ │ -# │ ├─────┴──────────┐ ├─────┴──────────┐│ │ ├─────┴──────────┐ ├─────┴──────────┐ │ -# │ │192.168.241.1/24│ │10.0.0.1/24 ││ │ │10.0.0.2/24 │ │192.168.241.2/24│ │ -# │ │fd00::1/24 │ │fd00:aa::1/96 ││ │ │fd00:aa::2/96 │ │fd00::2/24 │ │ -# │ └────────────────┘ └────────────────┘│ │ └────────────────┘ └────────────────┘ │ -# └────────────────────────────────────────┘ └────────────────────────────────────────┘ - -ip1 link add dev wg0 type wireguard -ip2 link add dev wg0 type wireguard -configure_peers -ip1 link add veth1 type veth peer name veth2 -ip1 link set veth2 netns $netns2 -n1 bash -c 'printf 0 > /proc/sys/net/ipv6/conf/all/accept_dad' -n2 bash -c 'printf 0 > /proc/sys/net/ipv6/conf/all/accept_dad' -n1 bash -c 'printf 0 > /proc/sys/net/ipv6/conf/veth1/accept_dad' -n2 bash -c 'printf 0 > /proc/sys/net/ipv6/conf/veth2/accept_dad' -n1 bash -c 'printf 1 > /proc/sys/net/ipv4/conf/veth1/promote_secondaries' - -# First we check that we aren't overly sticky and can fall over to new IPs when old ones are removed -ip1 addr add 10.0.0.1/24 dev veth1 -ip1 addr add fd00:aa::1/96 dev veth1 -ip2 addr add 10.0.0.2/24 dev veth2 -ip2 addr add fd00:aa::2/96 dev veth2 -ip1 link set veth1 up -ip2 link set veth2 up -waitiface $netns1 veth1 -waitiface $netns2 veth2 -n1 wg set wg0 peer "$pub2" endpoint 10.0.0.2:2 -n1 ping -W 1 -c 1 192.168.241.2 -ip1 addr add 10.0.0.10/24 dev veth1 -ip1 addr del 10.0.0.1/24 dev veth1 -n1 ping -W 1 -c 1 192.168.241.2 -n1 wg set wg0 peer "$pub2" endpoint [fd00:aa::2]:2 -n1 ping -W 1 -c 1 192.168.241.2 -ip1 addr add fd00:aa::10/96 dev veth1 -ip1 addr del fd00:aa::1/96 dev veth1 -n1 ping -W 1 -c 1 192.168.241.2 - -# Now we show that we can successfully do reply to sender routing -ip1 link set veth1 down -ip2 link set veth2 down -ip1 addr flush dev veth1 -ip2 addr flush dev veth2 -ip1 addr add 10.0.0.1/24 dev veth1 -ip1 addr add 10.0.0.2/24 dev veth1 -ip1 addr add fd00:aa::1/96 dev veth1 -ip1 addr add fd00:aa::2/96 dev veth1 -ip2 addr add 10.0.0.3/24 dev veth2 -ip2 addr add fd00:aa::3/96 dev veth2 -ip1 link set veth1 up -ip2 link set veth2 up -waitiface $netns1 veth1 -waitiface $netns2 veth2 -n2 wg set wg0 peer "$pub1" endpoint 10.0.0.1:1 -n2 ping -W 1 -c 1 192.168.241.1 -[[ $(n2 wg show wg0 endpoints) == "$pub1 10.0.0.1:1" ]] -n2 wg set wg0 peer "$pub1" endpoint [fd00:aa::1]:1 -n2 ping -W 1 -c 1 192.168.241.1 -[[ $(n2 wg show wg0 endpoints) == "$pub1 [fd00:aa::1]:1" ]] -n2 wg set wg0 peer "$pub1" endpoint 10.0.0.2:1 -n2 ping -W 1 -c 1 192.168.241.1 -[[ $(n2 wg show wg0 endpoints) == "$pub1 10.0.0.2:1" ]] -n2 wg set wg0 peer "$pub1" endpoint [fd00:aa::2]:1 -n2 ping -W 1 -c 1 192.168.241.1 -[[ $(n2 wg show wg0 endpoints) == "$pub1 [fd00:aa::2]:1" ]] - -# What happens if the inbound destination address belongs to a different interface as the default route? -ip1 link add dummy0 type dummy -ip1 addr add 10.50.0.1/24 dev dummy0 -ip1 link set dummy0 up -ip2 route add 10.50.0.0/24 dev veth2 -n2 wg set wg0 peer "$pub1" endpoint 10.50.0.1:1 -n2 ping -W 1 -c 1 192.168.241.1 -[[ $(n2 wg show wg0 endpoints) == "$pub1 10.50.0.1:1" ]] - -ip1 link del dummy0 -ip1 addr flush dev veth1 -ip2 addr flush dev veth2 -ip1 route flush dev veth1 -ip2 route flush dev veth2 - -# Now we see what happens if another interface route takes precedence over an ongoing one -ip1 link add veth3 type veth peer name veth4 -ip1 link set veth4 netns $netns2 -ip1 addr add 10.0.0.1/24 dev veth1 -ip2 addr add 10.0.0.2/24 dev veth2 -ip1 addr add 10.0.0.3/24 dev veth3 -ip1 link set veth1 up -ip2 link set veth2 up -ip1 link set veth3 up -ip2 link set veth4 up -waitiface $netns1 veth1 -waitiface $netns2 veth2 -waitiface $netns1 veth3 -waitiface $netns2 veth4 -ip1 route flush dev veth1 -ip1 route flush dev veth3 -ip1 route add 10.0.0.0/24 dev veth1 src 10.0.0.1 metric 2 -n1 wg set wg0 peer "$pub2" endpoint 10.0.0.2:2 -n1 ping -W 1 -c 1 192.168.241.2 -[[ $(n2 wg show wg0 endpoints) == "$pub1 10.0.0.1:1" ]] -ip1 route add 10.0.0.0/24 dev veth3 src 10.0.0.3 metric 1 -n1 bash -c 'printf 0 > /proc/sys/net/ipv4/conf/veth1/rp_filter' -n2 bash -c 'printf 0 > /proc/sys/net/ipv4/conf/veth4/rp_filter' -n1 bash -c 'printf 0 > /proc/sys/net/ipv4/conf/all/rp_filter' -n2 bash -c 'printf 0 > /proc/sys/net/ipv4/conf/all/rp_filter' -n1 ping -W 1 -c 1 192.168.241.2 -[[ $(n2 wg show wg0 endpoints) == "$pub1 10.0.0.3:1" ]] - -ip1 link del veth1 -ip1 link del veth3 -ip1 link del wg0 -ip2 link del wg0 - -# We test that Netlink/IPC is working properly by doing things that usually cause split responses -ip0 link add dev wg0 type wireguard -config=( "[Interface]" "PrivateKey=$(wg genkey)" "[Peer]" "PublicKey=$(wg genkey)" ) -for a in {1..255}; do - for b in {0..255}; do - config+=( "AllowedIPs=$a.$b.0.0/16,$a::$b/128" ) - done -done -n0 wg setconf wg0 <(printf '%s\n' "${config[@]}") -i=0 -for ip in $(n0 wg show wg0 allowed-ips); do - ((++i)) -done -((i == 255*256*2+1)) -ip0 link del wg0 -ip0 link add dev wg0 type wireguard -config=( "[Interface]" "PrivateKey=$(wg genkey)" ) -for a in {1..40}; do - config+=( "[Peer]" "PublicKey=$(wg genkey)" ) - for b in {1..52}; do - config+=( "AllowedIPs=$a.$b.0.0/16" ) - done -done -n0 wg setconf wg0 <(printf '%s\n' "${config[@]}") -i=0 -while read -r line; do - j=0 - for ip in $line; do - ((++j)) - done - ((j == 53)) - ((++i)) -done < <(n0 wg show wg0 allowed-ips) -((i == 40)) -ip0 link del wg0 -ip0 link add wg0 type wireguard -config=( ) -for i in {1..29}; do - config+=( "[Peer]" "PublicKey=$(wg genkey)" ) -done -config+=( "[Peer]" "PublicKey=$(wg genkey)" "AllowedIPs=255.2.3.4/32,abcd::255/128" ) -n0 wg setconf wg0 <(printf '%s\n' "${config[@]}") -n0 wg showconf wg0 > /dev/null -ip0 link del wg0 - -allowedips=( ) -for i in {1..197}; do - allowedips+=( abcd::$i ) -done -saved_ifs="$IFS" -IFS=, -allowedips="${allowedips[*]}" -IFS="$saved_ifs" -ip0 link add wg0 type wireguard -n0 wg set wg0 peer "$pub1" -n0 wg set wg0 peer "$pub2" allowed-ips "$allowedips" -{ - read -r pub allowedips - [[ $pub == "$pub1" && $allowedips == "(none)" ]] - read -r pub allowedips - [[ $pub == "$pub2" ]] - i=0 - for _ in $allowedips; do - ((++i)) - done - ((i == 197)) -} < <(n0 wg show wg0 allowed-ips) -ip0 link del wg0 - -! n0 wg show doesnotexist || false - -ip0 link add wg0 type wireguard -n0 wg set wg0 private-key <(echo "$key1") peer "$pub2" preshared-key <(echo "$psk") -[[ $(n0 wg show wg0 private-key) == "$key1" ]] -[[ $(n0 wg show wg0 preshared-keys) == "$pub2 $psk" ]] -n0 wg set wg0 private-key /dev/null peer "$pub2" preshared-key /dev/null -[[ $(n0 wg show wg0 private-key) == "(none)" ]] -[[ $(n0 wg show wg0 preshared-keys) == "$pub2 (none)" ]] -n0 wg set wg0 peer "$pub2" -n0 wg set wg0 private-key <(echo "$key2") -[[ $(n0 wg show wg0 public-key) == "$pub2" ]] -[[ -z $(n0 wg show wg0 peers) ]] -n0 wg set wg0 peer "$pub2" -[[ -z $(n0 wg show wg0 peers) ]] -n0 wg set wg0 private-key <(echo "$key1") -n0 wg set wg0 peer "$pub2" -[[ $(n0 wg show wg0 peers) == "$pub2" ]] -n0 wg set wg0 private-key <(echo "/${key1:1}") -[[ $(n0 wg show wg0 private-key) == "+${key1:1}" ]] -n0 wg set wg0 peer "$pub2" allowed-ips 0.0.0.0/0,10.0.0.0/8,100.0.0.0/10,172.16.0.0/12,192.168.0.0/16 -n0 wg set wg0 peer "$pub2" allowed-ips 0.0.0.0/0 -n0 wg set wg0 peer "$pub2" allowed-ips ::/0,1700::/111,5000::/4,e000::/37,9000::/75 -n0 wg set wg0 peer "$pub2" allowed-ips ::/0 -n0 wg set wg0 peer "$pub2" remove -for low_order_point in AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= AQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= 4Ot6fDtBuK4WVuP68Z/EatoJjeucMrH9hmIFFl9JuAA= X5yVvKNQjCSx0LFVnIPvWwREXMRYHI6G2CJO3dCfEVc= 7P///////////////////////////////////////38= 7f///////////////////////////////////////38= 7v///////////////////////////////////////38=; do - n0 wg set wg0 peer "$low_order_point" persistent-keepalive 1 endpoint 127.0.0.1:1111 -done -[[ -n $(n0 wg show wg0 peers) ]] -exec 4< <(n0 ncat -l -u -p 1111) -ncat_pid=$! -waitncatudp $netns0 $ncat_pid -ip0 link set wg0 up -! read -r -n 1 -t 2 <&4 || false -kill $ncat_pid -ip0 link del wg0 - -declare -A objects -while read -t 0.1 -r line 2>/dev/null || [[ $? -ne 142 ]]; do - [[ $line =~ .*(wg[0-9]+:\ [A-Z][a-z]+\ [0-9]+)\ .*(created|destroyed).* ]] || continue - objects["${BASH_REMATCH[1]}"]+="${BASH_REMATCH[2]}" -done < /dev/kmsg -alldeleted=1 -for object in "${!objects[@]}"; do - if [[ ${objects["$object"]} != *createddestroyed ]]; then - echo "Error: $object: merely ${objects["$object"]}" >&3 - alldeleted=0 - fi -done -[[ $alldeleted -eq 1 ]] -pretty "" "Objects that were created were also destroyed." diff --git a/tools/testing/selftests/wireguard/qemu/.gitignore b/tools/testing/selftests/wireguard/qemu/.gitignore deleted file mode 100644 index 415b542a9d599..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -build/ -distfiles/ diff --git a/tools/testing/selftests/wireguard/qemu/Makefile b/tools/testing/selftests/wireguard/qemu/Makefile deleted file mode 100644 index 28d477683e8ab..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/Makefile +++ /dev/null @@ -1,377 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - -PWD := $(shell pwd) - -CHOST := $(shell gcc -dumpmachine) -HOST_ARCH := $(firstword $(subst -, ,$(CHOST))) -ifneq (,$(ARCH)) -CBUILD := $(subst -gcc,,$(lastword $(subst /, ,$(firstword $(wildcard $(foreach bindir,$(subst :, ,$(PATH)),$(bindir)/$(ARCH)-*-gcc)))))) -ifeq (,$(CBUILD)) -$(error The toolchain for $(ARCH) is not installed) -endif -else -CBUILD := $(CHOST) -ARCH := $(firstword $(subst -, ,$(CBUILD))) -endif - -# Set these from the environment to override -KERNEL_PATH ?= $(PWD)/../../../../.. -BUILD_PATH ?= $(PWD)/build/$(ARCH) -DISTFILES_PATH ?= $(PWD)/distfiles -NR_CPUS ?= 4 - -MIRROR := https://download.wireguard.com/qemu-test/distfiles/ - -default: qemu - -# variable name, tarball project name, version, tarball extension, default URI base -define tar_download = -$(1)_VERSION := $(3) -$(1)_NAME := $(2)-$$($(1)_VERSION) -$(1)_TAR := $(DISTFILES_PATH)/$$($(1)_NAME)$(4) -$(1)_PATH := $(BUILD_PATH)/$$($(1)_NAME) -$(call file_download,$$($(1)_NAME)$(4),$(5),$(6)) -endef - -define file_download = -$(DISTFILES_PATH)/$(1): - mkdir -p $(DISTFILES_PATH) - flock -x $$@.lock -c '[ -f $$@ ] && exit 0; wget -O $$@.tmp $(MIRROR)$(1) || wget -O $$@.tmp $(2)$(1) || rm -f $$@.tmp; [ -f $$@.tmp ] || exit 1; if echo "$(3) $$@.tmp" | sha256sum -c -; then mv $$@.tmp $$@; else rm -f $$@.tmp; exit 71; fi' -endef - -$(eval $(call tar_download,MUSL,musl,1.1.24,.tar.gz,https://www.musl-libc.org/releases/,1370c9a812b2cf2a7d92802510cca0058cc37e66a7bedd70051f0a34015022a3)) -$(eval $(call tar_download,IPERF,iperf,3.7,.tar.gz,https://downloads.es.net/pub/iperf/,d846040224317caf2f75c843d309a950a7db23f9b44b94688ccbe557d6d1710c)) -$(eval $(call tar_download,BASH,bash,5.0,.tar.gz,https://ftp.gnu.org/gnu/bash/,b4a80f2ac66170b2913efbfb9f2594f1f76c7b1afd11f799e22035d63077fb4d)) -$(eval $(call tar_download,IPROUTE2,iproute2,5.4.0,.tar.xz,https://www.kernel.org/pub/linux/utils/net/iproute2/,fe97aa60a0d4c5ac830be18937e18dc3400ca713a33a89ad896ff1e3d46086ae)) -$(eval $(call tar_download,IPTABLES,iptables,1.8.4,.tar.bz2,https://www.netfilter.org/projects/iptables/files/,993a3a5490a544c2cbf2ef15cf7e7ed21af1845baf228318d5c36ef8827e157c)) -$(eval $(call tar_download,NMAP,nmap,7.80,.tar.bz2,https://nmap.org/dist/,fcfa5a0e42099e12e4bf7a68ebe6fde05553383a682e816a7ec9256ab4773faa)) -$(eval $(call tar_download,IPUTILS,iputils,s20190709,.tar.gz,https://github.com/iputils/iputils/archive/s20190709.tar.gz/#,a15720dd741d7538dd2645f9f516d193636ae4300ff7dbc8bfca757bf166490a)) -$(eval $(call tar_download,WIREGUARD_TOOLS,wireguard-tools,1.0.20200206,.tar.xz,https://git.zx2c4.com/wireguard-tools/snapshot/,f5207248c6a3c3e3bfc9ab30b91c1897b00802ed861e1f9faaed873366078c64)) - -KERNEL_BUILD_PATH := $(BUILD_PATH)/kernel$(if $(findstring yes,$(DEBUG_KERNEL)),-debug) -rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst *,%,$2),$d)) -WIREGUARD_SOURCES := $(call rwildcard,$(KERNEL_PATH)/drivers/net/wireguard/,*) - -export CFLAGS ?= -O3 -pipe -export LDFLAGS ?= -export CPPFLAGS := -I$(BUILD_PATH)/include - -ifeq ($(HOST_ARCH),$(ARCH)) -CROSS_COMPILE_FLAG := --host=$(CHOST) -CFLAGS += -march=native -STRIP := strip -else -$(info Cross compilation: building for $(CBUILD) using $(CHOST)) -CROSS_COMPILE_FLAG := --build=$(CBUILD) --host=$(CHOST) -export CROSS_COMPILE=$(CBUILD)- -STRIP := $(CBUILD)-strip -endif -ifeq ($(ARCH),aarch64) -QEMU_ARCH := aarch64 -KERNEL_ARCH := arm64 -KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/arch/arm64/boot/Image -ifeq ($(HOST_ARCH),$(ARCH)) -QEMU_MACHINE := -cpu host -machine virt,gic_version=host,accel=kvm -else -QEMU_MACHINE := -cpu cortex-a53 -machine virt -CFLAGS += -march=armv8-a -mtune=cortex-a53 -endif -else ifeq ($(ARCH),aarch64_be) -QEMU_ARCH := aarch64 -KERNEL_ARCH := arm64 -KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/arch/arm64/boot/Image -ifeq ($(HOST_ARCH),$(ARCH)) -QEMU_MACHINE := -cpu host -machine virt,gic_version=host,accel=kvm -else -QEMU_MACHINE := -cpu cortex-a53 -machine virt -CFLAGS += -march=armv8-a -mtune=cortex-a53 -endif -else ifeq ($(ARCH),arm) -QEMU_ARCH := arm -KERNEL_ARCH := arm -KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/arch/arm/boot/zImage -ifeq ($(HOST_ARCH),$(ARCH)) -QEMU_MACHINE := -cpu host -machine virt,gic_version=host,accel=kvm -else -QEMU_MACHINE := -cpu cortex-a15 -machine virt -CFLAGS += -march=armv7-a -mtune=cortex-a15 -mabi=aapcs-linux -endif -else ifeq ($(ARCH),armeb) -QEMU_ARCH := arm -KERNEL_ARCH := arm -KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/arch/arm/boot/zImage -ifeq ($(HOST_ARCH),$(ARCH)) -QEMU_MACHINE := -cpu host -machine virt,gic_version=host,accel=kvm -else -QEMU_MACHINE := -cpu cortex-a15 -machine virt -CFLAGS += -march=armv7-a -mabi=aapcs-linux # We don't pass -mtune=cortex-a15 due to a compiler bug on big endian. -LDFLAGS += -Wl,--be8 -endif -else ifeq ($(ARCH),x86_64) -QEMU_ARCH := x86_64 -KERNEL_ARCH := x86_64 -KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/arch/x86/boot/bzImage -ifeq ($(HOST_ARCH),$(ARCH)) -QEMU_MACHINE := -cpu host -machine q35,accel=kvm -else -QEMU_MACHINE := -cpu Skylake-Server -machine q35 -CFLAGS += -march=skylake-avx512 -endif -else ifeq ($(ARCH),i686) -QEMU_ARCH := i386 -KERNEL_ARCH := x86 -KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/arch/x86/boot/bzImage -ifeq ($(subst x86_64,i686,$(HOST_ARCH)),$(ARCH)) -QEMU_MACHINE := -cpu host -machine q35,accel=kvm -else -QEMU_MACHINE := -cpu coreduo -machine q35 -CFLAGS += -march=prescott -endif -else ifeq ($(ARCH),mips64) -QEMU_ARCH := mips64 -KERNEL_ARCH := mips -KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/vmlinux -ifeq ($(HOST_ARCH),$(ARCH)) -QEMU_MACHINE := -cpu host -machine malta,accel=kvm -CFLAGS += -EB -else -QEMU_MACHINE := -cpu MIPS64R2-generic -machine malta -smp 1 -CFLAGS += -march=mips64r2 -EB -endif -else ifeq ($(ARCH),mips64el) -QEMU_ARCH := mips64el -KERNEL_ARCH := mips -KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/vmlinux -ifeq ($(HOST_ARCH),$(ARCH)) -QEMU_MACHINE := -cpu host -machine malta,accel=kvm -CFLAGS += -EL -else -QEMU_MACHINE := -cpu MIPS64R2-generic -machine malta -smp 1 -CFLAGS += -march=mips64r2 -EL -endif -else ifeq ($(ARCH),mips) -QEMU_ARCH := mips -KERNEL_ARCH := mips -KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/vmlinux -ifeq ($(HOST_ARCH),$(ARCH)) -QEMU_MACHINE := -cpu host -machine malta,accel=kvm -CFLAGS += -EB -else -QEMU_MACHINE := -cpu 24Kf -machine malta -smp 1 -CFLAGS += -march=mips32r2 -EB -endif -else ifeq ($(ARCH),mipsel) -QEMU_ARCH := mipsel -KERNEL_ARCH := mips -KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/vmlinux -ifeq ($(HOST_ARCH),$(ARCH)) -QEMU_MACHINE := -cpu host -machine malta,accel=kvm -CFLAGS += -EL -else -QEMU_MACHINE := -cpu 24Kf -machine malta -smp 1 -CFLAGS += -march=mips32r2 -EL -endif -else ifeq ($(ARCH),powerpc64le) -QEMU_ARCH := ppc64 -KERNEL_ARCH := powerpc -KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/vmlinux -ifeq ($(HOST_ARCH),$(ARCH)) -QEMU_MACHINE := -cpu host,accel=kvm -machine pseries -else -QEMU_MACHINE := -machine pseries -endif -CFLAGS += -mcpu=powerpc64le -mlong-double-64 -else ifeq ($(ARCH),powerpc) -QEMU_ARCH := ppc -KERNEL_ARCH := powerpc -KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/arch/powerpc/boot/uImage -ifeq ($(HOST_ARCH),$(ARCH)) -QEMU_MACHINE := -cpu host,accel=kvm -machine ppce500 -else -QEMU_MACHINE := -machine ppce500 -endif -CFLAGS += -mcpu=powerpc -mlong-double-64 -msecure-plt -else ifeq ($(ARCH),m68k) -QEMU_ARCH := m68k -KERNEL_ARCH := m68k -KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/vmlinux -KERNEL_CMDLINE := $(shell sed -n 's/CONFIG_CMDLINE=\(.*\)/\1/p' arch/m68k.config) -ifeq ($(HOST_ARCH),$(ARCH)) -QEMU_MACHINE := -cpu host,accel=kvm -machine q800 -smp 1 -append $(KERNEL_CMDLINE) -else -QEMU_MACHINE := -machine q800 -smp 1 -append $(KERNEL_CMDLINE) -endif -else -$(error I only build: x86_64, i686, arm, armeb, aarch64, aarch64_be, mips, mipsel, mips64, mips64el, powerpc64le, powerpc, m68k) -endif - -REAL_CC := $(CBUILD)-gcc -MUSL_CC := $(BUILD_PATH)/musl-gcc -export CC := $(MUSL_CC) -USERSPACE_DEPS := $(MUSL_CC) $(BUILD_PATH)/include/.installed $(BUILD_PATH)/include/linux/.installed - -build: $(KERNEL_BZIMAGE) -qemu: $(KERNEL_BZIMAGE) - rm -f $(BUILD_PATH)/result - timeout --foreground 20m qemu-system-$(QEMU_ARCH) \ - -nodefaults \ - -nographic \ - -smp $(NR_CPUS) \ - $(QEMU_MACHINE) \ - -m $$(grep -q CONFIG_DEBUG_KMEMLEAK=y $(KERNEL_BUILD_PATH)/.config && echo 1G || echo 256M) \ - -serial stdio \ - -serial file:$(BUILD_PATH)/result \ - -no-reboot \ - -monitor none \ - -kernel $< - grep -Fq success $(BUILD_PATH)/result - -$(BUILD_PATH)/init-cpio-spec.txt: - mkdir -p $(BUILD_PATH) - echo "file /init $(BUILD_PATH)/init 755 0 0" > $@ - echo "file /init.sh $(PWD)/../netns.sh 755 0 0" >> $@ - echo "dir /dev 755 0 0" >> $@ - echo "nod /dev/console 644 0 0 c 5 1" >> $@ - echo "dir /bin 755 0 0" >> $@ - echo "file /bin/iperf3 $(IPERF_PATH)/src/iperf3 755 0 0" >> $@ - echo "file /bin/wg $(WIREGUARD_TOOLS_PATH)/src/wg 755 0 0" >> $@ - echo "file /bin/bash $(BASH_PATH)/bash 755 0 0" >> $@ - echo "file /bin/ip $(IPROUTE2_PATH)/ip/ip 755 0 0" >> $@ - echo "file /bin/ss $(IPROUTE2_PATH)/misc/ss 755 0 0" >> $@ - echo "file /bin/ping $(IPUTILS_PATH)/ping 755 0 0" >> $@ - echo "file /bin/ncat $(NMAP_PATH)/ncat/ncat 755 0 0" >> $@ - echo "file /bin/xtables-legacy-multi $(IPTABLES_PATH)/iptables/xtables-legacy-multi 755 0 0" >> $@ - echo "slink /bin/iptables xtables-legacy-multi 777 0 0" >> $@ - echo "slink /bin/ping6 ping 777 0 0" >> $@ - echo "dir /lib 755 0 0" >> $@ - echo "file /lib/libc.so $(MUSL_PATH)/lib/libc.so 755 0 0" >> $@ - echo "slink /lib/ld-linux.so.1 libc.so 777 0 0" >> $@ - -$(KERNEL_BUILD_PATH)/.config: kernel.config arch/$(ARCH).config - mkdir -p $(KERNEL_BUILD_PATH) - cp kernel.config $(KERNEL_BUILD_PATH)/minimal.config - printf 'CONFIG_NR_CPUS=$(NR_CPUS)\nCONFIG_INITRAMFS_SOURCE="$(BUILD_PATH)/init-cpio-spec.txt"\n' >> $(KERNEL_BUILD_PATH)/minimal.config - cat arch/$(ARCH).config >> $(KERNEL_BUILD_PATH)/minimal.config - $(MAKE) -C $(KERNEL_PATH) O=$(KERNEL_BUILD_PATH) ARCH=$(KERNEL_ARCH) allnoconfig - cd $(KERNEL_BUILD_PATH) && ARCH=$(KERNEL_ARCH) $(KERNEL_PATH)/scripts/kconfig/merge_config.sh -n $(KERNEL_BUILD_PATH)/.config $(KERNEL_BUILD_PATH)/minimal.config - $(if $(findstring yes,$(DEBUG_KERNEL)),cp debug.config $(KERNEL_BUILD_PATH) && cd $(KERNEL_BUILD_PATH) && ARCH=$(KERNEL_ARCH) $(KERNEL_PATH)/scripts/kconfig/merge_config.sh -n $(KERNEL_BUILD_PATH)/.config debug.config,) - -$(KERNEL_BZIMAGE): $(KERNEL_BUILD_PATH)/.config $(BUILD_PATH)/init-cpio-spec.txt $(MUSL_PATH)/lib/libc.so $(IPERF_PATH)/src/iperf3 $(IPUTILS_PATH)/ping $(BASH_PATH)/bash $(IPROUTE2_PATH)/misc/ss $(IPROUTE2_PATH)/ip/ip $(IPTABLES_PATH)/iptables/xtables-legacy-multi $(NMAP_PATH)/ncat/ncat $(WIREGUARD_TOOLS_PATH)/src/wg $(BUILD_PATH)/init ../netns.sh $(WIREGUARD_SOURCES) - $(MAKE) -C $(KERNEL_PATH) O=$(KERNEL_BUILD_PATH) ARCH=$(KERNEL_ARCH) CROSS_COMPILE=$(CROSS_COMPILE) - -$(BUILD_PATH)/include/linux/.installed: | $(KERNEL_BUILD_PATH)/.config - $(MAKE) -C $(KERNEL_PATH) O=$(KERNEL_BUILD_PATH) INSTALL_HDR_PATH=$(BUILD_PATH) ARCH=$(KERNEL_ARCH) CROSS_COMPILE=$(CROSS_COMPILE) headers_install - touch $@ - -$(MUSL_PATH)/lib/libc.so: $(MUSL_TAR) - mkdir -p $(BUILD_PATH) - flock -s $<.lock tar -C $(BUILD_PATH) -xf $< - cd $(MUSL_PATH) && CC=$(REAL_CC) ./configure --prefix=/ --disable-static --build=$(CBUILD) - $(MAKE) -C $(MUSL_PATH) - $(STRIP) -s $@ - -$(BUILD_PATH)/include/.installed: $(MUSL_PATH)/lib/libc.so - $(MAKE) -C $(MUSL_PATH) DESTDIR=$(BUILD_PATH) install-headers - touch $@ - -$(MUSL_CC): $(MUSL_PATH)/lib/libc.so - sh $(MUSL_PATH)/tools/musl-gcc.specs.sh $(BUILD_PATH)/include $(MUSL_PATH)/lib /lib/ld-linux.so.1 > $(BUILD_PATH)/musl-gcc.specs - printf '#!/bin/sh\nexec "$(REAL_CC)" --specs="$(BUILD_PATH)/musl-gcc.specs" "$$@"\n' > $(BUILD_PATH)/musl-gcc - chmod +x $(BUILD_PATH)/musl-gcc - -$(IPERF_PATH)/.installed: $(IPERF_TAR) - mkdir -p $(BUILD_PATH) - flock -s $<.lock tar -C $(BUILD_PATH) -xf $< - sed -i '1s/^/#include /' $(IPERF_PATH)/src/cjson.h $(IPERF_PATH)/src/timer.h - sed -i -r 's/-p?g//g' $(IPERF_PATH)/src/Makefile* - touch $@ - -$(IPERF_PATH)/src/iperf3: | $(IPERF_PATH)/.installed $(USERSPACE_DEPS) - cd $(IPERF_PATH) && CFLAGS="$(CFLAGS) -D_GNU_SOURCE" ./configure --prefix=/ $(CROSS_COMPILE_FLAG) --enable-static --disable-shared --with-openssl=no - $(MAKE) -C $(IPERF_PATH) - $(STRIP) -s $@ - -$(WIREGUARD_TOOLS_PATH)/.installed: $(WIREGUARD_TOOLS_TAR) - mkdir -p $(BUILD_PATH) - flock -s $<.lock tar -C $(BUILD_PATH) -xf $< - touch $@ - -$(WIREGUARD_TOOLS_PATH)/src/wg: | $(WIREGUARD_TOOLS_PATH)/.installed $(USERSPACE_DEPS) - $(MAKE) -C $(WIREGUARD_TOOLS_PATH)/src wg - $(STRIP) -s $@ - -$(BUILD_PATH)/init: init.c | $(USERSPACE_DEPS) - mkdir -p $(BUILD_PATH) - $(MUSL_CC) -o $@ $(CFLAGS) $(LDFLAGS) -std=gnu11 $< - $(STRIP) -s $@ - -$(IPUTILS_PATH)/.installed: $(IPUTILS_TAR) - mkdir -p $(BUILD_PATH) - flock -s $<.lock tar -C $(BUILD_PATH) -xf $< - touch $@ - -$(IPUTILS_PATH)/ping: | $(IPUTILS_PATH)/.installed $(USERSPACE_DEPS) - sed -i /atexit/d $(IPUTILS_PATH)/ping.c - cd $(IPUTILS_PATH) && $(CC) $(CFLAGS) -std=c99 -o $@ ping.c ping_common.c ping6_common.c iputils_common.c -D_GNU_SOURCE -D'IPUTILS_VERSION(f)=f' -lresolv $(LDFLAGS) - $(STRIP) -s $@ - -$(BASH_PATH)/.installed: $(BASH_TAR) - mkdir -p $(BUILD_PATH) - flock -s $<.lock tar -C $(BUILD_PATH) -xf $< - touch $@ - -$(BASH_PATH)/bash: | $(BASH_PATH)/.installed $(USERSPACE_DEPS) - cd $(BASH_PATH) && ./configure --prefix=/ $(CROSS_COMPILE_FLAG) --without-bash-malloc --disable-debugger --disable-help-builtin --disable-history --disable-multibyte --disable-progcomp --disable-readline --disable-mem-scramble - $(MAKE) -C $(BASH_PATH) - $(STRIP) -s $@ - -$(IPROUTE2_PATH)/.installed: $(IPROUTE2_TAR) - mkdir -p $(BUILD_PATH) - flock -s $<.lock tar -C $(BUILD_PATH) -xf $< - printf 'CC:=$(CC)\nPKG_CONFIG:=pkg-config\nTC_CONFIG_XT:=n\nTC_CONFIG_ATM:=n\nTC_CONFIG_IPSET:=n\nIP_CONFIG_SETNS:=y\nHAVE_ELF:=n\nHAVE_MNL:=n\nHAVE_BERKELEY_DB:=n\nHAVE_LATEX:=n\nHAVE_PDFLATEX:=n\nCFLAGS+=-DHAVE_SETNS\n' > $(IPROUTE2_PATH)/config.mk - printf 'lib: snapshot\n\t$$(MAKE) -C lib\nip/ip: lib\n\t$$(MAKE) -C ip ip\nmisc/ss: lib\n\t$$(MAKE) -C misc ss\n' >> $(IPROUTE2_PATH)/Makefile - touch $@ - -$(IPROUTE2_PATH)/ip/ip: | $(IPROUTE2_PATH)/.installed $(USERSPACE_DEPS) - $(MAKE) -C $(IPROUTE2_PATH) PREFIX=/ ip/ip - $(STRIP) -s $@ - -$(IPROUTE2_PATH)/misc/ss: | $(IPROUTE2_PATH)/.installed $(USERSPACE_DEPS) - $(MAKE) -C $(IPROUTE2_PATH) PREFIX=/ misc/ss - $(STRIP) -s $@ - -$(IPTABLES_PATH)/.installed: $(IPTABLES_TAR) - mkdir -p $(BUILD_PATH) - flock -s $<.lock tar -C $(BUILD_PATH) -xf $< - sed -i -e "/nfnetlink=[01]/s:=[01]:=0:" -e "/nfconntrack=[01]/s:=[01]:=0:" $(IPTABLES_PATH)/configure - touch $@ - -$(IPTABLES_PATH)/iptables/xtables-legacy-multi: | $(IPTABLES_PATH)/.installed $(USERSPACE_DEPS) - cd $(IPTABLES_PATH) && ./configure --prefix=/ $(CROSS_COMPILE_FLAG) --enable-static --disable-shared --disable-nftables --disable-bpf-compiler --disable-nfsynproxy --disable-libipq --disable-connlabel --with-kernel=$(BUILD_PATH)/include - $(MAKE) -C $(IPTABLES_PATH) - $(STRIP) -s $@ - -$(NMAP_PATH)/.installed: $(NMAP_TAR) - mkdir -p $(BUILD_PATH) - flock -s $<.lock tar -C $(BUILD_PATH) -xf $< - touch $@ - -$(NMAP_PATH)/ncat/ncat: | $(NMAP_PATH)/.installed $(USERSPACE_DEPS) - cd $(NMAP_PATH) && ./configure --prefix=/ $(CROSS_COMPILE_FLAG) --enable-static --disable-shared --without-ndiff --without-zenmap --without-nping --with-libpcap=included --with-libpcre=included --with-libdnet=included --without-liblua --with-liblinear=included --without-nmap-update --without-openssl --with-pcap=linux --without-libssh - $(MAKE) -C $(NMAP_PATH)/libpcap - $(MAKE) -C $(NMAP_PATH)/ncat - $(STRIP) -s $@ - -clean: - rm -rf $(BUILD_PATH) - -distclean: clean - rm -rf $(DISTFILES_PATH) - -menuconfig: $(KERNEL_BUILD_PATH)/.config - $(MAKE) -C $(KERNEL_PATH) O=$(KERNEL_BUILD_PATH) ARCH=$(KERNEL_ARCH) CROSS_COMPILE=$(CROSS_COMPILE) menuconfig - -.PHONY: qemu build clean distclean menuconfig -.DELETE_ON_ERROR: diff --git a/tools/testing/selftests/wireguard/qemu/arch/aarch64.config b/tools/testing/selftests/wireguard/qemu/arch/aarch64.config deleted file mode 100644 index 3d063bb247bbe..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/arch/aarch64.config +++ /dev/null @@ -1,5 +0,0 @@ -CONFIG_SERIAL_AMBA_PL011=y -CONFIG_SERIAL_AMBA_PL011_CONSOLE=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="console=ttyAMA0 wg.success=ttyAMA1" -CONFIG_FRAME_WARN=1280 diff --git a/tools/testing/selftests/wireguard/qemu/arch/aarch64_be.config b/tools/testing/selftests/wireguard/qemu/arch/aarch64_be.config deleted file mode 100644 index dbdc7e406a7ba..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/arch/aarch64_be.config +++ /dev/null @@ -1,6 +0,0 @@ -CONFIG_CPU_BIG_ENDIAN=y -CONFIG_SERIAL_AMBA_PL011=y -CONFIG_SERIAL_AMBA_PL011_CONSOLE=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="console=ttyAMA0 wg.success=ttyAMA1" -CONFIG_FRAME_WARN=1280 diff --git a/tools/testing/selftests/wireguard/qemu/arch/arm.config b/tools/testing/selftests/wireguard/qemu/arch/arm.config deleted file mode 100644 index 148f499054182..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/arch/arm.config +++ /dev/null @@ -1,9 +0,0 @@ -CONFIG_MMU=y -CONFIG_ARCH_MULTI_V7=y -CONFIG_ARCH_VIRT=y -CONFIG_THUMB2_KERNEL=n -CONFIG_SERIAL_AMBA_PL011=y -CONFIG_SERIAL_AMBA_PL011_CONSOLE=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="console=ttyAMA0 wg.success=ttyAMA1" -CONFIG_FRAME_WARN=1024 diff --git a/tools/testing/selftests/wireguard/qemu/arch/armeb.config b/tools/testing/selftests/wireguard/qemu/arch/armeb.config deleted file mode 100644 index bd76b07d00a2c..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/arch/armeb.config +++ /dev/null @@ -1,10 +0,0 @@ -CONFIG_MMU=y -CONFIG_ARCH_MULTI_V7=y -CONFIG_ARCH_VIRT=y -CONFIG_THUMB2_KERNEL=n -CONFIG_SERIAL_AMBA_PL011=y -CONFIG_SERIAL_AMBA_PL011_CONSOLE=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="console=ttyAMA0 wg.success=ttyAMA1" -CONFIG_CPU_BIG_ENDIAN=y -CONFIG_FRAME_WARN=1024 diff --git a/tools/testing/selftests/wireguard/qemu/arch/i686.config b/tools/testing/selftests/wireguard/qemu/arch/i686.config deleted file mode 100644 index a85025d7206ee..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/arch/i686.config +++ /dev/null @@ -1,5 +0,0 @@ -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" -CONFIG_FRAME_WARN=1024 diff --git a/tools/testing/selftests/wireguard/qemu/arch/m68k.config b/tools/testing/selftests/wireguard/qemu/arch/m68k.config deleted file mode 100644 index 62a15bdb877e7..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/arch/m68k.config +++ /dev/null @@ -1,9 +0,0 @@ -CONFIG_MMU=y -CONFIG_M68KCLASSIC=y -CONFIG_M68040=y -CONFIG_MAC=y -CONFIG_SERIAL_PMACZILOG=y -CONFIG_SERIAL_PMACZILOG_TTYS=y -CONFIG_SERIAL_PMACZILOG_CONSOLE=y -CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" -CONFIG_FRAME_WARN=1024 diff --git a/tools/testing/selftests/wireguard/qemu/arch/mips.config b/tools/testing/selftests/wireguard/qemu/arch/mips.config deleted file mode 100644 index df71d6b95546f..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/arch/mips.config +++ /dev/null @@ -1,11 +0,0 @@ -CONFIG_CPU_MIPS32_R2=y -CONFIG_MIPS_MALTA=y -CONFIG_MIPS_CPS=y -CONFIG_MIPS_FP_SUPPORT=y -CONFIG_POWER_RESET=y -CONFIG_POWER_RESET_SYSCON=y -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" -CONFIG_FRAME_WARN=1024 diff --git a/tools/testing/selftests/wireguard/qemu/arch/mips64.config b/tools/testing/selftests/wireguard/qemu/arch/mips64.config deleted file mode 100644 index 90c783f725c4d..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/arch/mips64.config +++ /dev/null @@ -1,14 +0,0 @@ -CONFIG_64BIT=y -CONFIG_CPU_MIPS64_R2=y -CONFIG_MIPS32_N32=y -CONFIG_CPU_HAS_MSA=y -CONFIG_MIPS_MALTA=y -CONFIG_MIPS_CPS=y -CONFIG_MIPS_FP_SUPPORT=y -CONFIG_POWER_RESET=y -CONFIG_POWER_RESET_SYSCON=y -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" -CONFIG_FRAME_WARN=1280 diff --git a/tools/testing/selftests/wireguard/qemu/arch/mips64el.config b/tools/testing/selftests/wireguard/qemu/arch/mips64el.config deleted file mode 100644 index 435b0b43e00cb..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/arch/mips64el.config +++ /dev/null @@ -1,15 +0,0 @@ -CONFIG_64BIT=y -CONFIG_CPU_MIPS64_R2=y -CONFIG_MIPS32_N32=y -CONFIG_CPU_HAS_MSA=y -CONFIG_MIPS_MALTA=y -CONFIG_CPU_LITTLE_ENDIAN=y -CONFIG_MIPS_CPS=y -CONFIG_MIPS_FP_SUPPORT=y -CONFIG_POWER_RESET=y -CONFIG_POWER_RESET_SYSCON=y -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" -CONFIG_FRAME_WARN=1280 diff --git a/tools/testing/selftests/wireguard/qemu/arch/mipsel.config b/tools/testing/selftests/wireguard/qemu/arch/mipsel.config deleted file mode 100644 index 62bb50c4a85fc..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/arch/mipsel.config +++ /dev/null @@ -1,12 +0,0 @@ -CONFIG_CPU_MIPS32_R2=y -CONFIG_MIPS_MALTA=y -CONFIG_CPU_LITTLE_ENDIAN=y -CONFIG_MIPS_CPS=y -CONFIG_MIPS_FP_SUPPORT=y -CONFIG_POWER_RESET=y -CONFIG_POWER_RESET_SYSCON=y -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" -CONFIG_FRAME_WARN=1024 diff --git a/tools/testing/selftests/wireguard/qemu/arch/powerpc.config b/tools/testing/selftests/wireguard/qemu/arch/powerpc.config deleted file mode 100644 index 57957093b71b8..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/arch/powerpc.config +++ /dev/null @@ -1,10 +0,0 @@ -CONFIG_PPC_QEMU_E500=y -CONFIG_FSL_SOC_BOOKE=y -CONFIG_PPC_85xx=y -CONFIG_PHYS_64BIT=y -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_MATH_EMULATION=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" -CONFIG_FRAME_WARN=1024 diff --git a/tools/testing/selftests/wireguard/qemu/arch/powerpc64le.config b/tools/testing/selftests/wireguard/qemu/arch/powerpc64le.config deleted file mode 100644 index 990c510a9cfa5..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/arch/powerpc64le.config +++ /dev/null @@ -1,12 +0,0 @@ -CONFIG_PPC64=y -CONFIG_PPC_PSERIES=y -CONFIG_ALTIVEC=y -CONFIG_VSX=y -CONFIG_PPC_OF_BOOT_TRAMPOLINE=y -CONFIG_PPC_RADIX_MMU=y -CONFIG_HVC_CONSOLE=y -CONFIG_CPU_LITTLE_ENDIAN=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="console=hvc0 wg.success=hvc1" -CONFIG_SECTION_MISMATCH_WARN_ONLY=y -CONFIG_FRAME_WARN=1280 diff --git a/tools/testing/selftests/wireguard/qemu/arch/x86_64.config b/tools/testing/selftests/wireguard/qemu/arch/x86_64.config deleted file mode 100644 index 00a1ef4869d58..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/arch/x86_64.config +++ /dev/null @@ -1,5 +0,0 @@ -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" -CONFIG_FRAME_WARN=1280 diff --git a/tools/testing/selftests/wireguard/qemu/debug.config b/tools/testing/selftests/wireguard/qemu/debug.config deleted file mode 100644 index b9c72706fe4db..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/debug.config +++ /dev/null @@ -1,67 +0,0 @@ -CONFIG_LOCALVERSION="-debug" -CONFIG_ENABLE_WARN_DEPRECATED=y -CONFIG_ENABLE_MUST_CHECK=y -CONFIG_FRAME_POINTER=y -CONFIG_STACK_VALIDATION=y -CONFIG_DEBUG_KERNEL=y -CONFIG_DEBUG_INFO=y -CONFIG_DEBUG_INFO_DWARF4=y -CONFIG_PAGE_EXTENSION=y -CONFIG_PAGE_POISONING=y -CONFIG_DEBUG_OBJECTS=y -CONFIG_DEBUG_OBJECTS_FREE=y -CONFIG_DEBUG_OBJECTS_TIMERS=y -CONFIG_DEBUG_OBJECTS_WORK=y -CONFIG_DEBUG_OBJECTS_RCU_HEAD=y -CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER=y -CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT=1 -CONFIG_SLUB_DEBUG_ON=y -CONFIG_DEBUG_VM=y -CONFIG_DEBUG_MEMORY_INIT=y -CONFIG_HAVE_DEBUG_STACKOVERFLOW=y -CONFIG_DEBUG_STACKOVERFLOW=y -CONFIG_HAVE_ARCH_KMEMCHECK=y -CONFIG_HAVE_ARCH_KASAN=y -CONFIG_KASAN=y -CONFIG_KASAN_INLINE=y -CONFIG_UBSAN=y -CONFIG_UBSAN_SANITIZE_ALL=y -CONFIG_UBSAN_NO_ALIGNMENT=y -CONFIG_UBSAN_NULL=y -CONFIG_DEBUG_KMEMLEAK=y -CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE=8192 -CONFIG_DEBUG_STACK_USAGE=y -CONFIG_DEBUG_SHIRQ=y -CONFIG_WQ_WATCHDOG=y -CONFIG_SCHED_DEBUG=y -CONFIG_SCHED_INFO=y -CONFIG_SCHEDSTATS=y -CONFIG_SCHED_STACK_END_CHECK=y -CONFIG_DEBUG_TIMEKEEPING=y -CONFIG_TIMER_STATS=y -CONFIG_DEBUG_PREEMPT=y -CONFIG_DEBUG_RT_MUTEXES=y -CONFIG_DEBUG_SPINLOCK=y -CONFIG_DEBUG_MUTEXES=y -CONFIG_DEBUG_LOCK_ALLOC=y -CONFIG_PROVE_LOCKING=y -CONFIG_LOCKDEP=y -CONFIG_DEBUG_ATOMIC_SLEEP=y -CONFIG_TRACE_IRQFLAGS=y -CONFIG_DEBUG_BUGVERBOSE=y -CONFIG_DEBUG_LIST=y -CONFIG_DEBUG_PI_LIST=y -CONFIG_PROVE_RCU=y -CONFIG_SPARSE_RCU_POINTER=y -CONFIG_RCU_CPU_STALL_TIMEOUT=21 -CONFIG_RCU_TRACE=y -CONFIG_RCU_EQS_DEBUG=y -CONFIG_USER_STACKTRACE_SUPPORT=y -CONFIG_DEBUG_SG=y -CONFIG_DEBUG_NOTIFIERS=y -CONFIG_DOUBLEFAULT=y -CONFIG_X86_DEBUG_FPU=y -CONFIG_DEBUG_SECTION_MISMATCH=y -CONFIG_DEBUG_PAGEALLOC=y -CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT=y -CONFIG_DEBUG_WW_MUTEX_SLOWPATH=y diff --git a/tools/testing/selftests/wireguard/qemu/init.c b/tools/testing/selftests/wireguard/qemu/init.c deleted file mode 100644 index c9698120ac9d8..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/init.c +++ /dev/null @@ -1,284 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. - */ - -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -__attribute__((noreturn)) static void poweroff(void) -{ - fflush(stdout); - fflush(stderr); - reboot(RB_AUTOBOOT); - sleep(30); - fprintf(stderr, "\x1b[37m\x1b[41m\x1b[1mFailed to power off!!!\x1b[0m\n"); - exit(1); -} - -static void panic(const char *what) -{ - fprintf(stderr, "\n\n\x1b[37m\x1b[41m\x1b[1mSOMETHING WENT HORRIBLY WRONG\x1b[0m\n\n \x1b[31m\x1b[1m%s: %s\x1b[0m\n\n\x1b[37m\x1b[44m\x1b[1mPower off...\x1b[0m\n\n", what, strerror(errno)); - poweroff(); -} - -#define pretty_message(msg) puts("\x1b[32m\x1b[1m" msg "\x1b[0m") - -static void print_banner(void) -{ - struct utsname utsname; - int len; - - if (uname(&utsname) < 0) - panic("uname"); - - len = strlen(" WireGuard Test Suite on ") + strlen(utsname.sysname) + strlen(utsname.release) + strlen(utsname.machine); - printf("\x1b[45m\x1b[33m\x1b[1m%*.s\x1b[0m\n\x1b[45m\x1b[33m\x1b[1m WireGuard Test Suite on %s %s %s \x1b[0m\n\x1b[45m\x1b[33m\x1b[1m%*.s\x1b[0m\n\n", len, "", utsname.sysname, utsname.release, utsname.machine, len, ""); -} - -static void seed_rng(void) -{ - int fd; - struct { - int entropy_count; - int buffer_size; - unsigned char buffer[256]; - } entropy = { - .entropy_count = sizeof(entropy.buffer) * 8, - .buffer_size = sizeof(entropy.buffer), - .buffer = "Adding real entropy is not actually important for these tests. Don't try this at home, kids!" - }; - - if (mknod("/dev/urandom", S_IFCHR | 0644, makedev(1, 9))) - panic("mknod(/dev/urandom)"); - fd = open("/dev/urandom", O_WRONLY); - if (fd < 0) - panic("open(urandom)"); - for (int i = 0; i < 256; ++i) { - if (ioctl(fd, RNDADDENTROPY, &entropy) < 0) - panic("ioctl(urandom)"); - } - close(fd); -} - -static void mount_filesystems(void) -{ - pretty_message("[+] Mounting filesystems..."); - mkdir("/dev", 0755); - mkdir("/proc", 0755); - mkdir("/sys", 0755); - mkdir("/tmp", 0755); - mkdir("/run", 0755); - mkdir("/var", 0755); - if (mount("none", "/dev", "devtmpfs", 0, NULL)) - panic("devtmpfs mount"); - if (mount("none", "/proc", "proc", 0, NULL)) - panic("procfs mount"); - if (mount("none", "/sys", "sysfs", 0, NULL)) - panic("sysfs mount"); - if (mount("none", "/tmp", "tmpfs", 0, NULL)) - panic("tmpfs mount"); - if (mount("none", "/run", "tmpfs", 0, NULL)) - panic("tmpfs mount"); - if (mount("none", "/sys/kernel/debug", "debugfs", 0, NULL)) - ; /* Not a problem if it fails.*/ - if (symlink("/run", "/var/run")) - panic("run symlink"); - if (symlink("/proc/self/fd", "/dev/fd")) - panic("fd symlink"); -} - -static void enable_logging(void) -{ - int fd; - pretty_message("[+] Enabling logging..."); - fd = open("/proc/sys/kernel/printk", O_WRONLY); - if (fd >= 0) { - if (write(fd, "9\n", 2) != 2) - panic("write(printk)"); - close(fd); - } - fd = open("/proc/sys/debug/exception-trace", O_WRONLY); - if (fd >= 0) { - if (write(fd, "1\n", 2) != 2) - panic("write(exception-trace)"); - close(fd); - } - fd = open("/proc/sys/kernel/panic_on_warn", O_WRONLY); - if (fd >= 0) { - if (write(fd, "1\n", 2) != 2) - panic("write(panic_on_warn)"); - close(fd); - } -} - -static void kmod_selftests(void) -{ - FILE *file; - char line[2048], *start, *pass; - bool success = true; - pretty_message("[+] Module self-tests:"); - file = fopen("/proc/kmsg", "r"); - if (!file) - panic("fopen(kmsg)"); - if (fcntl(fileno(file), F_SETFL, O_NONBLOCK) < 0) - panic("fcntl(kmsg, nonblock)"); - while (fgets(line, sizeof(line), file)) { - start = strstr(line, "wireguard: "); - if (!start) - continue; - start += 11; - *strchrnul(start, '\n') = '\0'; - if (strstr(start, "www.wireguard.com")) - break; - pass = strstr(start, ": pass"); - if (!pass || pass[6] != '\0') { - success = false; - printf(" \x1b[31m* %s\x1b[0m\n", start); - } else - printf(" \x1b[32m* %s\x1b[0m\n", start); - } - fclose(file); - if (!success) { - puts("\x1b[31m\x1b[1m[-] Tests failed! \u2639\x1b[0m"); - poweroff(); - } -} - -static void launch_tests(void) -{ - char cmdline[4096], *success_dev; - int status, fd; - pid_t pid; - - pretty_message("[+] Launching tests..."); - pid = fork(); - if (pid == -1) - panic("fork"); - else if (pid == 0) { - execl("/init.sh", "init", NULL); - panic("exec"); - } - if (waitpid(pid, &status, 0) < 0) - panic("waitpid"); - if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { - pretty_message("[+] Tests successful! :-)"); - fd = open("/proc/cmdline", O_RDONLY); - if (fd < 0) - panic("open(/proc/cmdline)"); - if (read(fd, cmdline, sizeof(cmdline) - 1) <= 0) - panic("read(/proc/cmdline)"); - cmdline[sizeof(cmdline) - 1] = '\0'; - for (success_dev = strtok(cmdline, " \n"); success_dev; success_dev = strtok(NULL, " \n")) { - if (strncmp(success_dev, "wg.success=", 11)) - continue; - memcpy(success_dev + 11 - 5, "/dev/", 5); - success_dev += 11 - 5; - break; - } - if (!success_dev || !strlen(success_dev)) - panic("Unable to find success device"); - - fd = open(success_dev, O_WRONLY); - if (fd < 0) - panic("open(success_dev)"); - if (write(fd, "success\n", 8) != 8) - panic("write(success_dev)"); - close(fd); - } else { - const char *why = "unknown cause"; - int what = -1; - - if (WIFEXITED(status)) { - why = "exit code"; - what = WEXITSTATUS(status); - } else if (WIFSIGNALED(status)) { - why = "signal"; - what = WTERMSIG(status); - } - printf("\x1b[31m\x1b[1m[-] Tests failed with %s %d! \u2639\x1b[0m\n", why, what); - } -} - -static void ensure_console(void) -{ - for (unsigned int i = 0; i < 1000; ++i) { - int fd = open("/dev/console", O_RDWR); - if (fd < 0) { - usleep(50000); - continue; - } - dup2(fd, 0); - dup2(fd, 1); - dup2(fd, 2); - close(fd); - if (write(1, "\0\0\0\0\n", 5) == 5) - return; - } - panic("Unable to open console device"); -} - -static void clear_leaks(void) -{ - int fd; - - fd = open("/sys/kernel/debug/kmemleak", O_WRONLY); - if (fd < 0) - return; - pretty_message("[+] Starting memory leak detection..."); - write(fd, "clear\n", 5); - close(fd); -} - -static void check_leaks(void) -{ - int fd; - - fd = open("/sys/kernel/debug/kmemleak", O_WRONLY); - if (fd < 0) - return; - pretty_message("[+] Scanning for memory leaks..."); - sleep(2); /* Wait for any grace periods. */ - write(fd, "scan\n", 5); - close(fd); - - fd = open("/sys/kernel/debug/kmemleak", O_RDONLY); - if (fd < 0) - return; - if (sendfile(1, fd, NULL, 0x7ffff000) > 0) - panic("Memory leaks encountered"); - close(fd); -} - -int main(int argc, char *argv[]) -{ - seed_rng(); - ensure_console(); - print_banner(); - mount_filesystems(); - kmod_selftests(); - enable_logging(); - clear_leaks(); - launch_tests(); - check_leaks(); - poweroff(); - return 1; -} diff --git a/tools/testing/selftests/wireguard/qemu/kernel.config b/tools/testing/selftests/wireguard/qemu/kernel.config deleted file mode 100644 index af9323a0b6e09..0000000000000 --- a/tools/testing/selftests/wireguard/qemu/kernel.config +++ /dev/null @@ -1,88 +0,0 @@ -CONFIG_LOCALVERSION="" -CONFIG_NET=y -CONFIG_NETDEVICES=y -CONFIG_NET_CORE=y -CONFIG_NET_IPIP=y -CONFIG_DUMMY=y -CONFIG_VETH=y -CONFIG_MULTIUSER=y -CONFIG_NAMESPACES=y -CONFIG_NET_NS=y -CONFIG_UNIX=y -CONFIG_INET=y -CONFIG_IPV6=y -CONFIG_NETFILTER=y -CONFIG_NETFILTER_ADVANCED=y -CONFIG_NF_CONNTRACK=y -CONFIG_NF_NAT=y -CONFIG_NETFILTER_XTABLES=y -CONFIG_NETFILTER_XT_NAT=y -CONFIG_NETFILTER_XT_MATCH_LENGTH=y -CONFIG_NF_CONNTRACK_IPV4=y -CONFIG_NF_NAT_IPV4=y -CONFIG_IP_NF_IPTABLES=y -CONFIG_IP_NF_FILTER=y -CONFIG_IP_NF_NAT=y -CONFIG_IP_ADVANCED_ROUTER=y -CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IPV6_MULTIPLE_TABLES=y -CONFIG_TTY=y -CONFIG_BINFMT_ELF=y -CONFIG_BINFMT_SCRIPT=y -CONFIG_VDSO=y -CONFIG_VIRTUALIZATION=y -CONFIG_HYPERVISOR_GUEST=y -CONFIG_PARAVIRT=y -CONFIG_KVM_GUEST=y -CONFIG_PARAVIRT_SPINLOCKS=y -CONFIG_PRINTK=y -CONFIG_KALLSYMS=y -CONFIG_BUG=y -CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE=y -CONFIG_JUMP_LABEL=y -CONFIG_EMBEDDED=n -CONFIG_BASE_FULL=y -CONFIG_FUTEX=y -CONFIG_SHMEM=y -CONFIG_SLUB=y -CONFIG_SPARSEMEM_VMEMMAP=y -CONFIG_SMP=y -CONFIG_SCHED_SMT=y -CONFIG_SCHED_MC=y -CONFIG_NUMA=y -CONFIG_PREEMPT=y -CONFIG_NO_HZ=y -CONFIG_NO_HZ_IDLE=y -CONFIG_NO_HZ_FULL=n -CONFIG_HZ_PERIODIC=n -CONFIG_HIGH_RES_TIMERS=y -CONFIG_COMPAT_32BIT_TIME=y -CONFIG_ARCH_RANDOM=y -CONFIG_FILE_LOCKING=y -CONFIG_POSIX_TIMERS=y -CONFIG_DEVTMPFS=y -CONFIG_PROC_FS=y -CONFIG_PROC_SYSCTL=y -CONFIG_SYSFS=y -CONFIG_TMPFS=y -CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15 -CONFIG_PRINTK_TIME=y -CONFIG_BLK_DEV_INITRD=y -CONFIG_LEGACY_VSYSCALL_NONE=y -CONFIG_KERNEL_GZIP=y -CONFIG_PANIC_ON_OOPS=y -CONFIG_BUG_ON_DATA_CORRUPTION=y -CONFIG_LOCKUP_DETECTOR=y -CONFIG_SOFTLOCKUP_DETECTOR=y -CONFIG_HARDLOCKUP_DETECTOR=y -CONFIG_WQ_WATCHDOG=y -CONFIG_DETECT_HUNG_TASK=y -CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y -CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y -CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y -CONFIG_PANIC_TIMEOUT=-1 -CONFIG_STACKTRACE=y -CONFIG_EARLY_PRINTK=y -CONFIG_GDB_SCRIPTS=y -CONFIG_WIREGUARD=y -CONFIG_WIREGUARD_DEBUG=y From c5e58d88054178092a1c5fc77f3c2b7c3b614902 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 24 Jan 2020 14:21:14 +0000 Subject: [PATCH 02/34] bpf: update jmp32 test cases to fix range bound deduction [ no upstream commit ] Since commit f2d67fec0b43 ("bpf: Undo incorrect __reg_bound_offset32 handling") has been backported to stable, we also need to update related test cases that started to (expectedly) fail on stable. Given the functionality has been reverted we need to move the result to REJECT. Reported-by: Naresh Kamboju Signed-off-by: Daniel Borkmann Signed-off-by: Greg Kroah-Hartman --- tools/testing/selftests/bpf/verifier/jmp32.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/bpf/verifier/jmp32.c b/tools/testing/selftests/bpf/verifier/jmp32.c index bf0322eb53464..e708a3f313f79 100644 --- a/tools/testing/selftests/bpf/verifier/jmp32.c +++ b/tools/testing/selftests/bpf/verifier/jmp32.c @@ -768,7 +768,8 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .fixup_map_hash_48b = { 4 }, - .result = ACCEPT, + .result = REJECT, + .errstr = "R8 unbounded memory access", .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { @@ -796,7 +797,8 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .fixup_map_hash_48b = { 4 }, - .result = ACCEPT, + .result = REJECT, + .errstr = "R8 unbounded memory access", .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { @@ -824,6 +826,7 @@ }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .fixup_map_hash_48b = { 4 }, - .result = ACCEPT, + .result = REJECT, + .errstr = "R8 unbounded memory access", .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, From 8924ef44fd34f11a6cbd0a686a952e3d6aef51a1 Mon Sep 17 00:00:00 2001 From: Jouni Malinen Date: Thu, 26 Mar 2020 15:51:34 +0100 Subject: [PATCH 03/34] mac80211: Check port authorization in the ieee80211_tx_dequeue() case commit ce2e1ca703071723ca2dd94d492a5ab6d15050da upstream. mac80211 used to check port authorization in the Data frame enqueue case when going through start_xmit(). However, that authorization status may change while the frame is waiting in a queue. Add a similar check in the dequeue case to avoid sending previously accepted frames after authorization change. This provides additional protection against potential leaking of frames after a station has been disconnected and the keys for it are being removed. Cc: stable@vger.kernel.org Signed-off-by: Jouni Malinen Link: https://lore.kernel.org/r/20200326155133.ced84317ea29.I34d4c47cd8cc8a4042b38a76f16a601fbcbfd9b3@changeid Signed-off-by: Johannes Berg Signed-off-by: Greg Kroah-Hartman --- net/mac80211/tx.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index 08ff42c3afd12..264dbe13ad5ba 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -3604,8 +3604,25 @@ struct sk_buff *ieee80211_tx_dequeue(struct ieee80211_hw *hw, tx.skb = skb; tx.sdata = vif_to_sdata(info->control.vif); - if (txq->sta) + if (txq->sta) { tx.sta = container_of(txq->sta, struct sta_info, sta); + /* + * Drop unicast frames to unauthorised stations unless they are + * EAPOL frames from the local station. + */ + if (unlikely(!ieee80211_vif_is_mesh(&tx.sdata->vif) && + tx.sdata->vif.type != NL80211_IFTYPE_OCB && + !is_multicast_ether_addr(hdr->addr1) && + !test_sta_flag(tx.sta, WLAN_STA_AUTHORIZED) && + (!(info->control.flags & + IEEE80211_TX_CTRL_PORT_CTRL_PROTO) || + !ether_addr_equal(tx.sdata->vif.addr, + hdr->addr2)))) { + I802_DEBUG_INC(local->tx_handlers_drop_unauth_port); + ieee80211_free_txskb(&local->hw, skb); + goto begin; + } + } /* * The key can be removed while the packet was queued, so need to call From 8f36778d4870b3b165ecde4987d1a0aa5bd303b6 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Sun, 29 Mar 2020 22:50:06 +0200 Subject: [PATCH 04/34] mac80211: fix authentication with iwlwifi/mvm commit be8c827f50a0bcd56361b31ada11dc0a3c2fd240 upstream. The original patch didn't copy the ieee80211_is_data() condition because on most drivers the management frames don't go through this path. However, they do on iwlwifi/mvm, so we do need to keep the condition here. Cc: stable@vger.kernel.org Fixes: ce2e1ca70307 ("mac80211: Check port authorization in the ieee80211_tx_dequeue() case") Signed-off-by: Johannes Berg Signed-off-by: David S. Miller Cc: Woody Suwalski Signed-off-by: Greg Kroah-Hartman --- net/mac80211/tx.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index 264dbe13ad5ba..23c4443a3921a 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -3610,7 +3610,8 @@ struct sk_buff *ieee80211_tx_dequeue(struct ieee80211_hw *hw, * Drop unicast frames to unauthorised stations unless they are * EAPOL frames from the local station. */ - if (unlikely(!ieee80211_vif_is_mesh(&tx.sdata->vif) && + if (unlikely(ieee80211_is_data(hdr->frame_control) && + !ieee80211_vif_is_mesh(&tx.sdata->vif) && tx.sdata->vif.type != NL80211_IFTYPE_OCB && !is_multicast_ether_addr(hdr->addr1) && !test_sta_flag(tx.sta, WLAN_STA_AUTHORIZED) && From 1430c0f56373641a0fcca1f9be67a82f4d0ec101 Mon Sep 17 00:00:00 2001 From: Lanqing Liu Date: Mon, 16 Mar 2020 11:13:33 +0800 Subject: [PATCH 05/34] serial: sprd: Fix a dereference warning commit efc176929a3505a30c3993ddd393b40893649bd2 upstream. We should validate if the 'sup' is NULL or not before freeing DMA memory, to fix below warning. "drivers/tty/serial/sprd_serial.c:1141 sprd_remove() error: we previously assumed 'sup' could be null (see line 1132)" Fixes: f4487db58eb7 ("serial: sprd: Add DMA mode support") Reported-by: Dan Carpenter Signed-off-by: Lanqing Liu Cc: stable Link: https://lore.kernel.org/r/e2bd92691538e95b04a2c2a728f3292e1617018f.1584325957.git.liuhhome@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/sprd_serial.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/tty/serial/sprd_serial.c b/drivers/tty/serial/sprd_serial.c index f60a59d9bf271..685041d14e012 100644 --- a/drivers/tty/serial/sprd_serial.c +++ b/drivers/tty/serial/sprd_serial.c @@ -1136,14 +1136,13 @@ static int sprd_remove(struct platform_device *dev) if (sup) { uart_remove_one_port(&sprd_uart_driver, &sup->port); sprd_port[sup->port.line] = NULL; + sprd_rx_free_buf(sup); sprd_ports_num--; } if (!sprd_ports_num) uart_unregister_driver(&sprd_uart_driver); - sprd_rx_free_buf(sup); - return 0; } From a0d8c14be7dbf61426fd1adb12a1aa2ea4464d4c Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Wed, 19 Feb 2020 08:39:43 +0100 Subject: [PATCH 06/34] vt: selection, introduce vc_is_sel commit dce05aa6eec977f1472abed95ccd71276b9a3864 upstream. Avoid global variables (namely sel_cons) by introducing vc_is_sel. It checks whether the parameter is the current selection console. This will help putting sel_cons to a struct later. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20200219073951.16151-1-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/selection.c | 5 +++++ drivers/tty/vt/vt.c | 7 ++++--- drivers/tty/vt/vt_ioctl.c | 2 +- include/linux/selection.h | 4 +++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/drivers/tty/vt/selection.c b/drivers/tty/vt/selection.c index d7d2e4b844bcd..7556139cd0da0 100644 --- a/drivers/tty/vt/selection.c +++ b/drivers/tty/vt/selection.c @@ -88,6 +88,11 @@ void clear_selection(void) } EXPORT_SYMBOL_GPL(clear_selection); +bool vc_is_sel(struct vc_data *vc) +{ + return vc == sel_cons; +} + /* * User settable table: what characters are to be considered alphabetic? * 128 bits. Locked by the console lock. diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index e9e27ba69d5d1..7d89e0657ccc3 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -890,8 +890,9 @@ static void hide_softcursor(struct vc_data *vc) static void hide_cursor(struct vc_data *vc) { - if (vc == sel_cons) + if (vc_is_sel(vc)) clear_selection(); + vc->vc_sw->con_cursor(vc, CM_ERASE); hide_softcursor(vc); } @@ -901,7 +902,7 @@ static void set_cursor(struct vc_data *vc) if (!con_is_fg(vc) || console_blanked || vc->vc_mode == KD_GRAPHICS) return; if (vc->vc_deccm) { - if (vc == sel_cons) + if (vc_is_sel(vc)) clear_selection(); add_softcursor(vc); if ((vc->vc_cursor_type & 0x0f) != 1) @@ -1207,7 +1208,7 @@ static int vc_do_resize(struct tty_struct *tty, struct vc_data *vc, } } - if (vc == sel_cons) + if (vc_is_sel(vc)) clear_selection(); old_rows = vc->vc_rows; diff --git a/drivers/tty/vt/vt_ioctl.c b/drivers/tty/vt/vt_ioctl.c index ee6c91ef1f6cf..bf4daa0c79300 100644 --- a/drivers/tty/vt/vt_ioctl.c +++ b/drivers/tty/vt/vt_ioctl.c @@ -43,7 +43,7 @@ char vt_dont_switch; extern struct tty_driver *console_driver; #define VT_IS_IN_USE(i) (console_driver->ttys[i] && console_driver->ttys[i]->count) -#define VT_BUSY(i) (VT_IS_IN_USE(i) || i == fg_console || vc_cons[i].d == sel_cons) +#define VT_BUSY(i) (VT_IS_IN_USE(i) || i == fg_console || vc_is_sel(vc_cons[i].d)) /* * Console (vt and kd) routines, as defined by USL SVR4 manual, and by diff --git a/include/linux/selection.h b/include/linux/selection.h index e2c1f96bf059b..5b890ef5b59fa 100644 --- a/include/linux/selection.h +++ b/include/linux/selection.h @@ -11,8 +11,8 @@ #include #include -extern struct vc_data *sel_cons; struct tty_struct; +struct vc_data; extern void clear_selection(void); extern int set_selection_user(const struct tiocl_selection __user *sel, @@ -24,6 +24,8 @@ extern int sel_loadlut(char __user *p); extern int mouse_reporting(void); extern void mouse_report(struct tty_struct * tty, int butt, int mrx, int mry); +bool vc_is_sel(struct vc_data *vc); + extern int console_blanked; extern const unsigned char color_table[]; From 6f405bc51d381f947d8675105e432574cb5d08c6 Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Wed, 19 Feb 2020 08:39:44 +0100 Subject: [PATCH 07/34] vt: ioctl, switch VT_IS_IN_USE and VT_BUSY to inlines commit e587e8f17433ddb26954f0edf5b2f95c42155ae9 upstream. These two were macros. Switch them to static inlines, so that it's more understandable what they are doing. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20200219073951.16151-2-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt_ioctl.c | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/drivers/tty/vt/vt_ioctl.c b/drivers/tty/vt/vt_ioctl.c index bf4daa0c79300..693d9d7ffb68f 100644 --- a/drivers/tty/vt/vt_ioctl.c +++ b/drivers/tty/vt/vt_ioctl.c @@ -40,10 +40,25 @@ #include char vt_dont_switch; -extern struct tty_driver *console_driver; -#define VT_IS_IN_USE(i) (console_driver->ttys[i] && console_driver->ttys[i]->count) -#define VT_BUSY(i) (VT_IS_IN_USE(i) || i == fg_console || vc_is_sel(vc_cons[i].d)) +static inline bool vt_in_use(unsigned int i) +{ + extern struct tty_driver *console_driver; + + return console_driver->ttys[i] && console_driver->ttys[i]->count; +} + +static inline bool vt_busy(int i) +{ + if (vt_in_use(i)) + return true; + if (i == fg_console) + return true; + if (vc_is_sel(vc_cons[i].d)) + return true; + + return false; +} /* * Console (vt and kd) routines, as defined by USL SVR4 manual, and by @@ -289,7 +304,7 @@ static int vt_disallocate(unsigned int vc_num) int ret = 0; console_lock(); - if (VT_BUSY(vc_num)) + if (vt_busy(vc_num)) ret = -EBUSY; else if (vc_num) vc = vc_deallocate(vc_num); @@ -311,7 +326,7 @@ static void vt_disallocate_all(void) console_lock(); for (i = 1; i < MAX_NR_CONSOLES; i++) - if (!VT_BUSY(i)) + if (!vt_busy(i)) vc[i] = vc_deallocate(i); else vc[i] = NULL; @@ -648,7 +663,7 @@ int vt_ioctl(struct tty_struct *tty, state = 1; /* /dev/tty0 is always open */ for (i = 0, mask = 2; i < MAX_NR_CONSOLES && mask; ++i, mask <<= 1) - if (VT_IS_IN_USE(i)) + if (vt_in_use(i)) state |= mask; ret = put_user(state, &vtstat->v_state); } @@ -661,7 +676,7 @@ int vt_ioctl(struct tty_struct *tty, case VT_OPENQRY: /* FIXME: locking ? - but then this is a stupid API */ for (i = 0; i < MAX_NR_CONSOLES; ++i) - if (! VT_IS_IN_USE(i)) + if (!vt_in_use(i)) break; uival = i < MAX_NR_CONSOLES ? (i+1) : -1; goto setint; From 157e906c45dd5c76c9f33b6e7d84377596551c1b Mon Sep 17 00:00:00 2001 From: Jiri Slaby Date: Wed, 19 Feb 2020 08:39:48 +0100 Subject: [PATCH 08/34] vt: switch vt_dont_switch to bool commit f400991bf872debffb01c46da882dc97d7e3248e upstream. vt_dont_switch is pure boolean, no need for whole char. Signed-off-by: Jiri Slaby Link: https://lore.kernel.org/r/20200219073951.16151-6-jslaby@suse.cz Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt_ioctl.c | 6 +++--- include/linux/vt_kern.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/tty/vt/vt_ioctl.c b/drivers/tty/vt/vt_ioctl.c index 693d9d7ffb68f..38948ac5fc49b 100644 --- a/drivers/tty/vt/vt_ioctl.c +++ b/drivers/tty/vt/vt_ioctl.c @@ -39,7 +39,7 @@ #include #include -char vt_dont_switch; +bool vt_dont_switch; static inline bool vt_in_use(unsigned int i) { @@ -1026,12 +1026,12 @@ int vt_ioctl(struct tty_struct *tty, case VT_LOCKSWITCH: if (!capable(CAP_SYS_TTY_CONFIG)) return -EPERM; - vt_dont_switch = 1; + vt_dont_switch = true; break; case VT_UNLOCKSWITCH: if (!capable(CAP_SYS_TTY_CONFIG)) return -EPERM; - vt_dont_switch = 0; + vt_dont_switch = false; break; case VT_GETHIFONTMASK: ret = put_user(vc->vc_hi_font_mask, diff --git a/include/linux/vt_kern.h b/include/linux/vt_kern.h index 8dc77e40bc038..ded5c48598f36 100644 --- a/include/linux/vt_kern.h +++ b/include/linux/vt_kern.h @@ -135,7 +135,7 @@ extern int do_unbind_con_driver(const struct consw *csw, int first, int last, int deflt); int vty_init(const struct file_operations *console_fops); -extern char vt_dont_switch; +extern bool vt_dont_switch; extern int default_utf8; extern int global_cursor_default; From 4c60e8e88e0db28b9d92d2f01280bf4c2e8b3be4 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 24 Feb 2020 00:03:26 -0800 Subject: [PATCH 09/34] vt: vt_ioctl: remove unnecessary console allocation checks commit 1aa6e058dd6cd04471b1f21298270014daf48ac9 upstream. The vc_cons_allocated() checks in vt_ioctl() and vt_compat_ioctl() are unnecessary because they can only be reached by calling ioctl() on an open tty, which implies the corresponding virtual console is allocated. And even if the virtual console *could* be freed concurrently, then these checks would be broken since they aren't done under console_lock, and the vc_data is dereferenced before them anyway. So, remove these unneeded checks to avoid confusion. Signed-off-by: Eric Biggers Link: https://lore.kernel.org/r/20200224080326.295046-1-ebiggers@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt_ioctl.c | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/drivers/tty/vt/vt_ioctl.c b/drivers/tty/vt/vt_ioctl.c index 38948ac5fc49b..7297997fcf04c 100644 --- a/drivers/tty/vt/vt_ioctl.c +++ b/drivers/tty/vt/vt_ioctl.c @@ -350,22 +350,13 @@ int vt_ioctl(struct tty_struct *tty, { struct vc_data *vc = tty->driver_data; struct console_font_op op; /* used in multiple places here */ - unsigned int console; + unsigned int console = vc->vc_num; unsigned char ucval; unsigned int uival; void __user *up = (void __user *)arg; int i, perm; int ret = 0; - console = vc->vc_num; - - - if (!vc_cons_allocated(console)) { /* impossible? */ - ret = -ENOIOCTLCMD; - goto out; - } - - /* * To have permissions to do most of the vt ioctls, we either have * to be the owner of the tty, or have CAP_SYS_TTY_CONFIG. @@ -1195,14 +1186,9 @@ long vt_compat_ioctl(struct tty_struct *tty, { struct vc_data *vc = tty->driver_data; struct console_font_op op; /* used in multiple places here */ - unsigned int console = vc->vc_num; void __user *up = compat_ptr(arg); int perm; - - if (!vc_cons_allocated(console)) /* impossible? */ - return -ENOIOCTLCMD; - /* * To have permissions to do most of the vt ioctls, we either have * to be the owner of the tty, or have CAP_SYS_TTY_CONFIG. From dafe1c8034fd5c7c3ae930aa76c8bc5333ee1096 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 21 Mar 2020 20:43:04 -0700 Subject: [PATCH 10/34] vt: vt_ioctl: fix VT_DISALLOCATE freeing in-use virtual console commit ca4463bf8438b403596edd0ec961ca0d4fbe0220 upstream. The VT_DISALLOCATE ioctl can free a virtual console while tty_release() is still running, causing a use-after-free in con_shutdown(). This occurs because VT_DISALLOCATE considers a virtual console's 'struct vc_data' to be unused as soon as the corresponding tty's refcount hits 0. But actually it may be still being closed. Fix this by making vc_data be reference-counted via the embedded 'struct tty_port'. A newly allocated virtual console has refcount 1. Opening it for the first time increments the refcount to 2. Closing it for the last time decrements the refcount (in tty_operations::cleanup() so that it happens late enough), as does VT_DISALLOCATE. Reproducer: #include #include #include #include int main() { if (fork()) { for (;;) close(open("/dev/tty5", O_RDWR)); } else { int fd = open("/dev/tty10", O_RDWR); for (;;) ioctl(fd, VT_DISALLOCATE, 5); } } KASAN report: BUG: KASAN: use-after-free in con_shutdown+0x76/0x80 drivers/tty/vt/vt.c:3278 Write of size 8 at addr ffff88806a4ec108 by task syz_vt/129 CPU: 0 PID: 129 Comm: syz_vt Not tainted 5.6.0-rc2 #11 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS ?-20191223_100556-anatol 04/01/2014 Call Trace: [...] con_shutdown+0x76/0x80 drivers/tty/vt/vt.c:3278 release_tty+0xa8/0x410 drivers/tty/tty_io.c:1514 tty_release_struct+0x34/0x50 drivers/tty/tty_io.c:1629 tty_release+0x984/0xed0 drivers/tty/tty_io.c:1789 [...] Allocated by task 129: [...] kzalloc include/linux/slab.h:669 [inline] vc_allocate drivers/tty/vt/vt.c:1085 [inline] vc_allocate+0x1ac/0x680 drivers/tty/vt/vt.c:1066 con_install+0x4d/0x3f0 drivers/tty/vt/vt.c:3229 tty_driver_install_tty drivers/tty/tty_io.c:1228 [inline] tty_init_dev+0x94/0x350 drivers/tty/tty_io.c:1341 tty_open_by_driver drivers/tty/tty_io.c:1987 [inline] tty_open+0x3ca/0xb30 drivers/tty/tty_io.c:2035 [...] Freed by task 130: [...] kfree+0xbf/0x1e0 mm/slab.c:3757 vt_disallocate drivers/tty/vt/vt_ioctl.c:300 [inline] vt_ioctl+0x16dc/0x1e30 drivers/tty/vt/vt_ioctl.c:818 tty_ioctl+0x9db/0x11b0 drivers/tty/tty_io.c:2660 [...] Fixes: 4001d7b7fc27 ("vt: push down the tty lock so we can see what is left to tackle") Cc: # v3.4+ Reported-by: syzbot+522643ab5729b0421998@syzkaller.appspotmail.com Acked-by: Jiri Slaby Signed-off-by: Eric Biggers Link: https://lore.kernel.org/r/20200322034305.210082-2-ebiggers@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt.c | 23 ++++++++++++++++++++++- drivers/tty/vt/vt_ioctl.c | 12 ++++-------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 7d89e0657ccc3..fa9433e6cdc79 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -1075,6 +1075,17 @@ static void visual_deinit(struct vc_data *vc) module_put(vc->vc_sw->owner); } +static void vc_port_destruct(struct tty_port *port) +{ + struct vc_data *vc = container_of(port, struct vc_data, port); + + kfree(vc); +} + +static const struct tty_port_operations vc_port_ops = { + .destruct = vc_port_destruct, +}; + int vc_allocate(unsigned int currcons) /* return 0 on success */ { struct vt_notifier_param param; @@ -1100,6 +1111,7 @@ int vc_allocate(unsigned int currcons) /* return 0 on success */ vc_cons[currcons].d = vc; tty_port_init(&vc->port); + vc->port.ops = &vc_port_ops; INIT_WORK(&vc_cons[currcons].SAK_work, vc_SAK); visual_init(vc, currcons, 1); @@ -3254,6 +3266,7 @@ static int con_install(struct tty_driver *driver, struct tty_struct *tty) tty->driver_data = vc; vc->port.tty = tty; + tty_port_get(&vc->port); if (!tty->winsize.ws_row && !tty->winsize.ws_col) { tty->winsize.ws_row = vc_cons[currcons].d->vc_rows; @@ -3289,6 +3302,13 @@ static void con_shutdown(struct tty_struct *tty) console_unlock(); } +static void con_cleanup(struct tty_struct *tty) +{ + struct vc_data *vc = tty->driver_data; + + tty_port_put(&vc->port); +} + static int default_color = 7; /* white */ static int default_italic_color = 2; // green (ASCII) static int default_underline_color = 3; // cyan (ASCII) @@ -3413,7 +3433,8 @@ static const struct tty_operations con_ops = { .throttle = con_throttle, .unthrottle = con_unthrottle, .resize = vt_resize, - .shutdown = con_shutdown + .shutdown = con_shutdown, + .cleanup = con_cleanup, }; static struct cdev vc0_cdev; diff --git a/drivers/tty/vt/vt_ioctl.c b/drivers/tty/vt/vt_ioctl.c index 7297997fcf04c..f62f498f63c05 100644 --- a/drivers/tty/vt/vt_ioctl.c +++ b/drivers/tty/vt/vt_ioctl.c @@ -310,10 +310,8 @@ static int vt_disallocate(unsigned int vc_num) vc = vc_deallocate(vc_num); console_unlock(); - if (vc && vc_num >= MIN_NR_CONSOLES) { - tty_port_destroy(&vc->port); - kfree(vc); - } + if (vc && vc_num >= MIN_NR_CONSOLES) + tty_port_put(&vc->port); return ret; } @@ -333,10 +331,8 @@ static void vt_disallocate_all(void) console_unlock(); for (i = 1; i < MAX_NR_CONSOLES; i++) { - if (vc[i] && i >= MIN_NR_CONSOLES) { - tty_port_destroy(&vc[i]->port); - kfree(vc[i]); - } + if (vc[i] && i >= MIN_NR_CONSOLES) + tty_port_put(&vc[i]->port); } } From 6799884b7dc5eb5b1bf94a3f24baa45fa8eaf197 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 21 Mar 2020 20:43:05 -0700 Subject: [PATCH 11/34] vt: vt_ioctl: fix use-after-free in vt_in_use() commit 7cf64b18b0b96e751178b8d0505d8466ff5a448f upstream. vt_in_use() dereferences console_driver->ttys[i] without proper locking. This is broken because the tty can be closed and freed concurrently. We could fix this by using 'READ_ONCE(console_driver->ttys[i]) != NULL' and skipping the check of tty_struct::count. But, looking at console_driver->ttys[i] isn't really appropriate anyway because even if it is NULL the tty can still be in the process of being closed. Instead, fix it by making vt_in_use() require console_lock() and check whether the vt is allocated and has port refcount > 1. This works since following the patch "vt: vt_ioctl: fix VT_DISALLOCATE freeing in-use virtual console" the port refcount is incremented while the vt is open. Reproducer (very unreliable, but it worked for me after a few minutes): #include #include int main() { int fd, nproc; struct vt_stat state; char ttyname[16]; fd = open("/dev/tty10", O_RDONLY); for (nproc = 1; nproc < 8; nproc *= 2) fork(); for (;;) { sprintf(ttyname, "/dev/tty%d", rand() % 8); close(open(ttyname, O_RDONLY)); ioctl(fd, VT_GETSTATE, &state); } } KASAN report: BUG: KASAN: use-after-free in vt_in_use drivers/tty/vt/vt_ioctl.c:48 [inline] BUG: KASAN: use-after-free in vt_ioctl+0x1ad3/0x1d70 drivers/tty/vt/vt_ioctl.c:657 Read of size 4 at addr ffff888065722468 by task syz-vt2/132 CPU: 0 PID: 132 Comm: syz-vt2 Not tainted 5.6.0-rc5-00130-g089b6d3654916 #13 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS ?-20191223_100556-anatol 04/01/2014 Call Trace: [...] vt_in_use drivers/tty/vt/vt_ioctl.c:48 [inline] vt_ioctl+0x1ad3/0x1d70 drivers/tty/vt/vt_ioctl.c:657 tty_ioctl+0x9db/0x11b0 drivers/tty/tty_io.c:2660 [...] Allocated by task 136: [...] kzalloc include/linux/slab.h:669 [inline] alloc_tty_struct+0x96/0x8a0 drivers/tty/tty_io.c:2982 tty_init_dev+0x23/0x350 drivers/tty/tty_io.c:1334 tty_open_by_driver drivers/tty/tty_io.c:1987 [inline] tty_open+0x3ca/0xb30 drivers/tty/tty_io.c:2035 [...] Freed by task 41: [...] kfree+0xbf/0x200 mm/slab.c:3757 free_tty_struct+0x8d/0xb0 drivers/tty/tty_io.c:177 release_one_tty+0x22d/0x2f0 drivers/tty/tty_io.c:1468 process_one_work+0x7f1/0x14b0 kernel/workqueue.c:2264 worker_thread+0x8b/0xc80 kernel/workqueue.c:2410 [...] Fixes: 4001d7b7fc27 ("vt: push down the tty lock so we can see what is left to tackle") Cc: # v3.4+ Acked-by: Jiri Slaby Signed-off-by: Eric Biggers Link: https://lore.kernel.org/r/20200322034305.210082-3-ebiggers@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/vt/vt_ioctl.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/tty/vt/vt_ioctl.c b/drivers/tty/vt/vt_ioctl.c index f62f498f63c05..daf61c28ba766 100644 --- a/drivers/tty/vt/vt_ioctl.c +++ b/drivers/tty/vt/vt_ioctl.c @@ -43,9 +43,15 @@ bool vt_dont_switch; static inline bool vt_in_use(unsigned int i) { - extern struct tty_driver *console_driver; + const struct vc_data *vc = vc_cons[i].d; - return console_driver->ttys[i] && console_driver->ttys[i]->count; + /* + * console_lock must be held to prevent the vc from being deallocated + * while we're checking whether it's in-use. + */ + WARN_CONSOLE_UNLOCKED(); + + return vc && kref_read(&vc->port.kref) > 1; } static inline bool vt_busy(int i) @@ -643,15 +649,16 @@ int vt_ioctl(struct tty_struct *tty, struct vt_stat __user *vtstat = up; unsigned short state, mask; - /* Review: FIXME: Console lock ? */ if (put_user(fg_console + 1, &vtstat->v_active)) ret = -EFAULT; else { state = 1; /* /dev/tty0 is always open */ + console_lock(); /* required by vt_in_use() */ for (i = 0, mask = 2; i < MAX_NR_CONSOLES && mask; ++i, mask <<= 1) if (vt_in_use(i)) state |= mask; + console_unlock(); ret = put_user(state, &vtstat->v_state); } break; @@ -661,10 +668,11 @@ int vt_ioctl(struct tty_struct *tty, * Returns the first available (non-opened) console. */ case VT_OPENQRY: - /* FIXME: locking ? - but then this is a stupid API */ + console_lock(); /* required by vt_in_use() */ for (i = 0; i < MAX_NR_CONSOLES; ++i) if (!vt_in_use(i)) break; + console_unlock(); uival = i < MAX_NR_CONSOLES ? (i+1) : -1; goto setint; From dd13b06c4d299a318f235b60140c47d2a874ff62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Georg=20M=C3=BCller?= Date: Mon, 3 Feb 2020 21:11:06 +0100 Subject: [PATCH 12/34] platform/x86: pmc_atom: Add Lex 2I385SW to critclk_systems DMI table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit 95b31e35239e5e1689e3d965d692a313c71bd8ab upstream. The Lex 2I385SW board has two Intel I211 ethernet controllers. Without this patch, only the first port is usable. The second port fails to start with the following message: igb: probe of 0000:02:00.0 failed with error -2 Fixes: 648e921888ad ("clk: x86: Stop marking clocks as CLK_IS_CRITICAL") Tested-by: Georg Müller Signed-off-by: Georg Müller Reviewed-by: Hans de Goede Signed-off-by: Andy Shevchenko Signed-off-by: Greg Kroah-Hartman --- drivers/platform/x86/pmc_atom.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/platform/x86/pmc_atom.c b/drivers/platform/x86/pmc_atom.c index 52ef1419b6715..2b1a3a6ee8dbf 100644 --- a/drivers/platform/x86/pmc_atom.c +++ b/drivers/platform/x86/pmc_atom.c @@ -383,6 +383,14 @@ static const struct dmi_system_id critclk_systems[] = { DMI_MATCH(DMI_PRODUCT_NAME, "3I380D"), }, }, + { + /* pmc_plt_clk* - are used for ethernet controllers */ + .ident = "Lex 2I385SW", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Lex BayTrail"), + DMI_MATCH(DMI_PRODUCT_NAME, "2I385SW"), + }, + }, { /* pmc_plt_clk* - are used for ethernet controllers */ .ident = "Beckhoff CB3163", From 14c36fce7a1756813eb7d87a29d5b7541e5e3e59 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 20 Mar 2020 10:48:13 +0100 Subject: [PATCH 13/34] bpf: Explicitly memset the bpf_attr structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit 8096f229421f7b22433775e928d506f0342e5907 upstream. For the bpf syscall, we are relying on the compiler to properly zero out the bpf_attr union that we copy userspace data into. Unfortunately that doesn't always work properly, padding and other oddities might not be correctly zeroed, and in some tests odd things have been found when the stack is pre-initialized to other values. Fix this by explicitly memsetting the structure to 0 before using it. Reported-by: Maciej Żenczykowski Reported-by: John Stultz Reported-by: Alexander Potapenko Reported-by: Alistair Delva Signed-off-by: Greg Kroah-Hartman Signed-off-by: Daniel Borkmann Acked-by: Yonghong Song Link: https://android-review.googlesource.com/c/kernel/common/+/1235490 Link: https://lore.kernel.org/bpf/20200320094813.GA421650@kroah.com Signed-off-by: Greg Kroah-Hartman --- kernel/bpf/syscall.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index e3461ec595706..8eee65175a781 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2988,7 +2988,7 @@ static int bpf_task_fd_query(const union bpf_attr *attr, SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size) { - union bpf_attr attr = {}; + union bpf_attr attr; int err; if (sysctl_unprivileged_bpf_disabled && !capable(CAP_SYS_ADMIN)) @@ -3000,6 +3000,7 @@ SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, siz size = min_t(u32, size, sizeof(attr)); /* copy attributes from user space, may be less than sizeof(bpf_attr) */ + memset(&attr, 0, sizeof(attr)); if (copy_from_user(&attr, uattr, size) != 0) return -EFAULT; From 25d734923b5ed80a809bc0187041d5b7c0e0d554 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 20 Mar 2020 17:22:58 +0100 Subject: [PATCH 14/34] bpf: Explicitly memset some bpf info structures declared on the stack commit 5c6f25887963f15492b604dd25cb149c501bbabf upstream. Trying to initialize a structure with "= {};" will not always clean out all padding locations in a structure. So be explicit and call memset to initialize everything for a number of bpf information structures that are then copied from userspace, sometimes from smaller memory locations than the size of the structure. Reported-by: Daniel Borkmann Signed-off-by: Greg Kroah-Hartman Signed-off-by: Daniel Borkmann Acked-by: Yonghong Song Link: https://lore.kernel.org/bpf/20200320162258.GA794295@kroah.com Signed-off-by: Greg Kroah-Hartman --- kernel/bpf/btf.c | 3 ++- kernel/bpf/syscall.c | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 9bf372120e127..c13f6aa3e8ef7 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -4218,7 +4218,7 @@ int btf_get_info_by_fd(const struct btf *btf, union bpf_attr __user *uattr) { struct bpf_btf_info __user *uinfo; - struct bpf_btf_info info = {}; + struct bpf_btf_info info; u32 info_copy, btf_copy; void __user *ubtf; u32 uinfo_len; @@ -4227,6 +4227,7 @@ int btf_get_info_by_fd(const struct btf *btf, uinfo_len = attr->info.info_len; info_copy = min_t(u32, uinfo_len, sizeof(info)); + memset(&info, 0, sizeof(info)); if (copy_from_user(&info, uinfo, info_copy)) return -EFAULT; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 8eee65175a781..4a0f8374f2fbb 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2477,7 +2477,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, union bpf_attr __user *uattr) { struct bpf_prog_info __user *uinfo = u64_to_user_ptr(attr->info.info); - struct bpf_prog_info info = {}; + struct bpf_prog_info info; u32 info_len = attr->info.info_len; struct bpf_prog_stats stats; char __user *uinsns; @@ -2489,6 +2489,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, return err; info_len = min_t(u32, sizeof(info), info_len); + memset(&info, 0, sizeof(info)); if (copy_from_user(&info, uinfo, info_len)) return -EFAULT; @@ -2752,7 +2753,7 @@ static int bpf_map_get_info_by_fd(struct bpf_map *map, union bpf_attr __user *uattr) { struct bpf_map_info __user *uinfo = u64_to_user_ptr(attr->info.info); - struct bpf_map_info info = {}; + struct bpf_map_info info; u32 info_len = attr->info.info_len; int err; @@ -2761,6 +2762,7 @@ static int bpf_map_get_info_by_fd(struct bpf_map *map, return err; info_len = min_t(u32, sizeof(info), info_len); + memset(&info, 0, sizeof(info)); info.type = map->map_type; info.id = map->id; info.key_size = map->key_size; From 886a2b173a508b22af5fedb896cf22449153e0c3 Mon Sep 17 00:00:00 2001 From: Golan Ben Ami Date: Wed, 18 Mar 2020 08:12:54 +0200 Subject: [PATCH 15/34] iwlwifi: don't send GEO_TX_POWER_LIMIT if no wgds table commit 0433ae556ec8fb588a0735ddb09d3eb9806df479 upstream. The GEO_TX_POWER_LIMIT command was sent although there is no wgds table, so the fw got wrong SAR values from the driver. Fix this by avoiding sending the command if no wgds tables are available. Signed-off-by: Golan Ben Ami Fixes: 39c1a9728f93 ("iwlwifi: refactor the SAR tables from mvm to acpi") Signed-off-by: Luca Coelho Tested-By: Jonathan McDowell Tested-by: Len Brown Signed-off-by: Kalle Valo Link: https://lore.kernel.org/r/iwlwifi.20200318081237.46db40617cc6.Id5cf852ec8c5dbf20ba86bad7b165a0c828f8b2e@changeid Cc: Felipe Contreras Signed-off-by: Greg Kroah-Hartman --- drivers/net/wireless/intel/iwlwifi/fw/acpi.c | 14 ++++++++------ drivers/net/wireless/intel/iwlwifi/fw/acpi.h | 14 ++++++++------ drivers/net/wireless/intel/iwlwifi/mvm/fw.c | 9 ++++++++- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/drivers/net/wireless/intel/iwlwifi/fw/acpi.c b/drivers/net/wireless/intel/iwlwifi/fw/acpi.c index 48d375a86d862..ba2aff3af0fe9 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/acpi.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/acpi.c @@ -6,7 +6,7 @@ * GPL LICENSE SUMMARY * * Copyright(c) 2017 Intel Deutschland GmbH - * Copyright (C) 2019 Intel Corporation + * Copyright (C) 2019 - 2020 Intel Corporation * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as @@ -27,7 +27,7 @@ * BSD LICENSE * * Copyright(c) 2017 Intel Deutschland GmbH - * Copyright (C) 2019 Intel Corporation + * Copyright (C) 2019 - 2020 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -491,13 +491,13 @@ int iwl_validate_sar_geo_profile(struct iwl_fw_runtime *fwrt, } IWL_EXPORT_SYMBOL(iwl_validate_sar_geo_profile); -void iwl_sar_geo_init(struct iwl_fw_runtime *fwrt, - struct iwl_per_chain_offset_group *table) +int iwl_sar_geo_init(struct iwl_fw_runtime *fwrt, + struct iwl_per_chain_offset_group *table) { int ret, i, j; if (!iwl_sar_geo_support(fwrt)) - return; + return -EOPNOTSUPP; ret = iwl_sar_get_wgds_table(fwrt); if (ret < 0) { @@ -505,7 +505,7 @@ void iwl_sar_geo_init(struct iwl_fw_runtime *fwrt, "Geo SAR BIOS table invalid or unavailable. (%d)\n", ret); /* we don't fail if the table is not available */ - return; + return -ENOENT; } BUILD_BUG_ON(ACPI_NUM_GEO_PROFILES * ACPI_WGDS_NUM_BANDS * @@ -530,5 +530,7 @@ void iwl_sar_geo_init(struct iwl_fw_runtime *fwrt, i, j, value[1], value[2], value[0]); } } + + return 0; } IWL_EXPORT_SYMBOL(iwl_sar_geo_init); diff --git a/drivers/net/wireless/intel/iwlwifi/fw/acpi.h b/drivers/net/wireless/intel/iwlwifi/fw/acpi.h index 4a6e8262974bd..5590e5cc8fbb7 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/acpi.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/acpi.h @@ -6,7 +6,7 @@ * GPL LICENSE SUMMARY * * Copyright(c) 2017 Intel Deutschland GmbH - * Copyright(c) 2018 - 2019 Intel Corporation + * Copyright(c) 2018 - 2020 Intel Corporation * * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as @@ -27,7 +27,7 @@ * BSD LICENSE * * Copyright(c) 2017 Intel Deutschland GmbH - * Copyright(c) 2018 - 2019 Intel Corporation + * Copyright(c) 2018 - 2020 Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -171,8 +171,9 @@ bool iwl_sar_geo_support(struct iwl_fw_runtime *fwrt); int iwl_validate_sar_geo_profile(struct iwl_fw_runtime *fwrt, struct iwl_host_cmd *cmd); -void iwl_sar_geo_init(struct iwl_fw_runtime *fwrt, - struct iwl_per_chain_offset_group *table); +int iwl_sar_geo_init(struct iwl_fw_runtime *fwrt, + struct iwl_per_chain_offset_group *table); + #else /* CONFIG_ACPI */ static inline void *iwl_acpi_get_object(struct device *dev, acpi_string method) @@ -243,9 +244,10 @@ static inline int iwl_validate_sar_geo_profile(struct iwl_fw_runtime *fwrt, return -ENOENT; } -static inline void iwl_sar_geo_init(struct iwl_fw_runtime *fwrt, - struct iwl_per_chain_offset_group *table) +static inline int iwl_sar_geo_init(struct iwl_fw_runtime *fwrt, + struct iwl_per_chain_offset_group *table) { + return -ENOENT; } #endif /* CONFIG_ACPI */ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c index c09624d8d7ee8..81b7da5815eb9 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c @@ -749,10 +749,17 @@ static int iwl_mvm_sar_geo_init(struct iwl_mvm *mvm) u16 cmd_wide_id = WIDE_ID(PHY_OPS_GROUP, GEO_TX_POWER_LIMIT); union geo_tx_power_profiles_cmd cmd; u16 len; + int ret; cmd.geo_cmd.ops = cpu_to_le32(IWL_PER_CHAIN_OFFSET_SET_TABLES); - iwl_sar_geo_init(&mvm->fwrt, cmd.geo_cmd.table); + ret = iwl_sar_geo_init(&mvm->fwrt, cmd.geo_cmd.table); + /* + * It is a valid scenario to not support SAR, or miss wgds table, + * but in that case there is no need to send the command. + */ + if (ret) + return 0; cmd.geo_cmd.table_revision = cpu_to_le32(mvm->fwrt.geo_rev); From 65c5f32156b835975ad445c35a99c183edf85a06 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Mon, 2 Mar 2020 12:12:25 +0100 Subject: [PATCH 16/34] gpiolib: acpi: Add quirk to ignore EC wakeups on HP x2 10 CHT + AXP288 model commit 0c625ccfe6f754d0896b8881f5c85bcb81699f1f upstream. There are at least 3 models of the HP x2 10 models: Bay Trail SoC + AXP288 PMIC Cherry Trail SoC + AXP288 PMIC Cherry Trail SoC + TI PMIC Like on the other HP x2 10 models we need to ignore wakeup for ACPI GPIO events on the external embedded-controller pin to avoid spurious wakeups on the HP x2 10 CHT + AXP288 model too. This commit adds an extra DMI based quirk for the HP x2 10 CHT + AXP288 model, ignoring wakeups for ACPI GPIO events on the EC interrupt pin on this model. This fixes spurious wakeups from suspend on this model. Fixes: aa23ca3d98f7 ("gpiolib: acpi: Add honor_wakeup module-option + quirk mechanism") Reported-and-tested-by: Marc Lehmann Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20200302111225.6641-4-hdegoede@redhat.com Acked-by: Mika Westerberg Signed-off-by: Linus Walleij Signed-off-by: Greg Kroah-Hartman --- drivers/gpio/gpiolib-acpi.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index d1ef060a5873a..0017367e94eea 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -1437,6 +1437,21 @@ static const struct dmi_system_id gpiolib_acpi_quirks[] = { .ignore_wake = "INT33FC:02@28", }, }, + { + /* + * HP X2 10 models with Cherry Trail SoC + AXP288 PMIC use an + * external embedded-controller connected via I2C + an ACPI GPIO + * event handler on INT33FF:01 pin 0, causing spurious wakeups. + */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "HP"), + DMI_MATCH(DMI_PRODUCT_NAME, "HP Pavilion x2 Detachable"), + DMI_MATCH(DMI_BOARD_NAME, "813E"), + }, + .driver_data = &(struct acpi_gpiolib_dmi_quirk) { + .ignore_wake = "INT33FF:01@0", + }, + }, {} /* Terminating entry */ }; From 2d0b3a5551696835d2715793b52d1c4e3bafd8cc Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Wed, 25 Mar 2020 15:25:47 +0100 Subject: [PATCH 17/34] net: ks8851-ml: Fix IO operations, again commit 8262e6f9b1034ede34548a04dec4c302d92c9497 upstream. This patch reverts 58292104832f ("net: ks8851-ml: Fix 16-bit IO operation") and edacb098ea9c ("net: ks8851-ml: Fix 16-bit data access"), because it turns out these were only necessary due to buggy hardware. This patch adds a check for such a buggy hardware to prevent any such mistakes again. While working further on the KS8851 driver, it came to light that the KS8851-16MLL is capable of switching bus endianness by a hardware strap, EESK pin. If this strap is incorrect, the IO accesses require such endian swapping as is being reverted by this patch. Such swapping also impacts the performance significantly. Hence, in addition to removing it, detect that the hardware is broken, report to user, and fail to bind with such hardware. Fixes: 58292104832f ("net: ks8851-ml: Fix 16-bit IO operation") Fixes: edacb098ea9c ("net: ks8851-ml: Fix 16-bit data access") Signed-off-by: Marek Vasut Cc: David S. Miller Cc: Lukas Wunner Cc: Petr Stetiar Cc: YueHaibing Signed-off-by: David S. Miller Signed-off-by: Greg Kroah-Hartman --- drivers/net/ethernet/micrel/ks8851_mll.c | 56 ++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/micrel/ks8851_mll.c b/drivers/net/ethernet/micrel/ks8851_mll.c index 58579baf3f7a0..45cc840d8e2e1 100644 --- a/drivers/net/ethernet/micrel/ks8851_mll.c +++ b/drivers/net/ethernet/micrel/ks8851_mll.c @@ -156,6 +156,50 @@ static int msg_enable; * chip is busy transferring packet data (RX/TX FIFO accesses). */ +/** + * ks_check_endian - Check whether endianness of the bus is correct + * @ks : The chip information + * + * The KS8851-16MLL EESK pin allows selecting the endianness of the 16bit + * bus. To maintain optimum performance, the bus endianness should be set + * such that it matches the endianness of the CPU. + */ + +static int ks_check_endian(struct ks_net *ks) +{ + u16 cider; + + /* + * Read CIDER register first, however read it the "wrong" way around. + * If the endian strap on the KS8851-16MLL in incorrect and the chip + * is operating in different endianness than the CPU, then the meaning + * of BE[3:0] byte-enable bits is also swapped such that: + * BE[3,2,1,0] becomes BE[1,0,3,2] + * + * Luckily for us, the byte-enable bits are the top four MSbits of + * the address register and the CIDER register is at offset 0xc0. + * Hence, by reading address 0xc0c0, which is not impacted by endian + * swapping, we assert either BE[3:2] or BE[1:0] while reading the + * CIDER register. + * + * If the bus configuration is correct, reading 0xc0c0 asserts + * BE[3:2] and this read returns 0x0000, because to read register + * with bottom two LSbits of address set to 0, BE[1:0] must be + * asserted. + * + * If the bus configuration is NOT correct, reading 0xc0c0 asserts + * BE[1:0] and this read returns non-zero 0x8872 value. + */ + iowrite16(BE3 | BE2 | KS_CIDER, ks->hw_addr_cmd); + cider = ioread16(ks->hw_addr); + if (!cider) + return 0; + + netdev_err(ks->netdev, "incorrect EESK endian strap setting\n"); + + return -EINVAL; +} + /** * ks_rdreg16 - read 16 bit register from device * @ks : The chip information @@ -166,7 +210,7 @@ static int msg_enable; static u16 ks_rdreg16(struct ks_net *ks, int offset) { - ks->cmd_reg_cache = (u16)offset | ((BE3 | BE2) >> (offset & 0x02)); + ks->cmd_reg_cache = (u16)offset | ((BE1 | BE0) << (offset & 0x02)); iowrite16(ks->cmd_reg_cache, ks->hw_addr_cmd); return ioread16(ks->hw_addr); } @@ -181,7 +225,7 @@ static u16 ks_rdreg16(struct ks_net *ks, int offset) static void ks_wrreg16(struct ks_net *ks, int offset, u16 value) { - ks->cmd_reg_cache = (u16)offset | ((BE3 | BE2) >> (offset & 0x02)); + ks->cmd_reg_cache = (u16)offset | ((BE1 | BE0) << (offset & 0x02)); iowrite16(ks->cmd_reg_cache, ks->hw_addr_cmd); iowrite16(value, ks->hw_addr); } @@ -197,7 +241,7 @@ static inline void ks_inblk(struct ks_net *ks, u16 *wptr, u32 len) { len >>= 1; while (len--) - *wptr++ = be16_to_cpu(ioread16(ks->hw_addr)); + *wptr++ = (u16)ioread16(ks->hw_addr); } /** @@ -211,7 +255,7 @@ static inline void ks_outblk(struct ks_net *ks, u16 *wptr, u32 len) { len >>= 1; while (len--) - iowrite16(cpu_to_be16(*wptr++), ks->hw_addr); + iowrite16(*wptr++, ks->hw_addr); } static void ks_disable_int(struct ks_net *ks) @@ -1218,6 +1262,10 @@ static int ks8851_probe(struct platform_device *pdev) goto err_free; } + err = ks_check_endian(ks); + if (err) + goto err_free; + netdev->irq = platform_get_irq(pdev, 0); if ((int)netdev->irq < 0) { From d3ef4fe494b0524e81cd37a142dd87220080a772 Mon Sep 17 00:00:00 2001 From: Leonard Crestez Date: Thu, 20 Feb 2020 18:29:32 +0200 Subject: [PATCH 18/34] clk: imx: Align imx sc clock msg structs to 4 commit a0ae04a25650fd51b7106e742d27333e502173c6 upstream. The imx SC api strongly assumes that messages are composed out of 4-bytes words but some of our message structs have odd sizeofs. This produces many oopses with CONFIG_KASAN=y. Fix by marking with __aligned(4). Fixes: fe37b4820417 ("clk: imx: add scu clock common part") Signed-off-by: Leonard Crestez Link: https://lkml.kernel.org/r/10e97a04980d933b2cfecb6b124bf9046b6e4f16.1582216144.git.leonard.crestez@nxp.com Signed-off-by: Stephen Boyd Signed-off-by: Greg Kroah-Hartman --- drivers/clk/imx/clk-scu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/clk/imx/clk-scu.c b/drivers/clk/imx/clk-scu.c index fbef740704d0c..3c5c42d8833ee 100644 --- a/drivers/clk/imx/clk-scu.c +++ b/drivers/clk/imx/clk-scu.c @@ -43,12 +43,12 @@ struct imx_sc_msg_req_set_clock_rate { __le32 rate; __le16 resource; u8 clk; -} __packed; +} __packed __aligned(4); struct req_get_clock_rate { __le16 resource; u8 clk; -} __packed; +} __packed __aligned(4); struct resp_get_clock_rate { __le32 rate; @@ -121,7 +121,7 @@ struct imx_sc_msg_req_clock_enable { u8 clk; u8 enable; u8 autog; -} __packed; +} __packed __aligned(4); static inline struct clk_scu *to_clk_scu(struct clk_hw *hw) { From ee01d005d0403b4cddcc06d9af1cae68c47daea2 Mon Sep 17 00:00:00 2001 From: Leonard Crestez Date: Thu, 20 Feb 2020 18:29:33 +0200 Subject: [PATCH 19/34] clk: imx: Align imx sc clock parent msg structs to 4 commit 8400ab8896324641243b57fc49b448023c07409a upstream. The imx SC api strongly assumes that messages are composed out of 4-bytes words but some of our message structs have odd sizeofs. This produces many oopses with CONFIG_KASAN=y. Fix by marking with __aligned(4). Fixes: 666aed2d13ee ("clk: imx: scu: add set parent support") Signed-off-by: Leonard Crestez Link: https://lkml.kernel.org/r/aad021e432b3062c142973d09b766656eec18fde.1582216144.git.leonard.crestez@nxp.com Signed-off-by: Stephen Boyd Signed-off-by: Greg Kroah-Hartman --- drivers/clk/imx/clk-scu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clk/imx/clk-scu.c b/drivers/clk/imx/clk-scu.c index 3c5c42d8833ee..b8b2072742a56 100644 --- a/drivers/clk/imx/clk-scu.c +++ b/drivers/clk/imx/clk-scu.c @@ -84,7 +84,7 @@ struct imx_sc_msg_get_clock_parent { struct req_get_clock_parent { __le16 resource; u8 clk; - } __packed req; + } __packed __aligned(4) req; struct resp_get_clock_parent { u8 parent; } resp; From 3dea08e7ac2954baa42f86af3ce81dfb0dd7a4cb Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Fri, 21 Feb 2020 09:10:30 -0800 Subject: [PATCH 20/34] clk: ti: am43xx: Fix clock parent for RTC clock commit 5f3d9b07b9bb4679922f0b2e2baa770e74a6bbd3 upstream. Currently enabling clkctrl clock on am4 can fail for RTC as the clock parent is wrong for RTC. Fixes: 76a1049b84dd ("clk: ti: am43xx: add new clkctrl data for am43xx") Signed-off-by: Tony Lindgren Link: https://lkml.kernel.org/r/20200221171030.39326-1-tony@atomide.com Acked-by: Tero Kristo Signed-off-by: Stephen Boyd Signed-off-by: Greg Kroah-Hartman --- drivers/clk/ti/clk-43xx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clk/ti/clk-43xx.c b/drivers/clk/ti/clk-43xx.c index af3e7805769e2..e5538d577ce57 100644 --- a/drivers/clk/ti/clk-43xx.c +++ b/drivers/clk/ti/clk-43xx.c @@ -78,7 +78,7 @@ static const struct omap_clkctrl_reg_data am4_gfx_l3_clkctrl_regs[] __initconst }; static const struct omap_clkctrl_reg_data am4_l4_rtc_clkctrl_regs[] __initconst = { - { AM4_L4_RTC_RTC_CLKCTRL, NULL, CLKF_SW_SUP, "clk_32768_ck" }, + { AM4_L4_RTC_RTC_CLKCTRL, NULL, CLKF_SW_SUP, "clkdiv32k_ick" }, { 0 }, }; From cf1d99484480a5a17c0dcf206e45402abe3476cc Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Tue, 10 Mar 2020 16:19:01 +0100 Subject: [PATCH 21/34] libceph: fix alloc_msg_with_page_vector() memory leaks commit e886274031200bb60965c1b9c49b7acda56a93bd upstream. Make it so that CEPH_MSG_DATA_PAGES data item can own pages, fixing a bunch of memory leaks for a page vector allocated in alloc_msg_with_page_vector(). Currently, only watch-notify messages trigger this allocation, and normally the page vector is freed either in handle_watch_notify() or by the caller of ceph_osdc_notify(). But if the message is freed before that (e.g. if the session faults while reading in the message or if the notify is stale), we leak the page vector. This was supposed to be fixed by switching to a message-owned pagelist, but that never happened. Fixes: 1907920324f1 ("libceph: support for sending notifies") Reported-by: Roman Penyaev Signed-off-by: Ilya Dryomov Reviewed-by: Roman Penyaev Signed-off-by: Greg Kroah-Hartman --- include/linux/ceph/messenger.h | 7 ++++--- net/ceph/messenger.c | 9 +++++++-- net/ceph/osd_client.c | 14 +++----------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/include/linux/ceph/messenger.h b/include/linux/ceph/messenger.h index c4458dc6a7576..76371aaae2d1c 100644 --- a/include/linux/ceph/messenger.h +++ b/include/linux/ceph/messenger.h @@ -175,9 +175,10 @@ struct ceph_msg_data { #endif /* CONFIG_BLOCK */ struct ceph_bvec_iter bvec_pos; struct { - struct page **pages; /* NOT OWNER. */ + struct page **pages; size_t length; /* total # bytes */ unsigned int alignment; /* first page */ + bool own_pages; }; struct ceph_pagelist *pagelist; }; @@ -356,8 +357,8 @@ extern void ceph_con_keepalive(struct ceph_connection *con); extern bool ceph_con_keepalive_expired(struct ceph_connection *con, unsigned long interval); -extern void ceph_msg_data_add_pages(struct ceph_msg *msg, struct page **pages, - size_t length, size_t alignment); +void ceph_msg_data_add_pages(struct ceph_msg *msg, struct page **pages, + size_t length, size_t alignment, bool own_pages); extern void ceph_msg_data_add_pagelist(struct ceph_msg *msg, struct ceph_pagelist *pagelist); #ifdef CONFIG_BLOCK diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 5b4bd8261002e..f8ca5edc5f2c9 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -3248,12 +3248,16 @@ static struct ceph_msg_data *ceph_msg_data_add(struct ceph_msg *msg) static void ceph_msg_data_destroy(struct ceph_msg_data *data) { - if (data->type == CEPH_MSG_DATA_PAGELIST) + if (data->type == CEPH_MSG_DATA_PAGES && data->own_pages) { + int num_pages = calc_pages_for(data->alignment, data->length); + ceph_release_page_vector(data->pages, num_pages); + } else if (data->type == CEPH_MSG_DATA_PAGELIST) { ceph_pagelist_release(data->pagelist); + } } void ceph_msg_data_add_pages(struct ceph_msg *msg, struct page **pages, - size_t length, size_t alignment) + size_t length, size_t alignment, bool own_pages) { struct ceph_msg_data *data; @@ -3265,6 +3269,7 @@ void ceph_msg_data_add_pages(struct ceph_msg *msg, struct page **pages, data->pages = pages; data->length = length; data->alignment = alignment & ~PAGE_MASK; + data->own_pages = own_pages; msg->data_length += length; } diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index ba45b074a3628..2352afa62d1fa 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -962,7 +962,7 @@ static void ceph_osdc_msg_data_add(struct ceph_msg *msg, BUG_ON(length > (u64) SIZE_MAX); if (length) ceph_msg_data_add_pages(msg, osd_data->pages, - length, osd_data->alignment); + length, osd_data->alignment, false); } else if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGELIST) { BUG_ON(!length); ceph_msg_data_add_pagelist(msg, osd_data->pagelist); @@ -4436,9 +4436,7 @@ static void handle_watch_notify(struct ceph_osd_client *osdc, CEPH_MSG_DATA_PAGES); *lreq->preply_pages = data->pages; *lreq->preply_len = data->length; - } else { - ceph_release_page_vector(data->pages, - calc_pages_for(0, data->length)); + data->own_pages = false; } } lreq->notify_finish_error = return_code; @@ -5500,9 +5498,6 @@ static struct ceph_msg *get_reply(struct ceph_connection *con, return m; } -/* - * TODO: switch to a msg-owned pagelist - */ static struct ceph_msg *alloc_msg_with_page_vector(struct ceph_msg_header *hdr) { struct ceph_msg *m; @@ -5516,7 +5511,6 @@ static struct ceph_msg *alloc_msg_with_page_vector(struct ceph_msg_header *hdr) if (data_len) { struct page **pages; - struct ceph_osd_data osd_data; pages = ceph_alloc_page_vector(calc_pages_for(0, data_len), GFP_NOIO); @@ -5525,9 +5519,7 @@ static struct ceph_msg *alloc_msg_with_page_vector(struct ceph_msg_header *hdr) return NULL; } - ceph_osd_data_pages_init(&osd_data, pages, data_len, 0, false, - false); - ceph_osdc_msg_data_add(m, &osd_data); + ceph_msg_data_add_pages(m, pages, data_len, 0, true); } return m; From 4fba4e808c0b357fd011fa0e84a81bf9af4b97cf Mon Sep 17 00:00:00 2001 From: Ilie Halip Date: Thu, 19 Mar 2020 23:45:28 +0200 Subject: [PATCH 22/34] arm64: alternative: fix build with clang integrated assembler commit 6f5459da2b8736720afdbd67c4bd2d1edba7d0e3 upstream. Building an arm64 defconfig with clang's integrated assembler, this error occurs: :2:2: error: unrecognized instruction mnemonic _ASM_EXTABLE 9999b, 9f ^ arch/arm64/mm/cache.S:50:1: note: while in macro instantiation user_alt 9f, "dc cvau, x4", "dc civac, x4", 0 ^ While GNU as seems fine with case-sensitive macro instantiations, clang doesn't, so use the actual macro name (_asm_extable) as in the rest of the file. Also checked that the generated assembly matches the GCC output. Reviewed-by: Nick Desaulniers Tested-by: Nick Desaulniers Fixes: 290622efc76e ("arm64: fix "dc cvau" cache operation on errata-affected core") Link: https://github.com/ClangBuiltLinux/linux/issues/924 Signed-off-by: Ilie Halip Signed-off-by: Will Deacon Signed-off-by: Greg Kroah-Hartman --- arch/arm64/include/asm/alternative.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h index 324e7d5ab37ed..5e5dc05d63a06 100644 --- a/arch/arm64/include/asm/alternative.h +++ b/arch/arm64/include/asm/alternative.h @@ -221,7 +221,7 @@ alternative_endif .macro user_alt, label, oldinstr, newinstr, cond 9999: alternative_insn "\oldinstr", "\newinstr", \cond - _ASM_EXTABLE 9999b, \label + _asm_extable 9999b, \label .endm /* From 0a431ada2c91a538b7e8c62a24f381dfcb1a5e97 Mon Sep 17 00:00:00 2001 From: disconnect3d Date: Mon, 9 Mar 2020 11:48:53 +0100 Subject: [PATCH 23/34] perf map: Fix off by one in strncpy() size argument commit db2c549407d4a76563c579e4768f7d6d32afefba upstream. This patch fixes an off-by-one error in strncpy size argument in tools/perf/util/map.c. The issue is that in: strncmp(filename, "/system/lib/", 11) the passed string literal: "/system/lib/" has 12 bytes (without the NULL byte) and the passed size argument is 11. As a result, the logic won't match the ending "/" byte and will pass filepaths that are stored in other directories e.g. "/system/libmalicious/bin" or just "/system/libmalicious". This functionality seems to be present only on Android. I assume the /system/ directory is only writable by the root user, so I don't think this bug has much (or any) security impact. Fixes: eca818369996 ("perf tools: Add automatic remapping of Android libraries") Signed-off-by: disconnect3d Cc: Alexander Shishkin Cc: Changbin Du Cc: Jiri Olsa Cc: John Keeping Cc: Mark Rutland Cc: Michael Lentine Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Song Liu Cc: Stephane Eranian Link: http://lore.kernel.org/lkml/20200309104855.3775-1-dominik.b.czarnota@gmail.com Signed-off-by: Arnaldo Carvalho de Melo Signed-off-by: Greg Kroah-Hartman --- tools/perf/util/map.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/util/map.c b/tools/perf/util/map.c index f67960bedebba..0c2434ae60d97 100644 --- a/tools/perf/util/map.c +++ b/tools/perf/util/map.c @@ -89,7 +89,7 @@ static inline bool replace_android_lib(const char *filename, char *newfilename) return true; } - if (!strncmp(filename, "/system/lib/", 11)) { + if (!strncmp(filename, "/system/lib/", 12)) { char *ndk, *app; const char *arch; size_t ndk_length; From 6327cc57f2dbda4c9dc78ba30867db71266fd59b Mon Sep 17 00:00:00 2001 From: Sungbo Eo Date: Sat, 21 Mar 2020 23:36:53 +0900 Subject: [PATCH 24/34] ARM: dts: oxnas: Fix clear-mask property commit deeabb4c1341a12bf8b599e6a2f4cfa4fd74738c upstream. Disable all rps-irq interrupts during driver initialization to prevent an accidental interrupt on GIC. Fixes: 84316f4ef141 ("ARM: boot: dts: Add Oxford Semiconductor OX810SE dtsi") Fixes: 38d4a53733f5 ("ARM: dts: Add support for OX820 and Pogoplug V3") Signed-off-by: Sungbo Eo Acked-by: Neil Armstrong Signed-off-by: Neil Armstrong Signed-off-by: Greg Kroah-Hartman --- arch/arm/boot/dts/ox810se.dtsi | 4 ++-- arch/arm/boot/dts/ox820.dtsi | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/arm/boot/dts/ox810se.dtsi b/arch/arm/boot/dts/ox810se.dtsi index 9f6c2b660ed39..0755e5864c4a3 100644 --- a/arch/arm/boot/dts/ox810se.dtsi +++ b/arch/arm/boot/dts/ox810se.dtsi @@ -323,8 +323,8 @@ interrupt-controller; reg = <0 0x200>; #interrupt-cells = <1>; - valid-mask = <0xFFFFFFFF>; - clear-mask = <0>; + valid-mask = <0xffffffff>; + clear-mask = <0xffffffff>; }; timer0: timer@200 { diff --git a/arch/arm/boot/dts/ox820.dtsi b/arch/arm/boot/dts/ox820.dtsi index c9b3277320639..90846a7655b49 100644 --- a/arch/arm/boot/dts/ox820.dtsi +++ b/arch/arm/boot/dts/ox820.dtsi @@ -240,8 +240,8 @@ reg = <0 0x200>; interrupts = ; #interrupt-cells = <1>; - valid-mask = <0xFFFFFFFF>; - clear-mask = <0>; + valid-mask = <0xffffffff>; + clear-mask = <0xffffffff>; }; timer0: timer@200 { From 0ec1c19e4314fe5e69d7bb73d068e34d7b943d36 Mon Sep 17 00:00:00 2001 From: Nicolas Saenz Julienne Date: Thu, 19 Mar 2020 20:00:13 +0100 Subject: [PATCH 25/34] ARM: dts: bcm283x: Fix vc4's firmware bus DMA limitations commit 55c7c0621078bd73e9d4d2a11eb36e61bc6fe998 upstream. The bus is virtual and devices have to inherit their DMA constraints from the underlying interconnect. So add an empty dma-ranges property to the bus node, implying the firmware bus' DMA constraints are identical to its parent's. Fixes: 7dbe8c62ceeb ("ARM: dts: Add minimal Raspberry Pi 4 support") Signed-off-by: Nicolas Saenz Julienne Signed-off-by: Florian Fainelli Signed-off-by: Greg Kroah-Hartman --- arch/arm/boot/dts/bcm2835-rpi.dtsi | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm/boot/dts/bcm2835-rpi.dtsi b/arch/arm/boot/dts/bcm2835-rpi.dtsi index 394c8a71b13be..fd2c766e0f710 100644 --- a/arch/arm/boot/dts/bcm2835-rpi.dtsi +++ b/arch/arm/boot/dts/bcm2835-rpi.dtsi @@ -15,6 +15,7 @@ firmware: firmware { compatible = "raspberrypi,bcm2835-firmware", "simple-bus"; mboxes = <&mailbox>; + dma-ranges; }; power: power { From 7e89f7695765bfe33cf378a686ab222395efc771 Mon Sep 17 00:00:00 2001 From: Nick Hudson Date: Thu, 12 Mar 2020 09:03:45 +0000 Subject: [PATCH 26/34] ARM: bcm2835-rpi-zero-w: Add missing pinctrl name commit 6687c201fdc3139315c2ea7ef96c157672805cdc upstream. Define the sdhci pinctrl state as "default" so it gets applied correctly and to match all other RPis. Fixes: 2c7c040c73e9 ("ARM: dts: bcm2835: Add Raspberry Pi Zero W") Signed-off-by: Nick Hudson Signed-off-by: Florian Fainelli Signed-off-by: Greg Kroah-Hartman --- arch/arm/boot/dts/bcm2835-rpi-zero-w.dts | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm/boot/dts/bcm2835-rpi-zero-w.dts b/arch/arm/boot/dts/bcm2835-rpi-zero-w.dts index b75af21069f95..4c3f606e5b8d8 100644 --- a/arch/arm/boot/dts/bcm2835-rpi-zero-w.dts +++ b/arch/arm/boot/dts/bcm2835-rpi-zero-w.dts @@ -112,6 +112,7 @@ &sdhci { #address-cells = <1>; #size-cells = <0>; + pinctrl-names = "default"; pinctrl-0 = <&emmc_gpio34 &gpclk2_gpio43>; bus-width = <4>; mmc-pwrseq = <&wifi_pwrseq>; From c45c4cb4b5440a72c9cbadeab5cd0d3b0c7867eb Mon Sep 17 00:00:00 2001 From: Marco Felsch Date: Thu, 27 Feb 2020 12:02:46 +0100 Subject: [PATCH 27/34] ARM: dts: imx6: phycore-som: fix arm and soc minimum voltage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commit 636b45b8efa91db05553840b6c0120d6fa6b94fa upstream. The current set minimum voltage of 730000µV seems to be wrong. I don't know the document which specifies that but the imx6qdl datasheets says that the minimum voltage should be 0.925V for VDD_ARM (LDO bypassed, lowest opp) and 1.15V for VDD_SOC (LDO bypassed, lowest opp). Fixes: ddec5d1c0047 ("ARM: dts: imx6: Add initial support for phyCORE-i.MX 6 SOM") Signed-off-by: Marco Felsch Signed-off-by: Shawn Guo Signed-off-by: Greg Kroah-Hartman --- arch/arm/boot/dts/imx6qdl-phytec-phycore-som.dtsi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm/boot/dts/imx6qdl-phytec-phycore-som.dtsi b/arch/arm/boot/dts/imx6qdl-phytec-phycore-som.dtsi index 881cea0b61ba2..31fa37d2fe471 100644 --- a/arch/arm/boot/dts/imx6qdl-phytec-phycore-som.dtsi +++ b/arch/arm/boot/dts/imx6qdl-phytec-phycore-som.dtsi @@ -107,14 +107,14 @@ regulators { vdd_arm: buck1 { regulator-name = "vdd_arm"; - regulator-min-microvolt = <730000>; + regulator-min-microvolt = <925000>; regulator-max-microvolt = <1380000>; regulator-always-on; }; vdd_soc: buck2 { regulator-name = "vdd_soc"; - regulator-min-microvolt = <730000>; + regulator-min-microvolt = <1150000>; regulator-max-microvolt = <1380000>; regulator-always-on; }; From fecf8f730d6155994e8c1d681708cb24df6b8325 Mon Sep 17 00:00:00 2001 From: Arthur Demchenkov Date: Sun, 8 Mar 2020 22:19:33 +0300 Subject: [PATCH 28/34] ARM: dts: N900: fix onenand timings commit 0c5220a3c1242c7a2451570ed5f5af69620aac75 upstream. Commit a758f50f10cf ("mtd: onenand: omap2: Configure driver from DT") started using DT specified timings for GPMC, and as a result the OneNAND stopped working on N900 as we had wrong values in the DT. Fix by updating the values to bootloader timings that have been tested to be working on Nokia N900 with OneNAND manufacturers: Samsung, Numonyx. Fixes: a758f50f10cf ("mtd: onenand: omap2: Configure driver from DT") Signed-off-by: Arthur Demchenkov Tested-by: Merlijn Wajer Reviewed-by: Roger Quadros Signed-off-by: Tony Lindgren Signed-off-by: Greg Kroah-Hartman --- arch/arm/boot/dts/omap3-n900.dts | 44 ++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/arch/arm/boot/dts/omap3-n900.dts b/arch/arm/boot/dts/omap3-n900.dts index a638e059135bc..b63e7d2cbeb46 100644 --- a/arch/arm/boot/dts/omap3-n900.dts +++ b/arch/arm/boot/dts/omap3-n900.dts @@ -849,34 +849,46 @@ compatible = "ti,omap2-onenand"; reg = <0 0 0x20000>; /* CS0, offset 0, IO size 128K */ + /* + * These timings are based on CONFIG_OMAP_GPMC_DEBUG=y reported + * bootloader set values when booted with v5.1 + * (OneNAND Manufacturer: Samsung): + * + * cs0 GPMC_CS_CONFIG1: 0xfb001202 + * cs0 GPMC_CS_CONFIG2: 0x00111100 + * cs0 GPMC_CS_CONFIG3: 0x00020200 + * cs0 GPMC_CS_CONFIG4: 0x11001102 + * cs0 GPMC_CS_CONFIG5: 0x03101616 + * cs0 GPMC_CS_CONFIG6: 0x90060000 + */ gpmc,sync-read; gpmc,sync-write; gpmc,burst-length = <16>; gpmc,burst-read; gpmc,burst-wrap; gpmc,burst-write; - gpmc,device-width = <2>; /* GPMC_DEVWIDTH_16BIT */ - gpmc,mux-add-data = <2>; /* GPMC_MUX_AD */ + gpmc,device-width = <2>; + gpmc,mux-add-data = <2>; gpmc,cs-on-ns = <0>; - gpmc,cs-rd-off-ns = <87>; - gpmc,cs-wr-off-ns = <87>; + gpmc,cs-rd-off-ns = <102>; + gpmc,cs-wr-off-ns = <102>; gpmc,adv-on-ns = <0>; - gpmc,adv-rd-off-ns = <10>; - gpmc,adv-wr-off-ns = <10>; - gpmc,oe-on-ns = <15>; - gpmc,oe-off-ns = <87>; + gpmc,adv-rd-off-ns = <12>; + gpmc,adv-wr-off-ns = <12>; + gpmc,oe-on-ns = <12>; + gpmc,oe-off-ns = <102>; gpmc,we-on-ns = <0>; - gpmc,we-off-ns = <87>; - gpmc,rd-cycle-ns = <112>; - gpmc,wr-cycle-ns = <112>; - gpmc,access-ns = <81>; - gpmc,page-burst-access-ns = <15>; + gpmc,we-off-ns = <102>; + gpmc,rd-cycle-ns = <132>; + gpmc,wr-cycle-ns = <132>; + gpmc,access-ns = <96>; + gpmc,page-burst-access-ns = <18>; gpmc,bus-turnaround-ns = <0>; gpmc,cycle2cycle-delay-ns = <0>; gpmc,wait-monitoring-ns = <0>; - gpmc,clk-activation-ns = <5>; - gpmc,wr-data-mux-bus-ns = <30>; - gpmc,wr-access-ns = <81>; + gpmc,clk-activation-ns = <6>; + gpmc,wr-data-mux-bus-ns = <36>; + gpmc,wr-access-ns = <96>; gpmc,sync-clk-ps = <15000>; /* From e38fa3065d126fb7ffa589e1ff733b469809f328 Mon Sep 17 00:00:00 2001 From: Chen-Yu Tsai Date: Wed, 11 Mar 2020 01:47:07 +0800 Subject: [PATCH 29/34] ARM: dts: sun8i: r40: Move AHCI device node based on address order commit fe3a04824f75786e39ed74e82fb6cb2534c95fe4 upstream. When the AHCI device node was added, it was added in the wrong location in the device tree file. The device nodes should be sorted by register address. Move the device node to before EHCI1, where it belongs. Fixes: 41c64d3318aa ("ARM: dts: sun8i: r40: add sata node") Acked-by: Maxime Ripard Reviewed-by: Andre Przywara Signed-off-by: Chen-Yu Tsai Signed-off-by: Greg Kroah-Hartman --- arch/arm/boot/dts/sun8i-r40.dtsi | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/arch/arm/boot/dts/sun8i-r40.dtsi b/arch/arm/boot/dts/sun8i-r40.dtsi index 421dfbbfd7ee4..c5a2d402f26d0 100644 --- a/arch/arm/boot/dts/sun8i-r40.dtsi +++ b/arch/arm/boot/dts/sun8i-r40.dtsi @@ -275,6 +275,16 @@ resets = <&ccu RST_BUS_CE>; }; + ahci: sata@1c18000 { + compatible = "allwinner,sun8i-r40-ahci"; + reg = <0x01c18000 0x1000>; + interrupts = ; + clocks = <&ccu CLK_BUS_SATA>, <&ccu CLK_SATA>; + resets = <&ccu RST_BUS_SATA>; + reset-names = "ahci"; + status = "disabled"; + }; + ehci1: usb@1c19000 { compatible = "allwinner,sun8i-r40-ehci", "generic-ehci"; reg = <0x01c19000 0x100>; @@ -566,17 +576,6 @@ #size-cells = <0>; }; - ahci: sata@1c18000 { - compatible = "allwinner,sun8i-r40-ahci"; - reg = <0x01c18000 0x1000>; - interrupts = ; - clocks = <&ccu CLK_BUS_SATA>, <&ccu CLK_SATA>; - resets = <&ccu RST_BUS_SATA>; - reset-names = "ahci"; - status = "disabled"; - - }; - gmac: ethernet@1c50000 { compatible = "allwinner,sun8i-r40-gmac"; syscon = <&ccu>; From 36514a23f1bb6d1b02dc7a15e7a8cb2f4b3401ca Mon Sep 17 00:00:00 2001 From: Madalin Bucur Date: Mon, 16 Mar 2020 14:05:57 +0200 Subject: [PATCH 30/34] arm64: dts: ls1043a-rdb: correct RGMII delay mode to rgmii-id commit 4022d808c45277693ea86478fab1f081ebf997e8 upstream. The correct setting for the RGMII ports on LS1043ARDB is to enable delay on both Rx and Tx so the interface mode used must be PHY_INTERFACE_MODE_RGMII_ID. Since commit 1b3047b5208a80 ("net: phy: realtek: add support for configuring the RX delay on RTL8211F") the Realtek 8211F PHY driver has control over the RGMII RX delay and it is disabling it for RGMII_TXID. The LS1043ARDB uses two such PHYs in RGMII_ID mode but in the device tree the mode was described as "rgmii_txid". This issue was not apparent at the time as the PHY driver took the same action for RGMII_TXID and RGMII_ID back then but it became visible (RX no longer working) after the above patch. Changing the phy-connection-type to "rgmii-id" to address the issue. Fixes: bf02f2ffe59c ("arm64: dts: add LS1043A DPAA FMan support") Signed-off-by: Madalin Bucur Signed-off-by: David S. Miller Signed-off-by: Greg Kroah-Hartman --- arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts index 4223a2352d45a..dde50c88f5e35 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts @@ -119,12 +119,12 @@ ethernet@e4000 { phy-handle = <&rgmii_phy1>; - phy-connection-type = "rgmii-txid"; + phy-connection-type = "rgmii-id"; }; ethernet@e6000 { phy-handle = <&rgmii_phy2>; - phy-connection-type = "rgmii-txid"; + phy-connection-type = "rgmii-id"; }; ethernet@e8000 { From f500e8136e67161eb133c33a14c41918e20aa978 Mon Sep 17 00:00:00 2001 From: Madalin Bucur Date: Mon, 16 Mar 2020 14:05:58 +0200 Subject: [PATCH 31/34] arm64: dts: ls1046ardb: set RGMII interfaces to RGMII_ID mode commit d79e9d7c1e4ba5f95f2ff3541880c40ea9722212 upstream. The correct setting for the RGMII ports on LS1046ARDB is to enable delay on both Rx and Tx so the interface mode used must be PHY_INTERFACE_MODE_RGMII_ID. Since commit 1b3047b5208a80 ("net: phy: realtek: add support for configuring the RX delay on RTL8211F") the Realtek 8211F PHY driver has control over the RGMII RX delay and it is disabling it for RGMII_TXID. The LS1046ARDB uses two such PHYs in RGMII_ID mode but in the device tree the mode was described as "rgmii". Changing the phy-connection-type to "rgmii-id" to address the issue. Fixes: 3fa395d2c48a ("arm64: dts: add LS1046A DPAA FMan nodes") Signed-off-by: Madalin Bucur Signed-off-by: David S. Miller Signed-off-by: Greg Kroah-Hartman --- arch/arm64/boot/dts/freescale/fsl-ls1046a-rdb.dts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1046a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1046a-rdb.dts index 0c742befb7612..da50db2f6900a 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1046a-rdb.dts +++ b/arch/arm64/boot/dts/freescale/fsl-ls1046a-rdb.dts @@ -131,12 +131,12 @@ &fman0 { ethernet@e4000 { phy-handle = <&rgmii_phy1>; - phy-connection-type = "rgmii"; + phy-connection-type = "rgmii-id"; }; ethernet@e6000 { phy-handle = <&rgmii_phy2>; - phy-connection-type = "rgmii"; + phy-connection-type = "rgmii-id"; }; ethernet@e8000 { From 43bc68e28bc817379b769d9e6352d72f1bfea233 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Thu, 2 Apr 2020 15:09:55 +0200 Subject: [PATCH 32/34] Linux 5.5.15 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 262892f82a150..2105fed0b3491 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 5 PATCHLEVEL = 5 -SUBLEVEL = 14 +SUBLEVEL = 15 EXTRAVERSION = NAME = Kleptomaniac Octopus From 6d9e81c5bacffa30e12c12234a03e58f9606e508 Mon Sep 17 00:00:00 2001 From: Steven Barrett Date: Thu, 2 Apr 2020 09:45:03 -0500 Subject: [PATCH 33/34] Update WireGuard for v5.5 - 2020-04-02 Source: https://git.zx2c4.com/wireguard-linux/log/?h=backport-5.5.y 3e69f3354c97 net: WireGuard secure network tunnel 541c818f504c icmp: introduce helper for nat'd source address in network device context 48aec5600f16 crypto: arm[64]/poly1305 - add artifact to .gitignore files 3abcb930dace crypto: x86/curve25519 - replace with formally verified implementation 4080c93f4e7e crypto: x86/curve25519 - support assemblers with no adx support 88af9018a82a crypto: Kconfig - allow tests to be disabled when manager is disabled 744d8679aa7b crypto: {arm,arm64,mips}/poly1305 - remove redundant non-reduction from emit f950970c2467 crypto: x86/poly1305 - wire up faster implementations for kernel 7a183de769a9 crypto: x86/poly1305 - import unmodified cryptogams implementation 76b9f883f67e crypto: poly1305 - add new 32 and 64-bit generic versions --- arch/arm64/crypto/chacha-neon-glue.c | 8 +- drivers/net/wireguard/queueing.h | 2 +- lib/crypto/chacha20poly1305-selftest.c | 1717 +----------------------- 3 files changed, 19 insertions(+), 1708 deletions(-) diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/crypto/chacha-neon-glue.c index 37ca3e8898484..c1f9660d104cf 100644 --- a/arch/arm64/crypto/chacha-neon-glue.c +++ b/arch/arm64/crypto/chacha-neon-glue.c @@ -55,10 +55,10 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, break; } chacha_4block_xor_neon(state, dst, src, nrounds, l); - bytes -= l; - src += l; - dst += l; - state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); + bytes -= CHACHA_BLOCK_SIZE * 5; + src += CHACHA_BLOCK_SIZE * 5; + dst += CHACHA_BLOCK_SIZE * 5; + state[12] += 5; } } diff --git a/drivers/net/wireguard/queueing.h b/drivers/net/wireguard/queueing.h index cf1e0e2376d81..3432232afe061 100644 --- a/drivers/net/wireguard/queueing.h +++ b/drivers/net/wireguard/queueing.h @@ -100,8 +100,8 @@ static inline void wg_reset_packet(struct sk_buff *skb) skb->dev = NULL; #ifdef CONFIG_NET_SCHED skb->tc_index = 0; - skb_reset_tc(skb); #endif + skb_reset_redirect(skb); skb->hdr_len = skb_headroom(skb); skb_reset_mac_header(skb); skb_reset_network_header(skb); diff --git a/lib/crypto/chacha20poly1305-selftest.c b/lib/crypto/chacha20poly1305-selftest.c index fa43deda2660d..465de46dbdef2 100644 --- a/lib/crypto/chacha20poly1305-selftest.c +++ b/lib/crypto/chacha20poly1305-selftest.c @@ -4,7 +4,6 @@ */ #include -#include #include #include @@ -1927,1104 +1926,6 @@ static const u8 enc_key012[] __initconst = { 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 }; -/* wycheproof - rfc7539 */ -static const u8 enc_input013[] __initconst = { - 0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61, - 0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c, - 0x65, 0x6d, 0x65, 0x6e, 0x20, 0x6f, 0x66, 0x20, - 0x74, 0x68, 0x65, 0x20, 0x63, 0x6c, 0x61, 0x73, - 0x73, 0x20, 0x6f, 0x66, 0x20, 0x27, 0x39, 0x39, - 0x3a, 0x20, 0x49, 0x66, 0x20, 0x49, 0x20, 0x63, - 0x6f, 0x75, 0x6c, 0x64, 0x20, 0x6f, 0x66, 0x66, - 0x65, 0x72, 0x20, 0x79, 0x6f, 0x75, 0x20, 0x6f, - 0x6e, 0x6c, 0x79, 0x20, 0x6f, 0x6e, 0x65, 0x20, - 0x74, 0x69, 0x70, 0x20, 0x66, 0x6f, 0x72, 0x20, - 0x74, 0x68, 0x65, 0x20, 0x66, 0x75, 0x74, 0x75, - 0x72, 0x65, 0x2c, 0x20, 0x73, 0x75, 0x6e, 0x73, - 0x63, 0x72, 0x65, 0x65, 0x6e, 0x20, 0x77, 0x6f, - 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69, - 0x74, 0x2e -}; -static const u8 enc_output013[] __initconst = { - 0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb, - 0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2, - 0xa4, 0xad, 0xed, 0x51, 0x29, 0x6e, 0x08, 0xfe, - 0xa9, 0xe2, 0xb5, 0xa7, 0x36, 0xee, 0x62, 0xd6, - 0x3d, 0xbe, 0xa4, 0x5e, 0x8c, 0xa9, 0x67, 0x12, - 0x82, 0xfa, 0xfb, 0x69, 0xda, 0x92, 0x72, 0x8b, - 0x1a, 0x71, 0xde, 0x0a, 0x9e, 0x06, 0x0b, 0x29, - 0x05, 0xd6, 0xa5, 0xb6, 0x7e, 0xcd, 0x3b, 0x36, - 0x92, 0xdd, 0xbd, 0x7f, 0x2d, 0x77, 0x8b, 0x8c, - 0x98, 0x03, 0xae, 0xe3, 0x28, 0x09, 0x1b, 0x58, - 0xfa, 0xb3, 0x24, 0xe4, 0xfa, 0xd6, 0x75, 0x94, - 0x55, 0x85, 0x80, 0x8b, 0x48, 0x31, 0xd7, 0xbc, - 0x3f, 0xf4, 0xde, 0xf0, 0x8e, 0x4b, 0x7a, 0x9d, - 0xe5, 0x76, 0xd2, 0x65, 0x86, 0xce, 0xc6, 0x4b, - 0x61, 0x16, 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, - 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, - 0x06, 0x91 -}; -static const u8 enc_assoc013[] __initconst = { - 0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, - 0xc4, 0xc5, 0xc6, 0xc7 -}; -static const u8 enc_nonce013[] __initconst = { - 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, - 0x44, 0x45, 0x46, 0x47 -}; -static const u8 enc_key013[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - -/* wycheproof - misc */ -static const u8 enc_input014[] __initconst = { }; -static const u8 enc_output014[] __initconst = { - 0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5, - 0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d -}; -static const u8 enc_assoc014[] __initconst = { }; -static const u8 enc_nonce014[] __initconst = { - 0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1, - 0xea, 0x12, 0x37, 0x9d -}; -static const u8 enc_key014[] __initconst = { - 0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96, - 0x5e, 0xa3, 0x71, 0xd5, 0xff, 0x07, 0x3c, 0xf0, - 0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, 0xb2, 0x08, - 0x42, 0x6e, 0x11, 0x40, 0x9c, 0x09, 0xb9, 0xb0 -}; - -/* wycheproof - misc */ -static const u8 enc_input015[] __initconst = { }; -static const u8 enc_output015[] __initconst = { - 0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b, - 0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09 -}; -static const u8 enc_assoc015[] __initconst = { - 0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10 -}; -static const u8 enc_nonce015[] __initconst = { - 0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16, - 0xa3, 0xc6, 0xf6, 0x89 -}; -static const u8 enc_key015[] __initconst = { - 0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb, - 0x20, 0x4d, 0xb2, 0xc3, 0xf5, 0xc7, 0x46, 0x22, - 0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, 0x19, 0x63, - 0x91, 0xdb, 0xb9, 0x57, 0x7a, 0x25, 0x07, 0x42 -}; - -/* wycheproof - misc */ -static const u8 enc_input016[] __initconst = { - 0x2a -}; -static const u8 enc_output016[] __initconst = { - 0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80, - 0x1e, 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75, - 0x22 -}; -static const u8 enc_assoc016[] __initconst = { }; -static const u8 enc_nonce016[] __initconst = { - 0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd, - 0xee, 0xab, 0x60, 0xf1 -}; -static const u8 enc_key016[] __initconst = { - 0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50, - 0x08, 0xf5, 0x48, 0x4b, 0x4c, 0xb8, 0x03, 0xfa, - 0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, 0x1f, 0x6a, - 0xb6, 0xae, 0xf4, 0x91, 0x6a, 0x76, 0x62, 0x73 -}; - -/* wycheproof - misc */ -static const u8 enc_input017[] __initconst = { - 0x51 -}; -static const u8 enc_output017[] __initconst = { - 0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7, - 0xc6, 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72, - 0xb9 -}; -static const u8 enc_assoc017[] __initconst = { - 0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53 -}; -static const u8 enc_nonce017[] __initconst = { - 0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2, - 0x78, 0x2f, 0x44, 0x03 -}; -static const u8 enc_key017[] __initconst = { - 0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5, - 0x2b, 0xdb, 0x4a, 0x70, 0xb4, 0x43, 0x19, 0x9f, - 0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, 0x20, 0xc5, - 0x5e, 0x4b, 0x70, 0xf0, 0xfd, 0xa6, 0x20, 0xee -}; - -/* wycheproof - misc */ -static const u8 enc_input018[] __initconst = { - 0x5c, 0x60 -}; -static const u8 enc_output018[] __initconst = { - 0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39, - 0xc1, 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22, - 0xe5, 0xe2 -}; -static const u8 enc_assoc018[] __initconst = { }; -static const u8 enc_nonce018[] __initconst = { - 0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34, - 0x7e, 0x03, 0xf2, 0xdb -}; -static const u8 enc_key018[] __initconst = { - 0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89, - 0x19, 0x49, 0x89, 0x74, 0x35, 0x07, 0xbf, 0x3e, - 0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, 0x69, 0x5f, - 0xe6, 0xc0, 0x25, 0xfd, 0x9b, 0xa4, 0xc1, 0x00 -}; - -/* wycheproof - misc */ -static const u8 enc_input019[] __initconst = { - 0xdd, 0xf2 -}; -static const u8 enc_output019[] __initconst = { - 0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec, - 0x2e, 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24, - 0x37, 0xa2 -}; -static const u8 enc_assoc019[] __initconst = { - 0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf -}; -static const u8 enc_nonce019[] __initconst = { - 0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90, - 0xb6, 0x04, 0x0a, 0xc6 -}; -static const u8 enc_key019[] __initconst = { - 0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48, - 0xaa, 0x20, 0x30, 0xea, 0xcf, 0xe7, 0x2b, 0xff, - 0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, 0x33, 0x44, - 0xe5, 0x71, 0x8f, 0xe0, 0xd7, 0xab, 0x1a, 0x58 -}; - -/* wycheproof - misc */ -static const u8 enc_input020[] __initconst = { - 0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31 -}; -static const u8 enc_output020[] __initconst = { - 0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed, - 0x7a, 0x43, 0x4e, 0x26, 0x02, 0xd3, 0x94, 0x28, - 0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42 -}; -static const u8 enc_assoc020[] __initconst = { }; -static const u8 enc_nonce020[] __initconst = { - 0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59, - 0x6d, 0xc5, 0x5b, 0xb7 -}; -static const u8 enc_key020[] __initconst = { - 0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f, - 0x1f, 0x70, 0x21, 0xea, 0xb6, 0x9b, 0x70, 0x3f, - 0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, 0xf5, 0xd3, - 0x4a, 0x37, 0x4f, 0xdb, 0xfc, 0x5a, 0xda, 0xc7 -}; - -/* wycheproof - misc */ -static const u8 enc_input021[] __initconst = { - 0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90 -}; -static const u8 enc_output021[] __initconst = { - 0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18, - 0x5f, 0x14, 0x08, 0xee, 0x7f, 0xbf, 0x18, 0xf5, - 0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba -}; -static const u8 enc_assoc021[] __initconst = { - 0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53 -}; -static const u8 enc_nonce021[] __initconst = { - 0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98, - 0xde, 0x94, 0x83, 0x96 -}; -static const u8 enc_key021[] __initconst = { - 0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5, - 0x16, 0xda, 0x6a, 0x33, 0x02, 0x2a, 0x1d, 0xd4, - 0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, 0xc5, 0xda, - 0x52, 0xa7, 0x30, 0xf3, 0x4d, 0x84, 0x0d, 0x7f -}; - -/* wycheproof - misc */ -static const u8 enc_input022[] __initconst = { - 0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed -}; -static const u8 enc_output022[] __initconst = { - 0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17, - 0xfb, 0xe8, 0xa6, 0xed, 0x8f, 0xab, 0xb1, 0x93, - 0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21 -}; -static const u8 enc_assoc022[] __initconst = { }; -static const u8 enc_nonce022[] __initconst = { - 0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2, - 0x36, 0x18, 0x23, 0xd3 -}; -static const u8 enc_key022[] __initconst = { - 0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf, - 0xf1, 0xa4, 0xbc, 0x05, 0x55, 0x0b, 0xa8, 0x1d, - 0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, 0x12, 0xc7, - 0xb0, 0x0e, 0x60, 0xe4, 0x8d, 0xb7, 0xe1, 0x52 -}; - -/* wycheproof - misc */ -static const u8 enc_input023[] __initconst = { - 0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf -}; -static const u8 enc_output023[] __initconst = { - 0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0, - 0x45, 0x30, 0xf5, 0x1a, 0xee, 0xe0, 0x24, 0xe0, - 0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18 -}; -static const u8 enc_assoc023[] __initconst = { - 0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d -}; -static const u8 enc_nonce023[] __initconst = { - 0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33, - 0x66, 0x48, 0x4a, 0x78 -}; -static const u8 enc_key023[] __initconst = { - 0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54, - 0xde, 0x7a, 0x4a, 0xc5, 0x51, 0x95, 0x0f, 0x1a, - 0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, 0x75, 0xfe, - 0x4a, 0xbd, 0x56, 0x08, 0xd7, 0xcd, 0xda, 0xdd -}; - -/* wycheproof - misc */ -static const u8 enc_input024[] __initconst = { - 0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c, - 0x36, 0x8d, 0x14, 0xe0 -}; -static const u8 enc_output024[] __initconst = { - 0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a, - 0x6b, 0x50, 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27, - 0x58, 0xc2, 0xe1, 0x4e, 0x36, 0xd4, 0xfc, 0x10, - 0x6d, 0xcb, 0x29, 0xb4 -}; -static const u8 enc_assoc024[] __initconst = { }; -static const u8 enc_nonce024[] __initconst = { - 0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e, - 0x07, 0x53, 0x86, 0x56 -}; -static const u8 enc_key024[] __initconst = { - 0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0, - 0x15, 0xbd, 0xe9, 0x3f, 0x0a, 0xbf, 0x48, 0x39, - 0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, 0x9c, 0xb5, - 0x0b, 0x39, 0xf7, 0xf8, 0xa9, 0xcc, 0x51, 0xbe -}; - -/* wycheproof - misc */ -static const u8 enc_input025[] __initconst = { - 0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7, - 0x59, 0xb1, 0xa0, 0xda -}; -static const u8 enc_output025[] __initconst = { - 0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38, - 0xd9, 0xe5, 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c, - 0x9d, 0x8f, 0xa9, 0x15, 0xfd, 0x84, 0x23, 0xa7, - 0xcf, 0x05, 0x07, 0x2f -}; -static const u8 enc_assoc025[] __initconst = { - 0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9 -}; -static const u8 enc_nonce025[] __initconst = { - 0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d, - 0xd9, 0x06, 0xe9, 0xce -}; -static const u8 enc_key025[] __initconst = { - 0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40, - 0xbc, 0x10, 0x20, 0xc8, 0x13, 0x6f, 0xf2, 0x70, - 0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, 0x6f, 0x1e, - 0x82, 0x51, 0x18, 0xb0, 0x06, 0xb9, 0x12, 0x57 -}; - -/* wycheproof - misc */ -static const u8 enc_input026[] __initconst = { - 0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac, - 0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07 -}; -static const u8 enc_output026[] __initconst = { - 0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf, - 0x53, 0x97, 0x0b, 0xe2, 0x21, 0x00, 0x42, 0x2a, - 0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, 0x6e, 0x79, - 0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2 -}; -static const u8 enc_assoc026[] __initconst = { }; -static const u8 enc_nonce026[] __initconst = { - 0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda, - 0xd4, 0x61, 0xd2, 0x3c -}; -static const u8 enc_key026[] __initconst = { - 0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda, - 0x90, 0x62, 0x19, 0xe0, 0x8c, 0x0d, 0x0d, 0x77, - 0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, 0x4f, 0xe0, - 0x70, 0x9a, 0xf7, 0x55, 0xee, 0xec, 0x6d, 0xcb -}; - -/* wycheproof - misc */ -static const u8 enc_input027[] __initconst = { - 0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f, - 0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73 -}; -static const u8 enc_output027[] __initconst = { - 0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81, - 0x55, 0x74, 0x55, 0xe5, 0x98, 0x0f, 0xed, 0xfe, - 0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, 0xd3, 0xa9, - 0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17 -}; -static const u8 enc_assoc027[] __initconst = { - 0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12 -}; -static const u8 enc_nonce027[] __initconst = { - 0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14, - 0xc5, 0x03, 0x5b, 0x6a -}; -static const u8 enc_key027[] __initconst = { - 0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f, - 0xb5, 0x82, 0x21, 0xb7, 0xed, 0x79, 0x67, 0x38, - 0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, 0xa6, 0x0b, - 0xbc, 0x8b, 0xc4, 0x89, 0x3a, 0x58, 0x89, 0x4d -}; - -/* wycheproof - misc */ -static const u8 enc_input028[] __initconst = { - 0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0, - 0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88 -}; -static const u8 enc_output028[] __initconst = { - 0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4, - 0x83, 0x50, 0x0c, 0xea, 0xba, 0xb6, 0x7a, 0x13, - 0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, 0xa2, 0x2a, - 0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c -}; -static const u8 enc_assoc028[] __initconst = { }; -static const u8 enc_nonce028[] __initconst = { - 0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8, - 0x47, 0x40, 0xad, 0x9b -}; -static const u8 enc_key028[] __initconst = { - 0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7, - 0xd3, 0xdb, 0x99, 0xa8, 0xf5, 0x4b, 0x15, 0xd7, - 0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, 0x97, 0x63, - 0xb0, 0x19, 0xc1, 0x69, 0x9f, 0x87, 0x67, 0x4a -}; - -/* wycheproof - misc */ -static const u8 enc_input029[] __initconst = { - 0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09, - 0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6 -}; -static const u8 enc_output029[] __initconst = { - 0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3, - 0xd3, 0xa8, 0xae, 0x9e, 0xc1, 0x41, 0xc0, 0x0c, - 0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, 0xd0, 0xdc, - 0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d -}; -static const u8 enc_assoc029[] __initconst = { - 0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff -}; -static const u8 enc_nonce029[] __initconst = { - 0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80, - 0x07, 0x1f, 0x52, 0x66 -}; -static const u8 enc_key029[] __initconst = { - 0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8, - 0xa8, 0x01, 0x9e, 0xde, 0xe3, 0xf2, 0x59, 0x14, - 0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, 0x6e, 0x1d, - 0x77, 0x1c, 0x86, 0x25, 0x71, 0xd2, 0xea, 0x1e -}; - -/* wycheproof - misc */ -static const u8 enc_input030[] __initconst = { - 0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15, - 0x91, 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e, - 0x1f -}; -static const u8 enc_output030[] __initconst = { - 0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd, - 0x12, 0xa0, 0x3b, 0xb8, 0x57, 0x97, 0x64, 0x74, - 0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, 0xc3, 0x80, - 0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08, - 0x13 -}; -static const u8 enc_assoc030[] __initconst = { }; -static const u8 enc_nonce030[] __initconst = { - 0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42, - 0xdc, 0x03, 0x44, 0x5d -}; -static const u8 enc_key030[] __initconst = { - 0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21, - 0xc0, 0xcc, 0x24, 0xc0, 0xc0, 0xe2, 0x4c, 0x1e, - 0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, 0x9a, 0x4b, - 0x16, 0x61, 0x76, 0x62, 0x96, 0x16, 0xd0, 0x11 -}; - -/* wycheproof - misc */ -static const u8 enc_input031[] __initconst = { - 0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88, - 0x7b, 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c, - 0x9c -}; -static const u8 enc_output031[] __initconst = { - 0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b, - 0x8d, 0x17, 0xcb, 0xbb, 0x9a, 0xbf, 0xad, 0x9e, - 0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, 0x27, 0x38, - 0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c, - 0xeb -}; -static const u8 enc_assoc031[] __initconst = { - 0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79 -}; -static const u8 enc_nonce031[] __initconst = { - 0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10, - 0x63, 0x3d, 0x99, 0x3d -}; -static const u8 enc_key031[] __initconst = { - 0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7, - 0x6c, 0x70, 0x37, 0xb4, 0x8f, 0xe3, 0x57, 0x7f, - 0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, 0xa9, 0x82, - 0x17, 0x6d, 0x18, 0x23, 0x06, 0xff, 0xb8, 0x70 -}; - -/* wycheproof - misc */ -static const u8 enc_input032[] __initconst = { - 0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66, - 0x5d, 0x9a, 0xe5, 0xef, 0xe2, 0x79, 0xe7, 0xf7, - 0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11 -}; -static const u8 enc_output032[] __initconst = { - 0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d, - 0x9d, 0x8d, 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0, - 0x0d, 0x75, 0xf3, 0x80, 0x74, 0xbf, 0xe7, 0x64, - 0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, 0xff, 0x7d, - 0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a -}; -static const u8 enc_assoc032[] __initconst = { }; -static const u8 enc_nonce032[] __initconst = { - 0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76, - 0x2c, 0x65, 0xf3, 0x1b -}; -static const u8 enc_key032[] __initconst = { - 0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87, - 0xb3, 0x3f, 0x1a, 0x70, 0x90, 0xb6, 0x11, 0x4f, - 0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, 0xf2, 0xc2, - 0x76, 0xbd, 0xcf, 0xdb, 0xc5, 0xce, 0x9c, 0xd7 -}; - -/* wycheproof - misc */ -static const u8 enc_input033[] __initconst = { - 0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d, - 0xe1, 0x20, 0x08, 0x06, 0xa9, 0xee, 0x8e, 0x4c, - 0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93 -}; -static const u8 enc_output033[] __initconst = { - 0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69, - 0x20, 0xe8, 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4, - 0x88, 0x9e, 0xbe, 0x7a, 0x0a, 0xd0, 0xe9, 0x00, - 0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, 0xda, 0x63, - 0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65 -}; -static const u8 enc_assoc033[] __initconst = { - 0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08 -}; -static const u8 enc_nonce033[] __initconst = { - 0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd, - 0x78, 0x34, 0xed, 0x55 -}; -static const u8 enc_key033[] __initconst = { - 0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed, - 0xda, 0x95, 0x4f, 0x1f, 0x73, 0x92, 0x23, 0xda, - 0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, 0x03, 0x3d, - 0x0f, 0xae, 0x4b, 0x02, 0x83, 0xd1, 0x8b, 0xe3 -}; - -/* wycheproof - misc */ -static const u8 enc_input034[] __initconst = { - 0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f, - 0x42, 0xcc, 0x8c, 0x20, 0x3a, 0xb2, 0x4e, 0x2c, - 0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, 0xd3, 0x26, - 0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29 -}; -static const u8 enc_output034[] __initconst = { - 0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab, - 0xb6, 0x88, 0x76, 0xa6, 0xe9, 0x6a, 0x48, 0xfb, - 0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1, - 0x89, 0xc9, 0xfd, 0x2d, 0xb9, 0x17, 0x46, 0x56, - 0x6d, 0x3c, 0xa1, 0x0e, 0x31, 0x1b, 0x69, 0x5f, - 0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93 -}; -static const u8 enc_assoc034[] __initconst = { }; -static const u8 enc_nonce034[] __initconst = { - 0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31, - 0x37, 0x1a, 0x6f, 0xd2 -}; -static const u8 enc_key034[] __initconst = { - 0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e, - 0x6c, 0x54, 0xa8, 0x34, 0x9b, 0xae, 0x50, 0xa2, - 0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, 0xa1, 0x15, - 0x2c, 0x42, 0x4f, 0x03, 0xb6, 0x67, 0x1d, 0x71 -}; - -/* wycheproof - misc */ -static const u8 enc_input035[] __initconst = { - 0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2, - 0xab, 0xc9, 0xab, 0x7f, 0xa2, 0x43, 0x03, 0x5f, - 0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, 0x39, 0x56, - 0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb -}; -static const u8 enc_output035[] __initconst = { - 0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20, - 0x08, 0x46, 0x27, 0x58, 0x00, 0x80, 0xdd, 0xe5, - 0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e, - 0xd6, 0xf3, 0xd3, 0x26, 0xbc, 0x7b, 0x70, 0x53, - 0x4d, 0x4a, 0xa2, 0x83, 0x5a, 0x52, 0xe7, 0x2d, - 0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f -}; -static const u8 enc_assoc035[] __initconst = { - 0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48 -}; -static const u8 enc_nonce035[] __initconst = { - 0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8, - 0xb8, 0x1a, 0x1f, 0x8b -}; -static const u8 enc_key035[] __initconst = { - 0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f, - 0xd1, 0x43, 0x58, 0xfe, 0x2f, 0xf2, 0xa7, 0x40, - 0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, 0xe6, 0xa4, - 0x37, 0xf1, 0x9e, 0xa4, 0x99, 0x11, 0xce, 0xc4 -}; - -/* wycheproof - misc */ -static const u8 enc_input036[] __initconst = { - 0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a, - 0xe8, 0x97, 0xdf, 0x7b, 0xee, 0xca, 0xc1, 0xdb, - 0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, 0x60, 0xce, - 0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78 -}; -static const u8 enc_output036[] __initconst = { - 0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76, - 0x0f, 0x5f, 0xe1, 0x97, 0x23, 0xb9, 0x81, 0x1e, - 0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50, - 0x81, 0xac, 0x43, 0x0b, 0x95, 0x3b, 0x0e, 0x21, - 0x22, 0x25, 0x41, 0xaf, 0x46, 0xb8, 0x65, 0x33, - 0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea -}; -static const u8 enc_assoc036[] __initconst = { }; -static const u8 enc_nonce036[] __initconst = { - 0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2, - 0x2b, 0x7e, 0x6e, 0x6a -}; -static const u8 enc_key036[] __initconst = { - 0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c, - 0xb3, 0xff, 0xe8, 0xb7, 0x86, 0xa2, 0xb3, 0xeb, - 0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, 0x57, 0xb9, - 0xdc, 0x8c, 0x7b, 0xc5, 0x35, 0x46, 0x34, 0xa3 -}; - -/* wycheproof - misc */ -static const u8 enc_input037[] __initconst = { - 0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14, - 0xa1, 0x3a, 0x52, 0x57, 0xed, 0x6c, 0xff, 0xd3, - 0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, 0xb1, 0x79, - 0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e -}; -static const u8 enc_output037[] __initconst = { - 0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b, - 0xf6, 0xbe, 0x95, 0x07, 0xed, 0x4d, 0x1b, 0xb2, - 0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29, - 0xda, 0xf5, 0x07, 0x62, 0x84, 0x6f, 0xf2, 0xf4, - 0x7b, 0x99, 0x7d, 0x93, 0xc9, 0x82, 0x18, 0x9d, - 0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32 -}; -static const u8 enc_assoc037[] __initconst = { - 0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59 -}; -static const u8 enc_nonce037[] __initconst = { - 0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5, - 0x34, 0x0d, 0xd1, 0xb8 -}; -static const u8 enc_key037[] __initconst = { - 0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4, - 0x10, 0x70, 0x2f, 0xea, 0x61, 0xbc, 0x87, 0x3f, - 0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, 0x4a, 0xbd, - 0xe2, 0x5b, 0x78, 0x6a, 0x2d, 0x97, 0xf1, 0x45 -}; - -/* wycheproof - misc */ -static const u8 enc_input038[] __initconst = { - 0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4, - 0x66, 0xcf, 0x9a, 0xbb, 0x34, 0x93, 0x15, 0x2e, - 0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11, - 0x71, 0x34, 0x6b, 0x4d, 0x2c, 0xeb, 0xf8, 0x04, - 0x1a, 0xa3, 0xce, 0xdc, 0x0d, 0xfd, 0x7b, 0x46, - 0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a -}; -static const u8 enc_output038[] __initconst = { - 0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6, - 0x2e, 0x3d, 0xa4, 0x0b, 0xe2, 0xcb, 0x77, 0x00, - 0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6, - 0x52, 0xe7, 0x6a, 0x43, 0x2c, 0x65, 0x8d, 0x27, - 0x09, 0x5f, 0x0e, 0xb8, 0xf9, 0x40, 0xc3, 0x24, - 0x98, 0x1e, 0xa9, 0x35, 0xe5, 0x07, 0xf9, 0x8f, - 0x04, 0x69, 0x56, 0xdb, 0x3a, 0x51, 0x29, 0x08, - 0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9 -}; -static const u8 enc_assoc038[] __initconst = { }; -static const u8 enc_nonce038[] __initconst = { - 0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9, - 0x0b, 0xef, 0x55, 0xd2 -}; -static const u8 enc_key038[] __initconst = { - 0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51, - 0x81, 0x51, 0x65, 0xd5, 0x30, 0x1b, 0x22, 0x63, - 0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, 0x78, 0xe3, - 0xa0, 0xa5, 0xe8, 0xe1, 0xaa, 0xe0, 0xf2, 0x64 -}; - -/* wycheproof - misc */ -static const u8 enc_input039[] __initconst = { - 0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74, - 0x8b, 0xfb, 0x77, 0x85, 0xff, 0x91, 0xee, 0xb3, - 0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d, - 0x5e, 0x7b, 0x0e, 0x79, 0xef, 0x65, 0x36, 0x59, - 0xa9, 0x97, 0x8d, 0x75, 0x54, 0x2e, 0xf9, 0x1c, - 0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9 -}; -static const u8 enc_output039[] __initconst = { - 0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec, - 0xce, 0x24, 0x2e, 0x0e, 0x88, 0xb1, 0x38, 0x04, - 0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e, - 0x19, 0x98, 0x6b, 0x3e, 0x31, 0x8c, 0xae, 0x8d, - 0x80, 0x61, 0x98, 0xfb, 0x4c, 0x52, 0x7c, 0xc3, - 0x93, 0x50, 0xeb, 0xdd, 0xea, 0xc5, 0x73, 0xc4, - 0xcb, 0xf0, 0xbe, 0xfd, 0xa0, 0xb7, 0x02, 0x42, - 0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3 -}; -static const u8 enc_assoc039[] __initconst = { - 0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84 -}; -static const u8 enc_nonce039[] __initconst = { - 0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b, - 0x31, 0xcd, 0x4d, 0x95 -}; -static const u8 enc_key039[] __initconst = { - 0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c, - 0xf5, 0xd2, 0x3c, 0x40, 0x2b, 0xdb, 0x0d, 0x25, - 0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, 0x73, 0xd4, - 0x2e, 0xb5, 0xbe, 0x10, 0x9f, 0xdc, 0xb2, 0xac -}; - -/* wycheproof - misc */ -static const u8 enc_input040[] __initconst = { - 0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e, - 0x00, 0x8f, 0xf8, 0x5d, 0x5d, 0xdc, 0x38, 0xdd, - 0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f, - 0x1e, 0x40, 0x79, 0xcb, 0x64, 0xd0, 0xdc, 0x8f, - 0x5e, 0x67, 0x11, 0xcd, 0x49, 0x21, 0xa7, 0x88, - 0x7d, 0xe7, 0x6e, 0x26, 0x78, 0xfd, 0xc6, 0x76, - 0x18, 0xf1, 0x18, 0x55, 0x86, 0xbf, 0xea, 0x9d, - 0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82 -}; -static const u8 enc_output040[] __initconst = { - 0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5, - 0x75, 0x5c, 0x08, 0xf7, 0x47, 0xc0, 0xf8, 0xd8, - 0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c, - 0x12, 0xbb, 0x51, 0xbb, 0x18, 0x72, 0xc8, 0xe8, - 0xb8, 0x77, 0x67, 0x8b, 0xec, 0x44, 0x2c, 0xfc, - 0xbb, 0x0f, 0xf4, 0x64, 0xa6, 0x4b, 0x74, 0x33, - 0x2c, 0xf0, 0x72, 0x89, 0x8c, 0x7e, 0x0e, 0xdd, - 0xf6, 0x23, 0x2e, 0xa6, 0xe2, 0x7e, 0xfe, 0x50, - 0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56, - 0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13 -}; -static const u8 enc_assoc040[] __initconst = { }; -static const u8 enc_nonce040[] __initconst = { - 0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28, - 0x23, 0xcc, 0x06, 0x5b -}; -static const u8 enc_key040[] __initconst = { - 0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0, - 0xb0, 0x44, 0x47, 0x67, 0xf8, 0x0a, 0x25, 0xb8, - 0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, 0x4d, 0x30, - 0x52, 0x22, 0x6b, 0xaa, 0x1c, 0x6f, 0xb7, 0x01 -}; - -/* wycheproof - misc */ -static const u8 enc_input041[] __initconst = { - 0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88, - 0x32, 0x43, 0x71, 0x36, 0xb4, 0xae, 0x80, 0x5d, - 0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19, - 0x3a, 0x25, 0x01, 0x62, 0x93, 0xed, 0xba, 0x44, - 0x3c, 0x58, 0xe0, 0x7e, 0x7b, 0x71, 0x95, 0xec, - 0x5b, 0xd8, 0x45, 0x82, 0xa9, 0xd5, 0x6c, 0x8d, - 0x4a, 0x10, 0x8c, 0x7d, 0x7c, 0xe3, 0x4e, 0x6c, - 0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17 -}; -static const u8 enc_output041[] __initconst = { - 0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16, - 0x14, 0xf6, 0x36, 0x03, 0x1e, 0xeb, 0x42, 0xf1, - 0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8, - 0x71, 0xee, 0x8e, 0x73, 0x82, 0x0c, 0x82, 0x97, - 0x49, 0xf1, 0xab, 0xb4, 0x29, 0x43, 0x67, 0x84, - 0x9f, 0xb6, 0xc2, 0xaa, 0x56, 0xbd, 0xa8, 0xa3, - 0x07, 0x8f, 0x72, 0x3d, 0x7c, 0x1c, 0x85, 0x20, - 0x24, 0xb0, 0x17, 0xb5, 0x89, 0x73, 0xfb, 0x1e, - 0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14, - 0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec -}; -static const u8 enc_assoc041[] __initconst = { - 0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2 -}; -static const u8 enc_nonce041[] __initconst = { - 0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d, - 0xe4, 0xeb, 0xb1, 0x9c -}; -static const u8 enc_key041[] __initconst = { - 0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9, - 0x99, 0x8a, 0x90, 0x87, 0x5d, 0xa1, 0xd3, 0xa2, - 0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, 0x97, 0xa5, - 0x72, 0x52, 0x35, 0x19, 0x0e, 0x10, 0xa9, 0x7e -}; - -/* wycheproof - misc */ -static const u8 enc_input042[] __initconst = { - 0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba, - 0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, 0x34, 0x58, - 0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06, - 0x9d, 0x12, 0xd4, 0x5c, 0x22, 0x21, 0x75, 0x64, - 0x12, 0x15, 0x88, 0x03, 0x22, 0x97, 0xef, 0xf5, - 0x67, 0x83, 0x74, 0x2a, 0x5f, 0xc2, 0x2d, 0x74, - 0x10, 0xff, 0xb2, 0x9d, 0x66, 0x09, 0x86, 0x61, - 0xd7, 0x6f, 0x12, 0x6c, 0x3c, 0x27, 0x68, 0x9e, - 0x43, 0xb3, 0x72, 0x67, 0xca, 0xc5, 0xa3, 0xa6, - 0xd3, 0xab, 0x49, 0xe3, 0x91, 0xda, 0x29, 0xcd, - 0x30, 0x54, 0xa5, 0x69, 0x2e, 0x28, 0x07, 0xe4, - 0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5, - 0x92 -}; -static const u8 enc_output042[] __initconst = { - 0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97, - 0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf, - 0x23, 0xd1, 0x05, 0xfb, 0x4c, 0x84, 0xcf, 0x98, - 0x51, 0x5e, 0x1b, 0xc9, 0xa6, 0x4f, 0x8a, 0xd5, - 0xbe, 0x8f, 0x07, 0x21, 0xbd, 0xe5, 0x06, 0x45, - 0xd0, 0x00, 0x83, 0xc3, 0xa2, 0x63, 0xa3, 0x10, - 0x53, 0xb7, 0x60, 0x24, 0x5f, 0x52, 0xae, 0x28, - 0x66, 0xa5, 0xec, 0x83, 0xb1, 0x9f, 0x61, 0xbe, - 0x1d, 0x30, 0xd5, 0xc5, 0xd9, 0xfe, 0xcc, 0x4c, - 0xbb, 0xe0, 0x8f, 0xd3, 0x85, 0x81, 0x3a, 0x2a, - 0xa3, 0x9a, 0x00, 0xff, 0x9c, 0x10, 0xf7, 0xf2, - 0x37, 0x02, 0xad, 0xd1, 0xe4, 0xb2, 0xff, 0xa3, - 0x1c, 0x41, 0x86, 0x5f, 0xc7, 0x1d, 0xe1, 0x2b, - 0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b, - 0xb0 -}; -static const u8 enc_assoc042[] __initconst = { }; -static const u8 enc_nonce042[] __initconst = { - 0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03, - 0xac, 0xde, 0x27, 0x99 -}; -static const u8 enc_key042[] __initconst = { - 0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28, - 0x4c, 0xa5, 0xfc, 0x0f, 0x8c, 0x5f, 0x2b, 0x8d, - 0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, 0xa6, 0x6a, - 0x41, 0x99, 0x5e, 0x78, 0x33, 0x51, 0xf9, 0x01 -}; - -/* wycheproof - misc */ -static const u8 enc_input043[] __initconst = { - 0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff, - 0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, 0xb2, 0xcc, - 0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc, - 0x16, 0x5d, 0x06, 0xc2, 0xf5, 0x10, 0x0f, 0xf5, - 0xe8, 0xab, 0x28, 0x82, 0xc4, 0x75, 0xaf, 0xcd, - 0x05, 0xcc, 0xd4, 0x9f, 0x2e, 0x7d, 0x8f, 0x55, - 0xef, 0x3a, 0x72, 0xe3, 0xdc, 0x51, 0xd6, 0x85, - 0x2b, 0x8e, 0x6b, 0x9e, 0x7a, 0xec, 0xe5, 0x7b, - 0xe6, 0x55, 0x6b, 0x0b, 0x6d, 0x94, 0x13, 0xe3, - 0x3f, 0xc5, 0xfc, 0x24, 0xa9, 0xa2, 0x05, 0xad, - 0x59, 0x57, 0x4b, 0xb3, 0x9d, 0x94, 0x4a, 0x92, - 0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31, - 0x76 -}; -static const u8 enc_output043[] __initconst = { - 0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5, - 0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06, - 0x3e, 0x58, 0xed, 0xee, 0x4b, 0xb1, 0x22, 0x7e, - 0x71, 0x10, 0xac, 0x4d, 0x26, 0x20, 0xc2, 0xae, - 0xc2, 0xf8, 0x48, 0xf5, 0x6d, 0xee, 0xb0, 0x37, - 0xa8, 0xdc, 0xed, 0x75, 0xaf, 0xa8, 0xa6, 0xc8, - 0x90, 0xe2, 0xde, 0xe4, 0x2f, 0x95, 0x0b, 0xb3, - 0x3d, 0x9e, 0x24, 0x24, 0xd0, 0x8a, 0x50, 0x5d, - 0x89, 0x95, 0x63, 0x97, 0x3e, 0xd3, 0x88, 0x70, - 0xf3, 0xde, 0x6e, 0xe2, 0xad, 0xc7, 0xfe, 0x07, - 0x2c, 0x36, 0x6c, 0x14, 0xe2, 0xcf, 0x7c, 0xa6, - 0x2f, 0xb3, 0xd3, 0x6b, 0xee, 0x11, 0x68, 0x54, - 0x61, 0xb7, 0x0d, 0x44, 0xef, 0x8c, 0x66, 0xc5, - 0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac, - 0xf6 -}; -static const u8 enc_assoc043[] __initconst = { - 0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30 -}; -static const u8 enc_nonce043[] __initconst = { - 0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63, - 0xe5, 0x22, 0x94, 0x60 -}; -static const u8 enc_key043[] __initconst = { - 0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c, - 0x3e, 0x9b, 0xdc, 0x48, 0x0d, 0xea, 0x97, 0xb4, - 0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, 0xff, 0x6b, - 0x15, 0xbd, 0xd2, 0xa4, 0xa5, 0x68, 0x2c, 0x4e -}; - -/* wycheproof - misc */ -static const u8 enc_input044[] __initconst = { - 0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68, - 0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2 -}; -static const u8 enc_output044[] __initconst = { - 0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6, - 0x11, 0x92, 0x1b, 0x43, 0x8f, 0x74, 0x9f, 0x0b, - 0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, 0x34, 0x02, - 0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4 -}; -static const u8 enc_assoc044[] __initconst = { - 0x02 -}; -static const u8 enc_nonce044[] __initconst = { - 0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21, - 0x02, 0xd5, 0x06, 0x56 -}; -static const u8 enc_key044[] __initconst = { - 0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32, - 0x72, 0x05, 0x06, 0x07, 0xb2, 0x64, 0x18, 0x50, - 0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, 0x8b, 0xe0, - 0x22, 0x77, 0x0f, 0x2c, 0xe2, 0xf3, 0x14, 0x3c -}; - -/* wycheproof - misc */ -static const u8 enc_input045[] __initconst = { - 0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44, - 0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8 -}; -static const u8 enc_output045[] __initconst = { - 0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1, - 0x4d, 0x1f, 0x8f, 0xff, 0xab, 0x6d, 0x9e, 0x60, - 0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, 0x15, 0xa1, - 0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58 -}; -static const u8 enc_assoc045[] __initconst = { - 0xb6, 0x48 -}; -static const u8 enc_nonce045[] __initconst = { - 0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9, - 0x5b, 0x3a, 0xa7, 0x13 -}; -static const u8 enc_key045[] __initconst = { - 0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41, - 0xac, 0x78, 0x36, 0xbc, 0xe2, 0x51, 0x85, 0xa0, - 0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, 0x18, 0x44, - 0x4b, 0x6e, 0xc7, 0x2c, 0x3b, 0xd8, 0xe4, 0xdc -}; - -/* wycheproof - misc */ -static const u8 enc_input046[] __initconst = { - 0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23, - 0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47 -}; -static const u8 enc_output046[] __initconst = { - 0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda, - 0xc0, 0x3e, 0x5f, 0x74, 0x2b, 0xff, 0x73, 0xa1, - 0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, 0x38, 0x2b, - 0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8 -}; -static const u8 enc_assoc046[] __initconst = { - 0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd, - 0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0 -}; -static const u8 enc_nonce046[] __initconst = { - 0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b, - 0xa4, 0x65, 0x96, 0xdf -}; -static const u8 enc_key046[] __initconst = { - 0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08, - 0x2a, 0x92, 0x0b, 0x86, 0x36, 0x4e, 0xb8, 0x96, - 0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, 0xd5, 0x89, - 0x29, 0xfc, 0xb3, 0x66, 0x90, 0xe6, 0x39, 0x4f -}; - -/* wycheproof - misc */ -static const u8 enc_input047[] __initconst = { - 0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf, - 0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2 -}; -static const u8 enc_output047[] __initconst = { - 0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb, - 0x17, 0x77, 0x2a, 0x1c, 0x50, 0x0c, 0x50, 0x95, - 0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, 0xa0, 0x34, - 0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f -}; -static const u8 enc_assoc047[] __initconst = { - 0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07, - 0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b -}; -static const u8 enc_nonce047[] __initconst = { - 0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6, - 0x80, 0x92, 0x66, 0xd9 -}; -static const u8 enc_key047[] __initconst = { - 0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91, - 0xcf, 0x32, 0x0b, 0x13, 0x9f, 0xb5, 0x94, 0x23, - 0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, 0x65, 0xa6, - 0x7d, 0x4f, 0x63, 0x3f, 0x2c, 0x08, 0xf0, 0x16 -}; - -/* wycheproof - misc */ -static const u8 enc_input048[] __initconst = { - 0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32, - 0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3 -}; -static const u8 enc_output048[] __initconst = { - 0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b, - 0x1f, 0xd2, 0xb5, 0xc3, 0xa1, 0x5e, 0x35, 0x68, - 0x33, 0x35, 0x11, 0x86, 0x17, 0x96, 0x97, 0x84, - 0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3 -}; -static const u8 enc_assoc048[] __initconst = { - 0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab, - 0x03, 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c, - 0x0e -}; -static const u8 enc_nonce048[] __initconst = { - 0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40, - 0xfc, 0x10, 0x68, 0xc3 -}; -static const u8 enc_key048[] __initconst = { - 0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b, - 0xa6, 0xe0, 0xff, 0x15, 0xc4, 0x62, 0x78, 0x97, - 0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, 0xd0, 0x0b, - 0x2e, 0x93, 0xfd, 0xdc, 0x4b, 0xa7, 0x76, 0x57 -}; - -/* wycheproof - misc */ -static const u8 enc_input049[] __initconst = { - 0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2, - 0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6 -}; -static const u8 enc_output049[] __initconst = { - 0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87, - 0x20, 0x9a, 0x16, 0xe2, 0x85, 0x14, 0x31, 0x11, - 0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, 0x11, 0x7e, - 0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6 -}; -static const u8 enc_assoc049[] __initconst = { - 0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8, - 0x1e, 0x37, 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94, - 0x80, 0xa2, 0xa4, 0xd3, 0x35, 0x2a, 0x4f, 0xf5, - 0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda -}; -static const u8 enc_nonce049[] __initconst = { - 0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf, - 0xad, 0x14, 0xd5, 0x3e -}; -static const u8 enc_key049[] __initconst = { - 0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d, - 0x04, 0x63, 0x0f, 0x4a, 0x42, 0xf3, 0x5c, 0xfc, - 0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, 0x2b, 0x47, - 0x84, 0xf6, 0xf9, 0x29, 0x03, 0x30, 0xbe, 0xe0 -}; - -/* wycheproof - misc */ -static const u8 enc_input050[] __initconst = { - 0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18, - 0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c -}; -static const u8 enc_output050[] __initconst = { - 0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6, - 0x60, 0x2b, 0xc7, 0x78, 0x05, 0x6f, 0x0d, 0xca, - 0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, 0xb4, 0x6b, - 0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04 -}; -static const u8 enc_assoc050[] __initconst = { - 0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22, - 0x45, 0x0f, 0xeb, 0x10, 0x73, 0x7d, 0x90, 0x07, - 0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, 0x4d, 0x90, - 0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37 -}; -static const u8 enc_nonce050[] __initconst = { - 0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28, - 0x6a, 0x7b, 0x76, 0x51 -}; -static const u8 enc_key050[] __initconst = { - 0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1, - 0xba, 0x0c, 0x46, 0x72, 0xd8, 0xab, 0xc4, 0x6c, - 0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, 0x26, 0x4a, - 0xb0, 0x6d, 0x6c, 0x58, 0x07, 0x20, 0x51, 0x30 -}; - -/* wycheproof - misc */ -static const u8 enc_input051[] __initconst = { - 0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59, - 0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6 -}; -static const u8 enc_output051[] __initconst = { - 0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70, - 0xde, 0x32, 0x56, 0x2c, 0xcf, 0x2c, 0x2b, 0xbd, - 0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, 0x19, 0xb4, - 0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43 -}; -static const u8 enc_assoc051[] __initconst = { - 0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92, - 0x47, 0x02, 0x19, 0x05, 0x40, 0x2e, 0xa2, 0x57, - 0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, 0x56, 0x75, - 0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88, - 0x73 -}; -static const u8 enc_nonce051[] __initconst = { - 0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0, - 0xa8, 0xfa, 0x89, 0x49 -}; -static const u8 enc_key051[] __initconst = { - 0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27, - 0x96, 0x46, 0xac, 0x3d, 0x17, 0x6b, 0x96, 0x74, - 0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, 0x46, 0x9c, - 0x65, 0x1e, 0xc7, 0xe1, 0x6b, 0x4c, 0x11, 0x99 -}; - -/* wycheproof - misc */ -static const u8 enc_input052[] __initconst = { - 0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3, - 0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed -}; -static const u8 enc_output052[] __initconst = { - 0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc, - 0xc9, 0x76, 0x67, 0x2c, 0x44, 0x15, 0xe6, 0x9b, - 0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, 0x5d, 0xed, - 0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32 -}; -static const u8 enc_assoc052[] __initconst = { - 0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a, - 0x78, 0xa3, 0xd8, 0x0e, 0xa2, 0x56, 0x66, 0x14, - 0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae, - 0x84, 0x58, 0x3d, 0xa7, 0x3d, 0x7f, 0xf8, 0x5c, - 0x6f, 0x42, 0xca, 0x42, 0x05, 0x6a, 0x97, 0x92, - 0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61 -}; -static const u8 enc_nonce052[] __initconst = { - 0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73, - 0x0e, 0xc3, 0x5d, 0x12 -}; -static const u8 enc_key052[] __initconst = { - 0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5, - 0x29, 0xc1, 0x1f, 0x2f, 0x5a, 0xdd, 0x9a, 0xaf, - 0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, 0x7a, 0x2c, - 0xb7, 0xf7, 0x97, 0x49, 0xbd, 0x90, 0xaa, 0xf4 -}; - /* wycheproof - misc */ static const u8 enc_input053[] __initconst = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, @@ -3858,126 +2759,6 @@ static const u8 enc_key073[] __initconst = { 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; -/* wycheproof - checking for int overflows */ -static const u8 enc_input074[] __initconst = { - 0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51, - 0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69, - 0xdf, 0x7e, 0x8b, 0x76, 0x24, 0x63, 0x33, 0x0f, - 0xac, 0xc1, 0x6a, 0x57, 0x26, 0xbe, 0x71, 0x90, - 0xc6, 0x3c, 0x5a, 0x1c, 0x92, 0x65, 0x84, 0xa0, - 0x96, 0x75, 0x68, 0x28, 0xdc, 0xdc, 0x64, 0xac, - 0xdf, 0x96, 0x3d, 0x93, 0x1b, 0xf1, 0xda, 0xe2, - 0x38, 0xf3, 0xf1, 0x57, 0x22, 0x4a, 0xc4, 0xb5, - 0x42, 0xd7, 0x85, 0xb0, 0xdd, 0x84, 0xdb, 0x6b, - 0xe3, 0xbc, 0x5a, 0x36, 0x63, 0xe8, 0x41, 0x49, - 0xff, 0xbe, 0xd0, 0x9e, 0x54, 0xf7, 0x8f, 0x16, - 0xa8, 0x22, 0x3b, 0x24, 0xcb, 0x01, 0x9f, 0x58, - 0xb2, 0x1b, 0x0e, 0x55, 0x1e, 0x7a, 0xa0, 0x73, - 0x27, 0x62, 0x95, 0x51, 0x37, 0x6c, 0xcb, 0xc3, - 0x93, 0x76, 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c, - 0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a -}; -static const u8 enc_output074[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85, - 0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca -}; -static const u8 enc_assoc074[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff -}; -static const u8 enc_nonce074[] __initconst = { - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x00, 0x02, 0x50, 0x6e -}; -static const u8 enc_key074[] __initconst = { - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 -}; - -/* wycheproof - checking for int overflows */ -static const u8 enc_input075[] __initconst = { - 0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75, - 0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56, - 0x9e, 0x16, 0xf1, 0x11, 0xb5, 0xeb, 0x26, 0xc2, - 0x2d, 0x85, 0x9e, 0x3f, 0xf8, 0x22, 0xec, 0xed, - 0x3a, 0x6d, 0xd9, 0xa6, 0x0f, 0x22, 0x95, 0x7f, - 0x7b, 0x7c, 0x85, 0x7e, 0x88, 0x22, 0xeb, 0x9f, - 0xe0, 0xb8, 0xd7, 0x02, 0x21, 0x41, 0xf2, 0xd0, - 0xb4, 0x8f, 0x4b, 0x56, 0x12, 0xd3, 0x22, 0xa8, - 0x8d, 0xd0, 0xfe, 0x0b, 0x4d, 0x91, 0x79, 0x32, - 0x4f, 0x7c, 0x6c, 0x9e, 0x99, 0x0e, 0xfb, 0xd8, - 0x0e, 0x5e, 0xd6, 0x77, 0x58, 0x26, 0x49, 0x8b, - 0x1e, 0xfe, 0x0f, 0x71, 0xa0, 0xf3, 0xec, 0x5b, - 0x29, 0xcb, 0x28, 0xc2, 0x54, 0x0a, 0x7d, 0xcd, - 0x51, 0xb7, 0xda, 0xae, 0xe0, 0xff, 0x4a, 0x7f, - 0x3a, 0xc1, 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1, - 0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94 -}; -static const u8 enc_output075[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7, - 0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5 -}; -static const u8 enc_assoc075[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff -}; -static const u8 enc_nonce075[] __initconst = { - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x00, 0x03, 0x18, 0xa5 -}; -static const u8 enc_key075[] __initconst = { - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, - 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 -}; - /* wycheproof - checking for int overflows */ static const u8 enc_input076[] __initconst = { 0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85, @@ -4568,286 +3349,6 @@ static const u8 enc_key085[] __initconst = { 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; -/* wycheproof - special case tag */ -static const u8 enc_input086[] __initconst = { - 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, - 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, - 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, - 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, - 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, - 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, - 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, - 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d -}; -static const u8 enc_output086[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f -}; -static const u8 enc_assoc086[] __initconst = { - 0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, 0xbb, 0xc1, - 0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00 -}; -static const u8 enc_nonce086[] __initconst = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b -}; -static const u8 enc_key086[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - -/* wycheproof - special case tag */ -static const u8 enc_input087[] __initconst = { - 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, - 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, - 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, - 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, - 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, - 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, - 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, - 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d -}; -static const u8 enc_output087[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; -static const u8 enc_assoc087[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, 0x0a, 0x2f, - 0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58 -}; -static const u8 enc_nonce087[] __initconst = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b -}; -static const u8 enc_key087[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - -/* wycheproof - special case tag */ -static const u8 enc_input088[] __initconst = { - 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, - 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, - 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, - 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, - 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, - 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, - 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, - 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d -}; -static const u8 enc_output088[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff -}; -static const u8 enc_assoc088[] __initconst = { - 0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, 0x96, 0x1f, - 0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00 -}; -static const u8 enc_nonce088[] __initconst = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b -}; -static const u8 enc_key088[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - -/* wycheproof - special case tag */ -static const u8 enc_input089[] __initconst = { - 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, - 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, - 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, - 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, - 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, - 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, - 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, - 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d -}; -static const u8 enc_output089[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, - 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 -}; -static const u8 enc_assoc089[] __initconst = { - 0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, 0x6e, 0xae, - 0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02 -}; -static const u8 enc_nonce089[] __initconst = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b -}; -static const u8 enc_key089[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - -/* wycheproof - special case tag */ -static const u8 enc_input090[] __initconst = { - 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, - 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, - 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, - 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, - 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, - 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, - 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, - 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d -}; -static const u8 enc_output090[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, - 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f -}; -static const u8 enc_assoc090[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, 0x07, 0xbe, - 0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3 -}; -static const u8 enc_nonce090[] __initconst = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b -}; -static const u8 enc_key090[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - -/* wycheproof - special case tag */ -static const u8 enc_input091[] __initconst = { - 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, - 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, - 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, - 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, - 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, - 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, - 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, - 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d -}; -static const u8 enc_output091[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 -}; -static const u8 enc_assoc091[] __initconst = { - 0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, 0xd8, 0x30, - 0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01 -}; -static const u8 enc_nonce091[] __initconst = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b -}; -static const u8 enc_key091[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - -/* wycheproof - special case tag */ -static const u8 enc_input092[] __initconst = { - 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, - 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, - 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, - 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, - 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, - 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, - 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, - 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d -}; -static const u8 enc_output092[] __initconst = { - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; -static const u8 enc_assoc092[] __initconst = { - 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - 0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, 0x72, 0x11, - 0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01 -}; -static const u8 enc_nonce092[] __initconst = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b -}; -static const u8 enc_key092[] __initconst = { - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f -}; - /* wycheproof - edge case intermediate sums in poly1305 */ static const u8 enc_input093[] __initconst = { 0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d, @@ -5954,86 +4455,6 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input011), sizeof(enc_assoc011), sizeof(enc_nonce011) }, { enc_input012, enc_output012, enc_assoc012, enc_nonce012, enc_key012, sizeof(enc_input012), sizeof(enc_assoc012), sizeof(enc_nonce012) }, - { enc_input013, enc_output013, enc_assoc013, enc_nonce013, enc_key013, - sizeof(enc_input013), sizeof(enc_assoc013), sizeof(enc_nonce013) }, - { enc_input014, enc_output014, enc_assoc014, enc_nonce014, enc_key014, - sizeof(enc_input014), sizeof(enc_assoc014), sizeof(enc_nonce014) }, - { enc_input015, enc_output015, enc_assoc015, enc_nonce015, enc_key015, - sizeof(enc_input015), sizeof(enc_assoc015), sizeof(enc_nonce015) }, - { enc_input016, enc_output016, enc_assoc016, enc_nonce016, enc_key016, - sizeof(enc_input016), sizeof(enc_assoc016), sizeof(enc_nonce016) }, - { enc_input017, enc_output017, enc_assoc017, enc_nonce017, enc_key017, - sizeof(enc_input017), sizeof(enc_assoc017), sizeof(enc_nonce017) }, - { enc_input018, enc_output018, enc_assoc018, enc_nonce018, enc_key018, - sizeof(enc_input018), sizeof(enc_assoc018), sizeof(enc_nonce018) }, - { enc_input019, enc_output019, enc_assoc019, enc_nonce019, enc_key019, - sizeof(enc_input019), sizeof(enc_assoc019), sizeof(enc_nonce019) }, - { enc_input020, enc_output020, enc_assoc020, enc_nonce020, enc_key020, - sizeof(enc_input020), sizeof(enc_assoc020), sizeof(enc_nonce020) }, - { enc_input021, enc_output021, enc_assoc021, enc_nonce021, enc_key021, - sizeof(enc_input021), sizeof(enc_assoc021), sizeof(enc_nonce021) }, - { enc_input022, enc_output022, enc_assoc022, enc_nonce022, enc_key022, - sizeof(enc_input022), sizeof(enc_assoc022), sizeof(enc_nonce022) }, - { enc_input023, enc_output023, enc_assoc023, enc_nonce023, enc_key023, - sizeof(enc_input023), sizeof(enc_assoc023), sizeof(enc_nonce023) }, - { enc_input024, enc_output024, enc_assoc024, enc_nonce024, enc_key024, - sizeof(enc_input024), sizeof(enc_assoc024), sizeof(enc_nonce024) }, - { enc_input025, enc_output025, enc_assoc025, enc_nonce025, enc_key025, - sizeof(enc_input025), sizeof(enc_assoc025), sizeof(enc_nonce025) }, - { enc_input026, enc_output026, enc_assoc026, enc_nonce026, enc_key026, - sizeof(enc_input026), sizeof(enc_assoc026), sizeof(enc_nonce026) }, - { enc_input027, enc_output027, enc_assoc027, enc_nonce027, enc_key027, - sizeof(enc_input027), sizeof(enc_assoc027), sizeof(enc_nonce027) }, - { enc_input028, enc_output028, enc_assoc028, enc_nonce028, enc_key028, - sizeof(enc_input028), sizeof(enc_assoc028), sizeof(enc_nonce028) }, - { enc_input029, enc_output029, enc_assoc029, enc_nonce029, enc_key029, - sizeof(enc_input029), sizeof(enc_assoc029), sizeof(enc_nonce029) }, - { enc_input030, enc_output030, enc_assoc030, enc_nonce030, enc_key030, - sizeof(enc_input030), sizeof(enc_assoc030), sizeof(enc_nonce030) }, - { enc_input031, enc_output031, enc_assoc031, enc_nonce031, enc_key031, - sizeof(enc_input031), sizeof(enc_assoc031), sizeof(enc_nonce031) }, - { enc_input032, enc_output032, enc_assoc032, enc_nonce032, enc_key032, - sizeof(enc_input032), sizeof(enc_assoc032), sizeof(enc_nonce032) }, - { enc_input033, enc_output033, enc_assoc033, enc_nonce033, enc_key033, - sizeof(enc_input033), sizeof(enc_assoc033), sizeof(enc_nonce033) }, - { enc_input034, enc_output034, enc_assoc034, enc_nonce034, enc_key034, - sizeof(enc_input034), sizeof(enc_assoc034), sizeof(enc_nonce034) }, - { enc_input035, enc_output035, enc_assoc035, enc_nonce035, enc_key035, - sizeof(enc_input035), sizeof(enc_assoc035), sizeof(enc_nonce035) }, - { enc_input036, enc_output036, enc_assoc036, enc_nonce036, enc_key036, - sizeof(enc_input036), sizeof(enc_assoc036), sizeof(enc_nonce036) }, - { enc_input037, enc_output037, enc_assoc037, enc_nonce037, enc_key037, - sizeof(enc_input037), sizeof(enc_assoc037), sizeof(enc_nonce037) }, - { enc_input038, enc_output038, enc_assoc038, enc_nonce038, enc_key038, - sizeof(enc_input038), sizeof(enc_assoc038), sizeof(enc_nonce038) }, - { enc_input039, enc_output039, enc_assoc039, enc_nonce039, enc_key039, - sizeof(enc_input039), sizeof(enc_assoc039), sizeof(enc_nonce039) }, - { enc_input040, enc_output040, enc_assoc040, enc_nonce040, enc_key040, - sizeof(enc_input040), sizeof(enc_assoc040), sizeof(enc_nonce040) }, - { enc_input041, enc_output041, enc_assoc041, enc_nonce041, enc_key041, - sizeof(enc_input041), sizeof(enc_assoc041), sizeof(enc_nonce041) }, - { enc_input042, enc_output042, enc_assoc042, enc_nonce042, enc_key042, - sizeof(enc_input042), sizeof(enc_assoc042), sizeof(enc_nonce042) }, - { enc_input043, enc_output043, enc_assoc043, enc_nonce043, enc_key043, - sizeof(enc_input043), sizeof(enc_assoc043), sizeof(enc_nonce043) }, - { enc_input044, enc_output044, enc_assoc044, enc_nonce044, enc_key044, - sizeof(enc_input044), sizeof(enc_assoc044), sizeof(enc_nonce044) }, - { enc_input045, enc_output045, enc_assoc045, enc_nonce045, enc_key045, - sizeof(enc_input045), sizeof(enc_assoc045), sizeof(enc_nonce045) }, - { enc_input046, enc_output046, enc_assoc046, enc_nonce046, enc_key046, - sizeof(enc_input046), sizeof(enc_assoc046), sizeof(enc_nonce046) }, - { enc_input047, enc_output047, enc_assoc047, enc_nonce047, enc_key047, - sizeof(enc_input047), sizeof(enc_assoc047), sizeof(enc_nonce047) }, - { enc_input048, enc_output048, enc_assoc048, enc_nonce048, enc_key048, - sizeof(enc_input048), sizeof(enc_assoc048), sizeof(enc_nonce048) }, - { enc_input049, enc_output049, enc_assoc049, enc_nonce049, enc_key049, - sizeof(enc_input049), sizeof(enc_assoc049), sizeof(enc_nonce049) }, - { enc_input050, enc_output050, enc_assoc050, enc_nonce050, enc_key050, - sizeof(enc_input050), sizeof(enc_assoc050), sizeof(enc_nonce050) }, - { enc_input051, enc_output051, enc_assoc051, enc_nonce051, enc_key051, - sizeof(enc_input051), sizeof(enc_assoc051), sizeof(enc_nonce051) }, - { enc_input052, enc_output052, enc_assoc052, enc_nonce052, enc_key052, - sizeof(enc_input052), sizeof(enc_assoc052), sizeof(enc_nonce052) }, { enc_input053, enc_output053, enc_assoc053, enc_nonce053, enc_key053, sizeof(enc_input053), sizeof(enc_assoc053), sizeof(enc_nonce053) }, { enc_input054, enc_output054, enc_assoc054, enc_nonce054, enc_key054, @@ -6076,10 +4497,6 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input072), sizeof(enc_assoc072), sizeof(enc_nonce072) }, { enc_input073, enc_output073, enc_assoc073, enc_nonce073, enc_key073, sizeof(enc_input073), sizeof(enc_assoc073), sizeof(enc_nonce073) }, - { enc_input074, enc_output074, enc_assoc074, enc_nonce074, enc_key074, - sizeof(enc_input074), sizeof(enc_assoc074), sizeof(enc_nonce074) }, - { enc_input075, enc_output075, enc_assoc075, enc_nonce075, enc_key075, - sizeof(enc_input075), sizeof(enc_assoc075), sizeof(enc_nonce075) }, { enc_input076, enc_output076, enc_assoc076, enc_nonce076, enc_key076, sizeof(enc_input076), sizeof(enc_assoc076), sizeof(enc_nonce076) }, { enc_input077, enc_output077, enc_assoc077, enc_nonce077, enc_key077, @@ -6100,20 +4517,6 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input084), sizeof(enc_assoc084), sizeof(enc_nonce084) }, { enc_input085, enc_output085, enc_assoc085, enc_nonce085, enc_key085, sizeof(enc_input085), sizeof(enc_assoc085), sizeof(enc_nonce085) }, - { enc_input086, enc_output086, enc_assoc086, enc_nonce086, enc_key086, - sizeof(enc_input086), sizeof(enc_assoc086), sizeof(enc_nonce086) }, - { enc_input087, enc_output087, enc_assoc087, enc_nonce087, enc_key087, - sizeof(enc_input087), sizeof(enc_assoc087), sizeof(enc_nonce087) }, - { enc_input088, enc_output088, enc_assoc088, enc_nonce088, enc_key088, - sizeof(enc_input088), sizeof(enc_assoc088), sizeof(enc_nonce088) }, - { enc_input089, enc_output089, enc_assoc089, enc_nonce089, enc_key089, - sizeof(enc_input089), sizeof(enc_assoc089), sizeof(enc_nonce089) }, - { enc_input090, enc_output090, enc_assoc090, enc_nonce090, enc_key090, - sizeof(enc_input090), sizeof(enc_assoc090), sizeof(enc_nonce090) }, - { enc_input091, enc_output091, enc_assoc091, enc_nonce091, enc_key091, - sizeof(enc_input091), sizeof(enc_assoc091), sizeof(enc_nonce091) }, - { enc_input092, enc_output092, enc_assoc092, enc_nonce092, enc_key092, - sizeof(enc_input092), sizeof(enc_assoc092), sizeof(enc_nonce092) }, { enc_input093, enc_output093, enc_assoc093, enc_nonce093, enc_key093, sizeof(enc_input093), sizeof(enc_assoc093), sizeof(enc_nonce093) }, { enc_input094, enc_output094, enc_assoc094, enc_nonce094, enc_key094, @@ -8821,43 +7224,6 @@ xchacha20poly1305_dec_vectors[] __initconst = { sizeof(xdec_input001), sizeof(xdec_assoc001), sizeof(xdec_nonce001) } }; -/* This is for the selftests-only, since it is only useful for the purpose of - * testing the underlying primitives and interactions. - */ -static void __init -chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, - const u8 *ad, const size_t ad_len, - const u8 nonce[12], - const u8 key[CHACHA20POLY1305_KEY_SIZE]) -{ - const u8 *pad0 = page_address(ZERO_PAGE(0)); - struct poly1305_desc_ctx poly1305_state; - u32 chacha20_state[CHACHA_STATE_WORDS]; - union { - u8 block0[POLY1305_KEY_SIZE]; - __le64 lens[2]; - } b = {{ 0 }}; - u8 bottom_row[16] = { 0 }; - u32 le_key[8]; - int i; - - memcpy(&bottom_row[4], nonce, 12); - for (i = 0; i < 8; ++i) - le_key[i] = get_unaligned_le32(key + sizeof(le_key[i]) * i); - chacha_init(chacha20_state, le_key, bottom_row); - chacha20_crypt(chacha20_state, b.block0, b.block0, sizeof(b.block0)); - poly1305_init(&poly1305_state, b.block0); - poly1305_update(&poly1305_state, ad, ad_len); - poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf); - chacha20_crypt(chacha20_state, dst, src, src_len); - poly1305_update(&poly1305_state, dst, src_len); - poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf); - b.lens[0] = cpu_to_le64(ad_len); - b.lens[1] = cpu_to_le64(src_len); - poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); - poly1305_final(&poly1305_state, dst + src_len); -} - static void __init chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, @@ -8867,9 +7233,6 @@ chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, if (nonce_len == 8) chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, get_unaligned_le64(nonce), key); - else if (nonce_len == 12) - chacha20poly1305_encrypt_bignonce(dst, src, src_len, ad, - ad_len, nonce, key); else BUG(); } @@ -8885,14 +7248,14 @@ decryption_success(bool func_ret, bool expect_failure, int memcmp_result) bool __init chacha20poly1305_selftest(void) { enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 }; - size_t i, j, k, total_len; - u8 *computed_output = NULL, *input = NULL; + size_t i; + u8 *computed_output = NULL, *heap_src = NULL; + struct scatterlist sg_src; bool success = true, ret; - struct scatterlist sg_src[3]; + heap_src = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - input = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - if (!computed_output || !input) { + if (!heap_src || !computed_output) { pr_err("chacha20poly1305 self-test malloc: FAIL\n"); success = false; goto out; @@ -8921,17 +7284,17 @@ bool __init chacha20poly1305_selftest(void) for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { if (chacha20poly1305_enc_vectors[i].nlen != 8) continue; - memcpy(computed_output, chacha20poly1305_enc_vectors[i].input, + memcpy(heap_src, chacha20poly1305_enc_vectors[i].input, chacha20poly1305_enc_vectors[i].ilen); - sg_init_one(sg_src, computed_output, + sg_init_one(&sg_src, heap_src, chacha20poly1305_enc_vectors[i].ilen + POLY1305_DIGEST_SIZE); - ret = chacha20poly1305_encrypt_sg_inplace(sg_src, + chacha20poly1305_encrypt_sg_inplace(&sg_src, chacha20poly1305_enc_vectors[i].ilen, chacha20poly1305_enc_vectors[i].assoc, chacha20poly1305_enc_vectors[i].alen, get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce), chacha20poly1305_enc_vectors[i].key); - if (!ret || memcmp(computed_output, + if (memcmp(heap_src, chacha20poly1305_enc_vectors[i].output, chacha20poly1305_enc_vectors[i].ilen + POLY1305_DIGEST_SIZE)) { @@ -8963,11 +7326,11 @@ bool __init chacha20poly1305_selftest(void) } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { - memcpy(computed_output, chacha20poly1305_dec_vectors[i].input, + memcpy(heap_src, chacha20poly1305_dec_vectors[i].input, chacha20poly1305_dec_vectors[i].ilen); - sg_init_one(sg_src, computed_output, + sg_init_one(&sg_src, heap_src, chacha20poly1305_dec_vectors[i].ilen); - ret = chacha20poly1305_decrypt_sg_inplace(sg_src, + ret = chacha20poly1305_decrypt_sg_inplace(&sg_src, chacha20poly1305_dec_vectors[i].ilen, chacha20poly1305_dec_vectors[i].assoc, chacha20poly1305_dec_vectors[i].alen, @@ -8975,7 +7338,7 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_dec_vectors[i].key); if (!decryption_success(ret, chacha20poly1305_dec_vectors[i].failure, - memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, + memcmp(heap_src, chacha20poly1305_dec_vectors[i].output, chacha20poly1305_dec_vectors[i].ilen - POLY1305_DIGEST_SIZE))) { pr_err("chacha20poly1305 sg decryption self-test %zu: FAIL\n", @@ -9002,7 +7365,6 @@ bool __init chacha20poly1305_selftest(void) success = false; } } - for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_dec_vectors); ++i) { memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); ret = xchacha20poly1305_decrypt(computed_output, @@ -9024,59 +7386,8 @@ bool __init chacha20poly1305_selftest(void) } } - for (total_len = POLY1305_DIGEST_SIZE; IS_ENABLED(DEBUG_CHACHA20POLY1305_SLOW_CHUNK_TEST) - && total_len <= 1 << 10; ++total_len) { - for (i = 0; i <= total_len; ++i) { - for (j = i; j <= total_len; ++j) { - k = 0; - sg_init_table(sg_src, 3); - if (i) - sg_set_buf(&sg_src[k++], input, i); - if (j - i) - sg_set_buf(&sg_src[k++], input + i, j - i); - if (total_len - j) - sg_set_buf(&sg_src[k++], input + j, total_len - j); - sg_init_marker(sg_src, k); - memset(computed_output, 0, total_len); - memset(input, 0, total_len); - - if (!chacha20poly1305_encrypt_sg_inplace(sg_src, - total_len - POLY1305_DIGEST_SIZE, NULL, 0, - 0, enc_key001)) - goto chunkfail; - chacha20poly1305_encrypt(computed_output, - computed_output, - total_len - POLY1305_DIGEST_SIZE, NULL, 0, 0, - enc_key001); - if (memcmp(computed_output, input, total_len)) - goto chunkfail; - if (!chacha20poly1305_decrypt(computed_output, - input, total_len, NULL, 0, 0, enc_key001)) - goto chunkfail; - for (k = 0; k < total_len - POLY1305_DIGEST_SIZE; ++k) { - if (computed_output[k]) - goto chunkfail; - } - if (!chacha20poly1305_decrypt_sg_inplace(sg_src, - total_len, NULL, 0, 0, enc_key001)) - goto chunkfail; - for (k = 0; k < total_len - POLY1305_DIGEST_SIZE; ++k) { - if (input[k]) - goto chunkfail; - } - continue; - - chunkfail: - pr_err("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL\n", - total_len, i, j); - success = false; - } - - } - } - out: + kfree(heap_src); kfree(computed_output); - kfree(input); return success; } From 1743f431c421a65f433084523ae8d8d81a719cd3 Mon Sep 17 00:00:00 2001 From: Steven Barrett Date: Thu, 2 Apr 2020 09:52:35 -0500 Subject: [PATCH 34/34] Restore "Add WireGuard for v5.5 - 2020-03-30" This reverts commit 820447db84465789238f6b81692ea4b4ad91f01a. --- MAINTAINERS | 8 + arch/arm/crypto/.gitignore | 1 + arch/arm/crypto/poly1305-glue.c | 18 +- arch/arm64/crypto/.gitignore | 1 + arch/arm64/crypto/chacha-neon-glue.c | 8 +- arch/arm64/crypto/poly1305-glue.c | 18 +- arch/mips/crypto/poly1305-glue.c | 18 +- arch/x86/Makefile | 5 +- arch/x86/crypto/.gitignore | 1 + arch/x86/crypto/Makefile | 18 +- arch/x86/crypto/curve25519-x86_64.c | 3546 +++++--------- arch/x86/crypto/poly1305-avx2-x86_64.S | 390 -- arch/x86/crypto/poly1305-sse2-x86_64.S | 590 --- arch/x86/crypto/poly1305-x86_64-cryptogams.pl | 4265 +++++++++++++++++ arch/x86/crypto/poly1305_glue.c | 304 +- crypto/Kconfig | 4 - crypto/adiantum.c | 4 +- crypto/nhpoly1305.c | 2 +- crypto/poly1305_generic.c | 25 +- drivers/net/Kconfig | 43 + drivers/net/Makefile | 1 + drivers/net/wireguard/Makefile | 18 + drivers/net/wireguard/allowedips.c | 377 ++ drivers/net/wireguard/allowedips.h | 59 + drivers/net/wireguard/cookie.c | 236 + drivers/net/wireguard/cookie.h | 59 + drivers/net/wireguard/device.c | 459 ++ drivers/net/wireguard/device.h | 65 + drivers/net/wireguard/main.c | 63 + drivers/net/wireguard/messages.h | 128 + drivers/net/wireguard/netlink.c | 636 +++ drivers/net/wireguard/netlink.h | 12 + drivers/net/wireguard/noise.c | 835 ++++ drivers/net/wireguard/noise.h | 137 + drivers/net/wireguard/peer.c | 237 + drivers/net/wireguard/peer.h | 83 + drivers/net/wireguard/peerlookup.c | 221 + drivers/net/wireguard/peerlookup.h | 64 + drivers/net/wireguard/queueing.c | 53 + drivers/net/wireguard/queueing.h | 200 + drivers/net/wireguard/ratelimiter.c | 223 + drivers/net/wireguard/ratelimiter.h | 19 + drivers/net/wireguard/receive.c | 597 +++ drivers/net/wireguard/selftest/allowedips.c | 683 +++ drivers/net/wireguard/selftest/counter.c | 104 + drivers/net/wireguard/selftest/ratelimiter.c | 226 + drivers/net/wireguard/send.c | 419 ++ drivers/net/wireguard/socket.c | 437 ++ drivers/net/wireguard/socket.h | 44 + drivers/net/wireguard/timers.c | 243 + drivers/net/wireguard/timers.h | 31 + drivers/net/wireguard/version.h | 1 + include/crypto/curve25519.h | 6 +- include/crypto/internal/poly1305.h | 45 +- include/crypto/nhpoly1305.h | 4 +- include/crypto/poly1305.h | 26 +- include/linux/icmpv6.h | 10 + include/linux/skbuff.h | 5 + include/net/icmp.h | 6 + include/uapi/linux/wireguard.h | 196 + lib/crypto/Kconfig | 2 +- lib/crypto/Makefile | 4 +- lib/crypto/chacha20poly1305-selftest.c | 1717 ++++++- lib/crypto/poly1305-donna32.c | 204 + lib/crypto/poly1305-donna64.c | 185 + lib/crypto/poly1305.c | 169 +- net/ipv4/icmp.c | 33 + net/ipv6/ip6_icmp.c | 34 + tools/testing/selftests/wireguard/netns.sh | 555 +++ .../selftests/wireguard/qemu/.gitignore | 2 + .../testing/selftests/wireguard/qemu/Makefile | 377 ++ .../wireguard/qemu/arch/aarch64.config | 5 + .../wireguard/qemu/arch/aarch64_be.config | 6 + .../selftests/wireguard/qemu/arch/arm.config | 9 + .../wireguard/qemu/arch/armeb.config | 10 + .../selftests/wireguard/qemu/arch/i686.config | 5 + .../selftests/wireguard/qemu/arch/m68k.config | 9 + .../selftests/wireguard/qemu/arch/mips.config | 11 + .../wireguard/qemu/arch/mips64.config | 14 + .../wireguard/qemu/arch/mips64el.config | 15 + .../wireguard/qemu/arch/mipsel.config | 12 + .../wireguard/qemu/arch/powerpc.config | 10 + .../wireguard/qemu/arch/powerpc64le.config | 12 + .../wireguard/qemu/arch/x86_64.config | 5 + .../selftests/wireguard/qemu/debug.config | 67 + tools/testing/selftests/wireguard/qemu/init.c | 284 ++ .../selftests/wireguard/qemu/kernel.config | 88 + 87 files changed, 16735 insertions(+), 3646 deletions(-) create mode 100644 arch/x86/crypto/.gitignore delete mode 100644 arch/x86/crypto/poly1305-avx2-x86_64.S delete mode 100644 arch/x86/crypto/poly1305-sse2-x86_64.S create mode 100644 arch/x86/crypto/poly1305-x86_64-cryptogams.pl create mode 100644 drivers/net/wireguard/Makefile create mode 100644 drivers/net/wireguard/allowedips.c create mode 100644 drivers/net/wireguard/allowedips.h create mode 100644 drivers/net/wireguard/cookie.c create mode 100644 drivers/net/wireguard/cookie.h create mode 100644 drivers/net/wireguard/device.c create mode 100644 drivers/net/wireguard/device.h create mode 100644 drivers/net/wireguard/main.c create mode 100644 drivers/net/wireguard/messages.h create mode 100644 drivers/net/wireguard/netlink.c create mode 100644 drivers/net/wireguard/netlink.h create mode 100644 drivers/net/wireguard/noise.c create mode 100644 drivers/net/wireguard/noise.h create mode 100644 drivers/net/wireguard/peer.c create mode 100644 drivers/net/wireguard/peer.h create mode 100644 drivers/net/wireguard/peerlookup.c create mode 100644 drivers/net/wireguard/peerlookup.h create mode 100644 drivers/net/wireguard/queueing.c create mode 100644 drivers/net/wireguard/queueing.h create mode 100644 drivers/net/wireguard/ratelimiter.c create mode 100644 drivers/net/wireguard/ratelimiter.h create mode 100644 drivers/net/wireguard/receive.c create mode 100644 drivers/net/wireguard/selftest/allowedips.c create mode 100644 drivers/net/wireguard/selftest/counter.c create mode 100644 drivers/net/wireguard/selftest/ratelimiter.c create mode 100644 drivers/net/wireguard/send.c create mode 100644 drivers/net/wireguard/socket.c create mode 100644 drivers/net/wireguard/socket.h create mode 100644 drivers/net/wireguard/timers.c create mode 100644 drivers/net/wireguard/timers.h create mode 100644 drivers/net/wireguard/version.h create mode 100644 include/uapi/linux/wireguard.h create mode 100644 lib/crypto/poly1305-donna32.c create mode 100644 lib/crypto/poly1305-donna64.c create mode 100755 tools/testing/selftests/wireguard/netns.sh create mode 100644 tools/testing/selftests/wireguard/qemu/.gitignore create mode 100644 tools/testing/selftests/wireguard/qemu/Makefile create mode 100644 tools/testing/selftests/wireguard/qemu/arch/aarch64.config create mode 100644 tools/testing/selftests/wireguard/qemu/arch/aarch64_be.config create mode 100644 tools/testing/selftests/wireguard/qemu/arch/arm.config create mode 100644 tools/testing/selftests/wireguard/qemu/arch/armeb.config create mode 100644 tools/testing/selftests/wireguard/qemu/arch/i686.config create mode 100644 tools/testing/selftests/wireguard/qemu/arch/m68k.config create mode 100644 tools/testing/selftests/wireguard/qemu/arch/mips.config create mode 100644 tools/testing/selftests/wireguard/qemu/arch/mips64.config create mode 100644 tools/testing/selftests/wireguard/qemu/arch/mips64el.config create mode 100644 tools/testing/selftests/wireguard/qemu/arch/mipsel.config create mode 100644 tools/testing/selftests/wireguard/qemu/arch/powerpc.config create mode 100644 tools/testing/selftests/wireguard/qemu/arch/powerpc64le.config create mode 100644 tools/testing/selftests/wireguard/qemu/arch/x86_64.config create mode 100644 tools/testing/selftests/wireguard/qemu/debug.config create mode 100644 tools/testing/selftests/wireguard/qemu/init.c create mode 100644 tools/testing/selftests/wireguard/qemu/kernel.config diff --git a/MAINTAINERS b/MAINTAINERS index 61856805620eb..d9adb2152f683 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17872,6 +17872,14 @@ L: linux-gpio@vger.kernel.org S: Maintained F: drivers/gpio/gpio-ws16c48.c +WIREGUARD SECURE NETWORK TUNNEL +M: Jason A. Donenfeld +S: Maintained +F: drivers/net/wireguard/ +F: tools/testing/selftests/wireguard/ +L: wireguard@lists.zx2c4.com +L: netdev@vger.kernel.org + WISTRON LAPTOP BUTTON DRIVER M: Miloslav Trmac S: Maintained diff --git a/arch/arm/crypto/.gitignore b/arch/arm/crypto/.gitignore index 31e1f538df7de..a3c7ad52a469f 100644 --- a/arch/arm/crypto/.gitignore +++ b/arch/arm/crypto/.gitignore @@ -1,3 +1,4 @@ aesbs-core.S sha256-core.S sha512-core.S +poly1305-core.S diff --git a/arch/arm/crypto/poly1305-glue.c b/arch/arm/crypto/poly1305-glue.c index abe3f2d587dcb..ceec04ec2f405 100644 --- a/arch/arm/crypto/poly1305-glue.c +++ b/arch/arm/crypto/poly1305-glue.c @@ -20,7 +20,7 @@ void poly1305_init_arm(void *state, const u8 *key); void poly1305_blocks_arm(void *state, const u8 *src, u32 len, u32 hibit); -void poly1305_emit_arm(void *state, __le32 *digest, const u32 *nonce); +void poly1305_emit_arm(void *state, u8 *digest, const u32 *nonce); void __weak poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit) { @@ -179,9 +179,6 @@ EXPORT_SYMBOL(poly1305_update_arch); void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) { - __le32 digest[4]; - u64 f = 0; - if (unlikely(dctx->buflen)) { dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, @@ -189,18 +186,7 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) poly1305_blocks_arm(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); } - poly1305_emit_arm(&dctx->h, digest, dctx->s); - - /* mac = (h + s) % (2^128) */ - f = (f >> 32) + le32_to_cpu(digest[0]); - put_unaligned_le32(f, dst); - f = (f >> 32) + le32_to_cpu(digest[1]); - put_unaligned_le32(f, dst + 4); - f = (f >> 32) + le32_to_cpu(digest[2]); - put_unaligned_le32(f, dst + 8); - f = (f >> 32) + le32_to_cpu(digest[3]); - put_unaligned_le32(f, dst + 12); - + poly1305_emit_arm(&dctx->h, dst, dctx->s); *dctx = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL(poly1305_final_arch); diff --git a/arch/arm64/crypto/.gitignore b/arch/arm64/crypto/.gitignore index 879df8781ed56..e403b1343328d 100644 --- a/arch/arm64/crypto/.gitignore +++ b/arch/arm64/crypto/.gitignore @@ -1,2 +1,3 @@ sha256-core.S sha512-core.S +poly1305-core.S diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/crypto/chacha-neon-glue.c index c1f9660d104cf..37ca3e8898484 100644 --- a/arch/arm64/crypto/chacha-neon-glue.c +++ b/arch/arm64/crypto/chacha-neon-glue.c @@ -55,10 +55,10 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, break; } chacha_4block_xor_neon(state, dst, src, nrounds, l); - bytes -= CHACHA_BLOCK_SIZE * 5; - src += CHACHA_BLOCK_SIZE * 5; - dst += CHACHA_BLOCK_SIZE * 5; - state[12] += 5; + bytes -= l; + src += l; + dst += l; + state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); } } diff --git a/arch/arm64/crypto/poly1305-glue.c b/arch/arm64/crypto/poly1305-glue.c index 83a2338a88263..e97b092f56b8f 100644 --- a/arch/arm64/crypto/poly1305-glue.c +++ b/arch/arm64/crypto/poly1305-glue.c @@ -21,7 +21,7 @@ asmlinkage void poly1305_init_arm64(void *state, const u8 *key); asmlinkage void poly1305_blocks(void *state, const u8 *src, u32 len, u32 hibit); asmlinkage void poly1305_blocks_neon(void *state, const u8 *src, u32 len, u32 hibit); -asmlinkage void poly1305_emit(void *state, __le32 *digest, const u32 *nonce); +asmlinkage void poly1305_emit(void *state, u8 *digest, const u32 *nonce); static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); @@ -162,9 +162,6 @@ EXPORT_SYMBOL(poly1305_update_arch); void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) { - __le32 digest[4]; - u64 f = 0; - if (unlikely(dctx->buflen)) { dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, @@ -172,18 +169,7 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) poly1305_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); } - poly1305_emit(&dctx->h, digest, dctx->s); - - /* mac = (h + s) % (2^128) */ - f = (f >> 32) + le32_to_cpu(digest[0]); - put_unaligned_le32(f, dst); - f = (f >> 32) + le32_to_cpu(digest[1]); - put_unaligned_le32(f, dst + 4); - f = (f >> 32) + le32_to_cpu(digest[2]); - put_unaligned_le32(f, dst + 8); - f = (f >> 32) + le32_to_cpu(digest[3]); - put_unaligned_le32(f, dst + 12); - + poly1305_emit(&dctx->h, dst, dctx->s); *dctx = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL(poly1305_final_arch); diff --git a/arch/mips/crypto/poly1305-glue.c b/arch/mips/crypto/poly1305-glue.c index b37d29cf5d0a8..fc881b46d9111 100644 --- a/arch/mips/crypto/poly1305-glue.c +++ b/arch/mips/crypto/poly1305-glue.c @@ -15,7 +15,7 @@ asmlinkage void poly1305_init_mips(void *state, const u8 *key); asmlinkage void poly1305_blocks_mips(void *state, const u8 *src, u32 len, u32 hibit); -asmlinkage void poly1305_emit_mips(void *state, __le32 *digest, const u32 *nonce); +asmlinkage void poly1305_emit_mips(void *state, u8 *digest, const u32 *nonce); void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 *key) { @@ -134,9 +134,6 @@ EXPORT_SYMBOL(poly1305_update_arch); void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) { - __le32 digest[4]; - u64 f = 0; - if (unlikely(dctx->buflen)) { dctx->buf[dctx->buflen++] = 1; memset(dctx->buf + dctx->buflen, 0, @@ -144,18 +141,7 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) poly1305_blocks_mips(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); } - poly1305_emit_mips(&dctx->h, digest, dctx->s); - - /* mac = (h + s) % (2^128) */ - f = (f >> 32) + le32_to_cpu(digest[0]); - put_unaligned_le32(f, dst); - f = (f >> 32) + le32_to_cpu(digest[1]); - put_unaligned_le32(f, dst + 4); - f = (f >> 32) + le32_to_cpu(digest[2]); - put_unaligned_le32(f, dst + 8); - f = (f >> 32) + le32_to_cpu(digest[3]); - put_unaligned_le32(f, dst + 12); - + poly1305_emit_mips(&dctx->h, dst, dctx->s); *dctx = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL(poly1305_final_arch); diff --git a/arch/x86/Makefile b/arch/x86/Makefile index dcbed7e3a0700..800e049850ec1 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -234,9 +234,10 @@ avx2_instr :=$(call as-instr,vpbroadcastb %xmm0$(comma)%ymm1,-DCONFIG_AS_AVX2=1) avx512_instr :=$(call as-instr,vpmovm2b %k1$(comma)%zmm5,-DCONFIG_AS_AVX512=1) sha1_ni_instr :=$(call as-instr,sha1msg1 %xmm0$(comma)%xmm1,-DCONFIG_AS_SHA1_NI=1) sha256_ni_instr :=$(call as-instr,sha256msg1 %xmm0$(comma)%xmm1,-DCONFIG_AS_SHA256_NI=1) +adx_instr := $(call as-instr,adox %r10$(comma)%r10,-DCONFIG_AS_ADX=1) -KBUILD_AFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(avx512_instr) $(sha1_ni_instr) $(sha256_ni_instr) -KBUILD_CFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(avx512_instr) $(sha1_ni_instr) $(sha256_ni_instr) +KBUILD_AFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(avx512_instr) $(sha1_ni_instr) $(sha256_ni_instr) $(adx_instr) +KBUILD_CFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr) $(avx512_instr) $(sha1_ni_instr) $(sha256_ni_instr) $(adx_instr) KBUILD_LDFLAGS := -m elf_$(UTS_MACHINE) diff --git a/arch/x86/crypto/.gitignore b/arch/x86/crypto/.gitignore new file mode 100644 index 0000000000000..30be0400a4392 --- /dev/null +++ b/arch/x86/crypto/.gitignore @@ -0,0 +1 @@ +poly1305-x86_64-cryptogams.S diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 958440eae27ec..8c2e9eadee8a0 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -11,6 +11,7 @@ avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\ avx512_supported :=$(call as-instr,vpmovm2b %k1$(comma)%zmm5,yes,no) sha1_ni_supported :=$(call as-instr,sha1msg1 %xmm0$(comma)%xmm1,yes,no) sha256_ni_supported :=$(call as-instr,sha256msg1 %xmm0$(comma)%xmm1,yes,no) +adx_supported := $(call as-instr,adox %r10$(comma)%r10,yes,no) obj-$(CONFIG_CRYPTO_GLUE_HELPER_X86) += glue_helper.o @@ -39,7 +40,11 @@ obj-$(CONFIG_CRYPTO_AEGIS128_AESNI_SSE2) += aegis128-aesni.o obj-$(CONFIG_CRYPTO_NHPOLY1305_SSE2) += nhpoly1305-sse2.o obj-$(CONFIG_CRYPTO_NHPOLY1305_AVX2) += nhpoly1305-avx2.o -obj-$(CONFIG_CRYPTO_CURVE25519_X86) += curve25519-x86_64.o + +# These modules require the assembler to support ADX. +ifeq ($(adx_supported),yes) + obj-$(CONFIG_CRYPTO_CURVE25519_X86) += curve25519-x86_64.o +endif # These modules require assembler to support AVX. ifeq ($(avx_supported),yes) @@ -73,6 +78,10 @@ aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o blake2s-x86_64-y := blake2s-core.o blake2s-glue.o +poly1305-x86_64-y := poly1305-x86_64-cryptogams.o poly1305_glue.o +ifneq ($(CONFIG_CRYPTO_POLY1305_X86_64),) +targets += poly1305-x86_64-cryptogams.S +endif ifeq ($(avx_supported),yes) camellia-aesni-avx-x86_64-y := camellia-aesni-avx-asm_64.o \ @@ -101,10 +110,8 @@ aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o aesni-intel-$(CONFIG_64BIT) += aesni-intel_avx-x86_64.o aes_ctrby8_avx-x86_64.o ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o -poly1305-x86_64-y := poly1305-sse2-x86_64.o poly1305_glue.o ifeq ($(avx2_supported),yes) sha1-ssse3-y += sha1_avx2_x86_64_asm.o -poly1305-x86_64-y += poly1305-avx2-x86_64.o endif ifeq ($(sha1_ni_supported),yes) sha1-ssse3-y += sha1_ni_asm.o @@ -118,3 +125,8 @@ sha256-ssse3-y += sha256_ni_asm.o endif sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o crct10dif-pclmul-y := crct10dif-pcl-asm_64.o crct10dif-pclmul_glue.o + +quiet_cmd_perlasm = PERLASM $@ + cmd_perlasm = $(PERL) $< > $@ +$(obj)/%.S: $(src)/%.pl FORCE + $(call if_changed,perlasm) diff --git a/arch/x86/crypto/curve25519-x86_64.c b/arch/x86/crypto/curve25519-x86_64.c index eec7d2d242396..8a17621f7d3a3 100644 --- a/arch/x86/crypto/curve25519-x86_64.c +++ b/arch/x86/crypto/curve25519-x86_64.c @@ -1,8 +1,7 @@ -// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +// SPDX-License-Identifier: GPL-2.0 OR MIT /* - * Copyright (c) 2017 Armando Faz . All Rights Reserved. - * Copyright (C) 2018-2019 Jason A. Donenfeld . All Rights Reserved. - * Copyright (C) 2018 Samuel Neves . All Rights Reserved. + * Copyright (C) 2020 Jason A. Donenfeld . All Rights Reserved. + * Copyright (c) 2016-2020 INRIA, CMU and Microsoft Corporation */ #include @@ -16,2337 +15,1378 @@ #include #include -static __ro_after_init DEFINE_STATIC_KEY_FALSE(curve25519_use_bmi2); -static __ro_after_init DEFINE_STATIC_KEY_FALSE(curve25519_use_adx); - -enum { NUM_WORDS_ELTFP25519 = 4 }; -typedef __aligned(32) u64 eltfp25519_1w[NUM_WORDS_ELTFP25519]; -typedef __aligned(32) u64 eltfp25519_1w_buffer[2 * NUM_WORDS_ELTFP25519]; - -#define mul_eltfp25519_1w_adx(c, a, b) do { \ - mul_256x256_integer_adx(m.buffer, a, b); \ - red_eltfp25519_1w_adx(c, m.buffer); \ -} while (0) - -#define mul_eltfp25519_1w_bmi2(c, a, b) do { \ - mul_256x256_integer_bmi2(m.buffer, a, b); \ - red_eltfp25519_1w_bmi2(c, m.buffer); \ -} while (0) - -#define sqr_eltfp25519_1w_adx(a) do { \ - sqr_256x256_integer_adx(m.buffer, a); \ - red_eltfp25519_1w_adx(a, m.buffer); \ -} while (0) - -#define sqr_eltfp25519_1w_bmi2(a) do { \ - sqr_256x256_integer_bmi2(m.buffer, a); \ - red_eltfp25519_1w_bmi2(a, m.buffer); \ -} while (0) - -#define mul_eltfp25519_2w_adx(c, a, b) do { \ - mul2_256x256_integer_adx(m.buffer, a, b); \ - red_eltfp25519_2w_adx(c, m.buffer); \ -} while (0) - -#define mul_eltfp25519_2w_bmi2(c, a, b) do { \ - mul2_256x256_integer_bmi2(m.buffer, a, b); \ - red_eltfp25519_2w_bmi2(c, m.buffer); \ -} while (0) - -#define sqr_eltfp25519_2w_adx(a) do { \ - sqr2_256x256_integer_adx(m.buffer, a); \ - red_eltfp25519_2w_adx(a, m.buffer); \ -} while (0) - -#define sqr_eltfp25519_2w_bmi2(a) do { \ - sqr2_256x256_integer_bmi2(m.buffer, a); \ - red_eltfp25519_2w_bmi2(a, m.buffer); \ -} while (0) - -#define sqrn_eltfp25519_1w_adx(a, times) do { \ - int ____counter = (times); \ - while (____counter-- > 0) \ - sqr_eltfp25519_1w_adx(a); \ -} while (0) - -#define sqrn_eltfp25519_1w_bmi2(a, times) do { \ - int ____counter = (times); \ - while (____counter-- > 0) \ - sqr_eltfp25519_1w_bmi2(a); \ -} while (0) - -#define copy_eltfp25519_1w(C, A) do { \ - (C)[0] = (A)[0]; \ - (C)[1] = (A)[1]; \ - (C)[2] = (A)[2]; \ - (C)[3] = (A)[3]; \ -} while (0) - -#define setzero_eltfp25519_1w(C) do { \ - (C)[0] = 0; \ - (C)[1] = 0; \ - (C)[2] = 0; \ - (C)[3] = 0; \ -} while (0) - -__aligned(32) static const u64 table_ladder_8k[252 * NUM_WORDS_ELTFP25519] = { - /* 1 */ 0xfffffffffffffff3UL, 0xffffffffffffffffUL, - 0xffffffffffffffffUL, 0x5fffffffffffffffUL, - /* 2 */ 0x6b8220f416aafe96UL, 0x82ebeb2b4f566a34UL, - 0xd5a9a5b075a5950fUL, 0x5142b2cf4b2488f4UL, - /* 3 */ 0x6aaebc750069680cUL, 0x89cf7820a0f99c41UL, - 0x2a58d9183b56d0f4UL, 0x4b5aca80e36011a4UL, - /* 4 */ 0x329132348c29745dUL, 0xf4a2e616e1642fd7UL, - 0x1e45bb03ff67bc34UL, 0x306912d0f42a9b4aUL, - /* 5 */ 0xff886507e6af7154UL, 0x04f50e13dfeec82fUL, - 0xaa512fe82abab5ceUL, 0x174e251a68d5f222UL, - /* 6 */ 0xcf96700d82028898UL, 0x1743e3370a2c02c5UL, - 0x379eec98b4e86eaaUL, 0x0c59888a51e0482eUL, - /* 7 */ 0xfbcbf1d699b5d189UL, 0xacaef0d58e9fdc84UL, - 0xc1c20d06231f7614UL, 0x2938218da274f972UL, - /* 8 */ 0xf6af49beff1d7f18UL, 0xcc541c22387ac9c2UL, - 0x96fcc9ef4015c56bUL, 0x69c1627c690913a9UL, - /* 9 */ 0x7a86fd2f4733db0eUL, 0xfdb8c4f29e087de9UL, - 0x095e4b1a8ea2a229UL, 0x1ad7a7c829b37a79UL, - /* 10 */ 0x342d89cad17ea0c0UL, 0x67bedda6cced2051UL, - 0x19ca31bf2bb42f74UL, 0x3df7b4c84980acbbUL, - /* 11 */ 0xa8c6444dc80ad883UL, 0xb91e440366e3ab85UL, - 0xc215cda00164f6d8UL, 0x3d867c6ef247e668UL, - /* 12 */ 0xc7dd582bcc3e658cUL, 0xfd2c4748ee0e5528UL, - 0xa0fd9b95cc9f4f71UL, 0x7529d871b0675ddfUL, - /* 13 */ 0xb8f568b42d3cbd78UL, 0x1233011b91f3da82UL, - 0x2dce6ccd4a7c3b62UL, 0x75e7fc8e9e498603UL, - /* 14 */ 0x2f4f13f1fcd0b6ecUL, 0xf1a8ca1f29ff7a45UL, - 0xc249c1a72981e29bUL, 0x6ebe0dbb8c83b56aUL, - /* 15 */ 0x7114fa8d170bb222UL, 0x65a2dcd5bf93935fUL, - 0xbdc41f68b59c979aUL, 0x2f0eef79a2ce9289UL, - /* 16 */ 0x42ecbf0c083c37ceUL, 0x2930bc09ec496322UL, - 0xf294b0c19cfeac0dUL, 0x3780aa4bedfabb80UL, - /* 17 */ 0x56c17d3e7cead929UL, 0xe7cb4beb2e5722c5UL, - 0x0ce931732dbfe15aUL, 0x41b883c7621052f8UL, - /* 18 */ 0xdbf75ca0c3d25350UL, 0x2936be086eb1e351UL, - 0xc936e03cb4a9b212UL, 0x1d45bf82322225aaUL, - /* 19 */ 0xe81ab1036a024cc5UL, 0xe212201c304c9a72UL, - 0xc5d73fba6832b1fcUL, 0x20ffdb5a4d839581UL, - /* 20 */ 0xa283d367be5d0fadUL, 0x6c2b25ca8b164475UL, - 0x9d4935467caaf22eUL, 0x5166408eee85ff49UL, - /* 21 */ 0x3c67baa2fab4e361UL, 0xb3e433c67ef35cefUL, - 0x5259729241159b1cUL, 0x6a621892d5b0ab33UL, - /* 22 */ 0x20b74a387555cdcbUL, 0x532aa10e1208923fUL, - 0xeaa17b7762281dd1UL, 0x61ab3443f05c44bfUL, - /* 23 */ 0x257a6c422324def8UL, 0x131c6c1017e3cf7fUL, - 0x23758739f630a257UL, 0x295a407a01a78580UL, - /* 24 */ 0xf8c443246d5da8d9UL, 0x19d775450c52fa5dUL, - 0x2afcfc92731bf83dUL, 0x7d10c8e81b2b4700UL, - /* 25 */ 0xc8e0271f70baa20bUL, 0x993748867ca63957UL, - 0x5412efb3cb7ed4bbUL, 0x3196d36173e62975UL, - /* 26 */ 0xde5bcad141c7dffcUL, 0x47cc8cd2b395c848UL, - 0xa34cd942e11af3cbUL, 0x0256dbf2d04ecec2UL, - /* 27 */ 0x875ab7e94b0e667fUL, 0xcad4dd83c0850d10UL, - 0x47f12e8f4e72c79fUL, 0x5f1a87bb8c85b19bUL, - /* 28 */ 0x7ae9d0b6437f51b8UL, 0x12c7ce5518879065UL, - 0x2ade09fe5cf77aeeUL, 0x23a05a2f7d2c5627UL, - /* 29 */ 0x5908e128f17c169aUL, 0xf77498dd8ad0852dUL, - 0x74b4c4ceab102f64UL, 0x183abadd10139845UL, - /* 30 */ 0xb165ba8daa92aaacUL, 0xd5c5ef9599386705UL, - 0xbe2f8f0cf8fc40d1UL, 0x2701e635ee204514UL, - /* 31 */ 0x629fa80020156514UL, 0xf223868764a8c1ceUL, - 0x5b894fff0b3f060eUL, 0x60d9944cf708a3faUL, - /* 32 */ 0xaeea001a1c7a201fUL, 0xebf16a633ee2ce63UL, - 0x6f7709594c7a07e1UL, 0x79b958150d0208cbUL, - /* 33 */ 0x24b55e5301d410e7UL, 0xe3a34edff3fdc84dUL, - 0xd88768e4904032d8UL, 0x131384427b3aaeecUL, - /* 34 */ 0x8405e51286234f14UL, 0x14dc4739adb4c529UL, - 0xb8a2b5b250634ffdUL, 0x2fe2a94ad8a7ff93UL, - /* 35 */ 0xec5c57efe843faddUL, 0x2843ce40f0bb9918UL, - 0xa4b561d6cf3d6305UL, 0x743629bde8fb777eUL, - /* 36 */ 0x343edd46bbaf738fUL, 0xed981828b101a651UL, - 0xa401760b882c797aUL, 0x1fc223e28dc88730UL, - /* 37 */ 0x48604e91fc0fba0eUL, 0xb637f78f052c6fa4UL, - 0x91ccac3d09e9239cUL, 0x23f7eed4437a687cUL, - /* 38 */ 0x5173b1118d9bd800UL, 0x29d641b63189d4a7UL, - 0xfdbf177988bbc586UL, 0x2959894fcad81df5UL, - /* 39 */ 0xaebc8ef3b4bbc899UL, 0x4148995ab26992b9UL, - 0x24e20b0134f92cfbUL, 0x40d158894a05dee8UL, - /* 40 */ 0x46b00b1185af76f6UL, 0x26bac77873187a79UL, - 0x3dc0bf95ab8fff5fUL, 0x2a608bd8945524d7UL, - /* 41 */ 0x26449588bd446302UL, 0x7c4bc21c0388439cUL, - 0x8e98a4f383bd11b2UL, 0x26218d7bc9d876b9UL, - /* 42 */ 0xe3081542997c178aUL, 0x3c2d29a86fb6606fUL, - 0x5c217736fa279374UL, 0x7dde05734afeb1faUL, - /* 43 */ 0x3bf10e3906d42babUL, 0xe4f7803e1980649cUL, - 0xe6053bf89595bf7aUL, 0x394faf38da245530UL, - /* 44 */ 0x7a8efb58896928f4UL, 0xfbc778e9cc6a113cUL, - 0x72670ce330af596fUL, 0x48f222a81d3d6cf7UL, - /* 45 */ 0xf01fce410d72caa7UL, 0x5a20ecc7213b5595UL, - 0x7bc21165c1fa1483UL, 0x07f89ae31da8a741UL, - /* 46 */ 0x05d2c2b4c6830ff9UL, 0xd43e330fc6316293UL, - 0xa5a5590a96d3a904UL, 0x705edb91a65333b6UL, - /* 47 */ 0x048ee15e0bb9a5f7UL, 0x3240cfca9e0aaf5dUL, - 0x8f4b71ceedc4a40bUL, 0x621c0da3de544a6dUL, - /* 48 */ 0x92872836a08c4091UL, 0xce8375b010c91445UL, - 0x8a72eb524f276394UL, 0x2667fcfa7ec83635UL, - /* 49 */ 0x7f4c173345e8752aUL, 0x061b47feee7079a5UL, - 0x25dd9afa9f86ff34UL, 0x3780cef5425dc89cUL, - /* 50 */ 0x1a46035a513bb4e9UL, 0x3e1ef379ac575adaUL, - 0xc78c5f1c5fa24b50UL, 0x321a967634fd9f22UL, - /* 51 */ 0x946707b8826e27faUL, 0x3dca84d64c506fd0UL, - 0xc189218075e91436UL, 0x6d9284169b3b8484UL, - /* 52 */ 0x3a67e840383f2ddfUL, 0x33eec9a30c4f9b75UL, - 0x3ec7c86fa783ef47UL, 0x26ec449fbac9fbc4UL, - /* 53 */ 0x5c0f38cba09b9e7dUL, 0x81168cc762a3478cUL, - 0x3e23b0d306fc121cUL, 0x5a238aa0a5efdcddUL, - /* 54 */ 0x1ba26121c4ea43ffUL, 0x36f8c77f7c8832b5UL, - 0x88fbea0b0adcf99aUL, 0x5ca9938ec25bebf9UL, - /* 55 */ 0xd5436a5e51fccda0UL, 0x1dbc4797c2cd893bUL, - 0x19346a65d3224a08UL, 0x0f5034e49b9af466UL, - /* 56 */ 0xf23c3967a1e0b96eUL, 0xe58b08fa867a4d88UL, - 0xfb2fabc6a7341679UL, 0x2a75381eb6026946UL, - /* 57 */ 0xc80a3be4c19420acUL, 0x66b1f6c681f2b6dcUL, - 0x7cf7036761e93388UL, 0x25abbbd8a660a4c4UL, - /* 58 */ 0x91ea12ba14fd5198UL, 0x684950fc4a3cffa9UL, - 0xf826842130f5ad28UL, 0x3ea988f75301a441UL, - /* 59 */ 0xc978109a695f8c6fUL, 0x1746eb4a0530c3f3UL, - 0x444d6d77b4459995UL, 0x75952b8c054e5cc7UL, - /* 60 */ 0xa3703f7915f4d6aaUL, 0x66c346202f2647d8UL, - 0xd01469df811d644bUL, 0x77fea47d81a5d71fUL, - /* 61 */ 0xc5e9529ef57ca381UL, 0x6eeeb4b9ce2f881aUL, - 0xb6e91a28e8009bd6UL, 0x4b80be3e9afc3fecUL, - /* 62 */ 0x7e3773c526aed2c5UL, 0x1b4afcb453c9a49dUL, - 0xa920bdd7baffb24dUL, 0x7c54699f122d400eUL, - /* 63 */ 0xef46c8e14fa94bc8UL, 0xe0b074ce2952ed5eUL, - 0xbea450e1dbd885d5UL, 0x61b68649320f712cUL, - /* 64 */ 0x8a485f7309ccbdd1UL, 0xbd06320d7d4d1a2dUL, - 0x25232973322dbef4UL, 0x445dc4758c17f770UL, - /* 65 */ 0xdb0434177cc8933cUL, 0xed6fe82175ea059fUL, - 0x1efebefdc053db34UL, 0x4adbe867c65daf99UL, - /* 66 */ 0x3acd71a2a90609dfUL, 0xe5e991856dd04050UL, - 0x1ec69b688157c23cUL, 0x697427f6885cfe4dUL, - /* 67 */ 0xd7be7b9b65e1a851UL, 0xa03d28d522c536ddUL, - 0x28399d658fd2b645UL, 0x49e5b7e17c2641e1UL, - /* 68 */ 0x6f8c3a98700457a4UL, 0x5078f0a25ebb6778UL, - 0xd13c3ccbc382960fUL, 0x2e003258a7df84b1UL, - /* 69 */ 0x8ad1f39be6296a1cUL, 0xc1eeaa652a5fbfb2UL, - 0x33ee0673fd26f3cbUL, 0x59256173a69d2cccUL, - /* 70 */ 0x41ea07aa4e18fc41UL, 0xd9fc19527c87a51eUL, - 0xbdaacb805831ca6fUL, 0x445b652dc916694fUL, - /* 71 */ 0xce92a3a7f2172315UL, 0x1edc282de11b9964UL, - 0xa1823aafe04c314aUL, 0x790a2d94437cf586UL, - /* 72 */ 0x71c447fb93f6e009UL, 0x8922a56722845276UL, - 0xbf70903b204f5169UL, 0x2f7a89891ba319feUL, - /* 73 */ 0x02a08eb577e2140cUL, 0xed9a4ed4427bdcf4UL, - 0x5253ec44e4323cd1UL, 0x3e88363c14e9355bUL, - /* 74 */ 0xaa66c14277110b8cUL, 0x1ae0391610a23390UL, - 0x2030bd12c93fc2a2UL, 0x3ee141579555c7abUL, - /* 75 */ 0x9214de3a6d6e7d41UL, 0x3ccdd88607f17efeUL, - 0x674f1288f8e11217UL, 0x5682250f329f93d0UL, - /* 76 */ 0x6cf00b136d2e396eUL, 0x6e4cf86f1014debfUL, - 0x5930b1b5bfcc4e83UL, 0x047069b48aba16b6UL, - /* 77 */ 0x0d4ce4ab69b20793UL, 0xb24db91a97d0fb9eUL, - 0xcdfa50f54e00d01dUL, 0x221b1085368bddb5UL, - /* 78 */ 0xe7e59468b1e3d8d2UL, 0x53c56563bd122f93UL, - 0xeee8a903e0663f09UL, 0x61efa662cbbe3d42UL, - /* 79 */ 0x2cf8ddddde6eab2aUL, 0x9bf80ad51435f231UL, - 0x5deadacec9f04973UL, 0x29275b5d41d29b27UL, - /* 80 */ 0xcfde0f0895ebf14fUL, 0xb9aab96b054905a7UL, - 0xcae80dd9a1c420fdUL, 0x0a63bf2f1673bbc7UL, - /* 81 */ 0x092f6e11958fbc8cUL, 0x672a81e804822fadUL, - 0xcac8351560d52517UL, 0x6f3f7722c8f192f8UL, - /* 82 */ 0xf8ba90ccc2e894b7UL, 0x2c7557a438ff9f0dUL, - 0x894d1d855ae52359UL, 0x68e122157b743d69UL, - /* 83 */ 0xd87e5570cfb919f3UL, 0x3f2cdecd95798db9UL, - 0x2121154710c0a2ceUL, 0x3c66a115246dc5b2UL, - /* 84 */ 0xcbedc562294ecb72UL, 0xba7143c36a280b16UL, - 0x9610c2efd4078b67UL, 0x6144735d946a4b1eUL, - /* 85 */ 0x536f111ed75b3350UL, 0x0211db8c2041d81bUL, - 0xf93cb1000e10413cUL, 0x149dfd3c039e8876UL, - /* 86 */ 0xd479dde46b63155bUL, 0xb66e15e93c837976UL, - 0xdafde43b1f13e038UL, 0x5fafda1a2e4b0b35UL, - /* 87 */ 0x3600bbdf17197581UL, 0x3972050bbe3cd2c2UL, - 0x5938906dbdd5be86UL, 0x34fce5e43f9b860fUL, - /* 88 */ 0x75a8a4cd42d14d02UL, 0x828dabc53441df65UL, - 0x33dcabedd2e131d3UL, 0x3ebad76fb814d25fUL, - /* 89 */ 0xd4906f566f70e10fUL, 0x5d12f7aa51690f5aUL, - 0x45adb16e76cefcf2UL, 0x01f768aead232999UL, - /* 90 */ 0x2b6cc77b6248febdUL, 0x3cd30628ec3aaffdUL, - 0xce1c0b80d4ef486aUL, 0x4c3bff2ea6f66c23UL, - /* 91 */ 0x3f2ec4094aeaeb5fUL, 0x61b19b286e372ca7UL, - 0x5eefa966de2a701dUL, 0x23b20565de55e3efUL, - /* 92 */ 0xe301ca5279d58557UL, 0x07b2d4ce27c2874fUL, - 0xa532cd8a9dcf1d67UL, 0x2a52fee23f2bff56UL, - /* 93 */ 0x8624efb37cd8663dUL, 0xbbc7ac20ffbd7594UL, - 0x57b85e9c82d37445UL, 0x7b3052cb86a6ec66UL, - /* 94 */ 0x3482f0ad2525e91eUL, 0x2cb68043d28edca0UL, - 0xaf4f6d052e1b003aUL, 0x185f8c2529781b0aUL, - /* 95 */ 0xaa41de5bd80ce0d6UL, 0x9407b2416853e9d6UL, - 0x563ec36e357f4c3aUL, 0x4cc4b8dd0e297bceUL, - /* 96 */ 0xa2fc1a52ffb8730eUL, 0x1811f16e67058e37UL, - 0x10f9a366cddf4ee1UL, 0x72f4a0c4a0b9f099UL, - /* 97 */ 0x8c16c06f663f4ea7UL, 0x693b3af74e970fbaUL, - 0x2102e7f1d69ec345UL, 0x0ba53cbc968a8089UL, - /* 98 */ 0xca3d9dc7fea15537UL, 0x4c6824bb51536493UL, - 0xb9886314844006b1UL, 0x40d2a72ab454cc60UL, - /* 99 */ 0x5936a1b712570975UL, 0x91b9d648debda657UL, - 0x3344094bb64330eaUL, 0x006ba10d12ee51d0UL, - /* 100 */ 0x19228468f5de5d58UL, 0x0eb12f4c38cc05b0UL, - 0xa1039f9dd5601990UL, 0x4502d4ce4fff0e0bUL, - /* 101 */ 0xeb2054106837c189UL, 0xd0f6544c6dd3b93cUL, - 0x40727064c416d74fUL, 0x6e15c6114b502ef0UL, - /* 102 */ 0x4df2a398cfb1a76bUL, 0x11256c7419f2f6b1UL, - 0x4a497962066e6043UL, 0x705b3aab41355b44UL, - /* 103 */ 0x365ef536d797b1d8UL, 0x00076bd622ddf0dbUL, - 0x3bbf33b0e0575a88UL, 0x3777aa05c8e4ca4dUL, - /* 104 */ 0x392745c85578db5fUL, 0x6fda4149dbae5ae2UL, - 0xb1f0b00b8adc9867UL, 0x09963437d36f1da3UL, - /* 105 */ 0x7e824e90a5dc3853UL, 0xccb5f6641f135cbdUL, - 0x6736d86c87ce8fccUL, 0x625f3ce26604249fUL, - /* 106 */ 0xaf8ac8059502f63fUL, 0x0c05e70a2e351469UL, - 0x35292e9c764b6305UL, 0x1a394360c7e23ac3UL, - /* 107 */ 0xd5c6d53251183264UL, 0x62065abd43c2b74fUL, - 0xb5fbf5d03b973f9bUL, 0x13a3da3661206e5eUL, - /* 108 */ 0xc6bd5837725d94e5UL, 0x18e30912205016c5UL, - 0x2088ce1570033c68UL, 0x7fba1f495c837987UL, - /* 109 */ 0x5a8c7423f2f9079dUL, 0x1735157b34023fc5UL, - 0xe4f9b49ad2fab351UL, 0x6691ff72c878e33cUL, - /* 110 */ 0x122c2adedc5eff3eUL, 0xf8dd4bf1d8956cf4UL, - 0xeb86205d9e9e5bdaUL, 0x049b92b9d975c743UL, - /* 111 */ 0xa5379730b0f6c05aUL, 0x72a0ffacc6f3a553UL, - 0xb0032c34b20dcd6dUL, 0x470e9dbc88d5164aUL, - /* 112 */ 0xb19cf10ca237c047UL, 0xb65466711f6c81a2UL, - 0xb3321bd16dd80b43UL, 0x48c14f600c5fbe8eUL, - /* 113 */ 0x66451c264aa6c803UL, 0xb66e3904a4fa7da6UL, - 0xd45f19b0b3128395UL, 0x31602627c3c9bc10UL, - /* 114 */ 0x3120dc4832e4e10dUL, 0xeb20c46756c717f7UL, - 0x00f52e3f67280294UL, 0x566d4fc14730c509UL, - /* 115 */ 0x7e3a5d40fd837206UL, 0xc1e926dc7159547aUL, - 0x216730fba68d6095UL, 0x22e8c3843f69cea7UL, - /* 116 */ 0x33d074e8930e4b2bUL, 0xb6e4350e84d15816UL, - 0x5534c26ad6ba2365UL, 0x7773c12f89f1f3f3UL, - /* 117 */ 0x8cba404da57962aaUL, 0x5b9897a81999ce56UL, - 0x508e862f121692fcUL, 0x3a81907fa093c291UL, - /* 118 */ 0x0dded0ff4725a510UL, 0x10d8cc10673fc503UL, - 0x5b9d151c9f1f4e89UL, 0x32a5c1d5cb09a44cUL, - /* 119 */ 0x1e0aa442b90541fbUL, 0x5f85eb7cc1b485dbUL, - 0xbee595ce8a9df2e5UL, 0x25e496c722422236UL, - /* 120 */ 0x5edf3c46cd0fe5b9UL, 0x34e75a7ed2a43388UL, - 0xe488de11d761e352UL, 0x0e878a01a085545cUL, - /* 121 */ 0xba493c77e021bb04UL, 0x2b4d1843c7df899aUL, - 0x9ea37a487ae80d67UL, 0x67a9958011e41794UL, - /* 122 */ 0x4b58051a6697b065UL, 0x47e33f7d8d6ba6d4UL, - 0xbb4da8d483ca46c1UL, 0x68becaa181c2db0dUL, - /* 123 */ 0x8d8980e90b989aa5UL, 0xf95eb14a2c93c99bUL, - 0x51c6c7c4796e73a2UL, 0x6e228363b5efb569UL, - /* 124 */ 0xc6bbc0b02dd624c8UL, 0x777eb47dec8170eeUL, - 0x3cde15a004cfafa9UL, 0x1dc6bc087160bf9bUL, - /* 125 */ 0x2e07e043eec34002UL, 0x18e9fc677a68dc7fUL, - 0xd8da03188bd15b9aUL, 0x48fbc3bb00568253UL, - /* 126 */ 0x57547d4cfb654ce1UL, 0xd3565b82a058e2adUL, - 0xf63eaf0bbf154478UL, 0x47531ef114dfbb18UL, - /* 127 */ 0xe1ec630a4278c587UL, 0x5507d546ca8e83f3UL, - 0x85e135c63adc0c2bUL, 0x0aa7efa85682844eUL, - /* 128 */ 0x72691ba8b3e1f615UL, 0x32b4e9701fbe3ffaUL, - 0x97b6d92e39bb7868UL, 0x2cfe53dea02e39e8UL, - /* 129 */ 0x687392cd85cd52b0UL, 0x27ff66c910e29831UL, - 0x97134556a9832d06UL, 0x269bb0360a84f8a0UL, - /* 130 */ 0x706e55457643f85cUL, 0x3734a48c9b597d1bUL, - 0x7aee91e8c6efa472UL, 0x5cd6abc198a9d9e0UL, - /* 131 */ 0x0e04de06cb3ce41aUL, 0xd8c6eb893402e138UL, - 0x904659bb686e3772UL, 0x7215c371746ba8c8UL, - /* 132 */ 0xfd12a97eeae4a2d9UL, 0x9514b7516394f2c5UL, - 0x266fd5809208f294UL, 0x5c847085619a26b9UL, - /* 133 */ 0x52985410fed694eaUL, 0x3c905b934a2ed254UL, - 0x10bb47692d3be467UL, 0x063b3d2d69e5e9e1UL, - /* 134 */ 0x472726eedda57debUL, 0xefb6c4ae10f41891UL, - 0x2b1641917b307614UL, 0x117c554fc4f45b7cUL, - /* 135 */ 0xc07cf3118f9d8812UL, 0x01dbd82050017939UL, - 0xd7e803f4171b2827UL, 0x1015e87487d225eaUL, - /* 136 */ 0xc58de3fed23acc4dUL, 0x50db91c294a7be2dUL, - 0x0b94d43d1c9cf457UL, 0x6b1640fa6e37524aUL, - /* 137 */ 0x692f346c5fda0d09UL, 0x200b1c59fa4d3151UL, - 0xb8c46f760777a296UL, 0x4b38395f3ffdfbcfUL, - /* 138 */ 0x18d25e00be54d671UL, 0x60d50582bec8aba6UL, - 0x87ad8f263b78b982UL, 0x50fdf64e9cda0432UL, - /* 139 */ 0x90f567aac578dcf0UL, 0xef1e9b0ef2a3133bUL, - 0x0eebba9242d9de71UL, 0x15473c9bf03101c7UL, - /* 140 */ 0x7c77e8ae56b78095UL, 0xb678e7666e6f078eUL, - 0x2da0b9615348ba1fUL, 0x7cf931c1ff733f0bUL, - /* 141 */ 0x26b357f50a0a366cUL, 0xe9708cf42b87d732UL, - 0xc13aeea5f91cb2c0UL, 0x35d90c991143bb4cUL, - /* 142 */ 0x47c1c404a9a0d9dcUL, 0x659e58451972d251UL, - 0x3875a8c473b38c31UL, 0x1fbd9ed379561f24UL, - /* 143 */ 0x11fabc6fd41ec28dUL, 0x7ef8dfe3cd2a2dcaUL, - 0x72e73b5d8c404595UL, 0x6135fa4954b72f27UL, - /* 144 */ 0xccfc32a2de24b69cUL, 0x3f55698c1f095d88UL, - 0xbe3350ed5ac3f929UL, 0x5e9bf806ca477eebUL, - /* 145 */ 0xe9ce8fb63c309f68UL, 0x5376f63565e1f9f4UL, - 0xd1afcfb35a6393f1UL, 0x6632a1ede5623506UL, - /* 146 */ 0x0b7d6c390c2ded4cUL, 0x56cb3281df04cb1fUL, - 0x66305a1249ecc3c7UL, 0x5d588b60a38ca72aUL, - /* 147 */ 0xa6ecbf78e8e5f42dUL, 0x86eeb44b3c8a3eecUL, - 0xec219c48fbd21604UL, 0x1aaf1af517c36731UL, - /* 148 */ 0xc306a2836769bde7UL, 0x208280622b1e2adbUL, - 0x8027f51ffbff94a6UL, 0x76cfa1ce1124f26bUL, - /* 149 */ 0x18eb00562422abb6UL, 0xf377c4d58f8c29c3UL, - 0x4dbbc207f531561aUL, 0x0253b7f082128a27UL, - /* 150 */ 0x3d1f091cb62c17e0UL, 0x4860e1abd64628a9UL, - 0x52d17436309d4253UL, 0x356f97e13efae576UL, - /* 151 */ 0xd351e11aa150535bUL, 0x3e6b45bb1dd878ccUL, - 0x0c776128bed92c98UL, 0x1d34ae93032885b8UL, - /* 152 */ 0x4ba0488ca85ba4c3UL, 0x985348c33c9ce6ceUL, - 0x66124c6f97bda770UL, 0x0f81a0290654124aUL, - /* 153 */ 0x9ed09ca6569b86fdUL, 0x811009fd18af9a2dUL, - 0xff08d03f93d8c20aUL, 0x52a148199faef26bUL, - /* 154 */ 0x3e03f9dc2d8d1b73UL, 0x4205801873961a70UL, - 0xc0d987f041a35970UL, 0x07aa1f15a1c0d549UL, - /* 155 */ 0xdfd46ce08cd27224UL, 0x6d0a024f934e4239UL, - 0x808a7a6399897b59UL, 0x0a4556e9e13d95a2UL, - /* 156 */ 0xd21a991fe9c13045UL, 0x9b0e8548fe7751b8UL, - 0x5da643cb4bf30035UL, 0x77db28d63940f721UL, - /* 157 */ 0xfc5eeb614adc9011UL, 0x5229419ae8c411ebUL, - 0x9ec3e7787d1dcf74UL, 0x340d053e216e4cb5UL, - /* 158 */ 0xcac7af39b48df2b4UL, 0xc0faec2871a10a94UL, - 0x140a69245ca575edUL, 0x0cf1c37134273a4cUL, - /* 159 */ 0xc8ee306ac224b8a5UL, 0x57eaee7ccb4930b0UL, - 0xa1e806bdaacbe74fUL, 0x7d9a62742eeb657dUL, - /* 160 */ 0x9eb6b6ef546c4830UL, 0x885cca1fddb36e2eUL, - 0xe6b9f383ef0d7105UL, 0x58654fef9d2e0412UL, - /* 161 */ 0xa905c4ffbe0e8e26UL, 0x942de5df9b31816eUL, - 0x497d723f802e88e1UL, 0x30684dea602f408dUL, - /* 162 */ 0x21e5a278a3e6cb34UL, 0xaefb6e6f5b151dc4UL, - 0xb30b8e049d77ca15UL, 0x28c3c9cf53b98981UL, - /* 163 */ 0x287fb721556cdd2aUL, 0x0d317ca897022274UL, - 0x7468c7423a543258UL, 0x4a7f11464eb5642fUL, - /* 164 */ 0xa237a4774d193aa6UL, 0xd865986ea92129a1UL, - 0x24c515ecf87c1a88UL, 0x604003575f39f5ebUL, - /* 165 */ 0x47b9f189570a9b27UL, 0x2b98cede465e4b78UL, - 0x026df551dbb85c20UL, 0x74fcd91047e21901UL, - /* 166 */ 0x13e2a90a23c1bfa3UL, 0x0cb0074e478519f6UL, - 0x5ff1cbbe3af6cf44UL, 0x67fe5438be812dbeUL, - /* 167 */ 0xd13cf64fa40f05b0UL, 0x054dfb2f32283787UL, - 0x4173915b7f0d2aeaUL, 0x482f144f1f610d4eUL, - /* 168 */ 0xf6210201b47f8234UL, 0x5d0ae1929e70b990UL, - 0xdcd7f455b049567cUL, 0x7e93d0f1f0916f01UL, - /* 169 */ 0xdd79cbf18a7db4faUL, 0xbe8391bf6f74c62fUL, - 0x027145d14b8291bdUL, 0x585a73ea2cbf1705UL, - /* 170 */ 0x485ca03e928a0db2UL, 0x10fc01a5742857e7UL, - 0x2f482edbd6d551a7UL, 0x0f0433b5048fdb8aUL, - /* 171 */ 0x60da2e8dd7dc6247UL, 0x88b4c9d38cd4819aUL, - 0x13033ac001f66697UL, 0x273b24fe3b367d75UL, - /* 172 */ 0xc6e8f66a31b3b9d4UL, 0x281514a494df49d5UL, - 0xd1726fdfc8b23da7UL, 0x4b3ae7d103dee548UL, - /* 173 */ 0xc6256e19ce4b9d7eUL, 0xff5c5cf186e3c61cUL, - 0xacc63ca34b8ec145UL, 0x74621888fee66574UL, - /* 174 */ 0x956f409645290a1eUL, 0xef0bf8e3263a962eUL, - 0xed6a50eb5ec2647bUL, 0x0694283a9dca7502UL, - /* 175 */ 0x769b963643a2dcd1UL, 0x42b7c8ea09fc5353UL, - 0x4f002aee13397eabUL, 0x63005e2c19b7d63aUL, - /* 176 */ 0xca6736da63023beaUL, 0x966c7f6db12a99b7UL, - 0xace09390c537c5e1UL, 0x0b696063a1aa89eeUL, - /* 177 */ 0xebb03e97288c56e5UL, 0x432a9f9f938c8be8UL, - 0xa6a5a93d5b717f71UL, 0x1a5fb4c3e18f9d97UL, - /* 178 */ 0x1c94e7ad1c60cdceUL, 0xee202a43fc02c4a0UL, - 0x8dafe4d867c46a20UL, 0x0a10263c8ac27b58UL, - /* 179 */ 0xd0dea9dfe4432a4aUL, 0x856af87bbe9277c5UL, - 0xce8472acc212c71aUL, 0x6f151b6d9bbb1e91UL, - /* 180 */ 0x26776c527ceed56aUL, 0x7d211cb7fbf8faecUL, - 0x37ae66a6fd4609ccUL, 0x1f81b702d2770c42UL, - /* 181 */ 0x2fb0b057eac58392UL, 0xe1dd89fe29744e9dUL, - 0xc964f8eb17beb4f8UL, 0x29571073c9a2d41eUL, - /* 182 */ 0xa948a18981c0e254UL, 0x2df6369b65b22830UL, - 0xa33eb2d75fcfd3c6UL, 0x078cd6ec4199a01fUL, - /* 183 */ 0x4a584a41ad900d2fUL, 0x32142b78e2c74c52UL, - 0x68c4e8338431c978UL, 0x7f69ea9008689fc2UL, - /* 184 */ 0x52f2c81e46a38265UL, 0xfd78072d04a832fdUL, - 0x8cd7d5fa25359e94UL, 0x4de71b7454cc29d2UL, - /* 185 */ 0x42eb60ad1eda6ac9UL, 0x0aad37dfdbc09c3aUL, - 0x81004b71e33cc191UL, 0x44e6be345122803cUL, - /* 186 */ 0x03fe8388ba1920dbUL, 0xf5d57c32150db008UL, - 0x49c8c4281af60c29UL, 0x21edb518de701aeeUL, - /* 187 */ 0x7fb63e418f06dc99UL, 0xa4460d99c166d7b8UL, - 0x24dd5248ce520a83UL, 0x5ec3ad712b928358UL, - /* 188 */ 0x15022a5fbd17930fUL, 0xa4f64a77d82570e3UL, - 0x12bc8d6915783712UL, 0x498194c0fc620abbUL, - /* 189 */ 0x38a2d9d255686c82UL, 0x785c6bd9193e21f0UL, - 0xe4d5c81ab24a5484UL, 0x56307860b2e20989UL, - /* 190 */ 0x429d55f78b4d74c4UL, 0x22f1834643350131UL, - 0x1e60c24598c71fffUL, 0x59f2f014979983efUL, - /* 191 */ 0x46a47d56eb494a44UL, 0x3e22a854d636a18eUL, - 0xb346e15274491c3bUL, 0x2ceafd4e5390cde7UL, - /* 192 */ 0xba8a8538be0d6675UL, 0x4b9074bb50818e23UL, - 0xcbdab89085d304c3UL, 0x61a24fe0e56192c4UL, - /* 193 */ 0xcb7615e6db525bcbUL, 0xdd7d8c35a567e4caUL, - 0xe6b4153acafcdd69UL, 0x2d668e097f3c9766UL, - /* 194 */ 0xa57e7e265ce55ef0UL, 0x5d9f4e527cd4b967UL, - 0xfbc83606492fd1e5UL, 0x090d52beb7c3f7aeUL, - /* 195 */ 0x09b9515a1e7b4d7cUL, 0x1f266a2599da44c0UL, - 0xa1c49548e2c55504UL, 0x7ef04287126f15ccUL, - /* 196 */ 0xfed1659dbd30ef15UL, 0x8b4ab9eec4e0277bUL, - 0x884d6236a5df3291UL, 0x1fd96ea6bf5cf788UL, - /* 197 */ 0x42a161981f190d9aUL, 0x61d849507e6052c1UL, - 0x9fe113bf285a2cd5UL, 0x7c22d676dbad85d8UL, - /* 198 */ 0x82e770ed2bfbd27dUL, 0x4c05b2ece996f5a5UL, - 0xcd40a9c2b0900150UL, 0x5895319213d9bf64UL, - /* 199 */ 0xe7cc5d703fea2e08UL, 0xb50c491258e2188cUL, - 0xcce30baa48205bf0UL, 0x537c659ccfa32d62UL, - /* 200 */ 0x37b6623a98cfc088UL, 0xfe9bed1fa4d6aca4UL, - 0x04d29b8e56a8d1b0UL, 0x725f71c40b519575UL, - /* 201 */ 0x28c7f89cd0339ce6UL, 0x8367b14469ddc18bUL, - 0x883ada83a6a1652cUL, 0x585f1974034d6c17UL, - /* 202 */ 0x89cfb266f1b19188UL, 0xe63b4863e7c35217UL, - 0xd88c9da6b4c0526aUL, 0x3e035c9df0954635UL, - /* 203 */ 0xdd9d5412fb45de9dUL, 0xdd684532e4cff40dUL, - 0x4b5c999b151d671cUL, 0x2d8c2cc811e7f690UL, - /* 204 */ 0x7f54be1d90055d40UL, 0xa464c5df464aaf40UL, - 0x33979624f0e917beUL, 0x2c018dc527356b30UL, - /* 205 */ 0xa5415024e330b3d4UL, 0x73ff3d96691652d3UL, - 0x94ec42c4ef9b59f1UL, 0x0747201618d08e5aUL, - /* 206 */ 0x4d6ca48aca411c53UL, 0x66415f2fcfa66119UL, - 0x9c4dd40051e227ffUL, 0x59810bc09a02f7ebUL, - /* 207 */ 0x2a7eb171b3dc101dUL, 0x441c5ab99ffef68eUL, - 0x32025c9b93b359eaUL, 0x5e8ce0a71e9d112fUL, - /* 208 */ 0xbfcccb92429503fdUL, 0xd271ba752f095d55UL, - 0x345ead5e972d091eUL, 0x18c8df11a83103baUL, - /* 209 */ 0x90cd949a9aed0f4cUL, 0xc5d1f4cb6660e37eUL, - 0xb8cac52d56c52e0bUL, 0x6e42e400c5808e0dUL, - /* 210 */ 0xa3b46966eeaefd23UL, 0x0c4f1f0be39ecdcaUL, - 0x189dc8c9d683a51dUL, 0x51f27f054c09351bUL, - /* 211 */ 0x4c487ccd2a320682UL, 0x587ea95bb3df1c96UL, - 0xc8ccf79e555cb8e8UL, 0x547dc829a206d73dUL, - /* 212 */ 0xb822a6cd80c39b06UL, 0xe96d54732000d4c6UL, - 0x28535b6f91463b4dUL, 0x228f4660e2486e1dUL, - /* 213 */ 0x98799538de8d3abfUL, 0x8cd8330045ebca6eUL, - 0x79952a008221e738UL, 0x4322e1a7535cd2bbUL, - /* 214 */ 0xb114c11819d1801cUL, 0x2016e4d84f3f5ec7UL, - 0xdd0e2df409260f4cUL, 0x5ec362c0ae5f7266UL, - /* 215 */ 0xc0462b18b8b2b4eeUL, 0x7cc8d950274d1afbUL, - 0xf25f7105436b02d2UL, 0x43bbf8dcbff9ccd3UL, - /* 216 */ 0xb6ad1767a039e9dfUL, 0xb0714da8f69d3583UL, - 0x5e55fa18b42931f5UL, 0x4ed5558f33c60961UL, - /* 217 */ 0x1fe37901c647a5ddUL, 0x593ddf1f8081d357UL, - 0x0249a4fd813fd7a6UL, 0x69acca274e9caf61UL, - /* 218 */ 0x047ba3ea330721c9UL, 0x83423fc20e7e1ea0UL, - 0x1df4c0af01314a60UL, 0x09a62dab89289527UL, - /* 219 */ 0xa5b325a49cc6cb00UL, 0xe94b5dc654b56cb6UL, - 0x3be28779adc994a0UL, 0x4296e8f8ba3a4aadUL, - /* 220 */ 0x328689761e451eabUL, 0x2e4d598bff59594aUL, - 0x49b96853d7a7084aUL, 0x4980a319601420a8UL, - /* 221 */ 0x9565b9e12f552c42UL, 0x8a5318db7100fe96UL, - 0x05c90b4d43add0d7UL, 0x538b4cd66a5d4edaUL, - /* 222 */ 0xf4e94fc3e89f039fUL, 0x592c9af26f618045UL, - 0x08a36eb5fd4b9550UL, 0x25fffaf6c2ed1419UL, - /* 223 */ 0x34434459cc79d354UL, 0xeeecbfb4b1d5476bUL, - 0xddeb34a061615d99UL, 0x5129cecceb64b773UL, - /* 224 */ 0xee43215894993520UL, 0x772f9c7cf14c0b3bUL, - 0xd2e2fce306bedad5UL, 0x715f42b546f06a97UL, - /* 225 */ 0x434ecdceda5b5f1aUL, 0x0da17115a49741a9UL, - 0x680bd77c73edad2eUL, 0x487c02354edd9041UL, - /* 226 */ 0xb8efeff3a70ed9c4UL, 0x56a32aa3e857e302UL, - 0xdf3a68bd48a2a5a0UL, 0x07f650b73176c444UL, - /* 227 */ 0xe38b9b1626e0ccb1UL, 0x79e053c18b09fb36UL, - 0x56d90319c9f94964UL, 0x1ca941e7ac9ff5c4UL, - /* 228 */ 0x49c4df29162fa0bbUL, 0x8488cf3282b33305UL, - 0x95dfda14cabb437dUL, 0x3391f78264d5ad86UL, - /* 229 */ 0x729ae06ae2b5095dUL, 0xd58a58d73259a946UL, - 0xe9834262d13921edUL, 0x27fedafaa54bb592UL, - /* 230 */ 0xa99dc5b829ad48bbUL, 0x5f025742499ee260UL, - 0x802c8ecd5d7513fdUL, 0x78ceb3ef3f6dd938UL, - /* 231 */ 0xc342f44f8a135d94UL, 0x7b9edb44828cdda3UL, - 0x9436d11a0537cfe7UL, 0x5064b164ec1ab4c8UL, - /* 232 */ 0x7020eccfd37eb2fcUL, 0x1f31ea3ed90d25fcUL, - 0x1b930d7bdfa1bb34UL, 0x5344467a48113044UL, - /* 233 */ 0x70073170f25e6dfbUL, 0xe385dc1a50114cc8UL, - 0x2348698ac8fc4f00UL, 0x2a77a55284dd40d8UL, - /* 234 */ 0xfe06afe0c98c6ce4UL, 0xc235df96dddfd6e4UL, - 0x1428d01e33bf1ed3UL, 0x785768ec9300bdafUL, - /* 235 */ 0x9702e57a91deb63bUL, 0x61bdb8bfe5ce8b80UL, - 0x645b426f3d1d58acUL, 0x4804a82227a557bcUL, - /* 236 */ 0x8e57048ab44d2601UL, 0x68d6501a4b3a6935UL, - 0xc39c9ec3f9e1c293UL, 0x4172f257d4de63e2UL, - /* 237 */ 0xd368b450330c6401UL, 0x040d3017418f2391UL, - 0x2c34bb6090b7d90dUL, 0x16f649228fdfd51fUL, - /* 238 */ 0xbea6818e2b928ef5UL, 0xe28ccf91cdc11e72UL, - 0x594aaa68e77a36cdUL, 0x313034806c7ffd0fUL, - /* 239 */ 0x8a9d27ac2249bd65UL, 0x19a3b464018e9512UL, - 0xc26ccff352b37ec7UL, 0x056f68341d797b21UL, - /* 240 */ 0x5e79d6757efd2327UL, 0xfabdbcb6553afe15UL, - 0xd3e7222c6eaf5a60UL, 0x7046c76d4dae743bUL, - /* 241 */ 0x660be872b18d4a55UL, 0x19992518574e1496UL, - 0xc103053a302bdcbbUL, 0x3ed8e9800b218e8eUL, - /* 242 */ 0x7b0b9239fa75e03eUL, 0xefe9fb684633c083UL, - 0x98a35fbe391a7793UL, 0x6065510fe2d0fe34UL, - /* 243 */ 0x55cb668548abad0cUL, 0xb4584548da87e527UL, - 0x2c43ecea0107c1ddUL, 0x526028809372de35UL, - /* 244 */ 0x3415c56af9213b1fUL, 0x5bee1a4d017e98dbUL, - 0x13f6b105b5cf709bUL, 0x5ff20e3482b29ab6UL, - /* 245 */ 0x0aa29c75cc2e6c90UL, 0xfc7d73ca3a70e206UL, - 0x899fc38fc4b5c515UL, 0x250386b124ffc207UL, - /* 246 */ 0x54ea28d5ae3d2b56UL, 0x9913149dd6de60ceUL, - 0x16694fc58f06d6c1UL, 0x46b23975eb018fc7UL, - /* 247 */ 0x470a6a0fb4b7b4e2UL, 0x5d92475a8f7253deUL, - 0xabeee5b52fbd3adbUL, 0x7fa20801a0806968UL, - /* 248 */ 0x76f3faf19f7714d2UL, 0xb3e840c12f4660c3UL, - 0x0fb4cd8df212744eUL, 0x4b065a251d3a2dd2UL, - /* 249 */ 0x5cebde383d77cd4aUL, 0x6adf39df882c9cb1UL, - 0xa2dd242eb09af759UL, 0x3147c0e50e5f6422UL, - /* 250 */ 0x164ca5101d1350dbUL, 0xf8d13479c33fc962UL, - 0xe640ce4d13e5da08UL, 0x4bdee0c45061f8baUL, - /* 251 */ 0xd7c46dc1a4edb1c9UL, 0x5514d7b6437fd98aUL, - 0x58942f6bb2a1c00bUL, 0x2dffb2ab1d70710eUL, - /* 252 */ 0xccdfcf2fc18b6d68UL, 0xa8ebcba8b7806167UL, - 0x980697f95e2937e3UL, 0x02fbba1cd0126e8cUL -}; - -/* c is two 512-bit products: c0[0:7]=a0[0:3]*b0[0:3] and c1[8:15]=a1[4:7]*b1[4:7] - * a is two 256-bit integers: a0[0:3] and a1[4:7] - * b is two 256-bit integers: b0[0:3] and b1[4:7] - */ -static void mul2_256x256_integer_adx(u64 *const c, const u64 *const a, - const u64 *const b) -{ - asm volatile( - "xorl %%r14d, %%r14d ;" - "movq (%1), %%rdx; " /* A[0] */ - "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ - "xorl %%r10d, %%r10d ;" - "movq %%r8, (%0) ;" - "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ - "adox %%r10, %%r15 ;" - "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ - "adox %%r8, %%rax ;" - "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ - "adox %%r10, %%rbx ;" - /******************************************/ - "adox %%r14, %%rcx ;" - - "movq 8(%1), %%rdx; " /* A[1] */ - "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ - "adox %%r15, %%r8 ;" - "movq %%r8, 8(%0) ;" - "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ - "adox %%r10, %%r9 ;" - "adcx %%r9, %%rax ;" - "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ - "adox %%r8, %%r11 ;" - "adcx %%r11, %%rbx ;" - "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ - "adox %%r10, %%r13 ;" - "adcx %%r13, %%rcx ;" - /******************************************/ - "adox %%r14, %%r15 ;" - "adcx %%r14, %%r15 ;" - - "movq 16(%1), %%rdx; " /* A[2] */ - "xorl %%r10d, %%r10d ;" - "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ - "adox %%rax, %%r8 ;" - "movq %%r8, 16(%0) ;" - "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ - "adox %%r10, %%r9 ;" - "adcx %%r9, %%rbx ;" - "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ - "adox %%r8, %%r11 ;" - "adcx %%r11, %%rcx ;" - "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ - "adox %%r10, %%r13 ;" - "adcx %%r13, %%r15 ;" - /******************************************/ - "adox %%r14, %%rax ;" - "adcx %%r14, %%rax ;" - - "movq 24(%1), %%rdx; " /* A[3] */ - "xorl %%r10d, %%r10d ;" - "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ - "adox %%rbx, %%r8 ;" - "movq %%r8, 24(%0) ;" - "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ - "adox %%r10, %%r9 ;" - "adcx %%r9, %%rcx ;" - "movq %%rcx, 32(%0) ;" - "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ - "adox %%r8, %%r11 ;" - "adcx %%r11, %%r15 ;" - "movq %%r15, 40(%0) ;" - "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ - "adox %%r10, %%r13 ;" - "adcx %%r13, %%rax ;" - "movq %%rax, 48(%0) ;" - /******************************************/ - "adox %%r14, %%rbx ;" - "adcx %%r14, %%rbx ;" - "movq %%rbx, 56(%0) ;" - - "movq 32(%1), %%rdx; " /* C[0] */ - "mulx 32(%2), %%r8, %%r15; " /* C[0]*D[0] */ - "xorl %%r10d, %%r10d ;" - "movq %%r8, 64(%0);" - "mulx 40(%2), %%r10, %%rax; " /* C[0]*D[1] */ - "adox %%r10, %%r15 ;" - "mulx 48(%2), %%r8, %%rbx; " /* C[0]*D[2] */ - "adox %%r8, %%rax ;" - "mulx 56(%2), %%r10, %%rcx; " /* C[0]*D[3] */ - "adox %%r10, %%rbx ;" - /******************************************/ - "adox %%r14, %%rcx ;" - - "movq 40(%1), %%rdx; " /* C[1] */ - "xorl %%r10d, %%r10d ;" - "mulx 32(%2), %%r8, %%r9; " /* C[1]*D[0] */ - "adox %%r15, %%r8 ;" - "movq %%r8, 72(%0);" - "mulx 40(%2), %%r10, %%r11; " /* C[1]*D[1] */ - "adox %%r10, %%r9 ;" - "adcx %%r9, %%rax ;" - "mulx 48(%2), %%r8, %%r13; " /* C[1]*D[2] */ - "adox %%r8, %%r11 ;" - "adcx %%r11, %%rbx ;" - "mulx 56(%2), %%r10, %%r15; " /* C[1]*D[3] */ - "adox %%r10, %%r13 ;" - "adcx %%r13, %%rcx ;" - /******************************************/ - "adox %%r14, %%r15 ;" - "adcx %%r14, %%r15 ;" - - "movq 48(%1), %%rdx; " /* C[2] */ - "xorl %%r10d, %%r10d ;" - "mulx 32(%2), %%r8, %%r9; " /* C[2]*D[0] */ - "adox %%rax, %%r8 ;" - "movq %%r8, 80(%0);" - "mulx 40(%2), %%r10, %%r11; " /* C[2]*D[1] */ - "adox %%r10, %%r9 ;" - "adcx %%r9, %%rbx ;" - "mulx 48(%2), %%r8, %%r13; " /* C[2]*D[2] */ - "adox %%r8, %%r11 ;" - "adcx %%r11, %%rcx ;" - "mulx 56(%2), %%r10, %%rax; " /* C[2]*D[3] */ - "adox %%r10, %%r13 ;" - "adcx %%r13, %%r15 ;" - /******************************************/ - "adox %%r14, %%rax ;" - "adcx %%r14, %%rax ;" - - "movq 56(%1), %%rdx; " /* C[3] */ - "xorl %%r10d, %%r10d ;" - "mulx 32(%2), %%r8, %%r9; " /* C[3]*D[0] */ - "adox %%rbx, %%r8 ;" - "movq %%r8, 88(%0);" - "mulx 40(%2), %%r10, %%r11; " /* C[3]*D[1] */ - "adox %%r10, %%r9 ;" - "adcx %%r9, %%rcx ;" - "movq %%rcx, 96(%0) ;" - "mulx 48(%2), %%r8, %%r13; " /* C[3]*D[2] */ - "adox %%r8, %%r11 ;" - "adcx %%r11, %%r15 ;" - "movq %%r15, 104(%0) ;" - "mulx 56(%2), %%r10, %%rbx; " /* C[3]*D[3] */ - "adox %%r10, %%r13 ;" - "adcx %%r13, %%rax ;" - "movq %%rax, 112(%0) ;" - /******************************************/ - "adox %%r14, %%rbx ;" - "adcx %%r14, %%rbx ;" - "movq %%rbx, 120(%0) ;" - : - : "r"(c), "r"(a), "r"(b) - : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", - "%r10", "%r11", "%r13", "%r14", "%r15"); -} - -static void mul2_256x256_integer_bmi2(u64 *const c, const u64 *const a, - const u64 *const b) +static __always_inline u64 eq_mask(u64 a, u64 b) { - asm volatile( - "movq (%1), %%rdx; " /* A[0] */ - "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ - "movq %%r8, (%0) ;" - "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ - "addq %%r10, %%r15 ;" - "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ - "adcq %%r8, %%rax ;" - "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ - "adcq %%r10, %%rbx ;" - /******************************************/ - "adcq $0, %%rcx ;" - - "movq 8(%1), %%rdx; " /* A[1] */ - "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ - "addq %%r15, %%r8 ;" - "movq %%r8, 8(%0) ;" - "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ - "adcq %%r10, %%r9 ;" - "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ - "adcq %%r8, %%r11 ;" - "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ - "adcq %%r10, %%r13 ;" - /******************************************/ - "adcq $0, %%r15 ;" - - "addq %%r9, %%rax ;" - "adcq %%r11, %%rbx ;" - "adcq %%r13, %%rcx ;" - "adcq $0, %%r15 ;" - - "movq 16(%1), %%rdx; " /* A[2] */ - "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ - "addq %%rax, %%r8 ;" - "movq %%r8, 16(%0) ;" - "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ - "adcq %%r10, %%r9 ;" - "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ - "adcq %%r8, %%r11 ;" - "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ - "adcq %%r10, %%r13 ;" - /******************************************/ - "adcq $0, %%rax ;" - - "addq %%r9, %%rbx ;" - "adcq %%r11, %%rcx ;" - "adcq %%r13, %%r15 ;" - "adcq $0, %%rax ;" - - "movq 24(%1), %%rdx; " /* A[3] */ - "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ - "addq %%rbx, %%r8 ;" - "movq %%r8, 24(%0) ;" - "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ - "adcq %%r10, %%r9 ;" - "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ - "adcq %%r8, %%r11 ;" - "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ - "adcq %%r10, %%r13 ;" - /******************************************/ - "adcq $0, %%rbx ;" - - "addq %%r9, %%rcx ;" - "movq %%rcx, 32(%0) ;" - "adcq %%r11, %%r15 ;" - "movq %%r15, 40(%0) ;" - "adcq %%r13, %%rax ;" - "movq %%rax, 48(%0) ;" - "adcq $0, %%rbx ;" - "movq %%rbx, 56(%0) ;" - - "movq 32(%1), %%rdx; " /* C[0] */ - "mulx 32(%2), %%r8, %%r15; " /* C[0]*D[0] */ - "movq %%r8, 64(%0) ;" - "mulx 40(%2), %%r10, %%rax; " /* C[0]*D[1] */ - "addq %%r10, %%r15 ;" - "mulx 48(%2), %%r8, %%rbx; " /* C[0]*D[2] */ - "adcq %%r8, %%rax ;" - "mulx 56(%2), %%r10, %%rcx; " /* C[0]*D[3] */ - "adcq %%r10, %%rbx ;" - /******************************************/ - "adcq $0, %%rcx ;" - - "movq 40(%1), %%rdx; " /* C[1] */ - "mulx 32(%2), %%r8, %%r9; " /* C[1]*D[0] */ - "addq %%r15, %%r8 ;" - "movq %%r8, 72(%0) ;" - "mulx 40(%2), %%r10, %%r11; " /* C[1]*D[1] */ - "adcq %%r10, %%r9 ;" - "mulx 48(%2), %%r8, %%r13; " /* C[1]*D[2] */ - "adcq %%r8, %%r11 ;" - "mulx 56(%2), %%r10, %%r15; " /* C[1]*D[3] */ - "adcq %%r10, %%r13 ;" - /******************************************/ - "adcq $0, %%r15 ;" - - "addq %%r9, %%rax ;" - "adcq %%r11, %%rbx ;" - "adcq %%r13, %%rcx ;" - "adcq $0, %%r15 ;" - - "movq 48(%1), %%rdx; " /* C[2] */ - "mulx 32(%2), %%r8, %%r9; " /* C[2]*D[0] */ - "addq %%rax, %%r8 ;" - "movq %%r8, 80(%0) ;" - "mulx 40(%2), %%r10, %%r11; " /* C[2]*D[1] */ - "adcq %%r10, %%r9 ;" - "mulx 48(%2), %%r8, %%r13; " /* C[2]*D[2] */ - "adcq %%r8, %%r11 ;" - "mulx 56(%2), %%r10, %%rax; " /* C[2]*D[3] */ - "adcq %%r10, %%r13 ;" - /******************************************/ - "adcq $0, %%rax ;" - - "addq %%r9, %%rbx ;" - "adcq %%r11, %%rcx ;" - "adcq %%r13, %%r15 ;" - "adcq $0, %%rax ;" - - "movq 56(%1), %%rdx; " /* C[3] */ - "mulx 32(%2), %%r8, %%r9; " /* C[3]*D[0] */ - "addq %%rbx, %%r8 ;" - "movq %%r8, 88(%0) ;" - "mulx 40(%2), %%r10, %%r11; " /* C[3]*D[1] */ - "adcq %%r10, %%r9 ;" - "mulx 48(%2), %%r8, %%r13; " /* C[3]*D[2] */ - "adcq %%r8, %%r11 ;" - "mulx 56(%2), %%r10, %%rbx; " /* C[3]*D[3] */ - "adcq %%r10, %%r13 ;" - /******************************************/ - "adcq $0, %%rbx ;" - - "addq %%r9, %%rcx ;" - "movq %%rcx, 96(%0) ;" - "adcq %%r11, %%r15 ;" - "movq %%r15, 104(%0) ;" - "adcq %%r13, %%rax ;" - "movq %%rax, 112(%0) ;" - "adcq $0, %%rbx ;" - "movq %%rbx, 120(%0) ;" - : - : "r"(c), "r"(a), "r"(b) - : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", - "%r10", "%r11", "%r13", "%r15"); + u64 x = a ^ b; + u64 minus_x = ~x + (u64)1U; + u64 x_or_minus_x = x | minus_x; + u64 xnx = x_or_minus_x >> (u32)63U; + return xnx - (u64)1U; } -static void sqr2_256x256_integer_adx(u64 *const c, const u64 *const a) +static __always_inline u64 gte_mask(u64 a, u64 b) { - asm volatile( - "movq (%1), %%rdx ;" /* A[0] */ - "mulx 8(%1), %%r8, %%r14 ;" /* A[1]*A[0] */ - "xorl %%r15d, %%r15d;" - "mulx 16(%1), %%r9, %%r10 ;" /* A[2]*A[0] */ - "adcx %%r14, %%r9 ;" - "mulx 24(%1), %%rax, %%rcx ;" /* A[3]*A[0] */ - "adcx %%rax, %%r10 ;" - "movq 24(%1), %%rdx ;" /* A[3] */ - "mulx 8(%1), %%r11, %%rbx ;" /* A[1]*A[3] */ - "adcx %%rcx, %%r11 ;" - "mulx 16(%1), %%rax, %%r13 ;" /* A[2]*A[3] */ - "adcx %%rax, %%rbx ;" - "movq 8(%1), %%rdx ;" /* A[1] */ - "adcx %%r15, %%r13 ;" - "mulx 16(%1), %%rax, %%rcx ;" /* A[2]*A[1] */ - "movq $0, %%r14 ;" - /******************************************/ - "adcx %%r15, %%r14 ;" - - "xorl %%r15d, %%r15d;" - "adox %%rax, %%r10 ;" - "adcx %%r8, %%r8 ;" - "adox %%rcx, %%r11 ;" - "adcx %%r9, %%r9 ;" - "adox %%r15, %%rbx ;" - "adcx %%r10, %%r10 ;" - "adox %%r15, %%r13 ;" - "adcx %%r11, %%r11 ;" - "adox %%r15, %%r14 ;" - "adcx %%rbx, %%rbx ;" - "adcx %%r13, %%r13 ;" - "adcx %%r14, %%r14 ;" - - "movq (%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ - /*******************/ - "movq %%rax, 0(%0) ;" - "addq %%rcx, %%r8 ;" - "movq %%r8, 8(%0) ;" - "movq 8(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ - "adcq %%rax, %%r9 ;" - "movq %%r9, 16(%0) ;" - "adcq %%rcx, %%r10 ;" - "movq %%r10, 24(%0) ;" - "movq 16(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ - "adcq %%rax, %%r11 ;" - "movq %%r11, 32(%0) ;" - "adcq %%rcx, %%rbx ;" - "movq %%rbx, 40(%0) ;" - "movq 24(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ - "adcq %%rax, %%r13 ;" - "movq %%r13, 48(%0) ;" - "adcq %%rcx, %%r14 ;" - "movq %%r14, 56(%0) ;" - - - "movq 32(%1), %%rdx ;" /* B[0] */ - "mulx 40(%1), %%r8, %%r14 ;" /* B[1]*B[0] */ - "xorl %%r15d, %%r15d;" - "mulx 48(%1), %%r9, %%r10 ;" /* B[2]*B[0] */ - "adcx %%r14, %%r9 ;" - "mulx 56(%1), %%rax, %%rcx ;" /* B[3]*B[0] */ - "adcx %%rax, %%r10 ;" - "movq 56(%1), %%rdx ;" /* B[3] */ - "mulx 40(%1), %%r11, %%rbx ;" /* B[1]*B[3] */ - "adcx %%rcx, %%r11 ;" - "mulx 48(%1), %%rax, %%r13 ;" /* B[2]*B[3] */ - "adcx %%rax, %%rbx ;" - "movq 40(%1), %%rdx ;" /* B[1] */ - "adcx %%r15, %%r13 ;" - "mulx 48(%1), %%rax, %%rcx ;" /* B[2]*B[1] */ - "movq $0, %%r14 ;" - /******************************************/ - "adcx %%r15, %%r14 ;" - - "xorl %%r15d, %%r15d;" - "adox %%rax, %%r10 ;" - "adcx %%r8, %%r8 ;" - "adox %%rcx, %%r11 ;" - "adcx %%r9, %%r9 ;" - "adox %%r15, %%rbx ;" - "adcx %%r10, %%r10 ;" - "adox %%r15, %%r13 ;" - "adcx %%r11, %%r11 ;" - "adox %%r15, %%r14 ;" - "adcx %%rbx, %%rbx ;" - "adcx %%r13, %%r13 ;" - "adcx %%r14, %%r14 ;" - - "movq 32(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* B[0]^2 */ - /*******************/ - "movq %%rax, 64(%0) ;" - "addq %%rcx, %%r8 ;" - "movq %%r8, 72(%0) ;" - "movq 40(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* B[1]^2 */ - "adcq %%rax, %%r9 ;" - "movq %%r9, 80(%0) ;" - "adcq %%rcx, %%r10 ;" - "movq %%r10, 88(%0) ;" - "movq 48(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* B[2]^2 */ - "adcq %%rax, %%r11 ;" - "movq %%r11, 96(%0) ;" - "adcq %%rcx, %%rbx ;" - "movq %%rbx, 104(%0) ;" - "movq 56(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* B[3]^2 */ - "adcq %%rax, %%r13 ;" - "movq %%r13, 112(%0) ;" - "adcq %%rcx, %%r14 ;" - "movq %%r14, 120(%0) ;" - : - : "r"(c), "r"(a) - : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", - "%r10", "%r11", "%r13", "%r14", "%r15"); + u64 x = a; + u64 y = b; + u64 x_xor_y = x ^ y; + u64 x_sub_y = x - y; + u64 x_sub_y_xor_y = x_sub_y ^ y; + u64 q = x_xor_y | x_sub_y_xor_y; + u64 x_xor_q = x ^ q; + u64 x_xor_q_ = x_xor_q >> (u32)63U; + return x_xor_q_ - (u64)1U; } -static void sqr2_256x256_integer_bmi2(u64 *const c, const u64 *const a) +/* Computes the addition of four-element f1 with value in f2 + * and returns the carry (if any) */ +static inline u64 add_scalar(u64 *out, const u64 *f1, u64 f2) { - asm volatile( - "movq 8(%1), %%rdx ;" /* A[1] */ - "mulx (%1), %%r8, %%r9 ;" /* A[0]*A[1] */ - "mulx 16(%1), %%r10, %%r11 ;" /* A[2]*A[1] */ - "mulx 24(%1), %%rcx, %%r14 ;" /* A[3]*A[1] */ - - "movq 16(%1), %%rdx ;" /* A[2] */ - "mulx 24(%1), %%r15, %%r13 ;" /* A[3]*A[2] */ - "mulx (%1), %%rax, %%rdx ;" /* A[0]*A[2] */ - - "addq %%rax, %%r9 ;" - "adcq %%rdx, %%r10 ;" - "adcq %%rcx, %%r11 ;" - "adcq %%r14, %%r15 ;" - "adcq $0, %%r13 ;" - "movq $0, %%r14 ;" - "adcq $0, %%r14 ;" - - "movq (%1), %%rdx ;" /* A[0] */ - "mulx 24(%1), %%rax, %%rcx ;" /* A[0]*A[3] */ - - "addq %%rax, %%r10 ;" - "adcq %%rcx, %%r11 ;" - "adcq $0, %%r15 ;" - "adcq $0, %%r13 ;" - "adcq $0, %%r14 ;" - - "shldq $1, %%r13, %%r14 ;" - "shldq $1, %%r15, %%r13 ;" - "shldq $1, %%r11, %%r15 ;" - "shldq $1, %%r10, %%r11 ;" - "shldq $1, %%r9, %%r10 ;" - "shldq $1, %%r8, %%r9 ;" - "shlq $1, %%r8 ;" - - /*******************/ - "mulx %%rdx, %%rax, %%rcx ; " /* A[0]^2 */ - /*******************/ - "movq %%rax, 0(%0) ;" - "addq %%rcx, %%r8 ;" - "movq %%r8, 8(%0) ;" - "movq 8(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ; " /* A[1]^2 */ - "adcq %%rax, %%r9 ;" - "movq %%r9, 16(%0) ;" - "adcq %%rcx, %%r10 ;" - "movq %%r10, 24(%0) ;" - "movq 16(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ; " /* A[2]^2 */ - "adcq %%rax, %%r11 ;" - "movq %%r11, 32(%0) ;" - "adcq %%rcx, %%r15 ;" - "movq %%r15, 40(%0) ;" - "movq 24(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ; " /* A[3]^2 */ - "adcq %%rax, %%r13 ;" - "movq %%r13, 48(%0) ;" - "adcq %%rcx, %%r14 ;" - "movq %%r14, 56(%0) ;" - - "movq 40(%1), %%rdx ;" /* B[1] */ - "mulx 32(%1), %%r8, %%r9 ;" /* B[0]*B[1] */ - "mulx 48(%1), %%r10, %%r11 ;" /* B[2]*B[1] */ - "mulx 56(%1), %%rcx, %%r14 ;" /* B[3]*B[1] */ - - "movq 48(%1), %%rdx ;" /* B[2] */ - "mulx 56(%1), %%r15, %%r13 ;" /* B[3]*B[2] */ - "mulx 32(%1), %%rax, %%rdx ;" /* B[0]*B[2] */ - - "addq %%rax, %%r9 ;" - "adcq %%rdx, %%r10 ;" - "adcq %%rcx, %%r11 ;" - "adcq %%r14, %%r15 ;" - "adcq $0, %%r13 ;" - "movq $0, %%r14 ;" - "adcq $0, %%r14 ;" - - "movq 32(%1), %%rdx ;" /* B[0] */ - "mulx 56(%1), %%rax, %%rcx ;" /* B[0]*B[3] */ - - "addq %%rax, %%r10 ;" - "adcq %%rcx, %%r11 ;" - "adcq $0, %%r15 ;" - "adcq $0, %%r13 ;" - "adcq $0, %%r14 ;" - - "shldq $1, %%r13, %%r14 ;" - "shldq $1, %%r15, %%r13 ;" - "shldq $1, %%r11, %%r15 ;" - "shldq $1, %%r10, %%r11 ;" - "shldq $1, %%r9, %%r10 ;" - "shldq $1, %%r8, %%r9 ;" - "shlq $1, %%r8 ;" - - /*******************/ - "mulx %%rdx, %%rax, %%rcx ; " /* B[0]^2 */ - /*******************/ - "movq %%rax, 64(%0) ;" - "addq %%rcx, %%r8 ;" - "movq %%r8, 72(%0) ;" - "movq 40(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ; " /* B[1]^2 */ - "adcq %%rax, %%r9 ;" - "movq %%r9, 80(%0) ;" - "adcq %%rcx, %%r10 ;" - "movq %%r10, 88(%0) ;" - "movq 48(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ; " /* B[2]^2 */ - "adcq %%rax, %%r11 ;" - "movq %%r11, 96(%0) ;" - "adcq %%rcx, %%r15 ;" - "movq %%r15, 104(%0) ;" - "movq 56(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ; " /* B[3]^2 */ - "adcq %%rax, %%r13 ;" - "movq %%r13, 112(%0) ;" - "adcq %%rcx, %%r14 ;" - "movq %%r14, 120(%0) ;" - : - : "r"(c), "r"(a) - : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", - "%r11", "%r13", "%r14", "%r15"); -} + u64 carry_r; -static void red_eltfp25519_2w_adx(u64 *const c, const u64 *const a) -{ asm volatile( - "movl $38, %%edx; " /* 2*c = 38 = 2^256 */ - "mulx 32(%1), %%r8, %%r10; " /* c*C[4] */ - "xorl %%ebx, %%ebx ;" - "adox (%1), %%r8 ;" - "mulx 40(%1), %%r9, %%r11; " /* c*C[5] */ - "adcx %%r10, %%r9 ;" - "adox 8(%1), %%r9 ;" - "mulx 48(%1), %%r10, %%rax; " /* c*C[6] */ - "adcx %%r11, %%r10 ;" - "adox 16(%1), %%r10 ;" - "mulx 56(%1), %%r11, %%rcx; " /* c*C[7] */ - "adcx %%rax, %%r11 ;" - "adox 24(%1), %%r11 ;" - /***************************************/ - "adcx %%rbx, %%rcx ;" - "adox %%rbx, %%rcx ;" - "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ - "adcx %%rcx, %%r8 ;" - "adcx %%rbx, %%r9 ;" - "movq %%r9, 8(%0) ;" - "adcx %%rbx, %%r10 ;" - "movq %%r10, 16(%0) ;" - "adcx %%rbx, %%r11 ;" - "movq %%r11, 24(%0) ;" - "mov $0, %%ecx ;" - "cmovc %%edx, %%ecx ;" - "addq %%rcx, %%r8 ;" - "movq %%r8, (%0) ;" - - "mulx 96(%1), %%r8, %%r10; " /* c*C[4] */ - "xorl %%ebx, %%ebx ;" - "adox 64(%1), %%r8 ;" - "mulx 104(%1), %%r9, %%r11; " /* c*C[5] */ - "adcx %%r10, %%r9 ;" - "adox 72(%1), %%r9 ;" - "mulx 112(%1), %%r10, %%rax; " /* c*C[6] */ - "adcx %%r11, %%r10 ;" - "adox 80(%1), %%r10 ;" - "mulx 120(%1), %%r11, %%rcx; " /* c*C[7] */ - "adcx %%rax, %%r11 ;" - "adox 88(%1), %%r11 ;" - /****************************************/ - "adcx %%rbx, %%rcx ;" - "adox %%rbx, %%rcx ;" - "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ - "adcx %%rcx, %%r8 ;" - "adcx %%rbx, %%r9 ;" - "movq %%r9, 40(%0) ;" - "adcx %%rbx, %%r10 ;" - "movq %%r10, 48(%0) ;" - "adcx %%rbx, %%r11 ;" - "movq %%r11, 56(%0) ;" - "mov $0, %%ecx ;" - "cmovc %%edx, %%ecx ;" - "addq %%rcx, %%r8 ;" - "movq %%r8, 32(%0) ;" - : - : "r"(c), "r"(a) - : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", - "%r10", "%r11"); -} + /* Clear registers to propagate the carry bit */ + " xor %%r8, %%r8;" + " xor %%r9, %%r9;" + " xor %%r10, %%r10;" + " xor %%r11, %%r11;" + " xor %1, %1;" + + /* Begin addition chain */ + " addq 0(%3), %0;" + " movq %0, 0(%2);" + " adcxq 8(%3), %%r8;" + " movq %%r8, 8(%2);" + " adcxq 16(%3), %%r9;" + " movq %%r9, 16(%2);" + " adcxq 24(%3), %%r10;" + " movq %%r10, 24(%2);" + + /* Return the carry bit in a register */ + " adcx %%r11, %1;" + : "+&r" (f2), "=&r" (carry_r) + : "r" (out), "r" (f1) + : "%r8", "%r9", "%r10", "%r11", "memory", "cc" + ); -static void red_eltfp25519_2w_bmi2(u64 *const c, const u64 *const a) -{ - asm volatile( - "movl $38, %%edx ; " /* 2*c = 38 = 2^256 */ - "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ - "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ - "addq %%r10, %%r9 ;" - "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ - "adcq %%r11, %%r10 ;" - "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ - "adcq %%rax, %%r11 ;" - /***************************************/ - "adcq $0, %%rcx ;" - "addq (%1), %%r8 ;" - "adcq 8(%1), %%r9 ;" - "adcq 16(%1), %%r10 ;" - "adcq 24(%1), %%r11 ;" - "adcq $0, %%rcx ;" - "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ - "addq %%rcx, %%r8 ;" - "adcq $0, %%r9 ;" - "movq %%r9, 8(%0) ;" - "adcq $0, %%r10 ;" - "movq %%r10, 16(%0) ;" - "adcq $0, %%r11 ;" - "movq %%r11, 24(%0) ;" - "mov $0, %%ecx ;" - "cmovc %%edx, %%ecx ;" - "addq %%rcx, %%r8 ;" - "movq %%r8, (%0) ;" - - "mulx 96(%1), %%r8, %%r10 ;" /* c*C[4] */ - "mulx 104(%1), %%r9, %%r11 ;" /* c*C[5] */ - "addq %%r10, %%r9 ;" - "mulx 112(%1), %%r10, %%rax ;" /* c*C[6] */ - "adcq %%r11, %%r10 ;" - "mulx 120(%1), %%r11, %%rcx ;" /* c*C[7] */ - "adcq %%rax, %%r11 ;" - /****************************************/ - "adcq $0, %%rcx ;" - "addq 64(%1), %%r8 ;" - "adcq 72(%1), %%r9 ;" - "adcq 80(%1), %%r10 ;" - "adcq 88(%1), %%r11 ;" - "adcq $0, %%rcx ;" - "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ - "addq %%rcx, %%r8 ;" - "adcq $0, %%r9 ;" - "movq %%r9, 40(%0) ;" - "adcq $0, %%r10 ;" - "movq %%r10, 48(%0) ;" - "adcq $0, %%r11 ;" - "movq %%r11, 56(%0) ;" - "mov $0, %%ecx ;" - "cmovc %%edx, %%ecx ;" - "addq %%rcx, %%r8 ;" - "movq %%r8, 32(%0) ;" - : - : "r"(c), "r"(a) - : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", - "%r11"); + return carry_r; } -static void mul_256x256_integer_adx(u64 *const c, const u64 *const a, - const u64 *const b) +/* Computes the field addition of two field elements */ +static inline void fadd(u64 *out, const u64 *f1, const u64 *f2) { asm volatile( - "movq (%1), %%rdx; " /* A[0] */ - "mulx (%2), %%r8, %%r9; " /* A[0]*B[0] */ - "xorl %%r10d, %%r10d ;" - "movq %%r8, (%0) ;" - "mulx 8(%2), %%r10, %%r11; " /* A[0]*B[1] */ - "adox %%r9, %%r10 ;" - "movq %%r10, 8(%0) ;" - "mulx 16(%2), %%r15, %%r13; " /* A[0]*B[2] */ - "adox %%r11, %%r15 ;" - "mulx 24(%2), %%r14, %%rdx; " /* A[0]*B[3] */ - "adox %%r13, %%r14 ;" - "movq $0, %%rax ;" - /******************************************/ - "adox %%rdx, %%rax ;" - - "movq 8(%1), %%rdx; " /* A[1] */ - "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ - "xorl %%r10d, %%r10d ;" - "adcx 8(%0), %%r8 ;" - "movq %%r8, 8(%0) ;" - "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ - "adox %%r9, %%r10 ;" - "adcx %%r15, %%r10 ;" - "movq %%r10, 16(%0) ;" - "mulx 16(%2), %%r15, %%r13; " /* A[1]*B[2] */ - "adox %%r11, %%r15 ;" - "adcx %%r14, %%r15 ;" - "movq $0, %%r8 ;" - "mulx 24(%2), %%r14, %%rdx; " /* A[1]*B[3] */ - "adox %%r13, %%r14 ;" - "adcx %%rax, %%r14 ;" - "movq $0, %%rax ;" - /******************************************/ - "adox %%rdx, %%rax ;" - "adcx %%r8, %%rax ;" - - "movq 16(%1), %%rdx; " /* A[2] */ - "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ - "xorl %%r10d, %%r10d ;" - "adcx 16(%0), %%r8 ;" - "movq %%r8, 16(%0) ;" - "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ - "adox %%r9, %%r10 ;" - "adcx %%r15, %%r10 ;" - "movq %%r10, 24(%0) ;" - "mulx 16(%2), %%r15, %%r13; " /* A[2]*B[2] */ - "adox %%r11, %%r15 ;" - "adcx %%r14, %%r15 ;" - "movq $0, %%r8 ;" - "mulx 24(%2), %%r14, %%rdx; " /* A[2]*B[3] */ - "adox %%r13, %%r14 ;" - "adcx %%rax, %%r14 ;" - "movq $0, %%rax ;" - /******************************************/ - "adox %%rdx, %%rax ;" - "adcx %%r8, %%rax ;" - - "movq 24(%1), %%rdx; " /* A[3] */ - "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ - "xorl %%r10d, %%r10d ;" - "adcx 24(%0), %%r8 ;" - "movq %%r8, 24(%0) ;" - "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ - "adox %%r9, %%r10 ;" - "adcx %%r15, %%r10 ;" - "movq %%r10, 32(%0) ;" - "mulx 16(%2), %%r15, %%r13; " /* A[3]*B[2] */ - "adox %%r11, %%r15 ;" - "adcx %%r14, %%r15 ;" - "movq %%r15, 40(%0) ;" - "movq $0, %%r8 ;" - "mulx 24(%2), %%r14, %%rdx; " /* A[3]*B[3] */ - "adox %%r13, %%r14 ;" - "adcx %%rax, %%r14 ;" - "movq %%r14, 48(%0) ;" - "movq $0, %%rax ;" - /******************************************/ - "adox %%rdx, %%rax ;" - "adcx %%r8, %%rax ;" - "movq %%rax, 56(%0) ;" - : - : "r"(c), "r"(a), "r"(b) - : "memory", "cc", "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", - "%r13", "%r14", "%r15"); + /* Compute the raw addition of f1 + f2 */ + " movq 0(%0), %%r8;" + " addq 0(%2), %%r8;" + " movq 8(%0), %%r9;" + " adcxq 8(%2), %%r9;" + " movq 16(%0), %%r10;" + " adcxq 16(%2), %%r10;" + " movq 24(%0), %%r11;" + " adcxq 24(%2), %%r11;" + + /* Wrap the result back into the field */ + + /* Step 1: Compute carry*38 */ + " mov $0, %%rax;" + " mov $38, %0;" + " cmovc %0, %%rax;" + + /* Step 2: Add carry*38 to the original sum */ + " xor %%rcx, %%rcx;" + " add %%rax, %%r8;" + " adcx %%rcx, %%r9;" + " movq %%r9, 8(%1);" + " adcx %%rcx, %%r10;" + " movq %%r10, 16(%1);" + " adcx %%rcx, %%r11;" + " movq %%r11, 24(%1);" + + /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ + " mov $0, %%rax;" + " cmovc %0, %%rax;" + " add %%rax, %%r8;" + " movq %%r8, 0(%1);" + : "+&r" (f2) + : "r" (out), "r" (f1) + : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc" + ); } -static void mul_256x256_integer_bmi2(u64 *const c, const u64 *const a, - const u64 *const b) +/* Computes the field substraction of two field elements */ +static inline void fsub(u64 *out, const u64 *f1, const u64 *f2) { asm volatile( - "movq (%1), %%rdx; " /* A[0] */ - "mulx (%2), %%r8, %%r15; " /* A[0]*B[0] */ - "movq %%r8, (%0) ;" - "mulx 8(%2), %%r10, %%rax; " /* A[0]*B[1] */ - "addq %%r10, %%r15 ;" - "mulx 16(%2), %%r8, %%rbx; " /* A[0]*B[2] */ - "adcq %%r8, %%rax ;" - "mulx 24(%2), %%r10, %%rcx; " /* A[0]*B[3] */ - "adcq %%r10, %%rbx ;" - /******************************************/ - "adcq $0, %%rcx ;" - - "movq 8(%1), %%rdx; " /* A[1] */ - "mulx (%2), %%r8, %%r9; " /* A[1]*B[0] */ - "addq %%r15, %%r8 ;" - "movq %%r8, 8(%0) ;" - "mulx 8(%2), %%r10, %%r11; " /* A[1]*B[1] */ - "adcq %%r10, %%r9 ;" - "mulx 16(%2), %%r8, %%r13; " /* A[1]*B[2] */ - "adcq %%r8, %%r11 ;" - "mulx 24(%2), %%r10, %%r15; " /* A[1]*B[3] */ - "adcq %%r10, %%r13 ;" - /******************************************/ - "adcq $0, %%r15 ;" - - "addq %%r9, %%rax ;" - "adcq %%r11, %%rbx ;" - "adcq %%r13, %%rcx ;" - "adcq $0, %%r15 ;" - - "movq 16(%1), %%rdx; " /* A[2] */ - "mulx (%2), %%r8, %%r9; " /* A[2]*B[0] */ - "addq %%rax, %%r8 ;" - "movq %%r8, 16(%0) ;" - "mulx 8(%2), %%r10, %%r11; " /* A[2]*B[1] */ - "adcq %%r10, %%r9 ;" - "mulx 16(%2), %%r8, %%r13; " /* A[2]*B[2] */ - "adcq %%r8, %%r11 ;" - "mulx 24(%2), %%r10, %%rax; " /* A[2]*B[3] */ - "adcq %%r10, %%r13 ;" - /******************************************/ - "adcq $0, %%rax ;" - - "addq %%r9, %%rbx ;" - "adcq %%r11, %%rcx ;" - "adcq %%r13, %%r15 ;" - "adcq $0, %%rax ;" - - "movq 24(%1), %%rdx; " /* A[3] */ - "mulx (%2), %%r8, %%r9; " /* A[3]*B[0] */ - "addq %%rbx, %%r8 ;" - "movq %%r8, 24(%0) ;" - "mulx 8(%2), %%r10, %%r11; " /* A[3]*B[1] */ - "adcq %%r10, %%r9 ;" - "mulx 16(%2), %%r8, %%r13; " /* A[3]*B[2] */ - "adcq %%r8, %%r11 ;" - "mulx 24(%2), %%r10, %%rbx; " /* A[3]*B[3] */ - "adcq %%r10, %%r13 ;" - /******************************************/ - "adcq $0, %%rbx ;" - - "addq %%r9, %%rcx ;" - "movq %%rcx, 32(%0) ;" - "adcq %%r11, %%r15 ;" - "movq %%r15, 40(%0) ;" - "adcq %%r13, %%rax ;" - "movq %%rax, 48(%0) ;" - "adcq $0, %%rbx ;" - "movq %%rbx, 56(%0) ;" - : - : "r"(c), "r"(a), "r"(b) - : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", - "%r10", "%r11", "%r13", "%r15"); + /* Compute the raw substraction of f1-f2 */ + " movq 0(%1), %%r8;" + " subq 0(%2), %%r8;" + " movq 8(%1), %%r9;" + " sbbq 8(%2), %%r9;" + " movq 16(%1), %%r10;" + " sbbq 16(%2), %%r10;" + " movq 24(%1), %%r11;" + " sbbq 24(%2), %%r11;" + + /* Wrap the result back into the field */ + + /* Step 1: Compute carry*38 */ + " mov $0, %%rax;" + " mov $38, %%rcx;" + " cmovc %%rcx, %%rax;" + + /* Step 2: Substract carry*38 from the original difference */ + " sub %%rax, %%r8;" + " sbb $0, %%r9;" + " sbb $0, %%r10;" + " sbb $0, %%r11;" + + /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ + " mov $0, %%rax;" + " cmovc %%rcx, %%rax;" + " sub %%rax, %%r8;" + + /* Store the result */ + " movq %%r8, 0(%0);" + " movq %%r9, 8(%0);" + " movq %%r10, 16(%0);" + " movq %%r11, 24(%0);" + : + : "r" (out), "r" (f1), "r" (f2) + : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc" + ); } -static void sqr_256x256_integer_adx(u64 *const c, const u64 *const a) +/* Computes a field multiplication: out <- f1 * f2 + * Uses the 8-element buffer tmp for intermediate results */ +static inline void fmul(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp) { asm volatile( - "movq (%1), %%rdx ;" /* A[0] */ - "mulx 8(%1), %%r8, %%r14 ;" /* A[1]*A[0] */ - "xorl %%r15d, %%r15d;" - "mulx 16(%1), %%r9, %%r10 ;" /* A[2]*A[0] */ - "adcx %%r14, %%r9 ;" - "mulx 24(%1), %%rax, %%rcx ;" /* A[3]*A[0] */ - "adcx %%rax, %%r10 ;" - "movq 24(%1), %%rdx ;" /* A[3] */ - "mulx 8(%1), %%r11, %%rbx ;" /* A[1]*A[3] */ - "adcx %%rcx, %%r11 ;" - "mulx 16(%1), %%rax, %%r13 ;" /* A[2]*A[3] */ - "adcx %%rax, %%rbx ;" - "movq 8(%1), %%rdx ;" /* A[1] */ - "adcx %%r15, %%r13 ;" - "mulx 16(%1), %%rax, %%rcx ;" /* A[2]*A[1] */ - "movq $0, %%r14 ;" - /******************************************/ - "adcx %%r15, %%r14 ;" - - "xorl %%r15d, %%r15d;" - "adox %%rax, %%r10 ;" - "adcx %%r8, %%r8 ;" - "adox %%rcx, %%r11 ;" - "adcx %%r9, %%r9 ;" - "adox %%r15, %%rbx ;" - "adcx %%r10, %%r10 ;" - "adox %%r15, %%r13 ;" - "adcx %%r11, %%r11 ;" - "adox %%r15, %%r14 ;" - "adcx %%rbx, %%rbx ;" - "adcx %%r13, %%r13 ;" - "adcx %%r14, %%r14 ;" - - "movq (%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ - /*******************/ - "movq %%rax, 0(%0) ;" - "addq %%rcx, %%r8 ;" - "movq %%r8, 8(%0) ;" - "movq 8(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ - "adcq %%rax, %%r9 ;" - "movq %%r9, 16(%0) ;" - "adcq %%rcx, %%r10 ;" - "movq %%r10, 24(%0) ;" - "movq 16(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ - "adcq %%rax, %%r11 ;" - "movq %%r11, 32(%0) ;" - "adcq %%rcx, %%rbx ;" - "movq %%rbx, 40(%0) ;" - "movq 24(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ - "adcq %%rax, %%r13 ;" - "movq %%r13, 48(%0) ;" - "adcq %%rcx, %%r14 ;" - "movq %%r14, 56(%0) ;" - : - : "r"(c), "r"(a) - : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", - "%r10", "%r11", "%r13", "%r14", "%r15"); + /* Compute the raw multiplication: tmp <- src1 * src2 */ + + /* Compute src1[0] * src2 */ + " movq 0(%1), %%rdx;" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " movq %%r8, 0(%0);" + " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);" + " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" + " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" + " adox %%rdx, %%rax;" + /* Compute src1[1] * src2 */ + " movq 8(%1), %%rdx;" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);" + " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);" + " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" + " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" + " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + /* Compute src1[2] * src2 */ + " movq 16(%1), %%rdx;" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);" + " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);" + " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" + " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" + " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + /* Compute src1[3] * src2 */ + " movq 24(%1), %%rdx;" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);" + " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);" + " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;" + " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;" + " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 56(%0);" + /* Line up pointers */ + " mov %0, %1;" + " mov %2, %0;" + + /* Wrap the result back into the field */ + + /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ + " mov $38, %%rdx;" + " mulxq 32(%1), %%r8, %%r13;" + " xor %3, %3;" + " adoxq 0(%1), %%r8;" + " mulxq 40(%1), %%r9, %%rbx;" + " adcx %%r13, %%r9;" + " adoxq 8(%1), %%r9;" + " mulxq 48(%1), %%r10, %%r13;" + " adcx %%rbx, %%r10;" + " adoxq 16(%1), %%r10;" + " mulxq 56(%1), %%r11, %%rax;" + " adcx %%r13, %%r11;" + " adoxq 24(%1), %%r11;" + " adcx %3, %%rax;" + " adox %3, %%rax;" + " imul %%rdx, %%rax;" + + /* Step 2: Fold the carry back into dst */ + " add %%rax, %%r8;" + " adcx %3, %%r9;" + " movq %%r9, 8(%0);" + " adcx %3, %%r10;" + " movq %%r10, 16(%0);" + " adcx %3, %%r11;" + " movq %%r11, 24(%0);" + + /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ + " mov $0, %%rax;" + " cmovc %%rdx, %%rax;" + " add %%rax, %%r8;" + " movq %%r8, 0(%0);" + : "+&r" (tmp), "+&r" (f1), "+&r" (out), "+&r" (f2) + : + : "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "memory", "cc" + ); } -static void sqr_256x256_integer_bmi2(u64 *const c, const u64 *const a) +/* Computes two field multiplications: + * out[0] <- f1[0] * f2[0] + * out[1] <- f1[1] * f2[1] + * Uses the 16-element buffer tmp for intermediate results. */ +static inline void fmul2(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp) { asm volatile( - "movq 8(%1), %%rdx ;" /* A[1] */ - "mulx (%1), %%r8, %%r9 ;" /* A[0]*A[1] */ - "mulx 16(%1), %%r10, %%r11 ;" /* A[2]*A[1] */ - "mulx 24(%1), %%rcx, %%r14 ;" /* A[3]*A[1] */ - - "movq 16(%1), %%rdx ;" /* A[2] */ - "mulx 24(%1), %%r15, %%r13 ;" /* A[3]*A[2] */ - "mulx (%1), %%rax, %%rdx ;" /* A[0]*A[2] */ - - "addq %%rax, %%r9 ;" - "adcq %%rdx, %%r10 ;" - "adcq %%rcx, %%r11 ;" - "adcq %%r14, %%r15 ;" - "adcq $0, %%r13 ;" - "movq $0, %%r14 ;" - "adcq $0, %%r14 ;" - - "movq (%1), %%rdx ;" /* A[0] */ - "mulx 24(%1), %%rax, %%rcx ;" /* A[0]*A[3] */ - - "addq %%rax, %%r10 ;" - "adcq %%rcx, %%r11 ;" - "adcq $0, %%r15 ;" - "adcq $0, %%r13 ;" - "adcq $0, %%r14 ;" - - "shldq $1, %%r13, %%r14 ;" - "shldq $1, %%r15, %%r13 ;" - "shldq $1, %%r11, %%r15 ;" - "shldq $1, %%r10, %%r11 ;" - "shldq $1, %%r9, %%r10 ;" - "shldq $1, %%r8, %%r9 ;" - "shlq $1, %%r8 ;" - - /*******************/ - "mulx %%rdx, %%rax, %%rcx ;" /* A[0]^2 */ - /*******************/ - "movq %%rax, 0(%0) ;" - "addq %%rcx, %%r8 ;" - "movq %%r8, 8(%0) ;" - "movq 8(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* A[1]^2 */ - "adcq %%rax, %%r9 ;" - "movq %%r9, 16(%0) ;" - "adcq %%rcx, %%r10 ;" - "movq %%r10, 24(%0) ;" - "movq 16(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* A[2]^2 */ - "adcq %%rax, %%r11 ;" - "movq %%r11, 32(%0) ;" - "adcq %%rcx, %%r15 ;" - "movq %%r15, 40(%0) ;" - "movq 24(%1), %%rdx ;" - "mulx %%rdx, %%rax, %%rcx ;" /* A[3]^2 */ - "adcq %%rax, %%r13 ;" - "movq %%r13, 48(%0) ;" - "adcq %%rcx, %%r14 ;" - "movq %%r14, 56(%0) ;" - : - : "r"(c), "r"(a) - : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", - "%r11", "%r13", "%r14", "%r15"); + /* Compute the raw multiplication tmp[0] <- f1[0] * f2[0] */ + + /* Compute src1[0] * src2 */ + " movq 0(%1), %%rdx;" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " movq %%r8, 0(%0);" + " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);" + " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" + " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" + " adox %%rdx, %%rax;" + /* Compute src1[1] * src2 */ + " movq 8(%1), %%rdx;" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);" + " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);" + " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" + " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" + " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + /* Compute src1[2] * src2 */ + " movq 16(%1), %%rdx;" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);" + " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);" + " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" + " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" + " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + /* Compute src1[3] * src2 */ + " movq 24(%1), %%rdx;" + " mulxq 0(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);" + " mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);" + " mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;" + " mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;" + " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 56(%0);" + + /* Compute the raw multiplication tmp[1] <- f1[1] * f2[1] */ + + /* Compute src1[0] * src2 */ + " movq 32(%1), %%rdx;" + " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " movq %%r8, 64(%0);" + " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 72(%0);" + " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" + " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;" + " adox %%rdx, %%rax;" + /* Compute src1[1] * src2 */ + " movq 40(%1), %%rdx;" + " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 72(%0), %%r8;" " movq %%r8, 72(%0);" + " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 80(%0);" + " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" + " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" + " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + /* Compute src1[2] * src2 */ + " movq 48(%1), %%rdx;" + " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 80(%0), %%r8;" " movq %%r8, 80(%0);" + " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 88(%0);" + " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;" + " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;" + " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" + /* Compute src1[3] * src2 */ + " movq 56(%1), %%rdx;" + " mulxq 32(%3), %%r8, %%r9;" " xor %%r10, %%r10;" " adcxq 88(%0), %%r8;" " movq %%r8, 88(%0);" + " mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 96(%0);" + " mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 104(%0);" " mov $0, %%r8;" + " mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 112(%0);" " mov $0, %%rax;" + " adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 120(%0);" + /* Line up pointers */ + " mov %0, %1;" + " mov %2, %0;" + + /* Wrap the results back into the field */ + + /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ + " mov $38, %%rdx;" + " mulxq 32(%1), %%r8, %%r13;" + " xor %3, %3;" + " adoxq 0(%1), %%r8;" + " mulxq 40(%1), %%r9, %%rbx;" + " adcx %%r13, %%r9;" + " adoxq 8(%1), %%r9;" + " mulxq 48(%1), %%r10, %%r13;" + " adcx %%rbx, %%r10;" + " adoxq 16(%1), %%r10;" + " mulxq 56(%1), %%r11, %%rax;" + " adcx %%r13, %%r11;" + " adoxq 24(%1), %%r11;" + " adcx %3, %%rax;" + " adox %3, %%rax;" + " imul %%rdx, %%rax;" + + /* Step 2: Fold the carry back into dst */ + " add %%rax, %%r8;" + " adcx %3, %%r9;" + " movq %%r9, 8(%0);" + " adcx %3, %%r10;" + " movq %%r10, 16(%0);" + " adcx %3, %%r11;" + " movq %%r11, 24(%0);" + + /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ + " mov $0, %%rax;" + " cmovc %%rdx, %%rax;" + " add %%rax, %%r8;" + " movq %%r8, 0(%0);" + + /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ + " mov $38, %%rdx;" + " mulxq 96(%1), %%r8, %%r13;" + " xor %3, %3;" + " adoxq 64(%1), %%r8;" + " mulxq 104(%1), %%r9, %%rbx;" + " adcx %%r13, %%r9;" + " adoxq 72(%1), %%r9;" + " mulxq 112(%1), %%r10, %%r13;" + " adcx %%rbx, %%r10;" + " adoxq 80(%1), %%r10;" + " mulxq 120(%1), %%r11, %%rax;" + " adcx %%r13, %%r11;" + " adoxq 88(%1), %%r11;" + " adcx %3, %%rax;" + " adox %3, %%rax;" + " imul %%rdx, %%rax;" + + /* Step 2: Fold the carry back into dst */ + " add %%rax, %%r8;" + " adcx %3, %%r9;" + " movq %%r9, 40(%0);" + " adcx %3, %%r10;" + " movq %%r10, 48(%0);" + " adcx %3, %%r11;" + " movq %%r11, 56(%0);" + + /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ + " mov $0, %%rax;" + " cmovc %%rdx, %%rax;" + " add %%rax, %%r8;" + " movq %%r8, 32(%0);" + : "+&r" (tmp), "+&r" (f1), "+&r" (out), "+&r" (f2) + : + : "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "memory", "cc" + ); } -static void red_eltfp25519_1w_adx(u64 *const c, const u64 *const a) +/* Computes the field multiplication of four-element f1 with value in f2 */ +static inline void fmul_scalar(u64 *out, const u64 *f1, u64 f2) { - asm volatile( - "movl $38, %%edx ;" /* 2*c = 38 = 2^256 */ - "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ - "xorl %%ebx, %%ebx ;" - "adox (%1), %%r8 ;" - "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ - "adcx %%r10, %%r9 ;" - "adox 8(%1), %%r9 ;" - "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ - "adcx %%r11, %%r10 ;" - "adox 16(%1), %%r10 ;" - "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ - "adcx %%rax, %%r11 ;" - "adox 24(%1), %%r11 ;" - /***************************************/ - "adcx %%rbx, %%rcx ;" - "adox %%rbx, %%rcx ;" - "imul %%rdx, %%rcx ;" /* c*C[4], cf=0, of=0 */ - "adcx %%rcx, %%r8 ;" - "adcx %%rbx, %%r9 ;" - "movq %%r9, 8(%0) ;" - "adcx %%rbx, %%r10 ;" - "movq %%r10, 16(%0) ;" - "adcx %%rbx, %%r11 ;" - "movq %%r11, 24(%0) ;" - "mov $0, %%ecx ;" - "cmovc %%edx, %%ecx ;" - "addq %%rcx, %%r8 ;" - "movq %%r8, (%0) ;" - : - : "r"(c), "r"(a) - : "memory", "cc", "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", - "%r10", "%r11"); -} + register u64 f2_r asm("rdx") = f2; -static void red_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a) -{ asm volatile( - "movl $38, %%edx ;" /* 2*c = 38 = 2^256 */ - "mulx 32(%1), %%r8, %%r10 ;" /* c*C[4] */ - "mulx 40(%1), %%r9, %%r11 ;" /* c*C[5] */ - "addq %%r10, %%r9 ;" - "mulx 48(%1), %%r10, %%rax ;" /* c*C[6] */ - "adcq %%r11, %%r10 ;" - "mulx 56(%1), %%r11, %%rcx ;" /* c*C[7] */ - "adcq %%rax, %%r11 ;" - /***************************************/ - "adcq $0, %%rcx ;" - "addq (%1), %%r8 ;" - "adcq 8(%1), %%r9 ;" - "adcq 16(%1), %%r10 ;" - "adcq 24(%1), %%r11 ;" - "adcq $0, %%rcx ;" - "imul %%rdx, %%rcx ;" /* c*C[4], cf=0 */ - "addq %%rcx, %%r8 ;" - "adcq $0, %%r9 ;" - "movq %%r9, 8(%0) ;" - "adcq $0, %%r10 ;" - "movq %%r10, 16(%0) ;" - "adcq $0, %%r11 ;" - "movq %%r11, 24(%0) ;" - "mov $0, %%ecx ;" - "cmovc %%edx, %%ecx ;" - "addq %%rcx, %%r8 ;" - "movq %%r8, (%0) ;" - : - : "r"(c), "r"(a) - : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", - "%r11"); + /* Compute the raw multiplication of f1*f2 */ + " mulxq 0(%2), %%r8, %%rcx;" /* f1[0]*f2 */ + " mulxq 8(%2), %%r9, %%rbx;" /* f1[1]*f2 */ + " add %%rcx, %%r9;" + " mov $0, %%rcx;" + " mulxq 16(%2), %%r10, %%r13;" /* f1[2]*f2 */ + " adcx %%rbx, %%r10;" + " mulxq 24(%2), %%r11, %%rax;" /* f1[3]*f2 */ + " adcx %%r13, %%r11;" + " adcx %%rcx, %%rax;" + + /* Wrap the result back into the field */ + + /* Step 1: Compute carry*38 */ + " mov $38, %%rdx;" + " imul %%rdx, %%rax;" + + /* Step 2: Fold the carry back into dst */ + " add %%rax, %%r8;" + " adcx %%rcx, %%r9;" + " movq %%r9, 8(%1);" + " adcx %%rcx, %%r10;" + " movq %%r10, 16(%1);" + " adcx %%rcx, %%r11;" + " movq %%r11, 24(%1);" + + /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ + " mov $0, %%rax;" + " cmovc %%rdx, %%rax;" + " add %%rax, %%r8;" + " movq %%r8, 0(%1);" + : "+&r" (f2_r) + : "r" (out), "r" (f1) + : "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "memory", "cc" + ); } -static __always_inline void -add_eltfp25519_1w_adx(u64 *const c, const u64 *const a, const u64 *const b) +/* Computes p1 <- bit ? p2 : p1 in constant time */ +static inline void cswap2(u64 bit, const u64 *p1, const u64 *p2) { asm volatile( - "mov $38, %%eax ;" - "xorl %%ecx, %%ecx ;" - "movq (%2), %%r8 ;" - "adcx (%1), %%r8 ;" - "movq 8(%2), %%r9 ;" - "adcx 8(%1), %%r9 ;" - "movq 16(%2), %%r10 ;" - "adcx 16(%1), %%r10 ;" - "movq 24(%2), %%r11 ;" - "adcx 24(%1), %%r11 ;" - "cmovc %%eax, %%ecx ;" - "xorl %%eax, %%eax ;" - "adcx %%rcx, %%r8 ;" - "adcx %%rax, %%r9 ;" - "movq %%r9, 8(%0) ;" - "adcx %%rax, %%r10 ;" - "movq %%r10, 16(%0) ;" - "adcx %%rax, %%r11 ;" - "movq %%r11, 24(%0) ;" - "mov $38, %%ecx ;" - "cmovc %%ecx, %%eax ;" - "addq %%rax, %%r8 ;" - "movq %%r8, (%0) ;" - : - : "r"(c), "r"(a), "r"(b) - : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); + /* Invert the polarity of bit to match cmov expectations */ + " add $18446744073709551615, %0;" + + /* cswap p1[0], p2[0] */ + " movq 0(%1), %%r8;" + " movq 0(%2), %%r9;" + " mov %%r8, %%r10;" + " cmovc %%r9, %%r8;" + " cmovc %%r10, %%r9;" + " movq %%r8, 0(%1);" + " movq %%r9, 0(%2);" + + /* cswap p1[1], p2[1] */ + " movq 8(%1), %%r8;" + " movq 8(%2), %%r9;" + " mov %%r8, %%r10;" + " cmovc %%r9, %%r8;" + " cmovc %%r10, %%r9;" + " movq %%r8, 8(%1);" + " movq %%r9, 8(%2);" + + /* cswap p1[2], p2[2] */ + " movq 16(%1), %%r8;" + " movq 16(%2), %%r9;" + " mov %%r8, %%r10;" + " cmovc %%r9, %%r8;" + " cmovc %%r10, %%r9;" + " movq %%r8, 16(%1);" + " movq %%r9, 16(%2);" + + /* cswap p1[3], p2[3] */ + " movq 24(%1), %%r8;" + " movq 24(%2), %%r9;" + " mov %%r8, %%r10;" + " cmovc %%r9, %%r8;" + " cmovc %%r10, %%r9;" + " movq %%r8, 24(%1);" + " movq %%r9, 24(%2);" + + /* cswap p1[4], p2[4] */ + " movq 32(%1), %%r8;" + " movq 32(%2), %%r9;" + " mov %%r8, %%r10;" + " cmovc %%r9, %%r8;" + " cmovc %%r10, %%r9;" + " movq %%r8, 32(%1);" + " movq %%r9, 32(%2);" + + /* cswap p1[5], p2[5] */ + " movq 40(%1), %%r8;" + " movq 40(%2), %%r9;" + " mov %%r8, %%r10;" + " cmovc %%r9, %%r8;" + " cmovc %%r10, %%r9;" + " movq %%r8, 40(%1);" + " movq %%r9, 40(%2);" + + /* cswap p1[6], p2[6] */ + " movq 48(%1), %%r8;" + " movq 48(%2), %%r9;" + " mov %%r8, %%r10;" + " cmovc %%r9, %%r8;" + " cmovc %%r10, %%r9;" + " movq %%r8, 48(%1);" + " movq %%r9, 48(%2);" + + /* cswap p1[7], p2[7] */ + " movq 56(%1), %%r8;" + " movq 56(%2), %%r9;" + " mov %%r8, %%r10;" + " cmovc %%r9, %%r8;" + " cmovc %%r10, %%r9;" + " movq %%r8, 56(%1);" + " movq %%r9, 56(%2);" + : "+&r" (bit) + : "r" (p1), "r" (p2) + : "%r8", "%r9", "%r10", "memory", "cc" + ); } -static __always_inline void -add_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a, const u64 *const b) +/* Computes the square of a field element: out <- f * f + * Uses the 8-element buffer tmp for intermediate results */ +static inline void fsqr(u64 *out, const u64 *f, u64 *tmp) { asm volatile( - "mov $38, %%eax ;" - "movq (%2), %%r8 ;" - "addq (%1), %%r8 ;" - "movq 8(%2), %%r9 ;" - "adcq 8(%1), %%r9 ;" - "movq 16(%2), %%r10 ;" - "adcq 16(%1), %%r10 ;" - "movq 24(%2), %%r11 ;" - "adcq 24(%1), %%r11 ;" - "mov $0, %%ecx ;" - "cmovc %%eax, %%ecx ;" - "addq %%rcx, %%r8 ;" - "adcq $0, %%r9 ;" - "movq %%r9, 8(%0) ;" - "adcq $0, %%r10 ;" - "movq %%r10, 16(%0) ;" - "adcq $0, %%r11 ;" - "movq %%r11, 24(%0) ;" - "mov $0, %%ecx ;" - "cmovc %%eax, %%ecx ;" - "addq %%rcx, %%r8 ;" - "movq %%r8, (%0) ;" - : - : "r"(c), "r"(a), "r"(b) - : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); + /* Compute the raw multiplication: tmp <- f * f */ + + /* Step 1: Compute all partial products */ + " movq 0(%1), %%rdx;" /* f[0] */ + " mulxq 8(%1), %%r8, %%r14;" " xor %%r15, %%r15;" /* f[1]*f[0] */ + " mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ + " mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ + " movq 24(%1), %%rdx;" /* f[3] */ + " mulxq 8(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */ + " mulxq 16(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */ + " movq 8(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */ + " mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ + + /* Step 2: Compute two parallel carry chains */ + " xor %%r15, %%r15;" + " adox %%rax, %%r10;" + " adcx %%r8, %%r8;" + " adox %%rcx, %%r11;" + " adcx %%r9, %%r9;" + " adox %%r15, %%rbx;" + " adcx %%r10, %%r10;" + " adox %%r15, %%r13;" + " adcx %%r11, %%r11;" + " adox %%r15, %%r14;" + " adcx %%rbx, %%rbx;" + " adcx %%r13, %%r13;" + " adcx %%r14, %%r14;" + + /* Step 3: Compute intermediate squares */ + " movq 0(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ + " movq %%rax, 0(%0);" + " add %%rcx, %%r8;" " movq %%r8, 8(%0);" + " movq 8(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ + " adcx %%rax, %%r9;" " movq %%r9, 16(%0);" + " adcx %%rcx, %%r10;" " movq %%r10, 24(%0);" + " movq 16(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ + " adcx %%rax, %%r11;" " movq %%r11, 32(%0);" + " adcx %%rcx, %%rbx;" " movq %%rbx, 40(%0);" + " movq 24(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ + " adcx %%rax, %%r13;" " movq %%r13, 48(%0);" + " adcx %%rcx, %%r14;" " movq %%r14, 56(%0);" + + /* Line up pointers */ + " mov %0, %1;" + " mov %2, %0;" + + /* Wrap the result back into the field */ + + /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ + " mov $38, %%rdx;" + " mulxq 32(%1), %%r8, %%r13;" + " xor %%rcx, %%rcx;" + " adoxq 0(%1), %%r8;" + " mulxq 40(%1), %%r9, %%rbx;" + " adcx %%r13, %%r9;" + " adoxq 8(%1), %%r9;" + " mulxq 48(%1), %%r10, %%r13;" + " adcx %%rbx, %%r10;" + " adoxq 16(%1), %%r10;" + " mulxq 56(%1), %%r11, %%rax;" + " adcx %%r13, %%r11;" + " adoxq 24(%1), %%r11;" + " adcx %%rcx, %%rax;" + " adox %%rcx, %%rax;" + " imul %%rdx, %%rax;" + + /* Step 2: Fold the carry back into dst */ + " add %%rax, %%r8;" + " adcx %%rcx, %%r9;" + " movq %%r9, 8(%0);" + " adcx %%rcx, %%r10;" + " movq %%r10, 16(%0);" + " adcx %%rcx, %%r11;" + " movq %%r11, 24(%0);" + + /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ + " mov $0, %%rax;" + " cmovc %%rdx, %%rax;" + " add %%rax, %%r8;" + " movq %%r8, 0(%0);" + : "+&r" (tmp), "+&r" (f), "+&r" (out) + : + : "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "%r15", "memory", "cc" + ); } -static __always_inline void -sub_eltfp25519_1w(u64 *const c, const u64 *const a, const u64 *const b) +/* Computes two field squarings: + * out[0] <- f[0] * f[0] + * out[1] <- f[1] * f[1] + * Uses the 16-element buffer tmp for intermediate results */ +static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp) { asm volatile( - "mov $38, %%eax ;" - "movq (%1), %%r8 ;" - "subq (%2), %%r8 ;" - "movq 8(%1), %%r9 ;" - "sbbq 8(%2), %%r9 ;" - "movq 16(%1), %%r10 ;" - "sbbq 16(%2), %%r10 ;" - "movq 24(%1), %%r11 ;" - "sbbq 24(%2), %%r11 ;" - "mov $0, %%ecx ;" - "cmovc %%eax, %%ecx ;" - "subq %%rcx, %%r8 ;" - "sbbq $0, %%r9 ;" - "movq %%r9, 8(%0) ;" - "sbbq $0, %%r10 ;" - "movq %%r10, 16(%0) ;" - "sbbq $0, %%r11 ;" - "movq %%r11, 24(%0) ;" - "mov $0, %%ecx ;" - "cmovc %%eax, %%ecx ;" - "subq %%rcx, %%r8 ;" - "movq %%r8, (%0) ;" - : - : "r"(c), "r"(a), "r"(b) - : "memory", "cc", "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11"); + /* Step 1: Compute all partial products */ + " movq 0(%1), %%rdx;" /* f[0] */ + " mulxq 8(%1), %%r8, %%r14;" " xor %%r15, %%r15;" /* f[1]*f[0] */ + " mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ + " mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ + " movq 24(%1), %%rdx;" /* f[3] */ + " mulxq 8(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */ + " mulxq 16(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */ + " movq 8(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */ + " mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ + + /* Step 2: Compute two parallel carry chains */ + " xor %%r15, %%r15;" + " adox %%rax, %%r10;" + " adcx %%r8, %%r8;" + " adox %%rcx, %%r11;" + " adcx %%r9, %%r9;" + " adox %%r15, %%rbx;" + " adcx %%r10, %%r10;" + " adox %%r15, %%r13;" + " adcx %%r11, %%r11;" + " adox %%r15, %%r14;" + " adcx %%rbx, %%rbx;" + " adcx %%r13, %%r13;" + " adcx %%r14, %%r14;" + + /* Step 3: Compute intermediate squares */ + " movq 0(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ + " movq %%rax, 0(%0);" + " add %%rcx, %%r8;" " movq %%r8, 8(%0);" + " movq 8(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ + " adcx %%rax, %%r9;" " movq %%r9, 16(%0);" + " adcx %%rcx, %%r10;" " movq %%r10, 24(%0);" + " movq 16(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ + " adcx %%rax, %%r11;" " movq %%r11, 32(%0);" + " adcx %%rcx, %%rbx;" " movq %%rbx, 40(%0);" + " movq 24(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ + " adcx %%rax, %%r13;" " movq %%r13, 48(%0);" + " adcx %%rcx, %%r14;" " movq %%r14, 56(%0);" + + /* Step 1: Compute all partial products */ + " movq 32(%1), %%rdx;" /* f[0] */ + " mulxq 40(%1), %%r8, %%r14;" " xor %%r15, %%r15;" /* f[1]*f[0] */ + " mulxq 48(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */ + " mulxq 56(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */ + " movq 56(%1), %%rdx;" /* f[3] */ + " mulxq 40(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */ + " mulxq 48(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */ + " movq 40(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */ + " mulxq 48(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */ + + /* Step 2: Compute two parallel carry chains */ + " xor %%r15, %%r15;" + " adox %%rax, %%r10;" + " adcx %%r8, %%r8;" + " adox %%rcx, %%r11;" + " adcx %%r9, %%r9;" + " adox %%r15, %%rbx;" + " adcx %%r10, %%r10;" + " adox %%r15, %%r13;" + " adcx %%r11, %%r11;" + " adox %%r15, %%r14;" + " adcx %%rbx, %%rbx;" + " adcx %%r13, %%r13;" + " adcx %%r14, %%r14;" + + /* Step 3: Compute intermediate squares */ + " movq 32(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */ + " movq %%rax, 64(%0);" + " add %%rcx, %%r8;" " movq %%r8, 72(%0);" + " movq 40(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */ + " adcx %%rax, %%r9;" " movq %%r9, 80(%0);" + " adcx %%rcx, %%r10;" " movq %%r10, 88(%0);" + " movq 48(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */ + " adcx %%rax, %%r11;" " movq %%r11, 96(%0);" + " adcx %%rcx, %%rbx;" " movq %%rbx, 104(%0);" + " movq 56(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */ + " adcx %%rax, %%r13;" " movq %%r13, 112(%0);" + " adcx %%rcx, %%r14;" " movq %%r14, 120(%0);" + + /* Line up pointers */ + " mov %0, %1;" + " mov %2, %0;" + + /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ + " mov $38, %%rdx;" + " mulxq 32(%1), %%r8, %%r13;" + " xor %%rcx, %%rcx;" + " adoxq 0(%1), %%r8;" + " mulxq 40(%1), %%r9, %%rbx;" + " adcx %%r13, %%r9;" + " adoxq 8(%1), %%r9;" + " mulxq 48(%1), %%r10, %%r13;" + " adcx %%rbx, %%r10;" + " adoxq 16(%1), %%r10;" + " mulxq 56(%1), %%r11, %%rax;" + " adcx %%r13, %%r11;" + " adoxq 24(%1), %%r11;" + " adcx %%rcx, %%rax;" + " adox %%rcx, %%rax;" + " imul %%rdx, %%rax;" + + /* Step 2: Fold the carry back into dst */ + " add %%rax, %%r8;" + " adcx %%rcx, %%r9;" + " movq %%r9, 8(%0);" + " adcx %%rcx, %%r10;" + " movq %%r10, 16(%0);" + " adcx %%rcx, %%r11;" + " movq %%r11, 24(%0);" + + /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ + " mov $0, %%rax;" + " cmovc %%rdx, %%rax;" + " add %%rax, %%r8;" + " movq %%r8, 0(%0);" + + /* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */ + " mov $38, %%rdx;" + " mulxq 96(%1), %%r8, %%r13;" + " xor %%rcx, %%rcx;" + " adoxq 64(%1), %%r8;" + " mulxq 104(%1), %%r9, %%rbx;" + " adcx %%r13, %%r9;" + " adoxq 72(%1), %%r9;" + " mulxq 112(%1), %%r10, %%r13;" + " adcx %%rbx, %%r10;" + " adoxq 80(%1), %%r10;" + " mulxq 120(%1), %%r11, %%rax;" + " adcx %%r13, %%r11;" + " adoxq 88(%1), %%r11;" + " adcx %%rcx, %%rax;" + " adox %%rcx, %%rax;" + " imul %%rdx, %%rax;" + + /* Step 2: Fold the carry back into dst */ + " add %%rax, %%r8;" + " adcx %%rcx, %%r9;" + " movq %%r9, 40(%0);" + " adcx %%rcx, %%r10;" + " movq %%r10, 48(%0);" + " adcx %%rcx, %%r11;" + " movq %%r11, 56(%0);" + + /* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */ + " mov $0, %%rax;" + " cmovc %%rdx, %%rax;" + " add %%rax, %%r8;" + " movq %%r8, 32(%0);" + : "+&r" (tmp), "+&r" (f), "+&r" (out) + : + : "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "%r15", "memory", "cc" + ); } -/* Multiplication by a24 = (A+2)/4 = (486662+2)/4 = 121666 */ -static __always_inline void -mul_a24_eltfp25519_1w(u64 *const c, const u64 *const a) +static void point_add_and_double(u64 *q, u64 *p01_tmp1, u64 *tmp2) { - const u64 a24 = 121666; - asm volatile( - "movq %2, %%rdx ;" - "mulx (%1), %%r8, %%r10 ;" - "mulx 8(%1), %%r9, %%r11 ;" - "addq %%r10, %%r9 ;" - "mulx 16(%1), %%r10, %%rax ;" - "adcq %%r11, %%r10 ;" - "mulx 24(%1), %%r11, %%rcx ;" - "adcq %%rax, %%r11 ;" - /**************************/ - "adcq $0, %%rcx ;" - "movl $38, %%edx ;" /* 2*c = 38 = 2^256 mod 2^255-19*/ - "imul %%rdx, %%rcx ;" - "addq %%rcx, %%r8 ;" - "adcq $0, %%r9 ;" - "movq %%r9, 8(%0) ;" - "adcq $0, %%r10 ;" - "movq %%r10, 16(%0) ;" - "adcq $0, %%r11 ;" - "movq %%r11, 24(%0) ;" - "mov $0, %%ecx ;" - "cmovc %%edx, %%ecx ;" - "addq %%rcx, %%r8 ;" - "movq %%r8, (%0) ;" - : - : "r"(c), "r"(a), "r"(a24) - : "memory", "cc", "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", - "%r11"); + u64 *nq = p01_tmp1; + u64 *nq_p1 = p01_tmp1 + (u32)8U; + u64 *tmp1 = p01_tmp1 + (u32)16U; + u64 *x1 = q; + u64 *x2 = nq; + u64 *z2 = nq + (u32)4U; + u64 *z3 = nq_p1 + (u32)4U; + u64 *a = tmp1; + u64 *b = tmp1 + (u32)4U; + u64 *ab = tmp1; + u64 *dc = tmp1 + (u32)8U; + u64 *x3; + u64 *z31; + u64 *d0; + u64 *c0; + u64 *a1; + u64 *b1; + u64 *d; + u64 *c; + u64 *ab1; + u64 *dc1; + fadd(a, x2, z2); + fsub(b, x2, z2); + x3 = nq_p1; + z31 = nq_p1 + (u32)4U; + d0 = dc; + c0 = dc + (u32)4U; + fadd(c0, x3, z31); + fsub(d0, x3, z31); + fmul2(dc, dc, ab, tmp2); + fadd(x3, d0, c0); + fsub(z31, d0, c0); + a1 = tmp1; + b1 = tmp1 + (u32)4U; + d = tmp1 + (u32)8U; + c = tmp1 + (u32)12U; + ab1 = tmp1; + dc1 = tmp1 + (u32)8U; + fsqr2(dc1, ab1, tmp2); + fsqr2(nq_p1, nq_p1, tmp2); + a1[0U] = c[0U]; + a1[1U] = c[1U]; + a1[2U] = c[2U]; + a1[3U] = c[3U]; + fsub(c, d, c); + fmul_scalar(b1, c, (u64)121665U); + fadd(b1, b1, d); + fmul2(nq, dc1, ab1, tmp2); + fmul(z3, z3, x1, tmp2); } -static void inv_eltfp25519_1w_adx(u64 *const c, const u64 *const a) +static void point_double(u64 *nq, u64 *tmp1, u64 *tmp2) { - struct { - eltfp25519_1w_buffer buffer; - eltfp25519_1w x0, x1, x2; - } __aligned(32) m; - u64 *T[4]; - - T[0] = m.x0; - T[1] = c; /* x^(-1) */ - T[2] = m.x1; - T[3] = m.x2; - - copy_eltfp25519_1w(T[1], a); - sqrn_eltfp25519_1w_adx(T[1], 1); - copy_eltfp25519_1w(T[2], T[1]); - sqrn_eltfp25519_1w_adx(T[2], 2); - mul_eltfp25519_1w_adx(T[0], a, T[2]); - mul_eltfp25519_1w_adx(T[1], T[1], T[0]); - copy_eltfp25519_1w(T[2], T[1]); - sqrn_eltfp25519_1w_adx(T[2], 1); - mul_eltfp25519_1w_adx(T[0], T[0], T[2]); - copy_eltfp25519_1w(T[2], T[0]); - sqrn_eltfp25519_1w_adx(T[2], 5); - mul_eltfp25519_1w_adx(T[0], T[0], T[2]); - copy_eltfp25519_1w(T[2], T[0]); - sqrn_eltfp25519_1w_adx(T[2], 10); - mul_eltfp25519_1w_adx(T[2], T[2], T[0]); - copy_eltfp25519_1w(T[3], T[2]); - sqrn_eltfp25519_1w_adx(T[3], 20); - mul_eltfp25519_1w_adx(T[3], T[3], T[2]); - sqrn_eltfp25519_1w_adx(T[3], 10); - mul_eltfp25519_1w_adx(T[3], T[3], T[0]); - copy_eltfp25519_1w(T[0], T[3]); - sqrn_eltfp25519_1w_adx(T[0], 50); - mul_eltfp25519_1w_adx(T[0], T[0], T[3]); - copy_eltfp25519_1w(T[2], T[0]); - sqrn_eltfp25519_1w_adx(T[2], 100); - mul_eltfp25519_1w_adx(T[2], T[2], T[0]); - sqrn_eltfp25519_1w_adx(T[2], 50); - mul_eltfp25519_1w_adx(T[2], T[2], T[3]); - sqrn_eltfp25519_1w_adx(T[2], 5); - mul_eltfp25519_1w_adx(T[1], T[1], T[2]); - - memzero_explicit(&m, sizeof(m)); + u64 *x2 = nq; + u64 *z2 = nq + (u32)4U; + u64 *a = tmp1; + u64 *b = tmp1 + (u32)4U; + u64 *d = tmp1 + (u32)8U; + u64 *c = tmp1 + (u32)12U; + u64 *ab = tmp1; + u64 *dc = tmp1 + (u32)8U; + fadd(a, x2, z2); + fsub(b, x2, z2); + fsqr2(dc, ab, tmp2); + a[0U] = c[0U]; + a[1U] = c[1U]; + a[2U] = c[2U]; + a[3U] = c[3U]; + fsub(c, d, c); + fmul_scalar(b, c, (u64)121665U); + fadd(b, b, d); + fmul2(nq, dc, ab, tmp2); } -static void inv_eltfp25519_1w_bmi2(u64 *const c, const u64 *const a) +static void montgomery_ladder(u64 *out, const u8 *key, u64 *init1) { - struct { - eltfp25519_1w_buffer buffer; - eltfp25519_1w x0, x1, x2; - } __aligned(32) m; - u64 *T[5]; - - T[0] = m.x0; - T[1] = c; /* x^(-1) */ - T[2] = m.x1; - T[3] = m.x2; - - copy_eltfp25519_1w(T[1], a); - sqrn_eltfp25519_1w_bmi2(T[1], 1); - copy_eltfp25519_1w(T[2], T[1]); - sqrn_eltfp25519_1w_bmi2(T[2], 2); - mul_eltfp25519_1w_bmi2(T[0], a, T[2]); - mul_eltfp25519_1w_bmi2(T[1], T[1], T[0]); - copy_eltfp25519_1w(T[2], T[1]); - sqrn_eltfp25519_1w_bmi2(T[2], 1); - mul_eltfp25519_1w_bmi2(T[0], T[0], T[2]); - copy_eltfp25519_1w(T[2], T[0]); - sqrn_eltfp25519_1w_bmi2(T[2], 5); - mul_eltfp25519_1w_bmi2(T[0], T[0], T[2]); - copy_eltfp25519_1w(T[2], T[0]); - sqrn_eltfp25519_1w_bmi2(T[2], 10); - mul_eltfp25519_1w_bmi2(T[2], T[2], T[0]); - copy_eltfp25519_1w(T[3], T[2]); - sqrn_eltfp25519_1w_bmi2(T[3], 20); - mul_eltfp25519_1w_bmi2(T[3], T[3], T[2]); - sqrn_eltfp25519_1w_bmi2(T[3], 10); - mul_eltfp25519_1w_bmi2(T[3], T[3], T[0]); - copy_eltfp25519_1w(T[0], T[3]); - sqrn_eltfp25519_1w_bmi2(T[0], 50); - mul_eltfp25519_1w_bmi2(T[0], T[0], T[3]); - copy_eltfp25519_1w(T[2], T[0]); - sqrn_eltfp25519_1w_bmi2(T[2], 100); - mul_eltfp25519_1w_bmi2(T[2], T[2], T[0]); - sqrn_eltfp25519_1w_bmi2(T[2], 50); - mul_eltfp25519_1w_bmi2(T[2], T[2], T[3]); - sqrn_eltfp25519_1w_bmi2(T[2], 5); - mul_eltfp25519_1w_bmi2(T[1], T[1], T[2]); - - memzero_explicit(&m, sizeof(m)); + u64 tmp2[16U] = { 0U }; + u64 p01_tmp1_swap[33U] = { 0U }; + u64 *p0 = p01_tmp1_swap; + u64 *p01 = p01_tmp1_swap; + u64 *p03 = p01; + u64 *p11 = p01 + (u32)8U; + u64 *x0; + u64 *z0; + u64 *p01_tmp1; + u64 *p01_tmp11; + u64 *nq10; + u64 *nq_p11; + u64 *swap1; + u64 sw0; + u64 *nq1; + u64 *tmp1; + memcpy(p11, init1, (u32)8U * sizeof(init1[0U])); + x0 = p03; + z0 = p03 + (u32)4U; + x0[0U] = (u64)1U; + x0[1U] = (u64)0U; + x0[2U] = (u64)0U; + x0[3U] = (u64)0U; + z0[0U] = (u64)0U; + z0[1U] = (u64)0U; + z0[2U] = (u64)0U; + z0[3U] = (u64)0U; + p01_tmp1 = p01_tmp1_swap; + p01_tmp11 = p01_tmp1_swap; + nq10 = p01_tmp1_swap; + nq_p11 = p01_tmp1_swap + (u32)8U; + swap1 = p01_tmp1_swap + (u32)32U; + cswap2((u64)1U, nq10, nq_p11); + point_add_and_double(init1, p01_tmp11, tmp2); + swap1[0U] = (u64)1U; + { + u32 i; + for (i = (u32)0U; i < (u32)251U; i = i + (u32)1U) { + u64 *p01_tmp12 = p01_tmp1_swap; + u64 *swap2 = p01_tmp1_swap + (u32)32U; + u64 *nq2 = p01_tmp12; + u64 *nq_p12 = p01_tmp12 + (u32)8U; + u64 bit = (u64)(key[((u32)253U - i) / (u32)8U] >> ((u32)253U - i) % (u32)8U & (u8)1U); + u64 sw = swap2[0U] ^ bit; + cswap2(sw, nq2, nq_p12); + point_add_and_double(init1, p01_tmp12, tmp2); + swap2[0U] = bit; + } + } + sw0 = swap1[0U]; + cswap2(sw0, nq10, nq_p11); + nq1 = p01_tmp1; + tmp1 = p01_tmp1 + (u32)16U; + point_double(nq1, tmp1, tmp2); + point_double(nq1, tmp1, tmp2); + point_double(nq1, tmp1, tmp2); + memcpy(out, p0, (u32)8U * sizeof(p0[0U])); + + memzero_explicit(tmp2, sizeof(tmp2)); + memzero_explicit(p01_tmp1_swap, sizeof(p01_tmp1_swap)); } -/* Given c, a 256-bit number, fred_eltfp25519_1w updates c - * with a number such that 0 <= C < 2**255-19. - */ -static __always_inline void fred_eltfp25519_1w(u64 *const c) +static void fsquare_times(u64 *o, const u64 *inp, u64 *tmp, u32 n1) { - u64 tmp0 = 38, tmp1 = 19; - asm volatile( - "btrq $63, %3 ;" /* Put bit 255 in carry flag and clear */ - "cmovncl %k5, %k4 ;" /* c[255] ? 38 : 19 */ - - /* Add either 19 or 38 to c */ - "addq %4, %0 ;" - "adcq $0, %1 ;" - "adcq $0, %2 ;" - "adcq $0, %3 ;" - - /* Test for bit 255 again; only triggered on overflow modulo 2^255-19 */ - "movl $0, %k4 ;" - "cmovnsl %k5, %k4 ;" /* c[255] ? 0 : 19 */ - "btrq $63, %3 ;" /* Clear bit 255 */ - - /* Subtract 19 if necessary */ - "subq %4, %0 ;" - "sbbq $0, %1 ;" - "sbbq $0, %2 ;" - "sbbq $0, %3 ;" - - : "+r"(c[0]), "+r"(c[1]), "+r"(c[2]), "+r"(c[3]), "+r"(tmp0), - "+r"(tmp1) - : - : "memory", "cc"); + u32 i; + fsqr(o, inp, tmp); + for (i = (u32)0U; i < n1 - (u32)1U; i = i + (u32)1U) + fsqr(o, o, tmp); } -static __always_inline void cswap(u8 bit, u64 *const px, u64 *const py) +static void finv(u64 *o, const u64 *i, u64 *tmp) { - u64 temp; - asm volatile( - "test %9, %9 ;" - "movq %0, %8 ;" - "cmovnzq %4, %0 ;" - "cmovnzq %8, %4 ;" - "movq %1, %8 ;" - "cmovnzq %5, %1 ;" - "cmovnzq %8, %5 ;" - "movq %2, %8 ;" - "cmovnzq %6, %2 ;" - "cmovnzq %8, %6 ;" - "movq %3, %8 ;" - "cmovnzq %7, %3 ;" - "cmovnzq %8, %7 ;" - : "+r"(px[0]), "+r"(px[1]), "+r"(px[2]), "+r"(px[3]), - "+r"(py[0]), "+r"(py[1]), "+r"(py[2]), "+r"(py[3]), - "=r"(temp) - : "r"(bit) - : "cc" - ); + u64 t1[16U] = { 0U }; + u64 *a0 = t1; + u64 *b = t1 + (u32)4U; + u64 *c = t1 + (u32)8U; + u64 *t00 = t1 + (u32)12U; + u64 *tmp1 = tmp; + u64 *a; + u64 *t0; + fsquare_times(a0, i, tmp1, (u32)1U); + fsquare_times(t00, a0, tmp1, (u32)2U); + fmul(b, t00, i, tmp); + fmul(a0, b, a0, tmp); + fsquare_times(t00, a0, tmp1, (u32)1U); + fmul(b, t00, b, tmp); + fsquare_times(t00, b, tmp1, (u32)5U); + fmul(b, t00, b, tmp); + fsquare_times(t00, b, tmp1, (u32)10U); + fmul(c, t00, b, tmp); + fsquare_times(t00, c, tmp1, (u32)20U); + fmul(t00, t00, c, tmp); + fsquare_times(t00, t00, tmp1, (u32)10U); + fmul(b, t00, b, tmp); + fsquare_times(t00, b, tmp1, (u32)50U); + fmul(c, t00, b, tmp); + fsquare_times(t00, c, tmp1, (u32)100U); + fmul(t00, t00, c, tmp); + fsquare_times(t00, t00, tmp1, (u32)50U); + fmul(t00, t00, b, tmp); + fsquare_times(t00, t00, tmp1, (u32)5U); + a = t1; + t0 = t1 + (u32)12U; + fmul(o, t0, a, tmp); } -static __always_inline void cselect(u8 bit, u64 *const px, const u64 *const py) +static void store_felem(u64 *b, u64 *f) { - asm volatile( - "test %4, %4 ;" - "cmovnzq %5, %0 ;" - "cmovnzq %6, %1 ;" - "cmovnzq %7, %2 ;" - "cmovnzq %8, %3 ;" - : "+r"(px[0]), "+r"(px[1]), "+r"(px[2]), "+r"(px[3]) - : "r"(bit), "rm"(py[0]), "rm"(py[1]), "rm"(py[2]), "rm"(py[3]) - : "cc" - ); + u64 f30 = f[3U]; + u64 top_bit0 = f30 >> (u32)63U; + u64 carry0; + u64 f31; + u64 top_bit; + u64 carry; + u64 f0; + u64 f1; + u64 f2; + u64 f3; + u64 m0; + u64 m1; + u64 m2; + u64 m3; + u64 mask; + u64 f0_; + u64 f1_; + u64 f2_; + u64 f3_; + u64 o0; + u64 o1; + u64 o2; + u64 o3; + f[3U] = f30 & (u64)0x7fffffffffffffffU; + carry0 = add_scalar(f, f, (u64)19U * top_bit0); + f31 = f[3U]; + top_bit = f31 >> (u32)63U; + f[3U] = f31 & (u64)0x7fffffffffffffffU; + carry = add_scalar(f, f, (u64)19U * top_bit); + f0 = f[0U]; + f1 = f[1U]; + f2 = f[2U]; + f3 = f[3U]; + m0 = gte_mask(f0, (u64)0xffffffffffffffedU); + m1 = eq_mask(f1, (u64)0xffffffffffffffffU); + m2 = eq_mask(f2, (u64)0xffffffffffffffffU); + m3 = eq_mask(f3, (u64)0x7fffffffffffffffU); + mask = ((m0 & m1) & m2) & m3; + f0_ = f0 - (mask & (u64)0xffffffffffffffedU); + f1_ = f1 - (mask & (u64)0xffffffffffffffffU); + f2_ = f2 - (mask & (u64)0xffffffffffffffffU); + f3_ = f3 - (mask & (u64)0x7fffffffffffffffU); + o0 = f0_; + o1 = f1_; + o2 = f2_; + o3 = f3_; + b[0U] = o0; + b[1U] = o1; + b[2U] = o2; + b[3U] = o3; } -static void curve25519_adx(u8 shared[CURVE25519_KEY_SIZE], - const u8 private_key[CURVE25519_KEY_SIZE], - const u8 session_key[CURVE25519_KEY_SIZE]) +static void encode_point(u8 *o, const u64 *i) { - struct { - u64 buffer[4 * NUM_WORDS_ELTFP25519]; - u64 coordinates[4 * NUM_WORDS_ELTFP25519]; - u64 workspace[6 * NUM_WORDS_ELTFP25519]; - u8 session[CURVE25519_KEY_SIZE]; - u8 private[CURVE25519_KEY_SIZE]; - } __aligned(32) m; - - int i = 0, j = 0; - u64 prev = 0; - u64 *const X1 = (u64 *)m.session; - u64 *const key = (u64 *)m.private; - u64 *const Px = m.coordinates + 0; - u64 *const Pz = m.coordinates + 4; - u64 *const Qx = m.coordinates + 8; - u64 *const Qz = m.coordinates + 12; - u64 *const X2 = Qx; - u64 *const Z2 = Qz; - u64 *const X3 = Px; - u64 *const Z3 = Pz; - u64 *const X2Z2 = Qx; - u64 *const X3Z3 = Px; - - u64 *const A = m.workspace + 0; - u64 *const B = m.workspace + 4; - u64 *const D = m.workspace + 8; - u64 *const C = m.workspace + 12; - u64 *const DA = m.workspace + 16; - u64 *const CB = m.workspace + 20; - u64 *const AB = A; - u64 *const DC = D; - u64 *const DACB = DA; - - memcpy(m.private, private_key, sizeof(m.private)); - memcpy(m.session, session_key, sizeof(m.session)); - - curve25519_clamp_secret(m.private); - - /* As in the draft: - * When receiving such an array, implementations of curve25519 - * MUST mask the most-significant bit in the final byte. This - * is done to preserve compatibility with point formats which - * reserve the sign bit for use in other protocols and to - * increase resistance to implementation fingerprinting - */ - m.session[CURVE25519_KEY_SIZE - 1] &= (1 << (255 % 8)) - 1; - - copy_eltfp25519_1w(Px, X1); - setzero_eltfp25519_1w(Pz); - setzero_eltfp25519_1w(Qx); - setzero_eltfp25519_1w(Qz); - - Pz[0] = 1; - Qx[0] = 1; - - /* main-loop */ - prev = 0; - j = 62; - for (i = 3; i >= 0; --i) { - while (j >= 0) { - u64 bit = (key[i] >> j) & 0x1; - u64 swap = bit ^ prev; - prev = bit; - - add_eltfp25519_1w_adx(A, X2, Z2); /* A = (X2+Z2) */ - sub_eltfp25519_1w(B, X2, Z2); /* B = (X2-Z2) */ - add_eltfp25519_1w_adx(C, X3, Z3); /* C = (X3+Z3) */ - sub_eltfp25519_1w(D, X3, Z3); /* D = (X3-Z3) */ - mul_eltfp25519_2w_adx(DACB, AB, DC); /* [DA|CB] = [A|B]*[D|C] */ - - cselect(swap, A, C); - cselect(swap, B, D); - - sqr_eltfp25519_2w_adx(AB); /* [AA|BB] = [A^2|B^2] */ - add_eltfp25519_1w_adx(X3, DA, CB); /* X3 = (DA+CB) */ - sub_eltfp25519_1w(Z3, DA, CB); /* Z3 = (DA-CB) */ - sqr_eltfp25519_2w_adx(X3Z3); /* [X3|Z3] = [(DA+CB)|(DA+CB)]^2 */ - - copy_eltfp25519_1w(X2, B); /* X2 = B^2 */ - sub_eltfp25519_1w(Z2, A, B); /* Z2 = E = AA-BB */ - - mul_a24_eltfp25519_1w(B, Z2); /* B = a24*E */ - add_eltfp25519_1w_adx(B, B, X2); /* B = a24*E+B */ - mul_eltfp25519_2w_adx(X2Z2, X2Z2, AB); /* [X2|Z2] = [B|E]*[A|a24*E+B] */ - mul_eltfp25519_1w_adx(Z3, Z3, X1); /* Z3 = Z3*X1 */ - --j; - } - j = 63; - } - - inv_eltfp25519_1w_adx(A, Qz); - mul_eltfp25519_1w_adx((u64 *)shared, Qx, A); - fred_eltfp25519_1w((u64 *)shared); - - memzero_explicit(&m, sizeof(m)); + const u64 *x = i; + const u64 *z = i + (u32)4U; + u64 tmp[4U] = { 0U }; + u64 tmp_w[16U] = { 0U }; + finv(tmp, z, tmp_w); + fmul(tmp, tmp, x, tmp_w); + store_felem((u64 *)o, tmp); } -static void curve25519_adx_base(u8 session_key[CURVE25519_KEY_SIZE], - const u8 private_key[CURVE25519_KEY_SIZE]) +static void curve25519_ever64(u8 *out, const u8 *priv, const u8 *pub) { - struct { - u64 buffer[4 * NUM_WORDS_ELTFP25519]; - u64 coordinates[4 * NUM_WORDS_ELTFP25519]; - u64 workspace[4 * NUM_WORDS_ELTFP25519]; - u8 private[CURVE25519_KEY_SIZE]; - } __aligned(32) m; - - const int ite[4] = { 64, 64, 64, 63 }; - const int q = 3; - u64 swap = 1; - - int i = 0, j = 0, k = 0; - u64 *const key = (u64 *)m.private; - u64 *const Ur1 = m.coordinates + 0; - u64 *const Zr1 = m.coordinates + 4; - u64 *const Ur2 = m.coordinates + 8; - u64 *const Zr2 = m.coordinates + 12; - - u64 *const UZr1 = m.coordinates + 0; - u64 *const ZUr2 = m.coordinates + 8; - - u64 *const A = m.workspace + 0; - u64 *const B = m.workspace + 4; - u64 *const C = m.workspace + 8; - u64 *const D = m.workspace + 12; - - u64 *const AB = m.workspace + 0; - u64 *const CD = m.workspace + 8; - - const u64 *const P = table_ladder_8k; - - memcpy(m.private, private_key, sizeof(m.private)); - - curve25519_clamp_secret(m.private); - - setzero_eltfp25519_1w(Ur1); - setzero_eltfp25519_1w(Zr1); - setzero_eltfp25519_1w(Zr2); - Ur1[0] = 1; - Zr1[0] = 1; - Zr2[0] = 1; - - /* G-S */ - Ur2[3] = 0x1eaecdeee27cab34UL; - Ur2[2] = 0xadc7a0b9235d48e2UL; - Ur2[1] = 0xbbf095ae14b2edf8UL; - Ur2[0] = 0x7e94e1fec82faabdUL; - - /* main-loop */ - j = q; - for (i = 0; i < NUM_WORDS_ELTFP25519; ++i) { - while (j < ite[i]) { - u64 bit = (key[i] >> j) & 0x1; - k = (64 * i + j - q); - swap = swap ^ bit; - cswap(swap, Ur1, Ur2); - cswap(swap, Zr1, Zr2); - swap = bit; - /* Addition */ - sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ - add_eltfp25519_1w_adx(A, Ur1, Zr1); /* A = Ur1+Zr1 */ - mul_eltfp25519_1w_adx(C, &P[4 * k], B); /* C = M0-B */ - sub_eltfp25519_1w(B, A, C); /* B = (Ur1+Zr1) - M*(Ur1-Zr1) */ - add_eltfp25519_1w_adx(A, A, C); /* A = (Ur1+Zr1) + M*(Ur1-Zr1) */ - sqr_eltfp25519_2w_adx(AB); /* A = A^2 | B = B^2 */ - mul_eltfp25519_2w_adx(UZr1, ZUr2, AB); /* Ur1 = Zr2*A | Zr1 = Ur2*B */ - ++j; + u64 init1[8U] = { 0U }; + u64 tmp[4U] = { 0U }; + u64 tmp3; + u64 *x; + u64 *z; + { + u32 i; + for (i = (u32)0U; i < (u32)4U; i = i + (u32)1U) { + u64 *os = tmp; + const u8 *bj = pub + i * (u32)8U; + u64 u = *(u64 *)bj; + u64 r = u; + u64 x0 = r; + os[i] = x0; } - j = 0; } - - /* Doubling */ - for (i = 0; i < q; ++i) { - add_eltfp25519_1w_adx(A, Ur1, Zr1); /* A = Ur1+Zr1 */ - sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ - sqr_eltfp25519_2w_adx(AB); /* A = A**2 B = B**2 */ - copy_eltfp25519_1w(C, B); /* C = B */ - sub_eltfp25519_1w(B, A, B); /* B = A-B */ - mul_a24_eltfp25519_1w(D, B); /* D = my_a24*B */ - add_eltfp25519_1w_adx(D, D, C); /* D = D+C */ - mul_eltfp25519_2w_adx(UZr1, AB, CD); /* Ur1 = A*B Zr1 = Zr1*A */ - } - - /* Convert to affine coordinates */ - inv_eltfp25519_1w_adx(A, Zr1); - mul_eltfp25519_1w_adx((u64 *)session_key, Ur1, A); - fred_eltfp25519_1w((u64 *)session_key); - - memzero_explicit(&m, sizeof(m)); + tmp3 = tmp[3U]; + tmp[3U] = tmp3 & (u64)0x7fffffffffffffffU; + x = init1; + z = init1 + (u32)4U; + z[0U] = (u64)1U; + z[1U] = (u64)0U; + z[2U] = (u64)0U; + z[3U] = (u64)0U; + x[0U] = tmp[0U]; + x[1U] = tmp[1U]; + x[2U] = tmp[2U]; + x[3U] = tmp[3U]; + montgomery_ladder(init1, priv, init1); + encode_point(out, init1); } -static void curve25519_bmi2(u8 shared[CURVE25519_KEY_SIZE], - const u8 private_key[CURVE25519_KEY_SIZE], - const u8 session_key[CURVE25519_KEY_SIZE]) -{ - struct { - u64 buffer[4 * NUM_WORDS_ELTFP25519]; - u64 coordinates[4 * NUM_WORDS_ELTFP25519]; - u64 workspace[6 * NUM_WORDS_ELTFP25519]; - u8 session[CURVE25519_KEY_SIZE]; - u8 private[CURVE25519_KEY_SIZE]; - } __aligned(32) m; - - int i = 0, j = 0; - u64 prev = 0; - u64 *const X1 = (u64 *)m.session; - u64 *const key = (u64 *)m.private; - u64 *const Px = m.coordinates + 0; - u64 *const Pz = m.coordinates + 4; - u64 *const Qx = m.coordinates + 8; - u64 *const Qz = m.coordinates + 12; - u64 *const X2 = Qx; - u64 *const Z2 = Qz; - u64 *const X3 = Px; - u64 *const Z3 = Pz; - u64 *const X2Z2 = Qx; - u64 *const X3Z3 = Px; - - u64 *const A = m.workspace + 0; - u64 *const B = m.workspace + 4; - u64 *const D = m.workspace + 8; - u64 *const C = m.workspace + 12; - u64 *const DA = m.workspace + 16; - u64 *const CB = m.workspace + 20; - u64 *const AB = A; - u64 *const DC = D; - u64 *const DACB = DA; - - memcpy(m.private, private_key, sizeof(m.private)); - memcpy(m.session, session_key, sizeof(m.session)); - - curve25519_clamp_secret(m.private); - - /* As in the draft: - * When receiving such an array, implementations of curve25519 - * MUST mask the most-significant bit in the final byte. This - * is done to preserve compatibility with point formats which - * reserve the sign bit for use in other protocols and to - * increase resistance to implementation fingerprinting - */ - m.session[CURVE25519_KEY_SIZE - 1] &= (1 << (255 % 8)) - 1; - - copy_eltfp25519_1w(Px, X1); - setzero_eltfp25519_1w(Pz); - setzero_eltfp25519_1w(Qx); - setzero_eltfp25519_1w(Qz); - - Pz[0] = 1; - Qx[0] = 1; - - /* main-loop */ - prev = 0; - j = 62; - for (i = 3; i >= 0; --i) { - while (j >= 0) { - u64 bit = (key[i] >> j) & 0x1; - u64 swap = bit ^ prev; - prev = bit; - - add_eltfp25519_1w_bmi2(A, X2, Z2); /* A = (X2+Z2) */ - sub_eltfp25519_1w(B, X2, Z2); /* B = (X2-Z2) */ - add_eltfp25519_1w_bmi2(C, X3, Z3); /* C = (X3+Z3) */ - sub_eltfp25519_1w(D, X3, Z3); /* D = (X3-Z3) */ - mul_eltfp25519_2w_bmi2(DACB, AB, DC); /* [DA|CB] = [A|B]*[D|C] */ - - cselect(swap, A, C); - cselect(swap, B, D); - - sqr_eltfp25519_2w_bmi2(AB); /* [AA|BB] = [A^2|B^2] */ - add_eltfp25519_1w_bmi2(X3, DA, CB); /* X3 = (DA+CB) */ - sub_eltfp25519_1w(Z3, DA, CB); /* Z3 = (DA-CB) */ - sqr_eltfp25519_2w_bmi2(X3Z3); /* [X3|Z3] = [(DA+CB)|(DA+CB)]^2 */ - - copy_eltfp25519_1w(X2, B); /* X2 = B^2 */ - sub_eltfp25519_1w(Z2, A, B); /* Z2 = E = AA-BB */ - - mul_a24_eltfp25519_1w(B, Z2); /* B = a24*E */ - add_eltfp25519_1w_bmi2(B, B, X2); /* B = a24*E+B */ - mul_eltfp25519_2w_bmi2(X2Z2, X2Z2, AB); /* [X2|Z2] = [B|E]*[A|a24*E+B] */ - mul_eltfp25519_1w_bmi2(Z3, Z3, X1); /* Z3 = Z3*X1 */ - --j; - } - j = 63; - } - - inv_eltfp25519_1w_bmi2(A, Qz); - mul_eltfp25519_1w_bmi2((u64 *)shared, Qx, A); - fred_eltfp25519_1w((u64 *)shared); +/* The below constants were generated using this sage script: + * + * #!/usr/bin/env sage + * import sys + * from sage.all import * + * def limbs(n): + * n = int(n) + * l = ((n >> 0) % 2^64, (n >> 64) % 2^64, (n >> 128) % 2^64, (n >> 192) % 2^64) + * return "0x%016xULL, 0x%016xULL, 0x%016xULL, 0x%016xULL" % l + * ec = EllipticCurve(GF(2^255 - 19), [0, 486662, 0, 1, 0]) + * p_minus_s = (ec.lift_x(9) - ec.lift_x(1))[0] + * print("static const u64 p_minus_s[] = { %s };\n" % limbs(p_minus_s)) + * print("static const u64 table_ladder[] = {") + * p = ec.lift_x(9) + * for i in range(252): + * l = (p[0] + p[2]) / (p[0] - p[2]) + * print(("\t%s" + ("," if i != 251 else "")) % limbs(l)) + * p = p * 2 + * print("};") + * + */ - memzero_explicit(&m, sizeof(m)); -} +static const u64 p_minus_s[] = { 0x816b1e0137d48290ULL, 0x440f6a51eb4d1207ULL, 0x52385f46dca2b71dULL, 0x215132111d8354cbULL }; + +static const u64 table_ladder[] = { + 0xfffffffffffffff3ULL, 0xffffffffffffffffULL, 0xffffffffffffffffULL, 0x5fffffffffffffffULL, + 0x6b8220f416aafe96ULL, 0x82ebeb2b4f566a34ULL, 0xd5a9a5b075a5950fULL, 0x5142b2cf4b2488f4ULL, + 0x6aaebc750069680cULL, 0x89cf7820a0f99c41ULL, 0x2a58d9183b56d0f4ULL, 0x4b5aca80e36011a4ULL, + 0x329132348c29745dULL, 0xf4a2e616e1642fd7ULL, 0x1e45bb03ff67bc34ULL, 0x306912d0f42a9b4aULL, + 0xff886507e6af7154ULL, 0x04f50e13dfeec82fULL, 0xaa512fe82abab5ceULL, 0x174e251a68d5f222ULL, + 0xcf96700d82028898ULL, 0x1743e3370a2c02c5ULL, 0x379eec98b4e86eaaULL, 0x0c59888a51e0482eULL, + 0xfbcbf1d699b5d189ULL, 0xacaef0d58e9fdc84ULL, 0xc1c20d06231f7614ULL, 0x2938218da274f972ULL, + 0xf6af49beff1d7f18ULL, 0xcc541c22387ac9c2ULL, 0x96fcc9ef4015c56bULL, 0x69c1627c690913a9ULL, + 0x7a86fd2f4733db0eULL, 0xfdb8c4f29e087de9ULL, 0x095e4b1a8ea2a229ULL, 0x1ad7a7c829b37a79ULL, + 0x342d89cad17ea0c0ULL, 0x67bedda6cced2051ULL, 0x19ca31bf2bb42f74ULL, 0x3df7b4c84980acbbULL, + 0xa8c6444dc80ad883ULL, 0xb91e440366e3ab85ULL, 0xc215cda00164f6d8ULL, 0x3d867c6ef247e668ULL, + 0xc7dd582bcc3e658cULL, 0xfd2c4748ee0e5528ULL, 0xa0fd9b95cc9f4f71ULL, 0x7529d871b0675ddfULL, + 0xb8f568b42d3cbd78ULL, 0x1233011b91f3da82ULL, 0x2dce6ccd4a7c3b62ULL, 0x75e7fc8e9e498603ULL, + 0x2f4f13f1fcd0b6ecULL, 0xf1a8ca1f29ff7a45ULL, 0xc249c1a72981e29bULL, 0x6ebe0dbb8c83b56aULL, + 0x7114fa8d170bb222ULL, 0x65a2dcd5bf93935fULL, 0xbdc41f68b59c979aULL, 0x2f0eef79a2ce9289ULL, + 0x42ecbf0c083c37ceULL, 0x2930bc09ec496322ULL, 0xf294b0c19cfeac0dULL, 0x3780aa4bedfabb80ULL, + 0x56c17d3e7cead929ULL, 0xe7cb4beb2e5722c5ULL, 0x0ce931732dbfe15aULL, 0x41b883c7621052f8ULL, + 0xdbf75ca0c3d25350ULL, 0x2936be086eb1e351ULL, 0xc936e03cb4a9b212ULL, 0x1d45bf82322225aaULL, + 0xe81ab1036a024cc5ULL, 0xe212201c304c9a72ULL, 0xc5d73fba6832b1fcULL, 0x20ffdb5a4d839581ULL, + 0xa283d367be5d0fadULL, 0x6c2b25ca8b164475ULL, 0x9d4935467caaf22eULL, 0x5166408eee85ff49ULL, + 0x3c67baa2fab4e361ULL, 0xb3e433c67ef35cefULL, 0x5259729241159b1cULL, 0x6a621892d5b0ab33ULL, + 0x20b74a387555cdcbULL, 0x532aa10e1208923fULL, 0xeaa17b7762281dd1ULL, 0x61ab3443f05c44bfULL, + 0x257a6c422324def8ULL, 0x131c6c1017e3cf7fULL, 0x23758739f630a257ULL, 0x295a407a01a78580ULL, + 0xf8c443246d5da8d9ULL, 0x19d775450c52fa5dULL, 0x2afcfc92731bf83dULL, 0x7d10c8e81b2b4700ULL, + 0xc8e0271f70baa20bULL, 0x993748867ca63957ULL, 0x5412efb3cb7ed4bbULL, 0x3196d36173e62975ULL, + 0xde5bcad141c7dffcULL, 0x47cc8cd2b395c848ULL, 0xa34cd942e11af3cbULL, 0x0256dbf2d04ecec2ULL, + 0x875ab7e94b0e667fULL, 0xcad4dd83c0850d10ULL, 0x47f12e8f4e72c79fULL, 0x5f1a87bb8c85b19bULL, + 0x7ae9d0b6437f51b8ULL, 0x12c7ce5518879065ULL, 0x2ade09fe5cf77aeeULL, 0x23a05a2f7d2c5627ULL, + 0x5908e128f17c169aULL, 0xf77498dd8ad0852dULL, 0x74b4c4ceab102f64ULL, 0x183abadd10139845ULL, + 0xb165ba8daa92aaacULL, 0xd5c5ef9599386705ULL, 0xbe2f8f0cf8fc40d1ULL, 0x2701e635ee204514ULL, + 0x629fa80020156514ULL, 0xf223868764a8c1ceULL, 0x5b894fff0b3f060eULL, 0x60d9944cf708a3faULL, + 0xaeea001a1c7a201fULL, 0xebf16a633ee2ce63ULL, 0x6f7709594c7a07e1ULL, 0x79b958150d0208cbULL, + 0x24b55e5301d410e7ULL, 0xe3a34edff3fdc84dULL, 0xd88768e4904032d8ULL, 0x131384427b3aaeecULL, + 0x8405e51286234f14ULL, 0x14dc4739adb4c529ULL, 0xb8a2b5b250634ffdULL, 0x2fe2a94ad8a7ff93ULL, + 0xec5c57efe843faddULL, 0x2843ce40f0bb9918ULL, 0xa4b561d6cf3d6305ULL, 0x743629bde8fb777eULL, + 0x343edd46bbaf738fULL, 0xed981828b101a651ULL, 0xa401760b882c797aULL, 0x1fc223e28dc88730ULL, + 0x48604e91fc0fba0eULL, 0xb637f78f052c6fa4ULL, 0x91ccac3d09e9239cULL, 0x23f7eed4437a687cULL, + 0x5173b1118d9bd800ULL, 0x29d641b63189d4a7ULL, 0xfdbf177988bbc586ULL, 0x2959894fcad81df5ULL, + 0xaebc8ef3b4bbc899ULL, 0x4148995ab26992b9ULL, 0x24e20b0134f92cfbULL, 0x40d158894a05dee8ULL, + 0x46b00b1185af76f6ULL, 0x26bac77873187a79ULL, 0x3dc0bf95ab8fff5fULL, 0x2a608bd8945524d7ULL, + 0x26449588bd446302ULL, 0x7c4bc21c0388439cULL, 0x8e98a4f383bd11b2ULL, 0x26218d7bc9d876b9ULL, + 0xe3081542997c178aULL, 0x3c2d29a86fb6606fULL, 0x5c217736fa279374ULL, 0x7dde05734afeb1faULL, + 0x3bf10e3906d42babULL, 0xe4f7803e1980649cULL, 0xe6053bf89595bf7aULL, 0x394faf38da245530ULL, + 0x7a8efb58896928f4ULL, 0xfbc778e9cc6a113cULL, 0x72670ce330af596fULL, 0x48f222a81d3d6cf7ULL, + 0xf01fce410d72caa7ULL, 0x5a20ecc7213b5595ULL, 0x7bc21165c1fa1483ULL, 0x07f89ae31da8a741ULL, + 0x05d2c2b4c6830ff9ULL, 0xd43e330fc6316293ULL, 0xa5a5590a96d3a904ULL, 0x705edb91a65333b6ULL, + 0x048ee15e0bb9a5f7ULL, 0x3240cfca9e0aaf5dULL, 0x8f4b71ceedc4a40bULL, 0x621c0da3de544a6dULL, + 0x92872836a08c4091ULL, 0xce8375b010c91445ULL, 0x8a72eb524f276394ULL, 0x2667fcfa7ec83635ULL, + 0x7f4c173345e8752aULL, 0x061b47feee7079a5ULL, 0x25dd9afa9f86ff34ULL, 0x3780cef5425dc89cULL, + 0x1a46035a513bb4e9ULL, 0x3e1ef379ac575adaULL, 0xc78c5f1c5fa24b50ULL, 0x321a967634fd9f22ULL, + 0x946707b8826e27faULL, 0x3dca84d64c506fd0ULL, 0xc189218075e91436ULL, 0x6d9284169b3b8484ULL, + 0x3a67e840383f2ddfULL, 0x33eec9a30c4f9b75ULL, 0x3ec7c86fa783ef47ULL, 0x26ec449fbac9fbc4ULL, + 0x5c0f38cba09b9e7dULL, 0x81168cc762a3478cULL, 0x3e23b0d306fc121cULL, 0x5a238aa0a5efdcddULL, + 0x1ba26121c4ea43ffULL, 0x36f8c77f7c8832b5ULL, 0x88fbea0b0adcf99aULL, 0x5ca9938ec25bebf9ULL, + 0xd5436a5e51fccda0ULL, 0x1dbc4797c2cd893bULL, 0x19346a65d3224a08ULL, 0x0f5034e49b9af466ULL, + 0xf23c3967a1e0b96eULL, 0xe58b08fa867a4d88ULL, 0xfb2fabc6a7341679ULL, 0x2a75381eb6026946ULL, + 0xc80a3be4c19420acULL, 0x66b1f6c681f2b6dcULL, 0x7cf7036761e93388ULL, 0x25abbbd8a660a4c4ULL, + 0x91ea12ba14fd5198ULL, 0x684950fc4a3cffa9ULL, 0xf826842130f5ad28ULL, 0x3ea988f75301a441ULL, + 0xc978109a695f8c6fULL, 0x1746eb4a0530c3f3ULL, 0x444d6d77b4459995ULL, 0x75952b8c054e5cc7ULL, + 0xa3703f7915f4d6aaULL, 0x66c346202f2647d8ULL, 0xd01469df811d644bULL, 0x77fea47d81a5d71fULL, + 0xc5e9529ef57ca381ULL, 0x6eeeb4b9ce2f881aULL, 0xb6e91a28e8009bd6ULL, 0x4b80be3e9afc3fecULL, + 0x7e3773c526aed2c5ULL, 0x1b4afcb453c9a49dULL, 0xa920bdd7baffb24dULL, 0x7c54699f122d400eULL, + 0xef46c8e14fa94bc8ULL, 0xe0b074ce2952ed5eULL, 0xbea450e1dbd885d5ULL, 0x61b68649320f712cULL, + 0x8a485f7309ccbdd1ULL, 0xbd06320d7d4d1a2dULL, 0x25232973322dbef4ULL, 0x445dc4758c17f770ULL, + 0xdb0434177cc8933cULL, 0xed6fe82175ea059fULL, 0x1efebefdc053db34ULL, 0x4adbe867c65daf99ULL, + 0x3acd71a2a90609dfULL, 0xe5e991856dd04050ULL, 0x1ec69b688157c23cULL, 0x697427f6885cfe4dULL, + 0xd7be7b9b65e1a851ULL, 0xa03d28d522c536ddULL, 0x28399d658fd2b645ULL, 0x49e5b7e17c2641e1ULL, + 0x6f8c3a98700457a4ULL, 0x5078f0a25ebb6778ULL, 0xd13c3ccbc382960fULL, 0x2e003258a7df84b1ULL, + 0x8ad1f39be6296a1cULL, 0xc1eeaa652a5fbfb2ULL, 0x33ee0673fd26f3cbULL, 0x59256173a69d2cccULL, + 0x41ea07aa4e18fc41ULL, 0xd9fc19527c87a51eULL, 0xbdaacb805831ca6fULL, 0x445b652dc916694fULL, + 0xce92a3a7f2172315ULL, 0x1edc282de11b9964ULL, 0xa1823aafe04c314aULL, 0x790a2d94437cf586ULL, + 0x71c447fb93f6e009ULL, 0x8922a56722845276ULL, 0xbf70903b204f5169ULL, 0x2f7a89891ba319feULL, + 0x02a08eb577e2140cULL, 0xed9a4ed4427bdcf4ULL, 0x5253ec44e4323cd1ULL, 0x3e88363c14e9355bULL, + 0xaa66c14277110b8cULL, 0x1ae0391610a23390ULL, 0x2030bd12c93fc2a2ULL, 0x3ee141579555c7abULL, + 0x9214de3a6d6e7d41ULL, 0x3ccdd88607f17efeULL, 0x674f1288f8e11217ULL, 0x5682250f329f93d0ULL, + 0x6cf00b136d2e396eULL, 0x6e4cf86f1014debfULL, 0x5930b1b5bfcc4e83ULL, 0x047069b48aba16b6ULL, + 0x0d4ce4ab69b20793ULL, 0xb24db91a97d0fb9eULL, 0xcdfa50f54e00d01dULL, 0x221b1085368bddb5ULL, + 0xe7e59468b1e3d8d2ULL, 0x53c56563bd122f93ULL, 0xeee8a903e0663f09ULL, 0x61efa662cbbe3d42ULL, + 0x2cf8ddddde6eab2aULL, 0x9bf80ad51435f231ULL, 0x5deadacec9f04973ULL, 0x29275b5d41d29b27ULL, + 0xcfde0f0895ebf14fULL, 0xb9aab96b054905a7ULL, 0xcae80dd9a1c420fdULL, 0x0a63bf2f1673bbc7ULL, + 0x092f6e11958fbc8cULL, 0x672a81e804822fadULL, 0xcac8351560d52517ULL, 0x6f3f7722c8f192f8ULL, + 0xf8ba90ccc2e894b7ULL, 0x2c7557a438ff9f0dULL, 0x894d1d855ae52359ULL, 0x68e122157b743d69ULL, + 0xd87e5570cfb919f3ULL, 0x3f2cdecd95798db9ULL, 0x2121154710c0a2ceULL, 0x3c66a115246dc5b2ULL, + 0xcbedc562294ecb72ULL, 0xba7143c36a280b16ULL, 0x9610c2efd4078b67ULL, 0x6144735d946a4b1eULL, + 0x536f111ed75b3350ULL, 0x0211db8c2041d81bULL, 0xf93cb1000e10413cULL, 0x149dfd3c039e8876ULL, + 0xd479dde46b63155bULL, 0xb66e15e93c837976ULL, 0xdafde43b1f13e038ULL, 0x5fafda1a2e4b0b35ULL, + 0x3600bbdf17197581ULL, 0x3972050bbe3cd2c2ULL, 0x5938906dbdd5be86ULL, 0x34fce5e43f9b860fULL, + 0x75a8a4cd42d14d02ULL, 0x828dabc53441df65ULL, 0x33dcabedd2e131d3ULL, 0x3ebad76fb814d25fULL, + 0xd4906f566f70e10fULL, 0x5d12f7aa51690f5aULL, 0x45adb16e76cefcf2ULL, 0x01f768aead232999ULL, + 0x2b6cc77b6248febdULL, 0x3cd30628ec3aaffdULL, 0xce1c0b80d4ef486aULL, 0x4c3bff2ea6f66c23ULL, + 0x3f2ec4094aeaeb5fULL, 0x61b19b286e372ca7ULL, 0x5eefa966de2a701dULL, 0x23b20565de55e3efULL, + 0xe301ca5279d58557ULL, 0x07b2d4ce27c2874fULL, 0xa532cd8a9dcf1d67ULL, 0x2a52fee23f2bff56ULL, + 0x8624efb37cd8663dULL, 0xbbc7ac20ffbd7594ULL, 0x57b85e9c82d37445ULL, 0x7b3052cb86a6ec66ULL, + 0x3482f0ad2525e91eULL, 0x2cb68043d28edca0ULL, 0xaf4f6d052e1b003aULL, 0x185f8c2529781b0aULL, + 0xaa41de5bd80ce0d6ULL, 0x9407b2416853e9d6ULL, 0x563ec36e357f4c3aULL, 0x4cc4b8dd0e297bceULL, + 0xa2fc1a52ffb8730eULL, 0x1811f16e67058e37ULL, 0x10f9a366cddf4ee1ULL, 0x72f4a0c4a0b9f099ULL, + 0x8c16c06f663f4ea7ULL, 0x693b3af74e970fbaULL, 0x2102e7f1d69ec345ULL, 0x0ba53cbc968a8089ULL, + 0xca3d9dc7fea15537ULL, 0x4c6824bb51536493ULL, 0xb9886314844006b1ULL, 0x40d2a72ab454cc60ULL, + 0x5936a1b712570975ULL, 0x91b9d648debda657ULL, 0x3344094bb64330eaULL, 0x006ba10d12ee51d0ULL, + 0x19228468f5de5d58ULL, 0x0eb12f4c38cc05b0ULL, 0xa1039f9dd5601990ULL, 0x4502d4ce4fff0e0bULL, + 0xeb2054106837c189ULL, 0xd0f6544c6dd3b93cULL, 0x40727064c416d74fULL, 0x6e15c6114b502ef0ULL, + 0x4df2a398cfb1a76bULL, 0x11256c7419f2f6b1ULL, 0x4a497962066e6043ULL, 0x705b3aab41355b44ULL, + 0x365ef536d797b1d8ULL, 0x00076bd622ddf0dbULL, 0x3bbf33b0e0575a88ULL, 0x3777aa05c8e4ca4dULL, + 0x392745c85578db5fULL, 0x6fda4149dbae5ae2ULL, 0xb1f0b00b8adc9867ULL, 0x09963437d36f1da3ULL, + 0x7e824e90a5dc3853ULL, 0xccb5f6641f135cbdULL, 0x6736d86c87ce8fccULL, 0x625f3ce26604249fULL, + 0xaf8ac8059502f63fULL, 0x0c05e70a2e351469ULL, 0x35292e9c764b6305ULL, 0x1a394360c7e23ac3ULL, + 0xd5c6d53251183264ULL, 0x62065abd43c2b74fULL, 0xb5fbf5d03b973f9bULL, 0x13a3da3661206e5eULL, + 0xc6bd5837725d94e5ULL, 0x18e30912205016c5ULL, 0x2088ce1570033c68ULL, 0x7fba1f495c837987ULL, + 0x5a8c7423f2f9079dULL, 0x1735157b34023fc5ULL, 0xe4f9b49ad2fab351ULL, 0x6691ff72c878e33cULL, + 0x122c2adedc5eff3eULL, 0xf8dd4bf1d8956cf4ULL, 0xeb86205d9e9e5bdaULL, 0x049b92b9d975c743ULL, + 0xa5379730b0f6c05aULL, 0x72a0ffacc6f3a553ULL, 0xb0032c34b20dcd6dULL, 0x470e9dbc88d5164aULL, + 0xb19cf10ca237c047ULL, 0xb65466711f6c81a2ULL, 0xb3321bd16dd80b43ULL, 0x48c14f600c5fbe8eULL, + 0x66451c264aa6c803ULL, 0xb66e3904a4fa7da6ULL, 0xd45f19b0b3128395ULL, 0x31602627c3c9bc10ULL, + 0x3120dc4832e4e10dULL, 0xeb20c46756c717f7ULL, 0x00f52e3f67280294ULL, 0x566d4fc14730c509ULL, + 0x7e3a5d40fd837206ULL, 0xc1e926dc7159547aULL, 0x216730fba68d6095ULL, 0x22e8c3843f69cea7ULL, + 0x33d074e8930e4b2bULL, 0xb6e4350e84d15816ULL, 0x5534c26ad6ba2365ULL, 0x7773c12f89f1f3f3ULL, + 0x8cba404da57962aaULL, 0x5b9897a81999ce56ULL, 0x508e862f121692fcULL, 0x3a81907fa093c291ULL, + 0x0dded0ff4725a510ULL, 0x10d8cc10673fc503ULL, 0x5b9d151c9f1f4e89ULL, 0x32a5c1d5cb09a44cULL, + 0x1e0aa442b90541fbULL, 0x5f85eb7cc1b485dbULL, 0xbee595ce8a9df2e5ULL, 0x25e496c722422236ULL, + 0x5edf3c46cd0fe5b9ULL, 0x34e75a7ed2a43388ULL, 0xe488de11d761e352ULL, 0x0e878a01a085545cULL, + 0xba493c77e021bb04ULL, 0x2b4d1843c7df899aULL, 0x9ea37a487ae80d67ULL, 0x67a9958011e41794ULL, + 0x4b58051a6697b065ULL, 0x47e33f7d8d6ba6d4ULL, 0xbb4da8d483ca46c1ULL, 0x68becaa181c2db0dULL, + 0x8d8980e90b989aa5ULL, 0xf95eb14a2c93c99bULL, 0x51c6c7c4796e73a2ULL, 0x6e228363b5efb569ULL, + 0xc6bbc0b02dd624c8ULL, 0x777eb47dec8170eeULL, 0x3cde15a004cfafa9ULL, 0x1dc6bc087160bf9bULL, + 0x2e07e043eec34002ULL, 0x18e9fc677a68dc7fULL, 0xd8da03188bd15b9aULL, 0x48fbc3bb00568253ULL, + 0x57547d4cfb654ce1ULL, 0xd3565b82a058e2adULL, 0xf63eaf0bbf154478ULL, 0x47531ef114dfbb18ULL, + 0xe1ec630a4278c587ULL, 0x5507d546ca8e83f3ULL, 0x85e135c63adc0c2bULL, 0x0aa7efa85682844eULL, + 0x72691ba8b3e1f615ULL, 0x32b4e9701fbe3ffaULL, 0x97b6d92e39bb7868ULL, 0x2cfe53dea02e39e8ULL, + 0x687392cd85cd52b0ULL, 0x27ff66c910e29831ULL, 0x97134556a9832d06ULL, 0x269bb0360a84f8a0ULL, + 0x706e55457643f85cULL, 0x3734a48c9b597d1bULL, 0x7aee91e8c6efa472ULL, 0x5cd6abc198a9d9e0ULL, + 0x0e04de06cb3ce41aULL, 0xd8c6eb893402e138ULL, 0x904659bb686e3772ULL, 0x7215c371746ba8c8ULL, + 0xfd12a97eeae4a2d9ULL, 0x9514b7516394f2c5ULL, 0x266fd5809208f294ULL, 0x5c847085619a26b9ULL, + 0x52985410fed694eaULL, 0x3c905b934a2ed254ULL, 0x10bb47692d3be467ULL, 0x063b3d2d69e5e9e1ULL, + 0x472726eedda57debULL, 0xefb6c4ae10f41891ULL, 0x2b1641917b307614ULL, 0x117c554fc4f45b7cULL, + 0xc07cf3118f9d8812ULL, 0x01dbd82050017939ULL, 0xd7e803f4171b2827ULL, 0x1015e87487d225eaULL, + 0xc58de3fed23acc4dULL, 0x50db91c294a7be2dULL, 0x0b94d43d1c9cf457ULL, 0x6b1640fa6e37524aULL, + 0x692f346c5fda0d09ULL, 0x200b1c59fa4d3151ULL, 0xb8c46f760777a296ULL, 0x4b38395f3ffdfbcfULL, + 0x18d25e00be54d671ULL, 0x60d50582bec8aba6ULL, 0x87ad8f263b78b982ULL, 0x50fdf64e9cda0432ULL, + 0x90f567aac578dcf0ULL, 0xef1e9b0ef2a3133bULL, 0x0eebba9242d9de71ULL, 0x15473c9bf03101c7ULL, + 0x7c77e8ae56b78095ULL, 0xb678e7666e6f078eULL, 0x2da0b9615348ba1fULL, 0x7cf931c1ff733f0bULL, + 0x26b357f50a0a366cULL, 0xe9708cf42b87d732ULL, 0xc13aeea5f91cb2c0ULL, 0x35d90c991143bb4cULL, + 0x47c1c404a9a0d9dcULL, 0x659e58451972d251ULL, 0x3875a8c473b38c31ULL, 0x1fbd9ed379561f24ULL, + 0x11fabc6fd41ec28dULL, 0x7ef8dfe3cd2a2dcaULL, 0x72e73b5d8c404595ULL, 0x6135fa4954b72f27ULL, + 0xccfc32a2de24b69cULL, 0x3f55698c1f095d88ULL, 0xbe3350ed5ac3f929ULL, 0x5e9bf806ca477eebULL, + 0xe9ce8fb63c309f68ULL, 0x5376f63565e1f9f4ULL, 0xd1afcfb35a6393f1ULL, 0x6632a1ede5623506ULL, + 0x0b7d6c390c2ded4cULL, 0x56cb3281df04cb1fULL, 0x66305a1249ecc3c7ULL, 0x5d588b60a38ca72aULL, + 0xa6ecbf78e8e5f42dULL, 0x86eeb44b3c8a3eecULL, 0xec219c48fbd21604ULL, 0x1aaf1af517c36731ULL, + 0xc306a2836769bde7ULL, 0x208280622b1e2adbULL, 0x8027f51ffbff94a6ULL, 0x76cfa1ce1124f26bULL, + 0x18eb00562422abb6ULL, 0xf377c4d58f8c29c3ULL, 0x4dbbc207f531561aULL, 0x0253b7f082128a27ULL, + 0x3d1f091cb62c17e0ULL, 0x4860e1abd64628a9ULL, 0x52d17436309d4253ULL, 0x356f97e13efae576ULL, + 0xd351e11aa150535bULL, 0x3e6b45bb1dd878ccULL, 0x0c776128bed92c98ULL, 0x1d34ae93032885b8ULL, + 0x4ba0488ca85ba4c3ULL, 0x985348c33c9ce6ceULL, 0x66124c6f97bda770ULL, 0x0f81a0290654124aULL, + 0x9ed09ca6569b86fdULL, 0x811009fd18af9a2dULL, 0xff08d03f93d8c20aULL, 0x52a148199faef26bULL, + 0x3e03f9dc2d8d1b73ULL, 0x4205801873961a70ULL, 0xc0d987f041a35970ULL, 0x07aa1f15a1c0d549ULL, + 0xdfd46ce08cd27224ULL, 0x6d0a024f934e4239ULL, 0x808a7a6399897b59ULL, 0x0a4556e9e13d95a2ULL, + 0xd21a991fe9c13045ULL, 0x9b0e8548fe7751b8ULL, 0x5da643cb4bf30035ULL, 0x77db28d63940f721ULL, + 0xfc5eeb614adc9011ULL, 0x5229419ae8c411ebULL, 0x9ec3e7787d1dcf74ULL, 0x340d053e216e4cb5ULL, + 0xcac7af39b48df2b4ULL, 0xc0faec2871a10a94ULL, 0x140a69245ca575edULL, 0x0cf1c37134273a4cULL, + 0xc8ee306ac224b8a5ULL, 0x57eaee7ccb4930b0ULL, 0xa1e806bdaacbe74fULL, 0x7d9a62742eeb657dULL, + 0x9eb6b6ef546c4830ULL, 0x885cca1fddb36e2eULL, 0xe6b9f383ef0d7105ULL, 0x58654fef9d2e0412ULL, + 0xa905c4ffbe0e8e26ULL, 0x942de5df9b31816eULL, 0x497d723f802e88e1ULL, 0x30684dea602f408dULL, + 0x21e5a278a3e6cb34ULL, 0xaefb6e6f5b151dc4ULL, 0xb30b8e049d77ca15ULL, 0x28c3c9cf53b98981ULL, + 0x287fb721556cdd2aULL, 0x0d317ca897022274ULL, 0x7468c7423a543258ULL, 0x4a7f11464eb5642fULL, + 0xa237a4774d193aa6ULL, 0xd865986ea92129a1ULL, 0x24c515ecf87c1a88ULL, 0x604003575f39f5ebULL, + 0x47b9f189570a9b27ULL, 0x2b98cede465e4b78ULL, 0x026df551dbb85c20ULL, 0x74fcd91047e21901ULL, + 0x13e2a90a23c1bfa3ULL, 0x0cb0074e478519f6ULL, 0x5ff1cbbe3af6cf44ULL, 0x67fe5438be812dbeULL, + 0xd13cf64fa40f05b0ULL, 0x054dfb2f32283787ULL, 0x4173915b7f0d2aeaULL, 0x482f144f1f610d4eULL, + 0xf6210201b47f8234ULL, 0x5d0ae1929e70b990ULL, 0xdcd7f455b049567cULL, 0x7e93d0f1f0916f01ULL, + 0xdd79cbf18a7db4faULL, 0xbe8391bf6f74c62fULL, 0x027145d14b8291bdULL, 0x585a73ea2cbf1705ULL, + 0x485ca03e928a0db2ULL, 0x10fc01a5742857e7ULL, 0x2f482edbd6d551a7ULL, 0x0f0433b5048fdb8aULL, + 0x60da2e8dd7dc6247ULL, 0x88b4c9d38cd4819aULL, 0x13033ac001f66697ULL, 0x273b24fe3b367d75ULL, + 0xc6e8f66a31b3b9d4ULL, 0x281514a494df49d5ULL, 0xd1726fdfc8b23da7ULL, 0x4b3ae7d103dee548ULL, + 0xc6256e19ce4b9d7eULL, 0xff5c5cf186e3c61cULL, 0xacc63ca34b8ec145ULL, 0x74621888fee66574ULL, + 0x956f409645290a1eULL, 0xef0bf8e3263a962eULL, 0xed6a50eb5ec2647bULL, 0x0694283a9dca7502ULL, + 0x769b963643a2dcd1ULL, 0x42b7c8ea09fc5353ULL, 0x4f002aee13397eabULL, 0x63005e2c19b7d63aULL, + 0xca6736da63023beaULL, 0x966c7f6db12a99b7ULL, 0xace09390c537c5e1ULL, 0x0b696063a1aa89eeULL, + 0xebb03e97288c56e5ULL, 0x432a9f9f938c8be8ULL, 0xa6a5a93d5b717f71ULL, 0x1a5fb4c3e18f9d97ULL, + 0x1c94e7ad1c60cdceULL, 0xee202a43fc02c4a0ULL, 0x8dafe4d867c46a20ULL, 0x0a10263c8ac27b58ULL, + 0xd0dea9dfe4432a4aULL, 0x856af87bbe9277c5ULL, 0xce8472acc212c71aULL, 0x6f151b6d9bbb1e91ULL, + 0x26776c527ceed56aULL, 0x7d211cb7fbf8faecULL, 0x37ae66a6fd4609ccULL, 0x1f81b702d2770c42ULL, + 0x2fb0b057eac58392ULL, 0xe1dd89fe29744e9dULL, 0xc964f8eb17beb4f8ULL, 0x29571073c9a2d41eULL, + 0xa948a18981c0e254ULL, 0x2df6369b65b22830ULL, 0xa33eb2d75fcfd3c6ULL, 0x078cd6ec4199a01fULL, + 0x4a584a41ad900d2fULL, 0x32142b78e2c74c52ULL, 0x68c4e8338431c978ULL, 0x7f69ea9008689fc2ULL, + 0x52f2c81e46a38265ULL, 0xfd78072d04a832fdULL, 0x8cd7d5fa25359e94ULL, 0x4de71b7454cc29d2ULL, + 0x42eb60ad1eda6ac9ULL, 0x0aad37dfdbc09c3aULL, 0x81004b71e33cc191ULL, 0x44e6be345122803cULL, + 0x03fe8388ba1920dbULL, 0xf5d57c32150db008ULL, 0x49c8c4281af60c29ULL, 0x21edb518de701aeeULL, + 0x7fb63e418f06dc99ULL, 0xa4460d99c166d7b8ULL, 0x24dd5248ce520a83ULL, 0x5ec3ad712b928358ULL, + 0x15022a5fbd17930fULL, 0xa4f64a77d82570e3ULL, 0x12bc8d6915783712ULL, 0x498194c0fc620abbULL, + 0x38a2d9d255686c82ULL, 0x785c6bd9193e21f0ULL, 0xe4d5c81ab24a5484ULL, 0x56307860b2e20989ULL, + 0x429d55f78b4d74c4ULL, 0x22f1834643350131ULL, 0x1e60c24598c71fffULL, 0x59f2f014979983efULL, + 0x46a47d56eb494a44ULL, 0x3e22a854d636a18eULL, 0xb346e15274491c3bULL, 0x2ceafd4e5390cde7ULL, + 0xba8a8538be0d6675ULL, 0x4b9074bb50818e23ULL, 0xcbdab89085d304c3ULL, 0x61a24fe0e56192c4ULL, + 0xcb7615e6db525bcbULL, 0xdd7d8c35a567e4caULL, 0xe6b4153acafcdd69ULL, 0x2d668e097f3c9766ULL, + 0xa57e7e265ce55ef0ULL, 0x5d9f4e527cd4b967ULL, 0xfbc83606492fd1e5ULL, 0x090d52beb7c3f7aeULL, + 0x09b9515a1e7b4d7cULL, 0x1f266a2599da44c0ULL, 0xa1c49548e2c55504ULL, 0x7ef04287126f15ccULL, + 0xfed1659dbd30ef15ULL, 0x8b4ab9eec4e0277bULL, 0x884d6236a5df3291ULL, 0x1fd96ea6bf5cf788ULL, + 0x42a161981f190d9aULL, 0x61d849507e6052c1ULL, 0x9fe113bf285a2cd5ULL, 0x7c22d676dbad85d8ULL, + 0x82e770ed2bfbd27dULL, 0x4c05b2ece996f5a5ULL, 0xcd40a9c2b0900150ULL, 0x5895319213d9bf64ULL, + 0xe7cc5d703fea2e08ULL, 0xb50c491258e2188cULL, 0xcce30baa48205bf0ULL, 0x537c659ccfa32d62ULL, + 0x37b6623a98cfc088ULL, 0xfe9bed1fa4d6aca4ULL, 0x04d29b8e56a8d1b0ULL, 0x725f71c40b519575ULL, + 0x28c7f89cd0339ce6ULL, 0x8367b14469ddc18bULL, 0x883ada83a6a1652cULL, 0x585f1974034d6c17ULL, + 0x89cfb266f1b19188ULL, 0xe63b4863e7c35217ULL, 0xd88c9da6b4c0526aULL, 0x3e035c9df0954635ULL, + 0xdd9d5412fb45de9dULL, 0xdd684532e4cff40dULL, 0x4b5c999b151d671cULL, 0x2d8c2cc811e7f690ULL, + 0x7f54be1d90055d40ULL, 0xa464c5df464aaf40ULL, 0x33979624f0e917beULL, 0x2c018dc527356b30ULL, + 0xa5415024e330b3d4ULL, 0x73ff3d96691652d3ULL, 0x94ec42c4ef9b59f1ULL, 0x0747201618d08e5aULL, + 0x4d6ca48aca411c53ULL, 0x66415f2fcfa66119ULL, 0x9c4dd40051e227ffULL, 0x59810bc09a02f7ebULL, + 0x2a7eb171b3dc101dULL, 0x441c5ab99ffef68eULL, 0x32025c9b93b359eaULL, 0x5e8ce0a71e9d112fULL, + 0xbfcccb92429503fdULL, 0xd271ba752f095d55ULL, 0x345ead5e972d091eULL, 0x18c8df11a83103baULL, + 0x90cd949a9aed0f4cULL, 0xc5d1f4cb6660e37eULL, 0xb8cac52d56c52e0bULL, 0x6e42e400c5808e0dULL, + 0xa3b46966eeaefd23ULL, 0x0c4f1f0be39ecdcaULL, 0x189dc8c9d683a51dULL, 0x51f27f054c09351bULL, + 0x4c487ccd2a320682ULL, 0x587ea95bb3df1c96ULL, 0xc8ccf79e555cb8e8ULL, 0x547dc829a206d73dULL, + 0xb822a6cd80c39b06ULL, 0xe96d54732000d4c6ULL, 0x28535b6f91463b4dULL, 0x228f4660e2486e1dULL, + 0x98799538de8d3abfULL, 0x8cd8330045ebca6eULL, 0x79952a008221e738ULL, 0x4322e1a7535cd2bbULL, + 0xb114c11819d1801cULL, 0x2016e4d84f3f5ec7ULL, 0xdd0e2df409260f4cULL, 0x5ec362c0ae5f7266ULL, + 0xc0462b18b8b2b4eeULL, 0x7cc8d950274d1afbULL, 0xf25f7105436b02d2ULL, 0x43bbf8dcbff9ccd3ULL, + 0xb6ad1767a039e9dfULL, 0xb0714da8f69d3583ULL, 0x5e55fa18b42931f5ULL, 0x4ed5558f33c60961ULL, + 0x1fe37901c647a5ddULL, 0x593ddf1f8081d357ULL, 0x0249a4fd813fd7a6ULL, 0x69acca274e9caf61ULL, + 0x047ba3ea330721c9ULL, 0x83423fc20e7e1ea0ULL, 0x1df4c0af01314a60ULL, 0x09a62dab89289527ULL, + 0xa5b325a49cc6cb00ULL, 0xe94b5dc654b56cb6ULL, 0x3be28779adc994a0ULL, 0x4296e8f8ba3a4aadULL, + 0x328689761e451eabULL, 0x2e4d598bff59594aULL, 0x49b96853d7a7084aULL, 0x4980a319601420a8ULL, + 0x9565b9e12f552c42ULL, 0x8a5318db7100fe96ULL, 0x05c90b4d43add0d7ULL, 0x538b4cd66a5d4edaULL, + 0xf4e94fc3e89f039fULL, 0x592c9af26f618045ULL, 0x08a36eb5fd4b9550ULL, 0x25fffaf6c2ed1419ULL, + 0x34434459cc79d354ULL, 0xeeecbfb4b1d5476bULL, 0xddeb34a061615d99ULL, 0x5129cecceb64b773ULL, + 0xee43215894993520ULL, 0x772f9c7cf14c0b3bULL, 0xd2e2fce306bedad5ULL, 0x715f42b546f06a97ULL, + 0x434ecdceda5b5f1aULL, 0x0da17115a49741a9ULL, 0x680bd77c73edad2eULL, 0x487c02354edd9041ULL, + 0xb8efeff3a70ed9c4ULL, 0x56a32aa3e857e302ULL, 0xdf3a68bd48a2a5a0ULL, 0x07f650b73176c444ULL, + 0xe38b9b1626e0ccb1ULL, 0x79e053c18b09fb36ULL, 0x56d90319c9f94964ULL, 0x1ca941e7ac9ff5c4ULL, + 0x49c4df29162fa0bbULL, 0x8488cf3282b33305ULL, 0x95dfda14cabb437dULL, 0x3391f78264d5ad86ULL, + 0x729ae06ae2b5095dULL, 0xd58a58d73259a946ULL, 0xe9834262d13921edULL, 0x27fedafaa54bb592ULL, + 0xa99dc5b829ad48bbULL, 0x5f025742499ee260ULL, 0x802c8ecd5d7513fdULL, 0x78ceb3ef3f6dd938ULL, + 0xc342f44f8a135d94ULL, 0x7b9edb44828cdda3ULL, 0x9436d11a0537cfe7ULL, 0x5064b164ec1ab4c8ULL, + 0x7020eccfd37eb2fcULL, 0x1f31ea3ed90d25fcULL, 0x1b930d7bdfa1bb34ULL, 0x5344467a48113044ULL, + 0x70073170f25e6dfbULL, 0xe385dc1a50114cc8ULL, 0x2348698ac8fc4f00ULL, 0x2a77a55284dd40d8ULL, + 0xfe06afe0c98c6ce4ULL, 0xc235df96dddfd6e4ULL, 0x1428d01e33bf1ed3ULL, 0x785768ec9300bdafULL, + 0x9702e57a91deb63bULL, 0x61bdb8bfe5ce8b80ULL, 0x645b426f3d1d58acULL, 0x4804a82227a557bcULL, + 0x8e57048ab44d2601ULL, 0x68d6501a4b3a6935ULL, 0xc39c9ec3f9e1c293ULL, 0x4172f257d4de63e2ULL, + 0xd368b450330c6401ULL, 0x040d3017418f2391ULL, 0x2c34bb6090b7d90dULL, 0x16f649228fdfd51fULL, + 0xbea6818e2b928ef5ULL, 0xe28ccf91cdc11e72ULL, 0x594aaa68e77a36cdULL, 0x313034806c7ffd0fULL, + 0x8a9d27ac2249bd65ULL, 0x19a3b464018e9512ULL, 0xc26ccff352b37ec7ULL, 0x056f68341d797b21ULL, + 0x5e79d6757efd2327ULL, 0xfabdbcb6553afe15ULL, 0xd3e7222c6eaf5a60ULL, 0x7046c76d4dae743bULL, + 0x660be872b18d4a55ULL, 0x19992518574e1496ULL, 0xc103053a302bdcbbULL, 0x3ed8e9800b218e8eULL, + 0x7b0b9239fa75e03eULL, 0xefe9fb684633c083ULL, 0x98a35fbe391a7793ULL, 0x6065510fe2d0fe34ULL, + 0x55cb668548abad0cULL, 0xb4584548da87e527ULL, 0x2c43ecea0107c1ddULL, 0x526028809372de35ULL, + 0x3415c56af9213b1fULL, 0x5bee1a4d017e98dbULL, 0x13f6b105b5cf709bULL, 0x5ff20e3482b29ab6ULL, + 0x0aa29c75cc2e6c90ULL, 0xfc7d73ca3a70e206ULL, 0x899fc38fc4b5c515ULL, 0x250386b124ffc207ULL, + 0x54ea28d5ae3d2b56ULL, 0x9913149dd6de60ceULL, 0x16694fc58f06d6c1ULL, 0x46b23975eb018fc7ULL, + 0x470a6a0fb4b7b4e2ULL, 0x5d92475a8f7253deULL, 0xabeee5b52fbd3adbULL, 0x7fa20801a0806968ULL, + 0x76f3faf19f7714d2ULL, 0xb3e840c12f4660c3ULL, 0x0fb4cd8df212744eULL, 0x4b065a251d3a2dd2ULL, + 0x5cebde383d77cd4aULL, 0x6adf39df882c9cb1ULL, 0xa2dd242eb09af759ULL, 0x3147c0e50e5f6422ULL, + 0x164ca5101d1350dbULL, 0xf8d13479c33fc962ULL, 0xe640ce4d13e5da08ULL, 0x4bdee0c45061f8baULL, + 0xd7c46dc1a4edb1c9ULL, 0x5514d7b6437fd98aULL, 0x58942f6bb2a1c00bULL, 0x2dffb2ab1d70710eULL, + 0xccdfcf2fc18b6d68ULL, 0xa8ebcba8b7806167ULL, 0x980697f95e2937e3ULL, 0x02fbba1cd0126e8cULL +}; -static void curve25519_bmi2_base(u8 session_key[CURVE25519_KEY_SIZE], - const u8 private_key[CURVE25519_KEY_SIZE]) +static void curve25519_ever64_base(u8 *out, const u8 *priv) { - struct { - u64 buffer[4 * NUM_WORDS_ELTFP25519]; - u64 coordinates[4 * NUM_WORDS_ELTFP25519]; - u64 workspace[4 * NUM_WORDS_ELTFP25519]; - u8 private[CURVE25519_KEY_SIZE]; - } __aligned(32) m; - - const int ite[4] = { 64, 64, 64, 63 }; - const int q = 3; u64 swap = 1; - - int i = 0, j = 0, k = 0; - u64 *const key = (u64 *)m.private; - u64 *const Ur1 = m.coordinates + 0; - u64 *const Zr1 = m.coordinates + 4; - u64 *const Ur2 = m.coordinates + 8; - u64 *const Zr2 = m.coordinates + 12; - - u64 *const UZr1 = m.coordinates + 0; - u64 *const ZUr2 = m.coordinates + 8; - - u64 *const A = m.workspace + 0; - u64 *const B = m.workspace + 4; - u64 *const C = m.workspace + 8; - u64 *const D = m.workspace + 12; - - u64 *const AB = m.workspace + 0; - u64 *const CD = m.workspace + 8; - - const u64 *const P = table_ladder_8k; - - memcpy(m.private, private_key, sizeof(m.private)); - - curve25519_clamp_secret(m.private); - - setzero_eltfp25519_1w(Ur1); - setzero_eltfp25519_1w(Zr1); - setzero_eltfp25519_1w(Zr2); - Ur1[0] = 1; - Zr1[0] = 1; - Zr2[0] = 1; - - /* G-S */ - Ur2[3] = 0x1eaecdeee27cab34UL; - Ur2[2] = 0xadc7a0b9235d48e2UL; - Ur2[1] = 0xbbf095ae14b2edf8UL; - Ur2[0] = 0x7e94e1fec82faabdUL; - - /* main-loop */ - j = q; - for (i = 0; i < NUM_WORDS_ELTFP25519; ++i) { - while (j < ite[i]) { - u64 bit = (key[i] >> j) & 0x1; - k = (64 * i + j - q); + int i, j, k; + u64 tmp[16 + 32 + 4]; + u64 *x1 = &tmp[0]; + u64 *z1 = &tmp[4]; + u64 *x2 = &tmp[8]; + u64 *z2 = &tmp[12]; + u64 *xz1 = &tmp[0]; + u64 *xz2 = &tmp[8]; + u64 *a = &tmp[0 + 16]; + u64 *b = &tmp[4 + 16]; + u64 *c = &tmp[8 + 16]; + u64 *ab = &tmp[0 + 16]; + u64 *abcd = &tmp[0 + 16]; + u64 *ef = &tmp[16 + 16]; + u64 *efgh = &tmp[16 + 16]; + u64 *key = &tmp[0 + 16 + 32]; + + memcpy(key, priv, 32); + ((u8 *)key)[0] &= 248; + ((u8 *)key)[31] = (((u8 *)key)[31] & 127) | 64; + + x1[0] = 1, x1[1] = x1[2] = x1[3] = 0; + z1[0] = 1, z1[1] = z1[2] = z1[3] = 0; + z2[0] = 1, z2[1] = z2[2] = z2[3] = 0; + memcpy(x2, p_minus_s, sizeof(p_minus_s)); + + j = 3; + for (i = 0; i < 4; ++i) { + while (j < (const int[]){ 64, 64, 64, 63 }[i]) { + u64 bit = (key[i] >> j) & 1; + k = (64 * i + j - 3); swap = swap ^ bit; - cswap(swap, Ur1, Ur2); - cswap(swap, Zr1, Zr2); + cswap2(swap, xz1, xz2); swap = bit; - /* Addition */ - sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ - add_eltfp25519_1w_bmi2(A, Ur1, Zr1); /* A = Ur1+Zr1 */ - mul_eltfp25519_1w_bmi2(C, &P[4 * k], B);/* C = M0-B */ - sub_eltfp25519_1w(B, A, C); /* B = (Ur1+Zr1) - M*(Ur1-Zr1) */ - add_eltfp25519_1w_bmi2(A, A, C); /* A = (Ur1+Zr1) + M*(Ur1-Zr1) */ - sqr_eltfp25519_2w_bmi2(AB); /* A = A^2 | B = B^2 */ - mul_eltfp25519_2w_bmi2(UZr1, ZUr2, AB); /* Ur1 = Zr2*A | Zr1 = Ur2*B */ + fsub(b, x1, z1); + fadd(a, x1, z1); + fmul(c, &table_ladder[4 * k], b, ef); + fsub(b, a, c); + fadd(a, a, c); + fsqr2(ab, ab, efgh); + fmul2(xz1, xz2, ab, efgh); ++j; } j = 0; } - /* Doubling */ - for (i = 0; i < q; ++i) { - add_eltfp25519_1w_bmi2(A, Ur1, Zr1); /* A = Ur1+Zr1 */ - sub_eltfp25519_1w(B, Ur1, Zr1); /* B = Ur1-Zr1 */ - sqr_eltfp25519_2w_bmi2(AB); /* A = A**2 B = B**2 */ - copy_eltfp25519_1w(C, B); /* C = B */ - sub_eltfp25519_1w(B, A, B); /* B = A-B */ - mul_a24_eltfp25519_1w(D, B); /* D = my_a24*B */ - add_eltfp25519_1w_bmi2(D, D, C); /* D = D+C */ - mul_eltfp25519_2w_bmi2(UZr1, AB, CD); /* Ur1 = A*B Zr1 = Zr1*A */ - } + point_double(xz1, abcd, efgh); + point_double(xz1, abcd, efgh); + point_double(xz1, abcd, efgh); + encode_point(out, xz1); - /* Convert to affine coordinates */ - inv_eltfp25519_1w_bmi2(A, Zr1); - mul_eltfp25519_1w_bmi2((u64 *)session_key, Ur1, A); - fred_eltfp25519_1w((u64 *)session_key); - - memzero_explicit(&m, sizeof(m)); + memzero_explicit(tmp, sizeof(tmp)); } +static __ro_after_init DEFINE_STATIC_KEY_FALSE(curve25519_use_bmi2_adx); + void curve25519_arch(u8 mypublic[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE], const u8 basepoint[CURVE25519_KEY_SIZE]) { - if (static_branch_likely(&curve25519_use_adx)) - curve25519_adx(mypublic, secret, basepoint); - else if (static_branch_likely(&curve25519_use_bmi2)) - curve25519_bmi2(mypublic, secret, basepoint); + if (static_branch_likely(&curve25519_use_bmi2_adx)) + curve25519_ever64(mypublic, secret, basepoint); else curve25519_generic(mypublic, secret, basepoint); } @@ -2355,10 +1395,8 @@ EXPORT_SYMBOL(curve25519_arch); void curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE]) { - if (static_branch_likely(&curve25519_use_adx)) - curve25519_adx_base(pub, secret); - else if (static_branch_likely(&curve25519_use_bmi2)) - curve25519_bmi2_base(pub, secret); + if (static_branch_likely(&curve25519_use_bmi2_adx)) + curve25519_ever64_base(pub, secret); else curve25519_generic(pub, secret, curve25519_base_point); } @@ -2449,12 +1487,11 @@ static struct kpp_alg curve25519_alg = { .max_size = curve25519_max_size, }; + static int __init curve25519_mod_init(void) { - if (boot_cpu_has(X86_FEATURE_BMI2)) - static_branch_enable(&curve25519_use_bmi2); - else if (boot_cpu_has(X86_FEATURE_ADX)) - static_branch_enable(&curve25519_use_adx); + if (boot_cpu_has(X86_FEATURE_BMI2) && boot_cpu_has(X86_FEATURE_ADX)) + static_branch_enable(&curve25519_use_bmi2_adx); else return 0; return IS_REACHABLE(CONFIG_CRYPTO_KPP) ? @@ -2474,3 +1511,4 @@ module_exit(curve25519_mod_exit); MODULE_ALIAS_CRYPTO("curve25519"); MODULE_ALIAS_CRYPTO("curve25519-x86"); MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Jason A. Donenfeld "); diff --git a/arch/x86/crypto/poly1305-avx2-x86_64.S b/arch/x86/crypto/poly1305-avx2-x86_64.S deleted file mode 100644 index d6063feda9daf..0000000000000 --- a/arch/x86/crypto/poly1305-avx2-x86_64.S +++ /dev/null @@ -1,390 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Poly1305 authenticator algorithm, RFC7539, x64 AVX2 functions - * - * Copyright (C) 2015 Martin Willi - */ - -#include - -.section .rodata.cst32.ANMASK, "aM", @progbits, 32 -.align 32 -ANMASK: .octa 0x0000000003ffffff0000000003ffffff - .octa 0x0000000003ffffff0000000003ffffff - -.section .rodata.cst32.ORMASK, "aM", @progbits, 32 -.align 32 -ORMASK: .octa 0x00000000010000000000000001000000 - .octa 0x00000000010000000000000001000000 - -.text - -#define h0 0x00(%rdi) -#define h1 0x04(%rdi) -#define h2 0x08(%rdi) -#define h3 0x0c(%rdi) -#define h4 0x10(%rdi) -#define r0 0x00(%rdx) -#define r1 0x04(%rdx) -#define r2 0x08(%rdx) -#define r3 0x0c(%rdx) -#define r4 0x10(%rdx) -#define u0 0x00(%r8) -#define u1 0x04(%r8) -#define u2 0x08(%r8) -#define u3 0x0c(%r8) -#define u4 0x10(%r8) -#define w0 0x14(%r8) -#define w1 0x18(%r8) -#define w2 0x1c(%r8) -#define w3 0x20(%r8) -#define w4 0x24(%r8) -#define y0 0x28(%r8) -#define y1 0x2c(%r8) -#define y2 0x30(%r8) -#define y3 0x34(%r8) -#define y4 0x38(%r8) -#define m %rsi -#define hc0 %ymm0 -#define hc1 %ymm1 -#define hc2 %ymm2 -#define hc3 %ymm3 -#define hc4 %ymm4 -#define hc0x %xmm0 -#define hc1x %xmm1 -#define hc2x %xmm2 -#define hc3x %xmm3 -#define hc4x %xmm4 -#define t1 %ymm5 -#define t2 %ymm6 -#define t1x %xmm5 -#define t2x %xmm6 -#define ruwy0 %ymm7 -#define ruwy1 %ymm8 -#define ruwy2 %ymm9 -#define ruwy3 %ymm10 -#define ruwy4 %ymm11 -#define ruwy0x %xmm7 -#define ruwy1x %xmm8 -#define ruwy2x %xmm9 -#define ruwy3x %xmm10 -#define ruwy4x %xmm11 -#define svxz1 %ymm12 -#define svxz2 %ymm13 -#define svxz3 %ymm14 -#define svxz4 %ymm15 -#define d0 %r9 -#define d1 %r10 -#define d2 %r11 -#define d3 %r12 -#define d4 %r13 - -SYM_FUNC_START(poly1305_4block_avx2) - # %rdi: Accumulator h[5] - # %rsi: 64 byte input block m - # %rdx: Poly1305 key r[5] - # %rcx: Quadblock count - # %r8: Poly1305 derived key r^2 u[5], r^3 w[5], r^4 y[5], - - # This four-block variant uses loop unrolled block processing. It - # requires 4 Poly1305 keys: r, r^2, r^3 and r^4: - # h = (h + m) * r => h = (h + m1) * r^4 + m2 * r^3 + m3 * r^2 + m4 * r - - vzeroupper - push %rbx - push %r12 - push %r13 - - # combine r0,u0,w0,y0 - vmovd y0,ruwy0x - vmovd w0,t1x - vpunpcklqdq t1,ruwy0,ruwy0 - vmovd u0,t1x - vmovd r0,t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,ruwy0,ruwy0 - - # combine r1,u1,w1,y1 and s1=r1*5,v1=u1*5,x1=w1*5,z1=y1*5 - vmovd y1,ruwy1x - vmovd w1,t1x - vpunpcklqdq t1,ruwy1,ruwy1 - vmovd u1,t1x - vmovd r1,t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,ruwy1,ruwy1 - vpslld $2,ruwy1,svxz1 - vpaddd ruwy1,svxz1,svxz1 - - # combine r2,u2,w2,y2 and s2=r2*5,v2=u2*5,x2=w2*5,z2=y2*5 - vmovd y2,ruwy2x - vmovd w2,t1x - vpunpcklqdq t1,ruwy2,ruwy2 - vmovd u2,t1x - vmovd r2,t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,ruwy2,ruwy2 - vpslld $2,ruwy2,svxz2 - vpaddd ruwy2,svxz2,svxz2 - - # combine r3,u3,w3,y3 and s3=r3*5,v3=u3*5,x3=w3*5,z3=y3*5 - vmovd y3,ruwy3x - vmovd w3,t1x - vpunpcklqdq t1,ruwy3,ruwy3 - vmovd u3,t1x - vmovd r3,t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,ruwy3,ruwy3 - vpslld $2,ruwy3,svxz3 - vpaddd ruwy3,svxz3,svxz3 - - # combine r4,u4,w4,y4 and s4=r4*5,v4=u4*5,x4=w4*5,z4=y4*5 - vmovd y4,ruwy4x - vmovd w4,t1x - vpunpcklqdq t1,ruwy4,ruwy4 - vmovd u4,t1x - vmovd r4,t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,ruwy4,ruwy4 - vpslld $2,ruwy4,svxz4 - vpaddd ruwy4,svxz4,svxz4 - -.Ldoblock4: - # hc0 = [m[48-51] & 0x3ffffff, m[32-35] & 0x3ffffff, - # m[16-19] & 0x3ffffff, m[ 0- 3] & 0x3ffffff + h0] - vmovd 0x00(m),hc0x - vmovd 0x10(m),t1x - vpunpcklqdq t1,hc0,hc0 - vmovd 0x20(m),t1x - vmovd 0x30(m),t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,hc0,hc0 - vpand ANMASK(%rip),hc0,hc0 - vmovd h0,t1x - vpaddd t1,hc0,hc0 - # hc1 = [(m[51-54] >> 2) & 0x3ffffff, (m[35-38] >> 2) & 0x3ffffff, - # (m[19-22] >> 2) & 0x3ffffff, (m[ 3- 6] >> 2) & 0x3ffffff + h1] - vmovd 0x03(m),hc1x - vmovd 0x13(m),t1x - vpunpcklqdq t1,hc1,hc1 - vmovd 0x23(m),t1x - vmovd 0x33(m),t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,hc1,hc1 - vpsrld $2,hc1,hc1 - vpand ANMASK(%rip),hc1,hc1 - vmovd h1,t1x - vpaddd t1,hc1,hc1 - # hc2 = [(m[54-57] >> 4) & 0x3ffffff, (m[38-41] >> 4) & 0x3ffffff, - # (m[22-25] >> 4) & 0x3ffffff, (m[ 6- 9] >> 4) & 0x3ffffff + h2] - vmovd 0x06(m),hc2x - vmovd 0x16(m),t1x - vpunpcklqdq t1,hc2,hc2 - vmovd 0x26(m),t1x - vmovd 0x36(m),t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,hc2,hc2 - vpsrld $4,hc2,hc2 - vpand ANMASK(%rip),hc2,hc2 - vmovd h2,t1x - vpaddd t1,hc2,hc2 - # hc3 = [(m[57-60] >> 6) & 0x3ffffff, (m[41-44] >> 6) & 0x3ffffff, - # (m[25-28] >> 6) & 0x3ffffff, (m[ 9-12] >> 6) & 0x3ffffff + h3] - vmovd 0x09(m),hc3x - vmovd 0x19(m),t1x - vpunpcklqdq t1,hc3,hc3 - vmovd 0x29(m),t1x - vmovd 0x39(m),t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,hc3,hc3 - vpsrld $6,hc3,hc3 - vpand ANMASK(%rip),hc3,hc3 - vmovd h3,t1x - vpaddd t1,hc3,hc3 - # hc4 = [(m[60-63] >> 8) | (1<<24), (m[44-47] >> 8) | (1<<24), - # (m[28-31] >> 8) | (1<<24), (m[12-15] >> 8) | (1<<24) + h4] - vmovd 0x0c(m),hc4x - vmovd 0x1c(m),t1x - vpunpcklqdq t1,hc4,hc4 - vmovd 0x2c(m),t1x - vmovd 0x3c(m),t2x - vpunpcklqdq t2,t1,t1 - vperm2i128 $0x20,t1,hc4,hc4 - vpsrld $8,hc4,hc4 - vpor ORMASK(%rip),hc4,hc4 - vmovd h4,t1x - vpaddd t1,hc4,hc4 - - # t1 = [ hc0[3] * r0, hc0[2] * u0, hc0[1] * w0, hc0[0] * y0 ] - vpmuludq hc0,ruwy0,t1 - # t1 += [ hc1[3] * s4, hc1[2] * v4, hc1[1] * x4, hc1[0] * z4 ] - vpmuludq hc1,svxz4,t2 - vpaddq t2,t1,t1 - # t1 += [ hc2[3] * s3, hc2[2] * v3, hc2[1] * x3, hc2[0] * z3 ] - vpmuludq hc2,svxz3,t2 - vpaddq t2,t1,t1 - # t1 += [ hc3[3] * s2, hc3[2] * v2, hc3[1] * x2, hc3[0] * z2 ] - vpmuludq hc3,svxz2,t2 - vpaddq t2,t1,t1 - # t1 += [ hc4[3] * s1, hc4[2] * v1, hc4[1] * x1, hc4[0] * z1 ] - vpmuludq hc4,svxz1,t2 - vpaddq t2,t1,t1 - # d0 = t1[0] + t1[1] + t[2] + t[3] - vpermq $0xee,t1,t2 - vpaddq t2,t1,t1 - vpsrldq $8,t1,t2 - vpaddq t2,t1,t1 - vmovq t1x,d0 - - # t1 = [ hc0[3] * r1, hc0[2] * u1,hc0[1] * w1, hc0[0] * y1 ] - vpmuludq hc0,ruwy1,t1 - # t1 += [ hc1[3] * r0, hc1[2] * u0, hc1[1] * w0, hc1[0] * y0 ] - vpmuludq hc1,ruwy0,t2 - vpaddq t2,t1,t1 - # t1 += [ hc2[3] * s4, hc2[2] * v4, hc2[1] * x4, hc2[0] * z4 ] - vpmuludq hc2,svxz4,t2 - vpaddq t2,t1,t1 - # t1 += [ hc3[3] * s3, hc3[2] * v3, hc3[1] * x3, hc3[0] * z3 ] - vpmuludq hc3,svxz3,t2 - vpaddq t2,t1,t1 - # t1 += [ hc4[3] * s2, hc4[2] * v2, hc4[1] * x2, hc4[0] * z2 ] - vpmuludq hc4,svxz2,t2 - vpaddq t2,t1,t1 - # d1 = t1[0] + t1[1] + t1[3] + t1[4] - vpermq $0xee,t1,t2 - vpaddq t2,t1,t1 - vpsrldq $8,t1,t2 - vpaddq t2,t1,t1 - vmovq t1x,d1 - - # t1 = [ hc0[3] * r2, hc0[2] * u2, hc0[1] * w2, hc0[0] * y2 ] - vpmuludq hc0,ruwy2,t1 - # t1 += [ hc1[3] * r1, hc1[2] * u1, hc1[1] * w1, hc1[0] * y1 ] - vpmuludq hc1,ruwy1,t2 - vpaddq t2,t1,t1 - # t1 += [ hc2[3] * r0, hc2[2] * u0, hc2[1] * w0, hc2[0] * y0 ] - vpmuludq hc2,ruwy0,t2 - vpaddq t2,t1,t1 - # t1 += [ hc3[3] * s4, hc3[2] * v4, hc3[1] * x4, hc3[0] * z4 ] - vpmuludq hc3,svxz4,t2 - vpaddq t2,t1,t1 - # t1 += [ hc4[3] * s3, hc4[2] * v3, hc4[1] * x3, hc4[0] * z3 ] - vpmuludq hc4,svxz3,t2 - vpaddq t2,t1,t1 - # d2 = t1[0] + t1[1] + t1[2] + t1[3] - vpermq $0xee,t1,t2 - vpaddq t2,t1,t1 - vpsrldq $8,t1,t2 - vpaddq t2,t1,t1 - vmovq t1x,d2 - - # t1 = [ hc0[3] * r3, hc0[2] * u3, hc0[1] * w3, hc0[0] * y3 ] - vpmuludq hc0,ruwy3,t1 - # t1 += [ hc1[3] * r2, hc1[2] * u2, hc1[1] * w2, hc1[0] * y2 ] - vpmuludq hc1,ruwy2,t2 - vpaddq t2,t1,t1 - # t1 += [ hc2[3] * r1, hc2[2] * u1, hc2[1] * w1, hc2[0] * y1 ] - vpmuludq hc2,ruwy1,t2 - vpaddq t2,t1,t1 - # t1 += [ hc3[3] * r0, hc3[2] * u0, hc3[1] * w0, hc3[0] * y0 ] - vpmuludq hc3,ruwy0,t2 - vpaddq t2,t1,t1 - # t1 += [ hc4[3] * s4, hc4[2] * v4, hc4[1] * x4, hc4[0] * z4 ] - vpmuludq hc4,svxz4,t2 - vpaddq t2,t1,t1 - # d3 = t1[0] + t1[1] + t1[2] + t1[3] - vpermq $0xee,t1,t2 - vpaddq t2,t1,t1 - vpsrldq $8,t1,t2 - vpaddq t2,t1,t1 - vmovq t1x,d3 - - # t1 = [ hc0[3] * r4, hc0[2] * u4, hc0[1] * w4, hc0[0] * y4 ] - vpmuludq hc0,ruwy4,t1 - # t1 += [ hc1[3] * r3, hc1[2] * u3, hc1[1] * w3, hc1[0] * y3 ] - vpmuludq hc1,ruwy3,t2 - vpaddq t2,t1,t1 - # t1 += [ hc2[3] * r2, hc2[2] * u2, hc2[1] * w2, hc2[0] * y2 ] - vpmuludq hc2,ruwy2,t2 - vpaddq t2,t1,t1 - # t1 += [ hc3[3] * r1, hc3[2] * u1, hc3[1] * w1, hc3[0] * y1 ] - vpmuludq hc3,ruwy1,t2 - vpaddq t2,t1,t1 - # t1 += [ hc4[3] * r0, hc4[2] * u0, hc4[1] * w0, hc4[0] * y0 ] - vpmuludq hc4,ruwy0,t2 - vpaddq t2,t1,t1 - # d4 = t1[0] + t1[1] + t1[2] + t1[3] - vpermq $0xee,t1,t2 - vpaddq t2,t1,t1 - vpsrldq $8,t1,t2 - vpaddq t2,t1,t1 - vmovq t1x,d4 - - # Now do a partial reduction mod (2^130)-5, carrying h0 -> h1 -> h2 -> - # h3 -> h4 -> h0 -> h1 to get h0,h2,h3,h4 < 2^26 and h1 < 2^26 + a small - # amount. Careful: we must not assume the carry bits 'd0 >> 26', - # 'd1 >> 26', 'd2 >> 26', 'd3 >> 26', and '(d4 >> 26) * 5' fit in 32-bit - # integers. It's true in a single-block implementation, but not here. - - # d1 += d0 >> 26 - mov d0,%rax - shr $26,%rax - add %rax,d1 - # h0 = d0 & 0x3ffffff - mov d0,%rbx - and $0x3ffffff,%ebx - - # d2 += d1 >> 26 - mov d1,%rax - shr $26,%rax - add %rax,d2 - # h1 = d1 & 0x3ffffff - mov d1,%rax - and $0x3ffffff,%eax - mov %eax,h1 - - # d3 += d2 >> 26 - mov d2,%rax - shr $26,%rax - add %rax,d3 - # h2 = d2 & 0x3ffffff - mov d2,%rax - and $0x3ffffff,%eax - mov %eax,h2 - - # d4 += d3 >> 26 - mov d3,%rax - shr $26,%rax - add %rax,d4 - # h3 = d3 & 0x3ffffff - mov d3,%rax - and $0x3ffffff,%eax - mov %eax,h3 - - # h0 += (d4 >> 26) * 5 - mov d4,%rax - shr $26,%rax - lea (%rax,%rax,4),%rax - add %rax,%rbx - # h4 = d4 & 0x3ffffff - mov d4,%rax - and $0x3ffffff,%eax - mov %eax,h4 - - # h1 += h0 >> 26 - mov %rbx,%rax - shr $26,%rax - add %eax,h1 - # h0 = h0 & 0x3ffffff - andl $0x3ffffff,%ebx - mov %ebx,h0 - - add $0x40,m - dec %rcx - jnz .Ldoblock4 - - vzeroupper - pop %r13 - pop %r12 - pop %rbx - ret -SYM_FUNC_END(poly1305_4block_avx2) diff --git a/arch/x86/crypto/poly1305-sse2-x86_64.S b/arch/x86/crypto/poly1305-sse2-x86_64.S deleted file mode 100644 index d8ea29b96640b..0000000000000 --- a/arch/x86/crypto/poly1305-sse2-x86_64.S +++ /dev/null @@ -1,590 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Poly1305 authenticator algorithm, RFC7539, x64 SSE2 functions - * - * Copyright (C) 2015 Martin Willi - */ - -#include - -.section .rodata.cst16.ANMASK, "aM", @progbits, 16 -.align 16 -ANMASK: .octa 0x0000000003ffffff0000000003ffffff - -.section .rodata.cst16.ORMASK, "aM", @progbits, 16 -.align 16 -ORMASK: .octa 0x00000000010000000000000001000000 - -.text - -#define h0 0x00(%rdi) -#define h1 0x04(%rdi) -#define h2 0x08(%rdi) -#define h3 0x0c(%rdi) -#define h4 0x10(%rdi) -#define r0 0x00(%rdx) -#define r1 0x04(%rdx) -#define r2 0x08(%rdx) -#define r3 0x0c(%rdx) -#define r4 0x10(%rdx) -#define s1 0x00(%rsp) -#define s2 0x04(%rsp) -#define s3 0x08(%rsp) -#define s4 0x0c(%rsp) -#define m %rsi -#define h01 %xmm0 -#define h23 %xmm1 -#define h44 %xmm2 -#define t1 %xmm3 -#define t2 %xmm4 -#define t3 %xmm5 -#define t4 %xmm6 -#define mask %xmm7 -#define d0 %r8 -#define d1 %r9 -#define d2 %r10 -#define d3 %r11 -#define d4 %r12 - -SYM_FUNC_START(poly1305_block_sse2) - # %rdi: Accumulator h[5] - # %rsi: 16 byte input block m - # %rdx: Poly1305 key r[5] - # %rcx: Block count - - # This single block variant tries to improve performance by doing two - # multiplications in parallel using SSE instructions. There is quite - # some quardword packing involved, hence the speedup is marginal. - - push %rbx - push %r12 - sub $0x10,%rsp - - # s1..s4 = r1..r4 * 5 - mov r1,%eax - lea (%eax,%eax,4),%eax - mov %eax,s1 - mov r2,%eax - lea (%eax,%eax,4),%eax - mov %eax,s2 - mov r3,%eax - lea (%eax,%eax,4),%eax - mov %eax,s3 - mov r4,%eax - lea (%eax,%eax,4),%eax - mov %eax,s4 - - movdqa ANMASK(%rip),mask - -.Ldoblock: - # h01 = [0, h1, 0, h0] - # h23 = [0, h3, 0, h2] - # h44 = [0, h4, 0, h4] - movd h0,h01 - movd h1,t1 - movd h2,h23 - movd h3,t2 - movd h4,h44 - punpcklqdq t1,h01 - punpcklqdq t2,h23 - punpcklqdq h44,h44 - - # h01 += [ (m[3-6] >> 2) & 0x3ffffff, m[0-3] & 0x3ffffff ] - movd 0x00(m),t1 - movd 0x03(m),t2 - psrld $2,t2 - punpcklqdq t2,t1 - pand mask,t1 - paddd t1,h01 - # h23 += [ (m[9-12] >> 6) & 0x3ffffff, (m[6-9] >> 4) & 0x3ffffff ] - movd 0x06(m),t1 - movd 0x09(m),t2 - psrld $4,t1 - psrld $6,t2 - punpcklqdq t2,t1 - pand mask,t1 - paddd t1,h23 - # h44 += [ (m[12-15] >> 8) | (1 << 24), (m[12-15] >> 8) | (1 << 24) ] - mov 0x0c(m),%eax - shr $8,%eax - or $0x01000000,%eax - movd %eax,t1 - pshufd $0xc4,t1,t1 - paddd t1,h44 - - # t1[0] = h0 * r0 + h2 * s3 - # t1[1] = h1 * s4 + h3 * s2 - movd r0,t1 - movd s4,t2 - punpcklqdq t2,t1 - pmuludq h01,t1 - movd s3,t2 - movd s2,t3 - punpcklqdq t3,t2 - pmuludq h23,t2 - paddq t2,t1 - # t2[0] = h0 * r1 + h2 * s4 - # t2[1] = h1 * r0 + h3 * s3 - movd r1,t2 - movd r0,t3 - punpcklqdq t3,t2 - pmuludq h01,t2 - movd s4,t3 - movd s3,t4 - punpcklqdq t4,t3 - pmuludq h23,t3 - paddq t3,t2 - # t3[0] = h4 * s1 - # t3[1] = h4 * s2 - movd s1,t3 - movd s2,t4 - punpcklqdq t4,t3 - pmuludq h44,t3 - # d0 = t1[0] + t1[1] + t3[0] - # d1 = t2[0] + t2[1] + t3[1] - movdqa t1,t4 - punpcklqdq t2,t4 - punpckhqdq t2,t1 - paddq t4,t1 - paddq t3,t1 - movq t1,d0 - psrldq $8,t1 - movq t1,d1 - - # t1[0] = h0 * r2 + h2 * r0 - # t1[1] = h1 * r1 + h3 * s4 - movd r2,t1 - movd r1,t2 - punpcklqdq t2,t1 - pmuludq h01,t1 - movd r0,t2 - movd s4,t3 - punpcklqdq t3,t2 - pmuludq h23,t2 - paddq t2,t1 - # t2[0] = h0 * r3 + h2 * r1 - # t2[1] = h1 * r2 + h3 * r0 - movd r3,t2 - movd r2,t3 - punpcklqdq t3,t2 - pmuludq h01,t2 - movd r1,t3 - movd r0,t4 - punpcklqdq t4,t3 - pmuludq h23,t3 - paddq t3,t2 - # t3[0] = h4 * s3 - # t3[1] = h4 * s4 - movd s3,t3 - movd s4,t4 - punpcklqdq t4,t3 - pmuludq h44,t3 - # d2 = t1[0] + t1[1] + t3[0] - # d3 = t2[0] + t2[1] + t3[1] - movdqa t1,t4 - punpcklqdq t2,t4 - punpckhqdq t2,t1 - paddq t4,t1 - paddq t3,t1 - movq t1,d2 - psrldq $8,t1 - movq t1,d3 - - # t1[0] = h0 * r4 + h2 * r2 - # t1[1] = h1 * r3 + h3 * r1 - movd r4,t1 - movd r3,t2 - punpcklqdq t2,t1 - pmuludq h01,t1 - movd r2,t2 - movd r1,t3 - punpcklqdq t3,t2 - pmuludq h23,t2 - paddq t2,t1 - # t3[0] = h4 * r0 - movd r0,t3 - pmuludq h44,t3 - # d4 = t1[0] + t1[1] + t3[0] - movdqa t1,t4 - psrldq $8,t4 - paddq t4,t1 - paddq t3,t1 - movq t1,d4 - - # d1 += d0 >> 26 - mov d0,%rax - shr $26,%rax - add %rax,d1 - # h0 = d0 & 0x3ffffff - mov d0,%rbx - and $0x3ffffff,%ebx - - # d2 += d1 >> 26 - mov d1,%rax - shr $26,%rax - add %rax,d2 - # h1 = d1 & 0x3ffffff - mov d1,%rax - and $0x3ffffff,%eax - mov %eax,h1 - - # d3 += d2 >> 26 - mov d2,%rax - shr $26,%rax - add %rax,d3 - # h2 = d2 & 0x3ffffff - mov d2,%rax - and $0x3ffffff,%eax - mov %eax,h2 - - # d4 += d3 >> 26 - mov d3,%rax - shr $26,%rax - add %rax,d4 - # h3 = d3 & 0x3ffffff - mov d3,%rax - and $0x3ffffff,%eax - mov %eax,h3 - - # h0 += (d4 >> 26) * 5 - mov d4,%rax - shr $26,%rax - lea (%rax,%rax,4),%rax - add %rax,%rbx - # h4 = d4 & 0x3ffffff - mov d4,%rax - and $0x3ffffff,%eax - mov %eax,h4 - - # h1 += h0 >> 26 - mov %rbx,%rax - shr $26,%rax - add %eax,h1 - # h0 = h0 & 0x3ffffff - andl $0x3ffffff,%ebx - mov %ebx,h0 - - add $0x10,m - dec %rcx - jnz .Ldoblock - - # Zeroing of key material - mov %rcx,0x00(%rsp) - mov %rcx,0x08(%rsp) - - add $0x10,%rsp - pop %r12 - pop %rbx - ret -SYM_FUNC_END(poly1305_block_sse2) - - -#define u0 0x00(%r8) -#define u1 0x04(%r8) -#define u2 0x08(%r8) -#define u3 0x0c(%r8) -#define u4 0x10(%r8) -#define hc0 %xmm0 -#define hc1 %xmm1 -#define hc2 %xmm2 -#define hc3 %xmm5 -#define hc4 %xmm6 -#define ru0 %xmm7 -#define ru1 %xmm8 -#define ru2 %xmm9 -#define ru3 %xmm10 -#define ru4 %xmm11 -#define sv1 %xmm12 -#define sv2 %xmm13 -#define sv3 %xmm14 -#define sv4 %xmm15 -#undef d0 -#define d0 %r13 - -SYM_FUNC_START(poly1305_2block_sse2) - # %rdi: Accumulator h[5] - # %rsi: 16 byte input block m - # %rdx: Poly1305 key r[5] - # %rcx: Doubleblock count - # %r8: Poly1305 derived key r^2 u[5] - - # This two-block variant further improves performance by using loop - # unrolled block processing. This is more straight forward and does - # less byte shuffling, but requires a second Poly1305 key r^2: - # h = (h + m) * r => h = (h + m1) * r^2 + m2 * r - - push %rbx - push %r12 - push %r13 - - # combine r0,u0 - movd u0,ru0 - movd r0,t1 - punpcklqdq t1,ru0 - - # combine r1,u1 and s1=r1*5,v1=u1*5 - movd u1,ru1 - movd r1,t1 - punpcklqdq t1,ru1 - movdqa ru1,sv1 - pslld $2,sv1 - paddd ru1,sv1 - - # combine r2,u2 and s2=r2*5,v2=u2*5 - movd u2,ru2 - movd r2,t1 - punpcklqdq t1,ru2 - movdqa ru2,sv2 - pslld $2,sv2 - paddd ru2,sv2 - - # combine r3,u3 and s3=r3*5,v3=u3*5 - movd u3,ru3 - movd r3,t1 - punpcklqdq t1,ru3 - movdqa ru3,sv3 - pslld $2,sv3 - paddd ru3,sv3 - - # combine r4,u4 and s4=r4*5,v4=u4*5 - movd u4,ru4 - movd r4,t1 - punpcklqdq t1,ru4 - movdqa ru4,sv4 - pslld $2,sv4 - paddd ru4,sv4 - -.Ldoblock2: - # hc0 = [ m[16-19] & 0x3ffffff, h0 + m[0-3] & 0x3ffffff ] - movd 0x00(m),hc0 - movd 0x10(m),t1 - punpcklqdq t1,hc0 - pand ANMASK(%rip),hc0 - movd h0,t1 - paddd t1,hc0 - # hc1 = [ (m[19-22] >> 2) & 0x3ffffff, h1 + (m[3-6] >> 2) & 0x3ffffff ] - movd 0x03(m),hc1 - movd 0x13(m),t1 - punpcklqdq t1,hc1 - psrld $2,hc1 - pand ANMASK(%rip),hc1 - movd h1,t1 - paddd t1,hc1 - # hc2 = [ (m[22-25] >> 4) & 0x3ffffff, h2 + (m[6-9] >> 4) & 0x3ffffff ] - movd 0x06(m),hc2 - movd 0x16(m),t1 - punpcklqdq t1,hc2 - psrld $4,hc2 - pand ANMASK(%rip),hc2 - movd h2,t1 - paddd t1,hc2 - # hc3 = [ (m[25-28] >> 6) & 0x3ffffff, h3 + (m[9-12] >> 6) & 0x3ffffff ] - movd 0x09(m),hc3 - movd 0x19(m),t1 - punpcklqdq t1,hc3 - psrld $6,hc3 - pand ANMASK(%rip),hc3 - movd h3,t1 - paddd t1,hc3 - # hc4 = [ (m[28-31] >> 8) | (1<<24), h4 + (m[12-15] >> 8) | (1<<24) ] - movd 0x0c(m),hc4 - movd 0x1c(m),t1 - punpcklqdq t1,hc4 - psrld $8,hc4 - por ORMASK(%rip),hc4 - movd h4,t1 - paddd t1,hc4 - - # t1 = [ hc0[1] * r0, hc0[0] * u0 ] - movdqa ru0,t1 - pmuludq hc0,t1 - # t1 += [ hc1[1] * s4, hc1[0] * v4 ] - movdqa sv4,t2 - pmuludq hc1,t2 - paddq t2,t1 - # t1 += [ hc2[1] * s3, hc2[0] * v3 ] - movdqa sv3,t2 - pmuludq hc2,t2 - paddq t2,t1 - # t1 += [ hc3[1] * s2, hc3[0] * v2 ] - movdqa sv2,t2 - pmuludq hc3,t2 - paddq t2,t1 - # t1 += [ hc4[1] * s1, hc4[0] * v1 ] - movdqa sv1,t2 - pmuludq hc4,t2 - paddq t2,t1 - # d0 = t1[0] + t1[1] - movdqa t1,t2 - psrldq $8,t2 - paddq t2,t1 - movq t1,d0 - - # t1 = [ hc0[1] * r1, hc0[0] * u1 ] - movdqa ru1,t1 - pmuludq hc0,t1 - # t1 += [ hc1[1] * r0, hc1[0] * u0 ] - movdqa ru0,t2 - pmuludq hc1,t2 - paddq t2,t1 - # t1 += [ hc2[1] * s4, hc2[0] * v4 ] - movdqa sv4,t2 - pmuludq hc2,t2 - paddq t2,t1 - # t1 += [ hc3[1] * s3, hc3[0] * v3 ] - movdqa sv3,t2 - pmuludq hc3,t2 - paddq t2,t1 - # t1 += [ hc4[1] * s2, hc4[0] * v2 ] - movdqa sv2,t2 - pmuludq hc4,t2 - paddq t2,t1 - # d1 = t1[0] + t1[1] - movdqa t1,t2 - psrldq $8,t2 - paddq t2,t1 - movq t1,d1 - - # t1 = [ hc0[1] * r2, hc0[0] * u2 ] - movdqa ru2,t1 - pmuludq hc0,t1 - # t1 += [ hc1[1] * r1, hc1[0] * u1 ] - movdqa ru1,t2 - pmuludq hc1,t2 - paddq t2,t1 - # t1 += [ hc2[1] * r0, hc2[0] * u0 ] - movdqa ru0,t2 - pmuludq hc2,t2 - paddq t2,t1 - # t1 += [ hc3[1] * s4, hc3[0] * v4 ] - movdqa sv4,t2 - pmuludq hc3,t2 - paddq t2,t1 - # t1 += [ hc4[1] * s3, hc4[0] * v3 ] - movdqa sv3,t2 - pmuludq hc4,t2 - paddq t2,t1 - # d2 = t1[0] + t1[1] - movdqa t1,t2 - psrldq $8,t2 - paddq t2,t1 - movq t1,d2 - - # t1 = [ hc0[1] * r3, hc0[0] * u3 ] - movdqa ru3,t1 - pmuludq hc0,t1 - # t1 += [ hc1[1] * r2, hc1[0] * u2 ] - movdqa ru2,t2 - pmuludq hc1,t2 - paddq t2,t1 - # t1 += [ hc2[1] * r1, hc2[0] * u1 ] - movdqa ru1,t2 - pmuludq hc2,t2 - paddq t2,t1 - # t1 += [ hc3[1] * r0, hc3[0] * u0 ] - movdqa ru0,t2 - pmuludq hc3,t2 - paddq t2,t1 - # t1 += [ hc4[1] * s4, hc4[0] * v4 ] - movdqa sv4,t2 - pmuludq hc4,t2 - paddq t2,t1 - # d3 = t1[0] + t1[1] - movdqa t1,t2 - psrldq $8,t2 - paddq t2,t1 - movq t1,d3 - - # t1 = [ hc0[1] * r4, hc0[0] * u4 ] - movdqa ru4,t1 - pmuludq hc0,t1 - # t1 += [ hc1[1] * r3, hc1[0] * u3 ] - movdqa ru3,t2 - pmuludq hc1,t2 - paddq t2,t1 - # t1 += [ hc2[1] * r2, hc2[0] * u2 ] - movdqa ru2,t2 - pmuludq hc2,t2 - paddq t2,t1 - # t1 += [ hc3[1] * r1, hc3[0] * u1 ] - movdqa ru1,t2 - pmuludq hc3,t2 - paddq t2,t1 - # t1 += [ hc4[1] * r0, hc4[0] * u0 ] - movdqa ru0,t2 - pmuludq hc4,t2 - paddq t2,t1 - # d4 = t1[0] + t1[1] - movdqa t1,t2 - psrldq $8,t2 - paddq t2,t1 - movq t1,d4 - - # Now do a partial reduction mod (2^130)-5, carrying h0 -> h1 -> h2 -> - # h3 -> h4 -> h0 -> h1 to get h0,h2,h3,h4 < 2^26 and h1 < 2^26 + a small - # amount. Careful: we must not assume the carry bits 'd0 >> 26', - # 'd1 >> 26', 'd2 >> 26', 'd3 >> 26', and '(d4 >> 26) * 5' fit in 32-bit - # integers. It's true in a single-block implementation, but not here. - - # d1 += d0 >> 26 - mov d0,%rax - shr $26,%rax - add %rax,d1 - # h0 = d0 & 0x3ffffff - mov d0,%rbx - and $0x3ffffff,%ebx - - # d2 += d1 >> 26 - mov d1,%rax - shr $26,%rax - add %rax,d2 - # h1 = d1 & 0x3ffffff - mov d1,%rax - and $0x3ffffff,%eax - mov %eax,h1 - - # d3 += d2 >> 26 - mov d2,%rax - shr $26,%rax - add %rax,d3 - # h2 = d2 & 0x3ffffff - mov d2,%rax - and $0x3ffffff,%eax - mov %eax,h2 - - # d4 += d3 >> 26 - mov d3,%rax - shr $26,%rax - add %rax,d4 - # h3 = d3 & 0x3ffffff - mov d3,%rax - and $0x3ffffff,%eax - mov %eax,h3 - - # h0 += (d4 >> 26) * 5 - mov d4,%rax - shr $26,%rax - lea (%rax,%rax,4),%rax - add %rax,%rbx - # h4 = d4 & 0x3ffffff - mov d4,%rax - and $0x3ffffff,%eax - mov %eax,h4 - - # h1 += h0 >> 26 - mov %rbx,%rax - shr $26,%rax - add %eax,h1 - # h0 = h0 & 0x3ffffff - andl $0x3ffffff,%ebx - mov %ebx,h0 - - add $0x20,m - dec %rcx - jnz .Ldoblock2 - - pop %r13 - pop %r12 - pop %rbx - ret -SYM_FUNC_END(poly1305_2block_sse2) diff --git a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl new file mode 100644 index 0000000000000..7a6b5380a46fc --- /dev/null +++ b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl @@ -0,0 +1,4265 @@ +#!/usr/bin/env perl +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# Copyright (C) 2017-2018 Samuel Neves . All Rights Reserved. +# Copyright (C) 2017-2019 Jason A. Donenfeld . All Rights Reserved. +# Copyright (C) 2006-2017 CRYPTOGAMS by . All Rights Reserved. +# +# This code is taken from the OpenSSL project but the author, Andy Polyakov, +# has relicensed it under the licenses specified in the SPDX header above. +# The original headers, including the original license headers, are +# included below for completeness. +# +# ==================================================================== +# Written by Andy Polyakov for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# ==================================================================== +# +# This module implements Poly1305 hash for x86_64. +# +# March 2015 +# +# Initial release. +# +# December 2016 +# +# Add AVX512F+VL+BW code path. +# +# November 2017 +# +# Convert AVX512F+VL+BW code path to pure AVX512F, so that it can be +# executed even on Knights Landing. Trigger for modification was +# observation that AVX512 code paths can negatively affect overall +# Skylake-X system performance. Since we are likely to suppress +# AVX512F capability flag [at least on Skylake-X], conversion serves +# as kind of "investment protection". Note that next *lake processor, +# Cannonlake, has AVX512IFMA code path to execute... +# +# Numbers are cycles per processed byte with poly1305_blocks alone, +# measured with rdtsc at fixed clock frequency. +# +# IALU/gcc-4.8(*) AVX(**) AVX2 AVX-512 +# P4 4.46/+120% - +# Core 2 2.41/+90% - +# Westmere 1.88/+120% - +# Sandy Bridge 1.39/+140% 1.10 +# Haswell 1.14/+175% 1.11 0.65 +# Skylake[-X] 1.13/+120% 0.96 0.51 [0.35] +# Silvermont 2.83/+95% - +# Knights L 3.60/? 1.65 1.10 0.41(***) +# Goldmont 1.70/+180% - +# VIA Nano 1.82/+150% - +# Sledgehammer 1.38/+160% - +# Bulldozer 2.30/+130% 0.97 +# Ryzen 1.15/+200% 1.08 1.18 +# +# (*) improvement coefficients relative to clang are more modest and +# are ~50% on most processors, in both cases we are comparing to +# __int128 code; +# (**) SSE2 implementation was attempted, but among non-AVX processors +# it was faster than integer-only code only on older Intel P4 and +# Core processors, 50-30%, less newer processor is, but slower on +# contemporary ones, for example almost 2x slower on Atom, and as +# former are naturally disappearing, SSE2 is deemed unnecessary; +# (***) strangely enough performance seems to vary from core to core, +# listed result is best case; + +$flavour = shift; +$output = shift; +if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); +$kernel=0; $kernel=1 if (!$flavour && !$output); + +if (!$kernel) { + $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; + ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or + ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or + die "can't locate x86_64-xlate.pl"; + + open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; + *STDOUT=*OUT; + + if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` + =~ /GNU assembler version ([2-9]\.[0-9]+)/) { + $avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25); + } + + if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) && + `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) { + $avx = ($1>=2.09) + ($1>=2.10) + ($1>=2.12); + $avx += 1 if ($1==2.11 && $2>=8); + } + + if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) && + `ml64 2>&1` =~ /Version ([0-9]+)\./) { + $avx = ($1>=10) + ($1>=11); + } + + if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) { + $avx = ($2>=3.0) + ($2>3.0); + } +} else { + $avx = 4; # The kernel uses ifdefs for this. +} + +sub declare_function() { + my ($name, $align, $nargs) = @_; + if($kernel) { + $code .= ".align $align\n"; + $code .= "SYM_FUNC_START($name)\n"; + $code .= ".L$name:\n"; + } else { + $code .= ".globl $name\n"; + $code .= ".type $name,\@function,$nargs\n"; + $code .= ".align $align\n"; + $code .= "$name:\n"; + } +} + +sub end_function() { + my ($name) = @_; + if($kernel) { + $code .= "SYM_FUNC_END($name)\n"; + } else { + $code .= ".size $name,.-$name\n"; + } +} + +$code.=<<___ if $kernel; +#include +___ + +if ($avx) { +$code.=<<___ if $kernel; +.section .rodata +___ +$code.=<<___; +.align 64 +.Lconst: +.Lmask24: +.long 0x0ffffff,0,0x0ffffff,0,0x0ffffff,0,0x0ffffff,0 +.L129: +.long `1<<24`,0,`1<<24`,0,`1<<24`,0,`1<<24`,0 +.Lmask26: +.long 0x3ffffff,0,0x3ffffff,0,0x3ffffff,0,0x3ffffff,0 +.Lpermd_avx2: +.long 2,2,2,3,2,0,2,1 +.Lpermd_avx512: +.long 0,0,0,1, 0,2,0,3, 0,4,0,5, 0,6,0,7 + +.L2_44_inp_permd: +.long 0,1,1,2,2,3,7,7 +.L2_44_inp_shift: +.quad 0,12,24,64 +.L2_44_mask: +.quad 0xfffffffffff,0xfffffffffff,0x3ffffffffff,0xffffffffffffffff +.L2_44_shift_rgt: +.quad 44,44,42,64 +.L2_44_shift_lft: +.quad 8,8,10,64 + +.align 64 +.Lx_mask44: +.quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff +.quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff +.Lx_mask42: +.quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff +.quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff +___ +} +$code.=<<___ if (!$kernel); +.asciz "Poly1305 for x86_64, CRYPTOGAMS by " +.align 16 +___ + +my ($ctx,$inp,$len,$padbit)=("%rdi","%rsi","%rdx","%rcx"); +my ($mac,$nonce)=($inp,$len); # *_emit arguments +my ($d1,$d2,$d3, $r0,$r1,$s1)=("%r8","%r9","%rdi","%r11","%r12","%r13"); +my ($h0,$h1,$h2)=("%r14","%rbx","%r10"); + +sub poly1305_iteration { +# input: copy of $r1 in %rax, $h0-$h2, $r0-$r1 +# output: $h0-$h2 *= $r0-$r1 +$code.=<<___; + mulq $h0 # h0*r1 + mov %rax,$d2 + mov $r0,%rax + mov %rdx,$d3 + + mulq $h0 # h0*r0 + mov %rax,$h0 # future $h0 + mov $r0,%rax + mov %rdx,$d1 + + mulq $h1 # h1*r0 + add %rax,$d2 + mov $s1,%rax + adc %rdx,$d3 + + mulq $h1 # h1*s1 + mov $h2,$h1 # borrow $h1 + add %rax,$h0 + adc %rdx,$d1 + + imulq $s1,$h1 # h2*s1 + add $h1,$d2 + mov $d1,$h1 + adc \$0,$d3 + + imulq $r0,$h2 # h2*r0 + add $d2,$h1 + mov \$-4,%rax # mask value + adc $h2,$d3 + + and $d3,%rax # last reduction step + mov $d3,$h2 + shr \$2,$d3 + and \$3,$h2 + add $d3,%rax + add %rax,$h0 + adc \$0,$h1 + adc \$0,$h2 +___ +} + +######################################################################## +# Layout of opaque area is following. +# +# unsigned __int64 h[3]; # current hash value base 2^64 +# unsigned __int64 r[2]; # key value base 2^64 + +$code.=<<___; +.text +___ +$code.=<<___ if (!$kernel); +.extern OPENSSL_ia32cap_P + +.globl poly1305_init_x86_64 +.hidden poly1305_init_x86_64 +.globl poly1305_blocks_x86_64 +.hidden poly1305_blocks_x86_64 +.globl poly1305_emit_x86_64 +.hidden poly1305_emit_x86_64 +___ +&declare_function("poly1305_init_x86_64", 32, 3); +$code.=<<___; + xor %rax,%rax + mov %rax,0($ctx) # initialize hash value + mov %rax,8($ctx) + mov %rax,16($ctx) + + cmp \$0,$inp + je .Lno_key +___ +$code.=<<___ if (!$kernel); + lea poly1305_blocks_x86_64(%rip),%r10 + lea poly1305_emit_x86_64(%rip),%r11 +___ +$code.=<<___ if (!$kernel && $avx); + mov OPENSSL_ia32cap_P+4(%rip),%r9 + lea poly1305_blocks_avx(%rip),%rax + lea poly1305_emit_avx(%rip),%rcx + bt \$`60-32`,%r9 # AVX? + cmovc %rax,%r10 + cmovc %rcx,%r11 +___ +$code.=<<___ if (!$kernel && $avx>1); + lea poly1305_blocks_avx2(%rip),%rax + bt \$`5+32`,%r9 # AVX2? + cmovc %rax,%r10 +___ +$code.=<<___ if (!$kernel && $avx>3); + mov \$`(1<<31|1<<21|1<<16)`,%rax + shr \$32,%r9 + and %rax,%r9 + cmp %rax,%r9 + je .Linit_base2_44 +___ +$code.=<<___; + mov \$0x0ffffffc0fffffff,%rax + mov \$0x0ffffffc0ffffffc,%rcx + and 0($inp),%rax + and 8($inp),%rcx + mov %rax,24($ctx) + mov %rcx,32($ctx) +___ +$code.=<<___ if (!$kernel && $flavour !~ /elf32/); + mov %r10,0(%rdx) + mov %r11,8(%rdx) +___ +$code.=<<___ if (!$kernel && $flavour =~ /elf32/); + mov %r10d,0(%rdx) + mov %r11d,4(%rdx) +___ +$code.=<<___; + mov \$1,%eax +.Lno_key: + ret +___ +&end_function("poly1305_init_x86_64"); + +&declare_function("poly1305_blocks_x86_64", 32, 4); +$code.=<<___; +.cfi_startproc +.Lblocks: + shr \$4,$len + jz .Lno_data # too short + + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 + push $ctx +.cfi_push $ctx +.Lblocks_body: + + mov $len,%r15 # reassign $len + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + mov 0($ctx),$h0 # load hash value + mov 8($ctx),$h1 + mov 16($ctx),$h2 + + mov $s1,$r1 + shr \$2,$s1 + mov $r1,%rax + add $r1,$s1 # s1 = r1 + (r1 >> 2) + jmp .Loop + +.align 32 +.Loop: + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 +___ + + &poly1305_iteration(); + +$code.=<<___; + mov $r1,%rax + dec %r15 # len-=16 + jnz .Loop + + mov 0(%rsp),$ctx +.cfi_restore $ctx + + mov $h0,0($ctx) # store hash value + mov $h1,8($ctx) + mov $h2,16($ctx) + + mov 8(%rsp),%r15 +.cfi_restore %r15 + mov 16(%rsp),%r14 +.cfi_restore %r14 + mov 24(%rsp),%r13 +.cfi_restore %r13 + mov 32(%rsp),%r12 +.cfi_restore %r12 + mov 40(%rsp),%rbx +.cfi_restore %rbx + lea 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lno_data: +.Lblocks_epilogue: + ret +.cfi_endproc +___ +&end_function("poly1305_blocks_x86_64"); + +&declare_function("poly1305_emit_x86_64", 32, 3); +$code.=<<___; +.Lemit: + mov 0($ctx),%r8 # load hash value + mov 8($ctx),%r9 + mov 16($ctx),%r10 + + mov %r8,%rax + add \$5,%r8 # compare to modulus + mov %r9,%rcx + adc \$0,%r9 + adc \$0,%r10 + shr \$2,%r10 # did 130-bit value overflow? + cmovnz %r8,%rax + cmovnz %r9,%rcx + + add 0($nonce),%rax # accumulate nonce + adc 8($nonce),%rcx + mov %rax,0($mac) # write result + mov %rcx,8($mac) + + ret +___ +&end_function("poly1305_emit_x86_64"); +if ($avx) { + +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX\n"; +} + +######################################################################## +# Layout of opaque area is following. +# +# unsigned __int32 h[5]; # current hash value base 2^26 +# unsigned __int32 is_base2_26; +# unsigned __int64 r[2]; # key value base 2^64 +# unsigned __int64 pad; +# struct { unsigned __int32 r^2, r^1, r^4, r^3; } r[9]; +# +# where r^n are base 2^26 digits of degrees of multiplier key. There are +# 5 digits, but last four are interleaved with multiples of 5, totalling +# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4. + +my ($H0,$H1,$H2,$H3,$H4, $T0,$T1,$T2,$T3,$T4, $D0,$D1,$D2,$D3,$D4, $MASK) = + map("%xmm$_",(0..15)); + +$code.=<<___; +.type __poly1305_block,\@abi-omnipotent +.align 32 +__poly1305_block: + push $ctx +___ + &poly1305_iteration(); +$code.=<<___; + pop $ctx + ret +.size __poly1305_block,.-__poly1305_block + +.type __poly1305_init_avx,\@abi-omnipotent +.align 32 +__poly1305_init_avx: + push %rbp + mov %rsp,%rbp + mov $r0,$h0 + mov $r1,$h1 + xor $h2,$h2 + + lea 48+64($ctx),$ctx # size optimization + + mov $r1,%rax + call __poly1305_block # r^2 + + mov \$0x3ffffff,%eax # save interleaved r^2 and r base 2^26 + mov \$0x3ffffff,%edx + mov $h0,$d1 + and $h0#d,%eax + mov $r0,$d2 + and $r0#d,%edx + mov %eax,`16*0+0-64`($ctx) + shr \$26,$d1 + mov %edx,`16*0+4-64`($ctx) + shr \$26,$d2 + + mov \$0x3ffffff,%eax + mov \$0x3ffffff,%edx + and $d1#d,%eax + and $d2#d,%edx + mov %eax,`16*1+0-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov %edx,`16*1+4-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + mov %eax,`16*2+0-64`($ctx) + shr \$26,$d1 + mov %edx,`16*2+4-64`($ctx) + shr \$26,$d2 + + mov $h1,%rax + mov $r1,%rdx + shl \$12,%rax + shl \$12,%rdx + or $d1,%rax + or $d2,%rdx + and \$0x3ffffff,%eax + and \$0x3ffffff,%edx + mov %eax,`16*3+0-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov %edx,`16*3+4-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + mov %eax,`16*4+0-64`($ctx) + mov $h1,$d1 + mov %edx,`16*4+4-64`($ctx) + mov $r1,$d2 + + mov \$0x3ffffff,%eax + mov \$0x3ffffff,%edx + shr \$14,$d1 + shr \$14,$d2 + and $d1#d,%eax + and $d2#d,%edx + mov %eax,`16*5+0-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov %edx,`16*5+4-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + mov %eax,`16*6+0-64`($ctx) + shr \$26,$d1 + mov %edx,`16*6+4-64`($ctx) + shr \$26,$d2 + + mov $h2,%rax + shl \$24,%rax + or %rax,$d1 + mov $d1#d,`16*7+0-64`($ctx) + lea ($d1,$d1,4),$d1 # *5 + mov $d2#d,`16*7+4-64`($ctx) + lea ($d2,$d2,4),$d2 # *5 + mov $d1#d,`16*8+0-64`($ctx) + mov $d2#d,`16*8+4-64`($ctx) + + mov $r1,%rax + call __poly1305_block # r^3 + + mov \$0x3ffffff,%eax # save r^3 base 2^26 + mov $h0,$d1 + and $h0#d,%eax + shr \$26,$d1 + mov %eax,`16*0+12-64`($ctx) + + mov \$0x3ffffff,%edx + and $d1#d,%edx + mov %edx,`16*1+12-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*2+12-64`($ctx) + + mov $h1,%rax + shl \$12,%rax + or $d1,%rax + and \$0x3ffffff,%eax + mov %eax,`16*3+12-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov $h1,$d1 + mov %eax,`16*4+12-64`($ctx) + + mov \$0x3ffffff,%edx + shr \$14,$d1 + and $d1#d,%edx + mov %edx,`16*5+12-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*6+12-64`($ctx) + + mov $h2,%rax + shl \$24,%rax + or %rax,$d1 + mov $d1#d,`16*7+12-64`($ctx) + lea ($d1,$d1,4),$d1 # *5 + mov $d1#d,`16*8+12-64`($ctx) + + mov $r1,%rax + call __poly1305_block # r^4 + + mov \$0x3ffffff,%eax # save r^4 base 2^26 + mov $h0,$d1 + and $h0#d,%eax + shr \$26,$d1 + mov %eax,`16*0+8-64`($ctx) + + mov \$0x3ffffff,%edx + and $d1#d,%edx + mov %edx,`16*1+8-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*2+8-64`($ctx) + + mov $h1,%rax + shl \$12,%rax + or $d1,%rax + and \$0x3ffffff,%eax + mov %eax,`16*3+8-64`($ctx) + lea (%rax,%rax,4),%eax # *5 + mov $h1,$d1 + mov %eax,`16*4+8-64`($ctx) + + mov \$0x3ffffff,%edx + shr \$14,$d1 + and $d1#d,%edx + mov %edx,`16*5+8-64`($ctx) + lea (%rdx,%rdx,4),%edx # *5 + shr \$26,$d1 + mov %edx,`16*6+8-64`($ctx) + + mov $h2,%rax + shl \$24,%rax + or %rax,$d1 + mov $d1#d,`16*7+8-64`($ctx) + lea ($d1,$d1,4),$d1 # *5 + mov $d1#d,`16*8+8-64`($ctx) + + lea -48-64($ctx),$ctx # size [de-]optimization + pop %rbp + ret +.size __poly1305_init_avx,.-__poly1305_init_avx +___ + +&declare_function("poly1305_blocks_avx", 32, 4); +$code.=<<___; +.cfi_startproc + mov 20($ctx),%r8d # is_base2_26 + cmp \$128,$len + jae .Lblocks_avx + test %r8d,%r8d + jz .Lblocks + +.Lblocks_avx: + and \$-16,$len + jz .Lno_data_avx + + vzeroupper + + test %r8d,%r8d + jz .Lbase2_64_avx + + test \$31,$len + jz .Leven_avx + + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lblocks_avx_body: + + mov $len,%r15 # reassign $len + + mov 0($ctx),$d1 # load hash value + mov 8($ctx),$d2 + mov 16($ctx),$h2#d + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + ################################# base 2^26 -> base 2^64 + mov $d1#d,$h0#d + and \$`-1*(1<<31)`,$d1 + mov $d2,$r1 # borrow $r1 + mov $d2#d,$h1#d + and \$`-1*(1<<31)`,$d2 + + shr \$6,$d1 + shl \$52,$r1 + add $d1,$h0 + shr \$12,$h1 + shr \$18,$d2 + add $r1,$h0 + adc $d2,$h1 + + mov $h2,$d1 + shl \$40,$d1 + shr \$24,$h2 + add $d1,$h1 + adc \$0,$h2 # can be partially reduced... + + mov \$-4,$d2 # ... so reduce + mov $h2,$d1 + and $h2,$d2 + shr \$2,$d1 + and \$3,$h2 + add $d2,$d1 # =*5 + add $d1,$h0 + adc \$0,$h1 + adc \$0,$h2 + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + + call __poly1305_block + + test $padbit,$padbit # if $padbit is zero, + jz .Lstore_base2_64_avx # store hash in base 2^64 format + + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$r0 + mov $h1,$r1 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$r0 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $r0,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$r1 + and \$0x3ffffff,$h1 # h[3] + or $r1,$h2 # h[4] + + sub \$16,%r15 + jz .Lstore_base2_26_avx + + vmovd %rax#d,$H0 + vmovd %rdx#d,$H1 + vmovd $h0#d,$H2 + vmovd $h1#d,$H3 + vmovd $h2#d,$H4 + jmp .Lproceed_avx + +.align 32 +.Lstore_base2_64_avx: + mov $h0,0($ctx) + mov $h1,8($ctx) + mov $h2,16($ctx) # note that is_base2_26 is zeroed + jmp .Ldone_avx + +.align 16 +.Lstore_base2_26_avx: + mov %rax#d,0($ctx) # store hash value base 2^26 + mov %rdx#d,4($ctx) + mov $h0#d,8($ctx) + mov $h1#d,12($ctx) + mov $h2#d,16($ctx) +.align 16 +.Ldone_avx: + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lno_data_avx: +.Lblocks_avx_epilogue: + ret +.cfi_endproc + +.align 32 +.Lbase2_64_avx: +.cfi_startproc + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lbase2_64_avx_body: + + mov $len,%r15 # reassign $len + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + mov 0($ctx),$h0 # load hash value + mov 8($ctx),$h1 + mov 16($ctx),$h2#d + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + + test \$31,$len + jz .Linit_avx + + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + sub \$16,%r15 + + call __poly1305_block + +.Linit_avx: + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$d1 + mov $h1,$d2 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$d1 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $d1,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$d2 + and \$0x3ffffff,$h1 # h[3] + or $d2,$h2 # h[4] + + vmovd %rax#d,$H0 + vmovd %rdx#d,$H1 + vmovd $h0#d,$H2 + vmovd $h1#d,$H3 + vmovd $h2#d,$H4 + movl \$1,20($ctx) # set is_base2_26 + + call __poly1305_init_avx + +.Lproceed_avx: + mov %r15,$len + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lbase2_64_avx_epilogue: + jmp .Ldo_avx +.cfi_endproc + +.align 32 +.Leven_avx: +.cfi_startproc + vmovd 4*0($ctx),$H0 # load hash value + vmovd 4*1($ctx),$H1 + vmovd 4*2($ctx),$H2 + vmovd 4*3($ctx),$H3 + vmovd 4*4($ctx),$H4 + +.Ldo_avx: +___ +$code.=<<___ if (!$win64); + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 + and \$-32,%rsp + sub \$-8,%rsp + lea -0x58(%rsp),%r11 + sub \$0x178,%rsp +___ +$code.=<<___ if ($win64); + lea -0xf8(%rsp),%r11 + sub \$0x218,%rsp + vmovdqa %xmm6,0x50(%r11) + vmovdqa %xmm7,0x60(%r11) + vmovdqa %xmm8,0x70(%r11) + vmovdqa %xmm9,0x80(%r11) + vmovdqa %xmm10,0x90(%r11) + vmovdqa %xmm11,0xa0(%r11) + vmovdqa %xmm12,0xb0(%r11) + vmovdqa %xmm13,0xc0(%r11) + vmovdqa %xmm14,0xd0(%r11) + vmovdqa %xmm15,0xe0(%r11) +.Ldo_avx_body: +___ +$code.=<<___; + sub \$64,$len + lea -32($inp),%rax + cmovc %rax,$inp + + vmovdqu `16*3`($ctx),$D4 # preload r0^2 + lea `16*3+64`($ctx),$ctx # size optimization + lea .Lconst(%rip),%rcx + + ################################################################ + # load input + vmovdqu 16*2($inp),$T0 + vmovdqu 16*3($inp),$T1 + vmovdqa 64(%rcx),$MASK # .Lmask26 + + vpsrldq \$6,$T0,$T2 # splat input + vpsrldq \$6,$T1,$T3 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpunpcklqdq $T3,$T2,$T3 # 2:3 + + vpsrlq \$40,$T4,$T4 # 4 + vpsrlq \$26,$T0,$T1 + vpand $MASK,$T0,$T0 # 0 + vpsrlq \$4,$T3,$T2 + vpand $MASK,$T1,$T1 # 1 + vpsrlq \$30,$T3,$T3 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + jbe .Lskip_loop_avx + + # expand and copy pre-calculated table to stack + vmovdqu `16*1-64`($ctx),$D1 + vmovdqu `16*2-64`($ctx),$D2 + vpshufd \$0xEE,$D4,$D3 # 34xx -> 3434 + vpshufd \$0x44,$D4,$D0 # xx12 -> 1212 + vmovdqa $D3,-0x90(%r11) + vmovdqa $D0,0x00(%rsp) + vpshufd \$0xEE,$D1,$D4 + vmovdqu `16*3-64`($ctx),$D0 + vpshufd \$0x44,$D1,$D1 + vmovdqa $D4,-0x80(%r11) + vmovdqa $D1,0x10(%rsp) + vpshufd \$0xEE,$D2,$D3 + vmovdqu `16*4-64`($ctx),$D1 + vpshufd \$0x44,$D2,$D2 + vmovdqa $D3,-0x70(%r11) + vmovdqa $D2,0x20(%rsp) + vpshufd \$0xEE,$D0,$D4 + vmovdqu `16*5-64`($ctx),$D2 + vpshufd \$0x44,$D0,$D0 + vmovdqa $D4,-0x60(%r11) + vmovdqa $D0,0x30(%rsp) + vpshufd \$0xEE,$D1,$D3 + vmovdqu `16*6-64`($ctx),$D0 + vpshufd \$0x44,$D1,$D1 + vmovdqa $D3,-0x50(%r11) + vmovdqa $D1,0x40(%rsp) + vpshufd \$0xEE,$D2,$D4 + vmovdqu `16*7-64`($ctx),$D1 + vpshufd \$0x44,$D2,$D2 + vmovdqa $D4,-0x40(%r11) + vmovdqa $D2,0x50(%rsp) + vpshufd \$0xEE,$D0,$D3 + vmovdqu `16*8-64`($ctx),$D2 + vpshufd \$0x44,$D0,$D0 + vmovdqa $D3,-0x30(%r11) + vmovdqa $D0,0x60(%rsp) + vpshufd \$0xEE,$D1,$D4 + vpshufd \$0x44,$D1,$D1 + vmovdqa $D4,-0x20(%r11) + vmovdqa $D1,0x70(%rsp) + vpshufd \$0xEE,$D2,$D3 + vmovdqa 0x00(%rsp),$D4 # preload r0^2 + vpshufd \$0x44,$D2,$D2 + vmovdqa $D3,-0x10(%r11) + vmovdqa $D2,0x80(%rsp) + + jmp .Loop_avx + +.align 32 +.Loop_avx: + ################################################################ + # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 + # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r + # \___________________/ + # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 + # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r + # \___________________/ \____________________/ + # + # Note that we start with inp[2:3]*r^2. This is because it + # doesn't depend on reduction in previous iteration. + ################################################################ + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + # + # though note that $Tx and $Hx are "reversed" in this section, + # and $D4 is preloaded with r0^2... + + vpmuludq $T0,$D4,$D0 # d0 = h0*r0 + vpmuludq $T1,$D4,$D1 # d1 = h1*r0 + vmovdqa $H2,0x20(%r11) # offload hash + vpmuludq $T2,$D4,$D2 # d3 = h2*r0 + vmovdqa 0x10(%rsp),$H2 # r1^2 + vpmuludq $T3,$D4,$D3 # d3 = h3*r0 + vpmuludq $T4,$D4,$D4 # d4 = h4*r0 + + vmovdqa $H0,0x00(%r11) # + vpmuludq 0x20(%rsp),$T4,$H0 # h4*s1 + vmovdqa $H1,0x10(%r11) # + vpmuludq $T3,$H2,$H1 # h3*r1 + vpaddq $H0,$D0,$D0 # d0 += h4*s1 + vpaddq $H1,$D4,$D4 # d4 += h3*r1 + vmovdqa $H3,0x30(%r11) # + vpmuludq $T2,$H2,$H0 # h2*r1 + vpmuludq $T1,$H2,$H1 # h1*r1 + vpaddq $H0,$D3,$D3 # d3 += h2*r1 + vmovdqa 0x30(%rsp),$H3 # r2^2 + vpaddq $H1,$D2,$D2 # d2 += h1*r1 + vmovdqa $H4,0x40(%r11) # + vpmuludq $T0,$H2,$H2 # h0*r1 + vpmuludq $T2,$H3,$H0 # h2*r2 + vpaddq $H2,$D1,$D1 # d1 += h0*r1 + + vmovdqa 0x40(%rsp),$H4 # s2^2 + vpaddq $H0,$D4,$D4 # d4 += h2*r2 + vpmuludq $T1,$H3,$H1 # h1*r2 + vpmuludq $T0,$H3,$H3 # h0*r2 + vpaddq $H1,$D3,$D3 # d3 += h1*r2 + vmovdqa 0x50(%rsp),$H2 # r3^2 + vpaddq $H3,$D2,$D2 # d2 += h0*r2 + vpmuludq $T4,$H4,$H0 # h4*s2 + vpmuludq $T3,$H4,$H4 # h3*s2 + vpaddq $H0,$D1,$D1 # d1 += h4*s2 + vmovdqa 0x60(%rsp),$H3 # s3^2 + vpaddq $H4,$D0,$D0 # d0 += h3*s2 + + vmovdqa 0x80(%rsp),$H4 # s4^2 + vpmuludq $T1,$H2,$H1 # h1*r3 + vpmuludq $T0,$H2,$H2 # h0*r3 + vpaddq $H1,$D4,$D4 # d4 += h1*r3 + vpaddq $H2,$D3,$D3 # d3 += h0*r3 + vpmuludq $T4,$H3,$H0 # h4*s3 + vpmuludq $T3,$H3,$H1 # h3*s3 + vpaddq $H0,$D2,$D2 # d2 += h4*s3 + vmovdqu 16*0($inp),$H0 # load input + vpaddq $H1,$D1,$D1 # d1 += h3*s3 + vpmuludq $T2,$H3,$H3 # h2*s3 + vpmuludq $T2,$H4,$T2 # h2*s4 + vpaddq $H3,$D0,$D0 # d0 += h2*s3 + + vmovdqu 16*1($inp),$H1 # + vpaddq $T2,$D1,$D1 # d1 += h2*s4 + vpmuludq $T3,$H4,$T3 # h3*s4 + vpmuludq $T4,$H4,$T4 # h4*s4 + vpsrldq \$6,$H0,$H2 # splat input + vpaddq $T3,$D2,$D2 # d2 += h3*s4 + vpaddq $T4,$D3,$D3 # d3 += h4*s4 + vpsrldq \$6,$H1,$H3 # + vpmuludq 0x70(%rsp),$T0,$T4 # h0*r4 + vpmuludq $T1,$H4,$T0 # h1*s4 + vpunpckhqdq $H1,$H0,$H4 # 4 + vpaddq $T4,$D4,$D4 # d4 += h0*r4 + vmovdqa -0x90(%r11),$T4 # r0^4 + vpaddq $T0,$D0,$D0 # d0 += h1*s4 + + vpunpcklqdq $H1,$H0,$H0 # 0:1 + vpunpcklqdq $H3,$H2,$H3 # 2:3 + + #vpsrlq \$40,$H4,$H4 # 4 + vpsrldq \$`40/8`,$H4,$H4 # 4 + vpsrlq \$26,$H0,$H1 + vpand $MASK,$H0,$H0 # 0 + vpsrlq \$4,$H3,$H2 + vpand $MASK,$H1,$H1 # 1 + vpand 0(%rcx),$H4,$H4 # .Lmask24 + vpsrlq \$30,$H3,$H3 + vpand $MASK,$H2,$H2 # 2 + vpand $MASK,$H3,$H3 # 3 + vpor 32(%rcx),$H4,$H4 # padbit, yes, always + + vpaddq 0x00(%r11),$H0,$H0 # add hash value + vpaddq 0x10(%r11),$H1,$H1 + vpaddq 0x20(%r11),$H2,$H2 + vpaddq 0x30(%r11),$H3,$H3 + vpaddq 0x40(%r11),$H4,$H4 + + lea 16*2($inp),%rax + lea 16*4($inp),$inp + sub \$64,$len + cmovc %rax,$inp + + ################################################################ + # Now we accumulate (inp[0:1]+hash)*r^4 + ################################################################ + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + + vpmuludq $H0,$T4,$T0 # h0*r0 + vpmuludq $H1,$T4,$T1 # h1*r0 + vpaddq $T0,$D0,$D0 + vpaddq $T1,$D1,$D1 + vmovdqa -0x80(%r11),$T2 # r1^4 + vpmuludq $H2,$T4,$T0 # h2*r0 + vpmuludq $H3,$T4,$T1 # h3*r0 + vpaddq $T0,$D2,$D2 + vpaddq $T1,$D3,$D3 + vpmuludq $H4,$T4,$T4 # h4*r0 + vpmuludq -0x70(%r11),$H4,$T0 # h4*s1 + vpaddq $T4,$D4,$D4 + + vpaddq $T0,$D0,$D0 # d0 += h4*s1 + vpmuludq $H2,$T2,$T1 # h2*r1 + vpmuludq $H3,$T2,$T0 # h3*r1 + vpaddq $T1,$D3,$D3 # d3 += h2*r1 + vmovdqa -0x60(%r11),$T3 # r2^4 + vpaddq $T0,$D4,$D4 # d4 += h3*r1 + vpmuludq $H1,$T2,$T1 # h1*r1 + vpmuludq $H0,$T2,$T2 # h0*r1 + vpaddq $T1,$D2,$D2 # d2 += h1*r1 + vpaddq $T2,$D1,$D1 # d1 += h0*r1 + + vmovdqa -0x50(%r11),$T4 # s2^4 + vpmuludq $H2,$T3,$T0 # h2*r2 + vpmuludq $H1,$T3,$T1 # h1*r2 + vpaddq $T0,$D4,$D4 # d4 += h2*r2 + vpaddq $T1,$D3,$D3 # d3 += h1*r2 + vmovdqa -0x40(%r11),$T2 # r3^4 + vpmuludq $H0,$T3,$T3 # h0*r2 + vpmuludq $H4,$T4,$T0 # h4*s2 + vpaddq $T3,$D2,$D2 # d2 += h0*r2 + vpaddq $T0,$D1,$D1 # d1 += h4*s2 + vmovdqa -0x30(%r11),$T3 # s3^4 + vpmuludq $H3,$T4,$T4 # h3*s2 + vpmuludq $H1,$T2,$T1 # h1*r3 + vpaddq $T4,$D0,$D0 # d0 += h3*s2 + + vmovdqa -0x10(%r11),$T4 # s4^4 + vpaddq $T1,$D4,$D4 # d4 += h1*r3 + vpmuludq $H0,$T2,$T2 # h0*r3 + vpmuludq $H4,$T3,$T0 # h4*s3 + vpaddq $T2,$D3,$D3 # d3 += h0*r3 + vpaddq $T0,$D2,$D2 # d2 += h4*s3 + vmovdqu 16*2($inp),$T0 # load input + vpmuludq $H3,$T3,$T2 # h3*s3 + vpmuludq $H2,$T3,$T3 # h2*s3 + vpaddq $T2,$D1,$D1 # d1 += h3*s3 + vmovdqu 16*3($inp),$T1 # + vpaddq $T3,$D0,$D0 # d0 += h2*s3 + + vpmuludq $H2,$T4,$H2 # h2*s4 + vpmuludq $H3,$T4,$H3 # h3*s4 + vpsrldq \$6,$T0,$T2 # splat input + vpaddq $H2,$D1,$D1 # d1 += h2*s4 + vpmuludq $H4,$T4,$H4 # h4*s4 + vpsrldq \$6,$T1,$T3 # + vpaddq $H3,$D2,$H2 # h2 = d2 + h3*s4 + vpaddq $H4,$D3,$H3 # h3 = d3 + h4*s4 + vpmuludq -0x20(%r11),$H0,$H4 # h0*r4 + vpmuludq $H1,$T4,$H0 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 + vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 + + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpunpcklqdq $T3,$T2,$T3 # 2:3 + + #vpsrlq \$40,$T4,$T4 # 4 + vpsrldq \$`40/8`,$T4,$T4 # 4 + vpsrlq \$26,$T0,$T1 + vmovdqa 0x00(%rsp),$D4 # preload r0^2 + vpand $MASK,$T0,$T0 # 0 + vpsrlq \$4,$T3,$T2 + vpand $MASK,$T1,$T1 # 1 + vpand 0(%rcx),$T4,$T4 # .Lmask24 + vpsrlq \$30,$T3,$T3 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + ################################################################ + # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein + # and P. Schwabe + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$D1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D0 + vpand $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D0,$H0,$H0 + vpsllq \$2,$D0,$D0 + vpaddq $D0,$H0,$H0 # h4 -> h0 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + ja .Loop_avx + +.Lskip_loop_avx: + ################################################################ + # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 + + vpshufd \$0x10,$D4,$D4 # r0^n, xx12 -> x1x2 + add \$32,$len + jnz .Long_tail_avx + + vpaddq $H2,$T2,$T2 + vpaddq $H0,$T0,$T0 + vpaddq $H1,$T1,$T1 + vpaddq $H3,$T3,$T3 + vpaddq $H4,$T4,$T4 + +.Long_tail_avx: + vmovdqa $H2,0x20(%r11) + vmovdqa $H0,0x00(%r11) + vmovdqa $H1,0x10(%r11) + vmovdqa $H3,0x30(%r11) + vmovdqa $H4,0x40(%r11) + + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + + vpmuludq $T2,$D4,$D2 # d2 = h2*r0 + vpmuludq $T0,$D4,$D0 # d0 = h0*r0 + vpshufd \$0x10,`16*1-64`($ctx),$H2 # r1^n + vpmuludq $T1,$D4,$D1 # d1 = h1*r0 + vpmuludq $T3,$D4,$D3 # d3 = h3*r0 + vpmuludq $T4,$D4,$D4 # d4 = h4*r0 + + vpmuludq $T3,$H2,$H0 # h3*r1 + vpaddq $H0,$D4,$D4 # d4 += h3*r1 + vpshufd \$0x10,`16*2-64`($ctx),$H3 # s1^n + vpmuludq $T2,$H2,$H1 # h2*r1 + vpaddq $H1,$D3,$D3 # d3 += h2*r1 + vpshufd \$0x10,`16*3-64`($ctx),$H4 # r2^n + vpmuludq $T1,$H2,$H0 # h1*r1 + vpaddq $H0,$D2,$D2 # d2 += h1*r1 + vpmuludq $T0,$H2,$H2 # h0*r1 + vpaddq $H2,$D1,$D1 # d1 += h0*r1 + vpmuludq $T4,$H3,$H3 # h4*s1 + vpaddq $H3,$D0,$D0 # d0 += h4*s1 + + vpshufd \$0x10,`16*4-64`($ctx),$H2 # s2^n + vpmuludq $T2,$H4,$H1 # h2*r2 + vpaddq $H1,$D4,$D4 # d4 += h2*r2 + vpmuludq $T1,$H4,$H0 # h1*r2 + vpaddq $H0,$D3,$D3 # d3 += h1*r2 + vpshufd \$0x10,`16*5-64`($ctx),$H3 # r3^n + vpmuludq $T0,$H4,$H4 # h0*r2 + vpaddq $H4,$D2,$D2 # d2 += h0*r2 + vpmuludq $T4,$H2,$H1 # h4*s2 + vpaddq $H1,$D1,$D1 # d1 += h4*s2 + vpshufd \$0x10,`16*6-64`($ctx),$H4 # s3^n + vpmuludq $T3,$H2,$H2 # h3*s2 + vpaddq $H2,$D0,$D0 # d0 += h3*s2 + + vpmuludq $T1,$H3,$H0 # h1*r3 + vpaddq $H0,$D4,$D4 # d4 += h1*r3 + vpmuludq $T0,$H3,$H3 # h0*r3 + vpaddq $H3,$D3,$D3 # d3 += h0*r3 + vpshufd \$0x10,`16*7-64`($ctx),$H2 # r4^n + vpmuludq $T4,$H4,$H1 # h4*s3 + vpaddq $H1,$D2,$D2 # d2 += h4*s3 + vpshufd \$0x10,`16*8-64`($ctx),$H3 # s4^n + vpmuludq $T3,$H4,$H0 # h3*s3 + vpaddq $H0,$D1,$D1 # d1 += h3*s3 + vpmuludq $T2,$H4,$H4 # h2*s3 + vpaddq $H4,$D0,$D0 # d0 += h2*s3 + + vpmuludq $T0,$H2,$H2 # h0*r4 + vpaddq $H2,$D4,$D4 # h4 = d4 + h0*r4 + vpmuludq $T4,$H3,$H1 # h4*s4 + vpaddq $H1,$D3,$D3 # h3 = d3 + h4*s4 + vpmuludq $T3,$H3,$H0 # h3*s4 + vpaddq $H0,$D2,$D2 # h2 = d2 + h3*s4 + vpmuludq $T2,$H3,$H1 # h2*s4 + vpaddq $H1,$D1,$D1 # h1 = d1 + h2*s4 + vpmuludq $T1,$H3,$H3 # h1*s4 + vpaddq $H3,$D0,$D0 # h0 = d0 + h1*s4 + + jz .Lshort_tail_avx + + vmovdqu 16*0($inp),$H0 # load input + vmovdqu 16*1($inp),$H1 + + vpsrldq \$6,$H0,$H2 # splat input + vpsrldq \$6,$H1,$H3 + vpunpckhqdq $H1,$H0,$H4 # 4 + vpunpcklqdq $H1,$H0,$H0 # 0:1 + vpunpcklqdq $H3,$H2,$H3 # 2:3 + + vpsrlq \$40,$H4,$H4 # 4 + vpsrlq \$26,$H0,$H1 + vpand $MASK,$H0,$H0 # 0 + vpsrlq \$4,$H3,$H2 + vpand $MASK,$H1,$H1 # 1 + vpsrlq \$30,$H3,$H3 + vpand $MASK,$H2,$H2 # 2 + vpand $MASK,$H3,$H3 # 3 + vpor 32(%rcx),$H4,$H4 # padbit, yes, always + + vpshufd \$0x32,`16*0-64`($ctx),$T4 # r0^n, 34xx -> x3x4 + vpaddq 0x00(%r11),$H0,$H0 + vpaddq 0x10(%r11),$H1,$H1 + vpaddq 0x20(%r11),$H2,$H2 + vpaddq 0x30(%r11),$H3,$H3 + vpaddq 0x40(%r11),$H4,$H4 + + ################################################################ + # multiply (inp[0:1]+hash) by r^4:r^3 and accumulate + + vpmuludq $H0,$T4,$T0 # h0*r0 + vpaddq $T0,$D0,$D0 # d0 += h0*r0 + vpmuludq $H1,$T4,$T1 # h1*r0 + vpaddq $T1,$D1,$D1 # d1 += h1*r0 + vpmuludq $H2,$T4,$T0 # h2*r0 + vpaddq $T0,$D2,$D2 # d2 += h2*r0 + vpshufd \$0x32,`16*1-64`($ctx),$T2 # r1^n + vpmuludq $H3,$T4,$T1 # h3*r0 + vpaddq $T1,$D3,$D3 # d3 += h3*r0 + vpmuludq $H4,$T4,$T4 # h4*r0 + vpaddq $T4,$D4,$D4 # d4 += h4*r0 + + vpmuludq $H3,$T2,$T0 # h3*r1 + vpaddq $T0,$D4,$D4 # d4 += h3*r1 + vpshufd \$0x32,`16*2-64`($ctx),$T3 # s1 + vpmuludq $H2,$T2,$T1 # h2*r1 + vpaddq $T1,$D3,$D3 # d3 += h2*r1 + vpshufd \$0x32,`16*3-64`($ctx),$T4 # r2 + vpmuludq $H1,$T2,$T0 # h1*r1 + vpaddq $T0,$D2,$D2 # d2 += h1*r1 + vpmuludq $H0,$T2,$T2 # h0*r1 + vpaddq $T2,$D1,$D1 # d1 += h0*r1 + vpmuludq $H4,$T3,$T3 # h4*s1 + vpaddq $T3,$D0,$D0 # d0 += h4*s1 + + vpshufd \$0x32,`16*4-64`($ctx),$T2 # s2 + vpmuludq $H2,$T4,$T1 # h2*r2 + vpaddq $T1,$D4,$D4 # d4 += h2*r2 + vpmuludq $H1,$T4,$T0 # h1*r2 + vpaddq $T0,$D3,$D3 # d3 += h1*r2 + vpshufd \$0x32,`16*5-64`($ctx),$T3 # r3 + vpmuludq $H0,$T4,$T4 # h0*r2 + vpaddq $T4,$D2,$D2 # d2 += h0*r2 + vpmuludq $H4,$T2,$T1 # h4*s2 + vpaddq $T1,$D1,$D1 # d1 += h4*s2 + vpshufd \$0x32,`16*6-64`($ctx),$T4 # s3 + vpmuludq $H3,$T2,$T2 # h3*s2 + vpaddq $T2,$D0,$D0 # d0 += h3*s2 + + vpmuludq $H1,$T3,$T0 # h1*r3 + vpaddq $T0,$D4,$D4 # d4 += h1*r3 + vpmuludq $H0,$T3,$T3 # h0*r3 + vpaddq $T3,$D3,$D3 # d3 += h0*r3 + vpshufd \$0x32,`16*7-64`($ctx),$T2 # r4 + vpmuludq $H4,$T4,$T1 # h4*s3 + vpaddq $T1,$D2,$D2 # d2 += h4*s3 + vpshufd \$0x32,`16*8-64`($ctx),$T3 # s4 + vpmuludq $H3,$T4,$T0 # h3*s3 + vpaddq $T0,$D1,$D1 # d1 += h3*s3 + vpmuludq $H2,$T4,$T4 # h2*s3 + vpaddq $T4,$D0,$D0 # d0 += h2*s3 + + vpmuludq $H0,$T2,$T2 # h0*r4 + vpaddq $T2,$D4,$D4 # d4 += h0*r4 + vpmuludq $H4,$T3,$T1 # h4*s4 + vpaddq $T1,$D3,$D3 # d3 += h4*s4 + vpmuludq $H3,$T3,$T0 # h3*s4 + vpaddq $T0,$D2,$D2 # d2 += h3*s4 + vpmuludq $H2,$T3,$T1 # h2*s4 + vpaddq $T1,$D1,$D1 # d1 += h2*s4 + vpmuludq $H1,$T3,$T3 # h1*s4 + vpaddq $T3,$D0,$D0 # d0 += h1*s4 + +.Lshort_tail_avx: + ################################################################ + # horizontal addition + + vpsrldq \$8,$D4,$T4 + vpsrldq \$8,$D3,$T3 + vpsrldq \$8,$D1,$T1 + vpsrldq \$8,$D0,$T0 + vpsrldq \$8,$D2,$T2 + vpaddq $T3,$D3,$D3 + vpaddq $T4,$D4,$D4 + vpaddq $T0,$D0,$D0 + vpaddq $T1,$D1,$D1 + vpaddq $T2,$D2,$D2 + + ################################################################ + # lazy reduction + + vpsrlq \$26,$D3,$H3 + vpand $MASK,$D3,$D3 + vpaddq $H3,$D4,$D4 # h3 -> h4 + + vpsrlq \$26,$D0,$H0 + vpand $MASK,$D0,$D0 + vpaddq $H0,$D1,$D1 # h0 -> h1 + + vpsrlq \$26,$D4,$H4 + vpand $MASK,$D4,$D4 + + vpsrlq \$26,$D1,$H1 + vpand $MASK,$D1,$D1 + vpaddq $H1,$D2,$D2 # h1 -> h2 + + vpaddq $H4,$D0,$D0 + vpsllq \$2,$H4,$H4 + vpaddq $H4,$D0,$D0 # h4 -> h0 + + vpsrlq \$26,$D2,$H2 + vpand $MASK,$D2,$D2 + vpaddq $H2,$D3,$D3 # h2 -> h3 + + vpsrlq \$26,$D0,$H0 + vpand $MASK,$D0,$D0 + vpaddq $H0,$D1,$D1 # h0 -> h1 + + vpsrlq \$26,$D3,$H3 + vpand $MASK,$D3,$D3 + vpaddq $H3,$D4,$D4 # h3 -> h4 + + vmovd $D0,`4*0-48-64`($ctx) # save partially reduced + vmovd $D1,`4*1-48-64`($ctx) + vmovd $D2,`4*2-48-64`($ctx) + vmovd $D3,`4*3-48-64`($ctx) + vmovd $D4,`4*4-48-64`($ctx) +___ +$code.=<<___ if ($win64); + vmovdqa 0x50(%r11),%xmm6 + vmovdqa 0x60(%r11),%xmm7 + vmovdqa 0x70(%r11),%xmm8 + vmovdqa 0x80(%r11),%xmm9 + vmovdqa 0x90(%r11),%xmm10 + vmovdqa 0xa0(%r11),%xmm11 + vmovdqa 0xb0(%r11),%xmm12 + vmovdqa 0xc0(%r11),%xmm13 + vmovdqa 0xd0(%r11),%xmm14 + vmovdqa 0xe0(%r11),%xmm15 + lea 0xf8(%r11),%rsp +.Ldo_avx_epilogue: +___ +$code.=<<___ if (!$win64); + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +___ +$code.=<<___; + vzeroupper + ret +.cfi_endproc +___ +&end_function("poly1305_blocks_avx"); + +&declare_function("poly1305_emit_avx", 32, 3); +$code.=<<___; + cmpl \$0,20($ctx) # is_base2_26? + je .Lemit + + mov 0($ctx),%eax # load hash value base 2^26 + mov 4($ctx),%ecx + mov 8($ctx),%r8d + mov 12($ctx),%r11d + mov 16($ctx),%r10d + + shl \$26,%rcx # base 2^26 -> base 2^64 + mov %r8,%r9 + shl \$52,%r8 + add %rcx,%rax + shr \$12,%r9 + add %rax,%r8 # h0 + adc \$0,%r9 + + shl \$14,%r11 + mov %r10,%rax + shr \$24,%r10 + add %r11,%r9 + shl \$40,%rax + add %rax,%r9 # h1 + adc \$0,%r10 # h2 + + mov %r10,%rax # could be partially reduced, so reduce + mov %r10,%rcx + and \$3,%r10 + shr \$2,%rax + and \$-4,%rcx + add %rcx,%rax + add %rax,%r8 + adc \$0,%r9 + adc \$0,%r10 + + mov %r8,%rax + add \$5,%r8 # compare to modulus + mov %r9,%rcx + adc \$0,%r9 + adc \$0,%r10 + shr \$2,%r10 # did 130-bit value overflow? + cmovnz %r8,%rax + cmovnz %r9,%rcx + + add 0($nonce),%rax # accumulate nonce + adc 8($nonce),%rcx + mov %rax,0($mac) # write result + mov %rcx,8($mac) + + ret +___ +&end_function("poly1305_emit_avx"); + +if ($kernel) { + $code .= "#endif\n"; +} + +if ($avx>1) { + +if ($kernel) { + $code .= "#ifdef CONFIG_AS_AVX2\n"; +} + +my ($H0,$H1,$H2,$H3,$H4, $MASK, $T4,$T0,$T1,$T2,$T3, $D0,$D1,$D2,$D3,$D4) = + map("%ymm$_",(0..15)); +my $S4=$MASK; + +sub poly1305_blocks_avxN { + my ($avx512) = @_; + my $suffix = $avx512 ? "_avx512" : ""; +$code.=<<___; +.cfi_startproc + mov 20($ctx),%r8d # is_base2_26 + cmp \$128,$len + jae .Lblocks_avx2$suffix + test %r8d,%r8d + jz .Lblocks + +.Lblocks_avx2$suffix: + and \$-16,$len + jz .Lno_data_avx2$suffix + + vzeroupper + + test %r8d,%r8d + jz .Lbase2_64_avx2$suffix + + test \$63,$len + jz .Leven_avx2$suffix + + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lblocks_avx2_body$suffix: + + mov $len,%r15 # reassign $len + + mov 0($ctx),$d1 # load hash value + mov 8($ctx),$d2 + mov 16($ctx),$h2#d + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + ################################# base 2^26 -> base 2^64 + mov $d1#d,$h0#d + and \$`-1*(1<<31)`,$d1 + mov $d2,$r1 # borrow $r1 + mov $d2#d,$h1#d + and \$`-1*(1<<31)`,$d2 + + shr \$6,$d1 + shl \$52,$r1 + add $d1,$h0 + shr \$12,$h1 + shr \$18,$d2 + add $r1,$h0 + adc $d2,$h1 + + mov $h2,$d1 + shl \$40,$d1 + shr \$24,$h2 + add $d1,$h1 + adc \$0,$h2 # can be partially reduced... + + mov \$-4,$d2 # ... so reduce + mov $h2,$d1 + and $h2,$d2 + shr \$2,$d1 + and \$3,$h2 + add $d2,$d1 # =*5 + add $d1,$h0 + adc \$0,$h1 + adc \$0,$h2 + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + +.Lbase2_26_pre_avx2$suffix: + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + sub \$16,%r15 + + call __poly1305_block + mov $r1,%rax + + test \$63,%r15 + jnz .Lbase2_26_pre_avx2$suffix + + test $padbit,$padbit # if $padbit is zero, + jz .Lstore_base2_64_avx2$suffix # store hash in base 2^64 format + + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$r0 + mov $h1,$r1 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$r0 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $r0,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$r1 + and \$0x3ffffff,$h1 # h[3] + or $r1,$h2 # h[4] + + test %r15,%r15 + jz .Lstore_base2_26_avx2$suffix + + vmovd %rax#d,%x#$H0 + vmovd %rdx#d,%x#$H1 + vmovd $h0#d,%x#$H2 + vmovd $h1#d,%x#$H3 + vmovd $h2#d,%x#$H4 + jmp .Lproceed_avx2$suffix + +.align 32 +.Lstore_base2_64_avx2$suffix: + mov $h0,0($ctx) + mov $h1,8($ctx) + mov $h2,16($ctx) # note that is_base2_26 is zeroed + jmp .Ldone_avx2$suffix + +.align 16 +.Lstore_base2_26_avx2$suffix: + mov %rax#d,0($ctx) # store hash value base 2^26 + mov %rdx#d,4($ctx) + mov $h0#d,8($ctx) + mov $h1#d,12($ctx) + mov $h2#d,16($ctx) +.align 16 +.Ldone_avx2$suffix: + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lno_data_avx2$suffix: +.Lblocks_avx2_epilogue$suffix: + ret +.cfi_endproc + +.align 32 +.Lbase2_64_avx2$suffix: +.cfi_startproc + push %rbp +.cfi_push %rbp + mov %rsp,%rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lbase2_64_avx2_body$suffix: + + mov $len,%r15 # reassign $len + + mov 24($ctx),$r0 # load r + mov 32($ctx),$s1 + + mov 0($ctx),$h0 # load hash value + mov 8($ctx),$h1 + mov 16($ctx),$h2#d + + mov $s1,$r1 + mov $s1,%rax + shr \$2,$s1 + add $r1,$s1 # s1 = r1 + (r1 >> 2) + + test \$63,$len + jz .Linit_avx2$suffix + +.Lbase2_64_pre_avx2$suffix: + add 0($inp),$h0 # accumulate input + adc 8($inp),$h1 + lea 16($inp),$inp + adc $padbit,$h2 + sub \$16,%r15 + + call __poly1305_block + mov $r1,%rax + + test \$63,%r15 + jnz .Lbase2_64_pre_avx2$suffix + +.Linit_avx2$suffix: + ################################# base 2^64 -> base 2^26 + mov $h0,%rax + mov $h0,%rdx + shr \$52,$h0 + mov $h1,$d1 + mov $h1,$d2 + shr \$26,%rdx + and \$0x3ffffff,%rax # h[0] + shl \$12,$d1 + and \$0x3ffffff,%rdx # h[1] + shr \$14,$h1 + or $d1,$h0 + shl \$24,$h2 + and \$0x3ffffff,$h0 # h[2] + shr \$40,$d2 + and \$0x3ffffff,$h1 # h[3] + or $d2,$h2 # h[4] + + vmovd %rax#d,%x#$H0 + vmovd %rdx#d,%x#$H1 + vmovd $h0#d,%x#$H2 + vmovd $h1#d,%x#$H3 + vmovd $h2#d,%x#$H4 + movl \$1,20($ctx) # set is_base2_26 + + call __poly1305_init_avx + +.Lproceed_avx2$suffix: + mov %r15,$len # restore $len +___ +$code.=<<___ if (!$kernel); + mov OPENSSL_ia32cap_P+8(%rip),%r9d + mov \$`(1<<31|1<<30|1<<16)`,%r11d +___ +$code.=<<___; + pop %r15 +.cfi_restore %r15 + pop %r14 +.cfi_restore %r14 + pop %r13 +.cfi_restore %r13 + pop %r12 +.cfi_restore %r12 + pop %rbx +.cfi_restore %rbx + pop %rbp +.cfi_restore %rbp +.Lbase2_64_avx2_epilogue$suffix: + jmp .Ldo_avx2$suffix +.cfi_endproc + +.align 32 +.Leven_avx2$suffix: +.cfi_startproc +___ +$code.=<<___ if (!$kernel); + mov OPENSSL_ia32cap_P+8(%rip),%r9d +___ +$code.=<<___; + vmovd 4*0($ctx),%x#$H0 # load hash value base 2^26 + vmovd 4*1($ctx),%x#$H1 + vmovd 4*2($ctx),%x#$H2 + vmovd 4*3($ctx),%x#$H3 + vmovd 4*4($ctx),%x#$H4 + +.Ldo_avx2$suffix: +___ +$code.=<<___ if (!$kernel && $avx>2); + cmp \$512,$len + jb .Lskip_avx512 + and %r11d,%r9d + test \$`1<<16`,%r9d # check for AVX512F + jnz .Lblocks_avx512 +.Lskip_avx512$suffix: +___ +$code.=<<___ if ($avx > 2 && $avx512 && $kernel); + cmp \$512,$len + jae .Lblocks_avx512 +___ +$code.=<<___ if (!$win64); + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 + sub \$0x128,%rsp +___ +$code.=<<___ if ($win64); + lea 8(%rsp),%r10 + sub \$0x1c8,%rsp + vmovdqa %xmm6,-0xb0(%r10) + vmovdqa %xmm7,-0xa0(%r10) + vmovdqa %xmm8,-0x90(%r10) + vmovdqa %xmm9,-0x80(%r10) + vmovdqa %xmm10,-0x70(%r10) + vmovdqa %xmm11,-0x60(%r10) + vmovdqa %xmm12,-0x50(%r10) + vmovdqa %xmm13,-0x40(%r10) + vmovdqa %xmm14,-0x30(%r10) + vmovdqa %xmm15,-0x20(%r10) +.Ldo_avx2_body$suffix: +___ +$code.=<<___; + lea .Lconst(%rip),%rcx + lea 48+64($ctx),$ctx # size optimization + vmovdqa 96(%rcx),$T0 # .Lpermd_avx2 + + # expand and copy pre-calculated table to stack + vmovdqu `16*0-64`($ctx),%x#$T2 + and \$-512,%rsp + vmovdqu `16*1-64`($ctx),%x#$T3 + vmovdqu `16*2-64`($ctx),%x#$T4 + vmovdqu `16*3-64`($ctx),%x#$D0 + vmovdqu `16*4-64`($ctx),%x#$D1 + vmovdqu `16*5-64`($ctx),%x#$D2 + lea 0x90(%rsp),%rax # size optimization + vmovdqu `16*6-64`($ctx),%x#$D3 + vpermd $T2,$T0,$T2 # 00003412 -> 14243444 + vmovdqu `16*7-64`($ctx),%x#$D4 + vpermd $T3,$T0,$T3 + vmovdqu `16*8-64`($ctx),%x#$MASK + vpermd $T4,$T0,$T4 + vmovdqa $T2,0x00(%rsp) + vpermd $D0,$T0,$D0 + vmovdqa $T3,0x20-0x90(%rax) + vpermd $D1,$T0,$D1 + vmovdqa $T4,0x40-0x90(%rax) + vpermd $D2,$T0,$D2 + vmovdqa $D0,0x60-0x90(%rax) + vpermd $D3,$T0,$D3 + vmovdqa $D1,0x80-0x90(%rax) + vpermd $D4,$T0,$D4 + vmovdqa $D2,0xa0-0x90(%rax) + vpermd $MASK,$T0,$MASK + vmovdqa $D3,0xc0-0x90(%rax) + vmovdqa $D4,0xe0-0x90(%rax) + vmovdqa $MASK,0x100-0x90(%rax) + vmovdqa 64(%rcx),$MASK # .Lmask26 + + ################################################################ + # load input + vmovdqu 16*0($inp),%x#$T0 + vmovdqu 16*1($inp),%x#$T1 + vinserti128 \$1,16*2($inp),$T0,$T0 + vinserti128 \$1,16*3($inp),$T1,$T1 + lea 16*4($inp),$inp + + vpsrldq \$6,$T0,$T2 # splat input + vpsrldq \$6,$T1,$T3 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpunpcklqdq $T3,$T2,$T2 # 2:3 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + + vpsrlq \$30,$T2,$T3 + vpsrlq \$4,$T2,$T2 + vpsrlq \$26,$T0,$T1 + vpsrlq \$40,$T4,$T4 # 4 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T0,$T0 # 0 + vpand $MASK,$T1,$T1 # 1 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + vpaddq $H2,$T2,$H2 # accumulate input + sub \$64,$len + jz .Ltail_avx2$suffix + jmp .Loop_avx2$suffix + +.align 32 +.Loop_avx2$suffix: + ################################################################ + # ((inp[0]*r^4+inp[4])*r^4+inp[ 8])*r^4 + # ((inp[1]*r^4+inp[5])*r^4+inp[ 9])*r^3 + # ((inp[2]*r^4+inp[6])*r^4+inp[10])*r^2 + # ((inp[3]*r^4+inp[7])*r^4+inp[11])*r^1 + # \________/\__________/ + ################################################################ + #vpaddq $H2,$T2,$H2 # accumulate input + vpaddq $H0,$T0,$H0 + vmovdqa `32*0`(%rsp),$T0 # r0^4 + vpaddq $H1,$T1,$H1 + vmovdqa `32*1`(%rsp),$T1 # r1^4 + vpaddq $H3,$T3,$H3 + vmovdqa `32*3`(%rsp),$T2 # r2^4 + vpaddq $H4,$T4,$H4 + vmovdqa `32*6-0x90`(%rax),$T3 # s3^4 + vmovdqa `32*8-0x90`(%rax),$S4 # s4^4 + + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + # + # however, as h2 is "chronologically" first one available pull + # corresponding operations up, so it's + # + # d4 = h2*r2 + h4*r0 + h3*r1 + h1*r3 + h0*r4 + # d3 = h2*r1 + h3*r0 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h2*5*r4 + h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + # d0 = h2*5*r3 + h0*r0 + h4*5*r1 + h3*5*r2 + h1*5*r4 + + vpmuludq $H2,$T0,$D2 # d2 = h2*r0 + vpmuludq $H2,$T1,$D3 # d3 = h2*r1 + vpmuludq $H2,$T2,$D4 # d4 = h2*r2 + vpmuludq $H2,$T3,$D0 # d0 = h2*s3 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + + vpmuludq $H0,$T1,$T4 # h0*r1 + vpmuludq $H1,$T1,$H2 # h1*r1, borrow $H2 as temp + vpaddq $T4,$D1,$D1 # d1 += h0*r1 + vpaddq $H2,$D2,$D2 # d2 += h1*r1 + vpmuludq $H3,$T1,$T4 # h3*r1 + vpmuludq `32*2`(%rsp),$H4,$H2 # h4*s1 + vpaddq $T4,$D4,$D4 # d4 += h3*r1 + vpaddq $H2,$D0,$D0 # d0 += h4*s1 + vmovdqa `32*4-0x90`(%rax),$T1 # s2 + + vpmuludq $H0,$T0,$T4 # h0*r0 + vpmuludq $H1,$T0,$H2 # h1*r0 + vpaddq $T4,$D0,$D0 # d0 += h0*r0 + vpaddq $H2,$D1,$D1 # d1 += h1*r0 + vpmuludq $H3,$T0,$T4 # h3*r0 + vpmuludq $H4,$T0,$H2 # h4*r0 + vmovdqu 16*0($inp),%x#$T0 # load input + vpaddq $T4,$D3,$D3 # d3 += h3*r0 + vpaddq $H2,$D4,$D4 # d4 += h4*r0 + vinserti128 \$1,16*2($inp),$T0,$T0 + + vpmuludq $H3,$T1,$T4 # h3*s2 + vpmuludq $H4,$T1,$H2 # h4*s2 + vmovdqu 16*1($inp),%x#$T1 + vpaddq $T4,$D0,$D0 # d0 += h3*s2 + vpaddq $H2,$D1,$D1 # d1 += h4*s2 + vmovdqa `32*5-0x90`(%rax),$H2 # r3 + vpmuludq $H1,$T2,$T4 # h1*r2 + vpmuludq $H0,$T2,$T2 # h0*r2 + vpaddq $T4,$D3,$D3 # d3 += h1*r2 + vpaddq $T2,$D2,$D2 # d2 += h0*r2 + vinserti128 \$1,16*3($inp),$T1,$T1 + lea 16*4($inp),$inp + + vpmuludq $H1,$H2,$T4 # h1*r3 + vpmuludq $H0,$H2,$H2 # h0*r3 + vpsrldq \$6,$T0,$T2 # splat input + vpaddq $T4,$D4,$D4 # d4 += h1*r3 + vpaddq $H2,$D3,$D3 # d3 += h0*r3 + vpmuludq $H3,$T3,$T4 # h3*s3 + vpmuludq $H4,$T3,$H2 # h4*s3 + vpsrldq \$6,$T1,$T3 + vpaddq $T4,$D1,$D1 # d1 += h3*s3 + vpaddq $H2,$D2,$D2 # d2 += h4*s3 + vpunpckhqdq $T1,$T0,$T4 # 4 + + vpmuludq $H3,$S4,$H3 # h3*s4 + vpmuludq $H4,$S4,$H4 # h4*s4 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpaddq $H3,$D2,$H2 # h2 = d2 + h3*r4 + vpaddq $H4,$D3,$H3 # h3 = d3 + h4*r4 + vpunpcklqdq $T3,$T2,$T3 # 2:3 + vpmuludq `32*7-0x90`(%rax),$H0,$H4 # h0*r4 + vpmuludq $H1,$S4,$H0 # h1*s4 + vmovdqa 64(%rcx),$MASK # .Lmask26 + vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 + vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 + + ################################################################ + # lazy reduction (interleaved with tail of input splat) + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$D1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D4 + vpand $MASK,$H4,$H4 + + vpsrlq \$4,$T3,$T2 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpand $MASK,$T2,$T2 # 2 + vpsrlq \$26,$T0,$T1 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpaddq $T2,$H2,$H2 # modulo-scheduled + vpsrlq \$30,$T3,$T3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$40,$T4,$T4 # 4 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpand $MASK,$T0,$T0 # 0 + vpand $MASK,$T1,$T1 # 1 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + + sub \$64,$len + jnz .Loop_avx2$suffix + + .byte 0x66,0x90 +.Ltail_avx2$suffix: + ################################################################ + # while above multiplications were by r^4 in all lanes, in last + # iteration we multiply least significant lane by r^4 and most + # significant one by r, so copy of above except that references + # to the precomputed table are displaced by 4... + + #vpaddq $H2,$T2,$H2 # accumulate input + vpaddq $H0,$T0,$H0 + vmovdqu `32*0+4`(%rsp),$T0 # r0^4 + vpaddq $H1,$T1,$H1 + vmovdqu `32*1+4`(%rsp),$T1 # r1^4 + vpaddq $H3,$T3,$H3 + vmovdqu `32*3+4`(%rsp),$T2 # r2^4 + vpaddq $H4,$T4,$H4 + vmovdqu `32*6+4-0x90`(%rax),$T3 # s3^4 + vmovdqu `32*8+4-0x90`(%rax),$S4 # s4^4 + + vpmuludq $H2,$T0,$D2 # d2 = h2*r0 + vpmuludq $H2,$T1,$D3 # d3 = h2*r1 + vpmuludq $H2,$T2,$D4 # d4 = h2*r2 + vpmuludq $H2,$T3,$D0 # d0 = h2*s3 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + + vpmuludq $H0,$T1,$T4 # h0*r1 + vpmuludq $H1,$T1,$H2 # h1*r1 + vpaddq $T4,$D1,$D1 # d1 += h0*r1 + vpaddq $H2,$D2,$D2 # d2 += h1*r1 + vpmuludq $H3,$T1,$T4 # h3*r1 + vpmuludq `32*2+4`(%rsp),$H4,$H2 # h4*s1 + vpaddq $T4,$D4,$D4 # d4 += h3*r1 + vpaddq $H2,$D0,$D0 # d0 += h4*s1 + + vpmuludq $H0,$T0,$T4 # h0*r0 + vpmuludq $H1,$T0,$H2 # h1*r0 + vpaddq $T4,$D0,$D0 # d0 += h0*r0 + vmovdqu `32*4+4-0x90`(%rax),$T1 # s2 + vpaddq $H2,$D1,$D1 # d1 += h1*r0 + vpmuludq $H3,$T0,$T4 # h3*r0 + vpmuludq $H4,$T0,$H2 # h4*r0 + vpaddq $T4,$D3,$D3 # d3 += h3*r0 + vpaddq $H2,$D4,$D4 # d4 += h4*r0 + + vpmuludq $H3,$T1,$T4 # h3*s2 + vpmuludq $H4,$T1,$H2 # h4*s2 + vpaddq $T4,$D0,$D0 # d0 += h3*s2 + vpaddq $H2,$D1,$D1 # d1 += h4*s2 + vmovdqu `32*5+4-0x90`(%rax),$H2 # r3 + vpmuludq $H1,$T2,$T4 # h1*r2 + vpmuludq $H0,$T2,$T2 # h0*r2 + vpaddq $T4,$D3,$D3 # d3 += h1*r2 + vpaddq $T2,$D2,$D2 # d2 += h0*r2 + + vpmuludq $H1,$H2,$T4 # h1*r3 + vpmuludq $H0,$H2,$H2 # h0*r3 + vpaddq $T4,$D4,$D4 # d4 += h1*r3 + vpaddq $H2,$D3,$D3 # d3 += h0*r3 + vpmuludq $H3,$T3,$T4 # h3*s3 + vpmuludq $H4,$T3,$H2 # h4*s3 + vpaddq $T4,$D1,$D1 # d1 += h3*s3 + vpaddq $H2,$D2,$D2 # d2 += h4*s3 + + vpmuludq $H3,$S4,$H3 # h3*s4 + vpmuludq $H4,$S4,$H4 # h4*s4 + vpaddq $H3,$D2,$H2 # h2 = d2 + h3*r4 + vpaddq $H4,$D3,$H3 # h3 = d3 + h4*r4 + vpmuludq `32*7+4-0x90`(%rax),$H0,$H4 # h0*r4 + vpmuludq $H1,$S4,$H0 # h1*s4 + vmovdqa 64(%rcx),$MASK # .Lmask26 + vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4 + vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4 + + ################################################################ + # horizontal addition + + vpsrldq \$8,$D1,$T1 + vpsrldq \$8,$H2,$T2 + vpsrldq \$8,$H3,$T3 + vpsrldq \$8,$H4,$T4 + vpsrldq \$8,$H0,$T0 + vpaddq $T1,$D1,$D1 + vpaddq $T2,$H2,$H2 + vpaddq $T3,$H3,$H3 + vpaddq $T4,$H4,$H4 + vpaddq $T0,$H0,$H0 + + vpermq \$0x2,$H3,$T3 + vpermq \$0x2,$H4,$T4 + vpermq \$0x2,$H0,$T0 + vpermq \$0x2,$D1,$T1 + vpermq \$0x2,$H2,$T2 + vpaddq $T3,$H3,$H3 + vpaddq $T4,$H4,$H4 + vpaddq $T0,$H0,$H0 + vpaddq $T1,$D1,$D1 + vpaddq $T2,$H2,$H2 + + ################################################################ + # lazy reduction + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$D1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D4 + vpand $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vmovd %x#$H0,`4*0-48-64`($ctx)# save partially reduced + vmovd %x#$H1,`4*1-48-64`($ctx) + vmovd %x#$H2,`4*2-48-64`($ctx) + vmovd %x#$H3,`4*3-48-64`($ctx) + vmovd %x#$H4,`4*4-48-64`($ctx) +___ +$code.=<<___ if ($win64); + vmovdqa -0xb0(%r10),%xmm6 + vmovdqa -0xa0(%r10),%xmm7 + vmovdqa -0x90(%r10),%xmm8 + vmovdqa -0x80(%r10),%xmm9 + vmovdqa -0x70(%r10),%xmm10 + vmovdqa -0x60(%r10),%xmm11 + vmovdqa -0x50(%r10),%xmm12 + vmovdqa -0x40(%r10),%xmm13 + vmovdqa -0x30(%r10),%xmm14 + vmovdqa -0x20(%r10),%xmm15 + lea -8(%r10),%rsp +.Ldo_avx2_epilogue$suffix: +___ +$code.=<<___ if (!$win64); + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +___ +$code.=<<___; + vzeroupper + ret +.cfi_endproc +___ +if($avx > 2 && $avx512) { +my ($R0,$R1,$R2,$R3,$R4, $S1,$S2,$S3,$S4) = map("%zmm$_",(16..24)); +my ($M0,$M1,$M2,$M3,$M4) = map("%zmm$_",(25..29)); +my $PADBIT="%zmm30"; + +map(s/%y/%z/,($T4,$T0,$T1,$T2,$T3)); # switch to %zmm domain +map(s/%y/%z/,($D0,$D1,$D2,$D3,$D4)); +map(s/%y/%z/,($H0,$H1,$H2,$H3,$H4)); +map(s/%y/%z/,($MASK)); + +$code.=<<___; +.cfi_startproc +.Lblocks_avx512: + mov \$15,%eax + kmovw %eax,%k2 +___ +$code.=<<___ if (!$win64); + lea 8(%rsp),%r10 +.cfi_def_cfa_register %r10 + sub \$0x128,%rsp +___ +$code.=<<___ if ($win64); + lea 8(%rsp),%r10 + sub \$0x1c8,%rsp + vmovdqa %xmm6,-0xb0(%r10) + vmovdqa %xmm7,-0xa0(%r10) + vmovdqa %xmm8,-0x90(%r10) + vmovdqa %xmm9,-0x80(%r10) + vmovdqa %xmm10,-0x70(%r10) + vmovdqa %xmm11,-0x60(%r10) + vmovdqa %xmm12,-0x50(%r10) + vmovdqa %xmm13,-0x40(%r10) + vmovdqa %xmm14,-0x30(%r10) + vmovdqa %xmm15,-0x20(%r10) +.Ldo_avx512_body: +___ +$code.=<<___; + lea .Lconst(%rip),%rcx + lea 48+64($ctx),$ctx # size optimization + vmovdqa 96(%rcx),%y#$T2 # .Lpermd_avx2 + + # expand pre-calculated table + vmovdqu `16*0-64`($ctx),%x#$D0 # will become expanded ${R0} + and \$-512,%rsp + vmovdqu `16*1-64`($ctx),%x#$D1 # will become ... ${R1} + mov \$0x20,%rax + vmovdqu `16*2-64`($ctx),%x#$T0 # ... ${S1} + vmovdqu `16*3-64`($ctx),%x#$D2 # ... ${R2} + vmovdqu `16*4-64`($ctx),%x#$T1 # ... ${S2} + vmovdqu `16*5-64`($ctx),%x#$D3 # ... ${R3} + vmovdqu `16*6-64`($ctx),%x#$T3 # ... ${S3} + vmovdqu `16*7-64`($ctx),%x#$D4 # ... ${R4} + vmovdqu `16*8-64`($ctx),%x#$T4 # ... ${S4} + vpermd $D0,$T2,$R0 # 00003412 -> 14243444 + vpbroadcastq 64(%rcx),$MASK # .Lmask26 + vpermd $D1,$T2,$R1 + vpermd $T0,$T2,$S1 + vpermd $D2,$T2,$R2 + vmovdqa64 $R0,0x00(%rsp){%k2} # save in case $len%128 != 0 + vpsrlq \$32,$R0,$T0 # 14243444 -> 01020304 + vpermd $T1,$T2,$S2 + vmovdqu64 $R1,0x00(%rsp,%rax){%k2} + vpsrlq \$32,$R1,$T1 + vpermd $D3,$T2,$R3 + vmovdqa64 $S1,0x40(%rsp){%k2} + vpermd $T3,$T2,$S3 + vpermd $D4,$T2,$R4 + vmovdqu64 $R2,0x40(%rsp,%rax){%k2} + vpermd $T4,$T2,$S4 + vmovdqa64 $S2,0x80(%rsp){%k2} + vmovdqu64 $R3,0x80(%rsp,%rax){%k2} + vmovdqa64 $S3,0xc0(%rsp){%k2} + vmovdqu64 $R4,0xc0(%rsp,%rax){%k2} + vmovdqa64 $S4,0x100(%rsp){%k2} + + ################################################################ + # calculate 5th through 8th powers of the key + # + # d0 = r0'*r0 + r1'*5*r4 + r2'*5*r3 + r3'*5*r2 + r4'*5*r1 + # d1 = r0'*r1 + r1'*r0 + r2'*5*r4 + r3'*5*r3 + r4'*5*r2 + # d2 = r0'*r2 + r1'*r1 + r2'*r0 + r3'*5*r4 + r4'*5*r3 + # d3 = r0'*r3 + r1'*r2 + r2'*r1 + r3'*r0 + r4'*5*r4 + # d4 = r0'*r4 + r1'*r3 + r2'*r2 + r3'*r1 + r4'*r0 + + vpmuludq $T0,$R0,$D0 # d0 = r0'*r0 + vpmuludq $T0,$R1,$D1 # d1 = r0'*r1 + vpmuludq $T0,$R2,$D2 # d2 = r0'*r2 + vpmuludq $T0,$R3,$D3 # d3 = r0'*r3 + vpmuludq $T0,$R4,$D4 # d4 = r0'*r4 + vpsrlq \$32,$R2,$T2 + + vpmuludq $T1,$S4,$M0 + vpmuludq $T1,$R0,$M1 + vpmuludq $T1,$R1,$M2 + vpmuludq $T1,$R2,$M3 + vpmuludq $T1,$R3,$M4 + vpsrlq \$32,$R3,$T3 + vpaddq $M0,$D0,$D0 # d0 += r1'*5*r4 + vpaddq $M1,$D1,$D1 # d1 += r1'*r0 + vpaddq $M2,$D2,$D2 # d2 += r1'*r1 + vpaddq $M3,$D3,$D3 # d3 += r1'*r2 + vpaddq $M4,$D4,$D4 # d4 += r1'*r3 + + vpmuludq $T2,$S3,$M0 + vpmuludq $T2,$S4,$M1 + vpmuludq $T2,$R1,$M3 + vpmuludq $T2,$R2,$M4 + vpmuludq $T2,$R0,$M2 + vpsrlq \$32,$R4,$T4 + vpaddq $M0,$D0,$D0 # d0 += r2'*5*r3 + vpaddq $M1,$D1,$D1 # d1 += r2'*5*r4 + vpaddq $M3,$D3,$D3 # d3 += r2'*r1 + vpaddq $M4,$D4,$D4 # d4 += r2'*r2 + vpaddq $M2,$D2,$D2 # d2 += r2'*r0 + + vpmuludq $T3,$S2,$M0 + vpmuludq $T3,$R0,$M3 + vpmuludq $T3,$R1,$M4 + vpmuludq $T3,$S3,$M1 + vpmuludq $T3,$S4,$M2 + vpaddq $M0,$D0,$D0 # d0 += r3'*5*r2 + vpaddq $M3,$D3,$D3 # d3 += r3'*r0 + vpaddq $M4,$D4,$D4 # d4 += r3'*r1 + vpaddq $M1,$D1,$D1 # d1 += r3'*5*r3 + vpaddq $M2,$D2,$D2 # d2 += r3'*5*r4 + + vpmuludq $T4,$S4,$M3 + vpmuludq $T4,$R0,$M4 + vpmuludq $T4,$S1,$M0 + vpmuludq $T4,$S2,$M1 + vpmuludq $T4,$S3,$M2 + vpaddq $M3,$D3,$D3 # d3 += r2'*5*r4 + vpaddq $M4,$D4,$D4 # d4 += r2'*r0 + vpaddq $M0,$D0,$D0 # d0 += r2'*5*r1 + vpaddq $M1,$D1,$D1 # d1 += r2'*5*r2 + vpaddq $M2,$D2,$D2 # d2 += r2'*5*r3 + + ################################################################ + # load input + vmovdqu64 16*0($inp),%z#$T3 + vmovdqu64 16*4($inp),%z#$T4 + lea 16*8($inp),$inp + + ################################################################ + # lazy reduction + + vpsrlq \$26,$D3,$M3 + vpandq $MASK,$D3,$D3 + vpaddq $M3,$D4,$D4 # d3 -> d4 + + vpsrlq \$26,$D0,$M0 + vpandq $MASK,$D0,$D0 + vpaddq $M0,$D1,$D1 # d0 -> d1 + + vpsrlq \$26,$D4,$M4 + vpandq $MASK,$D4,$D4 + + vpsrlq \$26,$D1,$M1 + vpandq $MASK,$D1,$D1 + vpaddq $M1,$D2,$D2 # d1 -> d2 + + vpaddq $M4,$D0,$D0 + vpsllq \$2,$M4,$M4 + vpaddq $M4,$D0,$D0 # d4 -> d0 + + vpsrlq \$26,$D2,$M2 + vpandq $MASK,$D2,$D2 + vpaddq $M2,$D3,$D3 # d2 -> d3 + + vpsrlq \$26,$D0,$M0 + vpandq $MASK,$D0,$D0 + vpaddq $M0,$D1,$D1 # d0 -> d1 + + vpsrlq \$26,$D3,$M3 + vpandq $MASK,$D3,$D3 + vpaddq $M3,$D4,$D4 # d3 -> d4 + + ################################################################ + # at this point we have 14243444 in $R0-$S4 and 05060708 in + # $D0-$D4, ... + + vpunpcklqdq $T4,$T3,$T0 # transpose input + vpunpckhqdq $T4,$T3,$T4 + + # ... since input 64-bit lanes are ordered as 73625140, we could + # "vperm" it to 76543210 (here and in each loop iteration), *or* + # we could just flow along, hence the goal for $R0-$S4 is + # 1858286838784888 ... + + vmovdqa32 128(%rcx),$M0 # .Lpermd_avx512: + mov \$0x7777,%eax + kmovw %eax,%k1 + + vpermd $R0,$M0,$R0 # 14243444 -> 1---2---3---4--- + vpermd $R1,$M0,$R1 + vpermd $R2,$M0,$R2 + vpermd $R3,$M0,$R3 + vpermd $R4,$M0,$R4 + + vpermd $D0,$M0,${R0}{%k1} # 05060708 -> 1858286838784888 + vpermd $D1,$M0,${R1}{%k1} + vpermd $D2,$M0,${R2}{%k1} + vpermd $D3,$M0,${R3}{%k1} + vpermd $D4,$M0,${R4}{%k1} + + vpslld \$2,$R1,$S1 # *5 + vpslld \$2,$R2,$S2 + vpslld \$2,$R3,$S3 + vpslld \$2,$R4,$S4 + vpaddd $R1,$S1,$S1 + vpaddd $R2,$S2,$S2 + vpaddd $R3,$S3,$S3 + vpaddd $R4,$S4,$S4 + + vpbroadcastq 32(%rcx),$PADBIT # .L129 + + vpsrlq \$52,$T0,$T2 # splat input + vpsllq \$12,$T4,$T3 + vporq $T3,$T2,$T2 + vpsrlq \$26,$T0,$T1 + vpsrlq \$14,$T4,$T3 + vpsrlq \$40,$T4,$T4 # 4 + vpandq $MASK,$T2,$T2 # 2 + vpandq $MASK,$T0,$T0 # 0 + #vpandq $MASK,$T1,$T1 # 1 + #vpandq $MASK,$T3,$T3 # 3 + #vporq $PADBIT,$T4,$T4 # padbit, yes, always + + vpaddq $H2,$T2,$H2 # accumulate input + sub \$192,$len + jbe .Ltail_avx512 + jmp .Loop_avx512 + +.align 32 +.Loop_avx512: + ################################################################ + # ((inp[0]*r^8+inp[ 8])*r^8+inp[16])*r^8 + # ((inp[1]*r^8+inp[ 9])*r^8+inp[17])*r^7 + # ((inp[2]*r^8+inp[10])*r^8+inp[18])*r^6 + # ((inp[3]*r^8+inp[11])*r^8+inp[19])*r^5 + # ((inp[4]*r^8+inp[12])*r^8+inp[20])*r^4 + # ((inp[5]*r^8+inp[13])*r^8+inp[21])*r^3 + # ((inp[6]*r^8+inp[14])*r^8+inp[22])*r^2 + # ((inp[7]*r^8+inp[15])*r^8+inp[23])*r^1 + # \________/\___________/ + ################################################################ + #vpaddq $H2,$T2,$H2 # accumulate input + + # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 + # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 + # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 + # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 + # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 + # + # however, as h2 is "chronologically" first one available pull + # corresponding operations up, so it's + # + # d3 = h2*r1 + h0*r3 + h1*r2 + h3*r0 + h4*5*r4 + # d4 = h2*r2 + h0*r4 + h1*r3 + h3*r1 + h4*r0 + # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1 + # d1 = h2*5*r4 + h0*r1 + h1*r0 + h3*5*r3 + h4*5*r2 + # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3 + + vpmuludq $H2,$R1,$D3 # d3 = h2*r1 + vpaddq $H0,$T0,$H0 + vpmuludq $H2,$R2,$D4 # d4 = h2*r2 + vpandq $MASK,$T1,$T1 # 1 + vpmuludq $H2,$S3,$D0 # d0 = h2*s3 + vpandq $MASK,$T3,$T3 # 3 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + vporq $PADBIT,$T4,$T4 # padbit, yes, always + vpmuludq $H2,$R0,$D2 # d2 = h2*r0 + vpaddq $H1,$T1,$H1 # accumulate input + vpaddq $H3,$T3,$H3 + vpaddq $H4,$T4,$H4 + + vmovdqu64 16*0($inp),$T3 # load input + vmovdqu64 16*4($inp),$T4 + lea 16*8($inp),$inp + vpmuludq $H0,$R3,$M3 + vpmuludq $H0,$R4,$M4 + vpmuludq $H0,$R0,$M0 + vpmuludq $H0,$R1,$M1 + vpaddq $M3,$D3,$D3 # d3 += h0*r3 + vpaddq $M4,$D4,$D4 # d4 += h0*r4 + vpaddq $M0,$D0,$D0 # d0 += h0*r0 + vpaddq $M1,$D1,$D1 # d1 += h0*r1 + + vpmuludq $H1,$R2,$M3 + vpmuludq $H1,$R3,$M4 + vpmuludq $H1,$S4,$M0 + vpmuludq $H0,$R2,$M2 + vpaddq $M3,$D3,$D3 # d3 += h1*r2 + vpaddq $M4,$D4,$D4 # d4 += h1*r3 + vpaddq $M0,$D0,$D0 # d0 += h1*s4 + vpaddq $M2,$D2,$D2 # d2 += h0*r2 + + vpunpcklqdq $T4,$T3,$T0 # transpose input + vpunpckhqdq $T4,$T3,$T4 + + vpmuludq $H3,$R0,$M3 + vpmuludq $H3,$R1,$M4 + vpmuludq $H1,$R0,$M1 + vpmuludq $H1,$R1,$M2 + vpaddq $M3,$D3,$D3 # d3 += h3*r0 + vpaddq $M4,$D4,$D4 # d4 += h3*r1 + vpaddq $M1,$D1,$D1 # d1 += h1*r0 + vpaddq $M2,$D2,$D2 # d2 += h1*r1 + + vpmuludq $H4,$S4,$M3 + vpmuludq $H4,$R0,$M4 + vpmuludq $H3,$S2,$M0 + vpmuludq $H3,$S3,$M1 + vpaddq $M3,$D3,$D3 # d3 += h4*s4 + vpmuludq $H3,$S4,$M2 + vpaddq $M4,$D4,$D4 # d4 += h4*r0 + vpaddq $M0,$D0,$D0 # d0 += h3*s2 + vpaddq $M1,$D1,$D1 # d1 += h3*s3 + vpaddq $M2,$D2,$D2 # d2 += h3*s4 + + vpmuludq $H4,$S1,$M0 + vpmuludq $H4,$S2,$M1 + vpmuludq $H4,$S3,$M2 + vpaddq $M0,$D0,$H0 # h0 = d0 + h4*s1 + vpaddq $M1,$D1,$H1 # h1 = d2 + h4*s2 + vpaddq $M2,$D2,$H2 # h2 = d3 + h4*s3 + + ################################################################ + # lazy reduction (interleaved with input splat) + + vpsrlq \$52,$T0,$T2 # splat input + vpsllq \$12,$T4,$T3 + + vpsrlq \$26,$D3,$H3 + vpandq $MASK,$D3,$D3 + vpaddq $H3,$D4,$H4 # h3 -> h4 + + vporq $T3,$T2,$T2 + + vpsrlq \$26,$H0,$D0 + vpandq $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpandq $MASK,$T2,$T2 # 2 + + vpsrlq \$26,$H4,$D4 + vpandq $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpandq $MASK,$H1,$H1 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpaddq $T2,$H2,$H2 # modulo-scheduled + vpsrlq \$26,$T0,$T1 + + vpsrlq \$26,$H2,$D2 + vpandq $MASK,$H2,$H2 + vpaddq $D2,$D3,$H3 # h2 -> h3 + + vpsrlq \$14,$T4,$T3 + + vpsrlq \$26,$H0,$D0 + vpandq $MASK,$H0,$H0 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$40,$T4,$T4 # 4 + + vpsrlq \$26,$H3,$D3 + vpandq $MASK,$H3,$H3 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpandq $MASK,$T0,$T0 # 0 + #vpandq $MASK,$T1,$T1 # 1 + #vpandq $MASK,$T3,$T3 # 3 + #vporq $PADBIT,$T4,$T4 # padbit, yes, always + + sub \$128,$len + ja .Loop_avx512 + +.Ltail_avx512: + ################################################################ + # while above multiplications were by r^8 in all lanes, in last + # iteration we multiply least significant lane by r^8 and most + # significant one by r, that's why table gets shifted... + + vpsrlq \$32,$R0,$R0 # 0105020603070408 + vpsrlq \$32,$R1,$R1 + vpsrlq \$32,$R2,$R2 + vpsrlq \$32,$S3,$S3 + vpsrlq \$32,$S4,$S4 + vpsrlq \$32,$R3,$R3 + vpsrlq \$32,$R4,$R4 + vpsrlq \$32,$S1,$S1 + vpsrlq \$32,$S2,$S2 + + ################################################################ + # load either next or last 64 byte of input + lea ($inp,$len),$inp + + #vpaddq $H2,$T2,$H2 # accumulate input + vpaddq $H0,$T0,$H0 + + vpmuludq $H2,$R1,$D3 # d3 = h2*r1 + vpmuludq $H2,$R2,$D4 # d4 = h2*r2 + vpmuludq $H2,$S3,$D0 # d0 = h2*s3 + vpandq $MASK,$T1,$T1 # 1 + vpmuludq $H2,$S4,$D1 # d1 = h2*s4 + vpandq $MASK,$T3,$T3 # 3 + vpmuludq $H2,$R0,$D2 # d2 = h2*r0 + vporq $PADBIT,$T4,$T4 # padbit, yes, always + vpaddq $H1,$T1,$H1 # accumulate input + vpaddq $H3,$T3,$H3 + vpaddq $H4,$T4,$H4 + + vmovdqu 16*0($inp),%x#$T0 + vpmuludq $H0,$R3,$M3 + vpmuludq $H0,$R4,$M4 + vpmuludq $H0,$R0,$M0 + vpmuludq $H0,$R1,$M1 + vpaddq $M3,$D3,$D3 # d3 += h0*r3 + vpaddq $M4,$D4,$D4 # d4 += h0*r4 + vpaddq $M0,$D0,$D0 # d0 += h0*r0 + vpaddq $M1,$D1,$D1 # d1 += h0*r1 + + vmovdqu 16*1($inp),%x#$T1 + vpmuludq $H1,$R2,$M3 + vpmuludq $H1,$R3,$M4 + vpmuludq $H1,$S4,$M0 + vpmuludq $H0,$R2,$M2 + vpaddq $M3,$D3,$D3 # d3 += h1*r2 + vpaddq $M4,$D4,$D4 # d4 += h1*r3 + vpaddq $M0,$D0,$D0 # d0 += h1*s4 + vpaddq $M2,$D2,$D2 # d2 += h0*r2 + + vinserti128 \$1,16*2($inp),%y#$T0,%y#$T0 + vpmuludq $H3,$R0,$M3 + vpmuludq $H3,$R1,$M4 + vpmuludq $H1,$R0,$M1 + vpmuludq $H1,$R1,$M2 + vpaddq $M3,$D3,$D3 # d3 += h3*r0 + vpaddq $M4,$D4,$D4 # d4 += h3*r1 + vpaddq $M1,$D1,$D1 # d1 += h1*r0 + vpaddq $M2,$D2,$D2 # d2 += h1*r1 + + vinserti128 \$1,16*3($inp),%y#$T1,%y#$T1 + vpmuludq $H4,$S4,$M3 + vpmuludq $H4,$R0,$M4 + vpmuludq $H3,$S2,$M0 + vpmuludq $H3,$S3,$M1 + vpmuludq $H3,$S4,$M2 + vpaddq $M3,$D3,$H3 # h3 = d3 + h4*s4 + vpaddq $M4,$D4,$D4 # d4 += h4*r0 + vpaddq $M0,$D0,$D0 # d0 += h3*s2 + vpaddq $M1,$D1,$D1 # d1 += h3*s3 + vpaddq $M2,$D2,$D2 # d2 += h3*s4 + + vpmuludq $H4,$S1,$M0 + vpmuludq $H4,$S2,$M1 + vpmuludq $H4,$S3,$M2 + vpaddq $M0,$D0,$H0 # h0 = d0 + h4*s1 + vpaddq $M1,$D1,$H1 # h1 = d2 + h4*s2 + vpaddq $M2,$D2,$H2 # h2 = d3 + h4*s3 + + ################################################################ + # horizontal addition + + mov \$1,%eax + vpermq \$0xb1,$H3,$D3 + vpermq \$0xb1,$D4,$H4 + vpermq \$0xb1,$H0,$D0 + vpermq \$0xb1,$H1,$D1 + vpermq \$0xb1,$H2,$D2 + vpaddq $D3,$H3,$H3 + vpaddq $D4,$H4,$H4 + vpaddq $D0,$H0,$H0 + vpaddq $D1,$H1,$H1 + vpaddq $D2,$H2,$H2 + + kmovw %eax,%k3 + vpermq \$0x2,$H3,$D3 + vpermq \$0x2,$H4,$D4 + vpermq \$0x2,$H0,$D0 + vpermq \$0x2,$H1,$D1 + vpermq \$0x2,$H2,$D2 + vpaddq $D3,$H3,$H3 + vpaddq $D4,$H4,$H4 + vpaddq $D0,$H0,$H0 + vpaddq $D1,$H1,$H1 + vpaddq $D2,$H2,$H2 + + vextracti64x4 \$0x1,$H3,%y#$D3 + vextracti64x4 \$0x1,$H4,%y#$D4 + vextracti64x4 \$0x1,$H0,%y#$D0 + vextracti64x4 \$0x1,$H1,%y#$D1 + vextracti64x4 \$0x1,$H2,%y#$D2 + vpaddq $D3,$H3,${H3}{%k3}{z} # keep single qword in case + vpaddq $D4,$H4,${H4}{%k3}{z} # it's passed to .Ltail_avx2 + vpaddq $D0,$H0,${H0}{%k3}{z} + vpaddq $D1,$H1,${H1}{%k3}{z} + vpaddq $D2,$H2,${H2}{%k3}{z} +___ +map(s/%z/%y/,($T0,$T1,$T2,$T3,$T4, $PADBIT)); +map(s/%z/%y/,($H0,$H1,$H2,$H3,$H4, $D0,$D1,$D2,$D3,$D4, $MASK)); +$code.=<<___; + ################################################################ + # lazy reduction (interleaved with input splat) + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpsrldq \$6,$T0,$T2 # splat input + vpsrldq \$6,$T1,$T3 + vpunpckhqdq $T1,$T0,$T4 # 4 + vpaddq $D3,$H4,$H4 # h3 -> h4 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpunpcklqdq $T3,$T2,$T2 # 2:3 + vpunpcklqdq $T1,$T0,$T0 # 0:1 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H4,$D4 + vpand $MASK,$H4,$H4 + + vpsrlq \$26,$H1,$D1 + vpand $MASK,$H1,$H1 + vpsrlq \$30,$T2,$T3 + vpsrlq \$4,$T2,$T2 + vpaddq $D1,$H2,$H2 # h1 -> h2 + + vpaddq $D4,$H0,$H0 + vpsllq \$2,$D4,$D4 + vpsrlq \$26,$T0,$T1 + vpsrlq \$40,$T4,$T4 # 4 + vpaddq $D4,$H0,$H0 # h4 -> h0 + + vpsrlq \$26,$H2,$D2 + vpand $MASK,$H2,$H2 + vpand $MASK,$T2,$T2 # 2 + vpand $MASK,$T0,$T0 # 0 + vpaddq $D2,$H3,$H3 # h2 -> h3 + + vpsrlq \$26,$H0,$D0 + vpand $MASK,$H0,$H0 + vpaddq $H2,$T2,$H2 # accumulate input for .Ltail_avx2 + vpand $MASK,$T1,$T1 # 1 + vpaddq $D0,$H1,$H1 # h0 -> h1 + + vpsrlq \$26,$H3,$D3 + vpand $MASK,$H3,$H3 + vpand $MASK,$T3,$T3 # 3 + vpor 32(%rcx),$T4,$T4 # padbit, yes, always + vpaddq $D3,$H4,$H4 # h3 -> h4 + + lea 0x90(%rsp),%rax # size optimization for .Ltail_avx2 + add \$64,$len + jnz .Ltail_avx2$suffix + + vpsubq $T2,$H2,$H2 # undo input accumulation + vmovd %x#$H0,`4*0-48-64`($ctx)# save partially reduced + vmovd %x#$H1,`4*1-48-64`($ctx) + vmovd %x#$H2,`4*2-48-64`($ctx) + vmovd %x#$H3,`4*3-48-64`($ctx) + vmovd %x#$H4,`4*4-48-64`($ctx) + vzeroall +___ +$code.=<<___ if ($win64); + movdqa -0xb0(%r10),%xmm6 + movdqa -0xa0(%r10),%xmm7 + movdqa -0x90(%r10),%xmm8 + movdqa -0x80(%r10),%xmm9 + movdqa -0x70(%r10),%xmm10 + movdqa -0x60(%r10),%xmm11 + movdqa -0x50(%r10),%xmm12 + movdqa -0x40(%r10),%xmm13 + movdqa -0x30(%r10),%xmm14 + movdqa -0x20(%r10),%xmm15 + lea -8(%r10),%rsp +.Ldo_avx512_epilogue: +___ +$code.=<<___ if (!$win64); + lea -8(%r10),%rsp +.cfi_def_cfa_register %rsp +___ +$code.=<<___; + ret +.cfi_endproc +___ + +} + +} + +&declare_function("poly1305_blocks_avx2", 32, 4); +poly1305_blocks_avxN(0); +&end_function("poly1305_blocks_avx2"); + +if($kernel) { + $code .= "#endif\n"; +} + +####################################################################### +if ($avx>2) { +# On entry we have input length divisible by 64. But since inner loop +# processes 128 bytes per iteration, cases when length is not divisible +# by 128 are handled by passing tail 64 bytes to .Ltail_avx2. For this +# reason stack layout is kept identical to poly1305_blocks_avx2. If not +# for this tail, we wouldn't have to even allocate stack frame... + +if($kernel) { + $code .= "#ifdef CONFIG_AS_AVX512\n"; +} + +&declare_function("poly1305_blocks_avx512", 32, 4); +poly1305_blocks_avxN(1); +&end_function("poly1305_blocks_avx512"); + +if ($kernel) { + $code .= "#endif\n"; +} + +if (!$kernel && $avx>3) { +######################################################################## +# VPMADD52 version using 2^44 radix. +# +# One can argue that base 2^52 would be more natural. Well, even though +# some operations would be more natural, one has to recognize couple of +# things. Base 2^52 doesn't provide advantage over base 2^44 if you look +# at amount of multiply-n-accumulate operations. Secondly, it makes it +# impossible to pre-compute multiples of 5 [referred to as s[]/sN in +# reference implementations], which means that more such operations +# would have to be performed in inner loop, which in turn makes critical +# path longer. In other words, even though base 2^44 reduction might +# look less elegant, overall critical path is actually shorter... + +######################################################################## +# Layout of opaque area is following. +# +# unsigned __int64 h[3]; # current hash value base 2^44 +# unsigned __int64 s[2]; # key value*20 base 2^44 +# unsigned __int64 r[3]; # key value base 2^44 +# struct { unsigned __int64 r^1, r^3, r^2, r^4; } R[4]; +# # r^n positions reflect +# # placement in register, not +# # memory, R[3] is R[1]*20 + +$code.=<<___; +.type poly1305_init_base2_44,\@function,3 +.align 32 +poly1305_init_base2_44: + xor %rax,%rax + mov %rax,0($ctx) # initialize hash value + mov %rax,8($ctx) + mov %rax,16($ctx) + +.Linit_base2_44: + lea poly1305_blocks_vpmadd52(%rip),%r10 + lea poly1305_emit_base2_44(%rip),%r11 + + mov \$0x0ffffffc0fffffff,%rax + mov \$0x0ffffffc0ffffffc,%rcx + and 0($inp),%rax + mov \$0x00000fffffffffff,%r8 + and 8($inp),%rcx + mov \$0x00000fffffffffff,%r9 + and %rax,%r8 + shrd \$44,%rcx,%rax + mov %r8,40($ctx) # r0 + and %r9,%rax + shr \$24,%rcx + mov %rax,48($ctx) # r1 + lea (%rax,%rax,4),%rax # *5 + mov %rcx,56($ctx) # r2 + shl \$2,%rax # magic <<2 + lea (%rcx,%rcx,4),%rcx # *5 + shl \$2,%rcx # magic <<2 + mov %rax,24($ctx) # s1 + mov %rcx,32($ctx) # s2 + movq \$-1,64($ctx) # write impossible value +___ +$code.=<<___ if ($flavour !~ /elf32/); + mov %r10,0(%rdx) + mov %r11,8(%rdx) +___ +$code.=<<___ if ($flavour =~ /elf32/); + mov %r10d,0(%rdx) + mov %r11d,4(%rdx) +___ +$code.=<<___; + mov \$1,%eax + ret +.size poly1305_init_base2_44,.-poly1305_init_base2_44 +___ +{ +my ($H0,$H1,$H2,$r2r1r0,$r1r0s2,$r0s2s1,$Dlo,$Dhi) = map("%ymm$_",(0..5,16,17)); +my ($T0,$inp_permd,$inp_shift,$PAD) = map("%ymm$_",(18..21)); +my ($reduc_mask,$reduc_rght,$reduc_left) = map("%ymm$_",(22..25)); + +$code.=<<___; +.type poly1305_blocks_vpmadd52,\@function,4 +.align 32 +poly1305_blocks_vpmadd52: + shr \$4,$len + jz .Lno_data_vpmadd52 # too short + + shl \$40,$padbit + mov 64($ctx),%r8 # peek on power of the key + + # if powers of the key are not calculated yet, process up to 3 + # blocks with this single-block subroutine, otherwise ensure that + # length is divisible by 2 blocks and pass the rest down to next + # subroutine... + + mov \$3,%rax + mov \$1,%r10 + cmp \$4,$len # is input long + cmovae %r10,%rax + test %r8,%r8 # is power value impossible? + cmovns %r10,%rax + + and $len,%rax # is input of favourable length? + jz .Lblocks_vpmadd52_4x + + sub %rax,$len + mov \$7,%r10d + mov \$1,%r11d + kmovw %r10d,%k7 + lea .L2_44_inp_permd(%rip),%r10 + kmovw %r11d,%k1 + + vmovq $padbit,%x#$PAD + vmovdqa64 0(%r10),$inp_permd # .L2_44_inp_permd + vmovdqa64 32(%r10),$inp_shift # .L2_44_inp_shift + vpermq \$0xcf,$PAD,$PAD + vmovdqa64 64(%r10),$reduc_mask # .L2_44_mask + + vmovdqu64 0($ctx),${Dlo}{%k7}{z} # load hash value + vmovdqu64 40($ctx),${r2r1r0}{%k7}{z} # load keys + vmovdqu64 32($ctx),${r1r0s2}{%k7}{z} + vmovdqu64 24($ctx),${r0s2s1}{%k7}{z} + + vmovdqa64 96(%r10),$reduc_rght # .L2_44_shift_rgt + vmovdqa64 128(%r10),$reduc_left # .L2_44_shift_lft + + jmp .Loop_vpmadd52 + +.align 32 +.Loop_vpmadd52: + vmovdqu32 0($inp),%x#$T0 # load input as ----3210 + lea 16($inp),$inp + + vpermd $T0,$inp_permd,$T0 # ----3210 -> --322110 + vpsrlvq $inp_shift,$T0,$T0 + vpandq $reduc_mask,$T0,$T0 + vporq $PAD,$T0,$T0 + + vpaddq $T0,$Dlo,$Dlo # accumulate input + + vpermq \$0,$Dlo,${H0}{%k7}{z} # smash hash value + vpermq \$0b01010101,$Dlo,${H1}{%k7}{z} + vpermq \$0b10101010,$Dlo,${H2}{%k7}{z} + + vpxord $Dlo,$Dlo,$Dlo + vpxord $Dhi,$Dhi,$Dhi + + vpmadd52luq $r2r1r0,$H0,$Dlo + vpmadd52huq $r2r1r0,$H0,$Dhi + + vpmadd52luq $r1r0s2,$H1,$Dlo + vpmadd52huq $r1r0s2,$H1,$Dhi + + vpmadd52luq $r0s2s1,$H2,$Dlo + vpmadd52huq $r0s2s1,$H2,$Dhi + + vpsrlvq $reduc_rght,$Dlo,$T0 # 0 in topmost qword + vpsllvq $reduc_left,$Dhi,$Dhi # 0 in topmost qword + vpandq $reduc_mask,$Dlo,$Dlo + + vpaddq $T0,$Dhi,$Dhi + + vpermq \$0b10010011,$Dhi,$Dhi # 0 in lowest qword + + vpaddq $Dhi,$Dlo,$Dlo # note topmost qword :-) + + vpsrlvq $reduc_rght,$Dlo,$T0 # 0 in topmost word + vpandq $reduc_mask,$Dlo,$Dlo + + vpermq \$0b10010011,$T0,$T0 + + vpaddq $T0,$Dlo,$Dlo + + vpermq \$0b10010011,$Dlo,${T0}{%k1}{z} + + vpaddq $T0,$Dlo,$Dlo + vpsllq \$2,$T0,$T0 + + vpaddq $T0,$Dlo,$Dlo + + dec %rax # len-=16 + jnz .Loop_vpmadd52 + + vmovdqu64 $Dlo,0($ctx){%k7} # store hash value + + test $len,$len + jnz .Lblocks_vpmadd52_4x + +.Lno_data_vpmadd52: + ret +.size poly1305_blocks_vpmadd52,.-poly1305_blocks_vpmadd52 +___ +} +{ +######################################################################## +# As implied by its name 4x subroutine processes 4 blocks in parallel +# (but handles even 4*n+2 blocks lengths). It takes up to 4th key power +# and is handled in 256-bit %ymm registers. + +my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17)); +my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23)); +my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31)); + +$code.=<<___; +.type poly1305_blocks_vpmadd52_4x,\@function,4 +.align 32 +poly1305_blocks_vpmadd52_4x: + shr \$4,$len + jz .Lno_data_vpmadd52_4x # too short + + shl \$40,$padbit + mov 64($ctx),%r8 # peek on power of the key + +.Lblocks_vpmadd52_4x: + vpbroadcastq $padbit,$PAD + + vmovdqa64 .Lx_mask44(%rip),$mask44 + mov \$5,%eax + vmovdqa64 .Lx_mask42(%rip),$mask42 + kmovw %eax,%k1 # used in 2x path + + test %r8,%r8 # is power value impossible? + js .Linit_vpmadd52 # if it is, then init R[4] + + vmovq 0($ctx),%x#$H0 # load current hash value + vmovq 8($ctx),%x#$H1 + vmovq 16($ctx),%x#$H2 + + test \$3,$len # is length 4*n+2? + jnz .Lblocks_vpmadd52_2x_do + +.Lblocks_vpmadd52_4x_do: + vpbroadcastq 64($ctx),$R0 # load 4th power of the key + vpbroadcastq 96($ctx),$R1 + vpbroadcastq 128($ctx),$R2 + vpbroadcastq 160($ctx),$S1 + +.Lblocks_vpmadd52_4x_key_loaded: + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S2,$S2 + + test \$7,$len # is len 8*n? + jz .Lblocks_vpmadd52_8x + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*2($inp),$T3 + lea 16*4($inp),$inp + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + + # at this point 64-bit lanes are ordered as 3-1-2-0 + + vpsrlq \$24,$T3,$T2 # splat the data + vporq $PAD,$T2,$T2 + vpaddq $T2,$H2,$H2 # accumulate input + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + sub \$4,$len + jz .Ltail_vpmadd52_4x + jmp .Loop_vpmadd52_4x + ud2 + +.align 32 +.Linit_vpmadd52: + vmovq 24($ctx),%x#$S1 # load key + vmovq 56($ctx),%x#$H2 + vmovq 32($ctx),%x#$S2 + vmovq 40($ctx),%x#$R0 + vmovq 48($ctx),%x#$R1 + + vmovdqa $R0,$H0 + vmovdqa $R1,$H1 + vmovdqa $H2,$R2 + + mov \$2,%eax + +.Lmul_init_vpmadd52: + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + dec %eax + jz .Ldone_init_vpmadd52 + + vpunpcklqdq $R1,$H1,$R1 # 1,2 + vpbroadcastq %x#$H1,%x#$H1 # 2,2 + vpunpcklqdq $R2,$H2,$R2 + vpbroadcastq %x#$H2,%x#$H2 + vpunpcklqdq $R0,$H0,$R0 + vpbroadcastq %x#$H0,%x#$H0 + + vpsllq \$2,$R1,$S1 # S1 = R1*5*4 + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R1,$S1,$S1 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S1,$S1 + vpsllq \$2,$S2,$S2 + + jmp .Lmul_init_vpmadd52 + ud2 + +.align 32 +.Ldone_init_vpmadd52: + vinserti128 \$1,%x#$R1,$H1,$R1 # 1,2,3,4 + vinserti128 \$1,%x#$R2,$H2,$R2 + vinserti128 \$1,%x#$R0,$H0,$R0 + + vpermq \$0b11011000,$R1,$R1 # 1,3,2,4 + vpermq \$0b11011000,$R2,$R2 + vpermq \$0b11011000,$R0,$R0 + + vpsllq \$2,$R1,$S1 # S1 = R1*5*4 + vpaddq $R1,$S1,$S1 + vpsllq \$2,$S1,$S1 + + vmovq 0($ctx),%x#$H0 # load current hash value + vmovq 8($ctx),%x#$H1 + vmovq 16($ctx),%x#$H2 + + test \$3,$len # is length 4*n+2? + jnz .Ldone_init_vpmadd52_2x + + vmovdqu64 $R0,64($ctx) # save key powers + vpbroadcastq %x#$R0,$R0 # broadcast 4th power + vmovdqu64 $R1,96($ctx) + vpbroadcastq %x#$R1,$R1 + vmovdqu64 $R2,128($ctx) + vpbroadcastq %x#$R2,$R2 + vmovdqu64 $S1,160($ctx) + vpbroadcastq %x#$S1,$S1 + + jmp .Lblocks_vpmadd52_4x_key_loaded + ud2 + +.align 32 +.Ldone_init_vpmadd52_2x: + vmovdqu64 $R0,64($ctx) # save key powers + vpsrldq \$8,$R0,$R0 # 0-1-0-2 + vmovdqu64 $R1,96($ctx) + vpsrldq \$8,$R1,$R1 + vmovdqu64 $R2,128($ctx) + vpsrldq \$8,$R2,$R2 + vmovdqu64 $S1,160($ctx) + vpsrldq \$8,$S1,$S1 + jmp .Lblocks_vpmadd52_2x_key_loaded + ud2 + +.align 32 +.Lblocks_vpmadd52_2x_do: + vmovdqu64 128+8($ctx),${R2}{%k1}{z}# load 2nd and 1st key powers + vmovdqu64 160+8($ctx),${S1}{%k1}{z} + vmovdqu64 64+8($ctx),${R0}{%k1}{z} + vmovdqu64 96+8($ctx),${R1}{%k1}{z} + +.Lblocks_vpmadd52_2x_key_loaded: + vmovdqu64 16*0($inp),$T2 # load data + vpxorq $T3,$T3,$T3 + lea 16*2($inp),$inp + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + + # at this point 64-bit lanes are ordered as x-1-x-0 + + vpsrlq \$24,$T3,$T2 # splat the data + vporq $PAD,$T2,$T2 + vpaddq $T2,$H2,$H2 # accumulate input + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + jmp .Ltail_vpmadd52_2x + ud2 + +.align 32 +.Loop_vpmadd52_4x: + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*2($inp),$T3 + lea 16*4($inp),$inp + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # partial reduction (interleaved with data splat) + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpsrlq \$24,$T3,$T2 + vporq $PAD,$T2,$T2 + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + sub \$4,$len # len-=64 + jnz .Loop_vpmadd52_4x + +.Ltail_vpmadd52_4x: + vmovdqu64 128($ctx),$R2 # load all key powers + vmovdqu64 160($ctx),$S1 + vmovdqu64 64($ctx),$R0 + vmovdqu64 96($ctx),$R1 + +.Ltail_vpmadd52_2x: + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S2,$S2 + + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # horizontal addition + + mov \$1,%eax + kmovw %eax,%k1 + vpsrldq \$8,$D0lo,$T0 + vpsrldq \$8,$D0hi,$H0 + vpsrldq \$8,$D1lo,$T1 + vpsrldq \$8,$D1hi,$H1 + vpaddq $T0,$D0lo,$D0lo + vpaddq $H0,$D0hi,$D0hi + vpsrldq \$8,$D2lo,$T2 + vpsrldq \$8,$D2hi,$H2 + vpaddq $T1,$D1lo,$D1lo + vpaddq $H1,$D1hi,$D1hi + vpermq \$0x2,$D0lo,$T0 + vpermq \$0x2,$D0hi,$H0 + vpaddq $T2,$D2lo,$D2lo + vpaddq $H2,$D2hi,$D2hi + + vpermq \$0x2,$D1lo,$T1 + vpermq \$0x2,$D1hi,$H1 + vpaddq $T0,$D0lo,${D0lo}{%k1}{z} + vpaddq $H0,$D0hi,${D0hi}{%k1}{z} + vpermq \$0x2,$D2lo,$T2 + vpermq \$0x2,$D2hi,$H2 + vpaddq $T1,$D1lo,${D1lo}{%k1}{z} + vpaddq $H1,$D1hi,${D1hi}{%k1}{z} + vpaddq $T2,$D2lo,${D2lo}{%k1}{z} + vpaddq $H2,$D2hi,${D2hi}{%k1}{z} + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + # at this point $len is + # either 4*n+2 or 0... + sub \$2,$len # len-=32 + ja .Lblocks_vpmadd52_4x_do + + vmovq %x#$H0,0($ctx) + vmovq %x#$H1,8($ctx) + vmovq %x#$H2,16($ctx) + vzeroall + +.Lno_data_vpmadd52_4x: + ret +.size poly1305_blocks_vpmadd52_4x,.-poly1305_blocks_vpmadd52_4x +___ +} +{ +######################################################################## +# As implied by its name 8x subroutine processes 8 blocks in parallel... +# This is intermediate version, as it's used only in cases when input +# length is either 8*n, 8*n+1 or 8*n+2... + +my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17)); +my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23)); +my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31)); +my ($RR0,$RR1,$RR2,$SS1,$SS2) = map("%ymm$_",(6..10)); + +$code.=<<___; +.type poly1305_blocks_vpmadd52_8x,\@function,4 +.align 32 +poly1305_blocks_vpmadd52_8x: + shr \$4,$len + jz .Lno_data_vpmadd52_8x # too short + + shl \$40,$padbit + mov 64($ctx),%r8 # peek on power of the key + + vmovdqa64 .Lx_mask44(%rip),$mask44 + vmovdqa64 .Lx_mask42(%rip),$mask42 + + test %r8,%r8 # is power value impossible? + js .Linit_vpmadd52 # if it is, then init R[4] + + vmovq 0($ctx),%x#$H0 # load current hash value + vmovq 8($ctx),%x#$H1 + vmovq 16($ctx),%x#$H2 + +.Lblocks_vpmadd52_8x: + ################################################################ + # fist we calculate more key powers + + vmovdqu64 128($ctx),$R2 # load 1-3-2-4 powers + vmovdqu64 160($ctx),$S1 + vmovdqu64 64($ctx),$R0 + vmovdqu64 96($ctx),$R1 + + vpsllq \$2,$R2,$S2 # S2 = R2*5*4 + vpaddq $R2,$S2,$S2 + vpsllq \$2,$S2,$S2 + + vpbroadcastq %x#$R2,$RR2 # broadcast 4th power + vpbroadcastq %x#$R0,$RR0 + vpbroadcastq %x#$R1,$RR1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $RR2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $RR2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $RR2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $RR2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $RR2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $RR2,$R0,$D2hi + + vpmadd52luq $RR0,$R0,$D0lo + vpmadd52huq $RR0,$R0,$D0hi + vpmadd52luq $RR0,$R1,$D1lo + vpmadd52huq $RR0,$R1,$D1hi + vpmadd52luq $RR0,$R2,$D2lo + vpmadd52huq $RR0,$R2,$D2hi + + vpmadd52luq $RR1,$S2,$D0lo + vpmadd52huq $RR1,$S2,$D0hi + vpmadd52luq $RR1,$R0,$D1lo + vpmadd52huq $RR1,$R0,$D1hi + vpmadd52luq $RR1,$R1,$D2lo + vpmadd52huq $RR1,$R1,$D2hi + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$RR0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$RR1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$RR2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$RR0,$RR0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$RR0,$RR0 + + vpsrlq \$44,$RR0,$tmp # additional step + vpandq $mask44,$RR0,$RR0 + + vpaddq $tmp,$RR1,$RR1 + + ################################################################ + # At this point Rx holds 1324 powers, RRx - 5768, and the goal + # is 15263748, which reflects how data is loaded... + + vpunpcklqdq $R2,$RR2,$T2 # 3748 + vpunpckhqdq $R2,$RR2,$R2 # 1526 + vpunpcklqdq $R0,$RR0,$T0 + vpunpckhqdq $R0,$RR0,$R0 + vpunpcklqdq $R1,$RR1,$T1 + vpunpckhqdq $R1,$RR1,$R1 +___ +######## switch to %zmm +map(s/%y/%z/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2); +map(s/%y/%z/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi); +map(s/%y/%z/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD); +map(s/%y/%z/, $RR0,$RR1,$RR2,$SS1,$SS2); + +$code.=<<___; + vshufi64x2 \$0x44,$R2,$T2,$RR2 # 15263748 + vshufi64x2 \$0x44,$R0,$T0,$RR0 + vshufi64x2 \$0x44,$R1,$T1,$RR1 + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*4($inp),$T3 + lea 16*8($inp),$inp + + vpsllq \$2,$RR2,$SS2 # S2 = R2*5*4 + vpsllq \$2,$RR1,$SS1 # S1 = R1*5*4 + vpaddq $RR2,$SS2,$SS2 + vpaddq $RR1,$SS1,$SS1 + vpsllq \$2,$SS2,$SS2 + vpsllq \$2,$SS1,$SS1 + + vpbroadcastq $padbit,$PAD + vpbroadcastq %x#$mask44,$mask44 + vpbroadcastq %x#$mask42,$mask42 + + vpbroadcastq %x#$SS1,$S1 # broadcast 8th power + vpbroadcastq %x#$SS2,$S2 + vpbroadcastq %x#$RR0,$R0 + vpbroadcastq %x#$RR1,$R1 + vpbroadcastq %x#$RR2,$R2 + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + + # at this point 64-bit lanes are ordered as 73625140 + + vpsrlq \$24,$T3,$T2 # splat the data + vporq $PAD,$T2,$T2 + vpaddq $T2,$H2,$H2 # accumulate input + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + sub \$8,$len + jz .Ltail_vpmadd52_8x + jmp .Loop_vpmadd52_8x + +.align 32 +.Loop_vpmadd52_8x: + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$S1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$S1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$S2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$S2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$R0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$R0,$D2hi + + vmovdqu64 16*0($inp),$T2 # load data + vmovdqu64 16*4($inp),$T3 + lea 16*8($inp),$inp + vpmadd52luq $H0,$R0,$D0lo + vpmadd52huq $H0,$R0,$D0hi + vpmadd52luq $H0,$R1,$D1lo + vpmadd52huq $H0,$R1,$D1hi + vpmadd52luq $H0,$R2,$D2lo + vpmadd52huq $H0,$R2,$D2hi + + vpunpcklqdq $T3,$T2,$T1 # transpose data + vpunpckhqdq $T3,$T2,$T3 + vpmadd52luq $H1,$S2,$D0lo + vpmadd52huq $H1,$S2,$D0hi + vpmadd52luq $H1,$R0,$D1lo + vpmadd52huq $H1,$R0,$D1hi + vpmadd52luq $H1,$R1,$D2lo + vpmadd52huq $H1,$R1,$D2hi + + ################################################################ + # partial reduction (interleaved with data splat) + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpsrlq \$24,$T3,$T2 + vporq $PAD,$T2,$T2 + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpandq $mask44,$T1,$T0 + vpsrlq \$44,$T1,$T1 + vpsllq \$20,$T3,$T3 + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vporq $T3,$T1,$T1 + vpandq $mask44,$T1,$T1 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + sub \$8,$len # len-=128 + jnz .Loop_vpmadd52_8x + +.Ltail_vpmadd52_8x: + #vpaddq $T2,$H2,$H2 # accumulate input + vpaddq $T0,$H0,$H0 + vpaddq $T1,$H1,$H1 + + vpxorq $D0lo,$D0lo,$D0lo + vpmadd52luq $H2,$SS1,$D0lo + vpxorq $D0hi,$D0hi,$D0hi + vpmadd52huq $H2,$SS1,$D0hi + vpxorq $D1lo,$D1lo,$D1lo + vpmadd52luq $H2,$SS2,$D1lo + vpxorq $D1hi,$D1hi,$D1hi + vpmadd52huq $H2,$SS2,$D1hi + vpxorq $D2lo,$D2lo,$D2lo + vpmadd52luq $H2,$RR0,$D2lo + vpxorq $D2hi,$D2hi,$D2hi + vpmadd52huq $H2,$RR0,$D2hi + + vpmadd52luq $H0,$RR0,$D0lo + vpmadd52huq $H0,$RR0,$D0hi + vpmadd52luq $H0,$RR1,$D1lo + vpmadd52huq $H0,$RR1,$D1hi + vpmadd52luq $H0,$RR2,$D2lo + vpmadd52huq $H0,$RR2,$D2hi + + vpmadd52luq $H1,$SS2,$D0lo + vpmadd52huq $H1,$SS2,$D0hi + vpmadd52luq $H1,$RR0,$D1lo + vpmadd52huq $H1,$RR0,$D1hi + vpmadd52luq $H1,$RR1,$D2lo + vpmadd52huq $H1,$RR1,$D2hi + + ################################################################ + # horizontal addition + + mov \$1,%eax + kmovw %eax,%k1 + vpsrldq \$8,$D0lo,$T0 + vpsrldq \$8,$D0hi,$H0 + vpsrldq \$8,$D1lo,$T1 + vpsrldq \$8,$D1hi,$H1 + vpaddq $T0,$D0lo,$D0lo + vpaddq $H0,$D0hi,$D0hi + vpsrldq \$8,$D2lo,$T2 + vpsrldq \$8,$D2hi,$H2 + vpaddq $T1,$D1lo,$D1lo + vpaddq $H1,$D1hi,$D1hi + vpermq \$0x2,$D0lo,$T0 + vpermq \$0x2,$D0hi,$H0 + vpaddq $T2,$D2lo,$D2lo + vpaddq $H2,$D2hi,$D2hi + + vpermq \$0x2,$D1lo,$T1 + vpermq \$0x2,$D1hi,$H1 + vpaddq $T0,$D0lo,$D0lo + vpaddq $H0,$D0hi,$D0hi + vpermq \$0x2,$D2lo,$T2 + vpermq \$0x2,$D2hi,$H2 + vpaddq $T1,$D1lo,$D1lo + vpaddq $H1,$D1hi,$D1hi + vextracti64x4 \$1,$D0lo,%y#$T0 + vextracti64x4 \$1,$D0hi,%y#$H0 + vpaddq $T2,$D2lo,$D2lo + vpaddq $H2,$D2hi,$D2hi + + vextracti64x4 \$1,$D1lo,%y#$T1 + vextracti64x4 \$1,$D1hi,%y#$H1 + vextracti64x4 \$1,$D2lo,%y#$T2 + vextracti64x4 \$1,$D2hi,%y#$H2 +___ +######## switch back to %ymm +map(s/%z/%y/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2); +map(s/%z/%y/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi); +map(s/%z/%y/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD); + +$code.=<<___; + vpaddq $T0,$D0lo,${D0lo}{%k1}{z} + vpaddq $H0,$D0hi,${D0hi}{%k1}{z} + vpaddq $T1,$D1lo,${D1lo}{%k1}{z} + vpaddq $H1,$D1hi,${D1hi}{%k1}{z} + vpaddq $T2,$D2lo,${D2lo}{%k1}{z} + vpaddq $H2,$D2hi,${D2hi}{%k1}{z} + + ################################################################ + # partial reduction + vpsrlq \$44,$D0lo,$tmp + vpsllq \$8,$D0hi,$D0hi + vpandq $mask44,$D0lo,$H0 + vpaddq $tmp,$D0hi,$D0hi + + vpaddq $D0hi,$D1lo,$D1lo + + vpsrlq \$44,$D1lo,$tmp + vpsllq \$8,$D1hi,$D1hi + vpandq $mask44,$D1lo,$H1 + vpaddq $tmp,$D1hi,$D1hi + + vpaddq $D1hi,$D2lo,$D2lo + + vpsrlq \$42,$D2lo,$tmp + vpsllq \$10,$D2hi,$D2hi + vpandq $mask42,$D2lo,$H2 + vpaddq $tmp,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + vpsllq \$2,$D2hi,$D2hi + + vpaddq $D2hi,$H0,$H0 + + vpsrlq \$44,$H0,$tmp # additional step + vpandq $mask44,$H0,$H0 + + vpaddq $tmp,$H1,$H1 + + ################################################################ + + vmovq %x#$H0,0($ctx) + vmovq %x#$H1,8($ctx) + vmovq %x#$H2,16($ctx) + vzeroall + +.Lno_data_vpmadd52_8x: + ret +.size poly1305_blocks_vpmadd52_8x,.-poly1305_blocks_vpmadd52_8x +___ +} +$code.=<<___; +.type poly1305_emit_base2_44,\@function,3 +.align 32 +poly1305_emit_base2_44: + mov 0($ctx),%r8 # load hash value + mov 8($ctx),%r9 + mov 16($ctx),%r10 + + mov %r9,%rax + shr \$20,%r9 + shl \$44,%rax + mov %r10,%rcx + shr \$40,%r10 + shl \$24,%rcx + + add %rax,%r8 + adc %rcx,%r9 + adc \$0,%r10 + + mov %r8,%rax + add \$5,%r8 # compare to modulus + mov %r9,%rcx + adc \$0,%r9 + adc \$0,%r10 + shr \$2,%r10 # did 130-bit value overflow? + cmovnz %r8,%rax + cmovnz %r9,%rcx + + add 0($nonce),%rax # accumulate nonce + adc 8($nonce),%rcx + mov %rax,0($mac) # write result + mov %rcx,8($mac) + + ret +.size poly1305_emit_base2_44,.-poly1305_emit_base2_44 +___ +} } } +} + +if (!$kernel) +{ # chacha20-poly1305 helpers +my ($out,$inp,$otp,$len)=$win64 ? ("%rcx","%rdx","%r8", "%r9") : # Win64 order + ("%rdi","%rsi","%rdx","%rcx"); # Unix order +$code.=<<___; +.globl xor128_encrypt_n_pad +.type xor128_encrypt_n_pad,\@abi-omnipotent +.align 16 +xor128_encrypt_n_pad: + sub $otp,$inp + sub $otp,$out + mov $len,%r10 # put len aside + shr \$4,$len # len / 16 + jz .Ltail_enc + nop +.Loop_enc_xmm: + movdqu ($inp,$otp),%xmm0 + pxor ($otp),%xmm0 + movdqu %xmm0,($out,$otp) + movdqa %xmm0,($otp) + lea 16($otp),$otp + dec $len + jnz .Loop_enc_xmm + + and \$15,%r10 # len % 16 + jz .Ldone_enc + +.Ltail_enc: + mov \$16,$len + sub %r10,$len + xor %eax,%eax +.Loop_enc_byte: + mov ($inp,$otp),%al + xor ($otp),%al + mov %al,($out,$otp) + mov %al,($otp) + lea 1($otp),$otp + dec %r10 + jnz .Loop_enc_byte + + xor %eax,%eax +.Loop_enc_pad: + mov %al,($otp) + lea 1($otp),$otp + dec $len + jnz .Loop_enc_pad + +.Ldone_enc: + mov $otp,%rax + ret +.size xor128_encrypt_n_pad,.-xor128_encrypt_n_pad + +.globl xor128_decrypt_n_pad +.type xor128_decrypt_n_pad,\@abi-omnipotent +.align 16 +xor128_decrypt_n_pad: + sub $otp,$inp + sub $otp,$out + mov $len,%r10 # put len aside + shr \$4,$len # len / 16 + jz .Ltail_dec + nop +.Loop_dec_xmm: + movdqu ($inp,$otp),%xmm0 + movdqa ($otp),%xmm1 + pxor %xmm0,%xmm1 + movdqu %xmm1,($out,$otp) + movdqa %xmm0,($otp) + lea 16($otp),$otp + dec $len + jnz .Loop_dec_xmm + + pxor %xmm1,%xmm1 + and \$15,%r10 # len % 16 + jz .Ldone_dec + +.Ltail_dec: + mov \$16,$len + sub %r10,$len + xor %eax,%eax + xor %r11,%r11 +.Loop_dec_byte: + mov ($inp,$otp),%r11b + mov ($otp),%al + xor %r11b,%al + mov %al,($out,$otp) + mov %r11b,($otp) + lea 1($otp),$otp + dec %r10 + jnz .Loop_dec_byte + + xor %eax,%eax +.Loop_dec_pad: + mov %al,($otp) + lea 1($otp),$otp + dec $len + jnz .Loop_dec_pad + +.Ldone_dec: + mov $otp,%rax + ret +.size xor128_decrypt_n_pad,.-xor128_decrypt_n_pad +___ +} + +# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, +# CONTEXT *context,DISPATCHER_CONTEXT *disp) +if ($win64) { +$rec="%rcx"; +$frame="%rdx"; +$context="%r8"; +$disp="%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type se_handler,\@abi-omnipotent +.align 16 +se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->Rip<.Lprologue + jb .Lcommon_seh_tail + + mov 152($context),%rax # pull context->Rsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=.Lepilogue + jae .Lcommon_seh_tail + + lea 48(%rax),%rax + + mov -8(%rax),%rbx + mov -16(%rax),%rbp + mov -24(%rax),%r12 + mov -32(%rax),%r13 + mov -40(%rax),%r14 + mov -48(%rax),%r15 + mov %rbx,144($context) # restore context->Rbx + mov %rbp,160($context) # restore context->Rbp + mov %r12,216($context) # restore context->R12 + mov %r13,224($context) # restore context->R13 + mov %r14,232($context) # restore context->R14 + mov %r15,240($context) # restore context->R14 + + jmp .Lcommon_seh_tail +.size se_handler,.-se_handler + +.type avx_handler,\@abi-omnipotent +.align 16 +avx_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64,%rsp + + mov 120($context),%rax # pull context->Rax + mov 248($context),%rbx # pull context->Rip + + mov 8($disp),%rsi # disp->ImageBase + mov 56($disp),%r11 # disp->HandlerData + + mov 0(%r11),%r10d # HandlerData[0] + lea (%rsi,%r10),%r10 # prologue label + cmp %r10,%rbx # context->RipRsp + + mov 4(%r11),%r10d # HandlerData[1] + lea (%rsi,%r10),%r10 # epilogue label + cmp %r10,%rbx # context->Rip>=epilogue label + jae .Lcommon_seh_tail + + mov 208($context),%rax # pull context->R11 + + lea 0x50(%rax),%rsi + lea 0xf8(%rax),%rax + lea 512($context),%rdi # &context.Xmm6 + mov \$20,%ecx + .long 0xa548f3fc # cld; rep movsq + +.Lcommon_seh_tail: + mov 8(%rax),%rdi + mov 16(%rax),%rsi + mov %rax,152($context) # restore context->Rsp + mov %rsi,168($context) # restore context->Rsi + mov %rdi,176($context) # restore context->Rdi + + mov 40($disp),%rdi # disp->ContextRecord + mov $context,%rsi # context + mov \$154,%ecx # sizeof(CONTEXT) + .long 0xa548f3fc # cld; rep movsq + + mov $disp,%rsi + xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER + mov 8(%rsi),%rdx # arg2, disp->ImageBase + mov 0(%rsi),%r8 # arg3, disp->ControlPc + mov 16(%rsi),%r9 # arg4, disp->FunctionEntry + mov 40(%rsi),%r10 # disp->ContextRecord + lea 56(%rsi),%r11 # &disp->HandlerData + lea 24(%rsi),%r12 # &disp->EstablisherFrame + mov %r10,32(%rsp) # arg5 + mov %r11,40(%rsp) # arg6 + mov %r12,48(%rsp) # arg7 + mov %rcx,56(%rsp) # arg8, (NULL) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1,%eax # ExceptionContinueSearch + add \$64,%rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size avx_handler,.-avx_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_poly1305_init_x86_64 + .rva .LSEH_end_poly1305_init_x86_64 + .rva .LSEH_info_poly1305_init_x86_64 + + .rva .LSEH_begin_poly1305_blocks_x86_64 + .rva .LSEH_end_poly1305_blocks_x86_64 + .rva .LSEH_info_poly1305_blocks_x86_64 + + .rva .LSEH_begin_poly1305_emit_x86_64 + .rva .LSEH_end_poly1305_emit_x86_64 + .rva .LSEH_info_poly1305_emit_x86_64 +___ +$code.=<<___ if ($avx); + .rva .LSEH_begin_poly1305_blocks_avx + .rva .Lbase2_64_avx + .rva .LSEH_info_poly1305_blocks_avx_1 + + .rva .Lbase2_64_avx + .rva .Leven_avx + .rva .LSEH_info_poly1305_blocks_avx_2 + + .rva .Leven_avx + .rva .LSEH_end_poly1305_blocks_avx + .rva .LSEH_info_poly1305_blocks_avx_3 + + .rva .LSEH_begin_poly1305_emit_avx + .rva .LSEH_end_poly1305_emit_avx + .rva .LSEH_info_poly1305_emit_avx +___ +$code.=<<___ if ($avx>1); + .rva .LSEH_begin_poly1305_blocks_avx2 + .rva .Lbase2_64_avx2 + .rva .LSEH_info_poly1305_blocks_avx2_1 + + .rva .Lbase2_64_avx2 + .rva .Leven_avx2 + .rva .LSEH_info_poly1305_blocks_avx2_2 + + .rva .Leven_avx2 + .rva .LSEH_end_poly1305_blocks_avx2 + .rva .LSEH_info_poly1305_blocks_avx2_3 +___ +$code.=<<___ if ($avx>2); + .rva .LSEH_begin_poly1305_blocks_avx512 + .rva .LSEH_end_poly1305_blocks_avx512 + .rva .LSEH_info_poly1305_blocks_avx512 +___ +$code.=<<___; +.section .xdata +.align 8 +.LSEH_info_poly1305_init_x86_64: + .byte 9,0,0,0 + .rva se_handler + .rva .LSEH_begin_poly1305_init_x86_64,.LSEH_begin_poly1305_init_x86_64 + +.LSEH_info_poly1305_blocks_x86_64: + .byte 9,0,0,0 + .rva se_handler + .rva .Lblocks_body,.Lblocks_epilogue + +.LSEH_info_poly1305_emit_x86_64: + .byte 9,0,0,0 + .rva se_handler + .rva .LSEH_begin_poly1305_emit_x86_64,.LSEH_begin_poly1305_emit_x86_64 +___ +$code.=<<___ if ($avx); +.LSEH_info_poly1305_blocks_avx_1: + .byte 9,0,0,0 + .rva se_handler + .rva .Lblocks_avx_body,.Lblocks_avx_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx_2: + .byte 9,0,0,0 + .rva se_handler + .rva .Lbase2_64_avx_body,.Lbase2_64_avx_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx_3: + .byte 9,0,0,0 + .rva avx_handler + .rva .Ldo_avx_body,.Ldo_avx_epilogue # HandlerData[] + +.LSEH_info_poly1305_emit_avx: + .byte 9,0,0,0 + .rva se_handler + .rva .LSEH_begin_poly1305_emit_avx,.LSEH_begin_poly1305_emit_avx +___ +$code.=<<___ if ($avx>1); +.LSEH_info_poly1305_blocks_avx2_1: + .byte 9,0,0,0 + .rva se_handler + .rva .Lblocks_avx2_body,.Lblocks_avx2_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx2_2: + .byte 9,0,0,0 + .rva se_handler + .rva .Lbase2_64_avx2_body,.Lbase2_64_avx2_epilogue # HandlerData[] + +.LSEH_info_poly1305_blocks_avx2_3: + .byte 9,0,0,0 + .rva avx_handler + .rva .Ldo_avx2_body,.Ldo_avx2_epilogue # HandlerData[] +___ +$code.=<<___ if ($avx>2); +.LSEH_info_poly1305_blocks_avx512: + .byte 9,0,0,0 + .rva avx_handler + .rva .Ldo_avx512_body,.Ldo_avx512_epilogue # HandlerData[] +___ +} + +open SELF,$0; +while() { + next if (/^#!/); + last if (!s/^#/\/\// and !/^$/); + print; +} +close SELF; + +foreach (split('\n',$code)) { + s/\`([^\`]*)\`/eval($1)/ge; + s/%r([a-z]+)#d/%e$1/g; + s/%r([0-9]+)#d/%r$1d/g; + s/%x#%[yz]/%x/g or s/%y#%z/%y/g or s/%z#%[yz]/%z/g; + + if ($kernel) { + s/(^\.type.*),[0-9]+$/\1/; + s/(^\.type.*),\@abi-omnipotent+$/\1,\@function/; + next if /^\.cfi.*/; + } + + print $_,"\n"; +} +close STDOUT; diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index 0cc4537e6617c..79bb58737d528 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -1,8 +1,6 @@ -// SPDX-License-Identifier: GPL-2.0-or-later +// SPDX-License-Identifier: GPL-2.0 OR MIT /* - * Poly1305 authenticator algorithm, RFC7539, SIMD glue code - * - * Copyright (C) 2015 Martin Willi + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. */ #include @@ -13,108 +11,166 @@ #include #include #include +#include #include -asmlinkage void poly1305_block_sse2(u32 *h, const u8 *src, - const u32 *r, unsigned int blocks); -asmlinkage void poly1305_2block_sse2(u32 *h, const u8 *src, const u32 *r, - unsigned int blocks, const u32 *u); -asmlinkage void poly1305_4block_avx2(u32 *h, const u8 *src, const u32 *r, - unsigned int blocks, const u32 *u); - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_simd); +asmlinkage void poly1305_init_x86_64(void *ctx, + const u8 key[POLY1305_KEY_SIZE]); +asmlinkage void poly1305_blocks_x86_64(void *ctx, const u8 *inp, + const size_t len, const u32 padbit); +asmlinkage void poly1305_emit_x86_64(void *ctx, u8 mac[POLY1305_DIGEST_SIZE], + const u32 nonce[4]); +asmlinkage void poly1305_emit_avx(void *ctx, u8 mac[POLY1305_DIGEST_SIZE], + const u32 nonce[4]); +asmlinkage void poly1305_blocks_avx(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_blocks_avx2(void *ctx, const u8 *inp, const size_t len, + const u32 padbit); +asmlinkage void poly1305_blocks_avx512(void *ctx, const u8 *inp, + const size_t len, const u32 padbit); + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx); static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx2); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(poly1305_use_avx512); + +struct poly1305_arch_internal { + union { + struct { + u32 h[5]; + u32 is_base2_26; + }; + u64 hs[3]; + }; + u64 r[2]; + u64 pad; + struct { u32 r2, r1, r4, r3; } rn[9]; +}; -static void poly1305_simd_mult(u32 *a, const u32 *b) +/* The AVX code uses base 2^26, while the scalar code uses base 2^64. If we hit + * the unfortunate situation of using AVX and then having to go back to scalar + * -- because the user is silly and has called the update function from two + * separate contexts -- then we need to convert back to the original base before + * proceeding. It is possible to reason that the initial reduction below is + * sufficient given the implementation invariants. However, for an avoidance of + * doubt and because this is not performance critical, we do the full reduction + * anyway. Z3 proof of below function: https://xn--4db.cc/ltPtHCKN/py + */ +static void convert_to_base2_64(void *ctx) { - u8 m[POLY1305_BLOCK_SIZE]; - - memset(m, 0, sizeof(m)); - /* The poly1305 block function adds a hi-bit to the accumulator which - * we don't need for key multiplication; compensate for it. */ - a[4] -= 1 << 24; - poly1305_block_sse2(a, m, b, 1); + struct poly1305_arch_internal *state = ctx; + u32 cy; + + if (!state->is_base2_26) + return; + + cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy; + cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy; + cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy; + cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy; + state->hs[0] = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0]; + state->hs[1] = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12); + state->hs[2] = state->h[4] >> 24; +#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1)) + cy = (state->hs[2] >> 2) + (state->hs[2] & ~3ULL); + state->hs[2] &= 3; + state->hs[0] += cy; + state->hs[1] += (cy = ULT(state->hs[0], cy)); + state->hs[2] += ULT(state->hs[1], cy); +#undef ULT + state->is_base2_26 = 0; } -static unsigned int poly1305_scalar_blocks(struct poly1305_desc_ctx *dctx, - const u8 *src, unsigned int srclen) +static void poly1305_simd_init(void *ctx, const u8 key[POLY1305_KEY_SIZE]) { - unsigned int datalen; - - if (unlikely(!dctx->sset)) { - datalen = crypto_poly1305_setdesckey(dctx, src, srclen); - src += srclen - datalen; - srclen = datalen; - } - if (srclen >= POLY1305_BLOCK_SIZE) { - poly1305_core_blocks(&dctx->h, dctx->r, src, - srclen / POLY1305_BLOCK_SIZE, 1); - srclen %= POLY1305_BLOCK_SIZE; - } - return srclen; + poly1305_init_x86_64(ctx, key); } -static unsigned int poly1305_simd_blocks(struct poly1305_desc_ctx *dctx, - const u8 *src, unsigned int srclen) +static void poly1305_simd_blocks(void *ctx, const u8 *inp, size_t len, + const u32 padbit) { - unsigned int blocks, datalen; - - if (unlikely(!dctx->sset)) { - datalen = crypto_poly1305_setdesckey(dctx, src, srclen); - src += srclen - datalen; - srclen = datalen; + struct poly1305_arch_internal *state = ctx; + + /* SIMD disables preemption, so relax after processing each page. */ + BUILD_BUG_ON(PAGE_SIZE < POLY1305_BLOCK_SIZE || + PAGE_SIZE % POLY1305_BLOCK_SIZE); + + if (!IS_ENABLED(CONFIG_AS_AVX) || !static_branch_likely(&poly1305_use_avx) || + (len < (POLY1305_BLOCK_SIZE * 18) && !state->is_base2_26) || + !crypto_simd_usable()) { + convert_to_base2_64(ctx); + poly1305_blocks_x86_64(ctx, inp, len, padbit); + return; } - if (IS_ENABLED(CONFIG_AS_AVX2) && - static_branch_likely(&poly1305_use_avx2) && - srclen >= POLY1305_BLOCK_SIZE * 4) { - if (unlikely(dctx->rset < 4)) { - if (dctx->rset < 2) { - dctx->r[1] = dctx->r[0]; - poly1305_simd_mult(dctx->r[1].r, dctx->r[0].r); - } - dctx->r[2] = dctx->r[1]; - poly1305_simd_mult(dctx->r[2].r, dctx->r[0].r); - dctx->r[3] = dctx->r[2]; - poly1305_simd_mult(dctx->r[3].r, dctx->r[0].r); - dctx->rset = 4; - } - blocks = srclen / (POLY1305_BLOCK_SIZE * 4); - poly1305_4block_avx2(dctx->h.h, src, dctx->r[0].r, blocks, - dctx->r[1].r); - src += POLY1305_BLOCK_SIZE * 4 * blocks; - srclen -= POLY1305_BLOCK_SIZE * 4 * blocks; + for (;;) { + const size_t bytes = min_t(size_t, len, PAGE_SIZE); + + kernel_fpu_begin(); + if (IS_ENABLED(CONFIG_AS_AVX512) && static_branch_likely(&poly1305_use_avx512)) + poly1305_blocks_avx512(ctx, inp, bytes, padbit); + else if (IS_ENABLED(CONFIG_AS_AVX2) && static_branch_likely(&poly1305_use_avx2)) + poly1305_blocks_avx2(ctx, inp, bytes, padbit); + else + poly1305_blocks_avx(ctx, inp, bytes, padbit); + kernel_fpu_end(); + len -= bytes; + if (!len) + break; + inp += bytes; } +} - if (likely(srclen >= POLY1305_BLOCK_SIZE * 2)) { - if (unlikely(dctx->rset < 2)) { - dctx->r[1] = dctx->r[0]; - poly1305_simd_mult(dctx->r[1].r, dctx->r[0].r); - dctx->rset = 2; - } - blocks = srclen / (POLY1305_BLOCK_SIZE * 2); - poly1305_2block_sse2(dctx->h.h, src, dctx->r[0].r, - blocks, dctx->r[1].r); - src += POLY1305_BLOCK_SIZE * 2 * blocks; - srclen -= POLY1305_BLOCK_SIZE * 2 * blocks; - } - if (srclen >= POLY1305_BLOCK_SIZE) { - poly1305_block_sse2(dctx->h.h, src, dctx->r[0].r, 1); - srclen -= POLY1305_BLOCK_SIZE; - } - return srclen; +static void poly1305_simd_emit(void *ctx, u8 mac[POLY1305_DIGEST_SIZE], + const u32 nonce[4]) +{ + if (!IS_ENABLED(CONFIG_AS_AVX) || !static_branch_likely(&poly1305_use_avx)) + poly1305_emit_x86_64(ctx, mac, nonce); + else + poly1305_emit_avx(ctx, mac, nonce); } -void poly1305_init_arch(struct poly1305_desc_ctx *desc, const u8 *key) +void poly1305_init_arch(struct poly1305_desc_ctx *dctx, const u8 *key) { - poly1305_init_generic(desc, key); + poly1305_simd_init(&dctx->h, key); + dctx->s[0] = get_unaligned_le32(&key[16]); + dctx->s[1] = get_unaligned_le32(&key[20]); + dctx->s[2] = get_unaligned_le32(&key[24]); + dctx->s[3] = get_unaligned_le32(&key[28]); + dctx->buflen = 0; + dctx->sset = true; } EXPORT_SYMBOL(poly1305_init_arch); +static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx, + const u8 *inp, unsigned int len) +{ + unsigned int acc = 0; + if (unlikely(!dctx->sset)) { + if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) { + poly1305_simd_init(&dctx->h, inp); + inp += POLY1305_BLOCK_SIZE; + len -= POLY1305_BLOCK_SIZE; + acc += POLY1305_BLOCK_SIZE; + dctx->rset = 1; + } + if (len >= POLY1305_BLOCK_SIZE) { + dctx->s[0] = get_unaligned_le32(&inp[0]); + dctx->s[1] = get_unaligned_le32(&inp[4]); + dctx->s[2] = get_unaligned_le32(&inp[8]); + dctx->s[3] = get_unaligned_le32(&inp[12]); + inp += POLY1305_BLOCK_SIZE; + len -= POLY1305_BLOCK_SIZE; + acc += POLY1305_BLOCK_SIZE; + dctx->sset = true; + } + } + return acc; +} + void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int srclen) { - unsigned int bytes; + unsigned int bytes, used; if (unlikely(dctx->buflen)) { bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); @@ -124,31 +180,19 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, dctx->buflen += bytes; if (dctx->buflen == POLY1305_BLOCK_SIZE) { - if (static_branch_likely(&poly1305_use_simd) && - likely(crypto_simd_usable())) { - kernel_fpu_begin(); - poly1305_simd_blocks(dctx, dctx->buf, - POLY1305_BLOCK_SIZE); - kernel_fpu_end(); - } else { - poly1305_scalar_blocks(dctx, dctx->buf, - POLY1305_BLOCK_SIZE); - } + if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf, POLY1305_BLOCK_SIZE))) + poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 1); dctx->buflen = 0; } } if (likely(srclen >= POLY1305_BLOCK_SIZE)) { - if (static_branch_likely(&poly1305_use_simd) && - likely(crypto_simd_usable())) { - kernel_fpu_begin(); - bytes = poly1305_simd_blocks(dctx, src, srclen); - kernel_fpu_end(); - } else { - bytes = poly1305_scalar_blocks(dctx, src, srclen); - } - src += srclen - bytes; - srclen = bytes; + bytes = round_down(srclen, POLY1305_BLOCK_SIZE); + srclen -= bytes; + used = crypto_poly1305_setdctxkey(dctx, src, bytes); + if (likely(bytes - used)) + poly1305_simd_blocks(&dctx->h, src + used, bytes - used, 1); + src += bytes; } if (unlikely(srclen)) { @@ -158,9 +202,17 @@ void poly1305_update_arch(struct poly1305_desc_ctx *dctx, const u8 *src, } EXPORT_SYMBOL(poly1305_update_arch); -void poly1305_final_arch(struct poly1305_desc_ctx *desc, u8 *digest) +void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) { - poly1305_final_generic(desc, digest); + if (unlikely(dctx->buflen)) { + dctx->buf[dctx->buflen++] = 1; + memset(dctx->buf + dctx->buflen, 0, + POLY1305_BLOCK_SIZE - dctx->buflen); + poly1305_simd_blocks(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); + } + + poly1305_simd_emit(&dctx->h, dst, dctx->s); + *dctx = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL(poly1305_final_arch); @@ -168,38 +220,34 @@ static int crypto_poly1305_init(struct shash_desc *desc) { struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - poly1305_core_init(&dctx->h); - dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - + *dctx = (struct poly1305_desc_ctx){}; return 0; } -static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) +static int crypto_poly1305_update(struct shash_desc *desc, + const u8 *src, unsigned int srclen) { struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - if (unlikely(!dctx->sset)) - return -ENOKEY; - - poly1305_final_generic(dctx, dst); + poly1305_update_arch(dctx, src, srclen); return 0; } -static int poly1305_simd_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) +static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) { struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - poly1305_update_arch(dctx, src, srclen); + if (unlikely(!dctx->sset)) + return -ENOKEY; + + poly1305_final_arch(dctx, dst); return 0; } static struct shash_alg alg = { .digestsize = POLY1305_DIGEST_SIZE, .init = crypto_poly1305_init, - .update = poly1305_simd_update, + .update = crypto_poly1305_update, .final = crypto_poly1305_final, .descsize = sizeof(struct poly1305_desc_ctx), .base = { @@ -213,17 +261,19 @@ static struct shash_alg alg = { static int __init poly1305_simd_mod_init(void) { - if (!boot_cpu_has(X86_FEATURE_XMM2)) - return 0; - - static_branch_enable(&poly1305_use_simd); - - if (IS_ENABLED(CONFIG_AS_AVX2) && - boot_cpu_has(X86_FEATURE_AVX) && + if (IS_ENABLED(CONFIG_AS_AVX) && boot_cpu_has(X86_FEATURE_AVX) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) + static_branch_enable(&poly1305_use_avx); + if (IS_ENABLED(CONFIG_AS_AVX2) && boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_AVX2) && cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) static_branch_enable(&poly1305_use_avx2); - + if (IS_ENABLED(CONFIG_AS_AVX512) && boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX512F) && + cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | XFEATURE_MASK_AVX512, NULL) && + /* Skylake downclocks unacceptably much when using zmm, but later generations are fast. */ + boot_cpu_data.x86_model != INTEL_FAM6_SKYLAKE_X) + static_branch_enable(&poly1305_use_avx512); return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? crypto_register_shash(&alg) : 0; } @@ -237,7 +287,7 @@ module_init(poly1305_simd_mod_init); module_exit(poly1305_simd_mod_exit); MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Martin Willi "); +MODULE_AUTHOR("Jason A. Donenfeld "); MODULE_DESCRIPTION("Poly1305 authenticator"); MODULE_ALIAS_CRYPTO("poly1305"); MODULE_ALIAS_CRYPTO("poly1305-simd"); diff --git a/crypto/Kconfig b/crypto/Kconfig index cdb51d4272d0c..c24a47406f8f5 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -136,8 +136,6 @@ config CRYPTO_USER Userspace configuration for cryptographic instantiations such as cbc(aes). -if CRYPTO_MANAGER2 - config CRYPTO_MANAGER_DISABLE_TESTS bool "Disable run-time self tests" default y @@ -155,8 +153,6 @@ config CRYPTO_MANAGER_EXTRA_TESTS This is intended for developer use only, as these tests take much longer to run than the normal self tests. -endif # if CRYPTO_MANAGER2 - config CRYPTO_GF128MUL tristate diff --git a/crypto/adiantum.c b/crypto/adiantum.c index 9dc53cf9b1f17..24440f6a69153 100644 --- a/crypto/adiantum.c +++ b/crypto/adiantum.c @@ -72,7 +72,7 @@ struct adiantum_tfm_ctx { struct crypto_skcipher *streamcipher; struct crypto_cipher *blockcipher; struct crypto_shash *hash; - struct poly1305_key header_hash_key; + struct poly1305_core_key header_hash_key; }; struct adiantum_request_ctx { @@ -249,7 +249,7 @@ static void adiantum_hash_header(struct skcipher_request *req) poly1305_core_blocks(&state, &tctx->header_hash_key, req->iv, TWEAK_SIZE / POLY1305_BLOCK_SIZE, 1); - poly1305_core_emit(&state, &rctx->header_hash); + poly1305_core_emit(&state, NULL, &rctx->header_hash); } /* Hash the left-hand part (the "bulk") of the message using NHPoly1305 */ diff --git a/crypto/nhpoly1305.c b/crypto/nhpoly1305.c index f6b6a52092b49..8a3006c3b51b9 100644 --- a/crypto/nhpoly1305.c +++ b/crypto/nhpoly1305.c @@ -210,7 +210,7 @@ int crypto_nhpoly1305_final_helper(struct shash_desc *desc, u8 *dst, nh_t nh_fn) if (state->nh_remaining) process_nh_hash_value(state, key); - poly1305_core_emit(&state->poly_state, dst); + poly1305_core_emit(&state->poly_state, NULL, dst); return 0; } EXPORT_SYMBOL(crypto_nhpoly1305_final_helper); diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c index 21edbd8c99fbc..94af47eb6fa69 100644 --- a/crypto/poly1305_generic.c +++ b/crypto/poly1305_generic.c @@ -31,6 +31,29 @@ static int crypto_poly1305_init(struct shash_desc *desc) return 0; } +static unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, + const u8 *src, unsigned int srclen) +{ + if (!dctx->sset) { + if (!dctx->rset && srclen >= POLY1305_BLOCK_SIZE) { + poly1305_core_setkey(&dctx->core_r, src); + src += POLY1305_BLOCK_SIZE; + srclen -= POLY1305_BLOCK_SIZE; + dctx->rset = 2; + } + if (srclen >= POLY1305_BLOCK_SIZE) { + dctx->s[0] = get_unaligned_le32(src + 0); + dctx->s[1] = get_unaligned_le32(src + 4); + dctx->s[2] = get_unaligned_le32(src + 8); + dctx->s[3] = get_unaligned_le32(src + 12); + src += POLY1305_BLOCK_SIZE; + srclen -= POLY1305_BLOCK_SIZE; + dctx->sset = true; + } + } + return srclen; +} + static void poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, unsigned int srclen) { @@ -42,7 +65,7 @@ static void poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, srclen = datalen; } - poly1305_core_blocks(&dctx->h, dctx->r, src, + poly1305_core_blocks(&dctx->h, &dctx->core_r, src, srclen / POLY1305_BLOCK_SIZE, 1); } diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig index e148bbd9e783e..537b1af285253 100644 --- a/drivers/net/Kconfig +++ b/drivers/net/Kconfig @@ -71,6 +71,49 @@ config DUMMY To compile this driver as a module, choose M here: the module will be called dummy. +config WIREGUARD + tristate "WireGuard secure network tunnel" + depends on NET && INET + depends on IPV6 || !IPV6 + select NET_UDP_TUNNEL + select DST_CACHE + select CRYPTO + select CRYPTO_LIB_CURVE25519 + select CRYPTO_LIB_CHACHA20POLY1305 + select CRYPTO_LIB_BLAKE2S + select CRYPTO_CHACHA20_X86_64 if X86 && 64BIT + select CRYPTO_POLY1305_X86_64 if X86 && 64BIT + select CRYPTO_BLAKE2S_X86 if X86 && 64BIT + select CRYPTO_CURVE25519_X86 if X86 && 64BIT + select ARM_CRYPTO if ARM + select ARM64_CRYPTO if ARM64 + select CRYPTO_CHACHA20_NEON if (ARM || ARM64) && KERNEL_MODE_NEON + select CRYPTO_POLY1305_NEON if ARM64 && KERNEL_MODE_NEON + select CRYPTO_POLY1305_ARM if ARM + select CRYPTO_CURVE25519_NEON if ARM && KERNEL_MODE_NEON + select CRYPTO_CHACHA_MIPS if CPU_MIPS32_R2 + select CRYPTO_POLY1305_MIPS if CPU_MIPS32 || (CPU_MIPS64 && 64BIT) + help + WireGuard is a secure, fast, and easy to use replacement for IPSec + that uses modern cryptography and clever networking tricks. It's + designed to be fairly general purpose and abstract enough to fit most + use cases, while at the same time remaining extremely simple to + configure. See www.wireguard.com for more info. + + It's safe to say Y or M here, as the driver is very lightweight and + is only in use when an administrator chooses to add an interface. + +config WIREGUARD_DEBUG + bool "Debugging checks and verbose messages" + depends on WIREGUARD + help + This will write log messages for handshake and other events + that occur for a WireGuard interface. It will also perform some + extra validation checks and unit tests at various points. This is + only useful for debugging. + + Say N here unless you know what you're doing. + config EQUALIZER tristate "EQL (serial line load balancing) support" ---help--- diff --git a/drivers/net/Makefile b/drivers/net/Makefile index 0d3ba056cda3c..953b7c12f0b0c 100644 --- a/drivers/net/Makefile +++ b/drivers/net/Makefile @@ -10,6 +10,7 @@ obj-$(CONFIG_BONDING) += bonding/ obj-$(CONFIG_IPVLAN) += ipvlan/ obj-$(CONFIG_IPVTAP) += ipvlan/ obj-$(CONFIG_DUMMY) += dummy.o +obj-$(CONFIG_WIREGUARD) += wireguard/ obj-$(CONFIG_EQUALIZER) += eql.o obj-$(CONFIG_IFB) += ifb.o obj-$(CONFIG_MACSEC) += macsec.o diff --git a/drivers/net/wireguard/Makefile b/drivers/net/wireguard/Makefile new file mode 100644 index 0000000000000..fc52b2cb500b3 --- /dev/null +++ b/drivers/net/wireguard/Makefile @@ -0,0 +1,18 @@ +ccflags-y := -O3 +ccflags-y += -D'pr_fmt(fmt)=KBUILD_MODNAME ": " fmt' +ccflags-$(CONFIG_WIREGUARD_DEBUG) += -DDEBUG +wireguard-y := main.o +wireguard-y += noise.o +wireguard-y += device.o +wireguard-y += peer.o +wireguard-y += timers.o +wireguard-y += queueing.o +wireguard-y += send.o +wireguard-y += receive.o +wireguard-y += socket.o +wireguard-y += peerlookup.o +wireguard-y += allowedips.o +wireguard-y += ratelimiter.o +wireguard-y += cookie.o +wireguard-y += netlink.o +obj-$(CONFIG_WIREGUARD) := wireguard.o diff --git a/drivers/net/wireguard/allowedips.c b/drivers/net/wireguard/allowedips.c new file mode 100644 index 0000000000000..3725e9cd85f4f --- /dev/null +++ b/drivers/net/wireguard/allowedips.c @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "allowedips.h" +#include "peer.h" + +static void swap_endian(u8 *dst, const u8 *src, u8 bits) +{ + if (bits == 32) { + *(u32 *)dst = be32_to_cpu(*(const __be32 *)src); + } else if (bits == 128) { + ((u64 *)dst)[0] = be64_to_cpu(((const __be64 *)src)[0]); + ((u64 *)dst)[1] = be64_to_cpu(((const __be64 *)src)[1]); + } +} + +static void copy_and_assign_cidr(struct allowedips_node *node, const u8 *src, + u8 cidr, u8 bits) +{ + node->cidr = cidr; + node->bit_at_a = cidr / 8U; +#ifdef __LITTLE_ENDIAN + node->bit_at_a ^= (bits / 8U - 1U) % 8U; +#endif + node->bit_at_b = 7U - (cidr % 8U); + node->bitlen = bits; + memcpy(node->bits, src, bits / 8U); +} +#define CHOOSE_NODE(parent, key) \ + parent->bit[(key[parent->bit_at_a] >> parent->bit_at_b) & 1] + +static void push_rcu(struct allowedips_node **stack, + struct allowedips_node __rcu *p, unsigned int *len) +{ + if (rcu_access_pointer(p)) { + WARN_ON(IS_ENABLED(DEBUG) && *len >= 128); + stack[(*len)++] = rcu_dereference_raw(p); + } +} + +static void root_free_rcu(struct rcu_head *rcu) +{ + struct allowedips_node *node, *stack[128] = { + container_of(rcu, struct allowedips_node, rcu) }; + unsigned int len = 1; + + while (len > 0 && (node = stack[--len])) { + push_rcu(stack, node->bit[0], &len); + push_rcu(stack, node->bit[1], &len); + kfree(node); + } +} + +static void root_remove_peer_lists(struct allowedips_node *root) +{ + struct allowedips_node *node, *stack[128] = { root }; + unsigned int len = 1; + + while (len > 0 && (node = stack[--len])) { + push_rcu(stack, node->bit[0], &len); + push_rcu(stack, node->bit[1], &len); + if (rcu_access_pointer(node->peer)) + list_del(&node->peer_list); + } +} + +static void walk_remove_by_peer(struct allowedips_node __rcu **top, + struct wg_peer *peer, struct mutex *lock) +{ +#define REF(p) rcu_access_pointer(p) +#define DEREF(p) rcu_dereference_protected(*(p), lockdep_is_held(lock)) +#define PUSH(p) ({ \ + WARN_ON(IS_ENABLED(DEBUG) && len >= 128); \ + stack[len++] = p; \ + }) + + struct allowedips_node __rcu **stack[128], **nptr; + struct allowedips_node *node, *prev; + unsigned int len; + + if (unlikely(!peer || !REF(*top))) + return; + + for (prev = NULL, len = 0, PUSH(top); len > 0; prev = node) { + nptr = stack[len - 1]; + node = DEREF(nptr); + if (!node) { + --len; + continue; + } + if (!prev || REF(prev->bit[0]) == node || + REF(prev->bit[1]) == node) { + if (REF(node->bit[0])) + PUSH(&node->bit[0]); + else if (REF(node->bit[1])) + PUSH(&node->bit[1]); + } else if (REF(node->bit[0]) == prev) { + if (REF(node->bit[1])) + PUSH(&node->bit[1]); + } else { + if (rcu_dereference_protected(node->peer, + lockdep_is_held(lock)) == peer) { + RCU_INIT_POINTER(node->peer, NULL); + list_del_init(&node->peer_list); + if (!node->bit[0] || !node->bit[1]) { + rcu_assign_pointer(*nptr, DEREF( + &node->bit[!REF(node->bit[0])])); + kfree_rcu(node, rcu); + node = DEREF(nptr); + } + } + --len; + } + } + +#undef REF +#undef DEREF +#undef PUSH +} + +static unsigned int fls128(u64 a, u64 b) +{ + return a ? fls64(a) + 64U : fls64(b); +} + +static u8 common_bits(const struct allowedips_node *node, const u8 *key, + u8 bits) +{ + if (bits == 32) + return 32U - fls(*(const u32 *)node->bits ^ *(const u32 *)key); + else if (bits == 128) + return 128U - fls128( + *(const u64 *)&node->bits[0] ^ *(const u64 *)&key[0], + *(const u64 *)&node->bits[8] ^ *(const u64 *)&key[8]); + return 0; +} + +static bool prefix_matches(const struct allowedips_node *node, const u8 *key, + u8 bits) +{ + /* This could be much faster if it actually just compared the common + * bits properly, by precomputing a mask bswap(~0 << (32 - cidr)), and + * the rest, but it turns out that common_bits is already super fast on + * modern processors, even taking into account the unfortunate bswap. + * So, we just inline it like this instead. + */ + return common_bits(node, key, bits) >= node->cidr; +} + +static struct allowedips_node *find_node(struct allowedips_node *trie, u8 bits, + const u8 *key) +{ + struct allowedips_node *node = trie, *found = NULL; + + while (node && prefix_matches(node, key, bits)) { + if (rcu_access_pointer(node->peer)) + found = node; + if (node->cidr == bits) + break; + node = rcu_dereference_bh(CHOOSE_NODE(node, key)); + } + return found; +} + +/* Returns a strong reference to a peer */ +static struct wg_peer *lookup(struct allowedips_node __rcu *root, u8 bits, + const void *be_ip) +{ + /* Aligned so it can be passed to fls/fls64 */ + u8 ip[16] __aligned(__alignof(u64)); + struct allowedips_node *node; + struct wg_peer *peer = NULL; + + swap_endian(ip, be_ip, bits); + + rcu_read_lock_bh(); +retry: + node = find_node(rcu_dereference_bh(root), bits, ip); + if (node) { + peer = wg_peer_get_maybe_zero(rcu_dereference_bh(node->peer)); + if (!peer) + goto retry; + } + rcu_read_unlock_bh(); + return peer; +} + +static bool node_placement(struct allowedips_node __rcu *trie, const u8 *key, + u8 cidr, u8 bits, struct allowedips_node **rnode, + struct mutex *lock) +{ + struct allowedips_node *node = rcu_dereference_protected(trie, + lockdep_is_held(lock)); + struct allowedips_node *parent = NULL; + bool exact = false; + + while (node && node->cidr <= cidr && prefix_matches(node, key, bits)) { + parent = node; + if (parent->cidr == cidr) { + exact = true; + break; + } + node = rcu_dereference_protected(CHOOSE_NODE(parent, key), + lockdep_is_held(lock)); + } + *rnode = parent; + return exact; +} + +static int add(struct allowedips_node __rcu **trie, u8 bits, const u8 *key, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + struct allowedips_node *node, *parent, *down, *newnode; + + if (unlikely(cidr > bits || !peer)) + return -EINVAL; + + if (!rcu_access_pointer(*trie)) { + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (unlikely(!node)) + return -ENOMEM; + RCU_INIT_POINTER(node->peer, peer); + list_add_tail(&node->peer_list, &peer->allowedips_list); + copy_and_assign_cidr(node, key, cidr, bits); + rcu_assign_pointer(*trie, node); + return 0; + } + if (node_placement(*trie, key, cidr, bits, &node, lock)) { + rcu_assign_pointer(node->peer, peer); + list_move_tail(&node->peer_list, &peer->allowedips_list); + return 0; + } + + newnode = kzalloc(sizeof(*newnode), GFP_KERNEL); + if (unlikely(!newnode)) + return -ENOMEM; + RCU_INIT_POINTER(newnode->peer, peer); + list_add_tail(&newnode->peer_list, &peer->allowedips_list); + copy_and_assign_cidr(newnode, key, cidr, bits); + + if (!node) { + down = rcu_dereference_protected(*trie, lockdep_is_held(lock)); + } else { + down = rcu_dereference_protected(CHOOSE_NODE(node, key), + lockdep_is_held(lock)); + if (!down) { + rcu_assign_pointer(CHOOSE_NODE(node, key), newnode); + return 0; + } + } + cidr = min(cidr, common_bits(down, key, bits)); + parent = node; + + if (newnode->cidr == cidr) { + rcu_assign_pointer(CHOOSE_NODE(newnode, down->bits), down); + if (!parent) + rcu_assign_pointer(*trie, newnode); + else + rcu_assign_pointer(CHOOSE_NODE(parent, newnode->bits), + newnode); + } else { + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (unlikely(!node)) { + list_del(&newnode->peer_list); + kfree(newnode); + return -ENOMEM; + } + INIT_LIST_HEAD(&node->peer_list); + copy_and_assign_cidr(node, newnode->bits, cidr, bits); + + rcu_assign_pointer(CHOOSE_NODE(node, down->bits), down); + rcu_assign_pointer(CHOOSE_NODE(node, newnode->bits), newnode); + if (!parent) + rcu_assign_pointer(*trie, node); + else + rcu_assign_pointer(CHOOSE_NODE(parent, node->bits), + node); + } + return 0; +} + +void wg_allowedips_init(struct allowedips *table) +{ + table->root4 = table->root6 = NULL; + table->seq = 1; +} + +void wg_allowedips_free(struct allowedips *table, struct mutex *lock) +{ + struct allowedips_node __rcu *old4 = table->root4, *old6 = table->root6; + + ++table->seq; + RCU_INIT_POINTER(table->root4, NULL); + RCU_INIT_POINTER(table->root6, NULL); + if (rcu_access_pointer(old4)) { + struct allowedips_node *node = rcu_dereference_protected(old4, + lockdep_is_held(lock)); + + root_remove_peer_lists(node); + call_rcu(&node->rcu, root_free_rcu); + } + if (rcu_access_pointer(old6)) { + struct allowedips_node *node = rcu_dereference_protected(old6, + lockdep_is_held(lock)); + + root_remove_peer_lists(node); + call_rcu(&node->rcu, root_free_rcu); + } +} + +int wg_allowedips_insert_v4(struct allowedips *table, const struct in_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + /* Aligned so it can be passed to fls */ + u8 key[4] __aligned(__alignof(u32)); + + ++table->seq; + swap_endian(key, (const u8 *)ip, 32); + return add(&table->root4, 32, key, cidr, peer, lock); +} + +int wg_allowedips_insert_v6(struct allowedips *table, const struct in6_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock) +{ + /* Aligned so it can be passed to fls64 */ + u8 key[16] __aligned(__alignof(u64)); + + ++table->seq; + swap_endian(key, (const u8 *)ip, 128); + return add(&table->root6, 128, key, cidr, peer, lock); +} + +void wg_allowedips_remove_by_peer(struct allowedips *table, + struct wg_peer *peer, struct mutex *lock) +{ + ++table->seq; + walk_remove_by_peer(&table->root4, peer, lock); + walk_remove_by_peer(&table->root6, peer, lock); +} + +int wg_allowedips_read_node(struct allowedips_node *node, u8 ip[16], u8 *cidr) +{ + const unsigned int cidr_bytes = DIV_ROUND_UP(node->cidr, 8U); + swap_endian(ip, node->bits, node->bitlen); + memset(ip + cidr_bytes, 0, node->bitlen / 8U - cidr_bytes); + if (node->cidr) + ip[cidr_bytes - 1U] &= ~0U << (-node->cidr % 8U); + + *cidr = node->cidr; + return node->bitlen == 32 ? AF_INET : AF_INET6; +} + +/* Returns a strong reference to a peer */ +struct wg_peer *wg_allowedips_lookup_dst(struct allowedips *table, + struct sk_buff *skb) +{ + if (skb->protocol == htons(ETH_P_IP)) + return lookup(table->root4, 32, &ip_hdr(skb)->daddr); + else if (skb->protocol == htons(ETH_P_IPV6)) + return lookup(table->root6, 128, &ipv6_hdr(skb)->daddr); + return NULL; +} + +/* Returns a strong reference to a peer */ +struct wg_peer *wg_allowedips_lookup_src(struct allowedips *table, + struct sk_buff *skb) +{ + if (skb->protocol == htons(ETH_P_IP)) + return lookup(table->root4, 32, &ip_hdr(skb)->saddr); + else if (skb->protocol == htons(ETH_P_IPV6)) + return lookup(table->root6, 128, &ipv6_hdr(skb)->saddr); + return NULL; +} + +#include "selftest/allowedips.c" diff --git a/drivers/net/wireguard/allowedips.h b/drivers/net/wireguard/allowedips.h new file mode 100644 index 0000000000000..e5c83cafcef4c --- /dev/null +++ b/drivers/net/wireguard/allowedips.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_ALLOWEDIPS_H +#define _WG_ALLOWEDIPS_H + +#include +#include +#include + +struct wg_peer; + +struct allowedips_node { + struct wg_peer __rcu *peer; + struct allowedips_node __rcu *bit[2]; + /* While it may seem scandalous that we waste space for v4, + * we're alloc'ing to the nearest power of 2 anyway, so this + * doesn't actually make a difference. + */ + u8 bits[16] __aligned(__alignof(u64)); + u8 cidr, bit_at_a, bit_at_b, bitlen; + + /* Keep rarely used list at bottom to be beyond cache line. */ + union { + struct list_head peer_list; + struct rcu_head rcu; + }; +}; + +struct allowedips { + struct allowedips_node __rcu *root4; + struct allowedips_node __rcu *root6; + u64 seq; +}; + +void wg_allowedips_init(struct allowedips *table); +void wg_allowedips_free(struct allowedips *table, struct mutex *mutex); +int wg_allowedips_insert_v4(struct allowedips *table, const struct in_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock); +int wg_allowedips_insert_v6(struct allowedips *table, const struct in6_addr *ip, + u8 cidr, struct wg_peer *peer, struct mutex *lock); +void wg_allowedips_remove_by_peer(struct allowedips *table, + struct wg_peer *peer, struct mutex *lock); +/* The ip input pointer should be __aligned(__alignof(u64))) */ +int wg_allowedips_read_node(struct allowedips_node *node, u8 ip[16], u8 *cidr); + +/* These return a strong reference to a peer: */ +struct wg_peer *wg_allowedips_lookup_dst(struct allowedips *table, + struct sk_buff *skb); +struct wg_peer *wg_allowedips_lookup_src(struct allowedips *table, + struct sk_buff *skb); + +#ifdef DEBUG +bool wg_allowedips_selftest(void); +#endif + +#endif /* _WG_ALLOWEDIPS_H */ diff --git a/drivers/net/wireguard/cookie.c b/drivers/net/wireguard/cookie.c new file mode 100644 index 0000000000000..4956f0499c198 --- /dev/null +++ b/drivers/net/wireguard/cookie.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "cookie.h" +#include "peer.h" +#include "device.h" +#include "messages.h" +#include "ratelimiter.h" +#include "timers.h" + +#include +#include + +#include +#include + +void wg_cookie_checker_init(struct cookie_checker *checker, + struct wg_device *wg) +{ + init_rwsem(&checker->secret_lock); + checker->secret_birthdate = ktime_get_coarse_boottime_ns(); + get_random_bytes(checker->secret, NOISE_HASH_LEN); + checker->device = wg; +} + +enum { COOKIE_KEY_LABEL_LEN = 8 }; +static const u8 mac1_key_label[COOKIE_KEY_LABEL_LEN] = "mac1----"; +static const u8 cookie_key_label[COOKIE_KEY_LABEL_LEN] = "cookie--"; + +static void precompute_key(u8 key[NOISE_SYMMETRIC_KEY_LEN], + const u8 pubkey[NOISE_PUBLIC_KEY_LEN], + const u8 label[COOKIE_KEY_LABEL_LEN]) +{ + struct blake2s_state blake; + + blake2s_init(&blake, NOISE_SYMMETRIC_KEY_LEN); + blake2s_update(&blake, label, COOKIE_KEY_LABEL_LEN); + blake2s_update(&blake, pubkey, NOISE_PUBLIC_KEY_LEN); + blake2s_final(&blake, key); +} + +/* Must hold peer->handshake.static_identity->lock */ +void wg_cookie_checker_precompute_device_keys(struct cookie_checker *checker) +{ + if (likely(checker->device->static_identity.has_identity)) { + precompute_key(checker->cookie_encryption_key, + checker->device->static_identity.static_public, + cookie_key_label); + precompute_key(checker->message_mac1_key, + checker->device->static_identity.static_public, + mac1_key_label); + } else { + memset(checker->cookie_encryption_key, 0, + NOISE_SYMMETRIC_KEY_LEN); + memset(checker->message_mac1_key, 0, NOISE_SYMMETRIC_KEY_LEN); + } +} + +void wg_cookie_checker_precompute_peer_keys(struct wg_peer *peer) +{ + precompute_key(peer->latest_cookie.cookie_decryption_key, + peer->handshake.remote_static, cookie_key_label); + precompute_key(peer->latest_cookie.message_mac1_key, + peer->handshake.remote_static, mac1_key_label); +} + +void wg_cookie_init(struct cookie *cookie) +{ + memset(cookie, 0, sizeof(*cookie)); + init_rwsem(&cookie->lock); +} + +static void compute_mac1(u8 mac1[COOKIE_LEN], const void *message, size_t len, + const u8 key[NOISE_SYMMETRIC_KEY_LEN]) +{ + len = len - sizeof(struct message_macs) + + offsetof(struct message_macs, mac1); + blake2s(mac1, message, key, COOKIE_LEN, len, NOISE_SYMMETRIC_KEY_LEN); +} + +static void compute_mac2(u8 mac2[COOKIE_LEN], const void *message, size_t len, + const u8 cookie[COOKIE_LEN]) +{ + len = len - sizeof(struct message_macs) + + offsetof(struct message_macs, mac2); + blake2s(mac2, message, cookie, COOKIE_LEN, len, COOKIE_LEN); +} + +static void make_cookie(u8 cookie[COOKIE_LEN], struct sk_buff *skb, + struct cookie_checker *checker) +{ + struct blake2s_state state; + + if (wg_birthdate_has_expired(checker->secret_birthdate, + COOKIE_SECRET_MAX_AGE)) { + down_write(&checker->secret_lock); + checker->secret_birthdate = ktime_get_coarse_boottime_ns(); + get_random_bytes(checker->secret, NOISE_HASH_LEN); + up_write(&checker->secret_lock); + } + + down_read(&checker->secret_lock); + + blake2s_init_key(&state, COOKIE_LEN, checker->secret, NOISE_HASH_LEN); + if (skb->protocol == htons(ETH_P_IP)) + blake2s_update(&state, (u8 *)&ip_hdr(skb)->saddr, + sizeof(struct in_addr)); + else if (skb->protocol == htons(ETH_P_IPV6)) + blake2s_update(&state, (u8 *)&ipv6_hdr(skb)->saddr, + sizeof(struct in6_addr)); + blake2s_update(&state, (u8 *)&udp_hdr(skb)->source, sizeof(__be16)); + blake2s_final(&state, cookie); + + up_read(&checker->secret_lock); +} + +enum cookie_mac_state wg_cookie_validate_packet(struct cookie_checker *checker, + struct sk_buff *skb, + bool check_cookie) +{ + struct message_macs *macs = (struct message_macs *) + (skb->data + skb->len - sizeof(*macs)); + enum cookie_mac_state ret; + u8 computed_mac[COOKIE_LEN]; + u8 cookie[COOKIE_LEN]; + + ret = INVALID_MAC; + compute_mac1(computed_mac, skb->data, skb->len, + checker->message_mac1_key); + if (crypto_memneq(computed_mac, macs->mac1, COOKIE_LEN)) + goto out; + + ret = VALID_MAC_BUT_NO_COOKIE; + + if (!check_cookie) + goto out; + + make_cookie(cookie, skb, checker); + + compute_mac2(computed_mac, skb->data, skb->len, cookie); + if (crypto_memneq(computed_mac, macs->mac2, COOKIE_LEN)) + goto out; + + ret = VALID_MAC_WITH_COOKIE_BUT_RATELIMITED; + if (!wg_ratelimiter_allow(skb, dev_net(checker->device->dev))) + goto out; + + ret = VALID_MAC_WITH_COOKIE; + +out: + return ret; +} + +void wg_cookie_add_mac_to_packet(void *message, size_t len, + struct wg_peer *peer) +{ + struct message_macs *macs = (struct message_macs *) + ((u8 *)message + len - sizeof(*macs)); + + down_write(&peer->latest_cookie.lock); + compute_mac1(macs->mac1, message, len, + peer->latest_cookie.message_mac1_key); + memcpy(peer->latest_cookie.last_mac1_sent, macs->mac1, COOKIE_LEN); + peer->latest_cookie.have_sent_mac1 = true; + up_write(&peer->latest_cookie.lock); + + down_read(&peer->latest_cookie.lock); + if (peer->latest_cookie.is_valid && + !wg_birthdate_has_expired(peer->latest_cookie.birthdate, + COOKIE_SECRET_MAX_AGE - COOKIE_SECRET_LATENCY)) + compute_mac2(macs->mac2, message, len, + peer->latest_cookie.cookie); + else + memset(macs->mac2, 0, COOKIE_LEN); + up_read(&peer->latest_cookie.lock); +} + +void wg_cookie_message_create(struct message_handshake_cookie *dst, + struct sk_buff *skb, __le32 index, + struct cookie_checker *checker) +{ + struct message_macs *macs = (struct message_macs *) + ((u8 *)skb->data + skb->len - sizeof(*macs)); + u8 cookie[COOKIE_LEN]; + + dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE); + dst->receiver_index = index; + get_random_bytes_wait(dst->nonce, COOKIE_NONCE_LEN); + + make_cookie(cookie, skb, checker); + xchacha20poly1305_encrypt(dst->encrypted_cookie, cookie, COOKIE_LEN, + macs->mac1, COOKIE_LEN, dst->nonce, + checker->cookie_encryption_key); +} + +void wg_cookie_message_consume(struct message_handshake_cookie *src, + struct wg_device *wg) +{ + struct wg_peer *peer = NULL; + u8 cookie[COOKIE_LEN]; + bool ret; + + if (unlikely(!wg_index_hashtable_lookup(wg->index_hashtable, + INDEX_HASHTABLE_HANDSHAKE | + INDEX_HASHTABLE_KEYPAIR, + src->receiver_index, &peer))) + return; + + down_read(&peer->latest_cookie.lock); + if (unlikely(!peer->latest_cookie.have_sent_mac1)) { + up_read(&peer->latest_cookie.lock); + goto out; + } + ret = xchacha20poly1305_decrypt( + cookie, src->encrypted_cookie, sizeof(src->encrypted_cookie), + peer->latest_cookie.last_mac1_sent, COOKIE_LEN, src->nonce, + peer->latest_cookie.cookie_decryption_key); + up_read(&peer->latest_cookie.lock); + + if (ret) { + down_write(&peer->latest_cookie.lock); + memcpy(peer->latest_cookie.cookie, cookie, COOKIE_LEN); + peer->latest_cookie.birthdate = ktime_get_coarse_boottime_ns(); + peer->latest_cookie.is_valid = true; + peer->latest_cookie.have_sent_mac1 = false; + up_write(&peer->latest_cookie.lock); + } else { + net_dbg_ratelimited("%s: Could not decrypt invalid cookie response\n", + wg->dev->name); + } + +out: + wg_peer_put(peer); +} diff --git a/drivers/net/wireguard/cookie.h b/drivers/net/wireguard/cookie.h new file mode 100644 index 0000000000000..c4bd61ca03f24 --- /dev/null +++ b/drivers/net/wireguard/cookie.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_COOKIE_H +#define _WG_COOKIE_H + +#include "messages.h" +#include + +struct wg_peer; + +struct cookie_checker { + u8 secret[NOISE_HASH_LEN]; + u8 cookie_encryption_key[NOISE_SYMMETRIC_KEY_LEN]; + u8 message_mac1_key[NOISE_SYMMETRIC_KEY_LEN]; + u64 secret_birthdate; + struct rw_semaphore secret_lock; + struct wg_device *device; +}; + +struct cookie { + u64 birthdate; + bool is_valid; + u8 cookie[COOKIE_LEN]; + bool have_sent_mac1; + u8 last_mac1_sent[COOKIE_LEN]; + u8 cookie_decryption_key[NOISE_SYMMETRIC_KEY_LEN]; + u8 message_mac1_key[NOISE_SYMMETRIC_KEY_LEN]; + struct rw_semaphore lock; +}; + +enum cookie_mac_state { + INVALID_MAC, + VALID_MAC_BUT_NO_COOKIE, + VALID_MAC_WITH_COOKIE_BUT_RATELIMITED, + VALID_MAC_WITH_COOKIE +}; + +void wg_cookie_checker_init(struct cookie_checker *checker, + struct wg_device *wg); +void wg_cookie_checker_precompute_device_keys(struct cookie_checker *checker); +void wg_cookie_checker_precompute_peer_keys(struct wg_peer *peer); +void wg_cookie_init(struct cookie *cookie); + +enum cookie_mac_state wg_cookie_validate_packet(struct cookie_checker *checker, + struct sk_buff *skb, + bool check_cookie); +void wg_cookie_add_mac_to_packet(void *message, size_t len, + struct wg_peer *peer); + +void wg_cookie_message_create(struct message_handshake_cookie *src, + struct sk_buff *skb, __le32 index, + struct cookie_checker *checker); +void wg_cookie_message_consume(struct message_handshake_cookie *src, + struct wg_device *wg); + +#endif /* _WG_COOKIE_H */ diff --git a/drivers/net/wireguard/device.c b/drivers/net/wireguard/device.c new file mode 100644 index 0000000000000..3ac3f8570ca1b --- /dev/null +++ b/drivers/net/wireguard/device.c @@ -0,0 +1,459 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" +#include "socket.h" +#include "timers.h" +#include "device.h" +#include "ratelimiter.h" +#include "peer.h" +#include "messages.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static LIST_HEAD(device_list); + +static int wg_open(struct net_device *dev) +{ + struct in_device *dev_v4 = __in_dev_get_rtnl(dev); + struct inet6_dev *dev_v6 = __in6_dev_get(dev); + struct wg_device *wg = netdev_priv(dev); + struct wg_peer *peer; + int ret; + + if (dev_v4) { + /* At some point we might put this check near the ip_rt_send_ + * redirect call of ip_forward in net/ipv4/ip_forward.c, similar + * to the current secpath check. + */ + IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); + IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; + } + if (dev_v6) + dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE; + + ret = wg_socket_init(wg, wg->incoming_port); + if (ret < 0) + return ret; + mutex_lock(&wg->device_update_lock); + list_for_each_entry(peer, &wg->peer_list, peer_list) { + wg_packet_send_staged_packets(peer); + if (peer->persistent_keepalive_interval) + wg_packet_send_keepalive(peer); + } + mutex_unlock(&wg->device_update_lock); + return 0; +} + +#ifdef CONFIG_PM_SLEEP +static int wg_pm_notification(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct wg_device *wg; + struct wg_peer *peer; + + /* If the machine is constantly suspending and resuming, as part of + * its normal operation rather than as a somewhat rare event, then we + * don't actually want to clear keys. + */ + if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) || IS_ENABLED(CONFIG_ANDROID)) + return 0; + + if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE) + return 0; + + rtnl_lock(); + list_for_each_entry(wg, &device_list, device_list) { + mutex_lock(&wg->device_update_lock); + list_for_each_entry(peer, &wg->peer_list, peer_list) { + del_timer(&peer->timer_zero_key_material); + wg_noise_handshake_clear(&peer->handshake); + wg_noise_keypairs_clear(&peer->keypairs); + } + mutex_unlock(&wg->device_update_lock); + } + rtnl_unlock(); + rcu_barrier(); + return 0; +} + +static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification }; +#endif + +static int wg_stop(struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + struct wg_peer *peer; + + mutex_lock(&wg->device_update_lock); + list_for_each_entry(peer, &wg->peer_list, peer_list) { + wg_packet_purge_staged_packets(peer); + wg_timers_stop(peer); + wg_noise_handshake_clear(&peer->handshake); + wg_noise_keypairs_clear(&peer->keypairs); + wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); + } + mutex_unlock(&wg->device_update_lock); + skb_queue_purge(&wg->incoming_handshakes); + wg_socket_reinit(wg, NULL, NULL); + return 0; +} + +static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + struct sk_buff_head packets; + struct wg_peer *peer; + struct sk_buff *next; + sa_family_t family; + u32 mtu; + int ret; + + if (unlikely(!wg_check_packet_protocol(skb))) { + ret = -EPROTONOSUPPORT; + net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name); + goto err; + } + + peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb); + if (unlikely(!peer)) { + ret = -ENOKEY; + if (skb->protocol == htons(ETH_P_IP)) + net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n", + dev->name, &ip_hdr(skb)->daddr); + else if (skb->protocol == htons(ETH_P_IPV6)) + net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n", + dev->name, &ipv6_hdr(skb)->daddr); + goto err; + } + + family = READ_ONCE(peer->endpoint.addr.sa_family); + if (unlikely(family != AF_INET && family != AF_INET6)) { + ret = -EDESTADDRREQ; + net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n", + dev->name, peer->internal_id); + goto err_peer; + } + + mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu; + + __skb_queue_head_init(&packets); + if (!skb_is_gso(skb)) { + skb_mark_not_on_list(skb); + } else { + struct sk_buff *segs = skb_gso_segment(skb, 0); + + if (unlikely(IS_ERR(segs))) { + ret = PTR_ERR(segs); + goto err_peer; + } + dev_kfree_skb(skb); + skb = segs; + } + + skb_list_walk_safe(skb, skb, next) { + skb_mark_not_on_list(skb); + + skb = skb_share_check(skb, GFP_ATOMIC); + if (unlikely(!skb)) + continue; + + /* We only need to keep the original dst around for icmp, + * so at this point we're in a position to drop it. + */ + skb_dst_drop(skb); + + PACKET_CB(skb)->mtu = mtu; + + __skb_queue_tail(&packets, skb); + } + + spin_lock_bh(&peer->staged_packet_queue.lock); + /* If the queue is getting too big, we start removing the oldest packets + * until it's small again. We do this before adding the new packet, so + * we don't remove GSO segments that are in excess. + */ + while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) { + dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue)); + ++dev->stats.tx_dropped; + } + skb_queue_splice_tail(&packets, &peer->staged_packet_queue); + spin_unlock_bh(&peer->staged_packet_queue.lock); + + wg_packet_send_staged_packets(peer); + + wg_peer_put(peer); + return NETDEV_TX_OK; + +err_peer: + wg_peer_put(peer); +err: + ++dev->stats.tx_errors; + if (skb->protocol == htons(ETH_P_IP)) + icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0); + else if (skb->protocol == htons(ETH_P_IPV6)) + icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0); + kfree_skb(skb); + return ret; +} + +static const struct net_device_ops netdev_ops = { + .ndo_open = wg_open, + .ndo_stop = wg_stop, + .ndo_start_xmit = wg_xmit, + .ndo_get_stats64 = ip_tunnel_get_stats64 +}; + +static void wg_destruct(struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + + rtnl_lock(); + list_del(&wg->device_list); + rtnl_unlock(); + mutex_lock(&wg->device_update_lock); + wg->incoming_port = 0; + wg_socket_reinit(wg, NULL, NULL); + /* The final references are cleared in the below calls to destroy_workqueue. */ + wg_peer_remove_all(wg); + destroy_workqueue(wg->handshake_receive_wq); + destroy_workqueue(wg->handshake_send_wq); + destroy_workqueue(wg->packet_crypt_wq); + wg_packet_queue_free(&wg->decrypt_queue, true); + wg_packet_queue_free(&wg->encrypt_queue, true); + rcu_barrier(); /* Wait for all the peers to be actually freed. */ + wg_ratelimiter_uninit(); + memzero_explicit(&wg->static_identity, sizeof(wg->static_identity)); + skb_queue_purge(&wg->incoming_handshakes); + free_percpu(dev->tstats); + free_percpu(wg->incoming_handshakes_worker); + if (wg->have_creating_net_ref) + put_net(wg->creating_net); + kvfree(wg->index_hashtable); + kvfree(wg->peer_hashtable); + mutex_unlock(&wg->device_update_lock); + + pr_debug("%s: Interface deleted\n", dev->name); + free_netdev(dev); +} + +static const struct device_type device_type = { .name = KBUILD_MODNAME }; + +static void wg_setup(struct net_device *dev) +{ + struct wg_device *wg = netdev_priv(dev); + enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM | + NETIF_F_SG | NETIF_F_GSO | + NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA }; + const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) + + max(sizeof(struct ipv6hdr), sizeof(struct iphdr)); + + dev->netdev_ops = &netdev_ops; + dev->hard_header_len = 0; + dev->addr_len = 0; + dev->needed_headroom = DATA_PACKET_HEAD_ROOM; + dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE); + dev->type = ARPHRD_NONE; + dev->flags = IFF_POINTOPOINT | IFF_NOARP; + dev->priv_flags |= IFF_NO_QUEUE; + dev->features |= NETIF_F_LLTX; + dev->features |= WG_NETDEV_FEATURES; + dev->hw_features |= WG_NETDEV_FEATURES; + dev->hw_enc_features |= WG_NETDEV_FEATURES; + dev->mtu = ETH_DATA_LEN - overhead; + dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead; + + SET_NETDEV_DEVTYPE(dev, &device_type); + + /* We need to keep the dst around in case of icmp replies. */ + netif_keep_dst(dev); + + memset(wg, 0, sizeof(*wg)); + wg->dev = dev; +} + +static int wg_newlink(struct net *src_net, struct net_device *dev, + struct nlattr *tb[], struct nlattr *data[], + struct netlink_ext_ack *extack) +{ + struct wg_device *wg = netdev_priv(dev); + int ret = -ENOMEM; + + wg->creating_net = src_net; + init_rwsem(&wg->static_identity.lock); + mutex_init(&wg->socket_update_lock); + mutex_init(&wg->device_update_lock); + skb_queue_head_init(&wg->incoming_handshakes); + wg_allowedips_init(&wg->peer_allowedips); + wg_cookie_checker_init(&wg->cookie_checker, wg); + INIT_LIST_HEAD(&wg->peer_list); + wg->device_update_gen = 1; + + wg->peer_hashtable = wg_pubkey_hashtable_alloc(); + if (!wg->peer_hashtable) + return ret; + + wg->index_hashtable = wg_index_hashtable_alloc(); + if (!wg->index_hashtable) + goto err_free_peer_hashtable; + + dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); + if (!dev->tstats) + goto err_free_index_hashtable; + + wg->incoming_handshakes_worker = + wg_packet_percpu_multicore_worker_alloc( + wg_packet_handshake_receive_worker, wg); + if (!wg->incoming_handshakes_worker) + goto err_free_tstats; + + wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s", + WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name); + if (!wg->handshake_receive_wq) + goto err_free_incoming_handshakes; + + wg->handshake_send_wq = alloc_workqueue("wg-kex-%s", + WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name); + if (!wg->handshake_send_wq) + goto err_destroy_handshake_receive; + + wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s", + WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name); + if (!wg->packet_crypt_wq) + goto err_destroy_handshake_send; + + ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker, + true, MAX_QUEUED_PACKETS); + if (ret < 0) + goto err_destroy_packet_crypt; + + ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker, + true, MAX_QUEUED_PACKETS); + if (ret < 0) + goto err_free_encrypt_queue; + + ret = wg_ratelimiter_init(); + if (ret < 0) + goto err_free_decrypt_queue; + + ret = register_netdevice(dev); + if (ret < 0) + goto err_uninit_ratelimiter; + + list_add(&wg->device_list, &device_list); + + /* We wait until the end to assign priv_destructor, so that + * register_netdevice doesn't call it for us if it fails. + */ + dev->priv_destructor = wg_destruct; + + pr_debug("%s: Interface created\n", dev->name); + return ret; + +err_uninit_ratelimiter: + wg_ratelimiter_uninit(); +err_free_decrypt_queue: + wg_packet_queue_free(&wg->decrypt_queue, true); +err_free_encrypt_queue: + wg_packet_queue_free(&wg->encrypt_queue, true); +err_destroy_packet_crypt: + destroy_workqueue(wg->packet_crypt_wq); +err_destroy_handshake_send: + destroy_workqueue(wg->handshake_send_wq); +err_destroy_handshake_receive: + destroy_workqueue(wg->handshake_receive_wq); +err_free_incoming_handshakes: + free_percpu(wg->incoming_handshakes_worker); +err_free_tstats: + free_percpu(dev->tstats); +err_free_index_hashtable: + kvfree(wg->index_hashtable); +err_free_peer_hashtable: + kvfree(wg->peer_hashtable); + return ret; +} + +static struct rtnl_link_ops link_ops __read_mostly = { + .kind = KBUILD_MODNAME, + .priv_size = sizeof(struct wg_device), + .setup = wg_setup, + .newlink = wg_newlink, +}; + +static int wg_netdevice_notification(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct net_device *dev = ((struct netdev_notifier_info *)data)->dev; + struct wg_device *wg = netdev_priv(dev); + + ASSERT_RTNL(); + + if (action != NETDEV_REGISTER || dev->netdev_ops != &netdev_ops) + return 0; + + if (dev_net(dev) == wg->creating_net && wg->have_creating_net_ref) { + put_net(wg->creating_net); + wg->have_creating_net_ref = false; + } else if (dev_net(dev) != wg->creating_net && + !wg->have_creating_net_ref) { + wg->have_creating_net_ref = true; + get_net(wg->creating_net); + } + return 0; +} + +static struct notifier_block netdevice_notifier = { + .notifier_call = wg_netdevice_notification +}; + +int __init wg_device_init(void) +{ + int ret; + +#ifdef CONFIG_PM_SLEEP + ret = register_pm_notifier(&pm_notifier); + if (ret) + return ret; +#endif + + ret = register_netdevice_notifier(&netdevice_notifier); + if (ret) + goto error_pm; + + ret = rtnl_link_register(&link_ops); + if (ret) + goto error_netdevice; + + return 0; + +error_netdevice: + unregister_netdevice_notifier(&netdevice_notifier); +error_pm: +#ifdef CONFIG_PM_SLEEP + unregister_pm_notifier(&pm_notifier); +#endif + return ret; +} + +void wg_device_uninit(void) +{ + rtnl_link_unregister(&link_ops); + unregister_netdevice_notifier(&netdevice_notifier); +#ifdef CONFIG_PM_SLEEP + unregister_pm_notifier(&pm_notifier); +#endif + rcu_barrier(); +} diff --git a/drivers/net/wireguard/device.h b/drivers/net/wireguard/device.h new file mode 100644 index 0000000000000..b15a8be9d8169 --- /dev/null +++ b/drivers/net/wireguard/device.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_DEVICE_H +#define _WG_DEVICE_H + +#include "noise.h" +#include "allowedips.h" +#include "peerlookup.h" +#include "cookie.h" + +#include +#include +#include +#include +#include +#include + +struct wg_device; + +struct multicore_worker { + void *ptr; + struct work_struct work; +}; + +struct crypt_queue { + struct ptr_ring ring; + union { + struct { + struct multicore_worker __percpu *worker; + int last_cpu; + }; + struct work_struct work; + }; +}; + +struct wg_device { + struct net_device *dev; + struct crypt_queue encrypt_queue, decrypt_queue; + struct sock __rcu *sock4, *sock6; + struct net *creating_net; + struct noise_static_identity static_identity; + struct workqueue_struct *handshake_receive_wq, *handshake_send_wq; + struct workqueue_struct *packet_crypt_wq; + struct sk_buff_head incoming_handshakes; + int incoming_handshake_cpu; + struct multicore_worker __percpu *incoming_handshakes_worker; + struct cookie_checker cookie_checker; + struct pubkey_hashtable *peer_hashtable; + struct index_hashtable *index_hashtable; + struct allowedips peer_allowedips; + struct mutex device_update_lock, socket_update_lock; + struct list_head device_list, peer_list; + unsigned int num_peers, device_update_gen; + u32 fwmark; + u16 incoming_port; + bool have_creating_net_ref; +}; + +int wg_device_init(void); +void wg_device_uninit(void); + +#endif /* _WG_DEVICE_H */ diff --git a/drivers/net/wireguard/main.c b/drivers/net/wireguard/main.c new file mode 100644 index 0000000000000..7a7d5f1a80fc7 --- /dev/null +++ b/drivers/net/wireguard/main.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "version.h" +#include "device.h" +#include "noise.h" +#include "queueing.h" +#include "ratelimiter.h" +#include "netlink.h" + +#include + +#include +#include +#include +#include + +static int __init mod_init(void) +{ + int ret; + +#ifdef DEBUG + if (!wg_allowedips_selftest() || !wg_packet_counter_selftest() || + !wg_ratelimiter_selftest()) + return -ENOTRECOVERABLE; +#endif + wg_noise_init(); + + ret = wg_device_init(); + if (ret < 0) + goto err_device; + + ret = wg_genetlink_init(); + if (ret < 0) + goto err_netlink; + + pr_info("WireGuard " WIREGUARD_VERSION " loaded. See www.wireguard.com for information.\n"); + pr_info("Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved.\n"); + + return 0; + +err_netlink: + wg_device_uninit(); +err_device: + return ret; +} + +static void __exit mod_exit(void) +{ + wg_genetlink_uninit(); + wg_device_uninit(); +} + +module_init(mod_init); +module_exit(mod_exit); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("WireGuard secure network tunnel"); +MODULE_AUTHOR("Jason A. Donenfeld "); +MODULE_VERSION(WIREGUARD_VERSION); +MODULE_ALIAS_RTNL_LINK(KBUILD_MODNAME); +MODULE_ALIAS_GENL_FAMILY(WG_GENL_NAME); diff --git a/drivers/net/wireguard/messages.h b/drivers/net/wireguard/messages.h new file mode 100644 index 0000000000000..b8a7b9ce32ba7 --- /dev/null +++ b/drivers/net/wireguard/messages.h @@ -0,0 +1,128 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_MESSAGES_H +#define _WG_MESSAGES_H + +#include +#include +#include + +#include +#include +#include + +enum noise_lengths { + NOISE_PUBLIC_KEY_LEN = CURVE25519_KEY_SIZE, + NOISE_SYMMETRIC_KEY_LEN = CHACHA20POLY1305_KEY_SIZE, + NOISE_TIMESTAMP_LEN = sizeof(u64) + sizeof(u32), + NOISE_AUTHTAG_LEN = CHACHA20POLY1305_AUTHTAG_SIZE, + NOISE_HASH_LEN = BLAKE2S_HASH_SIZE +}; + +#define noise_encrypted_len(plain_len) ((plain_len) + NOISE_AUTHTAG_LEN) + +enum cookie_values { + COOKIE_SECRET_MAX_AGE = 2 * 60, + COOKIE_SECRET_LATENCY = 5, + COOKIE_NONCE_LEN = XCHACHA20POLY1305_NONCE_SIZE, + COOKIE_LEN = 16 +}; + +enum counter_values { + COUNTER_BITS_TOTAL = 2048, + COUNTER_REDUNDANT_BITS = BITS_PER_LONG, + COUNTER_WINDOW_SIZE = COUNTER_BITS_TOTAL - COUNTER_REDUNDANT_BITS +}; + +enum limits { + REKEY_AFTER_MESSAGES = 1ULL << 60, + REJECT_AFTER_MESSAGES = U64_MAX - COUNTER_WINDOW_SIZE - 1, + REKEY_TIMEOUT = 5, + REKEY_TIMEOUT_JITTER_MAX_JIFFIES = HZ / 3, + REKEY_AFTER_TIME = 120, + REJECT_AFTER_TIME = 180, + INITIATIONS_PER_SECOND = 50, + MAX_PEERS_PER_DEVICE = 1U << 20, + KEEPALIVE_TIMEOUT = 10, + MAX_TIMER_HANDSHAKES = 90 / REKEY_TIMEOUT, + MAX_QUEUED_INCOMING_HANDSHAKES = 4096, /* TODO: replace this with DQL */ + MAX_STAGED_PACKETS = 128, + MAX_QUEUED_PACKETS = 1024 /* TODO: replace this with DQL */ +}; + +enum message_type { + MESSAGE_INVALID = 0, + MESSAGE_HANDSHAKE_INITIATION = 1, + MESSAGE_HANDSHAKE_RESPONSE = 2, + MESSAGE_HANDSHAKE_COOKIE = 3, + MESSAGE_DATA = 4 +}; + +struct message_header { + /* The actual layout of this that we want is: + * u8 type + * u8 reserved_zero[3] + * + * But it turns out that by encoding this as little endian, + * we achieve the same thing, and it makes checking faster. + */ + __le32 type; +}; + +struct message_macs { + u8 mac1[COOKIE_LEN]; + u8 mac2[COOKIE_LEN]; +}; + +struct message_handshake_initiation { + struct message_header header; + __le32 sender_index; + u8 unencrypted_ephemeral[NOISE_PUBLIC_KEY_LEN]; + u8 encrypted_static[noise_encrypted_len(NOISE_PUBLIC_KEY_LEN)]; + u8 encrypted_timestamp[noise_encrypted_len(NOISE_TIMESTAMP_LEN)]; + struct message_macs macs; +}; + +struct message_handshake_response { + struct message_header header; + __le32 sender_index; + __le32 receiver_index; + u8 unencrypted_ephemeral[NOISE_PUBLIC_KEY_LEN]; + u8 encrypted_nothing[noise_encrypted_len(0)]; + struct message_macs macs; +}; + +struct message_handshake_cookie { + struct message_header header; + __le32 receiver_index; + u8 nonce[COOKIE_NONCE_LEN]; + u8 encrypted_cookie[noise_encrypted_len(COOKIE_LEN)]; +}; + +struct message_data { + struct message_header header; + __le32 key_idx; + __le64 counter; + u8 encrypted_data[]; +}; + +#define message_data_len(plain_len) \ + (noise_encrypted_len(plain_len) + sizeof(struct message_data)) + +enum message_alignments { + MESSAGE_PADDING_MULTIPLE = 16, + MESSAGE_MINIMUM_LENGTH = message_data_len(0) +}; + +#define SKB_HEADER_LEN \ + (max(sizeof(struct iphdr), sizeof(struct ipv6hdr)) + \ + sizeof(struct udphdr) + NET_SKB_PAD) +#define DATA_PACKET_HEAD_ROOM \ + ALIGN(sizeof(struct message_data) + SKB_HEADER_LEN, 4) + +enum { HANDSHAKE_DSCP = 0x88 /* AF41, plus 00 ECN */ }; + +#endif /* _WG_MESSAGES_H */ diff --git a/drivers/net/wireguard/netlink.c b/drivers/net/wireguard/netlink.c new file mode 100644 index 0000000000000..802099c8828a6 --- /dev/null +++ b/drivers/net/wireguard/netlink.c @@ -0,0 +1,636 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "netlink.h" +#include "device.h" +#include "peer.h" +#include "socket.h" +#include "queueing.h" +#include "messages.h" + +#include + +#include +#include +#include +#include + +static struct genl_family genl_family; + +static const struct nla_policy device_policy[WGDEVICE_A_MAX + 1] = { + [WGDEVICE_A_IFINDEX] = { .type = NLA_U32 }, + [WGDEVICE_A_IFNAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 }, + [WGDEVICE_A_PRIVATE_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, + [WGDEVICE_A_PUBLIC_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, + [WGDEVICE_A_FLAGS] = { .type = NLA_U32 }, + [WGDEVICE_A_LISTEN_PORT] = { .type = NLA_U16 }, + [WGDEVICE_A_FWMARK] = { .type = NLA_U32 }, + [WGDEVICE_A_PEERS] = { .type = NLA_NESTED } +}; + +static const struct nla_policy peer_policy[WGPEER_A_MAX + 1] = { + [WGPEER_A_PUBLIC_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_PUBLIC_KEY_LEN }, + [WGPEER_A_PRESHARED_KEY] = { .type = NLA_EXACT_LEN, .len = NOISE_SYMMETRIC_KEY_LEN }, + [WGPEER_A_FLAGS] = { .type = NLA_U32 }, + [WGPEER_A_ENDPOINT] = { .type = NLA_MIN_LEN, .len = sizeof(struct sockaddr) }, + [WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL] = { .type = NLA_U16 }, + [WGPEER_A_LAST_HANDSHAKE_TIME] = { .type = NLA_EXACT_LEN, .len = sizeof(struct __kernel_timespec) }, + [WGPEER_A_RX_BYTES] = { .type = NLA_U64 }, + [WGPEER_A_TX_BYTES] = { .type = NLA_U64 }, + [WGPEER_A_ALLOWEDIPS] = { .type = NLA_NESTED }, + [WGPEER_A_PROTOCOL_VERSION] = { .type = NLA_U32 } +}; + +static const struct nla_policy allowedip_policy[WGALLOWEDIP_A_MAX + 1] = { + [WGALLOWEDIP_A_FAMILY] = { .type = NLA_U16 }, + [WGALLOWEDIP_A_IPADDR] = { .type = NLA_MIN_LEN, .len = sizeof(struct in_addr) }, + [WGALLOWEDIP_A_CIDR_MASK] = { .type = NLA_U8 } +}; + +static struct wg_device *lookup_interface(struct nlattr **attrs, + struct sk_buff *skb) +{ + struct net_device *dev = NULL; + + if (!attrs[WGDEVICE_A_IFINDEX] == !attrs[WGDEVICE_A_IFNAME]) + return ERR_PTR(-EBADR); + if (attrs[WGDEVICE_A_IFINDEX]) + dev = dev_get_by_index(sock_net(skb->sk), + nla_get_u32(attrs[WGDEVICE_A_IFINDEX])); + else if (attrs[WGDEVICE_A_IFNAME]) + dev = dev_get_by_name(sock_net(skb->sk), + nla_data(attrs[WGDEVICE_A_IFNAME])); + if (!dev) + return ERR_PTR(-ENODEV); + if (!dev->rtnl_link_ops || !dev->rtnl_link_ops->kind || + strcmp(dev->rtnl_link_ops->kind, KBUILD_MODNAME)) { + dev_put(dev); + return ERR_PTR(-EOPNOTSUPP); + } + return netdev_priv(dev); +} + +static int get_allowedips(struct sk_buff *skb, const u8 *ip, u8 cidr, + int family) +{ + struct nlattr *allowedip_nest; + + allowedip_nest = nla_nest_start(skb, 0); + if (!allowedip_nest) + return -EMSGSIZE; + + if (nla_put_u8(skb, WGALLOWEDIP_A_CIDR_MASK, cidr) || + nla_put_u16(skb, WGALLOWEDIP_A_FAMILY, family) || + nla_put(skb, WGALLOWEDIP_A_IPADDR, family == AF_INET6 ? + sizeof(struct in6_addr) : sizeof(struct in_addr), ip)) { + nla_nest_cancel(skb, allowedip_nest); + return -EMSGSIZE; + } + + nla_nest_end(skb, allowedip_nest); + return 0; +} + +struct dump_ctx { + struct wg_device *wg; + struct wg_peer *next_peer; + u64 allowedips_seq; + struct allowedips_node *next_allowedip; +}; + +#define DUMP_CTX(cb) ((struct dump_ctx *)(cb)->args) + +static int +get_peer(struct wg_peer *peer, struct sk_buff *skb, struct dump_ctx *ctx) +{ + + struct nlattr *allowedips_nest, *peer_nest = nla_nest_start(skb, 0); + struct allowedips_node *allowedips_node = ctx->next_allowedip; + bool fail; + + if (!peer_nest) + return -EMSGSIZE; + + down_read(&peer->handshake.lock); + fail = nla_put(skb, WGPEER_A_PUBLIC_KEY, NOISE_PUBLIC_KEY_LEN, + peer->handshake.remote_static); + up_read(&peer->handshake.lock); + if (fail) + goto err; + + if (!allowedips_node) { + const struct __kernel_timespec last_handshake = { + .tv_sec = peer->walltime_last_handshake.tv_sec, + .tv_nsec = peer->walltime_last_handshake.tv_nsec + }; + + down_read(&peer->handshake.lock); + fail = nla_put(skb, WGPEER_A_PRESHARED_KEY, + NOISE_SYMMETRIC_KEY_LEN, + peer->handshake.preshared_key); + up_read(&peer->handshake.lock); + if (fail) + goto err; + + if (nla_put(skb, WGPEER_A_LAST_HANDSHAKE_TIME, + sizeof(last_handshake), &last_handshake) || + nla_put_u16(skb, WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL, + peer->persistent_keepalive_interval) || + nla_put_u64_64bit(skb, WGPEER_A_TX_BYTES, peer->tx_bytes, + WGPEER_A_UNSPEC) || + nla_put_u64_64bit(skb, WGPEER_A_RX_BYTES, peer->rx_bytes, + WGPEER_A_UNSPEC) || + nla_put_u32(skb, WGPEER_A_PROTOCOL_VERSION, 1)) + goto err; + + read_lock_bh(&peer->endpoint_lock); + if (peer->endpoint.addr.sa_family == AF_INET) + fail = nla_put(skb, WGPEER_A_ENDPOINT, + sizeof(peer->endpoint.addr4), + &peer->endpoint.addr4); + else if (peer->endpoint.addr.sa_family == AF_INET6) + fail = nla_put(skb, WGPEER_A_ENDPOINT, + sizeof(peer->endpoint.addr6), + &peer->endpoint.addr6); + read_unlock_bh(&peer->endpoint_lock); + if (fail) + goto err; + allowedips_node = + list_first_entry_or_null(&peer->allowedips_list, + struct allowedips_node, peer_list); + } + if (!allowedips_node) + goto no_allowedips; + if (!ctx->allowedips_seq) + ctx->allowedips_seq = peer->device->peer_allowedips.seq; + else if (ctx->allowedips_seq != peer->device->peer_allowedips.seq) + goto no_allowedips; + + allowedips_nest = nla_nest_start(skb, WGPEER_A_ALLOWEDIPS); + if (!allowedips_nest) + goto err; + + list_for_each_entry_from(allowedips_node, &peer->allowedips_list, + peer_list) { + u8 cidr, ip[16] __aligned(__alignof(u64)); + int family; + + family = wg_allowedips_read_node(allowedips_node, ip, &cidr); + if (get_allowedips(skb, ip, cidr, family)) { + nla_nest_end(skb, allowedips_nest); + nla_nest_end(skb, peer_nest); + ctx->next_allowedip = allowedips_node; + return -EMSGSIZE; + } + } + nla_nest_end(skb, allowedips_nest); +no_allowedips: + nla_nest_end(skb, peer_nest); + ctx->next_allowedip = NULL; + ctx->allowedips_seq = 0; + return 0; +err: + nla_nest_cancel(skb, peer_nest); + return -EMSGSIZE; +} + +static int wg_get_device_start(struct netlink_callback *cb) +{ + struct wg_device *wg; + + wg = lookup_interface(genl_dumpit_info(cb)->attrs, cb->skb); + if (IS_ERR(wg)) + return PTR_ERR(wg); + DUMP_CTX(cb)->wg = wg; + return 0; +} + +static int wg_get_device_dump(struct sk_buff *skb, struct netlink_callback *cb) +{ + struct wg_peer *peer, *next_peer_cursor; + struct dump_ctx *ctx = DUMP_CTX(cb); + struct wg_device *wg = ctx->wg; + struct nlattr *peers_nest; + int ret = -EMSGSIZE; + bool done = true; + void *hdr; + + rtnl_lock(); + mutex_lock(&wg->device_update_lock); + cb->seq = wg->device_update_gen; + next_peer_cursor = ctx->next_peer; + + hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, + &genl_family, NLM_F_MULTI, WG_CMD_GET_DEVICE); + if (!hdr) + goto out; + genl_dump_check_consistent(cb, hdr); + + if (!ctx->next_peer) { + if (nla_put_u16(skb, WGDEVICE_A_LISTEN_PORT, + wg->incoming_port) || + nla_put_u32(skb, WGDEVICE_A_FWMARK, wg->fwmark) || + nla_put_u32(skb, WGDEVICE_A_IFINDEX, wg->dev->ifindex) || + nla_put_string(skb, WGDEVICE_A_IFNAME, wg->dev->name)) + goto out; + + down_read(&wg->static_identity.lock); + if (wg->static_identity.has_identity) { + if (nla_put(skb, WGDEVICE_A_PRIVATE_KEY, + NOISE_PUBLIC_KEY_LEN, + wg->static_identity.static_private) || + nla_put(skb, WGDEVICE_A_PUBLIC_KEY, + NOISE_PUBLIC_KEY_LEN, + wg->static_identity.static_public)) { + up_read(&wg->static_identity.lock); + goto out; + } + } + up_read(&wg->static_identity.lock); + } + + peers_nest = nla_nest_start(skb, WGDEVICE_A_PEERS); + if (!peers_nest) + goto out; + ret = 0; + /* If the last cursor was removed via list_del_init in peer_remove, then + * we just treat this the same as there being no more peers left. The + * reason is that seq_nr should indicate to userspace that this isn't a + * coherent dump anyway, so they'll try again. + */ + if (list_empty(&wg->peer_list) || + (ctx->next_peer && list_empty(&ctx->next_peer->peer_list))) { + nla_nest_cancel(skb, peers_nest); + goto out; + } + lockdep_assert_held(&wg->device_update_lock); + peer = list_prepare_entry(ctx->next_peer, &wg->peer_list, peer_list); + list_for_each_entry_continue(peer, &wg->peer_list, peer_list) { + if (get_peer(peer, skb, ctx)) { + done = false; + break; + } + next_peer_cursor = peer; + } + nla_nest_end(skb, peers_nest); + +out: + if (!ret && !done && next_peer_cursor) + wg_peer_get(next_peer_cursor); + wg_peer_put(ctx->next_peer); + mutex_unlock(&wg->device_update_lock); + rtnl_unlock(); + + if (ret) { + genlmsg_cancel(skb, hdr); + return ret; + } + genlmsg_end(skb, hdr); + if (done) { + ctx->next_peer = NULL; + return 0; + } + ctx->next_peer = next_peer_cursor; + return skb->len; + + /* At this point, we can't really deal ourselves with safely zeroing out + * the private key material after usage. This will need an additional API + * in the kernel for marking skbs as zero_on_free. + */ +} + +static int wg_get_device_done(struct netlink_callback *cb) +{ + struct dump_ctx *ctx = DUMP_CTX(cb); + + if (ctx->wg) + dev_put(ctx->wg->dev); + wg_peer_put(ctx->next_peer); + return 0; +} + +static int set_port(struct wg_device *wg, u16 port) +{ + struct wg_peer *peer; + + if (wg->incoming_port == port) + return 0; + list_for_each_entry(peer, &wg->peer_list, peer_list) + wg_socket_clear_peer_endpoint_src(peer); + if (!netif_running(wg->dev)) { + wg->incoming_port = port; + return 0; + } + return wg_socket_init(wg, port); +} + +static int set_allowedip(struct wg_peer *peer, struct nlattr **attrs) +{ + int ret = -EINVAL; + u16 family; + u8 cidr; + + if (!attrs[WGALLOWEDIP_A_FAMILY] || !attrs[WGALLOWEDIP_A_IPADDR] || + !attrs[WGALLOWEDIP_A_CIDR_MASK]) + return ret; + family = nla_get_u16(attrs[WGALLOWEDIP_A_FAMILY]); + cidr = nla_get_u8(attrs[WGALLOWEDIP_A_CIDR_MASK]); + + if (family == AF_INET && cidr <= 32 && + nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in_addr)) + ret = wg_allowedips_insert_v4( + &peer->device->peer_allowedips, + nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, + &peer->device->device_update_lock); + else if (family == AF_INET6 && cidr <= 128 && + nla_len(attrs[WGALLOWEDIP_A_IPADDR]) == sizeof(struct in6_addr)) + ret = wg_allowedips_insert_v6( + &peer->device->peer_allowedips, + nla_data(attrs[WGALLOWEDIP_A_IPADDR]), cidr, peer, + &peer->device->device_update_lock); + + return ret; +} + +static int set_peer(struct wg_device *wg, struct nlattr **attrs) +{ + u8 *public_key = NULL, *preshared_key = NULL; + struct wg_peer *peer = NULL; + u32 flags = 0; + int ret; + + ret = -EINVAL; + if (attrs[WGPEER_A_PUBLIC_KEY] && + nla_len(attrs[WGPEER_A_PUBLIC_KEY]) == NOISE_PUBLIC_KEY_LEN) + public_key = nla_data(attrs[WGPEER_A_PUBLIC_KEY]); + else + goto out; + if (attrs[WGPEER_A_PRESHARED_KEY] && + nla_len(attrs[WGPEER_A_PRESHARED_KEY]) == NOISE_SYMMETRIC_KEY_LEN) + preshared_key = nla_data(attrs[WGPEER_A_PRESHARED_KEY]); + + if (attrs[WGPEER_A_FLAGS]) + flags = nla_get_u32(attrs[WGPEER_A_FLAGS]); + ret = -EOPNOTSUPP; + if (flags & ~__WGPEER_F_ALL) + goto out; + + ret = -EPFNOSUPPORT; + if (attrs[WGPEER_A_PROTOCOL_VERSION]) { + if (nla_get_u32(attrs[WGPEER_A_PROTOCOL_VERSION]) != 1) + goto out; + } + + peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, + nla_data(attrs[WGPEER_A_PUBLIC_KEY])); + ret = 0; + if (!peer) { /* Peer doesn't exist yet. Add a new one. */ + if (flags & (WGPEER_F_REMOVE_ME | WGPEER_F_UPDATE_ONLY)) + goto out; + + /* The peer is new, so there aren't allowed IPs to remove. */ + flags &= ~WGPEER_F_REPLACE_ALLOWEDIPS; + + down_read(&wg->static_identity.lock); + if (wg->static_identity.has_identity && + !memcmp(nla_data(attrs[WGPEER_A_PUBLIC_KEY]), + wg->static_identity.static_public, + NOISE_PUBLIC_KEY_LEN)) { + /* We silently ignore peers that have the same public + * key as the device. The reason we do it silently is + * that we'd like for people to be able to reuse the + * same set of API calls across peers. + */ + up_read(&wg->static_identity.lock); + ret = 0; + goto out; + } + up_read(&wg->static_identity.lock); + + peer = wg_peer_create(wg, public_key, preshared_key); + if (IS_ERR(peer)) { + ret = PTR_ERR(peer); + peer = NULL; + goto out; + } + /* Take additional reference, as though we've just been + * looked up. + */ + wg_peer_get(peer); + } + + if (flags & WGPEER_F_REMOVE_ME) { + wg_peer_remove(peer); + goto out; + } + + if (preshared_key) { + down_write(&peer->handshake.lock); + memcpy(&peer->handshake.preshared_key, preshared_key, + NOISE_SYMMETRIC_KEY_LEN); + up_write(&peer->handshake.lock); + } + + if (attrs[WGPEER_A_ENDPOINT]) { + struct sockaddr *addr = nla_data(attrs[WGPEER_A_ENDPOINT]); + size_t len = nla_len(attrs[WGPEER_A_ENDPOINT]); + + if ((len == sizeof(struct sockaddr_in) && + addr->sa_family == AF_INET) || + (len == sizeof(struct sockaddr_in6) && + addr->sa_family == AF_INET6)) { + struct endpoint endpoint = { { { 0 } } }; + + memcpy(&endpoint.addr, addr, len); + wg_socket_set_peer_endpoint(peer, &endpoint); + } + } + + if (flags & WGPEER_F_REPLACE_ALLOWEDIPS) + wg_allowedips_remove_by_peer(&wg->peer_allowedips, peer, + &wg->device_update_lock); + + if (attrs[WGPEER_A_ALLOWEDIPS]) { + struct nlattr *attr, *allowedip[WGALLOWEDIP_A_MAX + 1]; + int rem; + + nla_for_each_nested(attr, attrs[WGPEER_A_ALLOWEDIPS], rem) { + ret = nla_parse_nested(allowedip, WGALLOWEDIP_A_MAX, + attr, allowedip_policy, NULL); + if (ret < 0) + goto out; + ret = set_allowedip(peer, allowedip); + if (ret < 0) + goto out; + } + } + + if (attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]) { + const u16 persistent_keepalive_interval = nla_get_u16( + attrs[WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL]); + const bool send_keepalive = + !peer->persistent_keepalive_interval && + persistent_keepalive_interval && + netif_running(wg->dev); + + peer->persistent_keepalive_interval = persistent_keepalive_interval; + if (send_keepalive) + wg_packet_send_keepalive(peer); + } + + if (netif_running(wg->dev)) + wg_packet_send_staged_packets(peer); + +out: + wg_peer_put(peer); + if (attrs[WGPEER_A_PRESHARED_KEY]) + memzero_explicit(nla_data(attrs[WGPEER_A_PRESHARED_KEY]), + nla_len(attrs[WGPEER_A_PRESHARED_KEY])); + return ret; +} + +static int wg_set_device(struct sk_buff *skb, struct genl_info *info) +{ + struct wg_device *wg = lookup_interface(info->attrs, skb); + u32 flags = 0; + int ret; + + if (IS_ERR(wg)) { + ret = PTR_ERR(wg); + goto out_nodev; + } + + rtnl_lock(); + mutex_lock(&wg->device_update_lock); + + if (info->attrs[WGDEVICE_A_FLAGS]) + flags = nla_get_u32(info->attrs[WGDEVICE_A_FLAGS]); + ret = -EOPNOTSUPP; + if (flags & ~__WGDEVICE_F_ALL) + goto out; + + ret = -EPERM; + if ((info->attrs[WGDEVICE_A_LISTEN_PORT] || + info->attrs[WGDEVICE_A_FWMARK]) && + !ns_capable(wg->creating_net->user_ns, CAP_NET_ADMIN)) + goto out; + + ++wg->device_update_gen; + + if (info->attrs[WGDEVICE_A_FWMARK]) { + struct wg_peer *peer; + + wg->fwmark = nla_get_u32(info->attrs[WGDEVICE_A_FWMARK]); + list_for_each_entry(peer, &wg->peer_list, peer_list) + wg_socket_clear_peer_endpoint_src(peer); + } + + if (info->attrs[WGDEVICE_A_LISTEN_PORT]) { + ret = set_port(wg, + nla_get_u16(info->attrs[WGDEVICE_A_LISTEN_PORT])); + if (ret) + goto out; + } + + if (flags & WGDEVICE_F_REPLACE_PEERS) + wg_peer_remove_all(wg); + + if (info->attrs[WGDEVICE_A_PRIVATE_KEY] && + nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY]) == + NOISE_PUBLIC_KEY_LEN) { + u8 *private_key = nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]); + u8 public_key[NOISE_PUBLIC_KEY_LEN]; + struct wg_peer *peer, *temp; + + if (!crypto_memneq(wg->static_identity.static_private, + private_key, NOISE_PUBLIC_KEY_LEN)) + goto skip_set_private_key; + + /* We remove before setting, to prevent race, which means doing + * two 25519-genpub ops. + */ + if (curve25519_generate_public(public_key, private_key)) { + peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, + public_key); + if (peer) { + wg_peer_put(peer); + wg_peer_remove(peer); + } + } + + down_write(&wg->static_identity.lock); + wg_noise_set_static_identity_private_key(&wg->static_identity, + private_key); + list_for_each_entry_safe(peer, temp, &wg->peer_list, + peer_list) { + wg_noise_precompute_static_static(peer); + wg_noise_expire_current_peer_keypairs(peer); + } + wg_cookie_checker_precompute_device_keys(&wg->cookie_checker); + up_write(&wg->static_identity.lock); + } +skip_set_private_key: + + if (info->attrs[WGDEVICE_A_PEERS]) { + struct nlattr *attr, *peer[WGPEER_A_MAX + 1]; + int rem; + + nla_for_each_nested(attr, info->attrs[WGDEVICE_A_PEERS], rem) { + ret = nla_parse_nested(peer, WGPEER_A_MAX, attr, + peer_policy, NULL); + if (ret < 0) + goto out; + ret = set_peer(wg, peer); + if (ret < 0) + goto out; + } + } + ret = 0; + +out: + mutex_unlock(&wg->device_update_lock); + rtnl_unlock(); + dev_put(wg->dev); +out_nodev: + if (info->attrs[WGDEVICE_A_PRIVATE_KEY]) + memzero_explicit(nla_data(info->attrs[WGDEVICE_A_PRIVATE_KEY]), + nla_len(info->attrs[WGDEVICE_A_PRIVATE_KEY])); + return ret; +} + +static const struct genl_ops genl_ops[] = { + { + .cmd = WG_CMD_GET_DEVICE, + .start = wg_get_device_start, + .dumpit = wg_get_device_dump, + .done = wg_get_device_done, + .flags = GENL_UNS_ADMIN_PERM + }, { + .cmd = WG_CMD_SET_DEVICE, + .doit = wg_set_device, + .flags = GENL_UNS_ADMIN_PERM + } +}; + +static struct genl_family genl_family __ro_after_init = { + .ops = genl_ops, + .n_ops = ARRAY_SIZE(genl_ops), + .name = WG_GENL_NAME, + .version = WG_GENL_VERSION, + .maxattr = WGDEVICE_A_MAX, + .module = THIS_MODULE, + .policy = device_policy, + .netnsok = true +}; + +int __init wg_genetlink_init(void) +{ + return genl_register_family(&genl_family); +} + +void __exit wg_genetlink_uninit(void) +{ + genl_unregister_family(&genl_family); +} diff --git a/drivers/net/wireguard/netlink.h b/drivers/net/wireguard/netlink.h new file mode 100644 index 0000000000000..15100d92e2e32 --- /dev/null +++ b/drivers/net/wireguard/netlink.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_NETLINK_H +#define _WG_NETLINK_H + +int wg_genetlink_init(void); +void wg_genetlink_uninit(void); + +#endif /* _WG_NETLINK_H */ diff --git a/drivers/net/wireguard/noise.c b/drivers/net/wireguard/noise.c new file mode 100644 index 0000000000000..708dc61c974f7 --- /dev/null +++ b/drivers/net/wireguard/noise.c @@ -0,0 +1,835 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "noise.h" +#include "device.h" +#include "peer.h" +#include "messages.h" +#include "queueing.h" +#include "peerlookup.h" + +#include +#include +#include +#include +#include +#include + +/* This implements Noise_IKpsk2: + * + * <- s + * ****** + * -> e, es, s, ss, {t} + * <- e, ee, se, psk, {} + */ + +static const u8 handshake_name[37] = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"; +static const u8 identifier_name[34] = "WireGuard v1 zx2c4 Jason@zx2c4.com"; +static u8 handshake_init_hash[NOISE_HASH_LEN] __ro_after_init; +static u8 handshake_init_chaining_key[NOISE_HASH_LEN] __ro_after_init; +static atomic64_t keypair_counter = ATOMIC64_INIT(0); + +void __init wg_noise_init(void) +{ + struct blake2s_state blake; + + blake2s(handshake_init_chaining_key, handshake_name, NULL, + NOISE_HASH_LEN, sizeof(handshake_name), 0); + blake2s_init(&blake, NOISE_HASH_LEN); + blake2s_update(&blake, handshake_init_chaining_key, NOISE_HASH_LEN); + blake2s_update(&blake, identifier_name, sizeof(identifier_name)); + blake2s_final(&blake, handshake_init_hash); +} + +/* Must hold peer->handshake.static_identity->lock */ +void wg_noise_precompute_static_static(struct wg_peer *peer) +{ + down_write(&peer->handshake.lock); + if (!peer->handshake.static_identity->has_identity || + !curve25519(peer->handshake.precomputed_static_static, + peer->handshake.static_identity->static_private, + peer->handshake.remote_static)) + memset(peer->handshake.precomputed_static_static, 0, + NOISE_PUBLIC_KEY_LEN); + up_write(&peer->handshake.lock); +} + +void wg_noise_handshake_init(struct noise_handshake *handshake, + struct noise_static_identity *static_identity, + const u8 peer_public_key[NOISE_PUBLIC_KEY_LEN], + const u8 peer_preshared_key[NOISE_SYMMETRIC_KEY_LEN], + struct wg_peer *peer) +{ + memset(handshake, 0, sizeof(*handshake)); + init_rwsem(&handshake->lock); + handshake->entry.type = INDEX_HASHTABLE_HANDSHAKE; + handshake->entry.peer = peer; + memcpy(handshake->remote_static, peer_public_key, NOISE_PUBLIC_KEY_LEN); + if (peer_preshared_key) + memcpy(handshake->preshared_key, peer_preshared_key, + NOISE_SYMMETRIC_KEY_LEN); + handshake->static_identity = static_identity; + handshake->state = HANDSHAKE_ZEROED; + wg_noise_precompute_static_static(peer); +} + +static void handshake_zero(struct noise_handshake *handshake) +{ + memset(&handshake->ephemeral_private, 0, NOISE_PUBLIC_KEY_LEN); + memset(&handshake->remote_ephemeral, 0, NOISE_PUBLIC_KEY_LEN); + memset(&handshake->hash, 0, NOISE_HASH_LEN); + memset(&handshake->chaining_key, 0, NOISE_HASH_LEN); + handshake->remote_index = 0; + handshake->state = HANDSHAKE_ZEROED; +} + +void wg_noise_handshake_clear(struct noise_handshake *handshake) +{ + wg_index_hashtable_remove( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); + down_write(&handshake->lock); + handshake_zero(handshake); + up_write(&handshake->lock); + wg_index_hashtable_remove( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); +} + +static struct noise_keypair *keypair_create(struct wg_peer *peer) +{ + struct noise_keypair *keypair = kzalloc(sizeof(*keypair), GFP_KERNEL); + + if (unlikely(!keypair)) + return NULL; + keypair->internal_id = atomic64_inc_return(&keypair_counter); + keypair->entry.type = INDEX_HASHTABLE_KEYPAIR; + keypair->entry.peer = peer; + kref_init(&keypair->refcount); + return keypair; +} + +static void keypair_free_rcu(struct rcu_head *rcu) +{ + kzfree(container_of(rcu, struct noise_keypair, rcu)); +} + +static void keypair_free_kref(struct kref *kref) +{ + struct noise_keypair *keypair = + container_of(kref, struct noise_keypair, refcount); + + net_dbg_ratelimited("%s: Keypair %llu destroyed for peer %llu\n", + keypair->entry.peer->device->dev->name, + keypair->internal_id, + keypair->entry.peer->internal_id); + wg_index_hashtable_remove(keypair->entry.peer->device->index_hashtable, + &keypair->entry); + call_rcu(&keypair->rcu, keypair_free_rcu); +} + +void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now) +{ + if (unlikely(!keypair)) + return; + if (unlikely(unreference_now)) + wg_index_hashtable_remove( + keypair->entry.peer->device->index_hashtable, + &keypair->entry); + kref_put(&keypair->refcount, keypair_free_kref); +} + +struct noise_keypair *wg_noise_keypair_get(struct noise_keypair *keypair) +{ + RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(), + "Taking noise keypair reference without holding the RCU BH read lock"); + if (unlikely(!keypair || !kref_get_unless_zero(&keypair->refcount))) + return NULL; + return keypair; +} + +void wg_noise_keypairs_clear(struct noise_keypairs *keypairs) +{ + struct noise_keypair *old; + + spin_lock_bh(&keypairs->keypair_update_lock); + + /* We zero the next_keypair before zeroing the others, so that + * wg_noise_received_with_keypair returns early before subsequent ones + * are zeroed. + */ + old = rcu_dereference_protected(keypairs->next_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + RCU_INIT_POINTER(keypairs->next_keypair, NULL); + wg_noise_keypair_put(old, true); + + old = rcu_dereference_protected(keypairs->previous_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + RCU_INIT_POINTER(keypairs->previous_keypair, NULL); + wg_noise_keypair_put(old, true); + + old = rcu_dereference_protected(keypairs->current_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + RCU_INIT_POINTER(keypairs->current_keypair, NULL); + wg_noise_keypair_put(old, true); + + spin_unlock_bh(&keypairs->keypair_update_lock); +} + +void wg_noise_expire_current_peer_keypairs(struct wg_peer *peer) +{ + struct noise_keypair *keypair; + + wg_noise_handshake_clear(&peer->handshake); + wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); + + spin_lock_bh(&peer->keypairs.keypair_update_lock); + keypair = rcu_dereference_protected(peer->keypairs.next_keypair, + lockdep_is_held(&peer->keypairs.keypair_update_lock)); + if (keypair) + keypair->sending.is_valid = false; + keypair = rcu_dereference_protected(peer->keypairs.current_keypair, + lockdep_is_held(&peer->keypairs.keypair_update_lock)); + if (keypair) + keypair->sending.is_valid = false; + spin_unlock_bh(&peer->keypairs.keypair_update_lock); +} + +static void add_new_keypair(struct noise_keypairs *keypairs, + struct noise_keypair *new_keypair) +{ + struct noise_keypair *previous_keypair, *next_keypair, *current_keypair; + + spin_lock_bh(&keypairs->keypair_update_lock); + previous_keypair = rcu_dereference_protected(keypairs->previous_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + next_keypair = rcu_dereference_protected(keypairs->next_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + current_keypair = rcu_dereference_protected(keypairs->current_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + if (new_keypair->i_am_the_initiator) { + /* If we're the initiator, it means we've sent a handshake, and + * received a confirmation response, which means this new + * keypair can now be used. + */ + if (next_keypair) { + /* If there already was a next keypair pending, we + * demote it to be the previous keypair, and free the + * existing current. Note that this means KCI can result + * in this transition. It would perhaps be more sound to + * always just get rid of the unused next keypair + * instead of putting it in the previous slot, but this + * might be a bit less robust. Something to think about + * for the future. + */ + RCU_INIT_POINTER(keypairs->next_keypair, NULL); + rcu_assign_pointer(keypairs->previous_keypair, + next_keypair); + wg_noise_keypair_put(current_keypair, true); + } else /* If there wasn't an existing next keypair, we replace + * the previous with the current one. + */ + rcu_assign_pointer(keypairs->previous_keypair, + current_keypair); + /* At this point we can get rid of the old previous keypair, and + * set up the new keypair. + */ + wg_noise_keypair_put(previous_keypair, true); + rcu_assign_pointer(keypairs->current_keypair, new_keypair); + } else { + /* If we're the responder, it means we can't use the new keypair + * until we receive confirmation via the first data packet, so + * we get rid of the existing previous one, the possibly + * existing next one, and slide in the new next one. + */ + rcu_assign_pointer(keypairs->next_keypair, new_keypair); + wg_noise_keypair_put(next_keypair, true); + RCU_INIT_POINTER(keypairs->previous_keypair, NULL); + wg_noise_keypair_put(previous_keypair, true); + } + spin_unlock_bh(&keypairs->keypair_update_lock); +} + +bool wg_noise_received_with_keypair(struct noise_keypairs *keypairs, + struct noise_keypair *received_keypair) +{ + struct noise_keypair *old_keypair; + bool key_is_new; + + /* We first check without taking the spinlock. */ + key_is_new = received_keypair == + rcu_access_pointer(keypairs->next_keypair); + if (likely(!key_is_new)) + return false; + + spin_lock_bh(&keypairs->keypair_update_lock); + /* After locking, we double check that things didn't change from + * beneath us. + */ + if (unlikely(received_keypair != + rcu_dereference_protected(keypairs->next_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)))) { + spin_unlock_bh(&keypairs->keypair_update_lock); + return false; + } + + /* When we've finally received the confirmation, we slide the next + * into the current, the current into the previous, and get rid of + * the old previous. + */ + old_keypair = rcu_dereference_protected(keypairs->previous_keypair, + lockdep_is_held(&keypairs->keypair_update_lock)); + rcu_assign_pointer(keypairs->previous_keypair, + rcu_dereference_protected(keypairs->current_keypair, + lockdep_is_held(&keypairs->keypair_update_lock))); + wg_noise_keypair_put(old_keypair, true); + rcu_assign_pointer(keypairs->current_keypair, received_keypair); + RCU_INIT_POINTER(keypairs->next_keypair, NULL); + + spin_unlock_bh(&keypairs->keypair_update_lock); + return true; +} + +/* Must hold static_identity->lock */ +void wg_noise_set_static_identity_private_key( + struct noise_static_identity *static_identity, + const u8 private_key[NOISE_PUBLIC_KEY_LEN]) +{ + memcpy(static_identity->static_private, private_key, + NOISE_PUBLIC_KEY_LEN); + curve25519_clamp_secret(static_identity->static_private); + static_identity->has_identity = curve25519_generate_public( + static_identity->static_public, private_key); +} + +/* This is Hugo Krawczyk's HKDF: + * - https://eprint.iacr.org/2010/264.pdf + * - https://tools.ietf.org/html/rfc5869 + */ +static void kdf(u8 *first_dst, u8 *second_dst, u8 *third_dst, const u8 *data, + size_t first_len, size_t second_len, size_t third_len, + size_t data_len, const u8 chaining_key[NOISE_HASH_LEN]) +{ + u8 output[BLAKE2S_HASH_SIZE + 1]; + u8 secret[BLAKE2S_HASH_SIZE]; + + WARN_ON(IS_ENABLED(DEBUG) && + (first_len > BLAKE2S_HASH_SIZE || + second_len > BLAKE2S_HASH_SIZE || + third_len > BLAKE2S_HASH_SIZE || + ((second_len || second_dst || third_len || third_dst) && + (!first_len || !first_dst)) || + ((third_len || third_dst) && (!second_len || !second_dst)))); + + /* Extract entropy from data into secret */ + blake2s256_hmac(secret, data, chaining_key, data_len, NOISE_HASH_LEN); + + if (!first_dst || !first_len) + goto out; + + /* Expand first key: key = secret, data = 0x1 */ + output[0] = 1; + blake2s256_hmac(output, output, secret, 1, BLAKE2S_HASH_SIZE); + memcpy(first_dst, output, first_len); + + if (!second_dst || !second_len) + goto out; + + /* Expand second key: key = secret, data = first-key || 0x2 */ + output[BLAKE2S_HASH_SIZE] = 2; + blake2s256_hmac(output, output, secret, BLAKE2S_HASH_SIZE + 1, + BLAKE2S_HASH_SIZE); + memcpy(second_dst, output, second_len); + + if (!third_dst || !third_len) + goto out; + + /* Expand third key: key = secret, data = second-key || 0x3 */ + output[BLAKE2S_HASH_SIZE] = 3; + blake2s256_hmac(output, output, secret, BLAKE2S_HASH_SIZE + 1, + BLAKE2S_HASH_SIZE); + memcpy(third_dst, output, third_len); + +out: + /* Clear sensitive data from stack */ + memzero_explicit(secret, BLAKE2S_HASH_SIZE); + memzero_explicit(output, BLAKE2S_HASH_SIZE + 1); +} + +static void symmetric_key_init(struct noise_symmetric_key *key) +{ + spin_lock_init(&key->counter.receive.lock); + atomic64_set(&key->counter.counter, 0); + memset(key->counter.receive.backtrack, 0, + sizeof(key->counter.receive.backtrack)); + key->birthdate = ktime_get_coarse_boottime_ns(); + key->is_valid = true; +} + +static void derive_keys(struct noise_symmetric_key *first_dst, + struct noise_symmetric_key *second_dst, + const u8 chaining_key[NOISE_HASH_LEN]) +{ + kdf(first_dst->key, second_dst->key, NULL, NULL, + NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, 0, 0, + chaining_key); + symmetric_key_init(first_dst); + symmetric_key_init(second_dst); +} + +static bool __must_check mix_dh(u8 chaining_key[NOISE_HASH_LEN], + u8 key[NOISE_SYMMETRIC_KEY_LEN], + const u8 private[NOISE_PUBLIC_KEY_LEN], + const u8 public[NOISE_PUBLIC_KEY_LEN]) +{ + u8 dh_calculation[NOISE_PUBLIC_KEY_LEN]; + + if (unlikely(!curve25519(dh_calculation, private, public))) + return false; + kdf(chaining_key, key, NULL, dh_calculation, NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, chaining_key); + memzero_explicit(dh_calculation, NOISE_PUBLIC_KEY_LEN); + return true; +} + +static bool __must_check mix_precomputed_dh(u8 chaining_key[NOISE_HASH_LEN], + u8 key[NOISE_SYMMETRIC_KEY_LEN], + const u8 precomputed[NOISE_PUBLIC_KEY_LEN]) +{ + static u8 zero_point[NOISE_PUBLIC_KEY_LEN]; + if (unlikely(!crypto_memneq(precomputed, zero_point, NOISE_PUBLIC_KEY_LEN))) + return false; + kdf(chaining_key, key, NULL, precomputed, NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, 0, NOISE_PUBLIC_KEY_LEN, + chaining_key); + return true; +} + +static void mix_hash(u8 hash[NOISE_HASH_LEN], const u8 *src, size_t src_len) +{ + struct blake2s_state blake; + + blake2s_init(&blake, NOISE_HASH_LEN); + blake2s_update(&blake, hash, NOISE_HASH_LEN); + blake2s_update(&blake, src, src_len); + blake2s_final(&blake, hash); +} + +static void mix_psk(u8 chaining_key[NOISE_HASH_LEN], u8 hash[NOISE_HASH_LEN], + u8 key[NOISE_SYMMETRIC_KEY_LEN], + const u8 psk[NOISE_SYMMETRIC_KEY_LEN]) +{ + u8 temp_hash[NOISE_HASH_LEN]; + + kdf(chaining_key, temp_hash, key, psk, NOISE_HASH_LEN, NOISE_HASH_LEN, + NOISE_SYMMETRIC_KEY_LEN, NOISE_SYMMETRIC_KEY_LEN, chaining_key); + mix_hash(hash, temp_hash, NOISE_HASH_LEN); + memzero_explicit(temp_hash, NOISE_HASH_LEN); +} + +static void handshake_init(u8 chaining_key[NOISE_HASH_LEN], + u8 hash[NOISE_HASH_LEN], + const u8 remote_static[NOISE_PUBLIC_KEY_LEN]) +{ + memcpy(hash, handshake_init_hash, NOISE_HASH_LEN); + memcpy(chaining_key, handshake_init_chaining_key, NOISE_HASH_LEN); + mix_hash(hash, remote_static, NOISE_PUBLIC_KEY_LEN); +} + +static void message_encrypt(u8 *dst_ciphertext, const u8 *src_plaintext, + size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN], + u8 hash[NOISE_HASH_LEN]) +{ + chacha20poly1305_encrypt(dst_ciphertext, src_plaintext, src_len, hash, + NOISE_HASH_LEN, + 0 /* Always zero for Noise_IK */, key); + mix_hash(hash, dst_ciphertext, noise_encrypted_len(src_len)); +} + +static bool message_decrypt(u8 *dst_plaintext, const u8 *src_ciphertext, + size_t src_len, u8 key[NOISE_SYMMETRIC_KEY_LEN], + u8 hash[NOISE_HASH_LEN]) +{ + if (!chacha20poly1305_decrypt(dst_plaintext, src_ciphertext, src_len, + hash, NOISE_HASH_LEN, + 0 /* Always zero for Noise_IK */, key)) + return false; + mix_hash(hash, src_ciphertext, src_len); + return true; +} + +static void message_ephemeral(u8 ephemeral_dst[NOISE_PUBLIC_KEY_LEN], + const u8 ephemeral_src[NOISE_PUBLIC_KEY_LEN], + u8 chaining_key[NOISE_HASH_LEN], + u8 hash[NOISE_HASH_LEN]) +{ + if (ephemeral_dst != ephemeral_src) + memcpy(ephemeral_dst, ephemeral_src, NOISE_PUBLIC_KEY_LEN); + mix_hash(hash, ephemeral_src, NOISE_PUBLIC_KEY_LEN); + kdf(chaining_key, NULL, NULL, ephemeral_src, NOISE_HASH_LEN, 0, 0, + NOISE_PUBLIC_KEY_LEN, chaining_key); +} + +static void tai64n_now(u8 output[NOISE_TIMESTAMP_LEN]) +{ + struct timespec64 now; + + ktime_get_real_ts64(&now); + + /* In order to prevent some sort of infoleak from precise timers, we + * round down the nanoseconds part to the closest rounded-down power of + * two to the maximum initiations per second allowed anyway by the + * implementation. + */ + now.tv_nsec = ALIGN_DOWN(now.tv_nsec, + rounddown_pow_of_two(NSEC_PER_SEC / INITIATIONS_PER_SECOND)); + + /* https://cr.yp.to/libtai/tai64.html */ + *(__be64 *)output = cpu_to_be64(0x400000000000000aULL + now.tv_sec); + *(__be32 *)(output + sizeof(__be64)) = cpu_to_be32(now.tv_nsec); +} + +bool +wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, + struct noise_handshake *handshake) +{ + u8 timestamp[NOISE_TIMESTAMP_LEN]; + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + bool ret = false; + + /* We need to wait for crng _before_ taking any locks, since + * curve25519_generate_secret uses get_random_bytes_wait. + */ + wait_for_random_bytes(); + + down_read(&handshake->static_identity->lock); + down_write(&handshake->lock); + + if (unlikely(!handshake->static_identity->has_identity)) + goto out; + + dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION); + + handshake_init(handshake->chaining_key, handshake->hash, + handshake->remote_static); + + /* e */ + curve25519_generate_secret(handshake->ephemeral_private); + if (!curve25519_generate_public(dst->unencrypted_ephemeral, + handshake->ephemeral_private)) + goto out; + message_ephemeral(dst->unencrypted_ephemeral, + dst->unencrypted_ephemeral, handshake->chaining_key, + handshake->hash); + + /* es */ + if (!mix_dh(handshake->chaining_key, key, handshake->ephemeral_private, + handshake->remote_static)) + goto out; + + /* s */ + message_encrypt(dst->encrypted_static, + handshake->static_identity->static_public, + NOISE_PUBLIC_KEY_LEN, key, handshake->hash); + + /* ss */ + if (!mix_precomputed_dh(handshake->chaining_key, key, + handshake->precomputed_static_static)) + goto out; + + /* {t} */ + tai64n_now(timestamp); + message_encrypt(dst->encrypted_timestamp, timestamp, + NOISE_TIMESTAMP_LEN, key, handshake->hash); + + dst->sender_index = wg_index_hashtable_insert( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); + + handshake->state = HANDSHAKE_CREATED_INITIATION; + ret = true; + +out: + up_write(&handshake->lock); + up_read(&handshake->static_identity->lock); + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + return ret; +} + +struct wg_peer * +wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, + struct wg_device *wg) +{ + struct wg_peer *peer = NULL, *ret_peer = NULL; + struct noise_handshake *handshake; + bool replay_attack, flood_attack; + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + u8 chaining_key[NOISE_HASH_LEN]; + u8 hash[NOISE_HASH_LEN]; + u8 s[NOISE_PUBLIC_KEY_LEN]; + u8 e[NOISE_PUBLIC_KEY_LEN]; + u8 t[NOISE_TIMESTAMP_LEN]; + u64 initiation_consumption; + + down_read(&wg->static_identity.lock); + if (unlikely(!wg->static_identity.has_identity)) + goto out; + + handshake_init(chaining_key, hash, wg->static_identity.static_public); + + /* e */ + message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash); + + /* es */ + if (!mix_dh(chaining_key, key, wg->static_identity.static_private, e)) + goto out; + + /* s */ + if (!message_decrypt(s, src->encrypted_static, + sizeof(src->encrypted_static), key, hash)) + goto out; + + /* Lookup which peer we're actually talking to */ + peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, s); + if (!peer) + goto out; + handshake = &peer->handshake; + + /* ss */ + if (!mix_precomputed_dh(chaining_key, key, + handshake->precomputed_static_static)) + goto out; + + /* {t} */ + if (!message_decrypt(t, src->encrypted_timestamp, + sizeof(src->encrypted_timestamp), key, hash)) + goto out; + + down_read(&handshake->lock); + replay_attack = memcmp(t, handshake->latest_timestamp, + NOISE_TIMESTAMP_LEN) <= 0; + flood_attack = (s64)handshake->last_initiation_consumption + + NSEC_PER_SEC / INITIATIONS_PER_SECOND > + (s64)ktime_get_coarse_boottime_ns(); + up_read(&handshake->lock); + if (replay_attack || flood_attack) + goto out; + + /* Success! Copy everything to peer */ + down_write(&handshake->lock); + memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN); + if (memcmp(t, handshake->latest_timestamp, NOISE_TIMESTAMP_LEN) > 0) + memcpy(handshake->latest_timestamp, t, NOISE_TIMESTAMP_LEN); + memcpy(handshake->hash, hash, NOISE_HASH_LEN); + memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN); + handshake->remote_index = src->sender_index; + if ((s64)(handshake->last_initiation_consumption - + (initiation_consumption = ktime_get_coarse_boottime_ns())) < 0) + handshake->last_initiation_consumption = initiation_consumption; + handshake->state = HANDSHAKE_CONSUMED_INITIATION; + up_write(&handshake->lock); + ret_peer = peer; + +out: + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + memzero_explicit(hash, NOISE_HASH_LEN); + memzero_explicit(chaining_key, NOISE_HASH_LEN); + up_read(&wg->static_identity.lock); + if (!ret_peer) + wg_peer_put(peer); + return ret_peer; +} + +bool wg_noise_handshake_create_response(struct message_handshake_response *dst, + struct noise_handshake *handshake) +{ + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + bool ret = false; + + /* We need to wait for crng _before_ taking any locks, since + * curve25519_generate_secret uses get_random_bytes_wait. + */ + wait_for_random_bytes(); + + down_read(&handshake->static_identity->lock); + down_write(&handshake->lock); + + if (handshake->state != HANDSHAKE_CONSUMED_INITIATION) + goto out; + + dst->header.type = cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE); + dst->receiver_index = handshake->remote_index; + + /* e */ + curve25519_generate_secret(handshake->ephemeral_private); + if (!curve25519_generate_public(dst->unencrypted_ephemeral, + handshake->ephemeral_private)) + goto out; + message_ephemeral(dst->unencrypted_ephemeral, + dst->unencrypted_ephemeral, handshake->chaining_key, + handshake->hash); + + /* ee */ + if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, + handshake->remote_ephemeral)) + goto out; + + /* se */ + if (!mix_dh(handshake->chaining_key, NULL, handshake->ephemeral_private, + handshake->remote_static)) + goto out; + + /* psk */ + mix_psk(handshake->chaining_key, handshake->hash, key, + handshake->preshared_key); + + /* {} */ + message_encrypt(dst->encrypted_nothing, NULL, 0, key, handshake->hash); + + dst->sender_index = wg_index_hashtable_insert( + handshake->entry.peer->device->index_hashtable, + &handshake->entry); + + handshake->state = HANDSHAKE_CREATED_RESPONSE; + ret = true; + +out: + up_write(&handshake->lock); + up_read(&handshake->static_identity->lock); + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + return ret; +} + +struct wg_peer * +wg_noise_handshake_consume_response(struct message_handshake_response *src, + struct wg_device *wg) +{ + enum noise_handshake_state state = HANDSHAKE_ZEROED; + struct wg_peer *peer = NULL, *ret_peer = NULL; + struct noise_handshake *handshake; + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + u8 hash[NOISE_HASH_LEN]; + u8 chaining_key[NOISE_HASH_LEN]; + u8 e[NOISE_PUBLIC_KEY_LEN]; + u8 ephemeral_private[NOISE_PUBLIC_KEY_LEN]; + u8 static_private[NOISE_PUBLIC_KEY_LEN]; + + down_read(&wg->static_identity.lock); + + if (unlikely(!wg->static_identity.has_identity)) + goto out; + + handshake = (struct noise_handshake *)wg_index_hashtable_lookup( + wg->index_hashtable, INDEX_HASHTABLE_HANDSHAKE, + src->receiver_index, &peer); + if (unlikely(!handshake)) + goto out; + + down_read(&handshake->lock); + state = handshake->state; + memcpy(hash, handshake->hash, NOISE_HASH_LEN); + memcpy(chaining_key, handshake->chaining_key, NOISE_HASH_LEN); + memcpy(ephemeral_private, handshake->ephemeral_private, + NOISE_PUBLIC_KEY_LEN); + up_read(&handshake->lock); + + if (state != HANDSHAKE_CREATED_INITIATION) + goto fail; + + /* e */ + message_ephemeral(e, src->unencrypted_ephemeral, chaining_key, hash); + + /* ee */ + if (!mix_dh(chaining_key, NULL, ephemeral_private, e)) + goto fail; + + /* se */ + if (!mix_dh(chaining_key, NULL, wg->static_identity.static_private, e)) + goto fail; + + /* psk */ + mix_psk(chaining_key, hash, key, handshake->preshared_key); + + /* {} */ + if (!message_decrypt(NULL, src->encrypted_nothing, + sizeof(src->encrypted_nothing), key, hash)) + goto fail; + + /* Success! Copy everything to peer */ + down_write(&handshake->lock); + /* It's important to check that the state is still the same, while we + * have an exclusive lock. + */ + if (handshake->state != state) { + up_write(&handshake->lock); + goto fail; + } + memcpy(handshake->remote_ephemeral, e, NOISE_PUBLIC_KEY_LEN); + memcpy(handshake->hash, hash, NOISE_HASH_LEN); + memcpy(handshake->chaining_key, chaining_key, NOISE_HASH_LEN); + handshake->remote_index = src->sender_index; + handshake->state = HANDSHAKE_CONSUMED_RESPONSE; + up_write(&handshake->lock); + ret_peer = peer; + goto out; + +fail: + wg_peer_put(peer); +out: + memzero_explicit(key, NOISE_SYMMETRIC_KEY_LEN); + memzero_explicit(hash, NOISE_HASH_LEN); + memzero_explicit(chaining_key, NOISE_HASH_LEN); + memzero_explicit(ephemeral_private, NOISE_PUBLIC_KEY_LEN); + memzero_explicit(static_private, NOISE_PUBLIC_KEY_LEN); + up_read(&wg->static_identity.lock); + return ret_peer; +} + +bool wg_noise_handshake_begin_session(struct noise_handshake *handshake, + struct noise_keypairs *keypairs) +{ + struct noise_keypair *new_keypair; + bool ret = false; + + down_write(&handshake->lock); + if (handshake->state != HANDSHAKE_CREATED_RESPONSE && + handshake->state != HANDSHAKE_CONSUMED_RESPONSE) + goto out; + + new_keypair = keypair_create(handshake->entry.peer); + if (!new_keypair) + goto out; + new_keypair->i_am_the_initiator = handshake->state == + HANDSHAKE_CONSUMED_RESPONSE; + new_keypair->remote_index = handshake->remote_index; + + if (new_keypair->i_am_the_initiator) + derive_keys(&new_keypair->sending, &new_keypair->receiving, + handshake->chaining_key); + else + derive_keys(&new_keypair->receiving, &new_keypair->sending, + handshake->chaining_key); + + handshake_zero(handshake); + rcu_read_lock_bh(); + if (likely(!READ_ONCE(container_of(handshake, struct wg_peer, + handshake)->is_dead))) { + add_new_keypair(keypairs, new_keypair); + net_dbg_ratelimited("%s: Keypair %llu created for peer %llu\n", + handshake->entry.peer->device->dev->name, + new_keypair->internal_id, + handshake->entry.peer->internal_id); + ret = wg_index_hashtable_replace( + handshake->entry.peer->device->index_hashtable, + &handshake->entry, &new_keypair->entry); + } else { + kzfree(new_keypair); + } + rcu_read_unlock_bh(); + +out: + up_write(&handshake->lock); + return ret; +} diff --git a/drivers/net/wireguard/noise.h b/drivers/net/wireguard/noise.h new file mode 100644 index 0000000000000..f532d59d3f19a --- /dev/null +++ b/drivers/net/wireguard/noise.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ +#ifndef _WG_NOISE_H +#define _WG_NOISE_H + +#include "messages.h" +#include "peerlookup.h" + +#include +#include +#include +#include +#include +#include + +union noise_counter { + struct { + u64 counter; + unsigned long backtrack[COUNTER_BITS_TOTAL / BITS_PER_LONG]; + spinlock_t lock; + } receive; + atomic64_t counter; +}; + +struct noise_symmetric_key { + u8 key[NOISE_SYMMETRIC_KEY_LEN]; + union noise_counter counter; + u64 birthdate; + bool is_valid; +}; + +struct noise_keypair { + struct index_hashtable_entry entry; + struct noise_symmetric_key sending; + struct noise_symmetric_key receiving; + __le32 remote_index; + bool i_am_the_initiator; + struct kref refcount; + struct rcu_head rcu; + u64 internal_id; +}; + +struct noise_keypairs { + struct noise_keypair __rcu *current_keypair; + struct noise_keypair __rcu *previous_keypair; + struct noise_keypair __rcu *next_keypair; + spinlock_t keypair_update_lock; +}; + +struct noise_static_identity { + u8 static_public[NOISE_PUBLIC_KEY_LEN]; + u8 static_private[NOISE_PUBLIC_KEY_LEN]; + struct rw_semaphore lock; + bool has_identity; +}; + +enum noise_handshake_state { + HANDSHAKE_ZEROED, + HANDSHAKE_CREATED_INITIATION, + HANDSHAKE_CONSUMED_INITIATION, + HANDSHAKE_CREATED_RESPONSE, + HANDSHAKE_CONSUMED_RESPONSE +}; + +struct noise_handshake { + struct index_hashtable_entry entry; + + enum noise_handshake_state state; + u64 last_initiation_consumption; + + struct noise_static_identity *static_identity; + + u8 ephemeral_private[NOISE_PUBLIC_KEY_LEN]; + u8 remote_static[NOISE_PUBLIC_KEY_LEN]; + u8 remote_ephemeral[NOISE_PUBLIC_KEY_LEN]; + u8 precomputed_static_static[NOISE_PUBLIC_KEY_LEN]; + + u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]; + + u8 hash[NOISE_HASH_LEN]; + u8 chaining_key[NOISE_HASH_LEN]; + + u8 latest_timestamp[NOISE_TIMESTAMP_LEN]; + __le32 remote_index; + + /* Protects all members except the immutable (after noise_handshake_ + * init): remote_static, precomputed_static_static, static_identity. + */ + struct rw_semaphore lock; +}; + +struct wg_device; + +void wg_noise_init(void); +void wg_noise_handshake_init(struct noise_handshake *handshake, + struct noise_static_identity *static_identity, + const u8 peer_public_key[NOISE_PUBLIC_KEY_LEN], + const u8 peer_preshared_key[NOISE_SYMMETRIC_KEY_LEN], + struct wg_peer *peer); +void wg_noise_handshake_clear(struct noise_handshake *handshake); +static inline void wg_noise_reset_last_sent_handshake(atomic64_t *handshake_ns) +{ + atomic64_set(handshake_ns, ktime_get_coarse_boottime_ns() - + (u64)(REKEY_TIMEOUT + 1) * NSEC_PER_SEC); +} + +void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now); +struct noise_keypair *wg_noise_keypair_get(struct noise_keypair *keypair); +void wg_noise_keypairs_clear(struct noise_keypairs *keypairs); +bool wg_noise_received_with_keypair(struct noise_keypairs *keypairs, + struct noise_keypair *received_keypair); +void wg_noise_expire_current_peer_keypairs(struct wg_peer *peer); + +void wg_noise_set_static_identity_private_key( + struct noise_static_identity *static_identity, + const u8 private_key[NOISE_PUBLIC_KEY_LEN]); +void wg_noise_precompute_static_static(struct wg_peer *peer); + +bool +wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst, + struct noise_handshake *handshake); +struct wg_peer * +wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src, + struct wg_device *wg); + +bool wg_noise_handshake_create_response(struct message_handshake_response *dst, + struct noise_handshake *handshake); +struct wg_peer * +wg_noise_handshake_consume_response(struct message_handshake_response *src, + struct wg_device *wg); + +bool wg_noise_handshake_begin_session(struct noise_handshake *handshake, + struct noise_keypairs *keypairs); + +#endif /* _WG_NOISE_H */ diff --git a/drivers/net/wireguard/peer.c b/drivers/net/wireguard/peer.c new file mode 100644 index 0000000000000..1d634bd3038f0 --- /dev/null +++ b/drivers/net/wireguard/peer.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "peer.h" +#include "device.h" +#include "queueing.h" +#include "timers.h" +#include "peerlookup.h" +#include "noise.h" + +#include +#include +#include +#include + +static atomic64_t peer_counter = ATOMIC64_INIT(0); + +struct wg_peer *wg_peer_create(struct wg_device *wg, + const u8 public_key[NOISE_PUBLIC_KEY_LEN], + const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]) +{ + struct wg_peer *peer; + int ret = -ENOMEM; + + lockdep_assert_held(&wg->device_update_lock); + + if (wg->num_peers >= MAX_PEERS_PER_DEVICE) + return ERR_PTR(ret); + + peer = kzalloc(sizeof(*peer), GFP_KERNEL); + if (unlikely(!peer)) + return ERR_PTR(ret); + peer->device = wg; + + wg_noise_handshake_init(&peer->handshake, &wg->static_identity, + public_key, preshared_key, peer); + if (dst_cache_init(&peer->endpoint_cache, GFP_KERNEL)) + goto err_1; + if (wg_packet_queue_init(&peer->tx_queue, wg_packet_tx_worker, false, + MAX_QUEUED_PACKETS)) + goto err_2; + if (wg_packet_queue_init(&peer->rx_queue, NULL, false, + MAX_QUEUED_PACKETS)) + goto err_3; + + peer->internal_id = atomic64_inc_return(&peer_counter); + peer->serial_work_cpu = nr_cpumask_bits; + wg_cookie_init(&peer->latest_cookie); + wg_timers_init(peer); + wg_cookie_checker_precompute_peer_keys(peer); + spin_lock_init(&peer->keypairs.keypair_update_lock); + INIT_WORK(&peer->transmit_handshake_work, + wg_packet_handshake_send_worker); + rwlock_init(&peer->endpoint_lock); + kref_init(&peer->refcount); + skb_queue_head_init(&peer->staged_packet_queue); + wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); + set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state); + netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll, + NAPI_POLL_WEIGHT); + napi_enable(&peer->napi); + list_add_tail(&peer->peer_list, &wg->peer_list); + INIT_LIST_HEAD(&peer->allowedips_list); + wg_pubkey_hashtable_add(wg->peer_hashtable, peer); + ++wg->num_peers; + pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id); + return peer; + +err_3: + wg_packet_queue_free(&peer->tx_queue, false); +err_2: + dst_cache_destroy(&peer->endpoint_cache); +err_1: + kfree(peer); + return ERR_PTR(ret); +} + +struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer) +{ + RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(), + "Taking peer reference without holding the RCU read lock"); + if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount))) + return NULL; + return peer; +} + +static void peer_make_dead(struct wg_peer *peer) +{ + /* Remove from configuration-time lookup structures. */ + list_del_init(&peer->peer_list); + wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer, + &peer->device->device_update_lock); + wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer); + + /* Mark as dead, so that we don't allow jumping contexts after. */ + WRITE_ONCE(peer->is_dead, true); + + /* The caller must now synchronize_rcu() for this to take effect. */ +} + +static void peer_remove_after_dead(struct wg_peer *peer) +{ + WARN_ON(!peer->is_dead); + + /* No more keypairs can be created for this peer, since is_dead protects + * add_new_keypair, so we can now destroy existing ones. + */ + wg_noise_keypairs_clear(&peer->keypairs); + + /* Destroy all ongoing timers that were in-flight at the beginning of + * this function. + */ + wg_timers_stop(peer); + + /* The transition between packet encryption/decryption queues isn't + * guarded by is_dead, but each reference's life is strictly bounded by + * two generations: once for parallel crypto and once for serial + * ingestion, so we can simply flush twice, and be sure that we no + * longer have references inside these queues. + */ + + /* a) For encrypt/decrypt. */ + flush_workqueue(peer->device->packet_crypt_wq); + /* b.1) For send (but not receive, since that's napi). */ + flush_workqueue(peer->device->packet_crypt_wq); + /* b.2.1) For receive (but not send, since that's wq). */ + napi_disable(&peer->napi); + /* b.2.1) It's now safe to remove the napi struct, which must be done + * here from process context. + */ + netif_napi_del(&peer->napi); + + /* Ensure any workstructs we own (like transmit_handshake_work or + * clear_peer_work) no longer are in use. + */ + flush_workqueue(peer->device->handshake_send_wq); + + /* After the above flushes, a peer might still be active in a few + * different contexts: 1) from xmit(), before hitting is_dead and + * returning, 2) from wg_packet_consume_data(), before hitting is_dead + * and returning, 3) from wg_receive_handshake_packet() after a point + * where it has processed an incoming handshake packet, but where + * all calls to pass it off to timers fails because of is_dead. We won't + * have new references in (1) eventually, because we're removed from + * allowedips; we won't have new references in (2) eventually, because + * wg_index_hashtable_lookup will always return NULL, since we removed + * all existing keypairs and no more can be created; we won't have new + * references in (3) eventually, because we're removed from the pubkey + * hash table, which allows for a maximum of one handshake response, + * via the still-uncleared index hashtable entry, but not more than one, + * and in wg_cookie_message_consume, the lookup eventually gets a peer + * with a refcount of zero, so no new reference is taken. + */ + + --peer->device->num_peers; + wg_peer_put(peer); +} + +/* We have a separate "remove" function make sure that all active places where + * a peer is currently operating will eventually come to an end and not pass + * their reference onto another context. + */ +void wg_peer_remove(struct wg_peer *peer) +{ + if (unlikely(!peer)) + return; + lockdep_assert_held(&peer->device->device_update_lock); + + peer_make_dead(peer); + synchronize_rcu(); + peer_remove_after_dead(peer); +} + +void wg_peer_remove_all(struct wg_device *wg) +{ + struct wg_peer *peer, *temp; + LIST_HEAD(dead_peers); + + lockdep_assert_held(&wg->device_update_lock); + + /* Avoid having to traverse individually for each one. */ + wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock); + + list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) { + peer_make_dead(peer); + list_add_tail(&peer->peer_list, &dead_peers); + } + synchronize_rcu(); + list_for_each_entry_safe(peer, temp, &dead_peers, peer_list) + peer_remove_after_dead(peer); +} + +static void rcu_release(struct rcu_head *rcu) +{ + struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu); + + dst_cache_destroy(&peer->endpoint_cache); + wg_packet_queue_free(&peer->rx_queue, false); + wg_packet_queue_free(&peer->tx_queue, false); + + /* The final zeroing takes care of clearing any remaining handshake key + * material and other potentially sensitive information. + */ + kzfree(peer); +} + +static void kref_release(struct kref *refcount) +{ + struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount); + + pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + + /* Remove ourself from dynamic runtime lookup structures, now that the + * last reference is gone. + */ + wg_index_hashtable_remove(peer->device->index_hashtable, + &peer->handshake.entry); + + /* Remove any lingering packets that didn't have a chance to be + * transmitted. + */ + wg_packet_purge_staged_packets(peer); + + /* Free the memory used. */ + call_rcu(&peer->rcu, rcu_release); +} + +void wg_peer_put(struct wg_peer *peer) +{ + if (unlikely(!peer)) + return; + kref_put(&peer->refcount, kref_release); +} diff --git a/drivers/net/wireguard/peer.h b/drivers/net/wireguard/peer.h new file mode 100644 index 0000000000000..23af409229972 --- /dev/null +++ b/drivers/net/wireguard/peer.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_PEER_H +#define _WG_PEER_H + +#include "device.h" +#include "noise.h" +#include "cookie.h" + +#include +#include +#include +#include +#include + +struct wg_device; + +struct endpoint { + union { + struct sockaddr addr; + struct sockaddr_in addr4; + struct sockaddr_in6 addr6; + }; + union { + struct { + struct in_addr src4; + /* Essentially the same as addr6->scope_id */ + int src_if4; + }; + struct in6_addr src6; + }; +}; + +struct wg_peer { + struct wg_device *device; + struct crypt_queue tx_queue, rx_queue; + struct sk_buff_head staged_packet_queue; + int serial_work_cpu; + struct noise_keypairs keypairs; + struct endpoint endpoint; + struct dst_cache endpoint_cache; + rwlock_t endpoint_lock; + struct noise_handshake handshake; + atomic64_t last_sent_handshake; + struct work_struct transmit_handshake_work, clear_peer_work; + struct cookie latest_cookie; + struct hlist_node pubkey_hash; + u64 rx_bytes, tx_bytes; + struct timer_list timer_retransmit_handshake, timer_send_keepalive; + struct timer_list timer_new_handshake, timer_zero_key_material; + struct timer_list timer_persistent_keepalive; + unsigned int timer_handshake_attempts; + u16 persistent_keepalive_interval; + bool timer_need_another_keepalive; + bool sent_lastminute_handshake; + struct timespec64 walltime_last_handshake; + struct kref refcount; + struct rcu_head rcu; + struct list_head peer_list; + struct list_head allowedips_list; + u64 internal_id; + struct napi_struct napi; + bool is_dead; +}; + +struct wg_peer *wg_peer_create(struct wg_device *wg, + const u8 public_key[NOISE_PUBLIC_KEY_LEN], + const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]); + +struct wg_peer *__must_check wg_peer_get_maybe_zero(struct wg_peer *peer); +static inline struct wg_peer *wg_peer_get(struct wg_peer *peer) +{ + kref_get(&peer->refcount); + return peer; +} +void wg_peer_put(struct wg_peer *peer); +void wg_peer_remove(struct wg_peer *peer); +void wg_peer_remove_all(struct wg_device *wg); + +#endif /* _WG_PEER_H */ diff --git a/drivers/net/wireguard/peerlookup.c b/drivers/net/wireguard/peerlookup.c new file mode 100644 index 0000000000000..e4deb331476b3 --- /dev/null +++ b/drivers/net/wireguard/peerlookup.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "peerlookup.h" +#include "peer.h" +#include "noise.h" + +static struct hlist_head *pubkey_bucket(struct pubkey_hashtable *table, + const u8 pubkey[NOISE_PUBLIC_KEY_LEN]) +{ + /* siphash gives us a secure 64bit number based on a random key. Since + * the bits are uniformly distributed, we can then mask off to get the + * bits we need. + */ + const u64 hash = siphash(pubkey, NOISE_PUBLIC_KEY_LEN, &table->key); + + return &table->hashtable[hash & (HASH_SIZE(table->hashtable) - 1)]; +} + +struct pubkey_hashtable *wg_pubkey_hashtable_alloc(void) +{ + struct pubkey_hashtable *table = kvmalloc(sizeof(*table), GFP_KERNEL); + + if (!table) + return NULL; + + get_random_bytes(&table->key, sizeof(table->key)); + hash_init(table->hashtable); + mutex_init(&table->lock); + return table; +} + +void wg_pubkey_hashtable_add(struct pubkey_hashtable *table, + struct wg_peer *peer) +{ + mutex_lock(&table->lock); + hlist_add_head_rcu(&peer->pubkey_hash, + pubkey_bucket(table, peer->handshake.remote_static)); + mutex_unlock(&table->lock); +} + +void wg_pubkey_hashtable_remove(struct pubkey_hashtable *table, + struct wg_peer *peer) +{ + mutex_lock(&table->lock); + hlist_del_init_rcu(&peer->pubkey_hash); + mutex_unlock(&table->lock); +} + +/* Returns a strong reference to a peer */ +struct wg_peer * +wg_pubkey_hashtable_lookup(struct pubkey_hashtable *table, + const u8 pubkey[NOISE_PUBLIC_KEY_LEN]) +{ + struct wg_peer *iter_peer, *peer = NULL; + + rcu_read_lock_bh(); + hlist_for_each_entry_rcu_bh(iter_peer, pubkey_bucket(table, pubkey), + pubkey_hash) { + if (!memcmp(pubkey, iter_peer->handshake.remote_static, + NOISE_PUBLIC_KEY_LEN)) { + peer = iter_peer; + break; + } + } + peer = wg_peer_get_maybe_zero(peer); + rcu_read_unlock_bh(); + return peer; +} + +static struct hlist_head *index_bucket(struct index_hashtable *table, + const __le32 index) +{ + /* Since the indices are random and thus all bits are uniformly + * distributed, we can find its bucket simply by masking. + */ + return &table->hashtable[(__force u32)index & + (HASH_SIZE(table->hashtable) - 1)]; +} + +struct index_hashtable *wg_index_hashtable_alloc(void) +{ + struct index_hashtable *table = kvmalloc(sizeof(*table), GFP_KERNEL); + + if (!table) + return NULL; + + hash_init(table->hashtable); + spin_lock_init(&table->lock); + return table; +} + +/* At the moment, we limit ourselves to 2^20 total peers, which generally might + * amount to 2^20*3 items in this hashtable. The algorithm below works by + * picking a random number and testing it. We can see that these limits mean we + * usually succeed pretty quickly: + * + * >>> def calculation(tries, size): + * ... return (size / 2**32)**(tries - 1) * (1 - (size / 2**32)) + * ... + * >>> calculation(1, 2**20 * 3) + * 0.999267578125 + * >>> calculation(2, 2**20 * 3) + * 0.0007318854331970215 + * >>> calculation(3, 2**20 * 3) + * 5.360489012673497e-07 + * >>> calculation(4, 2**20 * 3) + * 3.9261394135792216e-10 + * + * At the moment, we don't do any masking, so this algorithm isn't exactly + * constant time in either the random guessing or in the hash list lookup. We + * could require a minimum of 3 tries, which would successfully mask the + * guessing. this would not, however, help with the growing hash lengths, which + * is another thing to consider moving forward. + */ + +__le32 wg_index_hashtable_insert(struct index_hashtable *table, + struct index_hashtable_entry *entry) +{ + struct index_hashtable_entry *existing_entry; + + spin_lock_bh(&table->lock); + hlist_del_init_rcu(&entry->index_hash); + spin_unlock_bh(&table->lock); + + rcu_read_lock_bh(); + +search_unused_slot: + /* First we try to find an unused slot, randomly, while unlocked. */ + entry->index = (__force __le32)get_random_u32(); + hlist_for_each_entry_rcu_bh(existing_entry, + index_bucket(table, entry->index), + index_hash) { + if (existing_entry->index == entry->index) + /* If it's already in use, we continue searching. */ + goto search_unused_slot; + } + + /* Once we've found an unused slot, we lock it, and then double-check + * that nobody else stole it from us. + */ + spin_lock_bh(&table->lock); + hlist_for_each_entry_rcu_bh(existing_entry, + index_bucket(table, entry->index), + index_hash) { + if (existing_entry->index == entry->index) { + spin_unlock_bh(&table->lock); + /* If it was stolen, we start over. */ + goto search_unused_slot; + } + } + /* Otherwise, we know we have it exclusively (since we're locked), + * so we insert. + */ + hlist_add_head_rcu(&entry->index_hash, + index_bucket(table, entry->index)); + spin_unlock_bh(&table->lock); + + rcu_read_unlock_bh(); + + return entry->index; +} + +bool wg_index_hashtable_replace(struct index_hashtable *table, + struct index_hashtable_entry *old, + struct index_hashtable_entry *new) +{ + if (unlikely(hlist_unhashed(&old->index_hash))) + return false; + spin_lock_bh(&table->lock); + new->index = old->index; + hlist_replace_rcu(&old->index_hash, &new->index_hash); + + /* Calling init here NULLs out index_hash, and in fact after this + * function returns, it's theoretically possible for this to get + * reinserted elsewhere. That means the RCU lookup below might either + * terminate early or jump between buckets, in which case the packet + * simply gets dropped, which isn't terrible. + */ + INIT_HLIST_NODE(&old->index_hash); + spin_unlock_bh(&table->lock); + return true; +} + +void wg_index_hashtable_remove(struct index_hashtable *table, + struct index_hashtable_entry *entry) +{ + spin_lock_bh(&table->lock); + hlist_del_init_rcu(&entry->index_hash); + spin_unlock_bh(&table->lock); +} + +/* Returns a strong reference to a entry->peer */ +struct index_hashtable_entry * +wg_index_hashtable_lookup(struct index_hashtable *table, + const enum index_hashtable_type type_mask, + const __le32 index, struct wg_peer **peer) +{ + struct index_hashtable_entry *iter_entry, *entry = NULL; + + rcu_read_lock_bh(); + hlist_for_each_entry_rcu_bh(iter_entry, index_bucket(table, index), + index_hash) { + if (iter_entry->index == index) { + if (likely(iter_entry->type & type_mask)) + entry = iter_entry; + break; + } + } + if (likely(entry)) { + entry->peer = wg_peer_get_maybe_zero(entry->peer); + if (likely(entry->peer)) + *peer = entry->peer; + else + entry = NULL; + } + rcu_read_unlock_bh(); + return entry; +} diff --git a/drivers/net/wireguard/peerlookup.h b/drivers/net/wireguard/peerlookup.h new file mode 100644 index 0000000000000..ced8117976800 --- /dev/null +++ b/drivers/net/wireguard/peerlookup.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_PEERLOOKUP_H +#define _WG_PEERLOOKUP_H + +#include "messages.h" + +#include +#include +#include + +struct wg_peer; + +struct pubkey_hashtable { + /* TODO: move to rhashtable */ + DECLARE_HASHTABLE(hashtable, 11); + siphash_key_t key; + struct mutex lock; +}; + +struct pubkey_hashtable *wg_pubkey_hashtable_alloc(void); +void wg_pubkey_hashtable_add(struct pubkey_hashtable *table, + struct wg_peer *peer); +void wg_pubkey_hashtable_remove(struct pubkey_hashtable *table, + struct wg_peer *peer); +struct wg_peer * +wg_pubkey_hashtable_lookup(struct pubkey_hashtable *table, + const u8 pubkey[NOISE_PUBLIC_KEY_LEN]); + +struct index_hashtable { + /* TODO: move to rhashtable */ + DECLARE_HASHTABLE(hashtable, 13); + spinlock_t lock; +}; + +enum index_hashtable_type { + INDEX_HASHTABLE_HANDSHAKE = 1U << 0, + INDEX_HASHTABLE_KEYPAIR = 1U << 1 +}; + +struct index_hashtable_entry { + struct wg_peer *peer; + struct hlist_node index_hash; + enum index_hashtable_type type; + __le32 index; +}; + +struct index_hashtable *wg_index_hashtable_alloc(void); +__le32 wg_index_hashtable_insert(struct index_hashtable *table, + struct index_hashtable_entry *entry); +bool wg_index_hashtable_replace(struct index_hashtable *table, + struct index_hashtable_entry *old, + struct index_hashtable_entry *new); +void wg_index_hashtable_remove(struct index_hashtable *table, + struct index_hashtable_entry *entry); +struct index_hashtable_entry * +wg_index_hashtable_lookup(struct index_hashtable *table, + const enum index_hashtable_type type_mask, + const __le32 index, struct wg_peer **peer); + +#endif /* _WG_PEERLOOKUP_H */ diff --git a/drivers/net/wireguard/queueing.c b/drivers/net/wireguard/queueing.c new file mode 100644 index 0000000000000..5c964fcb994ec --- /dev/null +++ b/drivers/net/wireguard/queueing.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" + +struct multicore_worker __percpu * +wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr) +{ + int cpu; + struct multicore_worker __percpu *worker = + alloc_percpu(struct multicore_worker); + + if (!worker) + return NULL; + + for_each_possible_cpu(cpu) { + per_cpu_ptr(worker, cpu)->ptr = ptr; + INIT_WORK(&per_cpu_ptr(worker, cpu)->work, function); + } + return worker; +} + +int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, + bool multicore, unsigned int len) +{ + int ret; + + memset(queue, 0, sizeof(*queue)); + ret = ptr_ring_init(&queue->ring, len, GFP_KERNEL); + if (ret) + return ret; + if (function) { + if (multicore) { + queue->worker = wg_packet_percpu_multicore_worker_alloc( + function, queue); + if (!queue->worker) + return -ENOMEM; + } else { + INIT_WORK(&queue->work, function); + } + } + return 0; +} + +void wg_packet_queue_free(struct crypt_queue *queue, bool multicore) +{ + if (multicore) + free_percpu(queue->worker); + WARN_ON(!__ptr_ring_empty(&queue->ring)); + ptr_ring_cleanup(&queue->ring, NULL); +} diff --git a/drivers/net/wireguard/queueing.h b/drivers/net/wireguard/queueing.h new file mode 100644 index 0000000000000..cf1e0e2376d81 --- /dev/null +++ b/drivers/net/wireguard/queueing.h @@ -0,0 +1,200 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_QUEUEING_H +#define _WG_QUEUEING_H + +#include "peer.h" +#include +#include +#include +#include + +struct wg_device; +struct wg_peer; +struct multicore_worker; +struct crypt_queue; +struct sk_buff; + +/* queueing.c APIs: */ +int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function, + bool multicore, unsigned int len); +void wg_packet_queue_free(struct crypt_queue *queue, bool multicore); +struct multicore_worker __percpu * +wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr); + +/* receive.c APIs: */ +void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb); +void wg_packet_handshake_receive_worker(struct work_struct *work); +/* NAPI poll function: */ +int wg_packet_rx_poll(struct napi_struct *napi, int budget); +/* Workqueue worker: */ +void wg_packet_decrypt_worker(struct work_struct *work); + +/* send.c APIs: */ +void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, + bool is_retry); +void wg_packet_send_handshake_response(struct wg_peer *peer); +void wg_packet_send_handshake_cookie(struct wg_device *wg, + struct sk_buff *initiating_skb, + __le32 sender_index); +void wg_packet_send_keepalive(struct wg_peer *peer); +void wg_packet_purge_staged_packets(struct wg_peer *peer); +void wg_packet_send_staged_packets(struct wg_peer *peer); +/* Workqueue workers: */ +void wg_packet_handshake_send_worker(struct work_struct *work); +void wg_packet_tx_worker(struct work_struct *work); +void wg_packet_encrypt_worker(struct work_struct *work); + +enum packet_state { + PACKET_STATE_UNCRYPTED, + PACKET_STATE_CRYPTED, + PACKET_STATE_DEAD +}; + +struct packet_cb { + u64 nonce; + struct noise_keypair *keypair; + atomic_t state; + u32 mtu; + u8 ds; +}; + +#define PACKET_CB(skb) ((struct packet_cb *)((skb)->cb)) +#define PACKET_PEER(skb) (PACKET_CB(skb)->keypair->entry.peer) + +/* Returns either the correct skb->protocol value, or 0 if invalid. */ +static inline __be16 wg_examine_packet_protocol(struct sk_buff *skb) +{ + if (skb_network_header(skb) >= skb->head && + (skb_network_header(skb) + sizeof(struct iphdr)) <= + skb_tail_pointer(skb) && + ip_hdr(skb)->version == 4) + return htons(ETH_P_IP); + if (skb_network_header(skb) >= skb->head && + (skb_network_header(skb) + sizeof(struct ipv6hdr)) <= + skb_tail_pointer(skb) && + ipv6_hdr(skb)->version == 6) + return htons(ETH_P_IPV6); + return 0; +} + +static inline bool wg_check_packet_protocol(struct sk_buff *skb) +{ + __be16 real_protocol = wg_examine_packet_protocol(skb); + return real_protocol && skb->protocol == real_protocol; +} + +static inline void wg_reset_packet(struct sk_buff *skb) +{ + skb_scrub_packet(skb, true); + memset(&skb->headers_start, 0, + offsetof(struct sk_buff, headers_end) - + offsetof(struct sk_buff, headers_start)); + skb->queue_mapping = 0; + skb->nohdr = 0; + skb->peeked = 0; + skb->mac_len = 0; + skb->dev = NULL; +#ifdef CONFIG_NET_SCHED + skb->tc_index = 0; + skb_reset_tc(skb); +#endif + skb->hdr_len = skb_headroom(skb); + skb_reset_mac_header(skb); + skb_reset_network_header(skb); + skb_reset_transport_header(skb); + skb_probe_transport_header(skb); + skb_reset_inner_headers(skb); +} + +static inline int wg_cpumask_choose_online(int *stored_cpu, unsigned int id) +{ + unsigned int cpu = *stored_cpu, cpu_index, i; + + if (unlikely(cpu == nr_cpumask_bits || + !cpumask_test_cpu(cpu, cpu_online_mask))) { + cpu_index = id % cpumask_weight(cpu_online_mask); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < cpu_index; ++i) + cpu = cpumask_next(cpu, cpu_online_mask); + *stored_cpu = cpu; + } + return cpu; +} + +/* This function is racy, in the sense that next is unlocked, so it could return + * the same CPU twice. A race-free version of this would be to instead store an + * atomic sequence number, do an increment-and-return, and then iterate through + * every possible CPU until we get to that index -- choose_cpu. However that's + * a bit slower, and it doesn't seem like this potential race actually + * introduces any performance loss, so we live with it. + */ +static inline int wg_cpumask_next_online(int *next) +{ + int cpu = *next; + + while (unlikely(!cpumask_test_cpu(cpu, cpu_online_mask))) + cpu = cpumask_next(cpu, cpu_online_mask) % nr_cpumask_bits; + *next = cpumask_next(cpu, cpu_online_mask) % nr_cpumask_bits; + return cpu; +} + +static inline int wg_queue_enqueue_per_device_and_peer( + struct crypt_queue *device_queue, struct crypt_queue *peer_queue, + struct sk_buff *skb, struct workqueue_struct *wq, int *next_cpu) +{ + int cpu; + + atomic_set_release(&PACKET_CB(skb)->state, PACKET_STATE_UNCRYPTED); + /* We first queue this up for the peer ingestion, but the consumer + * will wait for the state to change to CRYPTED or DEAD before. + */ + if (unlikely(ptr_ring_produce_bh(&peer_queue->ring, skb))) + return -ENOSPC; + /* Then we queue it up in the device queue, which consumes the + * packet as soon as it can. + */ + cpu = wg_cpumask_next_online(next_cpu); + if (unlikely(ptr_ring_produce_bh(&device_queue->ring, skb))) + return -EPIPE; + queue_work_on(cpu, wq, &per_cpu_ptr(device_queue->worker, cpu)->work); + return 0; +} + +static inline void wg_queue_enqueue_per_peer(struct crypt_queue *queue, + struct sk_buff *skb, + enum packet_state state) +{ + /* We take a reference, because as soon as we call atomic_set, the + * peer can be freed from below us. + */ + struct wg_peer *peer = wg_peer_get(PACKET_PEER(skb)); + + atomic_set_release(&PACKET_CB(skb)->state, state); + queue_work_on(wg_cpumask_choose_online(&peer->serial_work_cpu, + peer->internal_id), + peer->device->packet_crypt_wq, &queue->work); + wg_peer_put(peer); +} + +static inline void wg_queue_enqueue_per_peer_napi(struct sk_buff *skb, + enum packet_state state) +{ + /* We take a reference, because as soon as we call atomic_set, the + * peer can be freed from below us. + */ + struct wg_peer *peer = wg_peer_get(PACKET_PEER(skb)); + + atomic_set_release(&PACKET_CB(skb)->state, state); + napi_schedule(&peer->napi); + wg_peer_put(peer); +} + +#ifdef DEBUG +bool wg_packet_counter_selftest(void); +#endif + +#endif /* _WG_QUEUEING_H */ diff --git a/drivers/net/wireguard/ratelimiter.c b/drivers/net/wireguard/ratelimiter.c new file mode 100644 index 0000000000000..3fedd1d21f5ee --- /dev/null +++ b/drivers/net/wireguard/ratelimiter.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "ratelimiter.h" +#include +#include +#include +#include + +static struct kmem_cache *entry_cache; +static hsiphash_key_t key; +static spinlock_t table_lock = __SPIN_LOCK_UNLOCKED("ratelimiter_table_lock"); +static DEFINE_MUTEX(init_lock); +static u64 init_refcnt; /* Protected by init_lock, hence not atomic. */ +static atomic_t total_entries = ATOMIC_INIT(0); +static unsigned int max_entries, table_size; +static void wg_ratelimiter_gc_entries(struct work_struct *); +static DECLARE_DEFERRABLE_WORK(gc_work, wg_ratelimiter_gc_entries); +static struct hlist_head *table_v4; +#if IS_ENABLED(CONFIG_IPV6) +static struct hlist_head *table_v6; +#endif + +struct ratelimiter_entry { + u64 last_time_ns, tokens, ip; + void *net; + spinlock_t lock; + struct hlist_node hash; + struct rcu_head rcu; +}; + +enum { + PACKETS_PER_SECOND = 20, + PACKETS_BURSTABLE = 5, + PACKET_COST = NSEC_PER_SEC / PACKETS_PER_SECOND, + TOKEN_MAX = PACKET_COST * PACKETS_BURSTABLE +}; + +static void entry_free(struct rcu_head *rcu) +{ + kmem_cache_free(entry_cache, + container_of(rcu, struct ratelimiter_entry, rcu)); + atomic_dec(&total_entries); +} + +static void entry_uninit(struct ratelimiter_entry *entry) +{ + hlist_del_rcu(&entry->hash); + call_rcu(&entry->rcu, entry_free); +} + +/* Calling this function with a NULL work uninits all entries. */ +static void wg_ratelimiter_gc_entries(struct work_struct *work) +{ + const u64 now = ktime_get_coarse_boottime_ns(); + struct ratelimiter_entry *entry; + struct hlist_node *temp; + unsigned int i; + + for (i = 0; i < table_size; ++i) { + spin_lock(&table_lock); + hlist_for_each_entry_safe(entry, temp, &table_v4[i], hash) { + if (unlikely(!work) || + now - entry->last_time_ns > NSEC_PER_SEC) + entry_uninit(entry); + } +#if IS_ENABLED(CONFIG_IPV6) + hlist_for_each_entry_safe(entry, temp, &table_v6[i], hash) { + if (unlikely(!work) || + now - entry->last_time_ns > NSEC_PER_SEC) + entry_uninit(entry); + } +#endif + spin_unlock(&table_lock); + if (likely(work)) + cond_resched(); + } + if (likely(work)) + queue_delayed_work(system_power_efficient_wq, &gc_work, HZ); +} + +bool wg_ratelimiter_allow(struct sk_buff *skb, struct net *net) +{ + /* We only take the bottom half of the net pointer, so that we can hash + * 3 words in the end. This way, siphash's len param fits into the final + * u32, and we don't incur an extra round. + */ + const u32 net_word = (unsigned long)net; + struct ratelimiter_entry *entry; + struct hlist_head *bucket; + u64 ip; + + if (skb->protocol == htons(ETH_P_IP)) { + ip = (u64 __force)ip_hdr(skb)->saddr; + bucket = &table_v4[hsiphash_2u32(net_word, ip, &key) & + (table_size - 1)]; + } +#if IS_ENABLED(CONFIG_IPV6) + else if (skb->protocol == htons(ETH_P_IPV6)) { + /* Only use 64 bits, so as to ratelimit the whole /64. */ + memcpy(&ip, &ipv6_hdr(skb)->saddr, sizeof(ip)); + bucket = &table_v6[hsiphash_3u32(net_word, ip >> 32, ip, &key) & + (table_size - 1)]; + } +#endif + else + return false; + rcu_read_lock(); + hlist_for_each_entry_rcu(entry, bucket, hash) { + if (entry->net == net && entry->ip == ip) { + u64 now, tokens; + bool ret; + /* Quasi-inspired by nft_limit.c, but this is actually a + * slightly different algorithm. Namely, we incorporate + * the burst as part of the maximum tokens, rather than + * as part of the rate. + */ + spin_lock(&entry->lock); + now = ktime_get_coarse_boottime_ns(); + tokens = min_t(u64, TOKEN_MAX, + entry->tokens + now - + entry->last_time_ns); + entry->last_time_ns = now; + ret = tokens >= PACKET_COST; + entry->tokens = ret ? tokens - PACKET_COST : tokens; + spin_unlock(&entry->lock); + rcu_read_unlock(); + return ret; + } + } + rcu_read_unlock(); + + if (atomic_inc_return(&total_entries) > max_entries) + goto err_oom; + + entry = kmem_cache_alloc(entry_cache, GFP_KERNEL); + if (unlikely(!entry)) + goto err_oom; + + entry->net = net; + entry->ip = ip; + INIT_HLIST_NODE(&entry->hash); + spin_lock_init(&entry->lock); + entry->last_time_ns = ktime_get_coarse_boottime_ns(); + entry->tokens = TOKEN_MAX - PACKET_COST; + spin_lock(&table_lock); + hlist_add_head_rcu(&entry->hash, bucket); + spin_unlock(&table_lock); + return true; + +err_oom: + atomic_dec(&total_entries); + return false; +} + +int wg_ratelimiter_init(void) +{ + mutex_lock(&init_lock); + if (++init_refcnt != 1) + goto out; + + entry_cache = KMEM_CACHE(ratelimiter_entry, 0); + if (!entry_cache) + goto err; + + /* xt_hashlimit.c uses a slightly different algorithm for ratelimiting, + * but what it shares in common is that it uses a massive hashtable. So, + * we borrow their wisdom about good table sizes on different systems + * dependent on RAM. This calculation here comes from there. + */ + table_size = (totalram_pages() > (1U << 30) / PAGE_SIZE) ? 8192 : + max_t(unsigned long, 16, roundup_pow_of_two( + (totalram_pages() << PAGE_SHIFT) / + (1U << 14) / sizeof(struct hlist_head))); + max_entries = table_size * 8; + + table_v4 = kvzalloc(table_size * sizeof(*table_v4), GFP_KERNEL); + if (unlikely(!table_v4)) + goto err_kmemcache; + +#if IS_ENABLED(CONFIG_IPV6) + table_v6 = kvzalloc(table_size * sizeof(*table_v6), GFP_KERNEL); + if (unlikely(!table_v6)) { + kvfree(table_v4); + goto err_kmemcache; + } +#endif + + queue_delayed_work(system_power_efficient_wq, &gc_work, HZ); + get_random_bytes(&key, sizeof(key)); +out: + mutex_unlock(&init_lock); + return 0; + +err_kmemcache: + kmem_cache_destroy(entry_cache); +err: + --init_refcnt; + mutex_unlock(&init_lock); + return -ENOMEM; +} + +void wg_ratelimiter_uninit(void) +{ + mutex_lock(&init_lock); + if (!init_refcnt || --init_refcnt) + goto out; + + cancel_delayed_work_sync(&gc_work); + wg_ratelimiter_gc_entries(NULL); + rcu_barrier(); + kvfree(table_v4); +#if IS_ENABLED(CONFIG_IPV6) + kvfree(table_v6); +#endif + kmem_cache_destroy(entry_cache); +out: + mutex_unlock(&init_lock); +} + +#include "selftest/ratelimiter.c" diff --git a/drivers/net/wireguard/ratelimiter.h b/drivers/net/wireguard/ratelimiter.h new file mode 100644 index 0000000000000..83067f71ea99e --- /dev/null +++ b/drivers/net/wireguard/ratelimiter.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_RATELIMITER_H +#define _WG_RATELIMITER_H + +#include + +int wg_ratelimiter_init(void); +void wg_ratelimiter_uninit(void); +bool wg_ratelimiter_allow(struct sk_buff *skb, struct net *net); + +#ifdef DEBUG +bool wg_ratelimiter_selftest(void); +#endif + +#endif /* _WG_RATELIMITER_H */ diff --git a/drivers/net/wireguard/receive.c b/drivers/net/wireguard/receive.c new file mode 100644 index 0000000000000..da3b782ab7d31 --- /dev/null +++ b/drivers/net/wireguard/receive.c @@ -0,0 +1,597 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" +#include "device.h" +#include "peer.h" +#include "timers.h" +#include "messages.h" +#include "cookie.h" +#include "socket.h" + +#include +#include +#include +#include + +/* Must be called with bh disabled. */ +static void update_rx_stats(struct wg_peer *peer, size_t len) +{ + struct pcpu_sw_netstats *tstats = + get_cpu_ptr(peer->device->dev->tstats); + + u64_stats_update_begin(&tstats->syncp); + ++tstats->rx_packets; + tstats->rx_bytes += len; + peer->rx_bytes += len; + u64_stats_update_end(&tstats->syncp); + put_cpu_ptr(tstats); +} + +#define SKB_TYPE_LE32(skb) (((struct message_header *)(skb)->data)->type) + +static size_t validate_header_len(struct sk_buff *skb) +{ + if (unlikely(skb->len < sizeof(struct message_header))) + return 0; + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_DATA) && + skb->len >= MESSAGE_MINIMUM_LENGTH) + return sizeof(struct message_data); + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION) && + skb->len == sizeof(struct message_handshake_initiation)) + return sizeof(struct message_handshake_initiation); + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) && + skb->len == sizeof(struct message_handshake_response)) + return sizeof(struct message_handshake_response); + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE) && + skb->len == sizeof(struct message_handshake_cookie)) + return sizeof(struct message_handshake_cookie); + return 0; +} + +static int prepare_skb_header(struct sk_buff *skb, struct wg_device *wg) +{ + size_t data_offset, data_len, header_len; + struct udphdr *udp; + + if (unlikely(!wg_check_packet_protocol(skb) || + skb_transport_header(skb) < skb->head || + (skb_transport_header(skb) + sizeof(struct udphdr)) > + skb_tail_pointer(skb))) + return -EINVAL; /* Bogus IP header */ + udp = udp_hdr(skb); + data_offset = (u8 *)udp - skb->data; + if (unlikely(data_offset > U16_MAX || + data_offset + sizeof(struct udphdr) > skb->len)) + /* Packet has offset at impossible location or isn't big enough + * to have UDP fields. + */ + return -EINVAL; + data_len = ntohs(udp->len); + if (unlikely(data_len < sizeof(struct udphdr) || + data_len > skb->len - data_offset)) + /* UDP packet is reporting too small of a size or lying about + * its size. + */ + return -EINVAL; + data_len -= sizeof(struct udphdr); + data_offset = (u8 *)udp + sizeof(struct udphdr) - skb->data; + if (unlikely(!pskb_may_pull(skb, + data_offset + sizeof(struct message_header)) || + pskb_trim(skb, data_len + data_offset) < 0)) + return -EINVAL; + skb_pull(skb, data_offset); + if (unlikely(skb->len != data_len)) + /* Final len does not agree with calculated len */ + return -EINVAL; + header_len = validate_header_len(skb); + if (unlikely(!header_len)) + return -EINVAL; + __skb_push(skb, data_offset); + if (unlikely(!pskb_may_pull(skb, data_offset + header_len))) + return -EINVAL; + __skb_pull(skb, data_offset); + return 0; +} + +static void wg_receive_handshake_packet(struct wg_device *wg, + struct sk_buff *skb) +{ + enum cookie_mac_state mac_state; + struct wg_peer *peer = NULL; + /* This is global, so that our load calculation applies to the whole + * system. We don't care about races with it at all. + */ + static u64 last_under_load; + bool packet_needs_cookie; + bool under_load; + + if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE)) { + net_dbg_skb_ratelimited("%s: Receiving cookie response from %pISpfsc\n", + wg->dev->name, skb); + wg_cookie_message_consume( + (struct message_handshake_cookie *)skb->data, wg); + return; + } + + under_load = skb_queue_len(&wg->incoming_handshakes) >= + MAX_QUEUED_INCOMING_HANDSHAKES / 8; + if (under_load) { + last_under_load = ktime_get_coarse_boottime_ns(); + } else if (last_under_load) { + under_load = !wg_birthdate_has_expired(last_under_load, 1); + if (!under_load) + last_under_load = 0; + } + mac_state = wg_cookie_validate_packet(&wg->cookie_checker, skb, + under_load); + if ((under_load && mac_state == VALID_MAC_WITH_COOKIE) || + (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE)) { + packet_needs_cookie = false; + } else if (under_load && mac_state == VALID_MAC_BUT_NO_COOKIE) { + packet_needs_cookie = true; + } else { + net_dbg_skb_ratelimited("%s: Invalid MAC of handshake, dropping packet from %pISpfsc\n", + wg->dev->name, skb); + return; + } + + switch (SKB_TYPE_LE32(skb)) { + case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): { + struct message_handshake_initiation *message = + (struct message_handshake_initiation *)skb->data; + + if (packet_needs_cookie) { + wg_packet_send_handshake_cookie(wg, skb, + message->sender_index); + return; + } + peer = wg_noise_handshake_consume_initiation(message, wg); + if (unlikely(!peer)) { + net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n", + wg->dev->name, skb); + return; + } + wg_socket_set_peer_endpoint_from_skb(peer, skb); + net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n", + wg->dev->name, peer->internal_id, + &peer->endpoint.addr); + wg_packet_send_handshake_response(peer); + break; + } + case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): { + struct message_handshake_response *message = + (struct message_handshake_response *)skb->data; + + if (packet_needs_cookie) { + wg_packet_send_handshake_cookie(wg, skb, + message->sender_index); + return; + } + peer = wg_noise_handshake_consume_response(message, wg); + if (unlikely(!peer)) { + net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n", + wg->dev->name, skb); + return; + } + wg_socket_set_peer_endpoint_from_skb(peer, skb); + net_dbg_ratelimited("%s: Receiving handshake response from peer %llu (%pISpfsc)\n", + wg->dev->name, peer->internal_id, + &peer->endpoint.addr); + if (wg_noise_handshake_begin_session(&peer->handshake, + &peer->keypairs)) { + wg_timers_session_derived(peer); + wg_timers_handshake_complete(peer); + /* Calling this function will either send any existing + * packets in the queue and not send a keepalive, which + * is the best case, Or, if there's nothing in the + * queue, it will send a keepalive, in order to give + * immediate confirmation of the session. + */ + wg_packet_send_keepalive(peer); + } + break; + } + } + + if (unlikely(!peer)) { + WARN(1, "Somehow a wrong type of packet wound up in the handshake queue!\n"); + return; + } + + local_bh_disable(); + update_rx_stats(peer, skb->len); + local_bh_enable(); + + wg_timers_any_authenticated_packet_received(peer); + wg_timers_any_authenticated_packet_traversal(peer); + wg_peer_put(peer); +} + +void wg_packet_handshake_receive_worker(struct work_struct *work) +{ + struct wg_device *wg = container_of(work, struct multicore_worker, + work)->ptr; + struct sk_buff *skb; + + while ((skb = skb_dequeue(&wg->incoming_handshakes)) != NULL) { + wg_receive_handshake_packet(wg, skb); + dev_kfree_skb(skb); + cond_resched(); + } +} + +static void keep_key_fresh(struct wg_peer *peer) +{ + struct noise_keypair *keypair; + bool send = false; + + if (peer->sent_lastminute_handshake) + return; + + rcu_read_lock_bh(); + keypair = rcu_dereference_bh(peer->keypairs.current_keypair); + if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) && + keypair->i_am_the_initiator && + unlikely(wg_birthdate_has_expired(keypair->sending.birthdate, + REJECT_AFTER_TIME - KEEPALIVE_TIMEOUT - REKEY_TIMEOUT))) + send = true; + rcu_read_unlock_bh(); + + if (send) { + peer->sent_lastminute_handshake = true; + wg_packet_send_queued_handshake_initiation(peer, false); + } +} + +static bool decrypt_packet(struct sk_buff *skb, struct noise_symmetric_key *key) +{ + struct scatterlist sg[MAX_SKB_FRAGS + 8]; + struct sk_buff *trailer; + unsigned int offset; + int num_frags; + + if (unlikely(!key)) + return false; + + if (unlikely(!READ_ONCE(key->is_valid) || + wg_birthdate_has_expired(key->birthdate, REJECT_AFTER_TIME) || + key->counter.receive.counter >= REJECT_AFTER_MESSAGES)) { + WRITE_ONCE(key->is_valid, false); + return false; + } + + PACKET_CB(skb)->nonce = + le64_to_cpu(((struct message_data *)skb->data)->counter); + + /* We ensure that the network header is part of the packet before we + * call skb_cow_data, so that there's no chance that data is removed + * from the skb, so that later we can extract the original endpoint. + */ + offset = skb->data - skb_network_header(skb); + skb_push(skb, offset); + num_frags = skb_cow_data(skb, 0, &trailer); + offset += sizeof(struct message_data); + skb_pull(skb, offset); + if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) + return false; + + sg_init_table(sg, num_frags); + if (skb_to_sgvec(skb, sg, 0, skb->len) <= 0) + return false; + + if (!chacha20poly1305_decrypt_sg_inplace(sg, skb->len, NULL, 0, + PACKET_CB(skb)->nonce, + key->key)) + return false; + + /* Another ugly situation of pushing and pulling the header so as to + * keep endpoint information intact. + */ + skb_push(skb, offset); + if (pskb_trim(skb, skb->len - noise_encrypted_len(0))) + return false; + skb_pull(skb, offset); + + return true; +} + +/* This is RFC6479, a replay detection bitmap algorithm that avoids bitshifts */ +static bool counter_validate(union noise_counter *counter, u64 their_counter) +{ + unsigned long index, index_current, top, i; + bool ret = false; + + spin_lock_bh(&counter->receive.lock); + + if (unlikely(counter->receive.counter >= REJECT_AFTER_MESSAGES + 1 || + their_counter >= REJECT_AFTER_MESSAGES)) + goto out; + + ++their_counter; + + if (unlikely((COUNTER_WINDOW_SIZE + their_counter) < + counter->receive.counter)) + goto out; + + index = their_counter >> ilog2(BITS_PER_LONG); + + if (likely(their_counter > counter->receive.counter)) { + index_current = counter->receive.counter >> ilog2(BITS_PER_LONG); + top = min_t(unsigned long, index - index_current, + COUNTER_BITS_TOTAL / BITS_PER_LONG); + for (i = 1; i <= top; ++i) + counter->receive.backtrack[(i + index_current) & + ((COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1)] = 0; + counter->receive.counter = their_counter; + } + + index &= (COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1; + ret = !test_and_set_bit(their_counter & (BITS_PER_LONG - 1), + &counter->receive.backtrack[index]); + +out: + spin_unlock_bh(&counter->receive.lock); + return ret; +} + +#include "selftest/counter.c" + +static void wg_packet_consume_data_done(struct wg_peer *peer, + struct sk_buff *skb, + struct endpoint *endpoint) +{ + struct net_device *dev = peer->device->dev; + unsigned int len, len_before_trim; + struct wg_peer *routed_peer; + + wg_socket_set_peer_endpoint(peer, endpoint); + + if (unlikely(wg_noise_received_with_keypair(&peer->keypairs, + PACKET_CB(skb)->keypair))) { + wg_timers_handshake_complete(peer); + wg_packet_send_staged_packets(peer); + } + + keep_key_fresh(peer); + + wg_timers_any_authenticated_packet_received(peer); + wg_timers_any_authenticated_packet_traversal(peer); + + /* A packet with length 0 is a keepalive packet */ + if (unlikely(!skb->len)) { + update_rx_stats(peer, message_data_len(0)); + net_dbg_ratelimited("%s: Receiving keepalive packet from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, + &peer->endpoint.addr); + goto packet_processed; + } + + wg_timers_data_received(peer); + + if (unlikely(skb_network_header(skb) < skb->head)) + goto dishonest_packet_size; + if (unlikely(!(pskb_network_may_pull(skb, sizeof(struct iphdr)) && + (ip_hdr(skb)->version == 4 || + (ip_hdr(skb)->version == 6 && + pskb_network_may_pull(skb, sizeof(struct ipv6hdr))))))) + goto dishonest_packet_type; + + skb->dev = dev; + /* We've already verified the Poly1305 auth tag, which means this packet + * was not modified in transit. We can therefore tell the networking + * stack that all checksums of every layer of encapsulation have already + * been checked "by the hardware" and therefore is unnecessary to check + * again in software. + */ + skb->ip_summed = CHECKSUM_UNNECESSARY; + skb->csum_level = ~0; /* All levels */ + skb->protocol = wg_examine_packet_protocol(skb); + if (skb->protocol == htons(ETH_P_IP)) { + len = ntohs(ip_hdr(skb)->tot_len); + if (unlikely(len < sizeof(struct iphdr))) + goto dishonest_packet_size; + if (INET_ECN_is_ce(PACKET_CB(skb)->ds)) + IP_ECN_set_ce(ip_hdr(skb)); + } else if (skb->protocol == htons(ETH_P_IPV6)) { + len = ntohs(ipv6_hdr(skb)->payload_len) + + sizeof(struct ipv6hdr); + if (INET_ECN_is_ce(PACKET_CB(skb)->ds)) + IP6_ECN_set_ce(skb, ipv6_hdr(skb)); + } else { + goto dishonest_packet_type; + } + + if (unlikely(len > skb->len)) + goto dishonest_packet_size; + len_before_trim = skb->len; + if (unlikely(pskb_trim(skb, len))) + goto packet_processed; + + routed_peer = wg_allowedips_lookup_src(&peer->device->peer_allowedips, + skb); + wg_peer_put(routed_peer); /* We don't need the extra reference. */ + + if (unlikely(routed_peer != peer)) + goto dishonest_packet_peer; + + if (unlikely(napi_gro_receive(&peer->napi, skb) == GRO_DROP)) { + ++dev->stats.rx_dropped; + net_dbg_ratelimited("%s: Failed to give packet to userspace from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, + &peer->endpoint.addr); + } else { + update_rx_stats(peer, message_data_len(len_before_trim)); + } + return; + +dishonest_packet_peer: + net_dbg_skb_ratelimited("%s: Packet has unallowed src IP (%pISc) from peer %llu (%pISpfsc)\n", + dev->name, skb, peer->internal_id, + &peer->endpoint.addr); + ++dev->stats.rx_errors; + ++dev->stats.rx_frame_errors; + goto packet_processed; +dishonest_packet_type: + net_dbg_ratelimited("%s: Packet is neither ipv4 nor ipv6 from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, &peer->endpoint.addr); + ++dev->stats.rx_errors; + ++dev->stats.rx_frame_errors; + goto packet_processed; +dishonest_packet_size: + net_dbg_ratelimited("%s: Packet has incorrect size from peer %llu (%pISpfsc)\n", + dev->name, peer->internal_id, &peer->endpoint.addr); + ++dev->stats.rx_errors; + ++dev->stats.rx_length_errors; + goto packet_processed; +packet_processed: + dev_kfree_skb(skb); +} + +int wg_packet_rx_poll(struct napi_struct *napi, int budget) +{ + struct wg_peer *peer = container_of(napi, struct wg_peer, napi); + struct crypt_queue *queue = &peer->rx_queue; + struct noise_keypair *keypair; + struct endpoint endpoint; + enum packet_state state; + struct sk_buff *skb; + int work_done = 0; + bool free; + + if (unlikely(budget <= 0)) + return 0; + + while ((skb = __ptr_ring_peek(&queue->ring)) != NULL && + (state = atomic_read_acquire(&PACKET_CB(skb)->state)) != + PACKET_STATE_UNCRYPTED) { + __ptr_ring_discard_one(&queue->ring); + peer = PACKET_PEER(skb); + keypair = PACKET_CB(skb)->keypair; + free = true; + + if (unlikely(state != PACKET_STATE_CRYPTED)) + goto next; + + if (unlikely(!counter_validate(&keypair->receiving.counter, + PACKET_CB(skb)->nonce))) { + net_dbg_ratelimited("%s: Packet has invalid nonce %llu (max %llu)\n", + peer->device->dev->name, + PACKET_CB(skb)->nonce, + keypair->receiving.counter.receive.counter); + goto next; + } + + if (unlikely(wg_socket_endpoint_from_skb(&endpoint, skb))) + goto next; + + wg_reset_packet(skb); + wg_packet_consume_data_done(peer, skb, &endpoint); + free = false; + +next: + wg_noise_keypair_put(keypair, false); + wg_peer_put(peer); + if (unlikely(free)) + dev_kfree_skb(skb); + + if (++work_done >= budget) + break; + } + + if (work_done < budget) + napi_complete_done(napi, work_done); + + return work_done; +} + +void wg_packet_decrypt_worker(struct work_struct *work) +{ + struct crypt_queue *queue = container_of(work, struct multicore_worker, + work)->ptr; + struct sk_buff *skb; + + while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) { + enum packet_state state = likely(decrypt_packet(skb, + &PACKET_CB(skb)->keypair->receiving)) ? + PACKET_STATE_CRYPTED : PACKET_STATE_DEAD; + wg_queue_enqueue_per_peer_napi(skb, state); + } +} + +static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb) +{ + __le32 idx = ((struct message_data *)skb->data)->key_idx; + struct wg_peer *peer = NULL; + int ret; + + rcu_read_lock_bh(); + PACKET_CB(skb)->keypair = + (struct noise_keypair *)wg_index_hashtable_lookup( + wg->index_hashtable, INDEX_HASHTABLE_KEYPAIR, idx, + &peer); + if (unlikely(!wg_noise_keypair_get(PACKET_CB(skb)->keypair))) + goto err_keypair; + + if (unlikely(READ_ONCE(peer->is_dead))) + goto err; + + ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, + &peer->rx_queue, skb, + wg->packet_crypt_wq, + &wg->decrypt_queue.last_cpu); + if (unlikely(ret == -EPIPE)) + wg_queue_enqueue_per_peer_napi(skb, PACKET_STATE_DEAD); + if (likely(!ret || ret == -EPIPE)) { + rcu_read_unlock_bh(); + return; + } +err: + wg_noise_keypair_put(PACKET_CB(skb)->keypair, false); +err_keypair: + rcu_read_unlock_bh(); + wg_peer_put(peer); + dev_kfree_skb(skb); +} + +void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb) +{ + if (unlikely(prepare_skb_header(skb, wg) < 0)) + goto err; + switch (SKB_TYPE_LE32(skb)) { + case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): + case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): + case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): { + int cpu; + + if (skb_queue_len(&wg->incoming_handshakes) > + MAX_QUEUED_INCOMING_HANDSHAKES || + unlikely(!rng_is_initialized())) { + net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n", + wg->dev->name, skb); + goto err; + } + skb_queue_tail(&wg->incoming_handshakes, skb); + /* Queues up a call to packet_process_queued_handshake_ + * packets(skb): + */ + cpu = wg_cpumask_next_online(&wg->incoming_handshake_cpu); + queue_work_on(cpu, wg->handshake_receive_wq, + &per_cpu_ptr(wg->incoming_handshakes_worker, cpu)->work); + break; + } + case cpu_to_le32(MESSAGE_DATA): + PACKET_CB(skb)->ds = ip_tunnel_get_dsfield(ip_hdr(skb), skb); + wg_packet_consume_data(wg, skb); + break; + default: + WARN(1, "Non-exhaustive parsing of packet header lead to unknown packet type!\n"); + goto err; + } + return; + +err: + dev_kfree_skb(skb); +} diff --git a/drivers/net/wireguard/selftest/allowedips.c b/drivers/net/wireguard/selftest/allowedips.c new file mode 100644 index 0000000000000..846db14cb046b --- /dev/null +++ b/drivers/net/wireguard/selftest/allowedips.c @@ -0,0 +1,683 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This contains some basic static unit tests for the allowedips data structure. + * It also has two additional modes that are disabled and meant to be used by + * folks directly playing with this file. If you define the macro + * DEBUG_PRINT_TRIE_GRAPHVIZ to be 1, then every time there's a full tree in + * memory, it will be printed out as KERN_DEBUG in a format that can be passed + * to graphviz (the dot command) to visualize it. If you define the macro + * DEBUG_RANDOM_TRIE to be 1, then there will be an extremely costly set of + * randomized tests done against a trivial implementation, which may take + * upwards of a half-hour to complete. There's no set of users who should be + * enabling these, and the only developers that should go anywhere near these + * nobs are the ones who are reading this comment. + */ + +#ifdef DEBUG + +#include + +static __init void swap_endian_and_apply_cidr(u8 *dst, const u8 *src, u8 bits, + u8 cidr) +{ + swap_endian(dst, src, bits); + memset(dst + (cidr + 7) / 8, 0, bits / 8 - (cidr + 7) / 8); + if (cidr) + dst[(cidr + 7) / 8 - 1] &= ~0U << ((8 - (cidr % 8)) % 8); +} + +static __init void print_node(struct allowedips_node *node, u8 bits) +{ + char *fmt_connection = KERN_DEBUG "\t\"%p/%d\" -> \"%p/%d\";\n"; + char *fmt_declaration = KERN_DEBUG + "\t\"%p/%d\"[style=%s, color=\"#%06x\"];\n"; + char *style = "dotted"; + u8 ip1[16], ip2[16]; + u32 color = 0; + + if (bits == 32) { + fmt_connection = KERN_DEBUG "\t\"%pI4/%d\" -> \"%pI4/%d\";\n"; + fmt_declaration = KERN_DEBUG + "\t\"%pI4/%d\"[style=%s, color=\"#%06x\"];\n"; + } else if (bits == 128) { + fmt_connection = KERN_DEBUG "\t\"%pI6/%d\" -> \"%pI6/%d\";\n"; + fmt_declaration = KERN_DEBUG + "\t\"%pI6/%d\"[style=%s, color=\"#%06x\"];\n"; + } + if (node->peer) { + hsiphash_key_t key = { { 0 } }; + + memcpy(&key, &node->peer, sizeof(node->peer)); + color = hsiphash_1u32(0xdeadbeef, &key) % 200 << 16 | + hsiphash_1u32(0xbabecafe, &key) % 200 << 8 | + hsiphash_1u32(0xabad1dea, &key) % 200; + style = "bold"; + } + swap_endian_and_apply_cidr(ip1, node->bits, bits, node->cidr); + printk(fmt_declaration, ip1, node->cidr, style, color); + if (node->bit[0]) { + swap_endian_and_apply_cidr(ip2, + rcu_dereference_raw(node->bit[0])->bits, bits, + node->cidr); + printk(fmt_connection, ip1, node->cidr, ip2, + rcu_dereference_raw(node->bit[0])->cidr); + print_node(rcu_dereference_raw(node->bit[0]), bits); + } + if (node->bit[1]) { + swap_endian_and_apply_cidr(ip2, + rcu_dereference_raw(node->bit[1])->bits, + bits, node->cidr); + printk(fmt_connection, ip1, node->cidr, ip2, + rcu_dereference_raw(node->bit[1])->cidr); + print_node(rcu_dereference_raw(node->bit[1]), bits); + } +} + +static __init void print_tree(struct allowedips_node __rcu *top, u8 bits) +{ + printk(KERN_DEBUG "digraph trie {\n"); + print_node(rcu_dereference_raw(top), bits); + printk(KERN_DEBUG "}\n"); +} + +enum { + NUM_PEERS = 2000, + NUM_RAND_ROUTES = 400, + NUM_MUTATED_ROUTES = 100, + NUM_QUERIES = NUM_RAND_ROUTES * NUM_MUTATED_ROUTES * 30 +}; + +struct horrible_allowedips { + struct hlist_head head; +}; + +struct horrible_allowedips_node { + struct hlist_node table; + union nf_inet_addr ip; + union nf_inet_addr mask; + u8 ip_version; + void *value; +}; + +static __init void horrible_allowedips_init(struct horrible_allowedips *table) +{ + INIT_HLIST_HEAD(&table->head); +} + +static __init void horrible_allowedips_free(struct horrible_allowedips *table) +{ + struct horrible_allowedips_node *node; + struct hlist_node *h; + + hlist_for_each_entry_safe(node, h, &table->head, table) { + hlist_del(&node->table); + kfree(node); + } +} + +static __init inline union nf_inet_addr horrible_cidr_to_mask(u8 cidr) +{ + union nf_inet_addr mask; + + memset(&mask, 0x00, 128 / 8); + memset(&mask, 0xff, cidr / 8); + if (cidr % 32) + mask.all[cidr / 32] = (__force u32)htonl( + (0xFFFFFFFFUL << (32 - (cidr % 32))) & 0xFFFFFFFFUL); + return mask; +} + +static __init inline u8 horrible_mask_to_cidr(union nf_inet_addr subnet) +{ + return hweight32(subnet.all[0]) + hweight32(subnet.all[1]) + + hweight32(subnet.all[2]) + hweight32(subnet.all[3]); +} + +static __init inline void +horrible_mask_self(struct horrible_allowedips_node *node) +{ + if (node->ip_version == 4) { + node->ip.ip &= node->mask.ip; + } else if (node->ip_version == 6) { + node->ip.ip6[0] &= node->mask.ip6[0]; + node->ip.ip6[1] &= node->mask.ip6[1]; + node->ip.ip6[2] &= node->mask.ip6[2]; + node->ip.ip6[3] &= node->mask.ip6[3]; + } +} + +static __init inline bool +horrible_match_v4(const struct horrible_allowedips_node *node, + struct in_addr *ip) +{ + return (ip->s_addr & node->mask.ip) == node->ip.ip; +} + +static __init inline bool +horrible_match_v6(const struct horrible_allowedips_node *node, + struct in6_addr *ip) +{ + return (ip->in6_u.u6_addr32[0] & node->mask.ip6[0]) == + node->ip.ip6[0] && + (ip->in6_u.u6_addr32[1] & node->mask.ip6[1]) == + node->ip.ip6[1] && + (ip->in6_u.u6_addr32[2] & node->mask.ip6[2]) == + node->ip.ip6[2] && + (ip->in6_u.u6_addr32[3] & node->mask.ip6[3]) == node->ip.ip6[3]; +} + +static __init void +horrible_insert_ordered(struct horrible_allowedips *table, + struct horrible_allowedips_node *node) +{ + struct horrible_allowedips_node *other = NULL, *where = NULL; + u8 my_cidr = horrible_mask_to_cidr(node->mask); + + hlist_for_each_entry(other, &table->head, table) { + if (!memcmp(&other->mask, &node->mask, + sizeof(union nf_inet_addr)) && + !memcmp(&other->ip, &node->ip, + sizeof(union nf_inet_addr)) && + other->ip_version == node->ip_version) { + other->value = node->value; + kfree(node); + return; + } + where = other; + if (horrible_mask_to_cidr(other->mask) <= my_cidr) + break; + } + if (!other && !where) + hlist_add_head(&node->table, &table->head); + else if (!other) + hlist_add_behind(&node->table, &where->table); + else + hlist_add_before(&node->table, &where->table); +} + +static __init int +horrible_allowedips_insert_v4(struct horrible_allowedips *table, + struct in_addr *ip, u8 cidr, void *value) +{ + struct horrible_allowedips_node *node = kzalloc(sizeof(*node), + GFP_KERNEL); + + if (unlikely(!node)) + return -ENOMEM; + node->ip.in = *ip; + node->mask = horrible_cidr_to_mask(cidr); + node->ip_version = 4; + node->value = value; + horrible_mask_self(node); + horrible_insert_ordered(table, node); + return 0; +} + +static __init int +horrible_allowedips_insert_v6(struct horrible_allowedips *table, + struct in6_addr *ip, u8 cidr, void *value) +{ + struct horrible_allowedips_node *node = kzalloc(sizeof(*node), + GFP_KERNEL); + + if (unlikely(!node)) + return -ENOMEM; + node->ip.in6 = *ip; + node->mask = horrible_cidr_to_mask(cidr); + node->ip_version = 6; + node->value = value; + horrible_mask_self(node); + horrible_insert_ordered(table, node); + return 0; +} + +static __init void * +horrible_allowedips_lookup_v4(struct horrible_allowedips *table, + struct in_addr *ip) +{ + struct horrible_allowedips_node *node; + void *ret = NULL; + + hlist_for_each_entry(node, &table->head, table) { + if (node->ip_version != 4) + continue; + if (horrible_match_v4(node, ip)) { + ret = node->value; + break; + } + } + return ret; +} + +static __init void * +horrible_allowedips_lookup_v6(struct horrible_allowedips *table, + struct in6_addr *ip) +{ + struct horrible_allowedips_node *node; + void *ret = NULL; + + hlist_for_each_entry(node, &table->head, table) { + if (node->ip_version != 6) + continue; + if (horrible_match_v6(node, ip)) { + ret = node->value; + break; + } + } + return ret; +} + +static __init bool randomized_test(void) +{ + unsigned int i, j, k, mutate_amount, cidr; + u8 ip[16], mutate_mask[16], mutated[16]; + struct wg_peer **peers, *peer; + struct horrible_allowedips h; + DEFINE_MUTEX(mutex); + struct allowedips t; + bool ret = false; + + mutex_init(&mutex); + + wg_allowedips_init(&t); + horrible_allowedips_init(&h); + + peers = kcalloc(NUM_PEERS, sizeof(*peers), GFP_KERNEL); + if (unlikely(!peers)) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free; + } + for (i = 0; i < NUM_PEERS; ++i) { + peers[i] = kzalloc(sizeof(*peers[i]), GFP_KERNEL); + if (unlikely(!peers[i])) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free; + } + kref_init(&peers[i]->refcount); + } + + mutex_lock(&mutex); + + for (i = 0; i < NUM_RAND_ROUTES; ++i) { + prandom_bytes(ip, 4); + cidr = prandom_u32_max(32) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v4(&t, (struct in_addr *)ip, cidr, + peer, &mutex) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v4(&h, (struct in_addr *)ip, + cidr, peer) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + for (j = 0; j < NUM_MUTATED_ROUTES; ++j) { + memcpy(mutated, ip, 4); + prandom_bytes(mutate_mask, 4); + mutate_amount = prandom_u32_max(32); + for (k = 0; k < mutate_amount / 8; ++k) + mutate_mask[k] = 0xff; + mutate_mask[k] = 0xff + << ((8 - (mutate_amount % 8)) % 8); + for (; k < 4; ++k) + mutate_mask[k] = 0; + for (k = 0; k < 4; ++k) + mutated[k] = (mutated[k] & mutate_mask[k]) | + (~mutate_mask[k] & + prandom_u32_max(256)); + cidr = prandom_u32_max(32) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v4(&t, + (struct in_addr *)mutated, + cidr, peer, &mutex) < 0) { + pr_err("allowedips random malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v4(&h, + (struct in_addr *)mutated, cidr, peer)) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + } + } + + for (i = 0; i < NUM_RAND_ROUTES; ++i) { + prandom_bytes(ip, 16); + cidr = prandom_u32_max(128) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v6(&t, (struct in6_addr *)ip, cidr, + peer, &mutex) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v6(&h, (struct in6_addr *)ip, + cidr, peer) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + for (j = 0; j < NUM_MUTATED_ROUTES; ++j) { + memcpy(mutated, ip, 16); + prandom_bytes(mutate_mask, 16); + mutate_amount = prandom_u32_max(128); + for (k = 0; k < mutate_amount / 8; ++k) + mutate_mask[k] = 0xff; + mutate_mask[k] = 0xff + << ((8 - (mutate_amount % 8)) % 8); + for (; k < 4; ++k) + mutate_mask[k] = 0; + for (k = 0; k < 4; ++k) + mutated[k] = (mutated[k] & mutate_mask[k]) | + (~mutate_mask[k] & + prandom_u32_max(256)); + cidr = prandom_u32_max(128) + 1; + peer = peers[prandom_u32_max(NUM_PEERS)]; + if (wg_allowedips_insert_v6(&t, + (struct in6_addr *)mutated, + cidr, peer, &mutex) < 0) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + if (horrible_allowedips_insert_v6( + &h, (struct in6_addr *)mutated, cidr, + peer)) { + pr_err("allowedips random self-test malloc: FAIL\n"); + goto free_locked; + } + } + } + + mutex_unlock(&mutex); + + if (IS_ENABLED(DEBUG_PRINT_TRIE_GRAPHVIZ)) { + print_tree(t.root4, 32); + print_tree(t.root6, 128); + } + + for (i = 0; i < NUM_QUERIES; ++i) { + prandom_bytes(ip, 4); + if (lookup(t.root4, 32, ip) != + horrible_allowedips_lookup_v4(&h, (struct in_addr *)ip)) { + pr_err("allowedips random self-test: FAIL\n"); + goto free; + } + } + + for (i = 0; i < NUM_QUERIES; ++i) { + prandom_bytes(ip, 16); + if (lookup(t.root6, 128, ip) != + horrible_allowedips_lookup_v6(&h, (struct in6_addr *)ip)) { + pr_err("allowedips random self-test: FAIL\n"); + goto free; + } + } + ret = true; + +free: + mutex_lock(&mutex); +free_locked: + wg_allowedips_free(&t, &mutex); + mutex_unlock(&mutex); + horrible_allowedips_free(&h); + if (peers) { + for (i = 0; i < NUM_PEERS; ++i) + kfree(peers[i]); + } + kfree(peers); + return ret; +} + +static __init inline struct in_addr *ip4(u8 a, u8 b, u8 c, u8 d) +{ + static struct in_addr ip; + u8 *split = (u8 *)&ip; + + split[0] = a; + split[1] = b; + split[2] = c; + split[3] = d; + return &ip; +} + +static __init inline struct in6_addr *ip6(u32 a, u32 b, u32 c, u32 d) +{ + static struct in6_addr ip; + __be32 *split = (__be32 *)&ip; + + split[0] = cpu_to_be32(a); + split[1] = cpu_to_be32(b); + split[2] = cpu_to_be32(c); + split[3] = cpu_to_be32(d); + return &ip; +} + +static __init struct wg_peer *init_peer(void) +{ + struct wg_peer *peer = kzalloc(sizeof(*peer), GFP_KERNEL); + + if (!peer) + return NULL; + kref_init(&peer->refcount); + INIT_LIST_HEAD(&peer->allowedips_list); + return peer; +} + +#define insert(version, mem, ipa, ipb, ipc, ipd, cidr) \ + wg_allowedips_insert_v##version(&t, ip##version(ipa, ipb, ipc, ipd), \ + cidr, mem, &mutex) + +#define maybe_fail() do { \ + ++i; \ + if (!_s) { \ + pr_info("allowedips self-test %zu: FAIL\n", i); \ + success = false; \ + } \ + } while (0) + +#define test(version, mem, ipa, ipb, ipc, ipd) do { \ + bool _s = lookup(t.root##version, (version) == 4 ? 32 : 128, \ + ip##version(ipa, ipb, ipc, ipd)) == (mem); \ + maybe_fail(); \ + } while (0) + +#define test_negative(version, mem, ipa, ipb, ipc, ipd) do { \ + bool _s = lookup(t.root##version, (version) == 4 ? 32 : 128, \ + ip##version(ipa, ipb, ipc, ipd)) != (mem); \ + maybe_fail(); \ + } while (0) + +#define test_boolean(cond) do { \ + bool _s = (cond); \ + maybe_fail(); \ + } while (0) + +bool __init wg_allowedips_selftest(void) +{ + bool found_a = false, found_b = false, found_c = false, found_d = false, + found_e = false, found_other = false; + struct wg_peer *a = init_peer(), *b = init_peer(), *c = init_peer(), + *d = init_peer(), *e = init_peer(), *f = init_peer(), + *g = init_peer(), *h = init_peer(); + struct allowedips_node *iter_node; + bool success = false; + struct allowedips t; + DEFINE_MUTEX(mutex); + struct in6_addr ip; + size_t i = 0, count = 0; + __be64 part; + + mutex_init(&mutex); + mutex_lock(&mutex); + wg_allowedips_init(&t); + + if (!a || !b || !c || !d || !e || !f || !g || !h) { + pr_err("allowedips self-test malloc: FAIL\n"); + goto free; + } + + insert(4, a, 192, 168, 4, 0, 24); + insert(4, b, 192, 168, 4, 4, 32); + insert(4, c, 192, 168, 0, 0, 16); + insert(4, d, 192, 95, 5, 64, 27); + /* replaces previous entry, and maskself is required */ + insert(4, c, 192, 95, 5, 65, 27); + insert(6, d, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128); + insert(6, c, 0x26075300, 0x60006b00, 0, 0, 64); + insert(4, e, 0, 0, 0, 0, 0); + insert(6, e, 0, 0, 0, 0, 0); + /* replaces previous entry */ + insert(6, f, 0, 0, 0, 0, 0); + insert(6, g, 0x24046800, 0, 0, 0, 32); + /* maskself is required */ + insert(6, h, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 64); + insert(6, a, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef, 128); + insert(6, c, 0x24446800, 0x40e40800, 0xdeaebeef, 0xdefbeef, 128); + insert(6, b, 0x24446800, 0xf0e40800, 0xeeaebeef, 0, 98); + insert(4, g, 64, 15, 112, 0, 20); + /* maskself is required */ + insert(4, h, 64, 15, 123, 211, 25); + insert(4, a, 10, 0, 0, 0, 25); + insert(4, b, 10, 0, 0, 128, 25); + insert(4, a, 10, 1, 0, 0, 30); + insert(4, b, 10, 1, 0, 4, 30); + insert(4, c, 10, 1, 0, 8, 29); + insert(4, d, 10, 1, 0, 16, 29); + + if (IS_ENABLED(DEBUG_PRINT_TRIE_GRAPHVIZ)) { + print_tree(t.root4, 32); + print_tree(t.root6, 128); + } + + success = true; + + test(4, a, 192, 168, 4, 20); + test(4, a, 192, 168, 4, 0); + test(4, b, 192, 168, 4, 4); + test(4, c, 192, 168, 200, 182); + test(4, c, 192, 95, 5, 68); + test(4, e, 192, 95, 5, 96); + test(6, d, 0x26075300, 0x60006b00, 0, 0xc05f0543); + test(6, c, 0x26075300, 0x60006b00, 0, 0xc02e01ee); + test(6, f, 0x26075300, 0x60006b01, 0, 0); + test(6, g, 0x24046800, 0x40040806, 0, 0x1006); + test(6, g, 0x24046800, 0x40040806, 0x1234, 0x5678); + test(6, f, 0x240467ff, 0x40040806, 0x1234, 0x5678); + test(6, f, 0x24046801, 0x40040806, 0x1234, 0x5678); + test(6, h, 0x24046800, 0x40040800, 0x1234, 0x5678); + test(6, h, 0x24046800, 0x40040800, 0, 0); + test(6, h, 0x24046800, 0x40040800, 0x10101010, 0x10101010); + test(6, a, 0x24046800, 0x40040800, 0xdeadbeef, 0xdeadbeef); + test(4, g, 64, 15, 116, 26); + test(4, g, 64, 15, 127, 3); + test(4, g, 64, 15, 123, 1); + test(4, h, 64, 15, 123, 128); + test(4, h, 64, 15, 123, 129); + test(4, a, 10, 0, 0, 52); + test(4, b, 10, 0, 0, 220); + test(4, a, 10, 1, 0, 2); + test(4, b, 10, 1, 0, 6); + test(4, c, 10, 1, 0, 10); + test(4, d, 10, 1, 0, 20); + + insert(4, a, 1, 0, 0, 0, 32); + insert(4, a, 64, 0, 0, 0, 32); + insert(4, a, 128, 0, 0, 0, 32); + insert(4, a, 192, 0, 0, 0, 32); + insert(4, a, 255, 0, 0, 0, 32); + wg_allowedips_remove_by_peer(&t, a, &mutex); + test_negative(4, a, 1, 0, 0, 0); + test_negative(4, a, 64, 0, 0, 0); + test_negative(4, a, 128, 0, 0, 0); + test_negative(4, a, 192, 0, 0, 0); + test_negative(4, a, 255, 0, 0, 0); + + wg_allowedips_free(&t, &mutex); + wg_allowedips_init(&t); + insert(4, a, 192, 168, 0, 0, 16); + insert(4, a, 192, 168, 0, 0, 24); + wg_allowedips_remove_by_peer(&t, a, &mutex); + test_negative(4, a, 192, 168, 0, 1); + + /* These will hit the WARN_ON(len >= 128) in free_node if something + * goes wrong. + */ + for (i = 0; i < 128; ++i) { + part = cpu_to_be64(~(1LLU << (i % 64))); + memset(&ip, 0xff, 16); + memcpy((u8 *)&ip + (i < 64) * 8, &part, 8); + wg_allowedips_insert_v6(&t, &ip, 128, a, &mutex); + } + + wg_allowedips_free(&t, &mutex); + + wg_allowedips_init(&t); + insert(4, a, 192, 95, 5, 93, 27); + insert(6, a, 0x26075300, 0x60006b00, 0, 0xc05f0543, 128); + insert(4, a, 10, 1, 0, 20, 29); + insert(6, a, 0x26075300, 0x6d8a6bf8, 0xdab1f1df, 0xc05f1523, 83); + insert(6, a, 0x26075300, 0x6d8a6bf8, 0xdab1f1df, 0xc05f1523, 21); + list_for_each_entry(iter_node, &a->allowedips_list, peer_list) { + u8 cidr, ip[16] __aligned(__alignof(u64)); + int family = wg_allowedips_read_node(iter_node, ip, &cidr); + + count++; + + if (cidr == 27 && family == AF_INET && + !memcmp(ip, ip4(192, 95, 5, 64), sizeof(struct in_addr))) + found_a = true; + else if (cidr == 128 && family == AF_INET6 && + !memcmp(ip, ip6(0x26075300, 0x60006b00, 0, 0xc05f0543), + sizeof(struct in6_addr))) + found_b = true; + else if (cidr == 29 && family == AF_INET && + !memcmp(ip, ip4(10, 1, 0, 16), sizeof(struct in_addr))) + found_c = true; + else if (cidr == 83 && family == AF_INET6 && + !memcmp(ip, ip6(0x26075300, 0x6d8a6bf8, 0xdab1e000, 0), + sizeof(struct in6_addr))) + found_d = true; + else if (cidr == 21 && family == AF_INET6 && + !memcmp(ip, ip6(0x26075000, 0, 0, 0), + sizeof(struct in6_addr))) + found_e = true; + else + found_other = true; + } + test_boolean(count == 5); + test_boolean(found_a); + test_boolean(found_b); + test_boolean(found_c); + test_boolean(found_d); + test_boolean(found_e); + test_boolean(!found_other); + + if (IS_ENABLED(DEBUG_RANDOM_TRIE) && success) + success = randomized_test(); + + if (success) + pr_info("allowedips self-tests: pass\n"); + +free: + wg_allowedips_free(&t, &mutex); + kfree(a); + kfree(b); + kfree(c); + kfree(d); + kfree(e); + kfree(f); + kfree(g); + kfree(h); + mutex_unlock(&mutex); + + return success; +} + +#undef test_negative +#undef test +#undef remove +#undef insert +#undef init_peer + +#endif diff --git a/drivers/net/wireguard/selftest/counter.c b/drivers/net/wireguard/selftest/counter.c new file mode 100644 index 0000000000000..f4fbb9072ed73 --- /dev/null +++ b/drivers/net/wireguard/selftest/counter.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifdef DEBUG +bool __init wg_packet_counter_selftest(void) +{ + unsigned int test_num = 0, i; + union noise_counter counter; + bool success = true; + +#define T_INIT do { \ + memset(&counter, 0, sizeof(union noise_counter)); \ + spin_lock_init(&counter.receive.lock); \ + } while (0) +#define T_LIM (COUNTER_WINDOW_SIZE + 1) +#define T(n, v) do { \ + ++test_num; \ + if (counter_validate(&counter, n) != (v)) { \ + pr_err("nonce counter self-test %u: FAIL\n", \ + test_num); \ + success = false; \ + } \ + } while (0) + + T_INIT; + /* 1 */ T(0, true); + /* 2 */ T(1, true); + /* 3 */ T(1, false); + /* 4 */ T(9, true); + /* 5 */ T(8, true); + /* 6 */ T(7, true); + /* 7 */ T(7, false); + /* 8 */ T(T_LIM, true); + /* 9 */ T(T_LIM - 1, true); + /* 10 */ T(T_LIM - 1, false); + /* 11 */ T(T_LIM - 2, true); + /* 12 */ T(2, true); + /* 13 */ T(2, false); + /* 14 */ T(T_LIM + 16, true); + /* 15 */ T(3, false); + /* 16 */ T(T_LIM + 16, false); + /* 17 */ T(T_LIM * 4, true); + /* 18 */ T(T_LIM * 4 - (T_LIM - 1), true); + /* 19 */ T(10, false); + /* 20 */ T(T_LIM * 4 - T_LIM, false); + /* 21 */ T(T_LIM * 4 - (T_LIM + 1), false); + /* 22 */ T(T_LIM * 4 - (T_LIM - 2), true); + /* 23 */ T(T_LIM * 4 + 1 - T_LIM, false); + /* 24 */ T(0, false); + /* 25 */ T(REJECT_AFTER_MESSAGES, false); + /* 26 */ T(REJECT_AFTER_MESSAGES - 1, true); + /* 27 */ T(REJECT_AFTER_MESSAGES, false); + /* 28 */ T(REJECT_AFTER_MESSAGES - 1, false); + /* 29 */ T(REJECT_AFTER_MESSAGES - 2, true); + /* 30 */ T(REJECT_AFTER_MESSAGES + 1, false); + /* 31 */ T(REJECT_AFTER_MESSAGES + 2, false); + /* 32 */ T(REJECT_AFTER_MESSAGES - 2, false); + /* 33 */ T(REJECT_AFTER_MESSAGES - 3, true); + /* 34 */ T(0, false); + + T_INIT; + for (i = 1; i <= COUNTER_WINDOW_SIZE; ++i) + T(i, true); + T(0, true); + T(0, false); + + T_INIT; + for (i = 2; i <= COUNTER_WINDOW_SIZE + 1; ++i) + T(i, true); + T(1, true); + T(0, false); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 0;) + T(i, true); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 2; i-- > 1;) + T(i, true); + T(0, false); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 1;) + T(i, true); + T(COUNTER_WINDOW_SIZE + 1, true); + T(0, false); + + T_INIT; + for (i = COUNTER_WINDOW_SIZE + 1; i-- > 1;) + T(i, true); + T(0, true); + T(COUNTER_WINDOW_SIZE + 1, true); + +#undef T +#undef T_LIM +#undef T_INIT + + if (success) + pr_info("nonce counter self-tests: pass\n"); + return success; +} +#endif diff --git a/drivers/net/wireguard/selftest/ratelimiter.c b/drivers/net/wireguard/selftest/ratelimiter.c new file mode 100644 index 0000000000000..bcd6462e45401 --- /dev/null +++ b/drivers/net/wireguard/selftest/ratelimiter.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifdef DEBUG + +#include + +static const struct { + bool result; + unsigned int msec_to_sleep_before; +} expected_results[] __initconst = { + [0 ... PACKETS_BURSTABLE - 1] = { true, 0 }, + [PACKETS_BURSTABLE] = { false, 0 }, + [PACKETS_BURSTABLE + 1] = { true, MSEC_PER_SEC / PACKETS_PER_SECOND }, + [PACKETS_BURSTABLE + 2] = { false, 0 }, + [PACKETS_BURSTABLE + 3] = { true, (MSEC_PER_SEC / PACKETS_PER_SECOND) * 2 }, + [PACKETS_BURSTABLE + 4] = { true, 0 }, + [PACKETS_BURSTABLE + 5] = { false, 0 } +}; + +static __init unsigned int maximum_jiffies_at_index(int index) +{ + unsigned int total_msecs = 2 * MSEC_PER_SEC / PACKETS_PER_SECOND / 3; + int i; + + for (i = 0; i <= index; ++i) + total_msecs += expected_results[i].msec_to_sleep_before; + return msecs_to_jiffies(total_msecs); +} + +static __init int timings_test(struct sk_buff *skb4, struct iphdr *hdr4, + struct sk_buff *skb6, struct ipv6hdr *hdr6, + int *test) +{ + unsigned long loop_start_time; + int i; + + wg_ratelimiter_gc_entries(NULL); + rcu_barrier(); + loop_start_time = jiffies; + + for (i = 0; i < ARRAY_SIZE(expected_results); ++i) { + if (expected_results[i].msec_to_sleep_before) + msleep(expected_results[i].msec_to_sleep_before); + + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (wg_ratelimiter_allow(skb4, &init_net) != + expected_results[i].result) + return -EXFULL; + ++(*test); + + hdr4->saddr = htonl(ntohl(hdr4->saddr) + i + 1); + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (!wg_ratelimiter_allow(skb4, &init_net)) + return -EXFULL; + ++(*test); + + hdr4->saddr = htonl(ntohl(hdr4->saddr) - i - 1); + +#if IS_ENABLED(CONFIG_IPV6) + hdr6->saddr.in6_u.u6_addr32[2] = htonl(i); + hdr6->saddr.in6_u.u6_addr32[3] = htonl(i); + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (wg_ratelimiter_allow(skb6, &init_net) != + expected_results[i].result) + return -EXFULL; + ++(*test); + + hdr6->saddr.in6_u.u6_addr32[0] = + htonl(ntohl(hdr6->saddr.in6_u.u6_addr32[0]) + i + 1); + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; + if (!wg_ratelimiter_allow(skb6, &init_net)) + return -EXFULL; + ++(*test); + + hdr6->saddr.in6_u.u6_addr32[0] = + htonl(ntohl(hdr6->saddr.in6_u.u6_addr32[0]) - i - 1); + + if (time_is_before_jiffies(loop_start_time + + maximum_jiffies_at_index(i))) + return -ETIMEDOUT; +#endif + } + return 0; +} + +static __init int capacity_test(struct sk_buff *skb4, struct iphdr *hdr4, + int *test) +{ + int i; + + wg_ratelimiter_gc_entries(NULL); + rcu_barrier(); + + if (atomic_read(&total_entries)) + return -EXFULL; + ++(*test); + + for (i = 0; i <= max_entries; ++i) { + hdr4->saddr = htonl(i); + if (wg_ratelimiter_allow(skb4, &init_net) != (i != max_entries)) + return -EXFULL; + ++(*test); + } + return 0; +} + +bool __init wg_ratelimiter_selftest(void) +{ + enum { TRIALS_BEFORE_GIVING_UP = 5000 }; + bool success = false; + int test = 0, trials; + struct sk_buff *skb4, *skb6; + struct iphdr *hdr4; + struct ipv6hdr *hdr6; + + if (IS_ENABLED(CONFIG_KASAN) || IS_ENABLED(CONFIG_UBSAN)) + return true; + + BUILD_BUG_ON(MSEC_PER_SEC % PACKETS_PER_SECOND != 0); + + if (wg_ratelimiter_init()) + goto out; + ++test; + if (wg_ratelimiter_init()) { + wg_ratelimiter_uninit(); + goto out; + } + ++test; + if (wg_ratelimiter_init()) { + wg_ratelimiter_uninit(); + wg_ratelimiter_uninit(); + goto out; + } + ++test; + + skb4 = alloc_skb(sizeof(struct iphdr), GFP_KERNEL); + if (unlikely(!skb4)) + goto err_nofree; + skb4->protocol = htons(ETH_P_IP); + hdr4 = (struct iphdr *)skb_put(skb4, sizeof(*hdr4)); + hdr4->saddr = htonl(8182); + skb_reset_network_header(skb4); + ++test; + +#if IS_ENABLED(CONFIG_IPV6) + skb6 = alloc_skb(sizeof(struct ipv6hdr), GFP_KERNEL); + if (unlikely(!skb6)) { + kfree_skb(skb4); + goto err_nofree; + } + skb6->protocol = htons(ETH_P_IPV6); + hdr6 = (struct ipv6hdr *)skb_put(skb6, sizeof(*hdr6)); + hdr6->saddr.in6_u.u6_addr32[0] = htonl(1212); + hdr6->saddr.in6_u.u6_addr32[1] = htonl(289188); + skb_reset_network_header(skb6); + ++test; +#endif + + for (trials = TRIALS_BEFORE_GIVING_UP;;) { + int test_count = 0, ret; + + ret = timings_test(skb4, hdr4, skb6, hdr6, &test_count); + if (ret == -ETIMEDOUT) { + if (!trials--) { + test += test_count; + goto err; + } + msleep(500); + continue; + } else if (ret < 0) { + test += test_count; + goto err; + } else { + test += test_count; + break; + } + } + + for (trials = TRIALS_BEFORE_GIVING_UP;;) { + int test_count = 0; + + if (capacity_test(skb4, hdr4, &test_count) < 0) { + if (!trials--) { + test += test_count; + goto err; + } + msleep(50); + continue; + } + test += test_count; + break; + } + + success = true; + +err: + kfree_skb(skb4); +#if IS_ENABLED(CONFIG_IPV6) + kfree_skb(skb6); +#endif +err_nofree: + wg_ratelimiter_uninit(); + wg_ratelimiter_uninit(); + wg_ratelimiter_uninit(); + /* Uninit one extra time to check underflow detection. */ + wg_ratelimiter_uninit(); +out: + if (success) + pr_info("ratelimiter self-tests: pass\n"); + else + pr_err("ratelimiter self-test %d: FAIL\n", test); + + return success; +} +#endif diff --git a/drivers/net/wireguard/send.c b/drivers/net/wireguard/send.c new file mode 100644 index 0000000000000..7348c10cbae3d --- /dev/null +++ b/drivers/net/wireguard/send.c @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "queueing.h" +#include "timers.h" +#include "device.h" +#include "peer.h" +#include "socket.h" +#include "messages.h" +#include "cookie.h" + +#include +#include +#include +#include +#include +#include + +static void wg_packet_send_handshake_initiation(struct wg_peer *peer) +{ + struct message_handshake_initiation packet; + + if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), + REKEY_TIMEOUT)) + return; /* This function is rate limited. */ + + atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); + net_dbg_ratelimited("%s: Sending handshake initiation to peer %llu (%pISpfsc)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + + if (wg_noise_handshake_create_initiation(&packet, &peer->handshake)) { + wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); + wg_timers_any_authenticated_packet_traversal(peer); + wg_timers_any_authenticated_packet_sent(peer); + atomic64_set(&peer->last_sent_handshake, + ktime_get_coarse_boottime_ns()); + wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet), + HANDSHAKE_DSCP); + wg_timers_handshake_initiated(peer); + } +} + +void wg_packet_handshake_send_worker(struct work_struct *work) +{ + struct wg_peer *peer = container_of(work, struct wg_peer, + transmit_handshake_work); + + wg_packet_send_handshake_initiation(peer); + wg_peer_put(peer); +} + +void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, + bool is_retry) +{ + if (!is_retry) + peer->timer_handshake_attempts = 0; + + rcu_read_lock_bh(); + /* We check last_sent_handshake here in addition to the actual function + * we're queueing up, so that we don't queue things if not strictly + * necessary: + */ + if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake), + REKEY_TIMEOUT) || + unlikely(READ_ONCE(peer->is_dead))) + goto out; + + wg_peer_get(peer); + /* Queues up calling packet_send_queued_handshakes(peer), where we do a + * peer_put(peer) after: + */ + if (!queue_work(peer->device->handshake_send_wq, + &peer->transmit_handshake_work)) + /* If the work was already queued, we want to drop the + * extra reference: + */ + wg_peer_put(peer); +out: + rcu_read_unlock_bh(); +} + +void wg_packet_send_handshake_response(struct wg_peer *peer) +{ + struct message_handshake_response packet; + + atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns()); + net_dbg_ratelimited("%s: Sending handshake response to peer %llu (%pISpfsc)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + + if (wg_noise_handshake_create_response(&packet, &peer->handshake)) { + wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer); + if (wg_noise_handshake_begin_session(&peer->handshake, + &peer->keypairs)) { + wg_timers_session_derived(peer); + wg_timers_any_authenticated_packet_traversal(peer); + wg_timers_any_authenticated_packet_sent(peer); + atomic64_set(&peer->last_sent_handshake, + ktime_get_coarse_boottime_ns()); + wg_socket_send_buffer_to_peer(peer, &packet, + sizeof(packet), + HANDSHAKE_DSCP); + } + } +} + +void wg_packet_send_handshake_cookie(struct wg_device *wg, + struct sk_buff *initiating_skb, + __le32 sender_index) +{ + struct message_handshake_cookie packet; + + net_dbg_skb_ratelimited("%s: Sending cookie response for denied handshake message for %pISpfsc\n", + wg->dev->name, initiating_skb); + wg_cookie_message_create(&packet, initiating_skb, sender_index, + &wg->cookie_checker); + wg_socket_send_buffer_as_reply_to_skb(wg, initiating_skb, &packet, + sizeof(packet)); +} + +static void keep_key_fresh(struct wg_peer *peer) +{ + struct noise_keypair *keypair; + bool send = false; + + rcu_read_lock_bh(); + keypair = rcu_dereference_bh(peer->keypairs.current_keypair); + if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) && + (unlikely(atomic64_read(&keypair->sending.counter.counter) > + REKEY_AFTER_MESSAGES) || + (keypair->i_am_the_initiator && + unlikely(wg_birthdate_has_expired(keypair->sending.birthdate, + REKEY_AFTER_TIME))))) + send = true; + rcu_read_unlock_bh(); + + if (send) + wg_packet_send_queued_handshake_initiation(peer, false); +} + +static unsigned int calculate_skb_padding(struct sk_buff *skb) +{ + unsigned int padded_size, last_unit = skb->len; + + if (unlikely(!PACKET_CB(skb)->mtu)) + return ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE) - last_unit; + + /* We do this modulo business with the MTU, just in case the networking + * layer gives us a packet that's bigger than the MTU. In that case, we + * wouldn't want the final subtraction to overflow in the case of the + * padded_size being clamped. Fortunately, that's very rarely the case, + * so we optimize for that not happening. + */ + if (unlikely(last_unit > PACKET_CB(skb)->mtu)) + last_unit %= PACKET_CB(skb)->mtu; + + padded_size = min(PACKET_CB(skb)->mtu, + ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE)); + return padded_size - last_unit; +} + +static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair) +{ + unsigned int padding_len, plaintext_len, trailer_len; + struct scatterlist sg[MAX_SKB_FRAGS + 8]; + struct message_data *header; + struct sk_buff *trailer; + int num_frags; + + /* Calculate lengths. */ + padding_len = calculate_skb_padding(skb); + trailer_len = padding_len + noise_encrypted_len(0); + plaintext_len = skb->len + padding_len; + + /* Expand data section to have room for padding and auth tag. */ + num_frags = skb_cow_data(skb, trailer_len, &trailer); + if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg))) + return false; + + /* Set the padding to zeros, and make sure it and the auth tag are part + * of the skb. + */ + memset(skb_tail_pointer(trailer), 0, padding_len); + + /* Expand head section to have room for our header and the network + * stack's headers. + */ + if (unlikely(skb_cow_head(skb, DATA_PACKET_HEAD_ROOM) < 0)) + return false; + + /* Finalize checksum calculation for the inner packet, if required. */ + if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL && + skb_checksum_help(skb))) + return false; + + /* Only after checksumming can we safely add on the padding at the end + * and the header. + */ + skb_set_inner_network_header(skb, 0); + header = (struct message_data *)skb_push(skb, sizeof(*header)); + header->header.type = cpu_to_le32(MESSAGE_DATA); + header->key_idx = keypair->remote_index; + header->counter = cpu_to_le64(PACKET_CB(skb)->nonce); + pskb_put(skb, trailer, trailer_len); + + /* Now we can encrypt the scattergather segments */ + sg_init_table(sg, num_frags); + if (skb_to_sgvec(skb, sg, sizeof(struct message_data), + noise_encrypted_len(plaintext_len)) <= 0) + return false; + return chacha20poly1305_encrypt_sg_inplace(sg, plaintext_len, NULL, 0, + PACKET_CB(skb)->nonce, + keypair->sending.key); +} + +void wg_packet_send_keepalive(struct wg_peer *peer) +{ + struct sk_buff *skb; + + if (skb_queue_empty(&peer->staged_packet_queue)) { + skb = alloc_skb(DATA_PACKET_HEAD_ROOM + MESSAGE_MINIMUM_LENGTH, + GFP_ATOMIC); + if (unlikely(!skb)) + return; + skb_reserve(skb, DATA_PACKET_HEAD_ROOM); + skb->dev = peer->device->dev; + PACKET_CB(skb)->mtu = skb->dev->mtu; + skb_queue_tail(&peer->staged_packet_queue, skb); + net_dbg_ratelimited("%s: Sending keepalive packet to peer %llu (%pISpfsc)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr); + } + + wg_packet_send_staged_packets(peer); +} + +static void wg_packet_create_data_done(struct sk_buff *first, + struct wg_peer *peer) +{ + struct sk_buff *skb, *next; + bool is_keepalive, data_sent = false; + + wg_timers_any_authenticated_packet_traversal(peer); + wg_timers_any_authenticated_packet_sent(peer); + skb_list_walk_safe(first, skb, next) { + is_keepalive = skb->len == message_data_len(0); + if (likely(!wg_socket_send_skb_to_peer(peer, skb, + PACKET_CB(skb)->ds) && !is_keepalive)) + data_sent = true; + } + + if (likely(data_sent)) + wg_timers_data_sent(peer); + + keep_key_fresh(peer); +} + +void wg_packet_tx_worker(struct work_struct *work) +{ + struct crypt_queue *queue = container_of(work, struct crypt_queue, + work); + struct noise_keypair *keypair; + enum packet_state state; + struct sk_buff *first; + struct wg_peer *peer; + + while ((first = __ptr_ring_peek(&queue->ring)) != NULL && + (state = atomic_read_acquire(&PACKET_CB(first)->state)) != + PACKET_STATE_UNCRYPTED) { + __ptr_ring_discard_one(&queue->ring); + peer = PACKET_PEER(first); + keypair = PACKET_CB(first)->keypair; + + if (likely(state == PACKET_STATE_CRYPTED)) + wg_packet_create_data_done(first, peer); + else + kfree_skb_list(first); + + wg_noise_keypair_put(keypair, false); + wg_peer_put(peer); + } +} + +void wg_packet_encrypt_worker(struct work_struct *work) +{ + struct crypt_queue *queue = container_of(work, struct multicore_worker, + work)->ptr; + struct sk_buff *first, *skb, *next; + + while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) { + enum packet_state state = PACKET_STATE_CRYPTED; + + skb_list_walk_safe(first, skb, next) { + if (likely(encrypt_packet(skb, + PACKET_CB(first)->keypair))) { + wg_reset_packet(skb); + } else { + state = PACKET_STATE_DEAD; + break; + } + } + wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, first, + state); + + } +} + +static void wg_packet_create_data(struct sk_buff *first) +{ + struct wg_peer *peer = PACKET_PEER(first); + struct wg_device *wg = peer->device; + int ret = -EINVAL; + + rcu_read_lock_bh(); + if (unlikely(READ_ONCE(peer->is_dead))) + goto err; + + ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, + &peer->tx_queue, first, + wg->packet_crypt_wq, + &wg->encrypt_queue.last_cpu); + if (unlikely(ret == -EPIPE)) + wg_queue_enqueue_per_peer(&peer->tx_queue, first, + PACKET_STATE_DEAD); +err: + rcu_read_unlock_bh(); + if (likely(!ret || ret == -EPIPE)) + return; + wg_noise_keypair_put(PACKET_CB(first)->keypair, false); + wg_peer_put(peer); + kfree_skb_list(first); +} + +void wg_packet_purge_staged_packets(struct wg_peer *peer) +{ + spin_lock_bh(&peer->staged_packet_queue.lock); + peer->device->dev->stats.tx_dropped += peer->staged_packet_queue.qlen; + __skb_queue_purge(&peer->staged_packet_queue); + spin_unlock_bh(&peer->staged_packet_queue.lock); +} + +void wg_packet_send_staged_packets(struct wg_peer *peer) +{ + struct noise_symmetric_key *key; + struct noise_keypair *keypair; + struct sk_buff_head packets; + struct sk_buff *skb; + + /* Steal the current queue into our local one. */ + __skb_queue_head_init(&packets); + spin_lock_bh(&peer->staged_packet_queue.lock); + skb_queue_splice_init(&peer->staged_packet_queue, &packets); + spin_unlock_bh(&peer->staged_packet_queue.lock); + if (unlikely(skb_queue_empty(&packets))) + return; + + /* First we make sure we have a valid reference to a valid key. */ + rcu_read_lock_bh(); + keypair = wg_noise_keypair_get( + rcu_dereference_bh(peer->keypairs.current_keypair)); + rcu_read_unlock_bh(); + if (unlikely(!keypair)) + goto out_nokey; + key = &keypair->sending; + if (unlikely(!READ_ONCE(key->is_valid))) + goto out_nokey; + if (unlikely(wg_birthdate_has_expired(key->birthdate, + REJECT_AFTER_TIME))) + goto out_invalid; + + /* After we know we have a somewhat valid key, we now try to assign + * nonces to all of the packets in the queue. If we can't assign nonces + * for all of them, we just consider it a failure and wait for the next + * handshake. + */ + skb_queue_walk(&packets, skb) { + /* 0 for no outer TOS: no leak. TODO: at some later point, we + * might consider using flowi->tos as outer instead. + */ + PACKET_CB(skb)->ds = ip_tunnel_ecn_encap(0, ip_hdr(skb), skb); + PACKET_CB(skb)->nonce = + atomic64_inc_return(&key->counter.counter) - 1; + if (unlikely(PACKET_CB(skb)->nonce >= REJECT_AFTER_MESSAGES)) + goto out_invalid; + } + + packets.prev->next = NULL; + wg_peer_get(keypair->entry.peer); + PACKET_CB(packets.next)->keypair = keypair; + wg_packet_create_data(packets.next); + return; + +out_invalid: + WRITE_ONCE(key->is_valid, false); +out_nokey: + wg_noise_keypair_put(keypair, false); + + /* We orphan the packets if we're waiting on a handshake, so that they + * don't block a socket's pool. + */ + skb_queue_walk(&packets, skb) + skb_orphan(skb); + /* Then we put them back on the top of the queue. We're not too + * concerned about accidentally getting things a little out of order if + * packets are being added really fast, because this queue is for before + * packets can even be sent and it's small anyway. + */ + spin_lock_bh(&peer->staged_packet_queue.lock); + skb_queue_splice(&packets, &peer->staged_packet_queue); + spin_unlock_bh(&peer->staged_packet_queue.lock); + + /* If we're exiting because there's something wrong with the key, it + * means we should initiate a new handshake. + */ + wg_packet_send_queued_handshake_initiation(peer, false); +} diff --git a/drivers/net/wireguard/socket.c b/drivers/net/wireguard/socket.c new file mode 100644 index 0000000000000..b0d6541582d31 --- /dev/null +++ b/drivers/net/wireguard/socket.c @@ -0,0 +1,437 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "device.h" +#include "peer.h" +#include "socket.h" +#include "queueing.h" +#include "messages.h" + +#include +#include +#include +#include +#include +#include +#include + +static int send4(struct wg_device *wg, struct sk_buff *skb, + struct endpoint *endpoint, u8 ds, struct dst_cache *cache) +{ + struct flowi4 fl = { + .saddr = endpoint->src4.s_addr, + .daddr = endpoint->addr4.sin_addr.s_addr, + .fl4_dport = endpoint->addr4.sin_port, + .flowi4_mark = wg->fwmark, + .flowi4_proto = IPPROTO_UDP + }; + struct rtable *rt = NULL; + struct sock *sock; + int ret = 0; + + skb_mark_not_on_list(skb); + skb->dev = wg->dev; + skb->mark = wg->fwmark; + + rcu_read_lock_bh(); + sock = rcu_dereference_bh(wg->sock4); + + if (unlikely(!sock)) { + ret = -ENONET; + goto err; + } + + fl.fl4_sport = inet_sk(sock)->inet_sport; + + if (cache) + rt = dst_cache_get_ip4(cache, &fl.saddr); + + if (!rt) { + security_sk_classify_flow(sock, flowi4_to_flowi(&fl)); + if (unlikely(!inet_confirm_addr(sock_net(sock), NULL, 0, + fl.saddr, RT_SCOPE_HOST))) { + endpoint->src4.s_addr = 0; + *(__force __be32 *)&endpoint->src_if4 = 0; + fl.saddr = 0; + if (cache) + dst_cache_reset(cache); + } + rt = ip_route_output_flow(sock_net(sock), &fl, sock); + if (unlikely(endpoint->src_if4 && ((IS_ERR(rt) && + PTR_ERR(rt) == -EINVAL) || (!IS_ERR(rt) && + rt->dst.dev->ifindex != endpoint->src_if4)))) { + endpoint->src4.s_addr = 0; + *(__force __be32 *)&endpoint->src_if4 = 0; + fl.saddr = 0; + if (cache) + dst_cache_reset(cache); + if (!IS_ERR(rt)) + ip_rt_put(rt); + rt = ip_route_output_flow(sock_net(sock), &fl, sock); + } + if (unlikely(IS_ERR(rt))) { + ret = PTR_ERR(rt); + net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n", + wg->dev->name, &endpoint->addr, ret); + goto err; + } else if (unlikely(rt->dst.dev == skb->dev)) { + ip_rt_put(rt); + ret = -ELOOP; + net_dbg_ratelimited("%s: Avoiding routing loop to %pISpfsc\n", + wg->dev->name, &endpoint->addr); + goto err; + } + if (cache) + dst_cache_set_ip4(cache, &rt->dst, fl.saddr); + } + + skb->ignore_df = 1; + udp_tunnel_xmit_skb(rt, sock, skb, fl.saddr, fl.daddr, ds, + ip4_dst_hoplimit(&rt->dst), 0, fl.fl4_sport, + fl.fl4_dport, false, false); + goto out; + +err: + kfree_skb(skb); +out: + rcu_read_unlock_bh(); + return ret; +} + +static int send6(struct wg_device *wg, struct sk_buff *skb, + struct endpoint *endpoint, u8 ds, struct dst_cache *cache) +{ +#if IS_ENABLED(CONFIG_IPV6) + struct flowi6 fl = { + .saddr = endpoint->src6, + .daddr = endpoint->addr6.sin6_addr, + .fl6_dport = endpoint->addr6.sin6_port, + .flowi6_mark = wg->fwmark, + .flowi6_oif = endpoint->addr6.sin6_scope_id, + .flowi6_proto = IPPROTO_UDP + /* TODO: addr->sin6_flowinfo */ + }; + struct dst_entry *dst = NULL; + struct sock *sock; + int ret = 0; + + skb_mark_not_on_list(skb); + skb->dev = wg->dev; + skb->mark = wg->fwmark; + + rcu_read_lock_bh(); + sock = rcu_dereference_bh(wg->sock6); + + if (unlikely(!sock)) { + ret = -ENONET; + goto err; + } + + fl.fl6_sport = inet_sk(sock)->inet_sport; + + if (cache) + dst = dst_cache_get_ip6(cache, &fl.saddr); + + if (!dst) { + security_sk_classify_flow(sock, flowi6_to_flowi(&fl)); + if (unlikely(!ipv6_addr_any(&fl.saddr) && + !ipv6_chk_addr(sock_net(sock), &fl.saddr, NULL, 0))) { + endpoint->src6 = fl.saddr = in6addr_any; + if (cache) + dst_cache_reset(cache); + } + dst = ipv6_stub->ipv6_dst_lookup_flow(sock_net(sock), sock, &fl, + NULL); + if (unlikely(IS_ERR(dst))) { + ret = PTR_ERR(dst); + net_dbg_ratelimited("%s: No route to %pISpfsc, error %d\n", + wg->dev->name, &endpoint->addr, ret); + goto err; + } else if (unlikely(dst->dev == skb->dev)) { + dst_release(dst); + ret = -ELOOP; + net_dbg_ratelimited("%s: Avoiding routing loop to %pISpfsc\n", + wg->dev->name, &endpoint->addr); + goto err; + } + if (cache) + dst_cache_set_ip6(cache, dst, &fl.saddr); + } + + skb->ignore_df = 1; + udp_tunnel6_xmit_skb(dst, sock, skb, skb->dev, &fl.saddr, &fl.daddr, ds, + ip6_dst_hoplimit(dst), 0, fl.fl6_sport, + fl.fl6_dport, false); + goto out; + +err: + kfree_skb(skb); +out: + rcu_read_unlock_bh(); + return ret; +#else + return -EAFNOSUPPORT; +#endif +} + +int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, u8 ds) +{ + size_t skb_len = skb->len; + int ret = -EAFNOSUPPORT; + + read_lock_bh(&peer->endpoint_lock); + if (peer->endpoint.addr.sa_family == AF_INET) + ret = send4(peer->device, skb, &peer->endpoint, ds, + &peer->endpoint_cache); + else if (peer->endpoint.addr.sa_family == AF_INET6) + ret = send6(peer->device, skb, &peer->endpoint, ds, + &peer->endpoint_cache); + else + dev_kfree_skb(skb); + if (likely(!ret)) + peer->tx_bytes += skb_len; + read_unlock_bh(&peer->endpoint_lock); + + return ret; +} + +int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *buffer, + size_t len, u8 ds) +{ + struct sk_buff *skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC); + + if (unlikely(!skb)) + return -ENOMEM; + + skb_reserve(skb, SKB_HEADER_LEN); + skb_set_inner_network_header(skb, 0); + skb_put_data(skb, buffer, len); + return wg_socket_send_skb_to_peer(peer, skb, ds); +} + +int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg, + struct sk_buff *in_skb, void *buffer, + size_t len) +{ + int ret = 0; + struct sk_buff *skb; + struct endpoint endpoint; + + if (unlikely(!in_skb)) + return -EINVAL; + ret = wg_socket_endpoint_from_skb(&endpoint, in_skb); + if (unlikely(ret < 0)) + return ret; + + skb = alloc_skb(len + SKB_HEADER_LEN, GFP_ATOMIC); + if (unlikely(!skb)) + return -ENOMEM; + skb_reserve(skb, SKB_HEADER_LEN); + skb_set_inner_network_header(skb, 0); + skb_put_data(skb, buffer, len); + + if (endpoint.addr.sa_family == AF_INET) + ret = send4(wg, skb, &endpoint, 0, NULL); + else if (endpoint.addr.sa_family == AF_INET6) + ret = send6(wg, skb, &endpoint, 0, NULL); + /* No other possibilities if the endpoint is valid, which it is, + * as we checked above. + */ + + return ret; +} + +int wg_socket_endpoint_from_skb(struct endpoint *endpoint, + const struct sk_buff *skb) +{ + memset(endpoint, 0, sizeof(*endpoint)); + if (skb->protocol == htons(ETH_P_IP)) { + endpoint->addr4.sin_family = AF_INET; + endpoint->addr4.sin_port = udp_hdr(skb)->source; + endpoint->addr4.sin_addr.s_addr = ip_hdr(skb)->saddr; + endpoint->src4.s_addr = ip_hdr(skb)->daddr; + endpoint->src_if4 = skb->skb_iif; + } else if (skb->protocol == htons(ETH_P_IPV6)) { + endpoint->addr6.sin6_family = AF_INET6; + endpoint->addr6.sin6_port = udp_hdr(skb)->source; + endpoint->addr6.sin6_addr = ipv6_hdr(skb)->saddr; + endpoint->addr6.sin6_scope_id = ipv6_iface_scope_id( + &ipv6_hdr(skb)->saddr, skb->skb_iif); + endpoint->src6 = ipv6_hdr(skb)->daddr; + } else { + return -EINVAL; + } + return 0; +} + +static bool endpoint_eq(const struct endpoint *a, const struct endpoint *b) +{ + return (a->addr.sa_family == AF_INET && b->addr.sa_family == AF_INET && + a->addr4.sin_port == b->addr4.sin_port && + a->addr4.sin_addr.s_addr == b->addr4.sin_addr.s_addr && + a->src4.s_addr == b->src4.s_addr && a->src_if4 == b->src_if4) || + (a->addr.sa_family == AF_INET6 && + b->addr.sa_family == AF_INET6 && + a->addr6.sin6_port == b->addr6.sin6_port && + ipv6_addr_equal(&a->addr6.sin6_addr, &b->addr6.sin6_addr) && + a->addr6.sin6_scope_id == b->addr6.sin6_scope_id && + ipv6_addr_equal(&a->src6, &b->src6)) || + unlikely(!a->addr.sa_family && !b->addr.sa_family); +} + +void wg_socket_set_peer_endpoint(struct wg_peer *peer, + const struct endpoint *endpoint) +{ + /* First we check unlocked, in order to optimize, since it's pretty rare + * that an endpoint will change. If we happen to be mid-write, and two + * CPUs wind up writing the same thing or something slightly different, + * it doesn't really matter much either. + */ + if (endpoint_eq(endpoint, &peer->endpoint)) + return; + write_lock_bh(&peer->endpoint_lock); + if (endpoint->addr.sa_family == AF_INET) { + peer->endpoint.addr4 = endpoint->addr4; + peer->endpoint.src4 = endpoint->src4; + peer->endpoint.src_if4 = endpoint->src_if4; + } else if (endpoint->addr.sa_family == AF_INET6) { + peer->endpoint.addr6 = endpoint->addr6; + peer->endpoint.src6 = endpoint->src6; + } else { + goto out; + } + dst_cache_reset(&peer->endpoint_cache); +out: + write_unlock_bh(&peer->endpoint_lock); +} + +void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, + const struct sk_buff *skb) +{ + struct endpoint endpoint; + + if (!wg_socket_endpoint_from_skb(&endpoint, skb)) + wg_socket_set_peer_endpoint(peer, &endpoint); +} + +void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer) +{ + write_lock_bh(&peer->endpoint_lock); + memset(&peer->endpoint.src6, 0, sizeof(peer->endpoint.src6)); + dst_cache_reset(&peer->endpoint_cache); + write_unlock_bh(&peer->endpoint_lock); +} + +static int wg_receive(struct sock *sk, struct sk_buff *skb) +{ + struct wg_device *wg; + + if (unlikely(!sk)) + goto err; + wg = sk->sk_user_data; + if (unlikely(!wg)) + goto err; + skb_mark_not_on_list(skb); + wg_packet_receive(wg, skb); + return 0; + +err: + kfree_skb(skb); + return 0; +} + +static void sock_free(struct sock *sock) +{ + if (unlikely(!sock)) + return; + sk_clear_memalloc(sock); + udp_tunnel_sock_release(sock->sk_socket); +} + +static void set_sock_opts(struct socket *sock) +{ + sock->sk->sk_allocation = GFP_ATOMIC; + sock->sk->sk_sndbuf = INT_MAX; + sk_set_memalloc(sock->sk); +} + +int wg_socket_init(struct wg_device *wg, u16 port) +{ + int ret; + struct udp_tunnel_sock_cfg cfg = { + .sk_user_data = wg, + .encap_type = 1, + .encap_rcv = wg_receive + }; + struct socket *new4 = NULL, *new6 = NULL; + struct udp_port_cfg port4 = { + .family = AF_INET, + .local_ip.s_addr = htonl(INADDR_ANY), + .local_udp_port = htons(port), + .use_udp_checksums = true + }; +#if IS_ENABLED(CONFIG_IPV6) + int retries = 0; + struct udp_port_cfg port6 = { + .family = AF_INET6, + .local_ip6 = IN6ADDR_ANY_INIT, + .use_udp6_tx_checksums = true, + .use_udp6_rx_checksums = true, + .ipv6_v6only = true + }; +#endif + +#if IS_ENABLED(CONFIG_IPV6) +retry: +#endif + + ret = udp_sock_create(wg->creating_net, &port4, &new4); + if (ret < 0) { + pr_err("%s: Could not create IPv4 socket\n", wg->dev->name); + return ret; + } + set_sock_opts(new4); + setup_udp_tunnel_sock(wg->creating_net, new4, &cfg); + +#if IS_ENABLED(CONFIG_IPV6) + if (ipv6_mod_enabled()) { + port6.local_udp_port = inet_sk(new4->sk)->inet_sport; + ret = udp_sock_create(wg->creating_net, &port6, &new6); + if (ret < 0) { + udp_tunnel_sock_release(new4); + if (ret == -EADDRINUSE && !port && retries++ < 100) + goto retry; + pr_err("%s: Could not create IPv6 socket\n", + wg->dev->name); + return ret; + } + set_sock_opts(new6); + setup_udp_tunnel_sock(wg->creating_net, new6, &cfg); + } +#endif + + wg_socket_reinit(wg, new4->sk, new6 ? new6->sk : NULL); + return 0; +} + +void wg_socket_reinit(struct wg_device *wg, struct sock *new4, + struct sock *new6) +{ + struct sock *old4, *old6; + + mutex_lock(&wg->socket_update_lock); + old4 = rcu_dereference_protected(wg->sock4, + lockdep_is_held(&wg->socket_update_lock)); + old6 = rcu_dereference_protected(wg->sock6, + lockdep_is_held(&wg->socket_update_lock)); + rcu_assign_pointer(wg->sock4, new4); + rcu_assign_pointer(wg->sock6, new6); + if (new4) + wg->incoming_port = ntohs(inet_sk(new4)->inet_sport); + mutex_unlock(&wg->socket_update_lock); + synchronize_rcu(); + sock_free(old4); + sock_free(old6); +} diff --git a/drivers/net/wireguard/socket.h b/drivers/net/wireguard/socket.h new file mode 100644 index 0000000000000..bab5848efbcdf --- /dev/null +++ b/drivers/net/wireguard/socket.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_SOCKET_H +#define _WG_SOCKET_H + +#include +#include +#include +#include + +int wg_socket_init(struct wg_device *wg, u16 port); +void wg_socket_reinit(struct wg_device *wg, struct sock *new4, + struct sock *new6); +int wg_socket_send_buffer_to_peer(struct wg_peer *peer, void *data, + size_t len, u8 ds); +int wg_socket_send_skb_to_peer(struct wg_peer *peer, struct sk_buff *skb, + u8 ds); +int wg_socket_send_buffer_as_reply_to_skb(struct wg_device *wg, + struct sk_buff *in_skb, + void *out_buffer, size_t len); + +int wg_socket_endpoint_from_skb(struct endpoint *endpoint, + const struct sk_buff *skb); +void wg_socket_set_peer_endpoint(struct wg_peer *peer, + const struct endpoint *endpoint); +void wg_socket_set_peer_endpoint_from_skb(struct wg_peer *peer, + const struct sk_buff *skb); +void wg_socket_clear_peer_endpoint_src(struct wg_peer *peer); + +#if defined(CONFIG_DYNAMIC_DEBUG) || defined(DEBUG) +#define net_dbg_skb_ratelimited(fmt, dev, skb, ...) do { \ + struct endpoint __endpoint; \ + wg_socket_endpoint_from_skb(&__endpoint, skb); \ + net_dbg_ratelimited(fmt, dev, &__endpoint.addr, \ + ##__VA_ARGS__); \ + } while (0) +#else +#define net_dbg_skb_ratelimited(fmt, skb, ...) +#endif + +#endif /* _WG_SOCKET_H */ diff --git a/drivers/net/wireguard/timers.c b/drivers/net/wireguard/timers.c new file mode 100644 index 0000000000000..d54d32ac9bc41 --- /dev/null +++ b/drivers/net/wireguard/timers.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#include "timers.h" +#include "device.h" +#include "peer.h" +#include "queueing.h" +#include "socket.h" + +/* + * - Timer for retransmitting the handshake if we don't hear back after + * `REKEY_TIMEOUT + jitter` ms. + * + * - Timer for sending empty packet if we have received a packet but after have + * not sent one for `KEEPALIVE_TIMEOUT` ms. + * + * - Timer for initiating new handshake if we have sent a packet but after have + * not received one (even empty) for `(KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) + + * jitter` ms. + * + * - Timer for zeroing out all ephemeral keys after `(REJECT_AFTER_TIME * 3)` ms + * if no new keys have been received. + * + * - Timer for, if enabled, sending an empty authenticated packet every user- + * specified seconds. + */ + +static inline void mod_peer_timer(struct wg_peer *peer, + struct timer_list *timer, + unsigned long expires) +{ + rcu_read_lock_bh(); + if (likely(netif_running(peer->device->dev) && + !READ_ONCE(peer->is_dead))) + mod_timer(timer, expires); + rcu_read_unlock_bh(); +} + +static void wg_expired_retransmit_handshake(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, + timer_retransmit_handshake); + + if (peer->timer_handshake_attempts > MAX_TIMER_HANDSHAKES) { + pr_debug("%s: Handshake for peer %llu (%pISpfsc) did not complete after %d attempts, giving up\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, MAX_TIMER_HANDSHAKES + 2); + + del_timer(&peer->timer_send_keepalive); + /* We drop all packets without a keypair and don't try again, + * if we try unsuccessfully for too long to make a handshake. + */ + wg_packet_purge_staged_packets(peer); + + /* We set a timer for destroying any residue that might be left + * of a partial exchange. + */ + if (!timer_pending(&peer->timer_zero_key_material)) + mod_peer_timer(peer, &peer->timer_zero_key_material, + jiffies + REJECT_AFTER_TIME * 3 * HZ); + } else { + ++peer->timer_handshake_attempts; + pr_debug("%s: Handshake for peer %llu (%pISpfsc) did not complete after %d seconds, retrying (try %d)\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, REKEY_TIMEOUT, + peer->timer_handshake_attempts + 1); + + /* We clear the endpoint address src address, in case this is + * the cause of trouble. + */ + wg_socket_clear_peer_endpoint_src(peer); + + wg_packet_send_queued_handshake_initiation(peer, true); + } +} + +static void wg_expired_send_keepalive(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, timer_send_keepalive); + + wg_packet_send_keepalive(peer); + if (peer->timer_need_another_keepalive) { + peer->timer_need_another_keepalive = false; + mod_peer_timer(peer, &peer->timer_send_keepalive, + jiffies + KEEPALIVE_TIMEOUT * HZ); + } +} + +static void wg_expired_new_handshake(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, timer_new_handshake); + + pr_debug("%s: Retrying handshake with peer %llu (%pISpfsc) because we stopped hearing back after %d seconds\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, KEEPALIVE_TIMEOUT + REKEY_TIMEOUT); + /* We clear the endpoint address src address, in case this is the cause + * of trouble. + */ + wg_socket_clear_peer_endpoint_src(peer); + wg_packet_send_queued_handshake_initiation(peer, false); +} + +static void wg_expired_zero_key_material(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, timer_zero_key_material); + + rcu_read_lock_bh(); + if (!READ_ONCE(peer->is_dead)) { + wg_peer_get(peer); + if (!queue_work(peer->device->handshake_send_wq, + &peer->clear_peer_work)) + /* If the work was already on the queue, we want to drop + * the extra reference. + */ + wg_peer_put(peer); + } + rcu_read_unlock_bh(); +} + +static void wg_queued_expired_zero_key_material(struct work_struct *work) +{ + struct wg_peer *peer = container_of(work, struct wg_peer, + clear_peer_work); + + pr_debug("%s: Zeroing out all keys for peer %llu (%pISpfsc), since we haven't received a new one in %d seconds\n", + peer->device->dev->name, peer->internal_id, + &peer->endpoint.addr, REJECT_AFTER_TIME * 3); + wg_noise_handshake_clear(&peer->handshake); + wg_noise_keypairs_clear(&peer->keypairs); + wg_peer_put(peer); +} + +static void wg_expired_send_persistent_keepalive(struct timer_list *timer) +{ + struct wg_peer *peer = from_timer(peer, timer, + timer_persistent_keepalive); + + if (likely(peer->persistent_keepalive_interval)) + wg_packet_send_keepalive(peer); +} + +/* Should be called after an authenticated data packet is sent. */ +void wg_timers_data_sent(struct wg_peer *peer) +{ + if (!timer_pending(&peer->timer_new_handshake)) + mod_peer_timer(peer, &peer->timer_new_handshake, + jiffies + (KEEPALIVE_TIMEOUT + REKEY_TIMEOUT) * HZ + + prandom_u32_max(REKEY_TIMEOUT_JITTER_MAX_JIFFIES)); +} + +/* Should be called after an authenticated data packet is received. */ +void wg_timers_data_received(struct wg_peer *peer) +{ + if (likely(netif_running(peer->device->dev))) { + if (!timer_pending(&peer->timer_send_keepalive)) + mod_peer_timer(peer, &peer->timer_send_keepalive, + jiffies + KEEPALIVE_TIMEOUT * HZ); + else + peer->timer_need_another_keepalive = true; + } +} + +/* Should be called after any type of authenticated packet is sent, whether + * keepalive, data, or handshake. + */ +void wg_timers_any_authenticated_packet_sent(struct wg_peer *peer) +{ + del_timer(&peer->timer_send_keepalive); +} + +/* Should be called after any type of authenticated packet is received, whether + * keepalive, data, or handshake. + */ +void wg_timers_any_authenticated_packet_received(struct wg_peer *peer) +{ + del_timer(&peer->timer_new_handshake); +} + +/* Should be called after a handshake initiation message is sent. */ +void wg_timers_handshake_initiated(struct wg_peer *peer) +{ + mod_peer_timer(peer, &peer->timer_retransmit_handshake, + jiffies + REKEY_TIMEOUT * HZ + + prandom_u32_max(REKEY_TIMEOUT_JITTER_MAX_JIFFIES)); +} + +/* Should be called after a handshake response message is received and processed + * or when getting key confirmation via the first data message. + */ +void wg_timers_handshake_complete(struct wg_peer *peer) +{ + del_timer(&peer->timer_retransmit_handshake); + peer->timer_handshake_attempts = 0; + peer->sent_lastminute_handshake = false; + ktime_get_real_ts64(&peer->walltime_last_handshake); +} + +/* Should be called after an ephemeral key is created, which is before sending a + * handshake response or after receiving a handshake response. + */ +void wg_timers_session_derived(struct wg_peer *peer) +{ + mod_peer_timer(peer, &peer->timer_zero_key_material, + jiffies + REJECT_AFTER_TIME * 3 * HZ); +} + +/* Should be called before a packet with authentication, whether + * keepalive, data, or handshakem is sent, or after one is received. + */ +void wg_timers_any_authenticated_packet_traversal(struct wg_peer *peer) +{ + if (peer->persistent_keepalive_interval) + mod_peer_timer(peer, &peer->timer_persistent_keepalive, + jiffies + peer->persistent_keepalive_interval * HZ); +} + +void wg_timers_init(struct wg_peer *peer) +{ + timer_setup(&peer->timer_retransmit_handshake, + wg_expired_retransmit_handshake, 0); + timer_setup(&peer->timer_send_keepalive, wg_expired_send_keepalive, 0); + timer_setup(&peer->timer_new_handshake, wg_expired_new_handshake, 0); + timer_setup(&peer->timer_zero_key_material, + wg_expired_zero_key_material, 0); + timer_setup(&peer->timer_persistent_keepalive, + wg_expired_send_persistent_keepalive, 0); + INIT_WORK(&peer->clear_peer_work, wg_queued_expired_zero_key_material); + peer->timer_handshake_attempts = 0; + peer->sent_lastminute_handshake = false; + peer->timer_need_another_keepalive = false; +} + +void wg_timers_stop(struct wg_peer *peer) +{ + del_timer_sync(&peer->timer_retransmit_handshake); + del_timer_sync(&peer->timer_send_keepalive); + del_timer_sync(&peer->timer_new_handshake); + del_timer_sync(&peer->timer_zero_key_material); + del_timer_sync(&peer->timer_persistent_keepalive); + flush_work(&peer->clear_peer_work); +} diff --git a/drivers/net/wireguard/timers.h b/drivers/net/wireguard/timers.h new file mode 100644 index 0000000000000..f0653dcb13266 --- /dev/null +++ b/drivers/net/wireguard/timers.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _WG_TIMERS_H +#define _WG_TIMERS_H + +#include + +struct wg_peer; + +void wg_timers_init(struct wg_peer *peer); +void wg_timers_stop(struct wg_peer *peer); +void wg_timers_data_sent(struct wg_peer *peer); +void wg_timers_data_received(struct wg_peer *peer); +void wg_timers_any_authenticated_packet_sent(struct wg_peer *peer); +void wg_timers_any_authenticated_packet_received(struct wg_peer *peer); +void wg_timers_handshake_initiated(struct wg_peer *peer); +void wg_timers_handshake_complete(struct wg_peer *peer); +void wg_timers_session_derived(struct wg_peer *peer); +void wg_timers_any_authenticated_packet_traversal(struct wg_peer *peer); + +static inline bool wg_birthdate_has_expired(u64 birthday_nanoseconds, + u64 expiration_seconds) +{ + return (s64)(birthday_nanoseconds + expiration_seconds * NSEC_PER_SEC) + <= (s64)ktime_get_coarse_boottime_ns(); +} + +#endif /* _WG_TIMERS_H */ diff --git a/drivers/net/wireguard/version.h b/drivers/net/wireguard/version.h new file mode 100644 index 0000000000000..a1a269a116344 --- /dev/null +++ b/drivers/net/wireguard/version.h @@ -0,0 +1 @@ +#define WIREGUARD_VERSION "1.0.0" diff --git a/include/crypto/curve25519.h b/include/crypto/curve25519.h index 4e6dc840b1592..9ecb3c1f0f15d 100644 --- a/include/crypto/curve25519.h +++ b/include/crypto/curve25519.h @@ -33,7 +33,8 @@ bool __must_check curve25519(u8 mypublic[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE], const u8 basepoint[CURVE25519_KEY_SIZE]) { - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CURVE25519)) + if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CURVE25519) && + (!IS_ENABLED(CONFIG_CRYPTO_CURVE25519_X86) || IS_ENABLED(CONFIG_AS_ADX))) curve25519_arch(mypublic, secret, basepoint); else curve25519_generic(mypublic, secret, basepoint); @@ -49,7 +50,8 @@ __must_check curve25519_generate_public(u8 pub[CURVE25519_KEY_SIZE], CURVE25519_KEY_SIZE))) return false; - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CURVE25519)) + if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CURVE25519) && + (!IS_ENABLED(CONFIG_CRYPTO_CURVE25519_X86) || IS_ENABLED(CONFIG_AS_ADX))) curve25519_base_arch(pub, secret); else curve25519_generic(pub, secret, curve25519_base_point); diff --git a/include/crypto/internal/poly1305.h b/include/crypto/internal/poly1305.h index 479b0cab2a1ad..064e52ca52480 100644 --- a/include/crypto/internal/poly1305.h +++ b/include/crypto/internal/poly1305.h @@ -11,48 +11,23 @@ #include /* - * Poly1305 core functions. These implement the ε-almost-∆-universal hash - * function underlying the Poly1305 MAC, i.e. they don't add an encrypted nonce - * ("s key") at the end. They also only support block-aligned inputs. + * Poly1305 core functions. These only accept whole blocks; the caller must + * handle any needed block buffering and padding. 'hibit' must be 1 for any + * full blocks, or 0 for the final block if it had to be padded. If 'nonce' is + * non-NULL, then it's added at the end to compute the Poly1305 MAC. Otherwise, + * only the ε-almost-∆-universal hash function (not the full MAC) is computed. */ -void poly1305_core_setkey(struct poly1305_key *key, const u8 *raw_key); + +void poly1305_core_setkey(struct poly1305_core_key *key, const u8 *raw_key); static inline void poly1305_core_init(struct poly1305_state *state) { *state = (struct poly1305_state){}; } void poly1305_core_blocks(struct poly1305_state *state, - const struct poly1305_key *key, const void *src, + const struct poly1305_core_key *key, const void *src, unsigned int nblocks, u32 hibit); -void poly1305_core_emit(const struct poly1305_state *state, void *dst); - -/* - * Poly1305 requires a unique key for each tag, which implies that we can't set - * it on the tfm that gets accessed by multiple users simultaneously. Instead we - * expect the key as the first 32 bytes in the update() call. - */ -static inline -unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, - const u8 *src, unsigned int srclen) -{ - if (!dctx->sset) { - if (!dctx->rset && srclen >= POLY1305_BLOCK_SIZE) { - poly1305_core_setkey(dctx->r, src); - src += POLY1305_BLOCK_SIZE; - srclen -= POLY1305_BLOCK_SIZE; - dctx->rset = 1; - } - if (srclen >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(src + 0); - dctx->s[1] = get_unaligned_le32(src + 4); - dctx->s[2] = get_unaligned_le32(src + 8); - dctx->s[3] = get_unaligned_le32(src + 12); - src += POLY1305_BLOCK_SIZE; - srclen -= POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - } - return srclen; -} +void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], + void *dst); #endif diff --git a/include/crypto/nhpoly1305.h b/include/crypto/nhpoly1305.h index 53c04423c582e..306925fea1904 100644 --- a/include/crypto/nhpoly1305.h +++ b/include/crypto/nhpoly1305.h @@ -7,7 +7,7 @@ #define _NHPOLY1305_H #include -#include +#include /* NH parameterization: */ @@ -33,7 +33,7 @@ #define NHPOLY1305_KEY_SIZE (POLY1305_BLOCK_SIZE + NH_KEY_BYTES) struct nhpoly1305_key { - struct poly1305_key poly_key; + struct poly1305_core_key poly_key; u32 nh_key[NH_KEY_WORDS]; }; diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h index 74c6e1cd73eee..f1f67fc749cf4 100644 --- a/include/crypto/poly1305.h +++ b/include/crypto/poly1305.h @@ -13,12 +13,29 @@ #define POLY1305_KEY_SIZE 32 #define POLY1305_DIGEST_SIZE 16 +/* The poly1305_key and poly1305_state types are mostly opaque and + * implementation-defined. Limbs might be in base 2^64 or base 2^26, or + * different yet. The union type provided keeps these 64-bit aligned for the + * case in which this is implemented using 64x64 multiplies. + */ + struct poly1305_key { - u32 r[5]; /* key, base 2^26 */ + union { + u32 r[5]; + u64 r64[3]; + }; +}; + +struct poly1305_core_key { + struct poly1305_key key; + struct poly1305_key precomputed_s; }; struct poly1305_state { - u32 h[5]; /* accumulator, base 2^26 */ + union { + u32 h[5]; + u64 h64[3]; + }; }; struct poly1305_desc_ctx { @@ -35,7 +52,10 @@ struct poly1305_desc_ctx { /* accumulator */ struct poly1305_state h; /* key */ - struct poly1305_key r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE]; + union { + struct poly1305_key opaque_r[CONFIG_CRYPTO_LIB_POLY1305_RSIZE]; + struct poly1305_core_key core_r; + }; }; void poly1305_init_arch(struct poly1305_desc_ctx *desc, const u8 *key); diff --git a/include/linux/icmpv6.h b/include/linux/icmpv6.h index ef1cbb5f454f7..33d3796023144 100644 --- a/include/linux/icmpv6.h +++ b/include/linux/icmpv6.h @@ -22,12 +22,22 @@ extern int inet6_unregister_icmp_sender(ip6_icmp_send_t *fn); int ip6_err_gen_icmpv6_unreach(struct sk_buff *skb, int nhs, int type, unsigned int data_len); +#if IS_ENABLED(CONFIG_NF_NAT) +void icmpv6_ndo_send(struct sk_buff *skb_in, u8 type, u8 code, __u32 info); +#else +#define icmpv6_ndo_send icmpv6_send +#endif + #else static inline void icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info) { +} +static inline void icmpv6_ndo_send(struct sk_buff *skb, + u8 type, u8 code, __u32 info) +{ } #endif diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index f0553ed6f4c64..ca0df5f019dfd 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1480,6 +1480,11 @@ static inline void skb_mark_not_on_list(struct sk_buff *skb) skb->next = NULL; } +/* Iterate through singly-linked GSO fragments of an skb. */ +#define skb_list_walk_safe(first, skb, next_skb) \ + for ((skb) = (first), (next_skb) = (skb) ? (skb)->next : NULL; (skb); \ + (skb) = (next_skb), (next_skb) = (skb) ? (skb)->next : NULL) + static inline void skb_list_del_init(struct sk_buff *skb) { __list_del_entry(&skb->list); diff --git a/include/net/icmp.h b/include/net/icmp.h index 5d4bfdba9adf0..9ac2d2672a938 100644 --- a/include/net/icmp.h +++ b/include/net/icmp.h @@ -43,6 +43,12 @@ static inline void icmp_send(struct sk_buff *skb_in, int type, int code, __be32 __icmp_send(skb_in, type, code, info, &IPCB(skb_in)->opt); } +#if IS_ENABLED(CONFIG_NF_NAT) +void icmp_ndo_send(struct sk_buff *skb_in, int type, int code, __be32 info); +#else +#define icmp_ndo_send icmp_send +#endif + int icmp_rcv(struct sk_buff *skb); int icmp_err(struct sk_buff *skb, u32 info); int icmp_init(void); diff --git a/include/uapi/linux/wireguard.h b/include/uapi/linux/wireguard.h new file mode 100644 index 0000000000000..ae88be14c9478 --- /dev/null +++ b/include/uapi/linux/wireguard.h @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * Documentation + * ============= + * + * The below enums and macros are for interfacing with WireGuard, using generic + * netlink, with family WG_GENL_NAME and version WG_GENL_VERSION. It defines two + * methods: get and set. Note that while they share many common attributes, + * these two functions actually accept a slightly different set of inputs and + * outputs. + * + * WG_CMD_GET_DEVICE + * ----------------- + * + * May only be called via NLM_F_REQUEST | NLM_F_DUMP. The command should contain + * one but not both of: + * + * WGDEVICE_A_IFINDEX: NLA_U32 + * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 + * + * The kernel will then return several messages (NLM_F_MULTI) containing the + * following tree of nested items: + * + * WGDEVICE_A_IFINDEX: NLA_U32 + * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 + * WGDEVICE_A_PRIVATE_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGDEVICE_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGDEVICE_A_LISTEN_PORT: NLA_U16 + * WGDEVICE_A_FWMARK: NLA_U32 + * WGDEVICE_A_PEERS: NLA_NESTED + * 0: NLA_NESTED + * WGPEER_A_PUBLIC_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGPEER_A_PRESHARED_KEY: NLA_EXACT_LEN, len WG_KEY_LEN + * WGPEER_A_ENDPOINT: NLA_MIN_LEN(struct sockaddr), struct sockaddr_in or struct sockaddr_in6 + * WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL: NLA_U16 + * WGPEER_A_LAST_HANDSHAKE_TIME: NLA_EXACT_LEN, struct __kernel_timespec + * WGPEER_A_RX_BYTES: NLA_U64 + * WGPEER_A_TX_BYTES: NLA_U64 + * WGPEER_A_ALLOWEDIPS: NLA_NESTED + * 0: NLA_NESTED + * WGALLOWEDIP_A_FAMILY: NLA_U16 + * WGALLOWEDIP_A_IPADDR: NLA_MIN_LEN(struct in_addr), struct in_addr or struct in6_addr + * WGALLOWEDIP_A_CIDR_MASK: NLA_U8 + * 0: NLA_NESTED + * ... + * 0: NLA_NESTED + * ... + * ... + * WGPEER_A_PROTOCOL_VERSION: NLA_U32 + * 0: NLA_NESTED + * ... + * ... + * + * It is possible that all of the allowed IPs of a single peer will not + * fit within a single netlink message. In that case, the same peer will + * be written in the following message, except it will only contain + * WGPEER_A_PUBLIC_KEY and WGPEER_A_ALLOWEDIPS. This may occur several + * times in a row for the same peer. It is then up to the receiver to + * coalesce adjacent peers. Likewise, it is possible that all peers will + * not fit within a single message. So, subsequent peers will be sent + * in following messages, except those will only contain WGDEVICE_A_IFNAME + * and WGDEVICE_A_PEERS. It is then up to the receiver to coalesce these + * messages to form the complete list of peers. + * + * Since this is an NLA_F_DUMP command, the final message will always be + * NLMSG_DONE, even if an error occurs. However, this NLMSG_DONE message + * contains an integer error code. It is either zero or a negative error + * code corresponding to the errno. + * + * WG_CMD_SET_DEVICE + * ----------------- + * + * May only be called via NLM_F_REQUEST. The command should contain the + * following tree of nested items, containing one but not both of + * WGDEVICE_A_IFINDEX and WGDEVICE_A_IFNAME: + * + * WGDEVICE_A_IFINDEX: NLA_U32 + * WGDEVICE_A_IFNAME: NLA_NUL_STRING, maxlen IFNAMSIZ - 1 + * WGDEVICE_A_FLAGS: NLA_U32, 0 or WGDEVICE_F_REPLACE_PEERS if all current + * peers should be removed prior to adding the list below. + * WGDEVICE_A_PRIVATE_KEY: len WG_KEY_LEN, all zeros to remove + * WGDEVICE_A_LISTEN_PORT: NLA_U16, 0 to choose randomly + * WGDEVICE_A_FWMARK: NLA_U32, 0 to disable + * WGDEVICE_A_PEERS: NLA_NESTED + * 0: NLA_NESTED + * WGPEER_A_PUBLIC_KEY: len WG_KEY_LEN + * WGPEER_A_FLAGS: NLA_U32, 0 and/or WGPEER_F_REMOVE_ME if the + * specified peer should not exist at the end of the + * operation, rather than added/updated and/or + * WGPEER_F_REPLACE_ALLOWEDIPS if all current allowed + * IPs of this peer should be removed prior to adding + * the list below and/or WGPEER_F_UPDATE_ONLY if the + * peer should only be set if it already exists. + * WGPEER_A_PRESHARED_KEY: len WG_KEY_LEN, all zeros to remove + * WGPEER_A_ENDPOINT: struct sockaddr_in or struct sockaddr_in6 + * WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL: NLA_U16, 0 to disable + * WGPEER_A_ALLOWEDIPS: NLA_NESTED + * 0: NLA_NESTED + * WGALLOWEDIP_A_FAMILY: NLA_U16 + * WGALLOWEDIP_A_IPADDR: struct in_addr or struct in6_addr + * WGALLOWEDIP_A_CIDR_MASK: NLA_U8 + * 0: NLA_NESTED + * ... + * 0: NLA_NESTED + * ... + * ... + * WGPEER_A_PROTOCOL_VERSION: NLA_U32, should not be set or used at + * all by most users of this API, as the + * most recent protocol will be used when + * this is unset. Otherwise, must be set + * to 1. + * 0: NLA_NESTED + * ... + * ... + * + * It is possible that the amount of configuration data exceeds that of + * the maximum message length accepted by the kernel. In that case, several + * messages should be sent one after another, with each successive one + * filling in information not contained in the prior. Note that if + * WGDEVICE_F_REPLACE_PEERS is specified in the first message, it probably + * should not be specified in fragments that come after, so that the list + * of peers is only cleared the first time but appended after. Likewise for + * peers, if WGPEER_F_REPLACE_ALLOWEDIPS is specified in the first message + * of a peer, it likely should not be specified in subsequent fragments. + * + * If an error occurs, NLMSG_ERROR will reply containing an errno. + */ + +#ifndef _WG_UAPI_WIREGUARD_H +#define _WG_UAPI_WIREGUARD_H + +#define WG_GENL_NAME "wireguard" +#define WG_GENL_VERSION 1 + +#define WG_KEY_LEN 32 + +enum wg_cmd { + WG_CMD_GET_DEVICE, + WG_CMD_SET_DEVICE, + __WG_CMD_MAX +}; +#define WG_CMD_MAX (__WG_CMD_MAX - 1) + +enum wgdevice_flag { + WGDEVICE_F_REPLACE_PEERS = 1U << 0, + __WGDEVICE_F_ALL = WGDEVICE_F_REPLACE_PEERS +}; +enum wgdevice_attribute { + WGDEVICE_A_UNSPEC, + WGDEVICE_A_IFINDEX, + WGDEVICE_A_IFNAME, + WGDEVICE_A_PRIVATE_KEY, + WGDEVICE_A_PUBLIC_KEY, + WGDEVICE_A_FLAGS, + WGDEVICE_A_LISTEN_PORT, + WGDEVICE_A_FWMARK, + WGDEVICE_A_PEERS, + __WGDEVICE_A_LAST +}; +#define WGDEVICE_A_MAX (__WGDEVICE_A_LAST - 1) + +enum wgpeer_flag { + WGPEER_F_REMOVE_ME = 1U << 0, + WGPEER_F_REPLACE_ALLOWEDIPS = 1U << 1, + WGPEER_F_UPDATE_ONLY = 1U << 2, + __WGPEER_F_ALL = WGPEER_F_REMOVE_ME | WGPEER_F_REPLACE_ALLOWEDIPS | + WGPEER_F_UPDATE_ONLY +}; +enum wgpeer_attribute { + WGPEER_A_UNSPEC, + WGPEER_A_PUBLIC_KEY, + WGPEER_A_PRESHARED_KEY, + WGPEER_A_FLAGS, + WGPEER_A_ENDPOINT, + WGPEER_A_PERSISTENT_KEEPALIVE_INTERVAL, + WGPEER_A_LAST_HANDSHAKE_TIME, + WGPEER_A_RX_BYTES, + WGPEER_A_TX_BYTES, + WGPEER_A_ALLOWEDIPS, + WGPEER_A_PROTOCOL_VERSION, + __WGPEER_A_LAST +}; +#define WGPEER_A_MAX (__WGPEER_A_LAST - 1) + +enum wgallowedip_attribute { + WGALLOWEDIP_A_UNSPEC, + WGALLOWEDIP_A_FAMILY, + WGALLOWEDIP_A_IPADDR, + WGALLOWEDIP_A_CIDR_MASK, + __WGALLOWEDIP_A_LAST +}; +#define WGALLOWEDIP_A_MAX (__WGALLOWEDIP_A_LAST - 1) + +#endif /* _WG_UAPI_WIREGUARD_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 0b2c4fce26d95..14c032de276e6 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -90,7 +90,7 @@ config CRYPTO_LIB_DES config CRYPTO_LIB_POLY1305_RSIZE int default 2 if MIPS - default 4 if X86_64 + default 11 if X86_64 default 9 if ARM || ARM64 default 1 diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 34a701ab8b92e..d19e5195569ee 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -28,7 +28,9 @@ obj-$(CONFIG_CRYPTO_LIB_DES) += libdes.o libdes-y := des.o obj-$(CONFIG_CRYPTO_LIB_POLY1305_GENERIC) += libpoly1305.o -libpoly1305-y := poly1305.o +libpoly1305-y := poly1305-donna32.o +libpoly1305-$(CONFIG_ARCH_SUPPORTS_INT128) := poly1305-donna64.o +libpoly1305-y += poly1305.o obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o libsha256-y := sha256.o diff --git a/lib/crypto/chacha20poly1305-selftest.c b/lib/crypto/chacha20poly1305-selftest.c index 465de46dbdef2..fa43deda2660d 100644 --- a/lib/crypto/chacha20poly1305-selftest.c +++ b/lib/crypto/chacha20poly1305-selftest.c @@ -4,6 +4,7 @@ */ #include +#include #include #include @@ -1926,6 +1927,1104 @@ static const u8 enc_key012[] __initconst = { 0x65, 0x91, 0x6e, 0x2a, 0x79, 0x22, 0xda, 0x64 }; +/* wycheproof - rfc7539 */ +static const u8 enc_input013[] __initconst = { + 0x4c, 0x61, 0x64, 0x69, 0x65, 0x73, 0x20, 0x61, + 0x6e, 0x64, 0x20, 0x47, 0x65, 0x6e, 0x74, 0x6c, + 0x65, 0x6d, 0x65, 0x6e, 0x20, 0x6f, 0x66, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x63, 0x6c, 0x61, 0x73, + 0x73, 0x20, 0x6f, 0x66, 0x20, 0x27, 0x39, 0x39, + 0x3a, 0x20, 0x49, 0x66, 0x20, 0x49, 0x20, 0x63, + 0x6f, 0x75, 0x6c, 0x64, 0x20, 0x6f, 0x66, 0x66, + 0x65, 0x72, 0x20, 0x79, 0x6f, 0x75, 0x20, 0x6f, + 0x6e, 0x6c, 0x79, 0x20, 0x6f, 0x6e, 0x65, 0x20, + 0x74, 0x69, 0x70, 0x20, 0x66, 0x6f, 0x72, 0x20, + 0x74, 0x68, 0x65, 0x20, 0x66, 0x75, 0x74, 0x75, + 0x72, 0x65, 0x2c, 0x20, 0x73, 0x75, 0x6e, 0x73, + 0x63, 0x72, 0x65, 0x65, 0x6e, 0x20, 0x77, 0x6f, + 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69, + 0x74, 0x2e +}; +static const u8 enc_output013[] __initconst = { + 0xd3, 0x1a, 0x8d, 0x34, 0x64, 0x8e, 0x60, 0xdb, + 0x7b, 0x86, 0xaf, 0xbc, 0x53, 0xef, 0x7e, 0xc2, + 0xa4, 0xad, 0xed, 0x51, 0x29, 0x6e, 0x08, 0xfe, + 0xa9, 0xe2, 0xb5, 0xa7, 0x36, 0xee, 0x62, 0xd6, + 0x3d, 0xbe, 0xa4, 0x5e, 0x8c, 0xa9, 0x67, 0x12, + 0x82, 0xfa, 0xfb, 0x69, 0xda, 0x92, 0x72, 0x8b, + 0x1a, 0x71, 0xde, 0x0a, 0x9e, 0x06, 0x0b, 0x29, + 0x05, 0xd6, 0xa5, 0xb6, 0x7e, 0xcd, 0x3b, 0x36, + 0x92, 0xdd, 0xbd, 0x7f, 0x2d, 0x77, 0x8b, 0x8c, + 0x98, 0x03, 0xae, 0xe3, 0x28, 0x09, 0x1b, 0x58, + 0xfa, 0xb3, 0x24, 0xe4, 0xfa, 0xd6, 0x75, 0x94, + 0x55, 0x85, 0x80, 0x8b, 0x48, 0x31, 0xd7, 0xbc, + 0x3f, 0xf4, 0xde, 0xf0, 0x8e, 0x4b, 0x7a, 0x9d, + 0xe5, 0x76, 0xd2, 0x65, 0x86, 0xce, 0xc6, 0x4b, + 0x61, 0x16, 0x1a, 0xe1, 0x0b, 0x59, 0x4f, 0x09, + 0xe2, 0x6a, 0x7e, 0x90, 0x2e, 0xcb, 0xd0, 0x60, + 0x06, 0x91 +}; +static const u8 enc_assoc013[] __initconst = { + 0x50, 0x51, 0x52, 0x53, 0xc0, 0xc1, 0xc2, 0xc3, + 0xc4, 0xc5, 0xc6, 0xc7 +}; +static const u8 enc_nonce013[] __initconst = { + 0x07, 0x00, 0x00, 0x00, 0x40, 0x41, 0x42, 0x43, + 0x44, 0x45, 0x46, 0x47 +}; +static const u8 enc_key013[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - misc */ +static const u8 enc_input014[] __initconst = { }; +static const u8 enc_output014[] __initconst = { + 0x76, 0xac, 0xb3, 0x42, 0xcf, 0x31, 0x66, 0xa5, + 0xb6, 0x3c, 0x0c, 0x0e, 0xa1, 0x38, 0x3c, 0x8d +}; +static const u8 enc_assoc014[] __initconst = { }; +static const u8 enc_nonce014[] __initconst = { + 0x4d, 0xa5, 0xbf, 0x8d, 0xfd, 0x58, 0x52, 0xc1, + 0xea, 0x12, 0x37, 0x9d +}; +static const u8 enc_key014[] __initconst = { + 0x80, 0xba, 0x31, 0x92, 0xc8, 0x03, 0xce, 0x96, + 0x5e, 0xa3, 0x71, 0xd5, 0xff, 0x07, 0x3c, 0xf0, + 0xf4, 0x3b, 0x6a, 0x2a, 0xb5, 0x76, 0xb2, 0x08, + 0x42, 0x6e, 0x11, 0x40, 0x9c, 0x09, 0xb9, 0xb0 +}; + +/* wycheproof - misc */ +static const u8 enc_input015[] __initconst = { }; +static const u8 enc_output015[] __initconst = { + 0x90, 0x6f, 0xa6, 0x28, 0x4b, 0x52, 0xf8, 0x7b, + 0x73, 0x59, 0xcb, 0xaa, 0x75, 0x63, 0xc7, 0x09 +}; +static const u8 enc_assoc015[] __initconst = { + 0xbd, 0x50, 0x67, 0x64, 0xf2, 0xd2, 0xc4, 0x10 +}; +static const u8 enc_nonce015[] __initconst = { + 0xa9, 0x2e, 0xf0, 0xac, 0x99, 0x1d, 0xd5, 0x16, + 0xa3, 0xc6, 0xf6, 0x89 +}; +static const u8 enc_key015[] __initconst = { + 0x7a, 0x4c, 0xd7, 0x59, 0x17, 0x2e, 0x02, 0xeb, + 0x20, 0x4d, 0xb2, 0xc3, 0xf5, 0xc7, 0x46, 0x22, + 0x7d, 0xf5, 0x84, 0xfc, 0x13, 0x45, 0x19, 0x63, + 0x91, 0xdb, 0xb9, 0x57, 0x7a, 0x25, 0x07, 0x42 +}; + +/* wycheproof - misc */ +static const u8 enc_input016[] __initconst = { + 0x2a +}; +static const u8 enc_output016[] __initconst = { + 0x3a, 0xca, 0xc2, 0x7d, 0xec, 0x09, 0x68, 0x80, + 0x1e, 0x9f, 0x6e, 0xde, 0xd6, 0x9d, 0x80, 0x75, + 0x22 +}; +static const u8 enc_assoc016[] __initconst = { }; +static const u8 enc_nonce016[] __initconst = { + 0x99, 0xe2, 0x3e, 0xc4, 0x89, 0x85, 0xbc, 0xcd, + 0xee, 0xab, 0x60, 0xf1 +}; +static const u8 enc_key016[] __initconst = { + 0xcc, 0x56, 0xb6, 0x80, 0x55, 0x2e, 0xb7, 0x50, + 0x08, 0xf5, 0x48, 0x4b, 0x4c, 0xb8, 0x03, 0xfa, + 0x50, 0x63, 0xeb, 0xd6, 0xea, 0xb9, 0x1f, 0x6a, + 0xb6, 0xae, 0xf4, 0x91, 0x6a, 0x76, 0x62, 0x73 +}; + +/* wycheproof - misc */ +static const u8 enc_input017[] __initconst = { + 0x51 +}; +static const u8 enc_output017[] __initconst = { + 0xc4, 0x16, 0x83, 0x10, 0xca, 0x45, 0xb1, 0xf7, + 0xc6, 0x6c, 0xad, 0x4e, 0x99, 0xe4, 0x3f, 0x72, + 0xb9 +}; +static const u8 enc_assoc017[] __initconst = { + 0x91, 0xca, 0x6c, 0x59, 0x2c, 0xbc, 0xca, 0x53 +}; +static const u8 enc_nonce017[] __initconst = { + 0xab, 0x0d, 0xca, 0x71, 0x6e, 0xe0, 0x51, 0xd2, + 0x78, 0x2f, 0x44, 0x03 +}; +static const u8 enc_key017[] __initconst = { + 0x46, 0xf0, 0x25, 0x49, 0x65, 0xf7, 0x69, 0xd5, + 0x2b, 0xdb, 0x4a, 0x70, 0xb4, 0x43, 0x19, 0x9f, + 0x8e, 0xf2, 0x07, 0x52, 0x0d, 0x12, 0x20, 0xc5, + 0x5e, 0x4b, 0x70, 0xf0, 0xfd, 0xa6, 0x20, 0xee +}; + +/* wycheproof - misc */ +static const u8 enc_input018[] __initconst = { + 0x5c, 0x60 +}; +static const u8 enc_output018[] __initconst = { + 0x4d, 0x13, 0x91, 0xe8, 0xb6, 0x1e, 0xfb, 0x39, + 0xc1, 0x22, 0x19, 0x54, 0x53, 0x07, 0x7b, 0x22, + 0xe5, 0xe2 +}; +static const u8 enc_assoc018[] __initconst = { }; +static const u8 enc_nonce018[] __initconst = { + 0x46, 0x1a, 0xf1, 0x22, 0xe9, 0xf2, 0xe0, 0x34, + 0x7e, 0x03, 0xf2, 0xdb +}; +static const u8 enc_key018[] __initconst = { + 0x2f, 0x7f, 0x7e, 0x4f, 0x59, 0x2b, 0xb3, 0x89, + 0x19, 0x49, 0x89, 0x74, 0x35, 0x07, 0xbf, 0x3e, + 0xe9, 0xcb, 0xde, 0x17, 0x86, 0xb6, 0x69, 0x5f, + 0xe6, 0xc0, 0x25, 0xfd, 0x9b, 0xa4, 0xc1, 0x00 +}; + +/* wycheproof - misc */ +static const u8 enc_input019[] __initconst = { + 0xdd, 0xf2 +}; +static const u8 enc_output019[] __initconst = { + 0xb6, 0x0d, 0xea, 0xd0, 0xfd, 0x46, 0x97, 0xec, + 0x2e, 0x55, 0x58, 0x23, 0x77, 0x19, 0xd0, 0x24, + 0x37, 0xa2 +}; +static const u8 enc_assoc019[] __initconst = { + 0x88, 0x36, 0x4f, 0xc8, 0x06, 0x05, 0x18, 0xbf +}; +static const u8 enc_nonce019[] __initconst = { + 0x61, 0x54, 0x6b, 0xa5, 0xf1, 0x72, 0x05, 0x90, + 0xb6, 0x04, 0x0a, 0xc6 +}; +static const u8 enc_key019[] __initconst = { + 0xc8, 0x83, 0x3d, 0xce, 0x5e, 0xa9, 0xf2, 0x48, + 0xaa, 0x20, 0x30, 0xea, 0xcf, 0xe7, 0x2b, 0xff, + 0xe6, 0x9a, 0x62, 0x0c, 0xaf, 0x79, 0x33, 0x44, + 0xe5, 0x71, 0x8f, 0xe0, 0xd7, 0xab, 0x1a, 0x58 +}; + +/* wycheproof - misc */ +static const u8 enc_input020[] __initconst = { + 0xab, 0x85, 0xe9, 0xc1, 0x57, 0x17, 0x31 +}; +static const u8 enc_output020[] __initconst = { + 0x5d, 0xfe, 0x34, 0x40, 0xdb, 0xb3, 0xc3, 0xed, + 0x7a, 0x43, 0x4e, 0x26, 0x02, 0xd3, 0x94, 0x28, + 0x1e, 0x0a, 0xfa, 0x9f, 0xb7, 0xaa, 0x42 +}; +static const u8 enc_assoc020[] __initconst = { }; +static const u8 enc_nonce020[] __initconst = { + 0x3c, 0x4e, 0x65, 0x4d, 0x66, 0x3f, 0xa4, 0x59, + 0x6d, 0xc5, 0x5b, 0xb7 +}; +static const u8 enc_key020[] __initconst = { + 0x55, 0x56, 0x81, 0x58, 0xd3, 0xa6, 0x48, 0x3f, + 0x1f, 0x70, 0x21, 0xea, 0xb6, 0x9b, 0x70, 0x3f, + 0x61, 0x42, 0x51, 0xca, 0xdc, 0x1a, 0xf5, 0xd3, + 0x4a, 0x37, 0x4f, 0xdb, 0xfc, 0x5a, 0xda, 0xc7 +}; + +/* wycheproof - misc */ +static const u8 enc_input021[] __initconst = { + 0x4e, 0xe5, 0xcd, 0xa2, 0x0d, 0x42, 0x90 +}; +static const u8 enc_output021[] __initconst = { + 0x4b, 0xd4, 0x72, 0x12, 0x94, 0x1c, 0xe3, 0x18, + 0x5f, 0x14, 0x08, 0xee, 0x7f, 0xbf, 0x18, 0xf5, + 0xab, 0xad, 0x6e, 0x22, 0x53, 0xa1, 0xba +}; +static const u8 enc_assoc021[] __initconst = { + 0x84, 0xe4, 0x6b, 0xe8, 0xc0, 0x91, 0x90, 0x53 +}; +static const u8 enc_nonce021[] __initconst = { + 0x58, 0x38, 0x93, 0x75, 0xc6, 0x9e, 0xe3, 0x98, + 0xde, 0x94, 0x83, 0x96 +}; +static const u8 enc_key021[] __initconst = { + 0xe3, 0xc0, 0x9e, 0x7f, 0xab, 0x1a, 0xef, 0xb5, + 0x16, 0xda, 0x6a, 0x33, 0x02, 0x2a, 0x1d, 0xd4, + 0xeb, 0x27, 0x2c, 0x80, 0xd5, 0x40, 0xc5, 0xda, + 0x52, 0xa7, 0x30, 0xf3, 0x4d, 0x84, 0x0d, 0x7f +}; + +/* wycheproof - misc */ +static const u8 enc_input022[] __initconst = { + 0xbe, 0x33, 0x08, 0xf7, 0x2a, 0x2c, 0x6a, 0xed +}; +static const u8 enc_output022[] __initconst = { + 0x8e, 0x94, 0x39, 0xa5, 0x6e, 0xee, 0xc8, 0x17, + 0xfb, 0xe8, 0xa6, 0xed, 0x8f, 0xab, 0xb1, 0x93, + 0x75, 0x39, 0xdd, 0x6c, 0x00, 0xe9, 0x00, 0x21 +}; +static const u8 enc_assoc022[] __initconst = { }; +static const u8 enc_nonce022[] __initconst = { + 0x4f, 0x07, 0xaf, 0xed, 0xfd, 0xc3, 0xb6, 0xc2, + 0x36, 0x18, 0x23, 0xd3 +}; +static const u8 enc_key022[] __initconst = { + 0x51, 0xe4, 0xbf, 0x2b, 0xad, 0x92, 0xb7, 0xaf, + 0xf1, 0xa4, 0xbc, 0x05, 0x55, 0x0b, 0xa8, 0x1d, + 0xf4, 0xb9, 0x6f, 0xab, 0xf4, 0x1c, 0x12, 0xc7, + 0xb0, 0x0e, 0x60, 0xe4, 0x8d, 0xb7, 0xe1, 0x52 +}; + +/* wycheproof - misc */ +static const u8 enc_input023[] __initconst = { + 0xa4, 0xc9, 0xc2, 0x80, 0x1b, 0x71, 0xf7, 0xdf +}; +static const u8 enc_output023[] __initconst = { + 0xb9, 0xb9, 0x10, 0x43, 0x3a, 0xf0, 0x52, 0xb0, + 0x45, 0x30, 0xf5, 0x1a, 0xee, 0xe0, 0x24, 0xe0, + 0xa4, 0x45, 0xa6, 0x32, 0x8f, 0xa6, 0x7a, 0x18 +}; +static const u8 enc_assoc023[] __initconst = { + 0x66, 0xc0, 0xae, 0x70, 0x07, 0x6c, 0xb1, 0x4d +}; +static const u8 enc_nonce023[] __initconst = { + 0xb4, 0xea, 0x66, 0x6e, 0xe1, 0x19, 0x56, 0x33, + 0x66, 0x48, 0x4a, 0x78 +}; +static const u8 enc_key023[] __initconst = { + 0x11, 0x31, 0xc1, 0x41, 0x85, 0x77, 0xa0, 0x54, + 0xde, 0x7a, 0x4a, 0xc5, 0x51, 0x95, 0x0f, 0x1a, + 0x05, 0x3f, 0x9a, 0xe4, 0x6e, 0x5b, 0x75, 0xfe, + 0x4a, 0xbd, 0x56, 0x08, 0xd7, 0xcd, 0xda, 0xdd +}; + +/* wycheproof - misc */ +static const u8 enc_input024[] __initconst = { + 0x42, 0xba, 0xae, 0x59, 0x78, 0xfe, 0xaf, 0x5c, + 0x36, 0x8d, 0x14, 0xe0 +}; +static const u8 enc_output024[] __initconst = { + 0xff, 0x7d, 0xc2, 0x03, 0xb2, 0x6c, 0x46, 0x7a, + 0x6b, 0x50, 0xdb, 0x33, 0x57, 0x8c, 0x0f, 0x27, + 0x58, 0xc2, 0xe1, 0x4e, 0x36, 0xd4, 0xfc, 0x10, + 0x6d, 0xcb, 0x29, 0xb4 +}; +static const u8 enc_assoc024[] __initconst = { }; +static const u8 enc_nonce024[] __initconst = { + 0x9a, 0x59, 0xfc, 0xe2, 0x6d, 0xf0, 0x00, 0x5e, + 0x07, 0x53, 0x86, 0x56 +}; +static const u8 enc_key024[] __initconst = { + 0x99, 0xb6, 0x2b, 0xd5, 0xaf, 0xbe, 0x3f, 0xb0, + 0x15, 0xbd, 0xe9, 0x3f, 0x0a, 0xbf, 0x48, 0x39, + 0x57, 0xa1, 0xc3, 0xeb, 0x3c, 0xa5, 0x9c, 0xb5, + 0x0b, 0x39, 0xf7, 0xf8, 0xa9, 0xcc, 0x51, 0xbe +}; + +/* wycheproof - misc */ +static const u8 enc_input025[] __initconst = { + 0xfd, 0xc8, 0x5b, 0x94, 0xa4, 0xb2, 0xa6, 0xb7, + 0x59, 0xb1, 0xa0, 0xda +}; +static const u8 enc_output025[] __initconst = { + 0x9f, 0x88, 0x16, 0xde, 0x09, 0x94, 0xe9, 0x38, + 0xd9, 0xe5, 0x3f, 0x95, 0xd0, 0x86, 0xfc, 0x6c, + 0x9d, 0x8f, 0xa9, 0x15, 0xfd, 0x84, 0x23, 0xa7, + 0xcf, 0x05, 0x07, 0x2f +}; +static const u8 enc_assoc025[] __initconst = { + 0xa5, 0x06, 0xe1, 0xa5, 0xc6, 0x90, 0x93, 0xf9 +}; +static const u8 enc_nonce025[] __initconst = { + 0x58, 0xdb, 0xd4, 0xad, 0x2c, 0x4a, 0xd3, 0x5d, + 0xd9, 0x06, 0xe9, 0xce +}; +static const u8 enc_key025[] __initconst = { + 0x85, 0xf3, 0x5b, 0x62, 0x82, 0xcf, 0xf4, 0x40, + 0xbc, 0x10, 0x20, 0xc8, 0x13, 0x6f, 0xf2, 0x70, + 0x31, 0x11, 0x0f, 0xa6, 0x3e, 0xc1, 0x6f, 0x1e, + 0x82, 0x51, 0x18, 0xb0, 0x06, 0xb9, 0x12, 0x57 +}; + +/* wycheproof - misc */ +static const u8 enc_input026[] __initconst = { + 0x51, 0xf8, 0xc1, 0xf7, 0x31, 0xea, 0x14, 0xac, + 0xdb, 0x21, 0x0a, 0x6d, 0x97, 0x3e, 0x07 +}; +static const u8 enc_output026[] __initconst = { + 0x0b, 0x29, 0x63, 0x8e, 0x1f, 0xbd, 0xd6, 0xdf, + 0x53, 0x97, 0x0b, 0xe2, 0x21, 0x00, 0x42, 0x2a, + 0x91, 0x34, 0x08, 0x7d, 0x67, 0xa4, 0x6e, 0x79, + 0x17, 0x8d, 0x0a, 0x93, 0xf5, 0xe1, 0xd2 +}; +static const u8 enc_assoc026[] __initconst = { }; +static const u8 enc_nonce026[] __initconst = { + 0x68, 0xab, 0x7f, 0xdb, 0xf6, 0x19, 0x01, 0xda, + 0xd4, 0x61, 0xd2, 0x3c +}; +static const u8 enc_key026[] __initconst = { + 0x67, 0x11, 0x96, 0x27, 0xbd, 0x98, 0x8e, 0xda, + 0x90, 0x62, 0x19, 0xe0, 0x8c, 0x0d, 0x0d, 0x77, + 0x9a, 0x07, 0xd2, 0x08, 0xce, 0x8a, 0x4f, 0xe0, + 0x70, 0x9a, 0xf7, 0x55, 0xee, 0xec, 0x6d, 0xcb +}; + +/* wycheproof - misc */ +static const u8 enc_input027[] __initconst = { + 0x97, 0x46, 0x9d, 0xa6, 0x67, 0xd6, 0x11, 0x0f, + 0x9c, 0xbd, 0xa1, 0xd1, 0xa2, 0x06, 0x73 +}; +static const u8 enc_output027[] __initconst = { + 0x32, 0xdb, 0x66, 0xc4, 0xa3, 0x81, 0x9d, 0x81, + 0x55, 0x74, 0x55, 0xe5, 0x98, 0x0f, 0xed, 0xfe, + 0xae, 0x30, 0xde, 0xc9, 0x4e, 0x6a, 0xd3, 0xa9, + 0xee, 0xa0, 0x6a, 0x0d, 0x70, 0x39, 0x17 +}; +static const u8 enc_assoc027[] __initconst = { + 0x64, 0x53, 0xa5, 0x33, 0x84, 0x63, 0x22, 0x12 +}; +static const u8 enc_nonce027[] __initconst = { + 0xd9, 0x5b, 0x32, 0x43, 0xaf, 0xae, 0xf7, 0x14, + 0xc5, 0x03, 0x5b, 0x6a +}; +static const u8 enc_key027[] __initconst = { + 0xe6, 0xf1, 0x11, 0x8d, 0x41, 0xe4, 0xb4, 0x3f, + 0xb5, 0x82, 0x21, 0xb7, 0xed, 0x79, 0x67, 0x38, + 0x34, 0xe0, 0xd8, 0xac, 0x5c, 0x4f, 0xa6, 0x0b, + 0xbc, 0x8b, 0xc4, 0x89, 0x3a, 0x58, 0x89, 0x4d +}; + +/* wycheproof - misc */ +static const u8 enc_input028[] __initconst = { + 0x54, 0x9b, 0x36, 0x5a, 0xf9, 0x13, 0xf3, 0xb0, + 0x81, 0x13, 0x1c, 0xcb, 0x6b, 0x82, 0x55, 0x88 +}; +static const u8 enc_output028[] __initconst = { + 0xe9, 0x11, 0x0e, 0x9f, 0x56, 0xab, 0x3c, 0xa4, + 0x83, 0x50, 0x0c, 0xea, 0xba, 0xb6, 0x7a, 0x13, + 0x83, 0x6c, 0xca, 0xbf, 0x15, 0xa6, 0xa2, 0x2a, + 0x51, 0xc1, 0x07, 0x1c, 0xfa, 0x68, 0xfa, 0x0c +}; +static const u8 enc_assoc028[] __initconst = { }; +static const u8 enc_nonce028[] __initconst = { + 0x2f, 0xcb, 0x1b, 0x38, 0xa9, 0x9e, 0x71, 0xb8, + 0x47, 0x40, 0xad, 0x9b +}; +static const u8 enc_key028[] __initconst = { + 0x59, 0xd4, 0xea, 0xfb, 0x4d, 0xe0, 0xcf, 0xc7, + 0xd3, 0xdb, 0x99, 0xa8, 0xf5, 0x4b, 0x15, 0xd7, + 0xb3, 0x9f, 0x0a, 0xcc, 0x8d, 0xa6, 0x97, 0x63, + 0xb0, 0x19, 0xc1, 0x69, 0x9f, 0x87, 0x67, 0x4a +}; + +/* wycheproof - misc */ +static const u8 enc_input029[] __initconst = { + 0x55, 0xa4, 0x65, 0x64, 0x4f, 0x5b, 0x65, 0x09, + 0x28, 0xcb, 0xee, 0x7c, 0x06, 0x32, 0x14, 0xd6 +}; +static const u8 enc_output029[] __initconst = { + 0xe4, 0xb1, 0x13, 0xcb, 0x77, 0x59, 0x45, 0xf3, + 0xd3, 0xa8, 0xae, 0x9e, 0xc1, 0x41, 0xc0, 0x0c, + 0x7c, 0x43, 0xf1, 0x6c, 0xe0, 0x96, 0xd0, 0xdc, + 0x27, 0xc9, 0x58, 0x49, 0xdc, 0x38, 0x3b, 0x7d +}; +static const u8 enc_assoc029[] __initconst = { + 0x03, 0x45, 0x85, 0x62, 0x1a, 0xf8, 0xd7, 0xff +}; +static const u8 enc_nonce029[] __initconst = { + 0x11, 0x8a, 0x69, 0x64, 0xc2, 0xd3, 0xe3, 0x80, + 0x07, 0x1f, 0x52, 0x66 +}; +static const u8 enc_key029[] __initconst = { + 0xb9, 0x07, 0xa4, 0x50, 0x75, 0x51, 0x3f, 0xe8, + 0xa8, 0x01, 0x9e, 0xde, 0xe3, 0xf2, 0x59, 0x14, + 0x87, 0xb2, 0xa0, 0x30, 0xb0, 0x3c, 0x6e, 0x1d, + 0x77, 0x1c, 0x86, 0x25, 0x71, 0xd2, 0xea, 0x1e +}; + +/* wycheproof - misc */ +static const u8 enc_input030[] __initconst = { + 0x3f, 0xf1, 0x51, 0x4b, 0x1c, 0x50, 0x39, 0x15, + 0x91, 0x8f, 0x0c, 0x0c, 0x31, 0x09, 0x4a, 0x6e, + 0x1f +}; +static const u8 enc_output030[] __initconst = { + 0x02, 0xcc, 0x3a, 0xcb, 0x5e, 0xe1, 0xfc, 0xdd, + 0x12, 0xa0, 0x3b, 0xb8, 0x57, 0x97, 0x64, 0x74, + 0xd3, 0xd8, 0x3b, 0x74, 0x63, 0xa2, 0xc3, 0x80, + 0x0f, 0xe9, 0x58, 0xc2, 0x8e, 0xaa, 0x29, 0x08, + 0x13 +}; +static const u8 enc_assoc030[] __initconst = { }; +static const u8 enc_nonce030[] __initconst = { + 0x45, 0xaa, 0xa3, 0xe5, 0xd1, 0x6d, 0x2d, 0x42, + 0xdc, 0x03, 0x44, 0x5d +}; +static const u8 enc_key030[] __initconst = { + 0x3b, 0x24, 0x58, 0xd8, 0x17, 0x6e, 0x16, 0x21, + 0xc0, 0xcc, 0x24, 0xc0, 0xc0, 0xe2, 0x4c, 0x1e, + 0x80, 0xd7, 0x2f, 0x7e, 0xe9, 0x14, 0x9a, 0x4b, + 0x16, 0x61, 0x76, 0x62, 0x96, 0x16, 0xd0, 0x11 +}; + +/* wycheproof - misc */ +static const u8 enc_input031[] __initconst = { + 0x63, 0x85, 0x8c, 0xa3, 0xe2, 0xce, 0x69, 0x88, + 0x7b, 0x57, 0x8a, 0x3c, 0x16, 0x7b, 0x42, 0x1c, + 0x9c +}; +static const u8 enc_output031[] __initconst = { + 0x35, 0x76, 0x64, 0x88, 0xd2, 0xbc, 0x7c, 0x2b, + 0x8d, 0x17, 0xcb, 0xbb, 0x9a, 0xbf, 0xad, 0x9e, + 0x6d, 0x1f, 0x39, 0x1e, 0x65, 0x7b, 0x27, 0x38, + 0xdd, 0xa0, 0x84, 0x48, 0xcb, 0xa2, 0x81, 0x1c, + 0xeb +}; +static const u8 enc_assoc031[] __initconst = { + 0x9a, 0xaf, 0x29, 0x9e, 0xee, 0xa7, 0x8f, 0x79 +}; +static const u8 enc_nonce031[] __initconst = { + 0xf0, 0x38, 0x4f, 0xb8, 0x76, 0x12, 0x14, 0x10, + 0x63, 0x3d, 0x99, 0x3d +}; +static const u8 enc_key031[] __initconst = { + 0xf6, 0x0c, 0x6a, 0x1b, 0x62, 0x57, 0x25, 0xf7, + 0x6c, 0x70, 0x37, 0xb4, 0x8f, 0xe3, 0x57, 0x7f, + 0xa7, 0xf7, 0xb8, 0x7b, 0x1b, 0xd5, 0xa9, 0x82, + 0x17, 0x6d, 0x18, 0x23, 0x06, 0xff, 0xb8, 0x70 +}; + +/* wycheproof - misc */ +static const u8 enc_input032[] __initconst = { + 0x10, 0xf1, 0xec, 0xf9, 0xc6, 0x05, 0x84, 0x66, + 0x5d, 0x9a, 0xe5, 0xef, 0xe2, 0x79, 0xe7, 0xf7, + 0x37, 0x7e, 0xea, 0x69, 0x16, 0xd2, 0xb1, 0x11 +}; +static const u8 enc_output032[] __initconst = { + 0x42, 0xf2, 0x6c, 0x56, 0xcb, 0x4b, 0xe2, 0x1d, + 0x9d, 0x8d, 0x0c, 0x80, 0xfc, 0x99, 0xdd, 0xe0, + 0x0d, 0x75, 0xf3, 0x80, 0x74, 0xbf, 0xe7, 0x64, + 0x54, 0xaa, 0x7e, 0x13, 0xd4, 0x8f, 0xff, 0x7d, + 0x75, 0x57, 0x03, 0x94, 0x57, 0x04, 0x0a, 0x3a +}; +static const u8 enc_assoc032[] __initconst = { }; +static const u8 enc_nonce032[] __initconst = { + 0xe6, 0xb1, 0xad, 0xf2, 0xfd, 0x58, 0xa8, 0x76, + 0x2c, 0x65, 0xf3, 0x1b +}; +static const u8 enc_key032[] __initconst = { + 0x02, 0x12, 0xa8, 0xde, 0x50, 0x07, 0xed, 0x87, + 0xb3, 0x3f, 0x1a, 0x70, 0x90, 0xb6, 0x11, 0x4f, + 0x9e, 0x08, 0xce, 0xfd, 0x96, 0x07, 0xf2, 0xc2, + 0x76, 0xbd, 0xcf, 0xdb, 0xc5, 0xce, 0x9c, 0xd7 +}; + +/* wycheproof - misc */ +static const u8 enc_input033[] __initconst = { + 0x92, 0x22, 0xf9, 0x01, 0x8e, 0x54, 0xfd, 0x6d, + 0xe1, 0x20, 0x08, 0x06, 0xa9, 0xee, 0x8e, 0x4c, + 0xc9, 0x04, 0xd2, 0x9f, 0x25, 0xcb, 0xa1, 0x93 +}; +static const u8 enc_output033[] __initconst = { + 0x12, 0x30, 0x32, 0x43, 0x7b, 0x4b, 0xfd, 0x69, + 0x20, 0xe8, 0xf7, 0xe7, 0xe0, 0x08, 0x7a, 0xe4, + 0x88, 0x9e, 0xbe, 0x7a, 0x0a, 0xd0, 0xe9, 0x00, + 0x3c, 0xf6, 0x8f, 0x17, 0x95, 0x50, 0xda, 0x63, + 0xd3, 0xb9, 0x6c, 0x2d, 0x55, 0x41, 0x18, 0x65 +}; +static const u8 enc_assoc033[] __initconst = { + 0x3e, 0x8b, 0xc5, 0xad, 0xe1, 0x82, 0xff, 0x08 +}; +static const u8 enc_nonce033[] __initconst = { + 0x6b, 0x28, 0x2e, 0xbe, 0xcc, 0x54, 0x1b, 0xcd, + 0x78, 0x34, 0xed, 0x55 +}; +static const u8 enc_key033[] __initconst = { + 0xc5, 0xbc, 0x09, 0x56, 0x56, 0x46, 0xe7, 0xed, + 0xda, 0x95, 0x4f, 0x1f, 0x73, 0x92, 0x23, 0xda, + 0xda, 0x20, 0xb9, 0x5c, 0x44, 0xab, 0x03, 0x3d, + 0x0f, 0xae, 0x4b, 0x02, 0x83, 0xd1, 0x8b, 0xe3 +}; + +/* wycheproof - misc */ +static const u8 enc_input034[] __initconst = { + 0xb0, 0x53, 0x99, 0x92, 0x86, 0xa2, 0x82, 0x4f, + 0x42, 0xcc, 0x8c, 0x20, 0x3a, 0xb2, 0x4e, 0x2c, + 0x97, 0xa6, 0x85, 0xad, 0xcc, 0x2a, 0xd3, 0x26, + 0x62, 0x55, 0x8e, 0x55, 0xa5, 0xc7, 0x29 +}; +static const u8 enc_output034[] __initconst = { + 0x45, 0xc7, 0xd6, 0xb5, 0x3a, 0xca, 0xd4, 0xab, + 0xb6, 0x88, 0x76, 0xa6, 0xe9, 0x6a, 0x48, 0xfb, + 0x59, 0x52, 0x4d, 0x2c, 0x92, 0xc9, 0xd8, 0xa1, + 0x89, 0xc9, 0xfd, 0x2d, 0xb9, 0x17, 0x46, 0x56, + 0x6d, 0x3c, 0xa1, 0x0e, 0x31, 0x1b, 0x69, 0x5f, + 0x3e, 0xae, 0x15, 0x51, 0x65, 0x24, 0x93 +}; +static const u8 enc_assoc034[] __initconst = { }; +static const u8 enc_nonce034[] __initconst = { + 0x04, 0xa9, 0xbe, 0x03, 0x50, 0x8a, 0x5f, 0x31, + 0x37, 0x1a, 0x6f, 0xd2 +}; +static const u8 enc_key034[] __initconst = { + 0x2e, 0xb5, 0x1c, 0x46, 0x9a, 0xa8, 0xeb, 0x9e, + 0x6c, 0x54, 0xa8, 0x34, 0x9b, 0xae, 0x50, 0xa2, + 0x0f, 0x0e, 0x38, 0x27, 0x11, 0xbb, 0xa1, 0x15, + 0x2c, 0x42, 0x4f, 0x03, 0xb6, 0x67, 0x1d, 0x71 +}; + +/* wycheproof - misc */ +static const u8 enc_input035[] __initconst = { + 0xf4, 0x52, 0x06, 0xab, 0xc2, 0x55, 0x52, 0xb2, + 0xab, 0xc9, 0xab, 0x7f, 0xa2, 0x43, 0x03, 0x5f, + 0xed, 0xaa, 0xdd, 0xc3, 0xb2, 0x29, 0x39, 0x56, + 0xf1, 0xea, 0x6e, 0x71, 0x56, 0xe7, 0xeb +}; +static const u8 enc_output035[] __initconst = { + 0x46, 0xa8, 0x0c, 0x41, 0x87, 0x02, 0x47, 0x20, + 0x08, 0x46, 0x27, 0x58, 0x00, 0x80, 0xdd, 0xe5, + 0xa3, 0xf4, 0xa1, 0x10, 0x93, 0xa7, 0x07, 0x6e, + 0xd6, 0xf3, 0xd3, 0x26, 0xbc, 0x7b, 0x70, 0x53, + 0x4d, 0x4a, 0xa2, 0x83, 0x5a, 0x52, 0xe7, 0x2d, + 0x14, 0xdf, 0x0e, 0x4f, 0x47, 0xf2, 0x5f +}; +static const u8 enc_assoc035[] __initconst = { + 0x37, 0x46, 0x18, 0xa0, 0x6e, 0xa9, 0x8a, 0x48 +}; +static const u8 enc_nonce035[] __initconst = { + 0x47, 0x0a, 0x33, 0x9e, 0xcb, 0x32, 0x19, 0xb8, + 0xb8, 0x1a, 0x1f, 0x8b +}; +static const u8 enc_key035[] __initconst = { + 0x7f, 0x5b, 0x74, 0xc0, 0x7e, 0xd1, 0xb4, 0x0f, + 0xd1, 0x43, 0x58, 0xfe, 0x2f, 0xf2, 0xa7, 0x40, + 0xc1, 0x16, 0xc7, 0x70, 0x65, 0x10, 0xe6, 0xa4, + 0x37, 0xf1, 0x9e, 0xa4, 0x99, 0x11, 0xce, 0xc4 +}; + +/* wycheproof - misc */ +static const u8 enc_input036[] __initconst = { + 0xb9, 0xc5, 0x54, 0xcb, 0xc3, 0x6a, 0xc1, 0x8a, + 0xe8, 0x97, 0xdf, 0x7b, 0xee, 0xca, 0xc1, 0xdb, + 0xeb, 0x4e, 0xaf, 0xa1, 0x56, 0xbb, 0x60, 0xce, + 0x2e, 0x5d, 0x48, 0xf0, 0x57, 0x15, 0xe6, 0x78 +}; +static const u8 enc_output036[] __initconst = { + 0xea, 0x29, 0xaf, 0xa4, 0x9d, 0x36, 0xe8, 0x76, + 0x0f, 0x5f, 0xe1, 0x97, 0x23, 0xb9, 0x81, 0x1e, + 0xd5, 0xd5, 0x19, 0x93, 0x4a, 0x44, 0x0f, 0x50, + 0x81, 0xac, 0x43, 0x0b, 0x95, 0x3b, 0x0e, 0x21, + 0x22, 0x25, 0x41, 0xaf, 0x46, 0xb8, 0x65, 0x33, + 0xc6, 0xb6, 0x8d, 0x2f, 0xf1, 0x08, 0xa7, 0xea +}; +static const u8 enc_assoc036[] __initconst = { }; +static const u8 enc_nonce036[] __initconst = { + 0x72, 0xcf, 0xd9, 0x0e, 0xf3, 0x02, 0x6c, 0xa2, + 0x2b, 0x7e, 0x6e, 0x6a +}; +static const u8 enc_key036[] __initconst = { + 0xe1, 0x73, 0x1d, 0x58, 0x54, 0xe1, 0xb7, 0x0c, + 0xb3, 0xff, 0xe8, 0xb7, 0x86, 0xa2, 0xb3, 0xeb, + 0xf0, 0x99, 0x43, 0x70, 0x95, 0x47, 0x57, 0xb9, + 0xdc, 0x8c, 0x7b, 0xc5, 0x35, 0x46, 0x34, 0xa3 +}; + +/* wycheproof - misc */ +static const u8 enc_input037[] __initconst = { + 0x6b, 0x26, 0x04, 0x99, 0x6c, 0xd3, 0x0c, 0x14, + 0xa1, 0x3a, 0x52, 0x57, 0xed, 0x6c, 0xff, 0xd3, + 0xbc, 0x5e, 0x29, 0xd6, 0xb9, 0x7e, 0xb1, 0x79, + 0x9e, 0xb3, 0x35, 0xe2, 0x81, 0xea, 0x45, 0x1e +}; +static const u8 enc_output037[] __initconst = { + 0x6d, 0xad, 0x63, 0x78, 0x97, 0x54, 0x4d, 0x8b, + 0xf6, 0xbe, 0x95, 0x07, 0xed, 0x4d, 0x1b, 0xb2, + 0xe9, 0x54, 0xbc, 0x42, 0x7e, 0x5d, 0xe7, 0x29, + 0xda, 0xf5, 0x07, 0x62, 0x84, 0x6f, 0xf2, 0xf4, + 0x7b, 0x99, 0x7d, 0x93, 0xc9, 0x82, 0x18, 0x9d, + 0x70, 0x95, 0xdc, 0x79, 0x4c, 0x74, 0x62, 0x32 +}; +static const u8 enc_assoc037[] __initconst = { + 0x23, 0x33, 0xe5, 0xce, 0x0f, 0x93, 0xb0, 0x59 +}; +static const u8 enc_nonce037[] __initconst = { + 0x26, 0x28, 0x80, 0xd4, 0x75, 0xf3, 0xda, 0xc5, + 0x34, 0x0d, 0xd1, 0xb8 +}; +static const u8 enc_key037[] __initconst = { + 0x27, 0xd8, 0x60, 0x63, 0x1b, 0x04, 0x85, 0xa4, + 0x10, 0x70, 0x2f, 0xea, 0x61, 0xbc, 0x87, 0x3f, + 0x34, 0x42, 0x26, 0x0c, 0xad, 0xed, 0x4a, 0xbd, + 0xe2, 0x5b, 0x78, 0x6a, 0x2d, 0x97, 0xf1, 0x45 +}; + +/* wycheproof - misc */ +static const u8 enc_input038[] __initconst = { + 0x97, 0x3d, 0x0c, 0x75, 0x38, 0x26, 0xba, 0xe4, + 0x66, 0xcf, 0x9a, 0xbb, 0x34, 0x93, 0x15, 0x2e, + 0x9d, 0xe7, 0x81, 0x9e, 0x2b, 0xd0, 0xc7, 0x11, + 0x71, 0x34, 0x6b, 0x4d, 0x2c, 0xeb, 0xf8, 0x04, + 0x1a, 0xa3, 0xce, 0xdc, 0x0d, 0xfd, 0x7b, 0x46, + 0x7e, 0x26, 0x22, 0x8b, 0xc8, 0x6c, 0x9a +}; +static const u8 enc_output038[] __initconst = { + 0xfb, 0xa7, 0x8a, 0xe4, 0xf9, 0xd8, 0x08, 0xa6, + 0x2e, 0x3d, 0xa4, 0x0b, 0xe2, 0xcb, 0x77, 0x00, + 0xc3, 0x61, 0x3d, 0x9e, 0xb2, 0xc5, 0x29, 0xc6, + 0x52, 0xe7, 0x6a, 0x43, 0x2c, 0x65, 0x8d, 0x27, + 0x09, 0x5f, 0x0e, 0xb8, 0xf9, 0x40, 0xc3, 0x24, + 0x98, 0x1e, 0xa9, 0x35, 0xe5, 0x07, 0xf9, 0x8f, + 0x04, 0x69, 0x56, 0xdb, 0x3a, 0x51, 0x29, 0x08, + 0xbd, 0x7a, 0xfc, 0x8f, 0x2a, 0xb0, 0xa9 +}; +static const u8 enc_assoc038[] __initconst = { }; +static const u8 enc_nonce038[] __initconst = { + 0xe7, 0x4a, 0x51, 0x5e, 0x7e, 0x21, 0x02, 0xb9, + 0x0b, 0xef, 0x55, 0xd2 +}; +static const u8 enc_key038[] __initconst = { + 0xcf, 0x0d, 0x40, 0xa4, 0x64, 0x4e, 0x5f, 0x51, + 0x81, 0x51, 0x65, 0xd5, 0x30, 0x1b, 0x22, 0x63, + 0x1f, 0x45, 0x44, 0xc4, 0x9a, 0x18, 0x78, 0xe3, + 0xa0, 0xa5, 0xe8, 0xe1, 0xaa, 0xe0, 0xf2, 0x64 +}; + +/* wycheproof - misc */ +static const u8 enc_input039[] __initconst = { + 0xa9, 0x89, 0x95, 0x50, 0x4d, 0xf1, 0x6f, 0x74, + 0x8b, 0xfb, 0x77, 0x85, 0xff, 0x91, 0xee, 0xb3, + 0xb6, 0x60, 0xea, 0x9e, 0xd3, 0x45, 0x0c, 0x3d, + 0x5e, 0x7b, 0x0e, 0x79, 0xef, 0x65, 0x36, 0x59, + 0xa9, 0x97, 0x8d, 0x75, 0x54, 0x2e, 0xf9, 0x1c, + 0x45, 0x67, 0x62, 0x21, 0x56, 0x40, 0xb9 +}; +static const u8 enc_output039[] __initconst = { + 0xa1, 0xff, 0xed, 0x80, 0x76, 0x18, 0x29, 0xec, + 0xce, 0x24, 0x2e, 0x0e, 0x88, 0xb1, 0x38, 0x04, + 0x90, 0x16, 0xbc, 0xa0, 0x18, 0xda, 0x2b, 0x6e, + 0x19, 0x98, 0x6b, 0x3e, 0x31, 0x8c, 0xae, 0x8d, + 0x80, 0x61, 0x98, 0xfb, 0x4c, 0x52, 0x7c, 0xc3, + 0x93, 0x50, 0xeb, 0xdd, 0xea, 0xc5, 0x73, 0xc4, + 0xcb, 0xf0, 0xbe, 0xfd, 0xa0, 0xb7, 0x02, 0x42, + 0xc6, 0x40, 0xd7, 0xcd, 0x02, 0xd7, 0xa3 +}; +static const u8 enc_assoc039[] __initconst = { + 0xb3, 0xe4, 0x06, 0x46, 0x83, 0xb0, 0x2d, 0x84 +}; +static const u8 enc_nonce039[] __initconst = { + 0xd4, 0xd8, 0x07, 0x34, 0x16, 0x83, 0x82, 0x5b, + 0x31, 0xcd, 0x4d, 0x95 +}; +static const u8 enc_key039[] __initconst = { + 0x6c, 0xbf, 0xd7, 0x1c, 0x64, 0x5d, 0x18, 0x4c, + 0xf5, 0xd2, 0x3c, 0x40, 0x2b, 0xdb, 0x0d, 0x25, + 0xec, 0x54, 0x89, 0x8c, 0x8a, 0x02, 0x73, 0xd4, + 0x2e, 0xb5, 0xbe, 0x10, 0x9f, 0xdc, 0xb2, 0xac +}; + +/* wycheproof - misc */ +static const u8 enc_input040[] __initconst = { + 0xd0, 0x96, 0x80, 0x31, 0x81, 0xbe, 0xef, 0x9e, + 0x00, 0x8f, 0xf8, 0x5d, 0x5d, 0xdc, 0x38, 0xdd, + 0xac, 0xf0, 0xf0, 0x9e, 0xe5, 0xf7, 0xe0, 0x7f, + 0x1e, 0x40, 0x79, 0xcb, 0x64, 0xd0, 0xdc, 0x8f, + 0x5e, 0x67, 0x11, 0xcd, 0x49, 0x21, 0xa7, 0x88, + 0x7d, 0xe7, 0x6e, 0x26, 0x78, 0xfd, 0xc6, 0x76, + 0x18, 0xf1, 0x18, 0x55, 0x86, 0xbf, 0xea, 0x9d, + 0x4c, 0x68, 0x5d, 0x50, 0xe4, 0xbb, 0x9a, 0x82 +}; +static const u8 enc_output040[] __initconst = { + 0x9a, 0x4e, 0xf2, 0x2b, 0x18, 0x16, 0x77, 0xb5, + 0x75, 0x5c, 0x08, 0xf7, 0x47, 0xc0, 0xf8, 0xd8, + 0xe8, 0xd4, 0xc1, 0x8a, 0x9c, 0xc2, 0x40, 0x5c, + 0x12, 0xbb, 0x51, 0xbb, 0x18, 0x72, 0xc8, 0xe8, + 0xb8, 0x77, 0x67, 0x8b, 0xec, 0x44, 0x2c, 0xfc, + 0xbb, 0x0f, 0xf4, 0x64, 0xa6, 0x4b, 0x74, 0x33, + 0x2c, 0xf0, 0x72, 0x89, 0x8c, 0x7e, 0x0e, 0xdd, + 0xf6, 0x23, 0x2e, 0xa6, 0xe2, 0x7e, 0xfe, 0x50, + 0x9f, 0xf3, 0x42, 0x7a, 0x0f, 0x32, 0xfa, 0x56, + 0x6d, 0x9c, 0xa0, 0xa7, 0x8a, 0xef, 0xc0, 0x13 +}; +static const u8 enc_assoc040[] __initconst = { }; +static const u8 enc_nonce040[] __initconst = { + 0xd6, 0x10, 0x40, 0xa3, 0x13, 0xed, 0x49, 0x28, + 0x23, 0xcc, 0x06, 0x5b +}; +static const u8 enc_key040[] __initconst = { + 0x5b, 0x1d, 0x10, 0x35, 0xc0, 0xb1, 0x7e, 0xe0, + 0xb0, 0x44, 0x47, 0x67, 0xf8, 0x0a, 0x25, 0xb8, + 0xc1, 0xb7, 0x41, 0xf4, 0xb5, 0x0a, 0x4d, 0x30, + 0x52, 0x22, 0x6b, 0xaa, 0x1c, 0x6f, 0xb7, 0x01 +}; + +/* wycheproof - misc */ +static const u8 enc_input041[] __initconst = { + 0x94, 0xee, 0x16, 0x6d, 0x6d, 0x6e, 0xcf, 0x88, + 0x32, 0x43, 0x71, 0x36, 0xb4, 0xae, 0x80, 0x5d, + 0x42, 0x88, 0x64, 0x35, 0x95, 0x86, 0xd9, 0x19, + 0x3a, 0x25, 0x01, 0x62, 0x93, 0xed, 0xba, 0x44, + 0x3c, 0x58, 0xe0, 0x7e, 0x7b, 0x71, 0x95, 0xec, + 0x5b, 0xd8, 0x45, 0x82, 0xa9, 0xd5, 0x6c, 0x8d, + 0x4a, 0x10, 0x8c, 0x7d, 0x7c, 0xe3, 0x4e, 0x6c, + 0x6f, 0x8e, 0xa1, 0xbe, 0xc0, 0x56, 0x73, 0x17 +}; +static const u8 enc_output041[] __initconst = { + 0x5f, 0xbb, 0xde, 0xcc, 0x34, 0xbe, 0x20, 0x16, + 0x14, 0xf6, 0x36, 0x03, 0x1e, 0xeb, 0x42, 0xf1, + 0xca, 0xce, 0x3c, 0x79, 0xa1, 0x2c, 0xff, 0xd8, + 0x71, 0xee, 0x8e, 0x73, 0x82, 0x0c, 0x82, 0x97, + 0x49, 0xf1, 0xab, 0xb4, 0x29, 0x43, 0x67, 0x84, + 0x9f, 0xb6, 0xc2, 0xaa, 0x56, 0xbd, 0xa8, 0xa3, + 0x07, 0x8f, 0x72, 0x3d, 0x7c, 0x1c, 0x85, 0x20, + 0x24, 0xb0, 0x17, 0xb5, 0x89, 0x73, 0xfb, 0x1e, + 0x09, 0x26, 0x3d, 0xa7, 0xb4, 0xcb, 0x92, 0x14, + 0x52, 0xf9, 0x7d, 0xca, 0x40, 0xf5, 0x80, 0xec +}; +static const u8 enc_assoc041[] __initconst = { + 0x71, 0x93, 0xf6, 0x23, 0x66, 0x33, 0x21, 0xa2 +}; +static const u8 enc_nonce041[] __initconst = { + 0xd3, 0x1c, 0x21, 0xab, 0xa1, 0x75, 0xb7, 0x0d, + 0xe4, 0xeb, 0xb1, 0x9c +}; +static const u8 enc_key041[] __initconst = { + 0x97, 0xd6, 0x35, 0xc4, 0xf4, 0x75, 0x74, 0xd9, + 0x99, 0x8a, 0x90, 0x87, 0x5d, 0xa1, 0xd3, 0xa2, + 0x84, 0xb7, 0x55, 0xb2, 0xd3, 0x92, 0x97, 0xa5, + 0x72, 0x52, 0x35, 0x19, 0x0e, 0x10, 0xa9, 0x7e +}; + +/* wycheproof - misc */ +static const u8 enc_input042[] __initconst = { + 0xb4, 0x29, 0xeb, 0x80, 0xfb, 0x8f, 0xe8, 0xba, + 0xed, 0xa0, 0xc8, 0x5b, 0x9c, 0x33, 0x34, 0x58, + 0xe7, 0xc2, 0x99, 0x2e, 0x55, 0x84, 0x75, 0x06, + 0x9d, 0x12, 0xd4, 0x5c, 0x22, 0x21, 0x75, 0x64, + 0x12, 0x15, 0x88, 0x03, 0x22, 0x97, 0xef, 0xf5, + 0x67, 0x83, 0x74, 0x2a, 0x5f, 0xc2, 0x2d, 0x74, + 0x10, 0xff, 0xb2, 0x9d, 0x66, 0x09, 0x86, 0x61, + 0xd7, 0x6f, 0x12, 0x6c, 0x3c, 0x27, 0x68, 0x9e, + 0x43, 0xb3, 0x72, 0x67, 0xca, 0xc5, 0xa3, 0xa6, + 0xd3, 0xab, 0x49, 0xe3, 0x91, 0xda, 0x29, 0xcd, + 0x30, 0x54, 0xa5, 0x69, 0x2e, 0x28, 0x07, 0xe4, + 0xc3, 0xea, 0x46, 0xc8, 0x76, 0x1d, 0x50, 0xf5, + 0x92 +}; +static const u8 enc_output042[] __initconst = { + 0xd0, 0x10, 0x2f, 0x6c, 0x25, 0x8b, 0xf4, 0x97, + 0x42, 0xce, 0xc3, 0x4c, 0xf2, 0xd0, 0xfe, 0xdf, + 0x23, 0xd1, 0x05, 0xfb, 0x4c, 0x84, 0xcf, 0x98, + 0x51, 0x5e, 0x1b, 0xc9, 0xa6, 0x4f, 0x8a, 0xd5, + 0xbe, 0x8f, 0x07, 0x21, 0xbd, 0xe5, 0x06, 0x45, + 0xd0, 0x00, 0x83, 0xc3, 0xa2, 0x63, 0xa3, 0x10, + 0x53, 0xb7, 0x60, 0x24, 0x5f, 0x52, 0xae, 0x28, + 0x66, 0xa5, 0xec, 0x83, 0xb1, 0x9f, 0x61, 0xbe, + 0x1d, 0x30, 0xd5, 0xc5, 0xd9, 0xfe, 0xcc, 0x4c, + 0xbb, 0xe0, 0x8f, 0xd3, 0x85, 0x81, 0x3a, 0x2a, + 0xa3, 0x9a, 0x00, 0xff, 0x9c, 0x10, 0xf7, 0xf2, + 0x37, 0x02, 0xad, 0xd1, 0xe4, 0xb2, 0xff, 0xa3, + 0x1c, 0x41, 0x86, 0x5f, 0xc7, 0x1d, 0xe1, 0x2b, + 0x19, 0x61, 0x21, 0x27, 0xce, 0x49, 0x99, 0x3b, + 0xb0 +}; +static const u8 enc_assoc042[] __initconst = { }; +static const u8 enc_nonce042[] __initconst = { + 0x17, 0xc8, 0x6a, 0x8a, 0xbb, 0xb7, 0xe0, 0x03, + 0xac, 0xde, 0x27, 0x99 +}; +static const u8 enc_key042[] __initconst = { + 0xfe, 0x6e, 0x55, 0xbd, 0xae, 0xd1, 0xf7, 0x28, + 0x4c, 0xa5, 0xfc, 0x0f, 0x8c, 0x5f, 0x2b, 0x8d, + 0xf5, 0x6d, 0xc0, 0xf4, 0x9e, 0x8c, 0xa6, 0x6a, + 0x41, 0x99, 0x5e, 0x78, 0x33, 0x51, 0xf9, 0x01 +}; + +/* wycheproof - misc */ +static const u8 enc_input043[] __initconst = { + 0xce, 0xb5, 0x34, 0xce, 0x50, 0xdc, 0x23, 0xff, + 0x63, 0x8a, 0xce, 0x3e, 0xf6, 0x3a, 0xb2, 0xcc, + 0x29, 0x73, 0xee, 0xad, 0xa8, 0x07, 0x85, 0xfc, + 0x16, 0x5d, 0x06, 0xc2, 0xf5, 0x10, 0x0f, 0xf5, + 0xe8, 0xab, 0x28, 0x82, 0xc4, 0x75, 0xaf, 0xcd, + 0x05, 0xcc, 0xd4, 0x9f, 0x2e, 0x7d, 0x8f, 0x55, + 0xef, 0x3a, 0x72, 0xe3, 0xdc, 0x51, 0xd6, 0x85, + 0x2b, 0x8e, 0x6b, 0x9e, 0x7a, 0xec, 0xe5, 0x7b, + 0xe6, 0x55, 0x6b, 0x0b, 0x6d, 0x94, 0x13, 0xe3, + 0x3f, 0xc5, 0xfc, 0x24, 0xa9, 0xa2, 0x05, 0xad, + 0x59, 0x57, 0x4b, 0xb3, 0x9d, 0x94, 0x4a, 0x92, + 0xdc, 0x47, 0x97, 0x0d, 0x84, 0xa6, 0xad, 0x31, + 0x76 +}; +static const u8 enc_output043[] __initconst = { + 0x75, 0x45, 0x39, 0x1b, 0x51, 0xde, 0x01, 0xd5, + 0xc5, 0x3d, 0xfa, 0xca, 0x77, 0x79, 0x09, 0x06, + 0x3e, 0x58, 0xed, 0xee, 0x4b, 0xb1, 0x22, 0x7e, + 0x71, 0x10, 0xac, 0x4d, 0x26, 0x20, 0xc2, 0xae, + 0xc2, 0xf8, 0x48, 0xf5, 0x6d, 0xee, 0xb0, 0x37, + 0xa8, 0xdc, 0xed, 0x75, 0xaf, 0xa8, 0xa6, 0xc8, + 0x90, 0xe2, 0xde, 0xe4, 0x2f, 0x95, 0x0b, 0xb3, + 0x3d, 0x9e, 0x24, 0x24, 0xd0, 0x8a, 0x50, 0x5d, + 0x89, 0x95, 0x63, 0x97, 0x3e, 0xd3, 0x88, 0x70, + 0xf3, 0xde, 0x6e, 0xe2, 0xad, 0xc7, 0xfe, 0x07, + 0x2c, 0x36, 0x6c, 0x14, 0xe2, 0xcf, 0x7c, 0xa6, + 0x2f, 0xb3, 0xd3, 0x6b, 0xee, 0x11, 0x68, 0x54, + 0x61, 0xb7, 0x0d, 0x44, 0xef, 0x8c, 0x66, 0xc5, + 0xc7, 0xbb, 0xf1, 0x0d, 0xca, 0xdd, 0x7f, 0xac, + 0xf6 +}; +static const u8 enc_assoc043[] __initconst = { + 0xa1, 0x1c, 0x40, 0xb6, 0x03, 0x76, 0x73, 0x30 +}; +static const u8 enc_nonce043[] __initconst = { + 0x46, 0x36, 0x2f, 0x45, 0xd6, 0x37, 0x9e, 0x63, + 0xe5, 0x22, 0x94, 0x60 +}; +static const u8 enc_key043[] __initconst = { + 0xaa, 0xbc, 0x06, 0x34, 0x74, 0xe6, 0x5c, 0x4c, + 0x3e, 0x9b, 0xdc, 0x48, 0x0d, 0xea, 0x97, 0xb4, + 0x51, 0x10, 0xc8, 0x61, 0x88, 0x46, 0xff, 0x6b, + 0x15, 0xbd, 0xd2, 0xa4, 0xa5, 0x68, 0x2c, 0x4e +}; + +/* wycheproof - misc */ +static const u8 enc_input044[] __initconst = { + 0xe5, 0xcc, 0xaa, 0x44, 0x1b, 0xc8, 0x14, 0x68, + 0x8f, 0x8f, 0x6e, 0x8f, 0x28, 0xb5, 0x00, 0xb2 +}; +static const u8 enc_output044[] __initconst = { + 0x7e, 0x72, 0xf5, 0xa1, 0x85, 0xaf, 0x16, 0xa6, + 0x11, 0x92, 0x1b, 0x43, 0x8f, 0x74, 0x9f, 0x0b, + 0x12, 0x42, 0xc6, 0x70, 0x73, 0x23, 0x34, 0x02, + 0x9a, 0xdf, 0xe1, 0xc5, 0x00, 0x16, 0x51, 0xe4 +}; +static const u8 enc_assoc044[] __initconst = { + 0x02 +}; +static const u8 enc_nonce044[] __initconst = { + 0x87, 0x34, 0x5f, 0x10, 0x55, 0xfd, 0x9e, 0x21, + 0x02, 0xd5, 0x06, 0x56 +}; +static const u8 enc_key044[] __initconst = { + 0x7d, 0x00, 0xb4, 0x80, 0x95, 0xad, 0xfa, 0x32, + 0x72, 0x05, 0x06, 0x07, 0xb2, 0x64, 0x18, 0x50, + 0x02, 0xba, 0x99, 0x95, 0x7c, 0x49, 0x8b, 0xe0, + 0x22, 0x77, 0x0f, 0x2c, 0xe2, 0xf3, 0x14, 0x3c +}; + +/* wycheproof - misc */ +static const u8 enc_input045[] __initconst = { + 0x02, 0xcd, 0xe1, 0x68, 0xfb, 0xa3, 0xf5, 0x44, + 0xbb, 0xd0, 0x33, 0x2f, 0x7a, 0xde, 0xad, 0xa8 +}; +static const u8 enc_output045[] __initconst = { + 0x85, 0xf2, 0x9a, 0x71, 0x95, 0x57, 0xcd, 0xd1, + 0x4d, 0x1f, 0x8f, 0xff, 0xab, 0x6d, 0x9e, 0x60, + 0x73, 0x2c, 0xa3, 0x2b, 0xec, 0xd5, 0x15, 0xa1, + 0xed, 0x35, 0x3f, 0x54, 0x2e, 0x99, 0x98, 0x58 +}; +static const u8 enc_assoc045[] __initconst = { + 0xb6, 0x48 +}; +static const u8 enc_nonce045[] __initconst = { + 0x87, 0xa3, 0x16, 0x3e, 0xc0, 0x59, 0x8a, 0xd9, + 0x5b, 0x3a, 0xa7, 0x13 +}; +static const u8 enc_key045[] __initconst = { + 0x64, 0x32, 0x71, 0x7f, 0x1d, 0xb8, 0x5e, 0x41, + 0xac, 0x78, 0x36, 0xbc, 0xe2, 0x51, 0x85, 0xa0, + 0x80, 0xd5, 0x76, 0x2b, 0x9e, 0x2b, 0x18, 0x44, + 0x4b, 0x6e, 0xc7, 0x2c, 0x3b, 0xd8, 0xe4, 0xdc +}; + +/* wycheproof - misc */ +static const u8 enc_input046[] __initconst = { + 0x16, 0xdd, 0xd2, 0x3f, 0xf5, 0x3f, 0x3d, 0x23, + 0xc0, 0x63, 0x34, 0x48, 0x70, 0x40, 0xeb, 0x47 +}; +static const u8 enc_output046[] __initconst = { + 0xc1, 0xb2, 0x95, 0x93, 0x6d, 0x56, 0xfa, 0xda, + 0xc0, 0x3e, 0x5f, 0x74, 0x2b, 0xff, 0x73, 0xa1, + 0x39, 0xc4, 0x57, 0xdb, 0xab, 0x66, 0x38, 0x2b, + 0xab, 0xb3, 0xb5, 0x58, 0x00, 0xcd, 0xa5, 0xb8 +}; +static const u8 enc_assoc046[] __initconst = { + 0xbd, 0x4c, 0xd0, 0x2f, 0xc7, 0x50, 0x2b, 0xbd, + 0xbd, 0xf6, 0xc9, 0xa3, 0xcb, 0xe8, 0xf0 +}; +static const u8 enc_nonce046[] __initconst = { + 0x6f, 0x57, 0x3a, 0xa8, 0x6b, 0xaa, 0x49, 0x2b, + 0xa4, 0x65, 0x96, 0xdf +}; +static const u8 enc_key046[] __initconst = { + 0x8e, 0x34, 0xcf, 0x73, 0xd2, 0x45, 0xa1, 0x08, + 0x2a, 0x92, 0x0b, 0x86, 0x36, 0x4e, 0xb8, 0x96, + 0xc4, 0x94, 0x64, 0x67, 0xbc, 0xb3, 0xd5, 0x89, + 0x29, 0xfc, 0xb3, 0x66, 0x90, 0xe6, 0x39, 0x4f +}; + +/* wycheproof - misc */ +static const u8 enc_input047[] __initconst = { + 0x62, 0x3b, 0x78, 0x50, 0xc3, 0x21, 0xe2, 0xcf, + 0x0c, 0x6f, 0xbc, 0xc8, 0xdf, 0xd1, 0xaf, 0xf2 +}; +static const u8 enc_output047[] __initconst = { + 0xc8, 0x4c, 0x9b, 0xb7, 0xc6, 0x1c, 0x1b, 0xcb, + 0x17, 0x77, 0x2a, 0x1c, 0x50, 0x0c, 0x50, 0x95, + 0xdb, 0xad, 0xf7, 0xa5, 0x13, 0x8c, 0xa0, 0x34, + 0x59, 0xa2, 0xcd, 0x65, 0x83, 0x1e, 0x09, 0x2f +}; +static const u8 enc_assoc047[] __initconst = { + 0x89, 0xcc, 0xe9, 0xfb, 0x47, 0x44, 0x1d, 0x07, + 0xe0, 0x24, 0x5a, 0x66, 0xfe, 0x8b, 0x77, 0x8b +}; +static const u8 enc_nonce047[] __initconst = { + 0x1a, 0x65, 0x18, 0xf0, 0x2e, 0xde, 0x1d, 0xa6, + 0x80, 0x92, 0x66, 0xd9 +}; +static const u8 enc_key047[] __initconst = { + 0xcb, 0x55, 0x75, 0xf5, 0xc7, 0xc4, 0x5c, 0x91, + 0xcf, 0x32, 0x0b, 0x13, 0x9f, 0xb5, 0x94, 0x23, + 0x75, 0x60, 0xd0, 0xa3, 0xe6, 0xf8, 0x65, 0xa6, + 0x7d, 0x4f, 0x63, 0x3f, 0x2c, 0x08, 0xf0, 0x16 +}; + +/* wycheproof - misc */ +static const u8 enc_input048[] __initconst = { + 0x87, 0xb3, 0xa4, 0xd7, 0xb2, 0x6d, 0x8d, 0x32, + 0x03, 0xa0, 0xde, 0x1d, 0x64, 0xef, 0x82, 0xe3 +}; +static const u8 enc_output048[] __initconst = { + 0x94, 0xbc, 0x80, 0x62, 0x1e, 0xd1, 0xe7, 0x1b, + 0x1f, 0xd2, 0xb5, 0xc3, 0xa1, 0x5e, 0x35, 0x68, + 0x33, 0x35, 0x11, 0x86, 0x17, 0x96, 0x97, 0x84, + 0x01, 0x59, 0x8b, 0x96, 0x37, 0x22, 0xf5, 0xb3 +}; +static const u8 enc_assoc048[] __initconst = { + 0xd1, 0x9f, 0x2d, 0x98, 0x90, 0x95, 0xf7, 0xab, + 0x03, 0xa5, 0xfd, 0xe8, 0x44, 0x16, 0xe0, 0x0c, + 0x0e +}; +static const u8 enc_nonce048[] __initconst = { + 0x56, 0x4d, 0xee, 0x49, 0xab, 0x00, 0xd2, 0x40, + 0xfc, 0x10, 0x68, 0xc3 +}; +static const u8 enc_key048[] __initconst = { + 0xa5, 0x56, 0x9e, 0x72, 0x9a, 0x69, 0xb2, 0x4b, + 0xa6, 0xe0, 0xff, 0x15, 0xc4, 0x62, 0x78, 0x97, + 0x43, 0x68, 0x24, 0xc9, 0x41, 0xe9, 0xd0, 0x0b, + 0x2e, 0x93, 0xfd, 0xdc, 0x4b, 0xa7, 0x76, 0x57 +}; + +/* wycheproof - misc */ +static const u8 enc_input049[] __initconst = { + 0xe6, 0x01, 0xb3, 0x85, 0x57, 0x79, 0x7d, 0xa2, + 0xf8, 0xa4, 0x10, 0x6a, 0x08, 0x9d, 0x1d, 0xa6 +}; +static const u8 enc_output049[] __initconst = { + 0x29, 0x9b, 0x5d, 0x3f, 0x3d, 0x03, 0xc0, 0x87, + 0x20, 0x9a, 0x16, 0xe2, 0x85, 0x14, 0x31, 0x11, + 0x4b, 0x45, 0x4e, 0xd1, 0x98, 0xde, 0x11, 0x7e, + 0x83, 0xec, 0x49, 0xfa, 0x8d, 0x85, 0x08, 0xd6 +}; +static const u8 enc_assoc049[] __initconst = { + 0x5e, 0x64, 0x70, 0xfa, 0xcd, 0x99, 0xc1, 0xd8, + 0x1e, 0x37, 0xcd, 0x44, 0x01, 0x5f, 0xe1, 0x94, + 0x80, 0xa2, 0xa4, 0xd3, 0x35, 0x2a, 0x4f, 0xf5, + 0x60, 0xc0, 0x64, 0x0f, 0xdb, 0xda +}; +static const u8 enc_nonce049[] __initconst = { + 0xdf, 0x87, 0x13, 0xe8, 0x7e, 0xc3, 0xdb, 0xcf, + 0xad, 0x14, 0xd5, 0x3e +}; +static const u8 enc_key049[] __initconst = { + 0x56, 0x20, 0x74, 0x65, 0xb4, 0xe4, 0x8e, 0x6d, + 0x04, 0x63, 0x0f, 0x4a, 0x42, 0xf3, 0x5c, 0xfc, + 0x16, 0x3a, 0xb2, 0x89, 0xc2, 0x2a, 0x2b, 0x47, + 0x84, 0xf6, 0xf9, 0x29, 0x03, 0x30, 0xbe, 0xe0 +}; + +/* wycheproof - misc */ +static const u8 enc_input050[] __initconst = { + 0xdc, 0x9e, 0x9e, 0xaf, 0x11, 0xe3, 0x14, 0x18, + 0x2d, 0xf6, 0xa4, 0xeb, 0xa1, 0x7a, 0xec, 0x9c +}; +static const u8 enc_output050[] __initconst = { + 0x60, 0x5b, 0xbf, 0x90, 0xae, 0xb9, 0x74, 0xf6, + 0x60, 0x2b, 0xc7, 0x78, 0x05, 0x6f, 0x0d, 0xca, + 0x38, 0xea, 0x23, 0xd9, 0x90, 0x54, 0xb4, 0x6b, + 0x42, 0xff, 0xe0, 0x04, 0x12, 0x9d, 0x22, 0x04 +}; +static const u8 enc_assoc050[] __initconst = { + 0xba, 0x44, 0x6f, 0x6f, 0x9a, 0x0c, 0xed, 0x22, + 0x45, 0x0f, 0xeb, 0x10, 0x73, 0x7d, 0x90, 0x07, + 0xfd, 0x69, 0xab, 0xc1, 0x9b, 0x1d, 0x4d, 0x90, + 0x49, 0xa5, 0x55, 0x1e, 0x86, 0xec, 0x2b, 0x37 +}; +static const u8 enc_nonce050[] __initconst = { + 0x8d, 0xf4, 0xb1, 0x5a, 0x88, 0x8c, 0x33, 0x28, + 0x6a, 0x7b, 0x76, 0x51 +}; +static const u8 enc_key050[] __initconst = { + 0x39, 0x37, 0x98, 0x6a, 0xf8, 0x6d, 0xaf, 0xc1, + 0xba, 0x0c, 0x46, 0x72, 0xd8, 0xab, 0xc4, 0x6c, + 0x20, 0x70, 0x62, 0x68, 0x2d, 0x9c, 0x26, 0x4a, + 0xb0, 0x6d, 0x6c, 0x58, 0x07, 0x20, 0x51, 0x30 +}; + +/* wycheproof - misc */ +static const u8 enc_input051[] __initconst = { + 0x81, 0xce, 0x84, 0xed, 0xe9, 0xb3, 0x58, 0x59, + 0xcc, 0x8c, 0x49, 0xa8, 0xf6, 0xbe, 0x7d, 0xc6 +}; +static const u8 enc_output051[] __initconst = { + 0x7b, 0x7c, 0xe0, 0xd8, 0x24, 0x80, 0x9a, 0x70, + 0xde, 0x32, 0x56, 0x2c, 0xcf, 0x2c, 0x2b, 0xbd, + 0x15, 0xd4, 0x4a, 0x00, 0xce, 0x0d, 0x19, 0xb4, + 0x23, 0x1f, 0x92, 0x1e, 0x22, 0xbc, 0x0a, 0x43 +}; +static const u8 enc_assoc051[] __initconst = { + 0xd4, 0x1a, 0x82, 0x8d, 0x5e, 0x71, 0x82, 0x92, + 0x47, 0x02, 0x19, 0x05, 0x40, 0x2e, 0xa2, 0x57, + 0xdc, 0xcb, 0xc3, 0xb8, 0x0f, 0xcd, 0x56, 0x75, + 0x05, 0x6b, 0x68, 0xbb, 0x59, 0xe6, 0x2e, 0x88, + 0x73 +}; +static const u8 enc_nonce051[] __initconst = { + 0xbe, 0x40, 0xe5, 0xf1, 0xa1, 0x18, 0x17, 0xa0, + 0xa8, 0xfa, 0x89, 0x49 +}; +static const u8 enc_key051[] __initconst = { + 0x36, 0x37, 0x2a, 0xbc, 0xdb, 0x78, 0xe0, 0x27, + 0x96, 0x46, 0xac, 0x3d, 0x17, 0x6b, 0x96, 0x74, + 0xe9, 0x15, 0x4e, 0xec, 0xf0, 0xd5, 0x46, 0x9c, + 0x65, 0x1e, 0xc7, 0xe1, 0x6b, 0x4c, 0x11, 0x99 +}; + +/* wycheproof - misc */ +static const u8 enc_input052[] __initconst = { + 0xa6, 0x67, 0x47, 0xc8, 0x9e, 0x85, 0x7a, 0xf3, + 0xa1, 0x8e, 0x2c, 0x79, 0x50, 0x00, 0x87, 0xed +}; +static const u8 enc_output052[] __initconst = { + 0xca, 0x82, 0xbf, 0xf3, 0xe2, 0xf3, 0x10, 0xcc, + 0xc9, 0x76, 0x67, 0x2c, 0x44, 0x15, 0xe6, 0x9b, + 0x57, 0x63, 0x8c, 0x62, 0xa5, 0xd8, 0x5d, 0xed, + 0x77, 0x4f, 0x91, 0x3c, 0x81, 0x3e, 0xa0, 0x32 +}; +static const u8 enc_assoc052[] __initconst = { + 0x3f, 0x2d, 0xd4, 0x9b, 0xbf, 0x09, 0xd6, 0x9a, + 0x78, 0xa3, 0xd8, 0x0e, 0xa2, 0x56, 0x66, 0x14, + 0xfc, 0x37, 0x94, 0x74, 0x19, 0x6c, 0x1a, 0xae, + 0x84, 0x58, 0x3d, 0xa7, 0x3d, 0x7f, 0xf8, 0x5c, + 0x6f, 0x42, 0xca, 0x42, 0x05, 0x6a, 0x97, 0x92, + 0xcc, 0x1b, 0x9f, 0xb3, 0xc7, 0xd2, 0x61 +}; +static const u8 enc_nonce052[] __initconst = { + 0x84, 0xc8, 0x7d, 0xae, 0x4e, 0xee, 0x27, 0x73, + 0x0e, 0xc3, 0x5d, 0x12 +}; +static const u8 enc_key052[] __initconst = { + 0x9f, 0x14, 0x79, 0xed, 0x09, 0x7d, 0x7f, 0xe5, + 0x29, 0xc1, 0x1f, 0x2f, 0x5a, 0xdd, 0x9a, 0xaf, + 0xf4, 0xa1, 0xca, 0x0b, 0x68, 0x99, 0x7a, 0x2c, + 0xb7, 0xf7, 0x97, 0x49, 0xbd, 0x90, 0xaa, 0xf4 +}; + /* wycheproof - misc */ static const u8 enc_input053[] __initconst = { 0x25, 0x6d, 0x40, 0x88, 0x80, 0x94, 0x17, 0x83, @@ -2759,6 +3858,126 @@ static const u8 enc_key073[] __initconst = { 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; +/* wycheproof - checking for int overflows */ +static const u8 enc_input074[] __initconst = { + 0xd4, 0x50, 0x0b, 0xf0, 0x09, 0x49, 0x35, 0x51, + 0xc3, 0x80, 0xad, 0xf5, 0x2c, 0x57, 0x3a, 0x69, + 0xdf, 0x7e, 0x8b, 0x76, 0x24, 0x63, 0x33, 0x0f, + 0xac, 0xc1, 0x6a, 0x57, 0x26, 0xbe, 0x71, 0x90, + 0xc6, 0x3c, 0x5a, 0x1c, 0x92, 0x65, 0x84, 0xa0, + 0x96, 0x75, 0x68, 0x28, 0xdc, 0xdc, 0x64, 0xac, + 0xdf, 0x96, 0x3d, 0x93, 0x1b, 0xf1, 0xda, 0xe2, + 0x38, 0xf3, 0xf1, 0x57, 0x22, 0x4a, 0xc4, 0xb5, + 0x42, 0xd7, 0x85, 0xb0, 0xdd, 0x84, 0xdb, 0x6b, + 0xe3, 0xbc, 0x5a, 0x36, 0x63, 0xe8, 0x41, 0x49, + 0xff, 0xbe, 0xd0, 0x9e, 0x54, 0xf7, 0x8f, 0x16, + 0xa8, 0x22, 0x3b, 0x24, 0xcb, 0x01, 0x9f, 0x58, + 0xb2, 0x1b, 0x0e, 0x55, 0x1e, 0x7a, 0xa0, 0x73, + 0x27, 0x62, 0x95, 0x51, 0x37, 0x6c, 0xcb, 0xc3, + 0x93, 0x76, 0x71, 0xa0, 0x62, 0x9b, 0xd9, 0x5c, + 0x99, 0x15, 0xc7, 0x85, 0x55, 0x77, 0x1e, 0x7a +}; +static const u8 enc_output074[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x0b, 0x30, 0x0d, 0x8d, 0xa5, 0x6c, 0x21, 0x85, + 0x75, 0x52, 0x79, 0x55, 0x3c, 0x4c, 0x82, 0xca +}; +static const u8 enc_assoc074[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce074[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x00, 0x02, 0x50, 0x6e +}; +static const u8 enc_key074[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + +/* wycheproof - checking for int overflows */ +static const u8 enc_input075[] __initconst = { + 0x7d, 0xe8, 0x7f, 0x67, 0x29, 0x94, 0x52, 0x75, + 0xd0, 0x65, 0x5d, 0xa4, 0xc7, 0xfd, 0xe4, 0x56, + 0x9e, 0x16, 0xf1, 0x11, 0xb5, 0xeb, 0x26, 0xc2, + 0x2d, 0x85, 0x9e, 0x3f, 0xf8, 0x22, 0xec, 0xed, + 0x3a, 0x6d, 0xd9, 0xa6, 0x0f, 0x22, 0x95, 0x7f, + 0x7b, 0x7c, 0x85, 0x7e, 0x88, 0x22, 0xeb, 0x9f, + 0xe0, 0xb8, 0xd7, 0x02, 0x21, 0x41, 0xf2, 0xd0, + 0xb4, 0x8f, 0x4b, 0x56, 0x12, 0xd3, 0x22, 0xa8, + 0x8d, 0xd0, 0xfe, 0x0b, 0x4d, 0x91, 0x79, 0x32, + 0x4f, 0x7c, 0x6c, 0x9e, 0x99, 0x0e, 0xfb, 0xd8, + 0x0e, 0x5e, 0xd6, 0x77, 0x58, 0x26, 0x49, 0x8b, + 0x1e, 0xfe, 0x0f, 0x71, 0xa0, 0xf3, 0xec, 0x5b, + 0x29, 0xcb, 0x28, 0xc2, 0x54, 0x0a, 0x7d, 0xcd, + 0x51, 0xb7, 0xda, 0xae, 0xe0, 0xff, 0x4a, 0x7f, + 0x3a, 0xc1, 0xee, 0x54, 0xc2, 0x9e, 0xe4, 0xc1, + 0x70, 0xde, 0x40, 0x8f, 0x66, 0x69, 0x21, 0x94 +}; +static const u8 enc_output075[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc5, 0x78, 0xe2, 0xaa, 0x44, 0xd3, 0x09, 0xb7, + 0xb6, 0xa5, 0x19, 0x3b, 0xdc, 0x61, 0x18, 0xf5 +}; +static const u8 enc_assoc075[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_nonce075[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x00, 0x03, 0x18, 0xa5 +}; +static const u8 enc_key075[] __initconst = { + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30 +}; + /* wycheproof - checking for int overflows */ static const u8 enc_input076[] __initconst = { 0x1b, 0x99, 0x6f, 0x9a, 0x3c, 0xcc, 0x67, 0x85, @@ -3349,6 +4568,286 @@ static const u8 enc_key085[] __initconst = { 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f }; +/* wycheproof - special case tag */ +static const u8 enc_input086[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output086[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f +}; +static const u8 enc_assoc086[] __initconst = { + 0x85, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xa6, 0x90, 0x2f, 0xcb, 0xc8, 0x83, 0xbb, 0xc1, + 0x80, 0xb2, 0x56, 0xae, 0x34, 0xad, 0x7f, 0x00 +}; +static const u8 enc_nonce086[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key086[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input087[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output087[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc087[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x24, 0x7e, 0x50, 0x64, 0x2a, 0x1c, 0x0a, 0x2f, + 0x8f, 0x77, 0x21, 0x96, 0x09, 0xdb, 0xa9, 0x58 +}; +static const u8 enc_nonce087[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key087[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input088[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output088[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const u8 enc_assoc088[] __initconst = { + 0x7c, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xd9, 0xe7, 0x2c, 0x06, 0x4a, 0xc8, 0x96, 0x1f, + 0x3f, 0xa5, 0x85, 0xe0, 0xe2, 0xab, 0xd6, 0x00 +}; +static const u8 enc_nonce088[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key088[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input089[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output089[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80 +}; +static const u8 enc_assoc089[] __initconst = { + 0x65, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x95, 0xaf, 0x0f, 0x4d, 0x0b, 0x68, 0x6e, 0xae, + 0xcc, 0xca, 0x43, 0x07, 0xd5, 0x96, 0xf5, 0x02 +}; +static const u8 enc_nonce089[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key089[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input090[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output090[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f, + 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, 0x7f +}; +static const u8 enc_assoc090[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x85, 0x40, 0xb4, 0x64, 0x35, 0x77, 0x07, 0xbe, + 0x3a, 0x39, 0xd5, 0x5c, 0x34, 0xf8, 0xbc, 0xb3 +}; +static const u8 enc_nonce090[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key090[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input091[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output091[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc091[] __initconst = { + 0x4f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x66, 0x23, 0xd9, 0x90, 0xb8, 0x98, 0xd8, 0x30, + 0xd2, 0x12, 0xaf, 0x23, 0x83, 0x33, 0x07, 0x01 +}; +static const u8 enc_nonce091[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key091[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + +/* wycheproof - special case tag */ +static const u8 enc_input092[] __initconst = { + 0x9a, 0x49, 0xc4, 0x0f, 0x8b, 0x48, 0xd7, 0xc6, + 0x6d, 0x1d, 0xb4, 0xe5, 0x3f, 0x20, 0xf2, 0xdd, + 0x4a, 0xaa, 0x24, 0x1d, 0xda, 0xb2, 0x6b, 0x5b, + 0xc0, 0xe2, 0x18, 0xb7, 0x2c, 0x33, 0x90, 0xf2, + 0xdf, 0x3e, 0xbd, 0x01, 0x76, 0x70, 0x44, 0x19, + 0x97, 0x2b, 0xcd, 0xbc, 0x6b, 0xbc, 0xb3, 0xe4, + 0xe7, 0x4a, 0x71, 0x52, 0x8e, 0xf5, 0x12, 0x63, + 0xce, 0x24, 0xe0, 0xd5, 0x75, 0xe0, 0xe4, 0x4d +}; +static const u8 enc_output092[] __initconst = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; +static const u8 enc_assoc092[] __initconst = { + 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x5f, 0x16, 0xd0, 0x9f, 0x17, 0x78, 0x72, 0x11, + 0xb7, 0xd4, 0x84, 0xe0, 0x24, 0xf8, 0x97, 0x01 +}; +static const u8 enc_nonce092[] __initconst = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b +}; +static const u8 enc_key092[] __initconst = { + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f +}; + /* wycheproof - edge case intermediate sums in poly1305 */ static const u8 enc_input093[] __initconst = { 0x00, 0x52, 0x35, 0xd2, 0xa9, 0x19, 0xf2, 0x8d, @@ -4455,6 +5954,86 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input011), sizeof(enc_assoc011), sizeof(enc_nonce011) }, { enc_input012, enc_output012, enc_assoc012, enc_nonce012, enc_key012, sizeof(enc_input012), sizeof(enc_assoc012), sizeof(enc_nonce012) }, + { enc_input013, enc_output013, enc_assoc013, enc_nonce013, enc_key013, + sizeof(enc_input013), sizeof(enc_assoc013), sizeof(enc_nonce013) }, + { enc_input014, enc_output014, enc_assoc014, enc_nonce014, enc_key014, + sizeof(enc_input014), sizeof(enc_assoc014), sizeof(enc_nonce014) }, + { enc_input015, enc_output015, enc_assoc015, enc_nonce015, enc_key015, + sizeof(enc_input015), sizeof(enc_assoc015), sizeof(enc_nonce015) }, + { enc_input016, enc_output016, enc_assoc016, enc_nonce016, enc_key016, + sizeof(enc_input016), sizeof(enc_assoc016), sizeof(enc_nonce016) }, + { enc_input017, enc_output017, enc_assoc017, enc_nonce017, enc_key017, + sizeof(enc_input017), sizeof(enc_assoc017), sizeof(enc_nonce017) }, + { enc_input018, enc_output018, enc_assoc018, enc_nonce018, enc_key018, + sizeof(enc_input018), sizeof(enc_assoc018), sizeof(enc_nonce018) }, + { enc_input019, enc_output019, enc_assoc019, enc_nonce019, enc_key019, + sizeof(enc_input019), sizeof(enc_assoc019), sizeof(enc_nonce019) }, + { enc_input020, enc_output020, enc_assoc020, enc_nonce020, enc_key020, + sizeof(enc_input020), sizeof(enc_assoc020), sizeof(enc_nonce020) }, + { enc_input021, enc_output021, enc_assoc021, enc_nonce021, enc_key021, + sizeof(enc_input021), sizeof(enc_assoc021), sizeof(enc_nonce021) }, + { enc_input022, enc_output022, enc_assoc022, enc_nonce022, enc_key022, + sizeof(enc_input022), sizeof(enc_assoc022), sizeof(enc_nonce022) }, + { enc_input023, enc_output023, enc_assoc023, enc_nonce023, enc_key023, + sizeof(enc_input023), sizeof(enc_assoc023), sizeof(enc_nonce023) }, + { enc_input024, enc_output024, enc_assoc024, enc_nonce024, enc_key024, + sizeof(enc_input024), sizeof(enc_assoc024), sizeof(enc_nonce024) }, + { enc_input025, enc_output025, enc_assoc025, enc_nonce025, enc_key025, + sizeof(enc_input025), sizeof(enc_assoc025), sizeof(enc_nonce025) }, + { enc_input026, enc_output026, enc_assoc026, enc_nonce026, enc_key026, + sizeof(enc_input026), sizeof(enc_assoc026), sizeof(enc_nonce026) }, + { enc_input027, enc_output027, enc_assoc027, enc_nonce027, enc_key027, + sizeof(enc_input027), sizeof(enc_assoc027), sizeof(enc_nonce027) }, + { enc_input028, enc_output028, enc_assoc028, enc_nonce028, enc_key028, + sizeof(enc_input028), sizeof(enc_assoc028), sizeof(enc_nonce028) }, + { enc_input029, enc_output029, enc_assoc029, enc_nonce029, enc_key029, + sizeof(enc_input029), sizeof(enc_assoc029), sizeof(enc_nonce029) }, + { enc_input030, enc_output030, enc_assoc030, enc_nonce030, enc_key030, + sizeof(enc_input030), sizeof(enc_assoc030), sizeof(enc_nonce030) }, + { enc_input031, enc_output031, enc_assoc031, enc_nonce031, enc_key031, + sizeof(enc_input031), sizeof(enc_assoc031), sizeof(enc_nonce031) }, + { enc_input032, enc_output032, enc_assoc032, enc_nonce032, enc_key032, + sizeof(enc_input032), sizeof(enc_assoc032), sizeof(enc_nonce032) }, + { enc_input033, enc_output033, enc_assoc033, enc_nonce033, enc_key033, + sizeof(enc_input033), sizeof(enc_assoc033), sizeof(enc_nonce033) }, + { enc_input034, enc_output034, enc_assoc034, enc_nonce034, enc_key034, + sizeof(enc_input034), sizeof(enc_assoc034), sizeof(enc_nonce034) }, + { enc_input035, enc_output035, enc_assoc035, enc_nonce035, enc_key035, + sizeof(enc_input035), sizeof(enc_assoc035), sizeof(enc_nonce035) }, + { enc_input036, enc_output036, enc_assoc036, enc_nonce036, enc_key036, + sizeof(enc_input036), sizeof(enc_assoc036), sizeof(enc_nonce036) }, + { enc_input037, enc_output037, enc_assoc037, enc_nonce037, enc_key037, + sizeof(enc_input037), sizeof(enc_assoc037), sizeof(enc_nonce037) }, + { enc_input038, enc_output038, enc_assoc038, enc_nonce038, enc_key038, + sizeof(enc_input038), sizeof(enc_assoc038), sizeof(enc_nonce038) }, + { enc_input039, enc_output039, enc_assoc039, enc_nonce039, enc_key039, + sizeof(enc_input039), sizeof(enc_assoc039), sizeof(enc_nonce039) }, + { enc_input040, enc_output040, enc_assoc040, enc_nonce040, enc_key040, + sizeof(enc_input040), sizeof(enc_assoc040), sizeof(enc_nonce040) }, + { enc_input041, enc_output041, enc_assoc041, enc_nonce041, enc_key041, + sizeof(enc_input041), sizeof(enc_assoc041), sizeof(enc_nonce041) }, + { enc_input042, enc_output042, enc_assoc042, enc_nonce042, enc_key042, + sizeof(enc_input042), sizeof(enc_assoc042), sizeof(enc_nonce042) }, + { enc_input043, enc_output043, enc_assoc043, enc_nonce043, enc_key043, + sizeof(enc_input043), sizeof(enc_assoc043), sizeof(enc_nonce043) }, + { enc_input044, enc_output044, enc_assoc044, enc_nonce044, enc_key044, + sizeof(enc_input044), sizeof(enc_assoc044), sizeof(enc_nonce044) }, + { enc_input045, enc_output045, enc_assoc045, enc_nonce045, enc_key045, + sizeof(enc_input045), sizeof(enc_assoc045), sizeof(enc_nonce045) }, + { enc_input046, enc_output046, enc_assoc046, enc_nonce046, enc_key046, + sizeof(enc_input046), sizeof(enc_assoc046), sizeof(enc_nonce046) }, + { enc_input047, enc_output047, enc_assoc047, enc_nonce047, enc_key047, + sizeof(enc_input047), sizeof(enc_assoc047), sizeof(enc_nonce047) }, + { enc_input048, enc_output048, enc_assoc048, enc_nonce048, enc_key048, + sizeof(enc_input048), sizeof(enc_assoc048), sizeof(enc_nonce048) }, + { enc_input049, enc_output049, enc_assoc049, enc_nonce049, enc_key049, + sizeof(enc_input049), sizeof(enc_assoc049), sizeof(enc_nonce049) }, + { enc_input050, enc_output050, enc_assoc050, enc_nonce050, enc_key050, + sizeof(enc_input050), sizeof(enc_assoc050), sizeof(enc_nonce050) }, + { enc_input051, enc_output051, enc_assoc051, enc_nonce051, enc_key051, + sizeof(enc_input051), sizeof(enc_assoc051), sizeof(enc_nonce051) }, + { enc_input052, enc_output052, enc_assoc052, enc_nonce052, enc_key052, + sizeof(enc_input052), sizeof(enc_assoc052), sizeof(enc_nonce052) }, { enc_input053, enc_output053, enc_assoc053, enc_nonce053, enc_key053, sizeof(enc_input053), sizeof(enc_assoc053), sizeof(enc_nonce053) }, { enc_input054, enc_output054, enc_assoc054, enc_nonce054, enc_key054, @@ -4497,6 +6076,10 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input072), sizeof(enc_assoc072), sizeof(enc_nonce072) }, { enc_input073, enc_output073, enc_assoc073, enc_nonce073, enc_key073, sizeof(enc_input073), sizeof(enc_assoc073), sizeof(enc_nonce073) }, + { enc_input074, enc_output074, enc_assoc074, enc_nonce074, enc_key074, + sizeof(enc_input074), sizeof(enc_assoc074), sizeof(enc_nonce074) }, + { enc_input075, enc_output075, enc_assoc075, enc_nonce075, enc_key075, + sizeof(enc_input075), sizeof(enc_assoc075), sizeof(enc_nonce075) }, { enc_input076, enc_output076, enc_assoc076, enc_nonce076, enc_key076, sizeof(enc_input076), sizeof(enc_assoc076), sizeof(enc_nonce076) }, { enc_input077, enc_output077, enc_assoc077, enc_nonce077, enc_key077, @@ -4517,6 +6100,20 @@ chacha20poly1305_enc_vectors[] __initconst = { sizeof(enc_input084), sizeof(enc_assoc084), sizeof(enc_nonce084) }, { enc_input085, enc_output085, enc_assoc085, enc_nonce085, enc_key085, sizeof(enc_input085), sizeof(enc_assoc085), sizeof(enc_nonce085) }, + { enc_input086, enc_output086, enc_assoc086, enc_nonce086, enc_key086, + sizeof(enc_input086), sizeof(enc_assoc086), sizeof(enc_nonce086) }, + { enc_input087, enc_output087, enc_assoc087, enc_nonce087, enc_key087, + sizeof(enc_input087), sizeof(enc_assoc087), sizeof(enc_nonce087) }, + { enc_input088, enc_output088, enc_assoc088, enc_nonce088, enc_key088, + sizeof(enc_input088), sizeof(enc_assoc088), sizeof(enc_nonce088) }, + { enc_input089, enc_output089, enc_assoc089, enc_nonce089, enc_key089, + sizeof(enc_input089), sizeof(enc_assoc089), sizeof(enc_nonce089) }, + { enc_input090, enc_output090, enc_assoc090, enc_nonce090, enc_key090, + sizeof(enc_input090), sizeof(enc_assoc090), sizeof(enc_nonce090) }, + { enc_input091, enc_output091, enc_assoc091, enc_nonce091, enc_key091, + sizeof(enc_input091), sizeof(enc_assoc091), sizeof(enc_nonce091) }, + { enc_input092, enc_output092, enc_assoc092, enc_nonce092, enc_key092, + sizeof(enc_input092), sizeof(enc_assoc092), sizeof(enc_nonce092) }, { enc_input093, enc_output093, enc_assoc093, enc_nonce093, enc_key093, sizeof(enc_input093), sizeof(enc_assoc093), sizeof(enc_nonce093) }, { enc_input094, enc_output094, enc_assoc094, enc_nonce094, enc_key094, @@ -7224,6 +8821,43 @@ xchacha20poly1305_dec_vectors[] __initconst = { sizeof(xdec_input001), sizeof(xdec_assoc001), sizeof(xdec_nonce001) } }; +/* This is for the selftests-only, since it is only useful for the purpose of + * testing the underlying primitives and interactions. + */ +static void __init +chacha20poly1305_encrypt_bignonce(u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 nonce[12], + const u8 key[CHACHA20POLY1305_KEY_SIZE]) +{ + const u8 *pad0 = page_address(ZERO_PAGE(0)); + struct poly1305_desc_ctx poly1305_state; + u32 chacha20_state[CHACHA_STATE_WORDS]; + union { + u8 block0[POLY1305_KEY_SIZE]; + __le64 lens[2]; + } b = {{ 0 }}; + u8 bottom_row[16] = { 0 }; + u32 le_key[8]; + int i; + + memcpy(&bottom_row[4], nonce, 12); + for (i = 0; i < 8; ++i) + le_key[i] = get_unaligned_le32(key + sizeof(le_key[i]) * i); + chacha_init(chacha20_state, le_key, bottom_row); + chacha20_crypt(chacha20_state, b.block0, b.block0, sizeof(b.block0)); + poly1305_init(&poly1305_state, b.block0); + poly1305_update(&poly1305_state, ad, ad_len); + poly1305_update(&poly1305_state, pad0, (0x10 - ad_len) & 0xf); + chacha20_crypt(chacha20_state, dst, src, src_len); + poly1305_update(&poly1305_state, dst, src_len); + poly1305_update(&poly1305_state, pad0, (0x10 - src_len) & 0xf); + b.lens[0] = cpu_to_le64(ad_len); + b.lens[1] = cpu_to_le64(src_len); + poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens)); + poly1305_final(&poly1305_state, dst + src_len); +} + static void __init chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, @@ -7233,6 +8867,9 @@ chacha20poly1305_selftest_encrypt(u8 *dst, const u8 *src, const size_t src_len, if (nonce_len == 8) chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, get_unaligned_le64(nonce), key); + else if (nonce_len == 12) + chacha20poly1305_encrypt_bignonce(dst, src, src_len, ad, + ad_len, nonce, key); else BUG(); } @@ -7248,14 +8885,14 @@ decryption_success(bool func_ret, bool expect_failure, int memcmp_result) bool __init chacha20poly1305_selftest(void) { enum { MAXIMUM_TEST_BUFFER_LEN = 1UL << 12 }; - size_t i; - u8 *computed_output = NULL, *heap_src = NULL; - struct scatterlist sg_src; + size_t i, j, k, total_len; + u8 *computed_output = NULL, *input = NULL; bool success = true, ret; + struct scatterlist sg_src[3]; - heap_src = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); computed_output = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); - if (!heap_src || !computed_output) { + input = kmalloc(MAXIMUM_TEST_BUFFER_LEN, GFP_KERNEL); + if (!computed_output || !input) { pr_err("chacha20poly1305 self-test malloc: FAIL\n"); success = false; goto out; @@ -7284,17 +8921,17 @@ bool __init chacha20poly1305_selftest(void) for (i = 0; i < ARRAY_SIZE(chacha20poly1305_enc_vectors); ++i) { if (chacha20poly1305_enc_vectors[i].nlen != 8) continue; - memcpy(heap_src, chacha20poly1305_enc_vectors[i].input, + memcpy(computed_output, chacha20poly1305_enc_vectors[i].input, chacha20poly1305_enc_vectors[i].ilen); - sg_init_one(&sg_src, heap_src, + sg_init_one(sg_src, computed_output, chacha20poly1305_enc_vectors[i].ilen + POLY1305_DIGEST_SIZE); - chacha20poly1305_encrypt_sg_inplace(&sg_src, + ret = chacha20poly1305_encrypt_sg_inplace(sg_src, chacha20poly1305_enc_vectors[i].ilen, chacha20poly1305_enc_vectors[i].assoc, chacha20poly1305_enc_vectors[i].alen, get_unaligned_le64(chacha20poly1305_enc_vectors[i].nonce), chacha20poly1305_enc_vectors[i].key); - if (memcmp(heap_src, + if (!ret || memcmp(computed_output, chacha20poly1305_enc_vectors[i].output, chacha20poly1305_enc_vectors[i].ilen + POLY1305_DIGEST_SIZE)) { @@ -7326,11 +8963,11 @@ bool __init chacha20poly1305_selftest(void) } for (i = 0; i < ARRAY_SIZE(chacha20poly1305_dec_vectors); ++i) { - memcpy(heap_src, chacha20poly1305_dec_vectors[i].input, + memcpy(computed_output, chacha20poly1305_dec_vectors[i].input, chacha20poly1305_dec_vectors[i].ilen); - sg_init_one(&sg_src, heap_src, + sg_init_one(sg_src, computed_output, chacha20poly1305_dec_vectors[i].ilen); - ret = chacha20poly1305_decrypt_sg_inplace(&sg_src, + ret = chacha20poly1305_decrypt_sg_inplace(sg_src, chacha20poly1305_dec_vectors[i].ilen, chacha20poly1305_dec_vectors[i].assoc, chacha20poly1305_dec_vectors[i].alen, @@ -7338,7 +8975,7 @@ bool __init chacha20poly1305_selftest(void) chacha20poly1305_dec_vectors[i].key); if (!decryption_success(ret, chacha20poly1305_dec_vectors[i].failure, - memcmp(heap_src, chacha20poly1305_dec_vectors[i].output, + memcmp(computed_output, chacha20poly1305_dec_vectors[i].output, chacha20poly1305_dec_vectors[i].ilen - POLY1305_DIGEST_SIZE))) { pr_err("chacha20poly1305 sg decryption self-test %zu: FAIL\n", @@ -7365,6 +9002,7 @@ bool __init chacha20poly1305_selftest(void) success = false; } } + for (i = 0; i < ARRAY_SIZE(xchacha20poly1305_dec_vectors); ++i) { memset(computed_output, 0, MAXIMUM_TEST_BUFFER_LEN); ret = xchacha20poly1305_decrypt(computed_output, @@ -7386,8 +9024,59 @@ bool __init chacha20poly1305_selftest(void) } } + for (total_len = POLY1305_DIGEST_SIZE; IS_ENABLED(DEBUG_CHACHA20POLY1305_SLOW_CHUNK_TEST) + && total_len <= 1 << 10; ++total_len) { + for (i = 0; i <= total_len; ++i) { + for (j = i; j <= total_len; ++j) { + k = 0; + sg_init_table(sg_src, 3); + if (i) + sg_set_buf(&sg_src[k++], input, i); + if (j - i) + sg_set_buf(&sg_src[k++], input + i, j - i); + if (total_len - j) + sg_set_buf(&sg_src[k++], input + j, total_len - j); + sg_init_marker(sg_src, k); + memset(computed_output, 0, total_len); + memset(input, 0, total_len); + + if (!chacha20poly1305_encrypt_sg_inplace(sg_src, + total_len - POLY1305_DIGEST_SIZE, NULL, 0, + 0, enc_key001)) + goto chunkfail; + chacha20poly1305_encrypt(computed_output, + computed_output, + total_len - POLY1305_DIGEST_SIZE, NULL, 0, 0, + enc_key001); + if (memcmp(computed_output, input, total_len)) + goto chunkfail; + if (!chacha20poly1305_decrypt(computed_output, + input, total_len, NULL, 0, 0, enc_key001)) + goto chunkfail; + for (k = 0; k < total_len - POLY1305_DIGEST_SIZE; ++k) { + if (computed_output[k]) + goto chunkfail; + } + if (!chacha20poly1305_decrypt_sg_inplace(sg_src, + total_len, NULL, 0, 0, enc_key001)) + goto chunkfail; + for (k = 0; k < total_len - POLY1305_DIGEST_SIZE; ++k) { + if (input[k]) + goto chunkfail; + } + continue; + + chunkfail: + pr_err("chacha20poly1305 chunked self-test %zu/%zu/%zu: FAIL\n", + total_len, i, j); + success = false; + } + + } + } + out: - kfree(heap_src); kfree(computed_output); + kfree(input); return success; } diff --git a/lib/crypto/poly1305-donna32.c b/lib/crypto/poly1305-donna32.c new file mode 100644 index 0000000000000..3cc77d94390b2 --- /dev/null +++ b/lib/crypto/poly1305-donna32.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is based in part on Andrew Moon's poly1305-donna, which is in the + * public domain. + */ + +#include +#include +#include + +void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[16]) +{ + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + key->key.r[0] = (get_unaligned_le32(&raw_key[0])) & 0x3ffffff; + key->key.r[1] = (get_unaligned_le32(&raw_key[3]) >> 2) & 0x3ffff03; + key->key.r[2] = (get_unaligned_le32(&raw_key[6]) >> 4) & 0x3ffc0ff; + key->key.r[3] = (get_unaligned_le32(&raw_key[9]) >> 6) & 0x3f03fff; + key->key.r[4] = (get_unaligned_le32(&raw_key[12]) >> 8) & 0x00fffff; + + /* s = 5*r */ + key->precomputed_s.r[0] = key->key.r[1] * 5; + key->precomputed_s.r[1] = key->key.r[2] * 5; + key->precomputed_s.r[2] = key->key.r[3] * 5; + key->precomputed_s.r[3] = key->key.r[4] * 5; +} +EXPORT_SYMBOL(poly1305_core_setkey); + +void poly1305_core_blocks(struct poly1305_state *state, + const struct poly1305_core_key *key, const void *src, + unsigned int nblocks, u32 hibit) +{ + const u8 *input = src; + u32 r0, r1, r2, r3, r4; + u32 s1, s2, s3, s4; + u32 h0, h1, h2, h3, h4; + u64 d0, d1, d2, d3, d4; + u32 c; + + if (!nblocks) + return; + + hibit <<= 24; + + r0 = key->key.r[0]; + r1 = key->key.r[1]; + r2 = key->key.r[2]; + r3 = key->key.r[3]; + r4 = key->key.r[4]; + + s1 = key->precomputed_s.r[0]; + s2 = key->precomputed_s.r[1]; + s3 = key->precomputed_s.r[2]; + s4 = key->precomputed_s.r[3]; + + h0 = state->h[0]; + h1 = state->h[1]; + h2 = state->h[2]; + h3 = state->h[3]; + h4 = state->h[4]; + + do { + /* h += m[i] */ + h0 += (get_unaligned_le32(&input[0])) & 0x3ffffff; + h1 += (get_unaligned_le32(&input[3]) >> 2) & 0x3ffffff; + h2 += (get_unaligned_le32(&input[6]) >> 4) & 0x3ffffff; + h3 += (get_unaligned_le32(&input[9]) >> 6) & 0x3ffffff; + h4 += (get_unaligned_le32(&input[12]) >> 8) | hibit; + + /* h *= r */ + d0 = ((u64)h0 * r0) + ((u64)h1 * s4) + + ((u64)h2 * s3) + ((u64)h3 * s2) + + ((u64)h4 * s1); + d1 = ((u64)h0 * r1) + ((u64)h1 * r0) + + ((u64)h2 * s4) + ((u64)h3 * s3) + + ((u64)h4 * s2); + d2 = ((u64)h0 * r2) + ((u64)h1 * r1) + + ((u64)h2 * r0) + ((u64)h3 * s4) + + ((u64)h4 * s3); + d3 = ((u64)h0 * r3) + ((u64)h1 * r2) + + ((u64)h2 * r1) + ((u64)h3 * r0) + + ((u64)h4 * s4); + d4 = ((u64)h0 * r4) + ((u64)h1 * r3) + + ((u64)h2 * r2) + ((u64)h3 * r1) + + ((u64)h4 * r0); + + /* (partial) h %= p */ + c = (u32)(d0 >> 26); + h0 = (u32)d0 & 0x3ffffff; + d1 += c; + c = (u32)(d1 >> 26); + h1 = (u32)d1 & 0x3ffffff; + d2 += c; + c = (u32)(d2 >> 26); + h2 = (u32)d2 & 0x3ffffff; + d3 += c; + c = (u32)(d3 >> 26); + h3 = (u32)d3 & 0x3ffffff; + d4 += c; + c = (u32)(d4 >> 26); + h4 = (u32)d4 & 0x3ffffff; + h0 += c * 5; + c = (h0 >> 26); + h0 = h0 & 0x3ffffff; + h1 += c; + + input += POLY1305_BLOCK_SIZE; + } while (--nblocks); + + state->h[0] = h0; + state->h[1] = h1; + state->h[2] = h2; + state->h[3] = h3; + state->h[4] = h4; +} +EXPORT_SYMBOL(poly1305_core_blocks); + +void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], + void *dst) +{ + u8 *mac = dst; + u32 h0, h1, h2, h3, h4, c; + u32 g0, g1, g2, g3, g4; + u64 f; + u32 mask; + + /* fully carry h */ + h0 = state->h[0]; + h1 = state->h[1]; + h2 = state->h[2]; + h3 = state->h[3]; + h4 = state->h[4]; + + c = h1 >> 26; + h1 = h1 & 0x3ffffff; + h2 += c; + c = h2 >> 26; + h2 = h2 & 0x3ffffff; + h3 += c; + c = h3 >> 26; + h3 = h3 & 0x3ffffff; + h4 += c; + c = h4 >> 26; + h4 = h4 & 0x3ffffff; + h0 += c * 5; + c = h0 >> 26; + h0 = h0 & 0x3ffffff; + h1 += c; + + /* compute h + -p */ + g0 = h0 + 5; + c = g0 >> 26; + g0 &= 0x3ffffff; + g1 = h1 + c; + c = g1 >> 26; + g1 &= 0x3ffffff; + g2 = h2 + c; + c = g2 >> 26; + g2 &= 0x3ffffff; + g3 = h3 + c; + c = g3 >> 26; + g3 &= 0x3ffffff; + g4 = h4 + c - (1UL << 26); + + /* select h if h < p, or h + -p if h >= p */ + mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1; + g0 &= mask; + g1 &= mask; + g2 &= mask; + g3 &= mask; + g4 &= mask; + mask = ~mask; + + h0 = (h0 & mask) | g0; + h1 = (h1 & mask) | g1; + h2 = (h2 & mask) | g2; + h3 = (h3 & mask) | g3; + h4 = (h4 & mask) | g4; + + /* h = h % (2^128) */ + h0 = ((h0) | (h1 << 26)) & 0xffffffff; + h1 = ((h1 >> 6) | (h2 << 20)) & 0xffffffff; + h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff; + h3 = ((h3 >> 18) | (h4 << 8)) & 0xffffffff; + + if (likely(nonce)) { + /* mac = (h + nonce) % (2^128) */ + f = (u64)h0 + nonce[0]; + h0 = (u32)f; + f = (u64)h1 + nonce[1] + (f >> 32); + h1 = (u32)f; + f = (u64)h2 + nonce[2] + (f >> 32); + h2 = (u32)f; + f = (u64)h3 + nonce[3] + (f >> 32); + h3 = (u32)f; + } + + put_unaligned_le32(h0, &mac[0]); + put_unaligned_le32(h1, &mac[4]); + put_unaligned_le32(h2, &mac[8]); + put_unaligned_le32(h3, &mac[12]); +} +EXPORT_SYMBOL(poly1305_core_emit); diff --git a/lib/crypto/poly1305-donna64.c b/lib/crypto/poly1305-donna64.c new file mode 100644 index 0000000000000..6ae181bb43450 --- /dev/null +++ b/lib/crypto/poly1305-donna64.c @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is based in part on Andrew Moon's poly1305-donna, which is in the + * public domain. + */ + +#include +#include +#include + +typedef __uint128_t u128; + +void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[16]) +{ + u64 t0, t1; + + /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ + t0 = get_unaligned_le64(&raw_key[0]); + t1 = get_unaligned_le64(&raw_key[8]); + + key->key.r64[0] = t0 & 0xffc0fffffffULL; + key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL; + key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL; + + /* s = 20*r */ + key->precomputed_s.r64[0] = key->key.r64[1] * 20; + key->precomputed_s.r64[1] = key->key.r64[2] * 20; +} +EXPORT_SYMBOL(poly1305_core_setkey); + +void poly1305_core_blocks(struct poly1305_state *state, + const struct poly1305_core_key *key, const void *src, + unsigned int nblocks, u32 hibit) +{ + const u8 *input = src; + u64 hibit64; + u64 r0, r1, r2; + u64 s1, s2; + u64 h0, h1, h2; + u64 c; + u128 d0, d1, d2, d; + + if (!nblocks) + return; + + hibit64 = ((u64)hibit) << 40; + + r0 = key->key.r64[0]; + r1 = key->key.r64[1]; + r2 = key->key.r64[2]; + + h0 = state->h64[0]; + h1 = state->h64[1]; + h2 = state->h64[2]; + + s1 = key->precomputed_s.r64[0]; + s2 = key->precomputed_s.r64[1]; + + do { + u64 t0, t1; + + /* h += m[i] */ + t0 = get_unaligned_le64(&input[0]); + t1 = get_unaligned_le64(&input[8]); + + h0 += t0 & 0xfffffffffffULL; + h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL; + h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64; + + /* h *= r */ + d0 = (u128)h0 * r0; + d = (u128)h1 * s2; + d0 += d; + d = (u128)h2 * s1; + d0 += d; + d1 = (u128)h0 * r1; + d = (u128)h1 * r0; + d1 += d; + d = (u128)h2 * s2; + d1 += d; + d2 = (u128)h0 * r2; + d = (u128)h1 * r1; + d2 += d; + d = (u128)h2 * r0; + d2 += d; + + /* (partial) h %= p */ + c = (u64)(d0 >> 44); + h0 = (u64)d0 & 0xfffffffffffULL; + d1 += c; + c = (u64)(d1 >> 44); + h1 = (u64)d1 & 0xfffffffffffULL; + d2 += c; + c = (u64)(d2 >> 42); + h2 = (u64)d2 & 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 = h0 & 0xfffffffffffULL; + h1 += c; + + input += POLY1305_BLOCK_SIZE; + } while (--nblocks); + + state->h64[0] = h0; + state->h64[1] = h1; + state->h64[2] = h2; +} +EXPORT_SYMBOL(poly1305_core_blocks); + +void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], + void *dst) +{ + u8 *mac = dst; + u64 h0, h1, h2, c; + u64 g0, g1, g2; + u64 t0, t1; + + /* fully carry h */ + h0 = state->h64[0]; + h1 = state->h64[1]; + h2 = state->h64[2]; + + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += c; + c = h2 >> 42; + h2 &= 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += c; + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += c; + c = h2 >> 42; + h2 &= 0x3ffffffffffULL; + h0 += c * 5; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += c; + + /* compute h + -p */ + g0 = h0 + 5; + c = g0 >> 44; + g0 &= 0xfffffffffffULL; + g1 = h1 + c; + c = g1 >> 44; + g1 &= 0xfffffffffffULL; + g2 = h2 + c - (1ULL << 42); + + /* select h if h < p, or h + -p if h >= p */ + c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1; + g0 &= c; + g1 &= c; + g2 &= c; + c = ~c; + h0 = (h0 & c) | g0; + h1 = (h1 & c) | g1; + h2 = (h2 & c) | g2; + + if (likely(nonce)) { + /* h = (h + nonce) */ + t0 = ((u64)nonce[1] << 32) | nonce[0]; + t1 = ((u64)nonce[3] << 32) | nonce[2]; + + h0 += t0 & 0xfffffffffffULL; + c = h0 >> 44; + h0 &= 0xfffffffffffULL; + h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c; + c = h1 >> 44; + h1 &= 0xfffffffffffULL; + h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c; + h2 &= 0x3ffffffffffULL; + } + + /* mac = h % (2^128) */ + h0 = h0 | (h1 << 44); + h1 = (h1 >> 20) | (h2 << 24); + + put_unaligned_le64(h0, &mac[0]); + put_unaligned_le64(h1, &mac[8]); +} +EXPORT_SYMBOL(poly1305_core_emit); diff --git a/lib/crypto/poly1305.c b/lib/crypto/poly1305.c index 32ec293c65ae7..9d2d14df0fee5 100644 --- a/lib/crypto/poly1305.c +++ b/lib/crypto/poly1305.c @@ -12,151 +12,9 @@ #include #include -static inline u64 mlt(u64 a, u64 b) -{ - return a * b; -} - -static inline u32 sr(u64 v, u_char n) -{ - return v >> n; -} - -static inline u32 and(u32 v, u32 mask) -{ - return v & mask; -} - -void poly1305_core_setkey(struct poly1305_key *key, const u8 *raw_key) -{ - /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */ - key->r[0] = (get_unaligned_le32(raw_key + 0) >> 0) & 0x3ffffff; - key->r[1] = (get_unaligned_le32(raw_key + 3) >> 2) & 0x3ffff03; - key->r[2] = (get_unaligned_le32(raw_key + 6) >> 4) & 0x3ffc0ff; - key->r[3] = (get_unaligned_le32(raw_key + 9) >> 6) & 0x3f03fff; - key->r[4] = (get_unaligned_le32(raw_key + 12) >> 8) & 0x00fffff; -} -EXPORT_SYMBOL_GPL(poly1305_core_setkey); - -void poly1305_core_blocks(struct poly1305_state *state, - const struct poly1305_key *key, const void *src, - unsigned int nblocks, u32 hibit) -{ - u32 r0, r1, r2, r3, r4; - u32 s1, s2, s3, s4; - u32 h0, h1, h2, h3, h4; - u64 d0, d1, d2, d3, d4; - - if (!nblocks) - return; - - r0 = key->r[0]; - r1 = key->r[1]; - r2 = key->r[2]; - r3 = key->r[3]; - r4 = key->r[4]; - - s1 = r1 * 5; - s2 = r2 * 5; - s3 = r3 * 5; - s4 = r4 * 5; - - h0 = state->h[0]; - h1 = state->h[1]; - h2 = state->h[2]; - h3 = state->h[3]; - h4 = state->h[4]; - - do { - /* h += m[i] */ - h0 += (get_unaligned_le32(src + 0) >> 0) & 0x3ffffff; - h1 += (get_unaligned_le32(src + 3) >> 2) & 0x3ffffff; - h2 += (get_unaligned_le32(src + 6) >> 4) & 0x3ffffff; - h3 += (get_unaligned_le32(src + 9) >> 6) & 0x3ffffff; - h4 += (get_unaligned_le32(src + 12) >> 8) | (hibit << 24); - - /* h *= r */ - d0 = mlt(h0, r0) + mlt(h1, s4) + mlt(h2, s3) + - mlt(h3, s2) + mlt(h4, s1); - d1 = mlt(h0, r1) + mlt(h1, r0) + mlt(h2, s4) + - mlt(h3, s3) + mlt(h4, s2); - d2 = mlt(h0, r2) + mlt(h1, r1) + mlt(h2, r0) + - mlt(h3, s4) + mlt(h4, s3); - d3 = mlt(h0, r3) + mlt(h1, r2) + mlt(h2, r1) + - mlt(h3, r0) + mlt(h4, s4); - d4 = mlt(h0, r4) + mlt(h1, r3) + mlt(h2, r2) + - mlt(h3, r1) + mlt(h4, r0); - - /* (partial) h %= p */ - d1 += sr(d0, 26); h0 = and(d0, 0x3ffffff); - d2 += sr(d1, 26); h1 = and(d1, 0x3ffffff); - d3 += sr(d2, 26); h2 = and(d2, 0x3ffffff); - d4 += sr(d3, 26); h3 = and(d3, 0x3ffffff); - h0 += sr(d4, 26) * 5; h4 = and(d4, 0x3ffffff); - h1 += h0 >> 26; h0 = h0 & 0x3ffffff; - - src += POLY1305_BLOCK_SIZE; - } while (--nblocks); - - state->h[0] = h0; - state->h[1] = h1; - state->h[2] = h2; - state->h[3] = h3; - state->h[4] = h4; -} -EXPORT_SYMBOL_GPL(poly1305_core_blocks); - -void poly1305_core_emit(const struct poly1305_state *state, void *dst) -{ - u32 h0, h1, h2, h3, h4; - u32 g0, g1, g2, g3, g4; - u32 mask; - - /* fully carry h */ - h0 = state->h[0]; - h1 = state->h[1]; - h2 = state->h[2]; - h3 = state->h[3]; - h4 = state->h[4]; - - h2 += (h1 >> 26); h1 = h1 & 0x3ffffff; - h3 += (h2 >> 26); h2 = h2 & 0x3ffffff; - h4 += (h3 >> 26); h3 = h3 & 0x3ffffff; - h0 += (h4 >> 26) * 5; h4 = h4 & 0x3ffffff; - h1 += (h0 >> 26); h0 = h0 & 0x3ffffff; - - /* compute h + -p */ - g0 = h0 + 5; - g1 = h1 + (g0 >> 26); g0 &= 0x3ffffff; - g2 = h2 + (g1 >> 26); g1 &= 0x3ffffff; - g3 = h3 + (g2 >> 26); g2 &= 0x3ffffff; - g4 = h4 + (g3 >> 26) - (1 << 26); g3 &= 0x3ffffff; - - /* select h if h < p, or h + -p if h >= p */ - mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1; - g0 &= mask; - g1 &= mask; - g2 &= mask; - g3 &= mask; - g4 &= mask; - mask = ~mask; - h0 = (h0 & mask) | g0; - h1 = (h1 & mask) | g1; - h2 = (h2 & mask) | g2; - h3 = (h3 & mask) | g3; - h4 = (h4 & mask) | g4; - - /* h = h % (2^128) */ - put_unaligned_le32((h0 >> 0) | (h1 << 26), dst + 0); - put_unaligned_le32((h1 >> 6) | (h2 << 20), dst + 4); - put_unaligned_le32((h2 >> 12) | (h3 << 14), dst + 8); - put_unaligned_le32((h3 >> 18) | (h4 << 8), dst + 12); -} -EXPORT_SYMBOL_GPL(poly1305_core_emit); - void poly1305_init_generic(struct poly1305_desc_ctx *desc, const u8 *key) { - poly1305_core_setkey(desc->r, key); + poly1305_core_setkey(&desc->core_r, key); desc->s[0] = get_unaligned_le32(key + 16); desc->s[1] = get_unaligned_le32(key + 20); desc->s[2] = get_unaligned_le32(key + 24); @@ -164,7 +22,7 @@ void poly1305_init_generic(struct poly1305_desc_ctx *desc, const u8 *key) poly1305_core_init(&desc->h); desc->buflen = 0; desc->sset = true; - desc->rset = 1; + desc->rset = 2; } EXPORT_SYMBOL_GPL(poly1305_init_generic); @@ -181,13 +39,14 @@ void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src, desc->buflen += bytes; if (desc->buflen == POLY1305_BLOCK_SIZE) { - poly1305_core_blocks(&desc->h, desc->r, desc->buf, 1, 1); + poly1305_core_blocks(&desc->h, &desc->core_r, desc->buf, + 1, 1); desc->buflen = 0; } } if (likely(nbytes >= POLY1305_BLOCK_SIZE)) { - poly1305_core_blocks(&desc->h, desc->r, src, + poly1305_core_blocks(&desc->h, &desc->core_r, src, nbytes / POLY1305_BLOCK_SIZE, 1); src += nbytes - (nbytes % POLY1305_BLOCK_SIZE); nbytes %= POLY1305_BLOCK_SIZE; @@ -202,28 +61,14 @@ EXPORT_SYMBOL_GPL(poly1305_update_generic); void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *dst) { - __le32 digest[4]; - u64 f = 0; - if (unlikely(desc->buflen)) { desc->buf[desc->buflen++] = 1; memset(desc->buf + desc->buflen, 0, POLY1305_BLOCK_SIZE - desc->buflen); - poly1305_core_blocks(&desc->h, desc->r, desc->buf, 1, 0); + poly1305_core_blocks(&desc->h, &desc->core_r, desc->buf, 1, 0); } - poly1305_core_emit(&desc->h, digest); - - /* mac = (h + s) % (2^128) */ - f = (f >> 32) + le32_to_cpu(digest[0]) + desc->s[0]; - put_unaligned_le32(f, dst + 0); - f = (f >> 32) + le32_to_cpu(digest[1]) + desc->s[1]; - put_unaligned_le32(f, dst + 4); - f = (f >> 32) + le32_to_cpu(digest[2]) + desc->s[2]; - put_unaligned_le32(f, dst + 8); - f = (f >> 32) + le32_to_cpu(digest[3]) + desc->s[3]; - put_unaligned_le32(f, dst + 12); - + poly1305_core_emit(&desc->h, desc->s, dst); *desc = (struct poly1305_desc_ctx){}; } EXPORT_SYMBOL_GPL(poly1305_final_generic); diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index 18068ed42f258..f369e7ce685b8 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -748,6 +748,39 @@ out:; } EXPORT_SYMBOL(__icmp_send); +#if IS_ENABLED(CONFIG_NF_NAT) +#include +void icmp_ndo_send(struct sk_buff *skb_in, int type, int code, __be32 info) +{ + struct sk_buff *cloned_skb = NULL; + enum ip_conntrack_info ctinfo; + struct nf_conn *ct; + __be32 orig_ip; + + ct = nf_ct_get(skb_in, &ctinfo); + if (!ct || !(ct->status & IPS_SRC_NAT)) { + icmp_send(skb_in, type, code, info); + return; + } + + if (skb_shared(skb_in)) + skb_in = cloned_skb = skb_clone(skb_in, GFP_ATOMIC); + + if (unlikely(!skb_in || skb_network_header(skb_in) < skb_in->head || + (skb_network_header(skb_in) + sizeof(struct iphdr)) > + skb_tail_pointer(skb_in) || skb_ensure_writable(skb_in, + skb_network_offset(skb_in) + sizeof(struct iphdr)))) + goto out; + + orig_ip = ip_hdr(skb_in)->saddr; + ip_hdr(skb_in)->saddr = ct->tuplehash[0].tuple.src.u3.ip; + icmp_send(skb_in, type, code, info); + ip_hdr(skb_in)->saddr = orig_ip; +out: + consume_skb(cloned_skb); +} +EXPORT_SYMBOL(icmp_ndo_send); +#endif static void icmp_socket_deliver(struct sk_buff *skb, u32 info) { diff --git a/net/ipv6/ip6_icmp.c b/net/ipv6/ip6_icmp.c index 02045494c24cc..e0086758b6ee3 100644 --- a/net/ipv6/ip6_icmp.c +++ b/net/ipv6/ip6_icmp.c @@ -45,4 +45,38 @@ void icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info) rcu_read_unlock(); } EXPORT_SYMBOL(icmpv6_send); + +#if IS_ENABLED(CONFIG_NF_NAT) +#include +void icmpv6_ndo_send(struct sk_buff *skb_in, u8 type, u8 code, __u32 info) +{ + struct sk_buff *cloned_skb = NULL; + enum ip_conntrack_info ctinfo; + struct in6_addr orig_ip; + struct nf_conn *ct; + + ct = nf_ct_get(skb_in, &ctinfo); + if (!ct || !(ct->status & IPS_SRC_NAT)) { + icmpv6_send(skb_in, type, code, info); + return; + } + + if (skb_shared(skb_in)) + skb_in = cloned_skb = skb_clone(skb_in, GFP_ATOMIC); + + if (unlikely(!skb_in || skb_network_header(skb_in) < skb_in->head || + (skb_network_header(skb_in) + sizeof(struct ipv6hdr)) > + skb_tail_pointer(skb_in) || skb_ensure_writable(skb_in, + skb_network_offset(skb_in) + sizeof(struct ipv6hdr)))) + goto out; + + orig_ip = ipv6_hdr(skb_in)->saddr; + ipv6_hdr(skb_in)->saddr = ct->tuplehash[0].tuple.src.u3.in6; + icmpv6_send(skb_in, type, code, info); + ipv6_hdr(skb_in)->saddr = orig_ip; +out: + consume_skb(cloned_skb); +} +EXPORT_SYMBOL(icmpv6_ndo_send); +#endif #endif diff --git a/tools/testing/selftests/wireguard/netns.sh b/tools/testing/selftests/wireguard/netns.sh new file mode 100755 index 0000000000000..936e1ca9410ec --- /dev/null +++ b/tools/testing/selftests/wireguard/netns.sh @@ -0,0 +1,555 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. +# +# This script tests the below topology: +# +# ┌─────────────────────┐ ┌──────────────────────────────────┐ ┌─────────────────────┐ +# │ $ns1 namespace │ │ $ns0 namespace │ │ $ns2 namespace │ +# │ │ │ │ │ │ +# │┌────────┐ │ │ ┌────────┐ │ │ ┌────────┐│ +# ││ wg0 │───────────┼───┼────────────│ lo │────────────┼───┼───────────│ wg0 ││ +# │├────────┴──────────┐│ │ ┌───────┴────────┴────────┐ │ │┌──────────┴────────┤│ +# ││192.168.241.1/24 ││ │ │(ns1) (ns2) │ │ ││192.168.241.2/24 ││ +# ││fd00::1/24 ││ │ │127.0.0.1:1 127.0.0.1:2│ │ ││fd00::2/24 ││ +# │└───────────────────┘│ │ │[::]:1 [::]:2 │ │ │└───────────────────┘│ +# └─────────────────────┘ │ └─────────────────────────┘ │ └─────────────────────┘ +# └──────────────────────────────────┘ +# +# After the topology is prepared we run a series of TCP/UDP iperf3 tests between the +# wireguard peers in $ns1 and $ns2. Note that $ns0 is the endpoint for the wg0 +# interfaces in $ns1 and $ns2. See https://www.wireguard.com/netns/ for further +# details on how this is accomplished. +set -e + +exec 3>&1 +export LANG=C +export WG_HIDE_KEYS=never +netns0="wg-test-$$-0" +netns1="wg-test-$$-1" +netns2="wg-test-$$-2" +pretty() { echo -e "\x1b[32m\x1b[1m[+] ${1:+NS$1: }${2}\x1b[0m" >&3; } +pp() { pretty "" "$*"; "$@"; } +maybe_exec() { if [[ $BASHPID -eq $$ ]]; then "$@"; else exec "$@"; fi; } +n0() { pretty 0 "$*"; maybe_exec ip netns exec $netns0 "$@"; } +n1() { pretty 1 "$*"; maybe_exec ip netns exec $netns1 "$@"; } +n2() { pretty 2 "$*"; maybe_exec ip netns exec $netns2 "$@"; } +ip0() { pretty 0 "ip $*"; ip -n $netns0 "$@"; } +ip1() { pretty 1 "ip $*"; ip -n $netns1 "$@"; } +ip2() { pretty 2 "ip $*"; ip -n $netns2 "$@"; } +sleep() { read -t "$1" -N 1 || true; } +waitiperf() { pretty "${1//*-}" "wait for iperf:5201 pid $2"; while [[ $(ss -N "$1" -tlpH 'sport = 5201') != *\"iperf3\",pid=$2,fd=* ]]; do sleep 0.1; done; } +waitncatudp() { pretty "${1//*-}" "wait for udp:1111 pid $2"; while [[ $(ss -N "$1" -ulpH 'sport = 1111') != *\"ncat\",pid=$2,fd=* ]]; do sleep 0.1; done; } +waitiface() { pretty "${1//*-}" "wait for $2 to come up"; ip netns exec "$1" bash -c "while [[ \$(< \"/sys/class/net/$2/operstate\") != up ]]; do read -t .1 -N 0 || true; done;"; } + +cleanup() { + set +e + exec 2>/dev/null + printf "$orig_message_cost" > /proc/sys/net/core/message_cost + ip0 link del dev wg0 + ip1 link del dev wg0 + ip2 link del dev wg0 + local to_kill="$(ip netns pids $netns0) $(ip netns pids $netns1) $(ip netns pids $netns2)" + [[ -n $to_kill ]] && kill $to_kill + pp ip netns del $netns1 + pp ip netns del $netns2 + pp ip netns del $netns0 + exit +} + +orig_message_cost="$(< /proc/sys/net/core/message_cost)" +trap cleanup EXIT +printf 0 > /proc/sys/net/core/message_cost + +ip netns del $netns0 2>/dev/null || true +ip netns del $netns1 2>/dev/null || true +ip netns del $netns2 2>/dev/null || true +pp ip netns add $netns0 +pp ip netns add $netns1 +pp ip netns add $netns2 +ip0 link set up dev lo + +ip0 link add dev wg0 type wireguard +ip0 link set wg0 netns $netns1 +ip0 link add dev wg0 type wireguard +ip0 link set wg0 netns $netns2 +key1="$(pp wg genkey)" +key2="$(pp wg genkey)" +key3="$(pp wg genkey)" +pub1="$(pp wg pubkey <<<"$key1")" +pub2="$(pp wg pubkey <<<"$key2")" +pub3="$(pp wg pubkey <<<"$key3")" +psk="$(pp wg genpsk)" +[[ -n $key1 && -n $key2 && -n $psk ]] + +configure_peers() { + ip1 addr add 192.168.241.1/24 dev wg0 + ip1 addr add fd00::1/24 dev wg0 + + ip2 addr add 192.168.241.2/24 dev wg0 + ip2 addr add fd00::2/24 dev wg0 + + n1 wg set wg0 \ + private-key <(echo "$key1") \ + listen-port 1 \ + peer "$pub2" \ + preshared-key <(echo "$psk") \ + allowed-ips 192.168.241.2/32,fd00::2/128 + n2 wg set wg0 \ + private-key <(echo "$key2") \ + listen-port 2 \ + peer "$pub1" \ + preshared-key <(echo "$psk") \ + allowed-ips 192.168.241.1/32,fd00::1/128 + + ip1 link set up dev wg0 + ip2 link set up dev wg0 +} +configure_peers + +tests() { + # Ping over IPv4 + n2 ping -c 10 -f -W 1 192.168.241.1 + n1 ping -c 10 -f -W 1 192.168.241.2 + + # Ping over IPv6 + n2 ping6 -c 10 -f -W 1 fd00::1 + n1 ping6 -c 10 -f -W 1 fd00::2 + + # TCP over IPv4 + n2 iperf3 -s -1 -B 192.168.241.2 & + waitiperf $netns2 $! + n1 iperf3 -Z -t 3 -c 192.168.241.2 + + # TCP over IPv6 + n1 iperf3 -s -1 -B fd00::1 & + waitiperf $netns1 $! + n2 iperf3 -Z -t 3 -c fd00::1 + + # UDP over IPv4 + n1 iperf3 -s -1 -B 192.168.241.1 & + waitiperf $netns1 $! + n2 iperf3 -Z -t 3 -b 0 -u -c 192.168.241.1 + + # UDP over IPv6 + n2 iperf3 -s -1 -B fd00::2 & + waitiperf $netns2 $! + n1 iperf3 -Z -t 3 -b 0 -u -c fd00::2 +} + +[[ $(ip1 link show dev wg0) =~ mtu\ ([0-9]+) ]] && orig_mtu="${BASH_REMATCH[1]}" +big_mtu=$(( 34816 - 1500 + $orig_mtu )) + +# Test using IPv4 as outer transport +n1 wg set wg0 peer "$pub2" endpoint 127.0.0.1:2 +n2 wg set wg0 peer "$pub1" endpoint 127.0.0.1:1 +# Before calling tests, we first make sure that the stats counters and timestamper are working +n2 ping -c 10 -f -W 1 192.168.241.1 +{ read _; read _; read _; read rx_bytes _; read _; read tx_bytes _; } < <(ip2 -stats link show dev wg0) +(( rx_bytes == 1372 && (tx_bytes == 1428 || tx_bytes == 1460) )) +{ read _; read _; read _; read rx_bytes _; read _; read tx_bytes _; } < <(ip1 -stats link show dev wg0) +(( tx_bytes == 1372 && (rx_bytes == 1428 || rx_bytes == 1460) )) +read _ rx_bytes tx_bytes < <(n2 wg show wg0 transfer) +(( rx_bytes == 1372 && (tx_bytes == 1428 || tx_bytes == 1460) )) +read _ rx_bytes tx_bytes < <(n1 wg show wg0 transfer) +(( tx_bytes == 1372 && (rx_bytes == 1428 || rx_bytes == 1460) )) +read _ timestamp < <(n1 wg show wg0 latest-handshakes) +(( timestamp != 0 )) + +tests +ip1 link set wg0 mtu $big_mtu +ip2 link set wg0 mtu $big_mtu +tests + +ip1 link set wg0 mtu $orig_mtu +ip2 link set wg0 mtu $orig_mtu + +# Test using IPv6 as outer transport +n1 wg set wg0 peer "$pub2" endpoint [::1]:2 +n2 wg set wg0 peer "$pub1" endpoint [::1]:1 +tests +ip1 link set wg0 mtu $big_mtu +ip2 link set wg0 mtu $big_mtu +tests + +# Test that route MTUs work with the padding +ip1 link set wg0 mtu 1300 +ip2 link set wg0 mtu 1300 +n1 wg set wg0 peer "$pub2" endpoint 127.0.0.1:2 +n2 wg set wg0 peer "$pub1" endpoint 127.0.0.1:1 +n0 iptables -A INPUT -m length --length 1360 -j DROP +n1 ip route add 192.168.241.2/32 dev wg0 mtu 1299 +n2 ip route add 192.168.241.1/32 dev wg0 mtu 1299 +n2 ping -c 1 -W 1 -s 1269 192.168.241.1 +n2 ip route delete 192.168.241.1/32 dev wg0 mtu 1299 +n1 ip route delete 192.168.241.2/32 dev wg0 mtu 1299 +n0 iptables -F INPUT + +ip1 link set wg0 mtu $orig_mtu +ip2 link set wg0 mtu $orig_mtu + +# Test using IPv4 that roaming works +ip0 -4 addr del 127.0.0.1/8 dev lo +ip0 -4 addr add 127.212.121.99/8 dev lo +n1 wg set wg0 listen-port 9999 +n1 wg set wg0 peer "$pub2" endpoint 127.0.0.1:2 +n1 ping6 -W 1 -c 1 fd00::2 +[[ $(n2 wg show wg0 endpoints) == "$pub1 127.212.121.99:9999" ]] + +# Test using IPv6 that roaming works +n1 wg set wg0 listen-port 9998 +n1 wg set wg0 peer "$pub2" endpoint [::1]:2 +n1 ping -W 1 -c 1 192.168.241.2 +[[ $(n2 wg show wg0 endpoints) == "$pub1 [::1]:9998" ]] + +# Test that crypto-RP filter works +n1 wg set wg0 peer "$pub2" allowed-ips 192.168.241.0/24 +exec 4< <(n1 ncat -l -u -p 1111) +ncat_pid=$! +waitncatudp $netns1 $ncat_pid +n2 ncat -u 192.168.241.1 1111 <<<"X" +read -r -N 1 -t 1 out <&4 && [[ $out == "X" ]] +kill $ncat_pid +more_specific_key="$(pp wg genkey | pp wg pubkey)" +n1 wg set wg0 peer "$more_specific_key" allowed-ips 192.168.241.2/32 +n2 wg set wg0 listen-port 9997 +exec 4< <(n1 ncat -l -u -p 1111) +ncat_pid=$! +waitncatudp $netns1 $ncat_pid +n2 ncat -u 192.168.241.1 1111 <<<"X" +! read -r -N 1 -t 1 out <&4 || false +kill $ncat_pid +n1 wg set wg0 peer "$more_specific_key" remove +[[ $(n1 wg show wg0 endpoints) == "$pub2 [::1]:9997" ]] + +# Test that we can change private keys keys and immediately handshake +n1 wg set wg0 private-key <(echo "$key1") peer "$pub2" preshared-key <(echo "$psk") allowed-ips 192.168.241.2/32 endpoint 127.0.0.1:2 +n2 wg set wg0 private-key <(echo "$key2") listen-port 2 peer "$pub1" preshared-key <(echo "$psk") allowed-ips 192.168.241.1/32 +n1 ping -W 1 -c 1 192.168.241.2 +n1 wg set wg0 private-key <(echo "$key3") +n2 wg set wg0 peer "$pub3" preshared-key <(echo "$psk") allowed-ips 192.168.241.1/32 peer "$pub1" remove +n1 ping -W 1 -c 1 192.168.241.2 + +ip1 link del wg0 +ip2 link del wg0 + +# Test using NAT. We now change the topology to this: +# ┌────────────────────────────────────────┐ ┌────────────────────────────────────────────────┐ ┌────────────────────────────────────────┐ +# │ $ns1 namespace │ │ $ns0 namespace │ │ $ns2 namespace │ +# │ │ │ │ │ │ +# │ ┌─────┐ ┌─────┐ │ │ ┌──────┐ ┌──────┐ │ │ ┌─────┐ ┌─────┐ │ +# │ │ wg0 │─────────────│vethc│───────────┼────┼────│vethrc│ │vethrs│──────────────┼─────┼──│veths│────────────│ wg0 │ │ +# │ ├─────┴──────────┐ ├─────┴──────────┐│ │ ├──────┴─────────┐ ├──────┴────────────┐ │ │ ├─────┴──────────┐ ├─────┴──────────┐ │ +# │ │192.168.241.1/24│ │192.168.1.100/24││ │ │192.168.1.1/24 │ │10.0.0.1/24 │ │ │ │10.0.0.100/24 │ │192.168.241.2/24│ │ +# │ │fd00::1/24 │ │ ││ │ │ │ │SNAT:192.168.1.0/24│ │ │ │ │ │fd00::2/24 │ │ +# │ └────────────────┘ └────────────────┘│ │ └────────────────┘ └───────────────────┘ │ │ └────────────────┘ └────────────────┘ │ +# └────────────────────────────────────────┘ └────────────────────────────────────────────────┘ └────────────────────────────────────────┘ + +ip1 link add dev wg0 type wireguard +ip2 link add dev wg0 type wireguard +configure_peers + +ip0 link add vethrc type veth peer name vethc +ip0 link add vethrs type veth peer name veths +ip0 link set vethc netns $netns1 +ip0 link set veths netns $netns2 +ip0 link set vethrc up +ip0 link set vethrs up +ip0 addr add 192.168.1.1/24 dev vethrc +ip0 addr add 10.0.0.1/24 dev vethrs +ip1 addr add 192.168.1.100/24 dev vethc +ip1 link set vethc up +ip1 route add default via 192.168.1.1 +ip2 addr add 10.0.0.100/24 dev veths +ip2 link set veths up +waitiface $netns0 vethrc +waitiface $netns0 vethrs +waitiface $netns1 vethc +waitiface $netns2 veths + +n0 bash -c 'printf 1 > /proc/sys/net/ipv4/ip_forward' +n0 bash -c 'printf 2 > /proc/sys/net/netfilter/nf_conntrack_udp_timeout' +n0 bash -c 'printf 2 > /proc/sys/net/netfilter/nf_conntrack_udp_timeout_stream' +n0 iptables -t nat -A POSTROUTING -s 192.168.1.0/24 -d 10.0.0.0/24 -j SNAT --to 10.0.0.1 + +n1 wg set wg0 peer "$pub2" endpoint 10.0.0.100:2 persistent-keepalive 1 +n1 ping -W 1 -c 1 192.168.241.2 +n2 ping -W 1 -c 1 192.168.241.1 +[[ $(n2 wg show wg0 endpoints) == "$pub1 10.0.0.1:1" ]] +# Demonstrate n2 can still send packets to n1, since persistent-keepalive will prevent connection tracking entry from expiring (to see entries: `n0 conntrack -L`). +pp sleep 3 +n2 ping -W 1 -c 1 192.168.241.1 +n1 wg set wg0 peer "$pub2" persistent-keepalive 0 + +# Do a wg-quick(8)-style policy routing for the default route, making sure vethc has a v6 address to tease out bugs. +ip1 -6 addr add fc00::9/96 dev vethc +ip1 -6 route add default via fc00::1 +ip2 -4 addr add 192.168.99.7/32 dev wg0 +ip2 -6 addr add abab::1111/128 dev wg0 +n1 wg set wg0 fwmark 51820 peer "$pub2" allowed-ips 192.168.99.7,abab::1111 +ip1 -6 route add default dev wg0 table 51820 +ip1 -6 rule add not fwmark 51820 table 51820 +ip1 -6 rule add table main suppress_prefixlength 0 +ip1 -4 route add default dev wg0 table 51820 +ip1 -4 rule add not fwmark 51820 table 51820 +ip1 -4 rule add table main suppress_prefixlength 0 +# Flood the pings instead of sending just one, to trigger routing table reference counting bugs. +n1 ping -W 1 -c 100 -f 192.168.99.7 +n1 ping -W 1 -c 100 -f abab::1111 + +# Have ns2 NAT into wg0 packets from ns0, but return an icmp error along the right route. +n2 iptables -t nat -A POSTROUTING -s 10.0.0.0/24 -d 192.168.241.0/24 -j SNAT --to 192.168.241.2 +n0 iptables -t filter -A INPUT \! -s 10.0.0.0/24 -i vethrs -j DROP # Manual rpfilter just to be explicit. +n2 bash -c 'printf 1 > /proc/sys/net/ipv4/ip_forward' +ip0 -4 route add 192.168.241.1 via 10.0.0.100 +n2 wg set wg0 peer "$pub1" remove +[[ $(! n0 ping -W 1 -c 1 192.168.241.1 || false) == *"From 10.0.0.100 icmp_seq=1 Destination Host Unreachable"* ]] + +n0 iptables -t nat -F +n0 iptables -t filter -F +n2 iptables -t nat -F +ip0 link del vethrc +ip0 link del vethrs +ip1 link del wg0 +ip2 link del wg0 + +# Test that saddr routing is sticky but not too sticky, changing to this topology: +# ┌────────────────────────────────────────┐ ┌────────────────────────────────────────┐ +# │ $ns1 namespace │ │ $ns2 namespace │ +# │ │ │ │ +# │ ┌─────┐ ┌─────┐ │ │ ┌─────┐ ┌─────┐ │ +# │ │ wg0 │─────────────│veth1│───────────┼────┼──│veth2│────────────│ wg0 │ │ +# │ ├─────┴──────────┐ ├─────┴──────────┐│ │ ├─────┴──────────┐ ├─────┴──────────┐ │ +# │ │192.168.241.1/24│ │10.0.0.1/24 ││ │ │10.0.0.2/24 │ │192.168.241.2/24│ │ +# │ │fd00::1/24 │ │fd00:aa::1/96 ││ │ │fd00:aa::2/96 │ │fd00::2/24 │ │ +# │ └────────────────┘ └────────────────┘│ │ └────────────────┘ └────────────────┘ │ +# └────────────────────────────────────────┘ └────────────────────────────────────────┘ + +ip1 link add dev wg0 type wireguard +ip2 link add dev wg0 type wireguard +configure_peers +ip1 link add veth1 type veth peer name veth2 +ip1 link set veth2 netns $netns2 +n1 bash -c 'printf 0 > /proc/sys/net/ipv6/conf/all/accept_dad' +n2 bash -c 'printf 0 > /proc/sys/net/ipv6/conf/all/accept_dad' +n1 bash -c 'printf 0 > /proc/sys/net/ipv6/conf/veth1/accept_dad' +n2 bash -c 'printf 0 > /proc/sys/net/ipv6/conf/veth2/accept_dad' +n1 bash -c 'printf 1 > /proc/sys/net/ipv4/conf/veth1/promote_secondaries' + +# First we check that we aren't overly sticky and can fall over to new IPs when old ones are removed +ip1 addr add 10.0.0.1/24 dev veth1 +ip1 addr add fd00:aa::1/96 dev veth1 +ip2 addr add 10.0.0.2/24 dev veth2 +ip2 addr add fd00:aa::2/96 dev veth2 +ip1 link set veth1 up +ip2 link set veth2 up +waitiface $netns1 veth1 +waitiface $netns2 veth2 +n1 wg set wg0 peer "$pub2" endpoint 10.0.0.2:2 +n1 ping -W 1 -c 1 192.168.241.2 +ip1 addr add 10.0.0.10/24 dev veth1 +ip1 addr del 10.0.0.1/24 dev veth1 +n1 ping -W 1 -c 1 192.168.241.2 +n1 wg set wg0 peer "$pub2" endpoint [fd00:aa::2]:2 +n1 ping -W 1 -c 1 192.168.241.2 +ip1 addr add fd00:aa::10/96 dev veth1 +ip1 addr del fd00:aa::1/96 dev veth1 +n1 ping -W 1 -c 1 192.168.241.2 + +# Now we show that we can successfully do reply to sender routing +ip1 link set veth1 down +ip2 link set veth2 down +ip1 addr flush dev veth1 +ip2 addr flush dev veth2 +ip1 addr add 10.0.0.1/24 dev veth1 +ip1 addr add 10.0.0.2/24 dev veth1 +ip1 addr add fd00:aa::1/96 dev veth1 +ip1 addr add fd00:aa::2/96 dev veth1 +ip2 addr add 10.0.0.3/24 dev veth2 +ip2 addr add fd00:aa::3/96 dev veth2 +ip1 link set veth1 up +ip2 link set veth2 up +waitiface $netns1 veth1 +waitiface $netns2 veth2 +n2 wg set wg0 peer "$pub1" endpoint 10.0.0.1:1 +n2 ping -W 1 -c 1 192.168.241.1 +[[ $(n2 wg show wg0 endpoints) == "$pub1 10.0.0.1:1" ]] +n2 wg set wg0 peer "$pub1" endpoint [fd00:aa::1]:1 +n2 ping -W 1 -c 1 192.168.241.1 +[[ $(n2 wg show wg0 endpoints) == "$pub1 [fd00:aa::1]:1" ]] +n2 wg set wg0 peer "$pub1" endpoint 10.0.0.2:1 +n2 ping -W 1 -c 1 192.168.241.1 +[[ $(n2 wg show wg0 endpoints) == "$pub1 10.0.0.2:1" ]] +n2 wg set wg0 peer "$pub1" endpoint [fd00:aa::2]:1 +n2 ping -W 1 -c 1 192.168.241.1 +[[ $(n2 wg show wg0 endpoints) == "$pub1 [fd00:aa::2]:1" ]] + +# What happens if the inbound destination address belongs to a different interface as the default route? +ip1 link add dummy0 type dummy +ip1 addr add 10.50.0.1/24 dev dummy0 +ip1 link set dummy0 up +ip2 route add 10.50.0.0/24 dev veth2 +n2 wg set wg0 peer "$pub1" endpoint 10.50.0.1:1 +n2 ping -W 1 -c 1 192.168.241.1 +[[ $(n2 wg show wg0 endpoints) == "$pub1 10.50.0.1:1" ]] + +ip1 link del dummy0 +ip1 addr flush dev veth1 +ip2 addr flush dev veth2 +ip1 route flush dev veth1 +ip2 route flush dev veth2 + +# Now we see what happens if another interface route takes precedence over an ongoing one +ip1 link add veth3 type veth peer name veth4 +ip1 link set veth4 netns $netns2 +ip1 addr add 10.0.0.1/24 dev veth1 +ip2 addr add 10.0.0.2/24 dev veth2 +ip1 addr add 10.0.0.3/24 dev veth3 +ip1 link set veth1 up +ip2 link set veth2 up +ip1 link set veth3 up +ip2 link set veth4 up +waitiface $netns1 veth1 +waitiface $netns2 veth2 +waitiface $netns1 veth3 +waitiface $netns2 veth4 +ip1 route flush dev veth1 +ip1 route flush dev veth3 +ip1 route add 10.0.0.0/24 dev veth1 src 10.0.0.1 metric 2 +n1 wg set wg0 peer "$pub2" endpoint 10.0.0.2:2 +n1 ping -W 1 -c 1 192.168.241.2 +[[ $(n2 wg show wg0 endpoints) == "$pub1 10.0.0.1:1" ]] +ip1 route add 10.0.0.0/24 dev veth3 src 10.0.0.3 metric 1 +n1 bash -c 'printf 0 > /proc/sys/net/ipv4/conf/veth1/rp_filter' +n2 bash -c 'printf 0 > /proc/sys/net/ipv4/conf/veth4/rp_filter' +n1 bash -c 'printf 0 > /proc/sys/net/ipv4/conf/all/rp_filter' +n2 bash -c 'printf 0 > /proc/sys/net/ipv4/conf/all/rp_filter' +n1 ping -W 1 -c 1 192.168.241.2 +[[ $(n2 wg show wg0 endpoints) == "$pub1 10.0.0.3:1" ]] + +ip1 link del veth1 +ip1 link del veth3 +ip1 link del wg0 +ip2 link del wg0 + +# We test that Netlink/IPC is working properly by doing things that usually cause split responses +ip0 link add dev wg0 type wireguard +config=( "[Interface]" "PrivateKey=$(wg genkey)" "[Peer]" "PublicKey=$(wg genkey)" ) +for a in {1..255}; do + for b in {0..255}; do + config+=( "AllowedIPs=$a.$b.0.0/16,$a::$b/128" ) + done +done +n0 wg setconf wg0 <(printf '%s\n' "${config[@]}") +i=0 +for ip in $(n0 wg show wg0 allowed-ips); do + ((++i)) +done +((i == 255*256*2+1)) +ip0 link del wg0 +ip0 link add dev wg0 type wireguard +config=( "[Interface]" "PrivateKey=$(wg genkey)" ) +for a in {1..40}; do + config+=( "[Peer]" "PublicKey=$(wg genkey)" ) + for b in {1..52}; do + config+=( "AllowedIPs=$a.$b.0.0/16" ) + done +done +n0 wg setconf wg0 <(printf '%s\n' "${config[@]}") +i=0 +while read -r line; do + j=0 + for ip in $line; do + ((++j)) + done + ((j == 53)) + ((++i)) +done < <(n0 wg show wg0 allowed-ips) +((i == 40)) +ip0 link del wg0 +ip0 link add wg0 type wireguard +config=( ) +for i in {1..29}; do + config+=( "[Peer]" "PublicKey=$(wg genkey)" ) +done +config+=( "[Peer]" "PublicKey=$(wg genkey)" "AllowedIPs=255.2.3.4/32,abcd::255/128" ) +n0 wg setconf wg0 <(printf '%s\n' "${config[@]}") +n0 wg showconf wg0 > /dev/null +ip0 link del wg0 + +allowedips=( ) +for i in {1..197}; do + allowedips+=( abcd::$i ) +done +saved_ifs="$IFS" +IFS=, +allowedips="${allowedips[*]}" +IFS="$saved_ifs" +ip0 link add wg0 type wireguard +n0 wg set wg0 peer "$pub1" +n0 wg set wg0 peer "$pub2" allowed-ips "$allowedips" +{ + read -r pub allowedips + [[ $pub == "$pub1" && $allowedips == "(none)" ]] + read -r pub allowedips + [[ $pub == "$pub2" ]] + i=0 + for _ in $allowedips; do + ((++i)) + done + ((i == 197)) +} < <(n0 wg show wg0 allowed-ips) +ip0 link del wg0 + +! n0 wg show doesnotexist || false + +ip0 link add wg0 type wireguard +n0 wg set wg0 private-key <(echo "$key1") peer "$pub2" preshared-key <(echo "$psk") +[[ $(n0 wg show wg0 private-key) == "$key1" ]] +[[ $(n0 wg show wg0 preshared-keys) == "$pub2 $psk" ]] +n0 wg set wg0 private-key /dev/null peer "$pub2" preshared-key /dev/null +[[ $(n0 wg show wg0 private-key) == "(none)" ]] +[[ $(n0 wg show wg0 preshared-keys) == "$pub2 (none)" ]] +n0 wg set wg0 peer "$pub2" +n0 wg set wg0 private-key <(echo "$key2") +[[ $(n0 wg show wg0 public-key) == "$pub2" ]] +[[ -z $(n0 wg show wg0 peers) ]] +n0 wg set wg0 peer "$pub2" +[[ -z $(n0 wg show wg0 peers) ]] +n0 wg set wg0 private-key <(echo "$key1") +n0 wg set wg0 peer "$pub2" +[[ $(n0 wg show wg0 peers) == "$pub2" ]] +n0 wg set wg0 private-key <(echo "/${key1:1}") +[[ $(n0 wg show wg0 private-key) == "+${key1:1}" ]] +n0 wg set wg0 peer "$pub2" allowed-ips 0.0.0.0/0,10.0.0.0/8,100.0.0.0/10,172.16.0.0/12,192.168.0.0/16 +n0 wg set wg0 peer "$pub2" allowed-ips 0.0.0.0/0 +n0 wg set wg0 peer "$pub2" allowed-ips ::/0,1700::/111,5000::/4,e000::/37,9000::/75 +n0 wg set wg0 peer "$pub2" allowed-ips ::/0 +n0 wg set wg0 peer "$pub2" remove +for low_order_point in AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= AQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= 4Ot6fDtBuK4WVuP68Z/EatoJjeucMrH9hmIFFl9JuAA= X5yVvKNQjCSx0LFVnIPvWwREXMRYHI6G2CJO3dCfEVc= 7P///////////////////////////////////////38= 7f///////////////////////////////////////38= 7v///////////////////////////////////////38=; do + n0 wg set wg0 peer "$low_order_point" persistent-keepalive 1 endpoint 127.0.0.1:1111 +done +[[ -n $(n0 wg show wg0 peers) ]] +exec 4< <(n0 ncat -l -u -p 1111) +ncat_pid=$! +waitncatudp $netns0 $ncat_pid +ip0 link set wg0 up +! read -r -n 1 -t 2 <&4 || false +kill $ncat_pid +ip0 link del wg0 + +declare -A objects +while read -t 0.1 -r line 2>/dev/null || [[ $? -ne 142 ]]; do + [[ $line =~ .*(wg[0-9]+:\ [A-Z][a-z]+\ [0-9]+)\ .*(created|destroyed).* ]] || continue + objects["${BASH_REMATCH[1]}"]+="${BASH_REMATCH[2]}" +done < /dev/kmsg +alldeleted=1 +for object in "${!objects[@]}"; do + if [[ ${objects["$object"]} != *createddestroyed ]]; then + echo "Error: $object: merely ${objects["$object"]}" >&3 + alldeleted=0 + fi +done +[[ $alldeleted -eq 1 ]] +pretty "" "Objects that were created were also destroyed." diff --git a/tools/testing/selftests/wireguard/qemu/.gitignore b/tools/testing/selftests/wireguard/qemu/.gitignore new file mode 100644 index 0000000000000..415b542a9d599 --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/.gitignore @@ -0,0 +1,2 @@ +build/ +distfiles/ diff --git a/tools/testing/selftests/wireguard/qemu/Makefile b/tools/testing/selftests/wireguard/qemu/Makefile new file mode 100644 index 0000000000000..28d477683e8ab --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/Makefile @@ -0,0 +1,377 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + +PWD := $(shell pwd) + +CHOST := $(shell gcc -dumpmachine) +HOST_ARCH := $(firstword $(subst -, ,$(CHOST))) +ifneq (,$(ARCH)) +CBUILD := $(subst -gcc,,$(lastword $(subst /, ,$(firstword $(wildcard $(foreach bindir,$(subst :, ,$(PATH)),$(bindir)/$(ARCH)-*-gcc)))))) +ifeq (,$(CBUILD)) +$(error The toolchain for $(ARCH) is not installed) +endif +else +CBUILD := $(CHOST) +ARCH := $(firstword $(subst -, ,$(CBUILD))) +endif + +# Set these from the environment to override +KERNEL_PATH ?= $(PWD)/../../../../.. +BUILD_PATH ?= $(PWD)/build/$(ARCH) +DISTFILES_PATH ?= $(PWD)/distfiles +NR_CPUS ?= 4 + +MIRROR := https://download.wireguard.com/qemu-test/distfiles/ + +default: qemu + +# variable name, tarball project name, version, tarball extension, default URI base +define tar_download = +$(1)_VERSION := $(3) +$(1)_NAME := $(2)-$$($(1)_VERSION) +$(1)_TAR := $(DISTFILES_PATH)/$$($(1)_NAME)$(4) +$(1)_PATH := $(BUILD_PATH)/$$($(1)_NAME) +$(call file_download,$$($(1)_NAME)$(4),$(5),$(6)) +endef + +define file_download = +$(DISTFILES_PATH)/$(1): + mkdir -p $(DISTFILES_PATH) + flock -x $$@.lock -c '[ -f $$@ ] && exit 0; wget -O $$@.tmp $(MIRROR)$(1) || wget -O $$@.tmp $(2)$(1) || rm -f $$@.tmp; [ -f $$@.tmp ] || exit 1; if echo "$(3) $$@.tmp" | sha256sum -c -; then mv $$@.tmp $$@; else rm -f $$@.tmp; exit 71; fi' +endef + +$(eval $(call tar_download,MUSL,musl,1.1.24,.tar.gz,https://www.musl-libc.org/releases/,1370c9a812b2cf2a7d92802510cca0058cc37e66a7bedd70051f0a34015022a3)) +$(eval $(call tar_download,IPERF,iperf,3.7,.tar.gz,https://downloads.es.net/pub/iperf/,d846040224317caf2f75c843d309a950a7db23f9b44b94688ccbe557d6d1710c)) +$(eval $(call tar_download,BASH,bash,5.0,.tar.gz,https://ftp.gnu.org/gnu/bash/,b4a80f2ac66170b2913efbfb9f2594f1f76c7b1afd11f799e22035d63077fb4d)) +$(eval $(call tar_download,IPROUTE2,iproute2,5.4.0,.tar.xz,https://www.kernel.org/pub/linux/utils/net/iproute2/,fe97aa60a0d4c5ac830be18937e18dc3400ca713a33a89ad896ff1e3d46086ae)) +$(eval $(call tar_download,IPTABLES,iptables,1.8.4,.tar.bz2,https://www.netfilter.org/projects/iptables/files/,993a3a5490a544c2cbf2ef15cf7e7ed21af1845baf228318d5c36ef8827e157c)) +$(eval $(call tar_download,NMAP,nmap,7.80,.tar.bz2,https://nmap.org/dist/,fcfa5a0e42099e12e4bf7a68ebe6fde05553383a682e816a7ec9256ab4773faa)) +$(eval $(call tar_download,IPUTILS,iputils,s20190709,.tar.gz,https://github.com/iputils/iputils/archive/s20190709.tar.gz/#,a15720dd741d7538dd2645f9f516d193636ae4300ff7dbc8bfca757bf166490a)) +$(eval $(call tar_download,WIREGUARD_TOOLS,wireguard-tools,1.0.20200206,.tar.xz,https://git.zx2c4.com/wireguard-tools/snapshot/,f5207248c6a3c3e3bfc9ab30b91c1897b00802ed861e1f9faaed873366078c64)) + +KERNEL_BUILD_PATH := $(BUILD_PATH)/kernel$(if $(findstring yes,$(DEBUG_KERNEL)),-debug) +rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst *,%,$2),$d)) +WIREGUARD_SOURCES := $(call rwildcard,$(KERNEL_PATH)/drivers/net/wireguard/,*) + +export CFLAGS ?= -O3 -pipe +export LDFLAGS ?= +export CPPFLAGS := -I$(BUILD_PATH)/include + +ifeq ($(HOST_ARCH),$(ARCH)) +CROSS_COMPILE_FLAG := --host=$(CHOST) +CFLAGS += -march=native +STRIP := strip +else +$(info Cross compilation: building for $(CBUILD) using $(CHOST)) +CROSS_COMPILE_FLAG := --build=$(CBUILD) --host=$(CHOST) +export CROSS_COMPILE=$(CBUILD)- +STRIP := $(CBUILD)-strip +endif +ifeq ($(ARCH),aarch64) +QEMU_ARCH := aarch64 +KERNEL_ARCH := arm64 +KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/arch/arm64/boot/Image +ifeq ($(HOST_ARCH),$(ARCH)) +QEMU_MACHINE := -cpu host -machine virt,gic_version=host,accel=kvm +else +QEMU_MACHINE := -cpu cortex-a53 -machine virt +CFLAGS += -march=armv8-a -mtune=cortex-a53 +endif +else ifeq ($(ARCH),aarch64_be) +QEMU_ARCH := aarch64 +KERNEL_ARCH := arm64 +KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/arch/arm64/boot/Image +ifeq ($(HOST_ARCH),$(ARCH)) +QEMU_MACHINE := -cpu host -machine virt,gic_version=host,accel=kvm +else +QEMU_MACHINE := -cpu cortex-a53 -machine virt +CFLAGS += -march=armv8-a -mtune=cortex-a53 +endif +else ifeq ($(ARCH),arm) +QEMU_ARCH := arm +KERNEL_ARCH := arm +KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/arch/arm/boot/zImage +ifeq ($(HOST_ARCH),$(ARCH)) +QEMU_MACHINE := -cpu host -machine virt,gic_version=host,accel=kvm +else +QEMU_MACHINE := -cpu cortex-a15 -machine virt +CFLAGS += -march=armv7-a -mtune=cortex-a15 -mabi=aapcs-linux +endif +else ifeq ($(ARCH),armeb) +QEMU_ARCH := arm +KERNEL_ARCH := arm +KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/arch/arm/boot/zImage +ifeq ($(HOST_ARCH),$(ARCH)) +QEMU_MACHINE := -cpu host -machine virt,gic_version=host,accel=kvm +else +QEMU_MACHINE := -cpu cortex-a15 -machine virt +CFLAGS += -march=armv7-a -mabi=aapcs-linux # We don't pass -mtune=cortex-a15 due to a compiler bug on big endian. +LDFLAGS += -Wl,--be8 +endif +else ifeq ($(ARCH),x86_64) +QEMU_ARCH := x86_64 +KERNEL_ARCH := x86_64 +KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/arch/x86/boot/bzImage +ifeq ($(HOST_ARCH),$(ARCH)) +QEMU_MACHINE := -cpu host -machine q35,accel=kvm +else +QEMU_MACHINE := -cpu Skylake-Server -machine q35 +CFLAGS += -march=skylake-avx512 +endif +else ifeq ($(ARCH),i686) +QEMU_ARCH := i386 +KERNEL_ARCH := x86 +KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/arch/x86/boot/bzImage +ifeq ($(subst x86_64,i686,$(HOST_ARCH)),$(ARCH)) +QEMU_MACHINE := -cpu host -machine q35,accel=kvm +else +QEMU_MACHINE := -cpu coreduo -machine q35 +CFLAGS += -march=prescott +endif +else ifeq ($(ARCH),mips64) +QEMU_ARCH := mips64 +KERNEL_ARCH := mips +KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/vmlinux +ifeq ($(HOST_ARCH),$(ARCH)) +QEMU_MACHINE := -cpu host -machine malta,accel=kvm +CFLAGS += -EB +else +QEMU_MACHINE := -cpu MIPS64R2-generic -machine malta -smp 1 +CFLAGS += -march=mips64r2 -EB +endif +else ifeq ($(ARCH),mips64el) +QEMU_ARCH := mips64el +KERNEL_ARCH := mips +KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/vmlinux +ifeq ($(HOST_ARCH),$(ARCH)) +QEMU_MACHINE := -cpu host -machine malta,accel=kvm +CFLAGS += -EL +else +QEMU_MACHINE := -cpu MIPS64R2-generic -machine malta -smp 1 +CFLAGS += -march=mips64r2 -EL +endif +else ifeq ($(ARCH),mips) +QEMU_ARCH := mips +KERNEL_ARCH := mips +KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/vmlinux +ifeq ($(HOST_ARCH),$(ARCH)) +QEMU_MACHINE := -cpu host -machine malta,accel=kvm +CFLAGS += -EB +else +QEMU_MACHINE := -cpu 24Kf -machine malta -smp 1 +CFLAGS += -march=mips32r2 -EB +endif +else ifeq ($(ARCH),mipsel) +QEMU_ARCH := mipsel +KERNEL_ARCH := mips +KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/vmlinux +ifeq ($(HOST_ARCH),$(ARCH)) +QEMU_MACHINE := -cpu host -machine malta,accel=kvm +CFLAGS += -EL +else +QEMU_MACHINE := -cpu 24Kf -machine malta -smp 1 +CFLAGS += -march=mips32r2 -EL +endif +else ifeq ($(ARCH),powerpc64le) +QEMU_ARCH := ppc64 +KERNEL_ARCH := powerpc +KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/vmlinux +ifeq ($(HOST_ARCH),$(ARCH)) +QEMU_MACHINE := -cpu host,accel=kvm -machine pseries +else +QEMU_MACHINE := -machine pseries +endif +CFLAGS += -mcpu=powerpc64le -mlong-double-64 +else ifeq ($(ARCH),powerpc) +QEMU_ARCH := ppc +KERNEL_ARCH := powerpc +KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/arch/powerpc/boot/uImage +ifeq ($(HOST_ARCH),$(ARCH)) +QEMU_MACHINE := -cpu host,accel=kvm -machine ppce500 +else +QEMU_MACHINE := -machine ppce500 +endif +CFLAGS += -mcpu=powerpc -mlong-double-64 -msecure-plt +else ifeq ($(ARCH),m68k) +QEMU_ARCH := m68k +KERNEL_ARCH := m68k +KERNEL_BZIMAGE := $(KERNEL_BUILD_PATH)/vmlinux +KERNEL_CMDLINE := $(shell sed -n 's/CONFIG_CMDLINE=\(.*\)/\1/p' arch/m68k.config) +ifeq ($(HOST_ARCH),$(ARCH)) +QEMU_MACHINE := -cpu host,accel=kvm -machine q800 -smp 1 -append $(KERNEL_CMDLINE) +else +QEMU_MACHINE := -machine q800 -smp 1 -append $(KERNEL_CMDLINE) +endif +else +$(error I only build: x86_64, i686, arm, armeb, aarch64, aarch64_be, mips, mipsel, mips64, mips64el, powerpc64le, powerpc, m68k) +endif + +REAL_CC := $(CBUILD)-gcc +MUSL_CC := $(BUILD_PATH)/musl-gcc +export CC := $(MUSL_CC) +USERSPACE_DEPS := $(MUSL_CC) $(BUILD_PATH)/include/.installed $(BUILD_PATH)/include/linux/.installed + +build: $(KERNEL_BZIMAGE) +qemu: $(KERNEL_BZIMAGE) + rm -f $(BUILD_PATH)/result + timeout --foreground 20m qemu-system-$(QEMU_ARCH) \ + -nodefaults \ + -nographic \ + -smp $(NR_CPUS) \ + $(QEMU_MACHINE) \ + -m $$(grep -q CONFIG_DEBUG_KMEMLEAK=y $(KERNEL_BUILD_PATH)/.config && echo 1G || echo 256M) \ + -serial stdio \ + -serial file:$(BUILD_PATH)/result \ + -no-reboot \ + -monitor none \ + -kernel $< + grep -Fq success $(BUILD_PATH)/result + +$(BUILD_PATH)/init-cpio-spec.txt: + mkdir -p $(BUILD_PATH) + echo "file /init $(BUILD_PATH)/init 755 0 0" > $@ + echo "file /init.sh $(PWD)/../netns.sh 755 0 0" >> $@ + echo "dir /dev 755 0 0" >> $@ + echo "nod /dev/console 644 0 0 c 5 1" >> $@ + echo "dir /bin 755 0 0" >> $@ + echo "file /bin/iperf3 $(IPERF_PATH)/src/iperf3 755 0 0" >> $@ + echo "file /bin/wg $(WIREGUARD_TOOLS_PATH)/src/wg 755 0 0" >> $@ + echo "file /bin/bash $(BASH_PATH)/bash 755 0 0" >> $@ + echo "file /bin/ip $(IPROUTE2_PATH)/ip/ip 755 0 0" >> $@ + echo "file /bin/ss $(IPROUTE2_PATH)/misc/ss 755 0 0" >> $@ + echo "file /bin/ping $(IPUTILS_PATH)/ping 755 0 0" >> $@ + echo "file /bin/ncat $(NMAP_PATH)/ncat/ncat 755 0 0" >> $@ + echo "file /bin/xtables-legacy-multi $(IPTABLES_PATH)/iptables/xtables-legacy-multi 755 0 0" >> $@ + echo "slink /bin/iptables xtables-legacy-multi 777 0 0" >> $@ + echo "slink /bin/ping6 ping 777 0 0" >> $@ + echo "dir /lib 755 0 0" >> $@ + echo "file /lib/libc.so $(MUSL_PATH)/lib/libc.so 755 0 0" >> $@ + echo "slink /lib/ld-linux.so.1 libc.so 777 0 0" >> $@ + +$(KERNEL_BUILD_PATH)/.config: kernel.config arch/$(ARCH).config + mkdir -p $(KERNEL_BUILD_PATH) + cp kernel.config $(KERNEL_BUILD_PATH)/minimal.config + printf 'CONFIG_NR_CPUS=$(NR_CPUS)\nCONFIG_INITRAMFS_SOURCE="$(BUILD_PATH)/init-cpio-spec.txt"\n' >> $(KERNEL_BUILD_PATH)/minimal.config + cat arch/$(ARCH).config >> $(KERNEL_BUILD_PATH)/minimal.config + $(MAKE) -C $(KERNEL_PATH) O=$(KERNEL_BUILD_PATH) ARCH=$(KERNEL_ARCH) allnoconfig + cd $(KERNEL_BUILD_PATH) && ARCH=$(KERNEL_ARCH) $(KERNEL_PATH)/scripts/kconfig/merge_config.sh -n $(KERNEL_BUILD_PATH)/.config $(KERNEL_BUILD_PATH)/minimal.config + $(if $(findstring yes,$(DEBUG_KERNEL)),cp debug.config $(KERNEL_BUILD_PATH) && cd $(KERNEL_BUILD_PATH) && ARCH=$(KERNEL_ARCH) $(KERNEL_PATH)/scripts/kconfig/merge_config.sh -n $(KERNEL_BUILD_PATH)/.config debug.config,) + +$(KERNEL_BZIMAGE): $(KERNEL_BUILD_PATH)/.config $(BUILD_PATH)/init-cpio-spec.txt $(MUSL_PATH)/lib/libc.so $(IPERF_PATH)/src/iperf3 $(IPUTILS_PATH)/ping $(BASH_PATH)/bash $(IPROUTE2_PATH)/misc/ss $(IPROUTE2_PATH)/ip/ip $(IPTABLES_PATH)/iptables/xtables-legacy-multi $(NMAP_PATH)/ncat/ncat $(WIREGUARD_TOOLS_PATH)/src/wg $(BUILD_PATH)/init ../netns.sh $(WIREGUARD_SOURCES) + $(MAKE) -C $(KERNEL_PATH) O=$(KERNEL_BUILD_PATH) ARCH=$(KERNEL_ARCH) CROSS_COMPILE=$(CROSS_COMPILE) + +$(BUILD_PATH)/include/linux/.installed: | $(KERNEL_BUILD_PATH)/.config + $(MAKE) -C $(KERNEL_PATH) O=$(KERNEL_BUILD_PATH) INSTALL_HDR_PATH=$(BUILD_PATH) ARCH=$(KERNEL_ARCH) CROSS_COMPILE=$(CROSS_COMPILE) headers_install + touch $@ + +$(MUSL_PATH)/lib/libc.so: $(MUSL_TAR) + mkdir -p $(BUILD_PATH) + flock -s $<.lock tar -C $(BUILD_PATH) -xf $< + cd $(MUSL_PATH) && CC=$(REAL_CC) ./configure --prefix=/ --disable-static --build=$(CBUILD) + $(MAKE) -C $(MUSL_PATH) + $(STRIP) -s $@ + +$(BUILD_PATH)/include/.installed: $(MUSL_PATH)/lib/libc.so + $(MAKE) -C $(MUSL_PATH) DESTDIR=$(BUILD_PATH) install-headers + touch $@ + +$(MUSL_CC): $(MUSL_PATH)/lib/libc.so + sh $(MUSL_PATH)/tools/musl-gcc.specs.sh $(BUILD_PATH)/include $(MUSL_PATH)/lib /lib/ld-linux.so.1 > $(BUILD_PATH)/musl-gcc.specs + printf '#!/bin/sh\nexec "$(REAL_CC)" --specs="$(BUILD_PATH)/musl-gcc.specs" "$$@"\n' > $(BUILD_PATH)/musl-gcc + chmod +x $(BUILD_PATH)/musl-gcc + +$(IPERF_PATH)/.installed: $(IPERF_TAR) + mkdir -p $(BUILD_PATH) + flock -s $<.lock tar -C $(BUILD_PATH) -xf $< + sed -i '1s/^/#include /' $(IPERF_PATH)/src/cjson.h $(IPERF_PATH)/src/timer.h + sed -i -r 's/-p?g//g' $(IPERF_PATH)/src/Makefile* + touch $@ + +$(IPERF_PATH)/src/iperf3: | $(IPERF_PATH)/.installed $(USERSPACE_DEPS) + cd $(IPERF_PATH) && CFLAGS="$(CFLAGS) -D_GNU_SOURCE" ./configure --prefix=/ $(CROSS_COMPILE_FLAG) --enable-static --disable-shared --with-openssl=no + $(MAKE) -C $(IPERF_PATH) + $(STRIP) -s $@ + +$(WIREGUARD_TOOLS_PATH)/.installed: $(WIREGUARD_TOOLS_TAR) + mkdir -p $(BUILD_PATH) + flock -s $<.lock tar -C $(BUILD_PATH) -xf $< + touch $@ + +$(WIREGUARD_TOOLS_PATH)/src/wg: | $(WIREGUARD_TOOLS_PATH)/.installed $(USERSPACE_DEPS) + $(MAKE) -C $(WIREGUARD_TOOLS_PATH)/src wg + $(STRIP) -s $@ + +$(BUILD_PATH)/init: init.c | $(USERSPACE_DEPS) + mkdir -p $(BUILD_PATH) + $(MUSL_CC) -o $@ $(CFLAGS) $(LDFLAGS) -std=gnu11 $< + $(STRIP) -s $@ + +$(IPUTILS_PATH)/.installed: $(IPUTILS_TAR) + mkdir -p $(BUILD_PATH) + flock -s $<.lock tar -C $(BUILD_PATH) -xf $< + touch $@ + +$(IPUTILS_PATH)/ping: | $(IPUTILS_PATH)/.installed $(USERSPACE_DEPS) + sed -i /atexit/d $(IPUTILS_PATH)/ping.c + cd $(IPUTILS_PATH) && $(CC) $(CFLAGS) -std=c99 -o $@ ping.c ping_common.c ping6_common.c iputils_common.c -D_GNU_SOURCE -D'IPUTILS_VERSION(f)=f' -lresolv $(LDFLAGS) + $(STRIP) -s $@ + +$(BASH_PATH)/.installed: $(BASH_TAR) + mkdir -p $(BUILD_PATH) + flock -s $<.lock tar -C $(BUILD_PATH) -xf $< + touch $@ + +$(BASH_PATH)/bash: | $(BASH_PATH)/.installed $(USERSPACE_DEPS) + cd $(BASH_PATH) && ./configure --prefix=/ $(CROSS_COMPILE_FLAG) --without-bash-malloc --disable-debugger --disable-help-builtin --disable-history --disable-multibyte --disable-progcomp --disable-readline --disable-mem-scramble + $(MAKE) -C $(BASH_PATH) + $(STRIP) -s $@ + +$(IPROUTE2_PATH)/.installed: $(IPROUTE2_TAR) + mkdir -p $(BUILD_PATH) + flock -s $<.lock tar -C $(BUILD_PATH) -xf $< + printf 'CC:=$(CC)\nPKG_CONFIG:=pkg-config\nTC_CONFIG_XT:=n\nTC_CONFIG_ATM:=n\nTC_CONFIG_IPSET:=n\nIP_CONFIG_SETNS:=y\nHAVE_ELF:=n\nHAVE_MNL:=n\nHAVE_BERKELEY_DB:=n\nHAVE_LATEX:=n\nHAVE_PDFLATEX:=n\nCFLAGS+=-DHAVE_SETNS\n' > $(IPROUTE2_PATH)/config.mk + printf 'lib: snapshot\n\t$$(MAKE) -C lib\nip/ip: lib\n\t$$(MAKE) -C ip ip\nmisc/ss: lib\n\t$$(MAKE) -C misc ss\n' >> $(IPROUTE2_PATH)/Makefile + touch $@ + +$(IPROUTE2_PATH)/ip/ip: | $(IPROUTE2_PATH)/.installed $(USERSPACE_DEPS) + $(MAKE) -C $(IPROUTE2_PATH) PREFIX=/ ip/ip + $(STRIP) -s $@ + +$(IPROUTE2_PATH)/misc/ss: | $(IPROUTE2_PATH)/.installed $(USERSPACE_DEPS) + $(MAKE) -C $(IPROUTE2_PATH) PREFIX=/ misc/ss + $(STRIP) -s $@ + +$(IPTABLES_PATH)/.installed: $(IPTABLES_TAR) + mkdir -p $(BUILD_PATH) + flock -s $<.lock tar -C $(BUILD_PATH) -xf $< + sed -i -e "/nfnetlink=[01]/s:=[01]:=0:" -e "/nfconntrack=[01]/s:=[01]:=0:" $(IPTABLES_PATH)/configure + touch $@ + +$(IPTABLES_PATH)/iptables/xtables-legacy-multi: | $(IPTABLES_PATH)/.installed $(USERSPACE_DEPS) + cd $(IPTABLES_PATH) && ./configure --prefix=/ $(CROSS_COMPILE_FLAG) --enable-static --disable-shared --disable-nftables --disable-bpf-compiler --disable-nfsynproxy --disable-libipq --disable-connlabel --with-kernel=$(BUILD_PATH)/include + $(MAKE) -C $(IPTABLES_PATH) + $(STRIP) -s $@ + +$(NMAP_PATH)/.installed: $(NMAP_TAR) + mkdir -p $(BUILD_PATH) + flock -s $<.lock tar -C $(BUILD_PATH) -xf $< + touch $@ + +$(NMAP_PATH)/ncat/ncat: | $(NMAP_PATH)/.installed $(USERSPACE_DEPS) + cd $(NMAP_PATH) && ./configure --prefix=/ $(CROSS_COMPILE_FLAG) --enable-static --disable-shared --without-ndiff --without-zenmap --without-nping --with-libpcap=included --with-libpcre=included --with-libdnet=included --without-liblua --with-liblinear=included --without-nmap-update --without-openssl --with-pcap=linux --without-libssh + $(MAKE) -C $(NMAP_PATH)/libpcap + $(MAKE) -C $(NMAP_PATH)/ncat + $(STRIP) -s $@ + +clean: + rm -rf $(BUILD_PATH) + +distclean: clean + rm -rf $(DISTFILES_PATH) + +menuconfig: $(KERNEL_BUILD_PATH)/.config + $(MAKE) -C $(KERNEL_PATH) O=$(KERNEL_BUILD_PATH) ARCH=$(KERNEL_ARCH) CROSS_COMPILE=$(CROSS_COMPILE) menuconfig + +.PHONY: qemu build clean distclean menuconfig +.DELETE_ON_ERROR: diff --git a/tools/testing/selftests/wireguard/qemu/arch/aarch64.config b/tools/testing/selftests/wireguard/qemu/arch/aarch64.config new file mode 100644 index 0000000000000..3d063bb247bbe --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/arch/aarch64.config @@ -0,0 +1,5 @@ +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_CMDLINE_BOOL=y +CONFIG_CMDLINE="console=ttyAMA0 wg.success=ttyAMA1" +CONFIG_FRAME_WARN=1280 diff --git a/tools/testing/selftests/wireguard/qemu/arch/aarch64_be.config b/tools/testing/selftests/wireguard/qemu/arch/aarch64_be.config new file mode 100644 index 0000000000000..dbdc7e406a7ba --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/arch/aarch64_be.config @@ -0,0 +1,6 @@ +CONFIG_CPU_BIG_ENDIAN=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_CMDLINE_BOOL=y +CONFIG_CMDLINE="console=ttyAMA0 wg.success=ttyAMA1" +CONFIG_FRAME_WARN=1280 diff --git a/tools/testing/selftests/wireguard/qemu/arch/arm.config b/tools/testing/selftests/wireguard/qemu/arch/arm.config new file mode 100644 index 0000000000000..148f499054182 --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/arch/arm.config @@ -0,0 +1,9 @@ +CONFIG_MMU=y +CONFIG_ARCH_MULTI_V7=y +CONFIG_ARCH_VIRT=y +CONFIG_THUMB2_KERNEL=n +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_CMDLINE_BOOL=y +CONFIG_CMDLINE="console=ttyAMA0 wg.success=ttyAMA1" +CONFIG_FRAME_WARN=1024 diff --git a/tools/testing/selftests/wireguard/qemu/arch/armeb.config b/tools/testing/selftests/wireguard/qemu/arch/armeb.config new file mode 100644 index 0000000000000..bd76b07d00a2c --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/arch/armeb.config @@ -0,0 +1,10 @@ +CONFIG_MMU=y +CONFIG_ARCH_MULTI_V7=y +CONFIG_ARCH_VIRT=y +CONFIG_THUMB2_KERNEL=n +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_CMDLINE_BOOL=y +CONFIG_CMDLINE="console=ttyAMA0 wg.success=ttyAMA1" +CONFIG_CPU_BIG_ENDIAN=y +CONFIG_FRAME_WARN=1024 diff --git a/tools/testing/selftests/wireguard/qemu/arch/i686.config b/tools/testing/selftests/wireguard/qemu/arch/i686.config new file mode 100644 index 0000000000000..a85025d7206ee --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/arch/i686.config @@ -0,0 +1,5 @@ +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_CMDLINE_BOOL=y +CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" +CONFIG_FRAME_WARN=1024 diff --git a/tools/testing/selftests/wireguard/qemu/arch/m68k.config b/tools/testing/selftests/wireguard/qemu/arch/m68k.config new file mode 100644 index 0000000000000..62a15bdb877e7 --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/arch/m68k.config @@ -0,0 +1,9 @@ +CONFIG_MMU=y +CONFIG_M68KCLASSIC=y +CONFIG_M68040=y +CONFIG_MAC=y +CONFIG_SERIAL_PMACZILOG=y +CONFIG_SERIAL_PMACZILOG_TTYS=y +CONFIG_SERIAL_PMACZILOG_CONSOLE=y +CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" +CONFIG_FRAME_WARN=1024 diff --git a/tools/testing/selftests/wireguard/qemu/arch/mips.config b/tools/testing/selftests/wireguard/qemu/arch/mips.config new file mode 100644 index 0000000000000..df71d6b95546f --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/arch/mips.config @@ -0,0 +1,11 @@ +CONFIG_CPU_MIPS32_R2=y +CONFIG_MIPS_MALTA=y +CONFIG_MIPS_CPS=y +CONFIG_MIPS_FP_SUPPORT=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_CMDLINE_BOOL=y +CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" +CONFIG_FRAME_WARN=1024 diff --git a/tools/testing/selftests/wireguard/qemu/arch/mips64.config b/tools/testing/selftests/wireguard/qemu/arch/mips64.config new file mode 100644 index 0000000000000..90c783f725c4d --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/arch/mips64.config @@ -0,0 +1,14 @@ +CONFIG_64BIT=y +CONFIG_CPU_MIPS64_R2=y +CONFIG_MIPS32_N32=y +CONFIG_CPU_HAS_MSA=y +CONFIG_MIPS_MALTA=y +CONFIG_MIPS_CPS=y +CONFIG_MIPS_FP_SUPPORT=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_CMDLINE_BOOL=y +CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" +CONFIG_FRAME_WARN=1280 diff --git a/tools/testing/selftests/wireguard/qemu/arch/mips64el.config b/tools/testing/selftests/wireguard/qemu/arch/mips64el.config new file mode 100644 index 0000000000000..435b0b43e00cb --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/arch/mips64el.config @@ -0,0 +1,15 @@ +CONFIG_64BIT=y +CONFIG_CPU_MIPS64_R2=y +CONFIG_MIPS32_N32=y +CONFIG_CPU_HAS_MSA=y +CONFIG_MIPS_MALTA=y +CONFIG_CPU_LITTLE_ENDIAN=y +CONFIG_MIPS_CPS=y +CONFIG_MIPS_FP_SUPPORT=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_CMDLINE_BOOL=y +CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" +CONFIG_FRAME_WARN=1280 diff --git a/tools/testing/selftests/wireguard/qemu/arch/mipsel.config b/tools/testing/selftests/wireguard/qemu/arch/mipsel.config new file mode 100644 index 0000000000000..62bb50c4a85fc --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/arch/mipsel.config @@ -0,0 +1,12 @@ +CONFIG_CPU_MIPS32_R2=y +CONFIG_MIPS_MALTA=y +CONFIG_CPU_LITTLE_ENDIAN=y +CONFIG_MIPS_CPS=y +CONFIG_MIPS_FP_SUPPORT=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_CMDLINE_BOOL=y +CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" +CONFIG_FRAME_WARN=1024 diff --git a/tools/testing/selftests/wireguard/qemu/arch/powerpc.config b/tools/testing/selftests/wireguard/qemu/arch/powerpc.config new file mode 100644 index 0000000000000..57957093b71b8 --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/arch/powerpc.config @@ -0,0 +1,10 @@ +CONFIG_PPC_QEMU_E500=y +CONFIG_FSL_SOC_BOOKE=y +CONFIG_PPC_85xx=y +CONFIG_PHYS_64BIT=y +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_MATH_EMULATION=y +CONFIG_CMDLINE_BOOL=y +CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" +CONFIG_FRAME_WARN=1024 diff --git a/tools/testing/selftests/wireguard/qemu/arch/powerpc64le.config b/tools/testing/selftests/wireguard/qemu/arch/powerpc64le.config new file mode 100644 index 0000000000000..990c510a9cfa5 --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/arch/powerpc64le.config @@ -0,0 +1,12 @@ +CONFIG_PPC64=y +CONFIG_PPC_PSERIES=y +CONFIG_ALTIVEC=y +CONFIG_VSX=y +CONFIG_PPC_OF_BOOT_TRAMPOLINE=y +CONFIG_PPC_RADIX_MMU=y +CONFIG_HVC_CONSOLE=y +CONFIG_CPU_LITTLE_ENDIAN=y +CONFIG_CMDLINE_BOOL=y +CONFIG_CMDLINE="console=hvc0 wg.success=hvc1" +CONFIG_SECTION_MISMATCH_WARN_ONLY=y +CONFIG_FRAME_WARN=1280 diff --git a/tools/testing/selftests/wireguard/qemu/arch/x86_64.config b/tools/testing/selftests/wireguard/qemu/arch/x86_64.config new file mode 100644 index 0000000000000..00a1ef4869d58 --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/arch/x86_64.config @@ -0,0 +1,5 @@ +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_CMDLINE_BOOL=y +CONFIG_CMDLINE="console=ttyS0 wg.success=ttyS1" +CONFIG_FRAME_WARN=1280 diff --git a/tools/testing/selftests/wireguard/qemu/debug.config b/tools/testing/selftests/wireguard/qemu/debug.config new file mode 100644 index 0000000000000..b9c72706fe4db --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/debug.config @@ -0,0 +1,67 @@ +CONFIG_LOCALVERSION="-debug" +CONFIG_ENABLE_WARN_DEPRECATED=y +CONFIG_ENABLE_MUST_CHECK=y +CONFIG_FRAME_POINTER=y +CONFIG_STACK_VALIDATION=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_INFO_DWARF4=y +CONFIG_PAGE_EXTENSION=y +CONFIG_PAGE_POISONING=y +CONFIG_DEBUG_OBJECTS=y +CONFIG_DEBUG_OBJECTS_FREE=y +CONFIG_DEBUG_OBJECTS_TIMERS=y +CONFIG_DEBUG_OBJECTS_WORK=y +CONFIG_DEBUG_OBJECTS_RCU_HEAD=y +CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER=y +CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT=1 +CONFIG_SLUB_DEBUG_ON=y +CONFIG_DEBUG_VM=y +CONFIG_DEBUG_MEMORY_INIT=y +CONFIG_HAVE_DEBUG_STACKOVERFLOW=y +CONFIG_DEBUG_STACKOVERFLOW=y +CONFIG_HAVE_ARCH_KMEMCHECK=y +CONFIG_HAVE_ARCH_KASAN=y +CONFIG_KASAN=y +CONFIG_KASAN_INLINE=y +CONFIG_UBSAN=y +CONFIG_UBSAN_SANITIZE_ALL=y +CONFIG_UBSAN_NO_ALIGNMENT=y +CONFIG_UBSAN_NULL=y +CONFIG_DEBUG_KMEMLEAK=y +CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE=8192 +CONFIG_DEBUG_STACK_USAGE=y +CONFIG_DEBUG_SHIRQ=y +CONFIG_WQ_WATCHDOG=y +CONFIG_SCHED_DEBUG=y +CONFIG_SCHED_INFO=y +CONFIG_SCHEDSTATS=y +CONFIG_SCHED_STACK_END_CHECK=y +CONFIG_DEBUG_TIMEKEEPING=y +CONFIG_TIMER_STATS=y +CONFIG_DEBUG_PREEMPT=y +CONFIG_DEBUG_RT_MUTEXES=y +CONFIG_DEBUG_SPINLOCK=y +CONFIG_DEBUG_MUTEXES=y +CONFIG_DEBUG_LOCK_ALLOC=y +CONFIG_PROVE_LOCKING=y +CONFIG_LOCKDEP=y +CONFIG_DEBUG_ATOMIC_SLEEP=y +CONFIG_TRACE_IRQFLAGS=y +CONFIG_DEBUG_BUGVERBOSE=y +CONFIG_DEBUG_LIST=y +CONFIG_DEBUG_PI_LIST=y +CONFIG_PROVE_RCU=y +CONFIG_SPARSE_RCU_POINTER=y +CONFIG_RCU_CPU_STALL_TIMEOUT=21 +CONFIG_RCU_TRACE=y +CONFIG_RCU_EQS_DEBUG=y +CONFIG_USER_STACKTRACE_SUPPORT=y +CONFIG_DEBUG_SG=y +CONFIG_DEBUG_NOTIFIERS=y +CONFIG_DOUBLEFAULT=y +CONFIG_X86_DEBUG_FPU=y +CONFIG_DEBUG_SECTION_MISMATCH=y +CONFIG_DEBUG_PAGEALLOC=y +CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT=y +CONFIG_DEBUG_WW_MUTEX_SLOWPATH=y diff --git a/tools/testing/selftests/wireguard/qemu/init.c b/tools/testing/selftests/wireguard/qemu/init.c new file mode 100644 index 0000000000000..c9698120ac9d8 --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/init.c @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +__attribute__((noreturn)) static void poweroff(void) +{ + fflush(stdout); + fflush(stderr); + reboot(RB_AUTOBOOT); + sleep(30); + fprintf(stderr, "\x1b[37m\x1b[41m\x1b[1mFailed to power off!!!\x1b[0m\n"); + exit(1); +} + +static void panic(const char *what) +{ + fprintf(stderr, "\n\n\x1b[37m\x1b[41m\x1b[1mSOMETHING WENT HORRIBLY WRONG\x1b[0m\n\n \x1b[31m\x1b[1m%s: %s\x1b[0m\n\n\x1b[37m\x1b[44m\x1b[1mPower off...\x1b[0m\n\n", what, strerror(errno)); + poweroff(); +} + +#define pretty_message(msg) puts("\x1b[32m\x1b[1m" msg "\x1b[0m") + +static void print_banner(void) +{ + struct utsname utsname; + int len; + + if (uname(&utsname) < 0) + panic("uname"); + + len = strlen(" WireGuard Test Suite on ") + strlen(utsname.sysname) + strlen(utsname.release) + strlen(utsname.machine); + printf("\x1b[45m\x1b[33m\x1b[1m%*.s\x1b[0m\n\x1b[45m\x1b[33m\x1b[1m WireGuard Test Suite on %s %s %s \x1b[0m\n\x1b[45m\x1b[33m\x1b[1m%*.s\x1b[0m\n\n", len, "", utsname.sysname, utsname.release, utsname.machine, len, ""); +} + +static void seed_rng(void) +{ + int fd; + struct { + int entropy_count; + int buffer_size; + unsigned char buffer[256]; + } entropy = { + .entropy_count = sizeof(entropy.buffer) * 8, + .buffer_size = sizeof(entropy.buffer), + .buffer = "Adding real entropy is not actually important for these tests. Don't try this at home, kids!" + }; + + if (mknod("/dev/urandom", S_IFCHR | 0644, makedev(1, 9))) + panic("mknod(/dev/urandom)"); + fd = open("/dev/urandom", O_WRONLY); + if (fd < 0) + panic("open(urandom)"); + for (int i = 0; i < 256; ++i) { + if (ioctl(fd, RNDADDENTROPY, &entropy) < 0) + panic("ioctl(urandom)"); + } + close(fd); +} + +static void mount_filesystems(void) +{ + pretty_message("[+] Mounting filesystems..."); + mkdir("/dev", 0755); + mkdir("/proc", 0755); + mkdir("/sys", 0755); + mkdir("/tmp", 0755); + mkdir("/run", 0755); + mkdir("/var", 0755); + if (mount("none", "/dev", "devtmpfs", 0, NULL)) + panic("devtmpfs mount"); + if (mount("none", "/proc", "proc", 0, NULL)) + panic("procfs mount"); + if (mount("none", "/sys", "sysfs", 0, NULL)) + panic("sysfs mount"); + if (mount("none", "/tmp", "tmpfs", 0, NULL)) + panic("tmpfs mount"); + if (mount("none", "/run", "tmpfs", 0, NULL)) + panic("tmpfs mount"); + if (mount("none", "/sys/kernel/debug", "debugfs", 0, NULL)) + ; /* Not a problem if it fails.*/ + if (symlink("/run", "/var/run")) + panic("run symlink"); + if (symlink("/proc/self/fd", "/dev/fd")) + panic("fd symlink"); +} + +static void enable_logging(void) +{ + int fd; + pretty_message("[+] Enabling logging..."); + fd = open("/proc/sys/kernel/printk", O_WRONLY); + if (fd >= 0) { + if (write(fd, "9\n", 2) != 2) + panic("write(printk)"); + close(fd); + } + fd = open("/proc/sys/debug/exception-trace", O_WRONLY); + if (fd >= 0) { + if (write(fd, "1\n", 2) != 2) + panic("write(exception-trace)"); + close(fd); + } + fd = open("/proc/sys/kernel/panic_on_warn", O_WRONLY); + if (fd >= 0) { + if (write(fd, "1\n", 2) != 2) + panic("write(panic_on_warn)"); + close(fd); + } +} + +static void kmod_selftests(void) +{ + FILE *file; + char line[2048], *start, *pass; + bool success = true; + pretty_message("[+] Module self-tests:"); + file = fopen("/proc/kmsg", "r"); + if (!file) + panic("fopen(kmsg)"); + if (fcntl(fileno(file), F_SETFL, O_NONBLOCK) < 0) + panic("fcntl(kmsg, nonblock)"); + while (fgets(line, sizeof(line), file)) { + start = strstr(line, "wireguard: "); + if (!start) + continue; + start += 11; + *strchrnul(start, '\n') = '\0'; + if (strstr(start, "www.wireguard.com")) + break; + pass = strstr(start, ": pass"); + if (!pass || pass[6] != '\0') { + success = false; + printf(" \x1b[31m* %s\x1b[0m\n", start); + } else + printf(" \x1b[32m* %s\x1b[0m\n", start); + } + fclose(file); + if (!success) { + puts("\x1b[31m\x1b[1m[-] Tests failed! \u2639\x1b[0m"); + poweroff(); + } +} + +static void launch_tests(void) +{ + char cmdline[4096], *success_dev; + int status, fd; + pid_t pid; + + pretty_message("[+] Launching tests..."); + pid = fork(); + if (pid == -1) + panic("fork"); + else if (pid == 0) { + execl("/init.sh", "init", NULL); + panic("exec"); + } + if (waitpid(pid, &status, 0) < 0) + panic("waitpid"); + if (WIFEXITED(status) && WEXITSTATUS(status) == 0) { + pretty_message("[+] Tests successful! :-)"); + fd = open("/proc/cmdline", O_RDONLY); + if (fd < 0) + panic("open(/proc/cmdline)"); + if (read(fd, cmdline, sizeof(cmdline) - 1) <= 0) + panic("read(/proc/cmdline)"); + cmdline[sizeof(cmdline) - 1] = '\0'; + for (success_dev = strtok(cmdline, " \n"); success_dev; success_dev = strtok(NULL, " \n")) { + if (strncmp(success_dev, "wg.success=", 11)) + continue; + memcpy(success_dev + 11 - 5, "/dev/", 5); + success_dev += 11 - 5; + break; + } + if (!success_dev || !strlen(success_dev)) + panic("Unable to find success device"); + + fd = open(success_dev, O_WRONLY); + if (fd < 0) + panic("open(success_dev)"); + if (write(fd, "success\n", 8) != 8) + panic("write(success_dev)"); + close(fd); + } else { + const char *why = "unknown cause"; + int what = -1; + + if (WIFEXITED(status)) { + why = "exit code"; + what = WEXITSTATUS(status); + } else if (WIFSIGNALED(status)) { + why = "signal"; + what = WTERMSIG(status); + } + printf("\x1b[31m\x1b[1m[-] Tests failed with %s %d! \u2639\x1b[0m\n", why, what); + } +} + +static void ensure_console(void) +{ + for (unsigned int i = 0; i < 1000; ++i) { + int fd = open("/dev/console", O_RDWR); + if (fd < 0) { + usleep(50000); + continue; + } + dup2(fd, 0); + dup2(fd, 1); + dup2(fd, 2); + close(fd); + if (write(1, "\0\0\0\0\n", 5) == 5) + return; + } + panic("Unable to open console device"); +} + +static void clear_leaks(void) +{ + int fd; + + fd = open("/sys/kernel/debug/kmemleak", O_WRONLY); + if (fd < 0) + return; + pretty_message("[+] Starting memory leak detection..."); + write(fd, "clear\n", 5); + close(fd); +} + +static void check_leaks(void) +{ + int fd; + + fd = open("/sys/kernel/debug/kmemleak", O_WRONLY); + if (fd < 0) + return; + pretty_message("[+] Scanning for memory leaks..."); + sleep(2); /* Wait for any grace periods. */ + write(fd, "scan\n", 5); + close(fd); + + fd = open("/sys/kernel/debug/kmemleak", O_RDONLY); + if (fd < 0) + return; + if (sendfile(1, fd, NULL, 0x7ffff000) > 0) + panic("Memory leaks encountered"); + close(fd); +} + +int main(int argc, char *argv[]) +{ + seed_rng(); + ensure_console(); + print_banner(); + mount_filesystems(); + kmod_selftests(); + enable_logging(); + clear_leaks(); + launch_tests(); + check_leaks(); + poweroff(); + return 1; +} diff --git a/tools/testing/selftests/wireguard/qemu/kernel.config b/tools/testing/selftests/wireguard/qemu/kernel.config new file mode 100644 index 0000000000000..af9323a0b6e09 --- /dev/null +++ b/tools/testing/selftests/wireguard/qemu/kernel.config @@ -0,0 +1,88 @@ +CONFIG_LOCALVERSION="" +CONFIG_NET=y +CONFIG_NETDEVICES=y +CONFIG_NET_CORE=y +CONFIG_NET_IPIP=y +CONFIG_DUMMY=y +CONFIG_VETH=y +CONFIG_MULTIUSER=y +CONFIG_NAMESPACES=y +CONFIG_NET_NS=y +CONFIG_UNIX=y +CONFIG_INET=y +CONFIG_IPV6=y +CONFIG_NETFILTER=y +CONFIG_NETFILTER_ADVANCED=y +CONFIG_NF_CONNTRACK=y +CONFIG_NF_NAT=y +CONFIG_NETFILTER_XTABLES=y +CONFIG_NETFILTER_XT_NAT=y +CONFIG_NETFILTER_XT_MATCH_LENGTH=y +CONFIG_NF_CONNTRACK_IPV4=y +CONFIG_NF_NAT_IPV4=y +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_FILTER=y +CONFIG_IP_NF_NAT=y +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_TTY=y +CONFIG_BINFMT_ELF=y +CONFIG_BINFMT_SCRIPT=y +CONFIG_VDSO=y +CONFIG_VIRTUALIZATION=y +CONFIG_HYPERVISOR_GUEST=y +CONFIG_PARAVIRT=y +CONFIG_KVM_GUEST=y +CONFIG_PARAVIRT_SPINLOCKS=y +CONFIG_PRINTK=y +CONFIG_KALLSYMS=y +CONFIG_BUG=y +CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE=y +CONFIG_JUMP_LABEL=y +CONFIG_EMBEDDED=n +CONFIG_BASE_FULL=y +CONFIG_FUTEX=y +CONFIG_SHMEM=y +CONFIG_SLUB=y +CONFIG_SPARSEMEM_VMEMMAP=y +CONFIG_SMP=y +CONFIG_SCHED_SMT=y +CONFIG_SCHED_MC=y +CONFIG_NUMA=y +CONFIG_PREEMPT=y +CONFIG_NO_HZ=y +CONFIG_NO_HZ_IDLE=y +CONFIG_NO_HZ_FULL=n +CONFIG_HZ_PERIODIC=n +CONFIG_HIGH_RES_TIMERS=y +CONFIG_COMPAT_32BIT_TIME=y +CONFIG_ARCH_RANDOM=y +CONFIG_FILE_LOCKING=y +CONFIG_POSIX_TIMERS=y +CONFIG_DEVTMPFS=y +CONFIG_PROC_FS=y +CONFIG_PROC_SYSCTL=y +CONFIG_SYSFS=y +CONFIG_TMPFS=y +CONFIG_CONSOLE_LOGLEVEL_DEFAULT=15 +CONFIG_PRINTK_TIME=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_LEGACY_VSYSCALL_NONE=y +CONFIG_KERNEL_GZIP=y +CONFIG_PANIC_ON_OOPS=y +CONFIG_BUG_ON_DATA_CORRUPTION=y +CONFIG_LOCKUP_DETECTOR=y +CONFIG_SOFTLOCKUP_DETECTOR=y +CONFIG_HARDLOCKUP_DETECTOR=y +CONFIG_WQ_WATCHDOG=y +CONFIG_DETECT_HUNG_TASK=y +CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y +CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y +CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y +CONFIG_PANIC_TIMEOUT=-1 +CONFIG_STACKTRACE=y +CONFIG_EARLY_PRINTK=y +CONFIG_GDB_SCRIPTS=y +CONFIG_WIREGUARD=y +CONFIG_WIREGUARD_DEBUG=y