Re: [PATCH v2 16/19] Add elliptic curve cryptography (ECC) helper functions

[Ahmad Fatoum <a.fatoum@pengutronix.de>]

Hello Sascha,

On 01.08.24 07:57, Sascha Hauer wrote:
> This ports the functions needed for supporting elliptic curve cryptography
> (ECC) from the Kernel. The code is taken from Linux-6.10 and mostly
> unchanged.
> 
> Signed-off-by: Sascha Hauer <s.hauer@pengutronix.de>

Acked-by: Ahmad Fatoum <a.fatoum@pengutronix.de>

> ---
>  crypto/Kconfig                |    3 +
>  crypto/Makefile               |    1 +
>  crypto/ecc.c                  | 1661 +++++++++++++++++++++++++++++++++
>  crypto/ecc_curve_defs.h       |  155 +++
>  include/crypto/ecc_curve.h    |   62 ++
>  include/crypto/ecdh.h         |   83 ++
>  include/crypto/internal/ecc.h |  278 ++++++
>  7 files changed, 2243 insertions(+)
>  create mode 100644 crypto/ecc.c
>  create mode 100644 crypto/ecc_curve_defs.h
>  create mode 100644 include/crypto/ecc_curve.h
>  create mode 100644 include/crypto/ecdh.h
>  create mode 100644 include/crypto/internal/ecc.h
> 
> diff --git a/crypto/Kconfig b/crypto/Kconfig
> index 78b499f646..e953ef5e15 100644
> --- a/crypto/Kconfig
> +++ b/crypto/Kconfig
> @@ -153,4 +153,7 @@ config JWT
>  	select BASE64
>  	select CRYPTO_RSA
>  
> +config CRYPTO_ECC
> +	bool
> +
>  endmenu
> diff --git a/crypto/Makefile b/crypto/Makefile
> index e84360a8c7..83c05761de 100644
> --- a/crypto/Makefile
> +++ b/crypto/Makefile
> @@ -18,6 +18,7 @@ obj-y	+= memneq.o
>  obj-$(CONFIG_CRYPTO_PBKDF2)	+= pbkdf2.o
>  obj-$(CONFIG_CRYPTO_RSA)	+= rsa.o
>  obj-$(CONFIG_CRYPTO_KEYSTORE)	+= keystore.o
> +obj-$(CONFIG_CRYPTO_ECC)	+= ecc.o
>  
>  obj-$(CONFIG_JWT)		+= jwt.o
>  
> diff --git a/crypto/ecc.c b/crypto/ecc.c
> new file mode 100644
> index 0000000000..a0ab962262
> --- /dev/null
> +++ b/crypto/ecc.c
> @@ -0,0 +1,1661 @@
> +/*
> + * Copyright (c) 2013, 2014 Kenneth MacKay. All rights reserved.
> + * Copyright (c) 2019 Vitaly Chikunov <vt@altlinux.org>
> + *
> + * Redistribution and use in source and binary forms, with or without
> + * modification, are permitted provided that the following conditions are
> + * met:
> + *  * Redistributions of source code must retain the above copyright
> + *   notice, this list of conditions and the following disclaimer.
> + *  * Redistributions in binary form must reproduce the above copyright
> + *    notice, this list of conditions and the following disclaimer in the
> + *    documentation and/or other materials provided with the distribution.
> + *
> + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +
> +#include <common.h>
> +#include <stdlib.h>
> +#include <crypto/ecc_curve.h>
> +#include <crypto/ecdh.h>
> +#include <crypto/internal/ecc.h>
> +#include <asm/unaligned.h>
> +
> +#include "ecc_curve_defs.h"
> +
> +typedef struct {
> +	u64 m_low;
> +	u64 m_high;
> +} uint128_t;
> +
> +/* Returns curv25519 curve param */
> +const struct ecc_curve *ecc_get_curve25519(void)
> +{
> +	return &ecc_25519;
> +}
> +EXPORT_SYMBOL(ecc_get_curve25519);
> +
> +const struct ecc_curve *ecc_get_curve(unsigned int curve_id)
> +{
> +	switch (curve_id) {
> +	case ECC_CURVE_NIST_P192:
> +		return &nist_p192;
> +	case ECC_CURVE_NIST_P256:
> +		return &nist_p256;
> +	case ECC_CURVE_NIST_P384:
> +		return &nist_p384;
> +	case ECC_CURVE_NIST_P521:
> +		return &nist_p521;
> +	default:
> +		return NULL;
> +	}
> +}
> +EXPORT_SYMBOL(ecc_get_curve);
> +
> +void ecc_digits_from_bytes(const u8 *in, unsigned int nbytes,
> +			   u64 *out, unsigned int ndigits)
> +{
> +	int diff = ndigits - DIV_ROUND_UP(nbytes, sizeof(u64));
> +	unsigned int o = nbytes & 7;
> +	__be64 msd = 0;
> +
> +	/* diff > 0: not enough input bytes: set most significant digits to 0 */
> +	if (diff > 0) {
> +		ndigits -= diff;
> +		memset(&out[ndigits - 1], 0, diff * sizeof(u64));
> +	}
> +
> +	if (o) {
> +		memcpy((u8 *)&msd + sizeof(msd) - o, in, o);
> +		out[--ndigits] = be64_to_cpu(msd);
> +		in += o;
> +	}
> +	ecc_swap_digits(in, out, ndigits);
> +}
> +EXPORT_SYMBOL(ecc_digits_from_bytes);
> +
> +static u64 *ecc_alloc_digits_space(unsigned int ndigits)
> +{
> +	size_t len = ndigits * sizeof(u64);
> +
> +	if (!len)
> +		return NULL;
> +
> +	return kmalloc(len, GFP_KERNEL);
> +}
> +
> +static void ecc_free_digits_space(u64 *space)
> +{
> +	kfree_sensitive(space);
> +}
> +
> +struct ecc_point *ecc_alloc_point(unsigned int ndigits)
> +{
> +	struct ecc_point *p = kmalloc(sizeof(*p), GFP_KERNEL);
> +
> +	if (!p)
> +		return NULL;
> +
> +	p->x = ecc_alloc_digits_space(ndigits);
> +	if (!p->x)
> +		goto err_alloc_x;
> +
> +	p->y = ecc_alloc_digits_space(ndigits);
> +	if (!p->y)
> +		goto err_alloc_y;
> +
> +	p->ndigits = ndigits;
> +
> +	return p;
> +
> +err_alloc_y:
> +	ecc_free_digits_space(p->x);
> +err_alloc_x:
> +	kfree(p);
> +	return NULL;
> +}
> +EXPORT_SYMBOL(ecc_alloc_point);
> +
> +void ecc_free_point(struct ecc_point *p)
> +{
> +	if (!p)
> +		return;
> +
> +	kfree_sensitive(p->x);
> +	kfree_sensitive(p->y);
> +	kfree_sensitive(p);
> +}
> +EXPORT_SYMBOL(ecc_free_point);
> +
> +static void vli_clear(u64 *vli, unsigned int ndigits)
> +{
> +	int i;
> +
> +	for (i = 0; i < ndigits; i++)
> +		vli[i] = 0;
> +}
> +
> +/* Returns true if vli == 0, false otherwise. */
> +bool vli_is_zero(const u64 *vli, unsigned int ndigits)
> +{
> +	int i;
> +
> +	for (i = 0; i < ndigits; i++) {
> +		if (vli[i])
> +			return false;
> +	}
> +
> +	return true;
> +}
> +EXPORT_SYMBOL(vli_is_zero);
> +
> +/* Returns nonzero if bit of vli is set. */
> +static u64 vli_test_bit(const u64 *vli, unsigned int bit)
> +{
> +	return (vli[bit / 64] & ((u64)1 << (bit % 64)));
> +}
> +
> +static bool vli_is_negative(const u64 *vli, unsigned int ndigits)
> +{
> +	return vli_test_bit(vli, ndigits * 64 - 1);
> +}
> +
> +/* Counts the number of 64-bit "digits" in vli. */
> +static unsigned int vli_num_digits(const u64 *vli, unsigned int ndigits)
> +{
> +	int i;
> +
> +	/* Search from the end until we find a non-zero digit.
> +	 * We do it in reverse because we expect that most digits will
> +	 * be nonzero.
> +	 */
> +	for (i = ndigits - 1; i >= 0 && vli[i] == 0; i--);
> +
> +	return (i + 1);
> +}
> +
> +/* Counts the number of bits required for vli. */
> +unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits)
> +{
> +	unsigned int i, num_digits;
> +	u64 digit;
> +
> +	num_digits = vli_num_digits(vli, ndigits);
> +	if (num_digits == 0)
> +		return 0;
> +
> +	digit = vli[num_digits - 1];
> +	for (i = 0; digit; i++)
> +		digit >>= 1;
> +
> +	return ((num_digits - 1) * 64 + i);
> +}
> +EXPORT_SYMBOL(vli_num_bits);
> +
> +/* Set dest from unaligned bit string src. */
> +void vli_from_be64(u64 *dest, const void *src, unsigned int ndigits)
> +{
> +	int i;
> +	const u64 *from = src;
> +
> +	for (i = 0; i < ndigits; i++)
> +		dest[i] = get_unaligned_be64(&from[ndigits - 1 - i]);
> +}
> +EXPORT_SYMBOL(vli_from_be64);
> +
> +void vli_from_le64(u64 *dest, const void *src, unsigned int ndigits)
> +{
> +	int i;
> +	const u64 *from = src;
> +
> +	for (i = 0; i < ndigits; i++)
> +		dest[i] = get_unaligned_le64(&from[i]);
> +}
> +EXPORT_SYMBOL(vli_from_le64);
> +
> +/* Sets dest = src. */
> +static void vli_set(u64 *dest, const u64 *src, unsigned int ndigits)
> +{
> +	int i;
> +
> +	for (i = 0; i < ndigits; i++)
> +		dest[i] = src[i];
> +}
> +
> +/* Returns sign of left - right. */
> +int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits)
> +{
> +	int i;
> +
> +	for (i = ndigits - 1; i >= 0; i--) {
> +		if (left[i] > right[i])
> +			return 1;
> +		else if (left[i] < right[i])
> +			return -1;
> +	}
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL(vli_cmp);
> +
> +/* Computes result = in << c, returning carry. Can modify in place
> + * (if result == in). 0 < shift < 64.
> + */
> +static u64 vli_lshift(u64 *result, const u64 *in, unsigned int shift,
> +		      unsigned int ndigits)
> +{
> +	u64 carry = 0;
> +	int i;
> +
> +	for (i = 0; i < ndigits; i++) {
> +		u64 temp = in[i];
> +
> +		result[i] = (temp << shift) | carry;
> +		carry = temp >> (64 - shift);
> +	}
> +
> +	return carry;
> +}
> +
> +/* Computes vli = vli >> 1. */
> +static void vli_rshift1(u64 *vli, unsigned int ndigits)
> +{
> +	u64 *end = vli;
> +	u64 carry = 0;
> +
> +	vli += ndigits;
> +
> +	while (vli-- > end) {
> +		u64 temp = *vli;
> +		*vli = (temp >> 1) | carry;
> +		carry = temp << 63;
> +	}
> +}
> +
> +/* Computes result = left + right, returning carry. Can modify in place. */
> +static u64 vli_add(u64 *result, const u64 *left, const u64 *right,
> +		   unsigned int ndigits)
> +{
> +	u64 carry = 0;
> +	int i;
> +
> +	for (i = 0; i < ndigits; i++) {
> +		u64 sum;
> +
> +		sum = left[i] + right[i] + carry;
> +		if (sum != left[i])
> +			carry = (sum < left[i]);
> +
> +		result[i] = sum;
> +	}
> +
> +	return carry;
> +}
> +
> +/* Computes result = left + right, returning carry. Can modify in place. */
> +static u64 vli_uadd(u64 *result, const u64 *left, u64 right,
> +		    unsigned int ndigits)
> +{
> +	u64 carry = right;
> +	int i;
> +
> +	for (i = 0; i < ndigits; i++) {
> +		u64 sum;
> +
> +		sum = left[i] + carry;
> +		if (sum != left[i])
> +			carry = (sum < left[i]);
> +		else
> +			carry = !!carry;
> +
> +		result[i] = sum;
> +	}
> +
> +	return carry;
> +}
> +
> +/* Computes result = left - right, returning borrow. Can modify in place. */
> +u64 vli_sub(u64 *result, const u64 *left, const u64 *right,
> +		   unsigned int ndigits)
> +{
> +	u64 borrow = 0;
> +	int i;
> +
> +	for (i = 0; i < ndigits; i++) {
> +		u64 diff;
> +
> +		diff = left[i] - right[i] - borrow;
> +		if (diff != left[i])
> +			borrow = (diff > left[i]);
> +
> +		result[i] = diff;
> +	}
> +
> +	return borrow;
> +}
> +EXPORT_SYMBOL(vli_sub);
> +
> +/* Computes result = left - right, returning borrow. Can modify in place. */
> +static u64 vli_usub(u64 *result, const u64 *left, u64 right,
> +	     unsigned int ndigits)
> +{
> +	u64 borrow = right;
> +	int i;
> +
> +	for (i = 0; i < ndigits; i++) {
> +		u64 diff;
> +
> +		diff = left[i] - borrow;
> +		if (diff != left[i])
> +			borrow = (diff > left[i]);
> +
> +		result[i] = diff;
> +	}
> +
> +	return borrow;
> +}
> +
> +static uint128_t mul_64_64(u64 left, u64 right)
> +{
> +	uint128_t result;
> +#if defined(CONFIG_ARCH_SUPPORTS_INT128)
> +	unsigned __int128 m = (unsigned __int128)left * right;
> +
> +	result.m_low  = m;
> +	result.m_high = m >> 64;
> +#else
> +	u64 a0 = left & 0xffffffffull;
> +	u64 a1 = left >> 32;
> +	u64 b0 = right & 0xffffffffull;
> +	u64 b1 = right >> 32;
> +	u64 m0 = a0 * b0;
> +	u64 m1 = a0 * b1;
> +	u64 m2 = a1 * b0;
> +	u64 m3 = a1 * b1;
> +
> +	m2 += (m0 >> 32);
> +	m2 += m1;
> +
> +	/* Overflow */
> +	if (m2 < m1)
> +		m3 += 0x100000000ull;
> +
> +	result.m_low = (m0 & 0xffffffffull) | (m2 << 32);
> +	result.m_high = m3 + (m2 >> 32);
> +#endif
> +	return result;
> +}
> +
> +static uint128_t add_128_128(uint128_t a, uint128_t b)
> +{
> +	uint128_t result;
> +
> +	result.m_low = a.m_low + b.m_low;
> +	result.m_high = a.m_high + b.m_high + (result.m_low < a.m_low);
> +
> +	return result;
> +}
> +
> +static void vli_mult(u64 *result, const u64 *left, const u64 *right,
> +		     unsigned int ndigits)
> +{
> +	uint128_t r01 = { 0, 0 };
> +	u64 r2 = 0;
> +	unsigned int i, k;
> +
> +	/* Compute each digit of result in sequence, maintaining the
> +	 * carries.
> +	 */
> +	for (k = 0; k < ndigits * 2 - 1; k++) {
> +		unsigned int min;
> +
> +		if (k < ndigits)
> +			min = 0;
> +		else
> +			min = (k + 1) - ndigits;
> +
> +		for (i = min; i <= k && i < ndigits; i++) {
> +			uint128_t product;
> +
> +			product = mul_64_64(left[i], right[k - i]);
> +
> +			r01 = add_128_128(r01, product);
> +			r2 += (r01.m_high < product.m_high);
> +		}
> +
> +		result[k] = r01.m_low;
> +		r01.m_low = r01.m_high;
> +		r01.m_high = r2;
> +		r2 = 0;
> +	}
> +
> +	result[ndigits * 2 - 1] = r01.m_low;
> +}
> +
> +/* Compute product = left * right, for a small right value. */
> +static void vli_umult(u64 *result, const u64 *left, u32 right,
> +		      unsigned int ndigits)
> +{
> +	uint128_t r01 = { 0 };
> +	unsigned int k;
> +
> +	for (k = 0; k < ndigits; k++) {
> +		uint128_t product;
> +
> +		product = mul_64_64(left[k], right);
> +		r01 = add_128_128(r01, product);
> +		/* no carry */
> +		result[k] = r01.m_low;
> +		r01.m_low = r01.m_high;
> +		r01.m_high = 0;
> +	}
> +	result[k] = r01.m_low;
> +	for (++k; k < ndigits * 2; k++)
> +		result[k] = 0;
> +}
> +
> +static void vli_square(u64 *result, const u64 *left, unsigned int ndigits)
> +{
> +	uint128_t r01 = { 0, 0 };
> +	u64 r2 = 0;
> +	int i, k;
> +
> +	for (k = 0; k < ndigits * 2 - 1; k++) {
> +		unsigned int min;
> +
> +		if (k < ndigits)
> +			min = 0;
> +		else
> +			min = (k + 1) - ndigits;
> +
> +		for (i = min; i <= k && i <= k - i; i++) {
> +			uint128_t product;
> +
> +			product = mul_64_64(left[i], left[k - i]);
> +
> +			if (i < k - i) {
> +				r2 += product.m_high >> 63;
> +				product.m_high = (product.m_high << 1) |
> +						 (product.m_low >> 63);
> +				product.m_low <<= 1;
> +			}
> +
> +			r01 = add_128_128(r01, product);
> +			r2 += (r01.m_high < product.m_high);
> +		}
> +
> +		result[k] = r01.m_low;
> +		r01.m_low = r01.m_high;
> +		r01.m_high = r2;
> +		r2 = 0;
> +	}
> +
> +	result[ndigits * 2 - 1] = r01.m_low;
> +}
> +
> +/* Computes result = (left + right) % mod.
> + * Assumes that left < mod and right < mod, result != mod.
> + */
> +static void vli_mod_add(u64 *result, const u64 *left, const u64 *right,
> +			const u64 *mod, unsigned int ndigits)
> +{
> +	u64 carry;
> +
> +	carry = vli_add(result, left, right, ndigits);
> +
> +	/* result > mod (result = mod + remainder), so subtract mod to
> +	 * get remainder.
> +	 */
> +	if (carry || vli_cmp(result, mod, ndigits) >= 0)
> +		vli_sub(result, result, mod, ndigits);
> +}
> +
> +/* Computes result = (left - right) % mod.
> + * Assumes that left < mod and right < mod, result != mod.
> + */
> +static void vli_mod_sub(u64 *result, const u64 *left, const u64 *right,
> +			const u64 *mod, unsigned int ndigits)
> +{
> +	u64 borrow = vli_sub(result, left, right, ndigits);
> +
> +	/* In this case, p_result == -diff == (max int) - diff.
> +	 * Since -x % d == d - x, we can get the correct result from
> +	 * result + mod (with overflow).
> +	 */
> +	if (borrow)
> +		vli_add(result, result, mod, ndigits);
> +}
> +
> +/*
> + * Computes result = product % mod
> + * for special form moduli: p = 2^k-c, for small c (note the minus sign)
> + *
> + * References:
> + * R. Crandall, C. Pomerance. Prime Numbers: A Computational Perspective.
> + * 9 Fast Algorithms for Large-Integer Arithmetic. 9.2.3 Moduli of special form
> + * Algorithm 9.2.13 (Fast mod operation for special-form moduli).
> + */
> +static void vli_mmod_special(u64 *result, const u64 *product,
> +			      const u64 *mod, unsigned int ndigits)
> +{
> +	u64 c = -mod[0];
> +	u64 t[ECC_MAX_DIGITS * 2];
> +	u64 r[ECC_MAX_DIGITS * 2];
> +
> +	vli_set(r, product, ndigits * 2);
> +	while (!vli_is_zero(r + ndigits, ndigits)) {
> +		vli_umult(t, r + ndigits, c, ndigits);
> +		vli_clear(r + ndigits, ndigits);
> +		vli_add(r, r, t, ndigits * 2);
> +	}
> +	vli_set(t, mod, ndigits);
> +	vli_clear(t + ndigits, ndigits);
> +	while (vli_cmp(r, t, ndigits * 2) >= 0)
> +		vli_sub(r, r, t, ndigits * 2);
> +	vli_set(result, r, ndigits);
> +}
> +
> +/*
> + * Computes result = product % mod
> + * for special form moduli: p = 2^{k-1}+c, for small c (note the plus sign)
> + * where k-1 does not fit into qword boundary by -1 bit (such as 255).
> +
> + * References (loosely based on):
> + * A. Menezes, P. van Oorschot, S. Vanstone. Handbook of Applied Cryptography.
> + * 14.3.4 Reduction methods for moduli of special form. Algorithm 14.47.
> + * URL: http://cacr.uwaterloo.ca/hac/about/chap14.pdf
> + *
> + * H. Cohen, G. Frey, R. Avanzi, C. Doche, T. Lange, K. Nguyen, F. Vercauteren.
> + * Handbook of Elliptic and Hyperelliptic Curve Cryptography.
> + * Algorithm 10.25 Fast reduction for special form moduli
> + */
> +static void vli_mmod_special2(u64 *result, const u64 *product,
> +			       const u64 *mod, unsigned int ndigits)
> +{
> +	u64 c2 = mod[0] * 2;
> +	u64 q[ECC_MAX_DIGITS];
> +	u64 r[ECC_MAX_DIGITS * 2];
> +	u64 m[ECC_MAX_DIGITS * 2]; /* expanded mod */
> +	int carry; /* last bit that doesn't fit into q */
> +	int i;
> +
> +	vli_set(m, mod, ndigits);
> +	vli_clear(m + ndigits, ndigits);
> +
> +	vli_set(r, product, ndigits);
> +	/* q and carry are top bits */
> +	vli_set(q, product + ndigits, ndigits);
> +	vli_clear(r + ndigits, ndigits);
> +	carry = vli_is_negative(r, ndigits);
> +	if (carry)
> +		r[ndigits - 1] &= (1ull << 63) - 1;
> +	for (i = 1; carry || !vli_is_zero(q, ndigits); i++) {
> +		u64 qc[ECC_MAX_DIGITS * 2];
> +
> +		vli_umult(qc, q, c2, ndigits);
> +		if (carry)
> +			vli_uadd(qc, qc, mod[0], ndigits * 2);
> +		vli_set(q, qc + ndigits, ndigits);
> +		vli_clear(qc + ndigits, ndigits);
> +		carry = vli_is_negative(qc, ndigits);
> +		if (carry)
> +			qc[ndigits - 1] &= (1ull << 63) - 1;
> +		if (i & 1)
> +			vli_sub(r, r, qc, ndigits * 2);
> +		else
> +			vli_add(r, r, qc, ndigits * 2);
> +	}
> +	while (vli_is_negative(r, ndigits * 2))
> +		vli_add(r, r, m, ndigits * 2);
> +	while (vli_cmp(r, m, ndigits * 2) >= 0)
> +		vli_sub(r, r, m, ndigits * 2);
> +
> +	vli_set(result, r, ndigits);
> +}
> +
> +/*
> + * Computes result = product % mod, where product is 2N words long.
> + * Reference: Ken MacKay's micro-ecc.
> + * Currently only designed to work for curve_p or curve_n.
> + */
> +static void vli_mmod_slow(u64 *result, u64 *product, const u64 *mod,
> +			  unsigned int ndigits)
> +{
> +	u64 mod_m[2 * ECC_MAX_DIGITS];
> +	u64 tmp[2 * ECC_MAX_DIGITS];
> +	u64 *v[2] = { tmp, product };
> +	u64 carry = 0;
> +	unsigned int i;
> +	/* Shift mod so its highest set bit is at the maximum position. */
> +	int shift = (ndigits * 2 * 64) - vli_num_bits(mod, ndigits);
> +	int word_shift = shift / 64;
> +	int bit_shift = shift % 64;
> +
> +	vli_clear(mod_m, word_shift);
> +	if (bit_shift > 0) {
> +		for (i = 0; i < ndigits; ++i) {
> +			mod_m[word_shift + i] = (mod[i] << bit_shift) | carry;
> +			carry = mod[i] >> (64 - bit_shift);
> +		}
> +	} else
> +		vli_set(mod_m + word_shift, mod, ndigits);
> +
> +	for (i = 1; shift >= 0; --shift) {
> +		u64 borrow = 0;
> +		unsigned int j;
> +
> +		for (j = 0; j < ndigits * 2; ++j) {
> +			u64 diff = v[i][j] - mod_m[j] - borrow;
> +
> +			if (diff != v[i][j])
> +				borrow = (diff > v[i][j]);
> +			v[1 - i][j] = diff;
> +		}
> +		i = !(i ^ borrow); /* Swap the index if there was no borrow */
> +		vli_rshift1(mod_m, ndigits);
> +		mod_m[ndigits - 1] |= mod_m[ndigits] << (64 - 1);
> +		vli_rshift1(mod_m + ndigits, ndigits);
> +	}
> +	vli_set(result, v[i], ndigits);
> +}
> +
> +/* Computes result = product % mod using Barrett's reduction with precomputed
> + * value mu appended to the mod after ndigits, mu = (2^{2w} / mod) and have
> + * length ndigits + 1, where mu * (2^w - 1) should not overflow ndigits
> + * boundary.
> + *
> + * Reference:
> + * R. Brent, P. Zimmermann. Modern Computer Arithmetic. 2010.
> + * 2.4.1 Barrett's algorithm. Algorithm 2.5.
> + */
> +static void vli_mmod_barrett(u64 *result, u64 *product, const u64 *mod,
> +			     unsigned int ndigits)
> +{
> +	u64 q[ECC_MAX_DIGITS * 2];
> +	u64 r[ECC_MAX_DIGITS * 2];
> +	const u64 *mu = mod + ndigits;
> +
> +	vli_mult(q, product + ndigits, mu, ndigits);
> +	if (mu[ndigits])
> +		vli_add(q + ndigits, q + ndigits, product + ndigits, ndigits);
> +	vli_mult(r, mod, q + ndigits, ndigits);
> +	vli_sub(r, product, r, ndigits * 2);
> +	while (!vli_is_zero(r + ndigits, ndigits) ||
> +	       vli_cmp(r, mod, ndigits) != -1) {
> +		u64 carry;
> +
> +		carry = vli_sub(r, r, mod, ndigits);
> +		vli_usub(r + ndigits, r + ndigits, carry, ndigits);
> +	}
> +	vli_set(result, r, ndigits);
> +}
> +
> +/* Computes p_result = p_product % curve_p.
> + * See algorithm 5 and 6 from
> + * http://www.isys.uni-klu.ac.at/PDF/2001-0126-MT.pdf
> + */
> +static void vli_mmod_fast_192(u64 *result, const u64 *product,
> +			      const u64 *curve_prime, u64 *tmp)
> +{
> +	const unsigned int ndigits = ECC_CURVE_NIST_P192_DIGITS;
> +	int carry;
> +
> +	vli_set(result, product, ndigits);
> +
> +	vli_set(tmp, &product[3], ndigits);
> +	carry = vli_add(result, result, tmp, ndigits);
> +
> +	tmp[0] = 0;
> +	tmp[1] = product[3];
> +	tmp[2] = product[4];
> +	carry += vli_add(result, result, tmp, ndigits);
> +
> +	tmp[0] = tmp[1] = product[5];
> +	tmp[2] = 0;
> +	carry += vli_add(result, result, tmp, ndigits);
> +
> +	while (carry || vli_cmp(curve_prime, result, ndigits) != 1)
> +		carry -= vli_sub(result, result, curve_prime, ndigits);
> +}
> +
> +/* Computes result = product % curve_prime
> + * from http://www.nsa.gov/ia/_files/nist-routines.pdf
> + */
> +static void vli_mmod_fast_256(u64 *result, const u64 *product,
> +			      const u64 *curve_prime, u64 *tmp)
> +{
> +	int carry;
> +	const unsigned int ndigits = ECC_CURVE_NIST_P256_DIGITS;
> +
> +	/* t */
> +	vli_set(result, product, ndigits);
> +
> +	/* s1 */
> +	tmp[0] = 0;
> +	tmp[1] = product[5] & 0xffffffff00000000ull;
> +	tmp[2] = product[6];
> +	tmp[3] = product[7];
> +	carry = vli_lshift(tmp, tmp, 1, ndigits);
> +	carry += vli_add(result, result, tmp, ndigits);
> +
> +	/* s2 */
> +	tmp[1] = product[6] << 32;
> +	tmp[2] = (product[6] >> 32) | (product[7] << 32);
> +	tmp[3] = product[7] >> 32;
> +	carry += vli_lshift(tmp, tmp, 1, ndigits);
> +	carry += vli_add(result, result, tmp, ndigits);
> +
> +	/* s3 */
> +	tmp[0] = product[4];
> +	tmp[1] = product[5] & 0xffffffff;
> +	tmp[2] = 0;
> +	tmp[3] = product[7];
> +	carry += vli_add(result, result, tmp, ndigits);
> +
> +	/* s4 */
> +	tmp[0] = (product[4] >> 32) | (product[5] << 32);
> +	tmp[1] = (product[5] >> 32) | (product[6] & 0xffffffff00000000ull);
> +	tmp[2] = product[7];
> +	tmp[3] = (product[6] >> 32) | (product[4] << 32);
> +	carry += vli_add(result, result, tmp, ndigits);
> +
> +	/* d1 */
> +	tmp[0] = (product[5] >> 32) | (product[6] << 32);
> +	tmp[1] = (product[6] >> 32);
> +	tmp[2] = 0;
> +	tmp[3] = (product[4] & 0xffffffff) | (product[5] << 32);
> +	carry -= vli_sub(result, result, tmp, ndigits);
> +
> +	/* d2 */
> +	tmp[0] = product[6];
> +	tmp[1] = product[7];
> +	tmp[2] = 0;
> +	tmp[3] = (product[4] >> 32) | (product[5] & 0xffffffff00000000ull);
> +	carry -= vli_sub(result, result, tmp, ndigits);
> +
> +	/* d3 */
> +	tmp[0] = (product[6] >> 32) | (product[7] << 32);
> +	tmp[1] = (product[7] >> 32) | (product[4] << 32);
> +	tmp[2] = (product[4] >> 32) | (product[5] << 32);
> +	tmp[3] = (product[6] << 32);
> +	carry -= vli_sub(result, result, tmp, ndigits);
> +
> +	/* d4 */
> +	tmp[0] = product[7];
> +	tmp[1] = product[4] & 0xffffffff00000000ull;
> +	tmp[2] = product[5];
> +	tmp[3] = product[6] & 0xffffffff00000000ull;
> +	carry -= vli_sub(result, result, tmp, ndigits);
> +
> +	if (carry < 0) {
> +		do {
> +			carry += vli_add(result, result, curve_prime, ndigits);
> +		} while (carry < 0);
> +	} else {
> +		while (carry || vli_cmp(curve_prime, result, ndigits) != 1)
> +			carry -= vli_sub(result, result, curve_prime, ndigits);
> +	}
> +}
> +
> +#define SL32OR32(x32, y32) (((u64)x32 << 32) | y32)
> +#define AND64H(x64)  (x64 & 0xffFFffFF00000000ull)
> +#define AND64L(x64)  (x64 & 0x00000000ffFFffFFull)
> +
> +/* Computes result = product % curve_prime
> + * from "Mathematical routines for the NIST prime elliptic curves"
> + */
> +static void vli_mmod_fast_384(u64 *result, const u64 *product,
> +				const u64 *curve_prime, u64 *tmp)
> +{
> +	int carry;
> +	const unsigned int ndigits = ECC_CURVE_NIST_P384_DIGITS;
> +
> +	/* t */
> +	vli_set(result, product, ndigits);
> +
> +	/* s1 */
> +	tmp[0] = 0;		// 0 || 0
> +	tmp[1] = 0;		// 0 || 0
> +	tmp[2] = SL32OR32(product[11], (product[10]>>32));	//a22||a21
> +	tmp[3] = product[11]>>32;	// 0 ||a23
> +	tmp[4] = 0;		// 0 || 0
> +	tmp[5] = 0;		// 0 || 0
> +	carry = vli_lshift(tmp, tmp, 1, ndigits);
> +	carry += vli_add(result, result, tmp, ndigits);
> +
> +	/* s2 */
> +	tmp[0] = product[6];	//a13||a12
> +	tmp[1] = product[7];	//a15||a14
> +	tmp[2] = product[8];	//a17||a16
> +	tmp[3] = product[9];	//a19||a18
> +	tmp[4] = product[10];	//a21||a20
> +	tmp[5] = product[11];	//a23||a22
> +	carry += vli_add(result, result, tmp, ndigits);
> +
> +	/* s3 */
> +	tmp[0] = SL32OR32(product[11], (product[10]>>32));	//a22||a21
> +	tmp[1] = SL32OR32(product[6], (product[11]>>32));	//a12||a23
> +	tmp[2] = SL32OR32(product[7], (product[6])>>32);	//a14||a13
> +	tmp[3] = SL32OR32(product[8], (product[7]>>32));	//a16||a15
> +	tmp[4] = SL32OR32(product[9], (product[8]>>32));	//a18||a17
> +	tmp[5] = SL32OR32(product[10], (product[9]>>32));	//a20||a19
> +	carry += vli_add(result, result, tmp, ndigits);
> +
> +	/* s4 */
> +	tmp[0] = AND64H(product[11]);	//a23|| 0
> +	tmp[1] = (product[10]<<32);	//a20|| 0
> +	tmp[2] = product[6];	//a13||a12
> +	tmp[3] = product[7];	//a15||a14
> +	tmp[4] = product[8];	//a17||a16
> +	tmp[5] = product[9];	//a19||a18
> +	carry += vli_add(result, result, tmp, ndigits);
> +
> +	/* s5 */
> +	tmp[0] = 0;		//  0|| 0
> +	tmp[1] = 0;		//  0|| 0
> +	tmp[2] = product[10];	//a21||a20
> +	tmp[3] = product[11];	//a23||a22
> +	tmp[4] = 0;		//  0|| 0
> +	tmp[5] = 0;		//  0|| 0
> +	carry += vli_add(result, result, tmp, ndigits);
> +
> +	/* s6 */
> +	tmp[0] = AND64L(product[10]);	// 0 ||a20
> +	tmp[1] = AND64H(product[10]);	//a21|| 0
> +	tmp[2] = product[11];	//a23||a22
> +	tmp[3] = 0;		// 0 || 0
> +	tmp[4] = 0;		// 0 || 0
> +	tmp[5] = 0;		// 0 || 0
> +	carry += vli_add(result, result, tmp, ndigits);
> +
> +	/* d1 */
> +	tmp[0] = SL32OR32(product[6], (product[11]>>32));	//a12||a23
> +	tmp[1] = SL32OR32(product[7], (product[6]>>32));	//a14||a13
> +	tmp[2] = SL32OR32(product[8], (product[7]>>32));	//a16||a15
> +	tmp[3] = SL32OR32(product[9], (product[8]>>32));	//a18||a17
> +	tmp[4] = SL32OR32(product[10], (product[9]>>32));	//a20||a19
> +	tmp[5] = SL32OR32(product[11], (product[10]>>32));	//a22||a21
> +	carry -= vli_sub(result, result, tmp, ndigits);
> +
> +	/* d2 */
> +	tmp[0] = (product[10]<<32);	//a20|| 0
> +	tmp[1] = SL32OR32(product[11], (product[10]>>32));	//a22||a21
> +	tmp[2] = (product[11]>>32);	// 0 ||a23
> +	tmp[3] = 0;		// 0 || 0
> +	tmp[4] = 0;		// 0 || 0
> +	tmp[5] = 0;		// 0 || 0
> +	carry -= vli_sub(result, result, tmp, ndigits);
> +
> +	/* d3 */
> +	tmp[0] = 0;		// 0 || 0
> +	tmp[1] = AND64H(product[11]);	//a23|| 0
> +	tmp[2] = product[11]>>32;	// 0 ||a23
> +	tmp[3] = 0;		// 0 || 0
> +	tmp[4] = 0;		// 0 || 0
> +	tmp[5] = 0;		// 0 || 0
> +	carry -= vli_sub(result, result, tmp, ndigits);
> +
> +	if (carry < 0) {
> +		do {
> +			carry += vli_add(result, result, curve_prime, ndigits);
> +		} while (carry < 0);
> +	} else {
> +		while (carry || vli_cmp(curve_prime, result, ndigits) != 1)
> +			carry -= vli_sub(result, result, curve_prime, ndigits);
> +	}
> +
> +}
> +
> +#undef SL32OR32
> +#undef AND64H
> +#undef AND64L
> +
> +/*
> + * Computes result = product % curve_prime
> + * from "Recommendations for Discrete Logarithm-Based Cryptography:
> + *       Elliptic Curve Domain Parameters" section G.1.4
> + */
> +static void vli_mmod_fast_521(u64 *result, const u64 *product,
> +			      const u64 *curve_prime, u64 *tmp)
> +{
> +	const unsigned int ndigits = ECC_CURVE_NIST_P521_DIGITS;
> +	size_t i;
> +
> +	/* Initialize result with lowest 521 bits from product */
> +	vli_set(result, product, ndigits);
> +	result[8] &= 0x1ff;
> +
> +	for (i = 0; i < ndigits; i++)
> +		tmp[i] = (product[8 + i] >> 9) | (product[9 + i] << 55);
> +	tmp[8] &= 0x1ff;
> +
> +	vli_mod_add(result, result, tmp, curve_prime, ndigits);
> +}
> +
> +/* Computes result = product % curve_prime for different curve_primes.
> + *
> + * Note that curve_primes are distinguished just by heuristic check and
> + * not by complete conformance check.
> + */
> +static bool vli_mmod_fast(u64 *result, u64 *product,
> +			  const struct ecc_curve *curve)
> +{
> +	u64 tmp[2 * ECC_MAX_DIGITS];
> +	const u64 *curve_prime = curve->p;
> +	const unsigned int ndigits = curve->g.ndigits;
> +
> +	/* All NIST curves have name prefix 'nist_' */
> +	if (strncmp(curve->name, "nist_", 5) != 0) {
> +		/* Try to handle Pseudo-Marsenne primes. */
> +		if (curve_prime[ndigits - 1] == -1ull) {
> +			vli_mmod_special(result, product, curve_prime,
> +					 ndigits);
> +			return true;
> +		} else if (curve_prime[ndigits - 1] == 1ull << 63 &&
> +			   curve_prime[ndigits - 2] == 0) {
> +			vli_mmod_special2(result, product, curve_prime,
> +					  ndigits);
> +			return true;
> +		}
> +		vli_mmod_barrett(result, product, curve_prime, ndigits);
> +		return true;
> +	}
> +
> +	switch (ndigits) {
> +	case ECC_CURVE_NIST_P192_DIGITS:
> +		vli_mmod_fast_192(result, product, curve_prime, tmp);
> +		break;
> +	case ECC_CURVE_NIST_P256_DIGITS:
> +		vli_mmod_fast_256(result, product, curve_prime, tmp);
> +		break;
> +	case ECC_CURVE_NIST_P384_DIGITS:
> +		vli_mmod_fast_384(result, product, curve_prime, tmp);
> +		break;
> +	case ECC_CURVE_NIST_P521_DIGITS:
> +		vli_mmod_fast_521(result, product, curve_prime, tmp);
> +		break;
> +	default:
> +		pr_err("ecc: unsupported digits size!\n");
> +		return false;
> +	}
> +
> +	return true;
> +}
> +
> +/* Computes result = (left * right) % mod.
> + * Assumes that mod is big enough curve order.
> + */
> +void vli_mod_mult_slow(u64 *result, const u64 *left, const u64 *right,
> +		       const u64 *mod, unsigned int ndigits)
> +{
> +	u64 product[ECC_MAX_DIGITS * 2];
> +
> +	vli_mult(product, left, right, ndigits);
> +	vli_mmod_slow(result, product, mod, ndigits);
> +}
> +EXPORT_SYMBOL(vli_mod_mult_slow);
> +
> +/* Computes result = (left * right) % curve_prime. */
> +static void vli_mod_mult_fast(u64 *result, const u64 *left, const u64 *right,
> +			      const struct ecc_curve *curve)
> +{
> +	u64 product[2 * ECC_MAX_DIGITS];
> +
> +	vli_mult(product, left, right, curve->g.ndigits);
> +	vli_mmod_fast(result, product, curve);
> +}
> +
> +/* Computes result = left^2 % curve_prime. */
> +static void vli_mod_square_fast(u64 *result, const u64 *left,
> +				const struct ecc_curve *curve)
> +{
> +	u64 product[2 * ECC_MAX_DIGITS];
> +
> +	vli_square(product, left, curve->g.ndigits);
> +	vli_mmod_fast(result, product, curve);
> +}
> +
> +#define EVEN(vli) (!(vli[0] & 1))
> +/* Computes result = (1 / p_input) % mod. All VLIs are the same size.
> + * See "From Euclid's GCD to Montgomery Multiplication to the Great Divide"
> + * https://labs.oracle.com/techrep/2001/smli_tr-2001-95.pdf
> + */
> +void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod,
> +			unsigned int ndigits)
> +{
> +	u64 a[ECC_MAX_DIGITS], b[ECC_MAX_DIGITS];
> +	u64 u[ECC_MAX_DIGITS], v[ECC_MAX_DIGITS];
> +	u64 carry;
> +	int cmp_result;
> +
> +	if (vli_is_zero(input, ndigits)) {
> +		vli_clear(result, ndigits);
> +		return;
> +	}
> +
> +	vli_set(a, input, ndigits);
> +	vli_set(b, mod, ndigits);
> +	vli_clear(u, ndigits);
> +	u[0] = 1;
> +	vli_clear(v, ndigits);
> +
> +	while ((cmp_result = vli_cmp(a, b, ndigits)) != 0) {
> +		carry = 0;
> +
> +		if (EVEN(a)) {
> +			vli_rshift1(a, ndigits);
> +
> +			if (!EVEN(u))
> +				carry = vli_add(u, u, mod, ndigits);
> +
> +			vli_rshift1(u, ndigits);
> +			if (carry)
> +				u[ndigits - 1] |= 0x8000000000000000ull;
> +		} else if (EVEN(b)) {
> +			vli_rshift1(b, ndigits);
> +
> +			if (!EVEN(v))
> +				carry = vli_add(v, v, mod, ndigits);
> +
> +			vli_rshift1(v, ndigits);
> +			if (carry)
> +				v[ndigits - 1] |= 0x8000000000000000ull;
> +		} else if (cmp_result > 0) {
> +			vli_sub(a, a, b, ndigits);
> +			vli_rshift1(a, ndigits);
> +
> +			if (vli_cmp(u, v, ndigits) < 0)
> +				vli_add(u, u, mod, ndigits);
> +
> +			vli_sub(u, u, v, ndigits);
> +			if (!EVEN(u))
> +				carry = vli_add(u, u, mod, ndigits);
> +
> +			vli_rshift1(u, ndigits);
> +			if (carry)
> +				u[ndigits - 1] |= 0x8000000000000000ull;
> +		} else {
> +			vli_sub(b, b, a, ndigits);
> +			vli_rshift1(b, ndigits);
> +
> +			if (vli_cmp(v, u, ndigits) < 0)
> +				vli_add(v, v, mod, ndigits);
> +
> +			vli_sub(v, v, u, ndigits);
> +			if (!EVEN(v))
> +				carry = vli_add(v, v, mod, ndigits);
> +
> +			vli_rshift1(v, ndigits);
> +			if (carry)
> +				v[ndigits - 1] |= 0x8000000000000000ull;
> +		}
> +	}
> +
> +	vli_set(result, u, ndigits);
> +}
> +EXPORT_SYMBOL(vli_mod_inv);
> +
> +/* ------ Point operations ------ */
> +
> +/* Returns true if p_point is the point at infinity, false otherwise. */
> +bool ecc_point_is_zero(const struct ecc_point *point)
> +{
> +	return (vli_is_zero(point->x, point->ndigits) &&
> +		vli_is_zero(point->y, point->ndigits));
> +}
> +EXPORT_SYMBOL(ecc_point_is_zero);
> +
> +/* Point multiplication algorithm using Montgomery's ladder with co-Z
> + * coordinates. From https://eprint.iacr.org/2011/338.pdf
> + */
> +
> +/* Double in place */
> +static void ecc_point_double_jacobian(u64 *x1, u64 *y1, u64 *z1,
> +					const struct ecc_curve *curve)
> +{
> +	/* t1 = x, t2 = y, t3 = z */
> +	u64 t4[ECC_MAX_DIGITS];
> +	u64 t5[ECC_MAX_DIGITS];
> +	const u64 *curve_prime = curve->p;
> +	const unsigned int ndigits = curve->g.ndigits;
> +
> +	if (vli_is_zero(z1, ndigits))
> +		return;
> +
> +	/* t4 = y1^2 */
> +	vli_mod_square_fast(t4, y1, curve);
> +	/* t5 = x1*y1^2 = A */
> +	vli_mod_mult_fast(t5, x1, t4, curve);
> +	/* t4 = y1^4 */
> +	vli_mod_square_fast(t4, t4, curve);
> +	/* t2 = y1*z1 = z3 */
> +	vli_mod_mult_fast(y1, y1, z1, curve);
> +	/* t3 = z1^2 */
> +	vli_mod_square_fast(z1, z1, curve);
> +
> +	/* t1 = x1 + z1^2 */
> +	vli_mod_add(x1, x1, z1, curve_prime, ndigits);
> +	/* t3 = 2*z1^2 */
> +	vli_mod_add(z1, z1, z1, curve_prime, ndigits);
> +	/* t3 = x1 - z1^2 */
> +	vli_mod_sub(z1, x1, z1, curve_prime, ndigits);
> +	/* t1 = x1^2 - z1^4 */
> +	vli_mod_mult_fast(x1, x1, z1, curve);
> +
> +	/* t3 = 2*(x1^2 - z1^4) */
> +	vli_mod_add(z1, x1, x1, curve_prime, ndigits);
> +	/* t1 = 3*(x1^2 - z1^4) */
> +	vli_mod_add(x1, x1, z1, curve_prime, ndigits);
> +	if (vli_test_bit(x1, 0)) {
> +		u64 carry = vli_add(x1, x1, curve_prime, ndigits);
> +
> +		vli_rshift1(x1, ndigits);
> +		x1[ndigits - 1] |= carry << 63;
> +	} else {
> +		vli_rshift1(x1, ndigits);
> +	}
> +	/* t1 = 3/2*(x1^2 - z1^4) = B */
> +
> +	/* t3 = B^2 */
> +	vli_mod_square_fast(z1, x1, curve);
> +	/* t3 = B^2 - A */
> +	vli_mod_sub(z1, z1, t5, curve_prime, ndigits);
> +	/* t3 = B^2 - 2A = x3 */
> +	vli_mod_sub(z1, z1, t5, curve_prime, ndigits);
> +	/* t5 = A - x3 */
> +	vli_mod_sub(t5, t5, z1, curve_prime, ndigits);
> +	/* t1 = B * (A - x3) */
> +	vli_mod_mult_fast(x1, x1, t5, curve);
> +	/* t4 = B * (A - x3) - y1^4 = y3 */
> +	vli_mod_sub(t4, x1, t4, curve_prime, ndigits);
> +
> +	vli_set(x1, z1, ndigits);
> +	vli_set(z1, y1, ndigits);
> +	vli_set(y1, t4, ndigits);
> +}
> +
> +/* Modify (x1, y1) => (x1 * z^2, y1 * z^3) */
> +static void apply_z(u64 *x1, u64 *y1, u64 *z, const struct ecc_curve *curve)
> +{
> +	u64 t1[ECC_MAX_DIGITS];
> +
> +	vli_mod_square_fast(t1, z, curve);		/* z^2 */
> +	vli_mod_mult_fast(x1, x1, t1, curve);	/* x1 * z^2 */
> +	vli_mod_mult_fast(t1, t1, z, curve);	/* z^3 */
> +	vli_mod_mult_fast(y1, y1, t1, curve);	/* y1 * z^3 */
> +}
> +
> +/* P = (x1, y1) => 2P, (x2, y2) => P' */
> +static void xycz_initial_double(u64 *x1, u64 *y1, u64 *x2, u64 *y2,
> +				u64 *p_initial_z, const struct ecc_curve *curve)
> +{
> +	u64 z[ECC_MAX_DIGITS];
> +	const unsigned int ndigits = curve->g.ndigits;
> +
> +	vli_set(x2, x1, ndigits);
> +	vli_set(y2, y1, ndigits);
> +
> +	vli_clear(z, ndigits);
> +	z[0] = 1;
> +
> +	if (p_initial_z)
> +		vli_set(z, p_initial_z, ndigits);
> +
> +	apply_z(x1, y1, z, curve);
> +
> +	ecc_point_double_jacobian(x1, y1, z, curve);
> +
> +	apply_z(x2, y2, z, curve);
> +}
> +
> +/* Input P = (x1, y1, Z), Q = (x2, y2, Z)
> + * Output P' = (x1', y1', Z3), P + Q = (x3, y3, Z3)
> + * or P => P', Q => P + Q
> + */
> +static void xycz_add(u64 *x1, u64 *y1, u64 *x2, u64 *y2,
> +			const struct ecc_curve *curve)
> +{
> +	/* t1 = X1, t2 = Y1, t3 = X2, t4 = Y2 */
> +	u64 t5[ECC_MAX_DIGITS];
> +	const u64 *curve_prime = curve->p;
> +	const unsigned int ndigits = curve->g.ndigits;
> +
> +	/* t5 = x2 - x1 */
> +	vli_mod_sub(t5, x2, x1, curve_prime, ndigits);
> +	/* t5 = (x2 - x1)^2 = A */
> +	vli_mod_square_fast(t5, t5, curve);
> +	/* t1 = x1*A = B */
> +	vli_mod_mult_fast(x1, x1, t5, curve);
> +	/* t3 = x2*A = C */
> +	vli_mod_mult_fast(x2, x2, t5, curve);
> +	/* t4 = y2 - y1 */
> +	vli_mod_sub(y2, y2, y1, curve_prime, ndigits);
> +	/* t5 = (y2 - y1)^2 = D */
> +	vli_mod_square_fast(t5, y2, curve);
> +
> +	/* t5 = D - B */
> +	vli_mod_sub(t5, t5, x1, curve_prime, ndigits);
> +	/* t5 = D - B - C = x3 */
> +	vli_mod_sub(t5, t5, x2, curve_prime, ndigits);
> +	/* t3 = C - B */
> +	vli_mod_sub(x2, x2, x1, curve_prime, ndigits);
> +	/* t2 = y1*(C - B) */
> +	vli_mod_mult_fast(y1, y1, x2, curve);
> +	/* t3 = B - x3 */
> +	vli_mod_sub(x2, x1, t5, curve_prime, ndigits);
> +	/* t4 = (y2 - y1)*(B - x3) */
> +	vli_mod_mult_fast(y2, y2, x2, curve);
> +	/* t4 = y3 */
> +	vli_mod_sub(y2, y2, y1, curve_prime, ndigits);
> +
> +	vli_set(x2, t5, ndigits);
> +}
> +
> +/* Input P = (x1, y1, Z), Q = (x2, y2, Z)
> + * Output P + Q = (x3, y3, Z3), P - Q = (x3', y3', Z3)
> + * or P => P - Q, Q => P + Q
> + */
> +static void xycz_add_c(u64 *x1, u64 *y1, u64 *x2, u64 *y2,
> +			const struct ecc_curve *curve)
> +{
> +	/* t1 = X1, t2 = Y1, t3 = X2, t4 = Y2 */
> +	u64 t5[ECC_MAX_DIGITS];
> +	u64 t6[ECC_MAX_DIGITS];
> +	u64 t7[ECC_MAX_DIGITS];
> +	const u64 *curve_prime = curve->p;
> +	const unsigned int ndigits = curve->g.ndigits;
> +
> +	/* t5 = x2 - x1 */
> +	vli_mod_sub(t5, x2, x1, curve_prime, ndigits);
> +	/* t5 = (x2 - x1)^2 = A */
> +	vli_mod_square_fast(t5, t5, curve);
> +	/* t1 = x1*A = B */
> +	vli_mod_mult_fast(x1, x1, t5, curve);
> +	/* t3 = x2*A = C */
> +	vli_mod_mult_fast(x2, x2, t5, curve);
> +	/* t4 = y2 + y1 */
> +	vli_mod_add(t5, y2, y1, curve_prime, ndigits);
> +	/* t4 = y2 - y1 */
> +	vli_mod_sub(y2, y2, y1, curve_prime, ndigits);
> +
> +	/* t6 = C - B */
> +	vli_mod_sub(t6, x2, x1, curve_prime, ndigits);
> +	/* t2 = y1 * (C - B) */
> +	vli_mod_mult_fast(y1, y1, t6, curve);
> +	/* t6 = B + C */
> +	vli_mod_add(t6, x1, x2, curve_prime, ndigits);
> +	/* t3 = (y2 - y1)^2 */
> +	vli_mod_square_fast(x2, y2, curve);
> +	/* t3 = x3 */
> +	vli_mod_sub(x2, x2, t6, curve_prime, ndigits);
> +
> +	/* t7 = B - x3 */
> +	vli_mod_sub(t7, x1, x2, curve_prime, ndigits);
> +	/* t4 = (y2 - y1)*(B - x3) */
> +	vli_mod_mult_fast(y2, y2, t7, curve);
> +	/* t4 = y3 */
> +	vli_mod_sub(y2, y2, y1, curve_prime, ndigits);
> +
> +	/* t7 = (y2 + y1)^2 = F */
> +	vli_mod_square_fast(t7, t5, curve);
> +	/* t7 = x3' */
> +	vli_mod_sub(t7, t7, t6, curve_prime, ndigits);
> +	/* t6 = x3' - B */
> +	vli_mod_sub(t6, t7, x1, curve_prime, ndigits);
> +	/* t6 = (y2 + y1)*(x3' - B) */
> +	vli_mod_mult_fast(t6, t6, t5, curve);
> +	/* t2 = y3' */
> +	vli_mod_sub(y1, t6, y1, curve_prime, ndigits);
> +
> +	vli_set(x1, t7, ndigits);
> +}
> +
> +static void ecc_point_mult(struct ecc_point *result,
> +			   const struct ecc_point *point, const u64 *scalar,
> +			   u64 *initial_z, const struct ecc_curve *curve,
> +			   unsigned int ndigits)
> +{
> +	/* R0 and R1 */
> +	u64 rx[2][ECC_MAX_DIGITS];
> +	u64 ry[2][ECC_MAX_DIGITS];
> +	u64 z[ECC_MAX_DIGITS];
> +	u64 sk[2][ECC_MAX_DIGITS];
> +	u64 *curve_prime = curve->p;
> +	int i, nb;
> +	int num_bits;
> +	int carry;
> +
> +	carry = vli_add(sk[0], scalar, curve->n, ndigits);
> +	vli_add(sk[1], sk[0], curve->n, ndigits);
> +	scalar = sk[!carry];
> +	if (curve->nbits == 521)	/* NIST P521 */
> +		num_bits = curve->nbits + 2;
> +	else
> +		num_bits = sizeof(u64) * ndigits * 8 + 1;
> +
> +	vli_set(rx[1], point->x, ndigits);
> +	vli_set(ry[1], point->y, ndigits);
> +
> +	xycz_initial_double(rx[1], ry[1], rx[0], ry[0], initial_z, curve);
> +
> +	for (i = num_bits - 2; i > 0; i--) {
> +		nb = !vli_test_bit(scalar, i);
> +		xycz_add_c(rx[1 - nb], ry[1 - nb], rx[nb], ry[nb], curve);
> +		xycz_add(rx[nb], ry[nb], rx[1 - nb], ry[1 - nb], curve);
> +	}
> +
> +	nb = !vli_test_bit(scalar, 0);
> +	xycz_add_c(rx[1 - nb], ry[1 - nb], rx[nb], ry[nb], curve);
> +
> +	/* Find final 1/Z value. */
> +	/* X1 - X0 */
> +	vli_mod_sub(z, rx[1], rx[0], curve_prime, ndigits);
> +	/* Yb * (X1 - X0) */
> +	vli_mod_mult_fast(z, z, ry[1 - nb], curve);
> +	/* xP * Yb * (X1 - X0) */
> +	vli_mod_mult_fast(z, z, point->x, curve);
> +
> +	/* 1 / (xP * Yb * (X1 - X0)) */
> +	vli_mod_inv(z, z, curve_prime, point->ndigits);
> +
> +	/* yP / (xP * Yb * (X1 - X0)) */
> +	vli_mod_mult_fast(z, z, point->y, curve);
> +	/* Xb * yP / (xP * Yb * (X1 - X0)) */
> +	vli_mod_mult_fast(z, z, rx[1 - nb], curve);
> +	/* End 1/Z calculation */
> +
> +	xycz_add(rx[nb], ry[nb], rx[1 - nb], ry[1 - nb], curve);
> +
> +	apply_z(rx[0], ry[0], z, curve);
> +
> +	vli_set(result->x, rx[0], ndigits);
> +	vli_set(result->y, ry[0], ndigits);
> +}
> +
> +/* Computes R = P + Q mod p */
> +static void ecc_point_add(const struct ecc_point *result,
> +		   const struct ecc_point *p, const struct ecc_point *q,
> +		   const struct ecc_curve *curve)
> +{
> +	u64 z[ECC_MAX_DIGITS];
> +	u64 px[ECC_MAX_DIGITS];
> +	u64 py[ECC_MAX_DIGITS];
> +	unsigned int ndigits = curve->g.ndigits;
> +
> +	vli_set(result->x, q->x, ndigits);
> +	vli_set(result->y, q->y, ndigits);
> +	vli_mod_sub(z, result->x, p->x, curve->p, ndigits);
> +	vli_set(px, p->x, ndigits);
> +	vli_set(py, p->y, ndigits);
> +	xycz_add(px, py, result->x, result->y, curve);
> +	vli_mod_inv(z, z, curve->p, ndigits);
> +	apply_z(result->x, result->y, z, curve);
> +}
> +
> +/* Computes R = u1P + u2Q mod p using Shamir's trick.
> + * Based on: Kenneth MacKay's micro-ecc (2014).
> + */
> +void ecc_point_mult_shamir(const struct ecc_point *result,
> +			   const u64 *u1, const struct ecc_point *p,
> +			   const u64 *u2, const struct ecc_point *q,
> +			   const struct ecc_curve *curve)
> +{
> +	u64 z[ECC_MAX_DIGITS];
> +	u64 sump[2][ECC_MAX_DIGITS];
> +	u64 *rx = result->x;
> +	u64 *ry = result->y;
> +	unsigned int ndigits = curve->g.ndigits;
> +	unsigned int num_bits;
> +	struct ecc_point sum = ECC_POINT_INIT(sump[0], sump[1], ndigits);
> +	const struct ecc_point *points[4];
> +	const struct ecc_point *point;
> +	unsigned int idx;
> +	int i;
> +
> +	ecc_point_add(&sum, p, q, curve);
> +	points[0] = NULL;
> +	points[1] = p;
> +	points[2] = q;
> +	points[3] = &sum;
> +
> +	num_bits = max(vli_num_bits(u1, ndigits), vli_num_bits(u2, ndigits));
> +	i = num_bits - 1;
> +	idx = !!vli_test_bit(u1, i);
> +	idx |= (!!vli_test_bit(u2, i)) << 1;
> +	point = points[idx];
> +
> +	vli_set(rx, point->x, ndigits);
> +	vli_set(ry, point->y, ndigits);
> +	vli_clear(z + 1, ndigits - 1);
> +	z[0] = 1;
> +
> +	for (--i; i >= 0; i--) {
> +		ecc_point_double_jacobian(rx, ry, z, curve);
> +		idx = !!vli_test_bit(u1, i);
> +		idx |= (!!vli_test_bit(u2, i)) << 1;
> +		point = points[idx];
> +		if (point) {
> +			u64 tx[ECC_MAX_DIGITS];
> +			u64 ty[ECC_MAX_DIGITS];
> +			u64 tz[ECC_MAX_DIGITS];
> +
> +			vli_set(tx, point->x, ndigits);
> +			vli_set(ty, point->y, ndigits);
> +			apply_z(tx, ty, z, curve);
> +			vli_mod_sub(tz, rx, tx, curve->p, ndigits);
> +			xycz_add(tx, ty, rx, ry, curve);
> +			vli_mod_mult_fast(z, z, tz, curve);
> +		}
> +	}
> +	vli_mod_inv(z, z, curve->p, ndigits);
> +	apply_z(rx, ry, z, curve);
> +}
> +EXPORT_SYMBOL(ecc_point_mult_shamir);
> +
> +/*
> + * This function performs checks equivalent to Appendix A.4.2 of FIPS 186-5.
> + * Whereas A.4.2 results in an integer in the interval [1, n-1], this function
> + * ensures that the integer is in the range of [2, n-3]. We are slightly
> + * stricter because of the currently used scalar multiplication algorithm.
> + */
> +static int __ecc_is_key_valid(const struct ecc_curve *curve,
> +			      const u64 *private_key, unsigned int ndigits)
> +{
> +	u64 one[ECC_MAX_DIGITS] = { 1, };
> +	u64 res[ECC_MAX_DIGITS];
> +
> +	if (!private_key)
> +		return -EINVAL;
> +
> +	if (curve->g.ndigits != ndigits)
> +		return -EINVAL;
> +
> +	/* Make sure the private key is in the range [2, n-3]. */
> +	if (vli_cmp(one, private_key, ndigits) != -1)
> +		return -EINVAL;
> +	vli_sub(res, curve->n, one, ndigits);
> +	vli_sub(res, res, one, ndigits);
> +	if (vli_cmp(res, private_key, ndigits) != 1)
> +		return -EINVAL;
> +
> +	return 0;
> +}
> +
> +int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
> +		     const u64 *private_key, unsigned int private_key_len)
> +{
> +	int nbytes;
> +	const struct ecc_curve *curve = ecc_get_curve(curve_id);
> +
> +	nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
> +
> +	if (private_key_len != nbytes)
> +		return -EINVAL;
> +
> +	return __ecc_is_key_valid(curve, private_key, ndigits);
> +}
> +EXPORT_SYMBOL(ecc_is_key_valid);
> +
> +int ecc_make_pub_key(unsigned int curve_id, unsigned int ndigits,
> +		     const u64 *private_key, u64 *public_key)
> +{
> +	int ret = 0;
> +	struct ecc_point *pk;
> +	const struct ecc_curve *curve = ecc_get_curve(curve_id);
> +
> +	if (!private_key) {
> +		ret = -EINVAL;
> +		goto out;
> +	}
> +
> +	pk = ecc_alloc_point(ndigits);
> +	if (!pk) {
> +		ret = -ENOMEM;
> +		goto out;
> +	}
> +
> +	ecc_point_mult(pk, &curve->g, private_key, NULL, curve, ndigits);
> +
> +	/* SP800-56A rev 3 5.6.2.1.3 key check */
> +	if (ecc_is_pubkey_valid_full(curve, pk)) {
> +		ret = -EAGAIN;
> +		goto err_free_point;
> +	}
> +
> +	ecc_swap_digits(pk->x, public_key, ndigits);
> +	ecc_swap_digits(pk->y, &public_key[ndigits], ndigits);
> +
> +err_free_point:
> +	ecc_free_point(pk);
> +out:
> +	return ret;
> +}
> +EXPORT_SYMBOL(ecc_make_pub_key);
> +
> +/* SP800-56A section 5.6.2.3.4 partial verification: ephemeral keys only */
> +int ecc_is_pubkey_valid_partial(const struct ecc_curve *curve,
> +				struct ecc_point *pk)
> +{
> +	u64 yy[ECC_MAX_DIGITS], xxx[ECC_MAX_DIGITS], w[ECC_MAX_DIGITS];
> +
> +	if (WARN_ON(pk->ndigits != curve->g.ndigits))
> +		return -EINVAL;
> +
> +	/* Check 1: Verify key is not the zero point. */
> +	if (ecc_point_is_zero(pk))
> +		return -EINVAL;
> +
> +	/* Check 2: Verify key is in the range [1, p-1]. */
> +	if (vli_cmp(curve->p, pk->x, pk->ndigits) != 1)
> +		return -EINVAL;
> +	if (vli_cmp(curve->p, pk->y, pk->ndigits) != 1)
> +		return -EINVAL;
> +
> +	/* Check 3: Verify that y^2 == (x^3 + a·x + b) mod p */
> +	vli_mod_square_fast(yy, pk->y, curve); /* y^2 */
> +	vli_mod_square_fast(xxx, pk->x, curve); /* x^2 */
> +	vli_mod_mult_fast(xxx, xxx, pk->x, curve); /* x^3 */
> +	vli_mod_mult_fast(w, curve->a, pk->x, curve); /* a·x */
> +	vli_mod_add(w, w, curve->b, curve->p, pk->ndigits); /* a·x + b */
> +	vli_mod_add(w, w, xxx, curve->p, pk->ndigits); /* x^3 + a·x + b */
> +	if (vli_cmp(yy, w, pk->ndigits) != 0) /* Equation */
> +		return -EINVAL;
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL(ecc_is_pubkey_valid_partial);
> +
> +/* SP800-56A section 5.6.2.3.3 full verification */
> +int ecc_is_pubkey_valid_full(const struct ecc_curve *curve,
> +			     struct ecc_point *pk)
> +{
> +	struct ecc_point *nQ;
> +
> +	/* Checks 1 through 3 */
> +	int ret = ecc_is_pubkey_valid_partial(curve, pk);
> +
> +	if (ret)
> +		return ret;
> +
> +	/* Check 4: Verify that nQ is the zero point. */
> +	nQ = ecc_alloc_point(pk->ndigits);
> +	if (!nQ)
> +		return -ENOMEM;
> +
> +	ecc_point_mult(nQ, pk, curve->n, NULL, curve, pk->ndigits);
> +	if (!ecc_point_is_zero(nQ))
> +		ret = -EINVAL;
> +
> +	ecc_free_point(nQ);
> +
> +	return ret;
> +}
> +EXPORT_SYMBOL(ecc_is_pubkey_valid_full);
> +
> +int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
> +			      const u64 *private_key, const u64 *public_key,
> +			      u64 *secret)
> +{
> +	int ret = 0;
> +	struct ecc_point *product, *pk;
> +	u64 rand_z[ECC_MAX_DIGITS];
> +	unsigned int nbytes;
> +	const struct ecc_curve *curve = ecc_get_curve(curve_id);
> +
> +	if (!private_key || !public_key || ndigits > ARRAY_SIZE(rand_z)) {
> +		ret = -EINVAL;
> +		goto out;
> +	}
> +
> +	nbytes = ndigits << ECC_DIGITS_TO_BYTES_SHIFT;
> +
> +	get_random_bytes(rand_z, nbytes);
> +
> +	pk = ecc_alloc_point(ndigits);
> +	if (!pk) {
> +		ret = -ENOMEM;
> +		goto out;
> +	}
> +
> +	ecc_swap_digits(public_key, pk->x, ndigits);
> +	ecc_swap_digits(&public_key[ndigits], pk->y, ndigits);
> +	ret = ecc_is_pubkey_valid_partial(curve, pk);
> +	if (ret)
> +		goto err_alloc_product;
> +
> +	product = ecc_alloc_point(ndigits);
> +	if (!product) {
> +		ret = -ENOMEM;
> +		goto err_alloc_product;
> +	}
> +
> +	ecc_point_mult(product, pk, private_key, rand_z, curve, ndigits);
> +
> +	if (ecc_point_is_zero(product)) {
> +		ret = -EFAULT;
> +		goto err_validity;
> +	}
> +
> +	ecc_swap_digits(product->x, secret, ndigits);
> +
> +err_validity:
> +	memzero_explicit(rand_z, sizeof(rand_z));
> +	ecc_free_point(product);
> +err_alloc_product:
> +	ecc_free_point(pk);
> +out:
> +	return ret;
> +}
> +EXPORT_SYMBOL(crypto_ecdh_shared_secret);
> +
> +MODULE_LICENSE("Dual BSD/GPL");
> diff --git a/crypto/ecc_curve_defs.h b/crypto/ecc_curve_defs.h
> new file mode 100644
> index 0000000000..0ecade7d02
> --- /dev/null
> +++ b/crypto/ecc_curve_defs.h
> @@ -0,0 +1,155 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef _CRYTO_ECC_CURVE_DEFS_H
> +#define _CRYTO_ECC_CURVE_DEFS_H
> +
> +/* NIST P-192: a = p - 3 */
> +static u64 nist_p192_g_x[] = { 0xF4FF0AFD82FF1012ull, 0x7CBF20EB43A18800ull,
> +				0x188DA80EB03090F6ull };
> +static u64 nist_p192_g_y[] = { 0x73F977A11E794811ull, 0x631011ED6B24CDD5ull,
> +				0x07192B95FFC8DA78ull };
> +static u64 nist_p192_p[] = { 0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFEull,
> +				0xFFFFFFFFFFFFFFFFull };
> +static u64 nist_p192_n[] = { 0x146BC9B1B4D22831ull, 0xFFFFFFFF99DEF836ull,
> +				0xFFFFFFFFFFFFFFFFull };
> +static u64 nist_p192_a[] = { 0xFFFFFFFFFFFFFFFCull, 0xFFFFFFFFFFFFFFFEull,
> +				0xFFFFFFFFFFFFFFFFull };
> +static u64 nist_p192_b[] = { 0xFEB8DEECC146B9B1ull, 0x0FA7E9AB72243049ull,
> +				0x64210519E59C80E7ull };
> +static struct ecc_curve nist_p192 = {
> +	.name = "nist_192",
> +	.nbits = 192,
> +	.g = {
> +		.x = nist_p192_g_x,
> +		.y = nist_p192_g_y,
> +		.ndigits = 3,
> +	},
> +	.p = nist_p192_p,
> +	.n = nist_p192_n,
> +	.a = nist_p192_a,
> +	.b = nist_p192_b
> +};
> +
> +/* NIST P-256: a = p - 3 */
> +static u64 nist_p256_g_x[] = { 0xF4A13945D898C296ull, 0x77037D812DEB33A0ull,
> +				0xF8BCE6E563A440F2ull, 0x6B17D1F2E12C4247ull };
> +static u64 nist_p256_g_y[] = { 0xCBB6406837BF51F5ull, 0x2BCE33576B315ECEull,
> +				0x8EE7EB4A7C0F9E16ull, 0x4FE342E2FE1A7F9Bull };
> +static u64 nist_p256_p[] = { 0xFFFFFFFFFFFFFFFFull, 0x00000000FFFFFFFFull,
> +				0x0000000000000000ull, 0xFFFFFFFF00000001ull };
> +static u64 nist_p256_n[] = { 0xF3B9CAC2FC632551ull, 0xBCE6FAADA7179E84ull,
> +				0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFF00000000ull };
> +static u64 nist_p256_a[] = { 0xFFFFFFFFFFFFFFFCull, 0x00000000FFFFFFFFull,
> +				0x0000000000000000ull, 0xFFFFFFFF00000001ull };
> +static u64 nist_p256_b[] = { 0x3BCE3C3E27D2604Bull, 0x651D06B0CC53B0F6ull,
> +				0xB3EBBD55769886BCull, 0x5AC635D8AA3A93E7ull };
> +static struct ecc_curve nist_p256 = {
> +	.name = "nist_256",
> +	.nbits = 256,
> +	.g = {
> +		.x = nist_p256_g_x,
> +		.y = nist_p256_g_y,
> +		.ndigits = 4,
> +	},
> +	.p = nist_p256_p,
> +	.n = nist_p256_n,
> +	.a = nist_p256_a,
> +	.b = nist_p256_b
> +};
> +
> +/* NIST P-384 */
> +static u64 nist_p384_g_x[] = { 0x3A545E3872760AB7ull, 0x5502F25DBF55296Cull,
> +				0x59F741E082542A38ull, 0x6E1D3B628BA79B98ull,
> +				0x8Eb1C71EF320AD74ull, 0xAA87CA22BE8B0537ull };
> +static u64 nist_p384_g_y[] = { 0x7A431D7C90EA0E5Full, 0x0A60B1CE1D7E819Dull,
> +				0xE9DA3113B5F0B8C0ull, 0xF8F41DBD289A147Cull,
> +				0x5D9E98BF9292DC29ull, 0x3617DE4A96262C6Full };
> +static u64 nist_p384_p[] = { 0x00000000FFFFFFFFull, 0xFFFFFFFF00000000ull,
> +				0xFFFFFFFFFFFFFFFEull, 0xFFFFFFFFFFFFFFFFull,
> +				0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull };
> +static u64 nist_p384_n[] = { 0xECEC196ACCC52973ull, 0x581A0DB248B0A77Aull,
> +				0xC7634D81F4372DDFull, 0xFFFFFFFFFFFFFFFFull,
> +				0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull };
> +static u64 nist_p384_a[] = { 0x00000000FFFFFFFCull, 0xFFFFFFFF00000000ull,
> +				0xFFFFFFFFFFFFFFFEull, 0xFFFFFFFFFFFFFFFFull,
> +				0xFFFFFFFFFFFFFFFFull, 0xFFFFFFFFFFFFFFFFull };
> +static u64 nist_p384_b[] = { 0x2a85c8edd3ec2aefull, 0xc656398d8a2ed19dull,
> +				0x0314088f5013875aull, 0x181d9c6efe814112ull,
> +				0x988e056be3f82d19ull, 0xb3312fa7e23ee7e4ull };
> +static struct ecc_curve nist_p384 = {
> +	.name = "nist_384",
> +	.nbits = 384,
> +	.g = {
> +		.x = nist_p384_g_x,
> +		.y = nist_p384_g_y,
> +		.ndigits = 6,
> +	},
> +	.p = nist_p384_p,
> +	.n = nist_p384_n,
> +	.a = nist_p384_a,
> +	.b = nist_p384_b
> +};
> +
> +/* NIST P-521 */
> +static u64 nist_p521_g_x[] = { 0xf97e7e31c2e5bd66ull, 0x3348b3c1856a429bull,
> +				0xfe1dc127a2ffa8deull, 0xa14b5e77efe75928ull,
> +				0xf828af606b4d3dbaull, 0x9c648139053fb521ull,
> +				0x9e3ecb662395b442ull, 0x858e06b70404e9cdull,
> +				0xc6ull };
> +static u64 nist_p521_g_y[] = { 0x88be94769fd16650ull, 0x353c7086a272c240ull,
> +				0xc550b9013fad0761ull, 0x97ee72995ef42640ull,
> +				0x17afbd17273e662cull, 0x98f54449579b4468ull,
> +				0x5c8a5fb42c7d1bd9ull, 0x39296a789a3bc004ull,
> +				0x118ull };
> +static u64 nist_p521_p[] = { 0xffffffffffffffffull, 0xffffffffffffffffull,
> +				0xffffffffffffffffull, 0xffffffffffffffffull,
> +				0xffffffffffffffffull, 0xffffffffffffffffull,
> +				0xffffffffffffffffull, 0xffffffffffffffffull,
> +				0x1ffull };
> +static u64 nist_p521_n[] = { 0xbb6fb71e91386409ull, 0x3bb5c9b8899c47aeull,
> +				0x7fcc0148f709a5d0ull, 0x51868783bf2f966bull,
> +				0xfffffffffffffffaull, 0xffffffffffffffffull,
> +				0xffffffffffffffffull, 0xffffffffffffffffull,
> +				0x1ffull };
> +static u64 nist_p521_a[] = { 0xfffffffffffffffcull, 0xffffffffffffffffull,
> +				0xffffffffffffffffull, 0xffffffffffffffffull,
> +				0xffffffffffffffffull, 0xffffffffffffffffull,
> +				0xffffffffffffffffull, 0xffffffffffffffffull,
> +				0x1ffull };
> +static u64 nist_p521_b[] = { 0xef451fd46b503f00ull, 0x3573df883d2c34f1ull,
> +				0x1652c0bd3bb1bf07ull, 0x56193951ec7e937bull,
> +				0xb8b489918ef109e1ull, 0xa2da725b99b315f3ull,
> +				0x929a21a0b68540eeull, 0x953eb9618e1c9a1full,
> +				0x051ull };
> +static struct ecc_curve nist_p521 = {
> +	.name = "nist_521",
> +	.nbits = 521,
> +	.g = {
> +		.x = nist_p521_g_x,
> +		.y = nist_p521_g_y,
> +		.ndigits = 9,
> +	},
> +	.p = nist_p521_p,
> +	.n = nist_p521_n,
> +	.a = nist_p521_a,
> +	.b = nist_p521_b
> +};
> +
> +/* curve25519 */
> +static u64 curve25519_g_x[] = { 0x0000000000000009, 0x0000000000000000,
> +				0x0000000000000000, 0x0000000000000000 };
> +static u64 curve25519_p[] = { 0xffffffffffffffed, 0xffffffffffffffff,
> +				0xffffffffffffffff, 0x7fffffffffffffff };
> +static u64 curve25519_a[] = { 0x000000000001DB41, 0x0000000000000000,
> +				0x0000000000000000, 0x0000000000000000 };
> +static const struct ecc_curve ecc_25519 = {
> +	.name = "curve25519",
> +	.nbits = 255,
> +	.g = {
> +		.x = curve25519_g_x,
> +		.ndigits = 4,
> +	},
> +	.p = curve25519_p,
> +	.a = curve25519_a,
> +};
> +
> +#endif
> diff --git a/include/crypto/ecc_curve.h b/include/crypto/ecc_curve.h
> new file mode 100644
> index 0000000000..7d90c5e822
> --- /dev/null
> +++ b/include/crypto/ecc_curve.h
> @@ -0,0 +1,62 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/* Copyright (c) 2021 HiSilicon */
> +
> +#ifndef _CRYTO_ECC_CURVE_H
> +#define _CRYTO_ECC_CURVE_H
> +
> +#include <linux/types.h>
> +
> +/**
> + * struct ecc_point - elliptic curve point in affine coordinates
> + *
> + * @x:		X coordinate in vli form.
> + * @y:		Y coordinate in vli form.
> + * @ndigits:	Length of vlis in u64 qwords.
> + */
> +struct ecc_point {
> +	u64 *x;
> +	u64 *y;
> +	u8 ndigits;
> +};
> +
> +/**
> + * struct ecc_curve - definition of elliptic curve
> + *
> + * @name:	Short name of the curve.
> + * @nbits:	The number of bits of a curve.
> + * @g:		Generator point of the curve.
> + * @p:		Prime number, if Barrett's reduction is used for this curve
> + *		pre-calculated value 'mu' is appended to the @p after ndigits.
> + *		Use of Barrett's reduction is heuristically determined in
> + *		vli_mmod_fast().
> + * @n:		Order of the curve group.
> + * @a:		Curve parameter a.
> + * @b:		Curve parameter b.
> + */
> +struct ecc_curve {
> +	char *name;
> +	u32 nbits;
> +	struct ecc_point g;
> +	u64 *p;
> +	u64 *n;
> +	u64 *a;
> +	u64 *b;
> +};
> +
> +/**
> + * ecc_get_curve() - get elliptic curve;
> + * @curve_id:           Curves IDs:
> + *                      defined in 'include/crypto/ecdh.h';
> + *
> + * Returns curve if get curve succssful, NULL otherwise
> + */
> +const struct ecc_curve *ecc_get_curve(unsigned int curve_id);
> +
> +/**
> + * ecc_get_curve25519() - get curve25519 curve;
> + *
> + * Returns curve25519
> + */
> +const struct ecc_curve *ecc_get_curve25519(void);
> +
> +#endif
> diff --git a/include/crypto/ecdh.h b/include/crypto/ecdh.h
> new file mode 100644
> index 0000000000..9784ecdd2f
> --- /dev/null
> +++ b/include/crypto/ecdh.h
> @@ -0,0 +1,83 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +/*
> + * ECDH params to be used with kpp API
> + *
> + * Copyright (c) 2016, Intel Corporation
> + * Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
> + */
> +#ifndef _CRYPTO_ECDH_
> +#define _CRYPTO_ECDH_
> +
> +/**
> + * DOC: ECDH Helper Functions
> + *
> + * To use ECDH with the KPP cipher API, the following data structure and
> + * functions should be used.
> + *
> + * The ECC curves known to the ECDH implementation are specified in this
> + * header file.
> + *
> + * To use ECDH with KPP, the following functions should be used to operate on
> + * an ECDH private key. The packet private key that can be set with
> + * the KPP API function call of crypto_kpp_set_secret.
> + */
> +
> +/* Curves IDs */
> +#define ECC_CURVE_NIST_P192	0x0001
> +#define ECC_CURVE_NIST_P256	0x0002
> +#define ECC_CURVE_NIST_P384	0x0003
> +#define ECC_CURVE_NIST_P521	0x0004
> +
> +/**
> + * struct ecdh - define an ECDH private key
> + *
> + * @key:	Private ECDH key
> + * @key_size:	Size of the private ECDH key
> + */
> +struct ecdh {
> +	char *key;
> +	unsigned short key_size;
> +};
> +
> +/**
> + * crypto_ecdh_key_len() - Obtain the size of the private ECDH key
> + * @params:	private ECDH key
> + *
> + * This function returns the packet ECDH key size. A caller can use that
> + * with the provided ECDH private key reference to obtain the required
> + * memory size to hold a packet key.
> + *
> + * Return: size of the key in bytes
> + */
> +unsigned int crypto_ecdh_key_len(const struct ecdh *params);
> +
> +/**
> + * crypto_ecdh_encode_key() - encode the private key
> + * @buf:	Buffer allocated by the caller to hold the packet ECDH
> + *		private key. The buffer should be at least crypto_ecdh_key_len
> + *		bytes in size.
> + * @len:	Length of the packet private key buffer
> + * @p:		Buffer with the caller-specified private key
> + *
> + * The ECDH implementations operate on a packet representation of the private
> + * key.
> + *
> + * Return:	-EINVAL if buffer has insufficient size, 0 on success
> + */
> +int crypto_ecdh_encode_key(char *buf, unsigned int len, const struct ecdh *p);
> +
> +/**
> + * crypto_ecdh_decode_key() - decode a private key
> + * @buf:	Buffer holding a packet key that should be decoded
> + * @len:	Length of the packet private key buffer
> + * @p:		Buffer allocated by the caller that is filled with the
> + *		unpacked ECDH private key.
> + *
> + * The unpacking obtains the private key by pointing @p to the correct location
> + * in @buf. Thus, both pointers refer to the same memory.
> + *
> + * Return:	-EINVAL if buffer has insufficient size, 0 on success
> + */
> +int crypto_ecdh_decode_key(const char *buf, unsigned int len, struct ecdh *p);
> +
> +#endif
> diff --git a/include/crypto/internal/ecc.h b/include/crypto/internal/ecc.h
> new file mode 100644
> index 0000000000..f191491cc0
> --- /dev/null
> +++ b/include/crypto/internal/ecc.h
> @@ -0,0 +1,278 @@
> +/*
> + * Copyright (c) 2013, Kenneth MacKay
> + * All rights reserved.
> + *
> + * Redistribution and use in source and binary forms, with or without
> + * modification, are permitted provided that the following conditions are
> + * met:
> + *  * Redistributions of source code must retain the above copyright
> + *   notice, this list of conditions and the following disclaimer.
> + *  * Redistributions in binary form must reproduce the above copyright
> + *    notice, this list of conditions and the following disclaimer in the
> + *    documentation and/or other materials provided with the distribution.
> + *
> + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
> + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
> + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
> + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
> + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
> + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
> + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
> + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
> + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
> + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
> + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
> + */
> +#ifndef _CRYPTO_ECC_H
> +#define _CRYPTO_ECC_H
> +
> +#include <crypto/ecc_curve.h>
> +#include <asm/unaligned.h>
> +
> +/* One digit is u64 qword. */
> +#define ECC_CURVE_NIST_P192_DIGITS  3
> +#define ECC_CURVE_NIST_P256_DIGITS  4
> +#define ECC_CURVE_NIST_P384_DIGITS  6
> +#define ECC_CURVE_NIST_P521_DIGITS  9
> +#define ECC_MAX_DIGITS              DIV_ROUND_UP(521, 64) /* NIST P521 */
> +
> +#define ECC_DIGITS_TO_BYTES_SHIFT 3
> +
> +#define ECC_MAX_BYTES (ECC_MAX_DIGITS << ECC_DIGITS_TO_BYTES_SHIFT)
> +
> +#define ECC_POINT_INIT(x, y, ndigits)	(struct ecc_point) { x, y, ndigits }
> +
> +/**
> + * ecc_swap_digits() - Copy ndigits from big endian array to native array
> + * @in:       Input array
> + * @out:      Output array
> + * @ndigits:  Number of digits to copy
> + */
> +static inline void ecc_swap_digits(const void *in, u64 *out, unsigned int ndigits)
> +{
> +	const __be64 *src = (__force __be64 *)in;
> +	int i;
> +
> +	for (i = 0; i < ndigits; i++)
> +		out[i] = get_unaligned_be64(&src[ndigits - 1 - i]);
> +}
> +
> +/**
> + * ecc_digits_from_bytes() - Create ndigits-sized digits array from byte array
> + * @in:       Input byte array
> + * @nbytes    Size of input byte array
> + * @out       Output digits array
> + * @ndigits:  Number of digits to create from byte array
> + */
> +void ecc_digits_from_bytes(const u8 *in, unsigned int nbytes,
> +			   u64 *out, unsigned int ndigits);
> +
> +/**
> + * ecc_is_key_valid() - Validate a given ECDH private key
> + *
> + * @curve_id:		id representing the curve to use
> + * @ndigits:		curve's number of digits
> + * @private_key:	private key to be used for the given curve
> + * @private_key_len:	private key length
> + *
> + * Returns 0 if the key is acceptable, a negative value otherwise
> + */
> +int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits,
> +		     const u64 *private_key, unsigned int private_key_len);
> +
> +/**
> + * ecc_make_pub_key() - Compute an ECC public key
> + *
> + * @curve_id:		id representing the curve to use
> + * @ndigits:		curve's number of digits
> + * @private_key:	pregenerated private key for the given curve
> + * @public_key:		buffer for storing the generated public key
> + *
> + * Returns 0 if the public key was generated successfully, a negative value
> + * if an error occurred.
> + */
> +int ecc_make_pub_key(const unsigned int curve_id, unsigned int ndigits,
> +		     const u64 *private_key, u64 *public_key);
> +
> +/**
> + * crypto_ecdh_shared_secret() - Compute a shared secret
> + *
> + * @curve_id:		id representing the curve to use
> + * @ndigits:		curve's number of digits
> + * @private_key:	private key of part A
> + * @public_key:		public key of counterpart B
> + * @secret:		buffer for storing the calculated shared secret
> + *
> + * Note: It is recommended that you hash the result of crypto_ecdh_shared_secret
> + * before using it for symmetric encryption or HMAC.
> + *
> + * Returns 0 if the shared secret was generated successfully, a negative value
> + * if an error occurred.
> + */
> +int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits,
> +			      const u64 *private_key, const u64 *public_key,
> +			      u64 *secret);
> +
> +/**
> + * ecc_is_pubkey_valid_partial() - Partial public key validation
> + *
> + * @curve:		elliptic curve domain parameters
> + * @pk:			public key as a point
> + *
> + * Valdiate public key according to SP800-56A section 5.6.2.3.4 ECC Partial
> + * Public-Key Validation Routine.
> + *
> + * Note: There is no check that the public key is in the correct elliptic curve
> + * subgroup.
> + *
> + * Return: 0 if validation is successful, -EINVAL if validation is failed.
> + */
> +int ecc_is_pubkey_valid_partial(const struct ecc_curve *curve,
> +				struct ecc_point *pk);
> +
> +/**
> + * ecc_is_pubkey_valid_full() - Full public key validation
> + *
> + * @curve:		elliptic curve domain parameters
> + * @pk:			public key as a point
> + *
> + * Valdiate public key according to SP800-56A section 5.6.2.3.3 ECC Full
> + * Public-Key Validation Routine.
> + *
> + * Return: 0 if validation is successful, -EINVAL if validation is failed.
> + */
> +int ecc_is_pubkey_valid_full(const struct ecc_curve *curve,
> +			     struct ecc_point *pk);
> +
> +/**
> + * vli_is_zero() - Determine is vli is zero
> + *
> + * @vli:		vli to check.
> + * @ndigits:		length of the @vli
> + */
> +bool vli_is_zero(const u64 *vli, unsigned int ndigits);
> +
> +/**
> + * vli_cmp() - compare left and right vlis
> + *
> + * @left:		vli
> + * @right:		vli
> + * @ndigits:		length of both vlis
> + *
> + * Returns sign of @left - @right, i.e. -1 if @left < @right,
> + * 0 if @left == @right, 1 if @left > @right.
> + */
> +int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits);
> +
> +/**
> + * vli_sub() - Subtracts right from left
> + *
> + * @result:		where to write result
> + * @left:		vli
> + * @right		vli
> + * @ndigits:		length of all vlis
> + *
> + * Note: can modify in-place.
> + *
> + * Return: carry bit.
> + */
> +u64 vli_sub(u64 *result, const u64 *left, const u64 *right,
> +	    unsigned int ndigits);
> +
> +/**
> + * vli_from_be64() - Load vli from big-endian u64 array
> + *
> + * @dest:		destination vli
> + * @src:		source array of u64 BE values
> + * @ndigits:		length of both vli and array
> + */
> +void vli_from_be64(u64 *dest, const void *src, unsigned int ndigits);
> +
> +/**
> + * vli_from_le64() - Load vli from little-endian u64 array
> + *
> + * @dest:		destination vli
> + * @src:		source array of u64 LE values
> + * @ndigits:		length of both vli and array
> + */
> +void vli_from_le64(u64 *dest, const void *src, unsigned int ndigits);
> +
> +/**
> + * vli_mod_inv() - Modular inversion
> + *
> + * @result:		where to write vli number
> + * @input:		vli value to operate on
> + * @mod:		modulus
> + * @ndigits:		length of all vlis
> + */
> +void vli_mod_inv(u64 *result, const u64 *input, const u64 *mod,
> +		 unsigned int ndigits);
> +
> +/**
> + * vli_mod_mult_slow() - Modular multiplication
> + *
> + * @result:		where to write result value
> + * @left:		vli number to multiply with @right
> + * @right:		vli number to multiply with @left
> + * @mod:		modulus
> + * @ndigits:		length of all vlis
> + *
> + * Note: Assumes that mod is big enough curve order.
> + */
> +void vli_mod_mult_slow(u64 *result, const u64 *left, const u64 *right,
> +		       const u64 *mod, unsigned int ndigits);
> +
> +/**
> + * vli_num_bits() - Counts the number of bits required for vli.
> + *
> + * @vli:		vli to check.
> + * @ndigits:		Length of the @vli
> + *
> + * Return: The number of bits required to represent @vli.
> + */
> +unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits);
> +
> +/**
> + * ecc_aloc_point() - Allocate ECC point.
> + *
> + * @ndigits:		Length of vlis in u64 qwords.
> + *
> + * Return: Pointer to the allocated point or NULL if allocation failed.
> + */
> +struct ecc_point *ecc_alloc_point(unsigned int ndigits);
> +
> +/**
> + * ecc_free_point() - Free ECC point.
> + *
> + * @p:			The point to free.
> + */
> +void ecc_free_point(struct ecc_point *p);
> +
> +/**
> + * ecc_point_is_zero() - Check if point is zero.
> + *
> + * @p:			Point to check for zero.
> + *
> + * Return: true if point is the point at infinity, false otherwise.
> + */
> +bool ecc_point_is_zero(const struct ecc_point *point);
> +
> +/**
> + * ecc_point_mult_shamir() - Add two points multiplied by scalars
> + *
> + * @result:		resulting point
> + * @x:			scalar to multiply with @p
> + * @p:			point to multiply with @x
> + * @y:			scalar to multiply with @q
> + * @q:			point to multiply with @y
> + * @curve:		curve
> + *
> + * Returns result = x * p + x * q over the curve.
> + * This works faster than two multiplications and addition.
> + */
> +void ecc_point_mult_shamir(const struct ecc_point *result,
> +			   const u64 *x, const struct ecc_point *p,
> +			   const u64 *y, const struct ecc_point *q,
> +			   const struct ecc_curve *curve);
> +
> +#endif

-- 
Pengutronix e.K.                           |                             |
Steuerwalder Str. 21                       | http://www.pengutronix.de/  |
31137 Hildesheim, Germany                  | Phone: +49-5121-206917-0    |
Amtsgericht Hildesheim, HRA 2686           | Fax:   +49-5121-206917-5555 |