blob: 3291b553c5b4040cdb7a405f19e5528d802b4455 [file] [log] [blame]
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001/*
2 * Copyright (C) 2021 Denys Vlasenko
3 *
4 * Licensed under GPLv2, see file LICENSE in this source tree.
5 */
6#include "tls.h"
7
8#define SP_DEBUG 0
9#define FIXED_SECRET 0
10#define FIXED_PEER_PUBKEY 0
11
Denys Vlasenko3b411eb2021-10-05 20:00:50 +020012#define ALLOW_ASM 1
13
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +020014#if SP_DEBUG
15# define dbg(...) fprintf(stderr, __VA_ARGS__)
16static void dump_hex(const char *fmt, const void *vp, int len)
17{
18 char hexbuf[32 * 1024 + 4];
19 const uint8_t *p = vp;
20
21 bin2hex(hexbuf, (void*)p, len)[0] = '\0';
22 dbg(fmt, hexbuf);
23}
24#else
25# define dbg(...) ((void)0)
26# define dump_hex(...) ((void)0)
27#endif
28
Denys Vlasenko3b411eb2021-10-05 20:00:50 +020029typedef uint32_t sp_digit;
30typedef int32_t signed_sp_digit;
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +020031
Denys Vlasenko4bc9da12021-11-27 11:28:11 +010032/* 64-bit optimizations:
33 * if BB_UNALIGNED_MEMACCESS_OK && ULONG_MAX > 0xffffffff,
34 * then loads and stores can be done in 64-bit chunks.
35 *
36 * A narrower case is when arch is also little-endian (such as x86_64),
37 * then "LSW first", uint32[8] and uint64[4] representations are equivalent,
38 * and arithmetic can be done in 64 bits too.
39 */
40#if defined(__GNUC__) && defined(__x86_64__)
41# define UNALIGNED_LE_64BIT 1
42#else
43# define UNALIGNED_LE_64BIT 0
44#endif
45
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +020046/* The code below is taken from parts of
47 * wolfssl-3.15.3/wolfcrypt/src/sp_c32.c
48 * and heavily modified.
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +020049 */
50
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +020051typedef struct sp_point {
Denys Vlasenko3b411eb2021-10-05 20:00:50 +020052 sp_digit x[2 * 8];
53 sp_digit y[2 * 8];
54 sp_digit z[2 * 8];
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +020055 int infinity;
56} sp_point;
57
58/* The modulus (prime) of the curve P256. */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +020059static const sp_digit p256_mod[8] = {
60 0xffffffff,0xffffffff,0xffffffff,0x00000000,
61 0x00000000,0x00000000,0x00000001,0xffffffff,
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +020062};
63
64#define p256_mp_mod ((sp_digit)0x000001)
65
Denys Vlasenko3b411eb2021-10-05 20:00:50 +020066/* Normalize the values in each word to 32 bits - NOP */
67#define sp_256_norm_8(a) ((void)0)
Denys Vlasenko77145182021-10-01 13:51:39 +020068
Denys Vlasenkoe7305052021-10-05 13:30:48 +020069/* Write r as big endian to byte array.
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +020070 * Fixed length number of bytes written: 32
71 *
72 * r A single precision integer.
73 * a Byte array.
74 */
Denys Vlasenko4bc9da12021-11-27 11:28:11 +010075#if BB_UNALIGNED_MEMACCESS_OK && ULONG_MAX > 0xffffffff
76static void sp_256_to_bin_8(const sp_digit* rr, uint8_t* a)
77{
78 int i;
79 const uint64_t* r = (void*)rr;
80
81 sp_256_norm_8(rr);
82
83 r += 4;
84 for (i = 0; i < 4; i++) {
85 r--;
86 move_to_unaligned64(a, SWAP_BE64(*r));
87 a += 8;
88 }
89}
90#else
Denys Vlasenko3b411eb2021-10-05 20:00:50 +020091static void sp_256_to_bin_8(const sp_digit* r, uint8_t* a)
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +020092{
Denys Vlasenko3b411eb2021-10-05 20:00:50 +020093 int i;
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +020094
Denys Vlasenko3b411eb2021-10-05 20:00:50 +020095 sp_256_norm_8(r);
Denys Vlasenko77145182021-10-01 13:51:39 +020096
Denys Vlasenko3b411eb2021-10-05 20:00:50 +020097 r += 8;
98 for (i = 0; i < 8; i++) {
99 r--;
100 move_to_unaligned32(a, SWAP_BE32(*r));
101 a += 4;
Denys Vlasenko12040122021-04-26 20:24:34 +0200102 }
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200103}
Denys Vlasenko4bc9da12021-11-27 11:28:11 +0100104#endif
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200105
Denys Vlasenkoe7305052021-10-05 13:30:48 +0200106/* Read big endian unsigned byte array into r.
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200107 *
108 * r A single precision integer.
109 * a Byte array.
110 * n Number of bytes in array to read.
111 */
Denys Vlasenko4bc9da12021-11-27 11:28:11 +0100112#if BB_UNALIGNED_MEMACCESS_OK && ULONG_MAX > 0xffffffff
113static void sp_256_from_bin_8(sp_digit* rr, const uint8_t* a)
114{
115 int i;
116 uint64_t* r = (void*)rr;
117
118 r += 4;
119 for (i = 0; i < 4; i++) {
120 uint64_t v;
121 move_from_unaligned64(v, a);
122 *--r = SWAP_BE64(v);
123 a += 8;
124 }
125}
126#else
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200127static void sp_256_from_bin_8(sp_digit* r, const uint8_t* a)
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200128{
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200129 int i;
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200130
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200131 r += 8;
132 for (i = 0; i < 8; i++) {
133 sp_digit v;
134 move_from_unaligned32(v, a);
135 *--r = SWAP_BE32(v);
136 a += 4;
Denys Vlasenko12040122021-04-26 20:24:34 +0200137 }
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200138}
Denys Vlasenko4bc9da12021-11-27 11:28:11 +0100139#endif
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200140
Denys Vlasenko137864f2021-10-05 13:47:42 +0200141#if SP_DEBUG
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200142static void dump_256(const char *fmt, const sp_digit* r)
Denys Vlasenko137864f2021-10-05 13:47:42 +0200143{
Denys Vlasenko137864f2021-10-05 13:47:42 +0200144 uint8_t b32[32];
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200145 sp_256_to_bin_8(r, b32);
Denys Vlasenko137864f2021-10-05 13:47:42 +0200146 dump_hex(fmt, b32, 32);
147}
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200148static void dump_512(const char *fmt, const sp_digit* r)
Denys Vlasenko137864f2021-10-05 13:47:42 +0200149{
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200150 uint8_t b64[64];
151 sp_256_to_bin_8(r, b64 + 32);
152 sp_256_to_bin_8(r+8, b64);
153 dump_hex(fmt, b64, 64);
Denys Vlasenko137864f2021-10-05 13:47:42 +0200154}
155#else
156# define dump_256(...) ((void)0)
157# define dump_512(...) ((void)0)
158#endif
159
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200160/* Convert a point of big-endian 32-byte x,y pair to type sp_point. */
161static void sp_256_point_from_bin2x32(sp_point* p, const uint8_t *bin2x32)
162{
Denys Vlasenko12040122021-04-26 20:24:34 +0200163 memset(p, 0, sizeof(*p));
164 /*p->infinity = 0;*/
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200165 sp_256_from_bin_8(p->x, bin2x32);
166 sp_256_from_bin_8(p->y, bin2x32 + 32);
Denys Vlasenkoe7305052021-10-05 13:30:48 +0200167 p->z[0] = 1; /* p->z = 1 */
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200168}
169
Denys Vlasenkob3b17132021-04-26 16:53:53 +0200170/* Compare a with b.
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200171 *
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200172 * return -ve, 0 or +ve if a is less than, equal to or greater than b
173 * respectively.
174 */
Denys Vlasenko4bc9da12021-11-27 11:28:11 +0100175#if UNALIGNED_LE_64BIT
176static signed_sp_digit sp_256_cmp_8(const sp_digit* aa, const sp_digit* bb)
177{
178 const uint64_t* a = (void*)aa;
179 const uint64_t* b = (void*)bb;
180 int i;
181 for (i = 3; i >= 0; i--) {
182 if (a[i] == b[i])
183 continue;
184 return (a[i] > b[i]) * 2 - 1;
185 }
186 return 0;
187}
188#else
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200189static signed_sp_digit sp_256_cmp_8(const sp_digit* a, const sp_digit* b)
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200190{
Denys Vlasenko12040122021-04-26 20:24:34 +0200191 int i;
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200192 for (i = 7; i >= 0; i--) {
193/* signed_sp_digit r = a[i] - b[i];
194 * if (r != 0)
195 * return r;
196 * does not work: think about a[i]=0, b[i]=0xffffffff
197 */
198 if (a[i] == b[i])
199 continue;
200 return (a[i] > b[i]) * 2 - 1;
Denys Vlasenko12040122021-04-26 20:24:34 +0200201 }
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200202 return 0;
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200203}
Denys Vlasenko4bc9da12021-11-27 11:28:11 +0100204#endif
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200205
206/* Compare two numbers to determine if they are equal.
207 *
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200208 * return 1 when equal and 0 otherwise.
209 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200210static int sp_256_cmp_equal_8(const sp_digit* a, const sp_digit* b)
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200211{
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200212 return sp_256_cmp_8(a, b) == 0;
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200213}
214
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200215/* Add b to a into r. (r = a + b). Return !0 on overflow */
216static int sp_256_add_8(sp_digit* r, const sp_digit* a, const sp_digit* b)
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200217{
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200218#if ALLOW_ASM && defined(__GNUC__) && defined(__i386__)
219 sp_digit reg;
220 asm volatile (
221"\n movl (%0), %3"
222"\n addl (%1), %3"
223"\n movl %3, (%2)"
224"\n"
225"\n movl 1*4(%0), %3"
226"\n adcl 1*4(%1), %3"
227"\n movl %3, 1*4(%2)"
228"\n"
229"\n movl 2*4(%0), %3"
230"\n adcl 2*4(%1), %3"
231"\n movl %3, 2*4(%2)"
232"\n"
233"\n movl 3*4(%0), %3"
234"\n adcl 3*4(%1), %3"
235"\n movl %3, 3*4(%2)"
236"\n"
237"\n movl 4*4(%0), %3"
238"\n adcl 4*4(%1), %3"
239"\n movl %3, 4*4(%2)"
240"\n"
241"\n movl 5*4(%0), %3"
242"\n adcl 5*4(%1), %3"
243"\n movl %3, 5*4(%2)"
244"\n"
245"\n movl 6*4(%0), %3"
246"\n adcl 6*4(%1), %3"
247"\n movl %3, 6*4(%2)"
248"\n"
249"\n movl 7*4(%0), %3"
250"\n adcl 7*4(%1), %3"
251"\n movl %3, 7*4(%2)"
252"\n"
253"\n sbbl %3, %3"
254"\n"
255 : "=r" (a), "=r" (b), "=r" (r), "=r" (reg)
256 : "0" (a), "1" (b), "2" (r)
257 : "memory"
258 );
259 return reg;
Denys Vlasenko911344a2021-10-06 17:17:34 +0200260#elif ALLOW_ASM && defined(__GNUC__) && defined(__x86_64__)
Denys Vlasenko911344a2021-10-06 17:17:34 +0200261 uint64_t reg;
262 asm volatile (
263"\n movq (%0), %3"
264"\n addq (%1), %3"
265"\n movq %3, (%2)"
266"\n"
267"\n movq 1*8(%0), %3"
268"\n adcq 1*8(%1), %3"
269"\n movq %3, 1*8(%2)"
270"\n"
271"\n movq 2*8(%0), %3"
272"\n adcq 2*8(%1), %3"
273"\n movq %3, 2*8(%2)"
274"\n"
275"\n movq 3*8(%0), %3"
276"\n adcq 3*8(%1), %3"
277"\n movq %3, 3*8(%2)"
278"\n"
279"\n sbbq %3, %3"
280"\n"
281 : "=r" (a), "=r" (b), "=r" (r), "=r" (reg)
282 : "0" (a), "1" (b), "2" (r)
283 : "memory"
284 );
285 return reg;
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200286#else
Denys Vlasenko12040122021-04-26 20:24:34 +0200287 int i;
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200288 sp_digit carry;
289
290 carry = 0;
291 for (i = 0; i < 8; i++) {
292 sp_digit w, v;
293 w = b[i] + carry;
294 v = a[i];
295 if (w != 0) {
296 v = a[i] + w;
297 carry = (v < a[i]);
298 /* hope compiler detects above as "carry flag set" */
299 }
300 /* else: b + carry == 0, two cases:
301 * b:ffffffff, carry:1
302 * b:00000000, carry:0
303 * in either case, r[i] = a[i] and carry remains unchanged
304 */
305 r[i] = v;
306 }
307 return carry;
308#endif
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200309}
310
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200311/* Sub b from a into r. (r = a - b). Return !0 on underflow */
312static int sp_256_sub_8(sp_digit* r, const sp_digit* a, const sp_digit* b)
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200313{
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200314#if ALLOW_ASM && defined(__GNUC__) && defined(__i386__)
315 sp_digit reg;
316 asm volatile (
317"\n movl (%0), %3"
318"\n subl (%1), %3"
319"\n movl %3, (%2)"
320"\n"
321"\n movl 1*4(%0), %3"
322"\n sbbl 1*4(%1), %3"
323"\n movl %3, 1*4(%2)"
324"\n"
325"\n movl 2*4(%0), %3"
326"\n sbbl 2*4(%1), %3"
327"\n movl %3, 2*4(%2)"
328"\n"
329"\n movl 3*4(%0), %3"
330"\n sbbl 3*4(%1), %3"
331"\n movl %3, 3*4(%2)"
332"\n"
333"\n movl 4*4(%0), %3"
334"\n sbbl 4*4(%1), %3"
335"\n movl %3, 4*4(%2)"
336"\n"
337"\n movl 5*4(%0), %3"
338"\n sbbl 5*4(%1), %3"
339"\n movl %3, 5*4(%2)"
340"\n"
341"\n movl 6*4(%0), %3"
342"\n sbbl 6*4(%1), %3"
343"\n movl %3, 6*4(%2)"
344"\n"
345"\n movl 7*4(%0), %3"
346"\n sbbl 7*4(%1), %3"
347"\n movl %3, 7*4(%2)"
348"\n"
349"\n sbbl %3, %3"
350"\n"
351 : "=r" (a), "=r" (b), "=r" (r), "=r" (reg)
352 : "0" (a), "1" (b), "2" (r)
353 : "memory"
354 );
355 return reg;
Denys Vlasenko911344a2021-10-06 17:17:34 +0200356#elif ALLOW_ASM && defined(__GNUC__) && defined(__x86_64__)
Denys Vlasenko911344a2021-10-06 17:17:34 +0200357 uint64_t reg;
358 asm volatile (
359"\n movq (%0), %3"
360"\n subq (%1), %3"
361"\n movq %3, (%2)"
362"\n"
363"\n movq 1*8(%0), %3"
364"\n sbbq 1*8(%1), %3"
365"\n movq %3, 1*8(%2)"
366"\n"
367"\n movq 2*8(%0), %3"
368"\n sbbq 2*8(%1), %3"
369"\n movq %3, 2*8(%2)"
370"\n"
371"\n movq 3*8(%0), %3"
372"\n sbbq 3*8(%1), %3"
373"\n movq %3, 3*8(%2)"
374"\n"
375"\n sbbq %3, %3"
376"\n"
377 : "=r" (a), "=r" (b), "=r" (r), "=r" (reg)
378 : "0" (a), "1" (b), "2" (r)
379 : "memory"
380 );
381 return reg;
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200382#else
Denys Vlasenko12040122021-04-26 20:24:34 +0200383 int i;
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200384 sp_digit borrow;
385
386 borrow = 0;
387 for (i = 0; i < 8; i++) {
388 sp_digit w, v;
389 w = b[i] + borrow;
390 v = a[i];
391 if (w != 0) {
392 v = a[i] - w;
393 borrow = (v > a[i]);
394 /* hope compiler detects above as "carry flag set" */
395 }
396 /* else: b + borrow == 0, two cases:
397 * b:ffffffff, borrow:1
398 * b:00000000, borrow:0
399 * in either case, r[i] = a[i] and borrow remains unchanged
400 */
401 r[i] = v;
402 }
403 return borrow;
404#endif
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200405}
406
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200407/* Sub p256_mod from r. (r = r - p256_mod). */
Denys Vlasenko87e3f2e2021-10-06 19:59:39 +0200408#if ALLOW_ASM && defined(__GNUC__) && defined(__i386__)
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200409static void sp_256_sub_8_p256_mod(sp_digit* r)
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200410{
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200411//p256_mod[7..0] = ffffffff 00000001 00000000 00000000 00000000 ffffffff ffffffff ffffffff
412 asm volatile (
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200413"\n subl $0xffffffff, (%0)"
414"\n sbbl $0xffffffff, 1*4(%0)"
415"\n sbbl $0xffffffff, 2*4(%0)"
416"\n sbbl $0, 3*4(%0)"
417"\n sbbl $0, 4*4(%0)"
418"\n sbbl $0, 5*4(%0)"
419"\n sbbl $1, 6*4(%0)"
420"\n sbbl $0xffffffff, 7*4(%0)"
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200421"\n"
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200422 : "=r" (r)
423 : "0" (r)
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200424 : "memory"
425 );
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200426}
Denys Vlasenko87e3f2e2021-10-06 19:59:39 +0200427#elif ALLOW_ASM && defined(__GNUC__) && defined(__x86_64__)
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200428static void sp_256_sub_8_p256_mod(sp_digit* r)
Denys Vlasenko87e3f2e2021-10-06 19:59:39 +0200429{
430 uint64_t reg;
431 uint64_t ooff;
432//p256_mod[3..0] = ffffffff00000001 0000000000000000 00000000ffffffff ffffffffffffffff
433 asm volatile (
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200434"\n addq $1, (%0)" // adding 1 is the same as subtracting ffffffffffffffff
Denys Vlasenko87e3f2e2021-10-06 19:59:39 +0200435"\n cmc" // only carry bit needs inverting
Denys Vlasenko17e6fb02021-10-06 21:22:36 +0200436"\n"
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200437"\n sbbq %1, 1*8(%0)" // %1 holds 00000000ffffffff
Denys Vlasenko17e6fb02021-10-06 21:22:36 +0200438"\n"
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200439"\n sbbq $0, 2*8(%0)"
Denys Vlasenko87e3f2e2021-10-06 19:59:39 +0200440"\n"
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200441"\n movq 3*8(%0), %2"
442"\n sbbq $0, %2" // adding 00000000ffffffff (in %1)
443"\n addq %1, %2" // is the same as subtracting ffffffff00000001
444"\n movq %2, 3*8(%0)"
Denys Vlasenko87e3f2e2021-10-06 19:59:39 +0200445"\n"
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200446 : "=r" (r), "=r" (ooff), "=r" (reg)
447 : "0" (r), "1" (0x00000000ffffffff)
Denys Vlasenko87e3f2e2021-10-06 19:59:39 +0200448 : "memory"
449 );
450}
Denys Vlasenko567eefc2021-10-06 14:25:10 +0200451#else
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200452static void sp_256_sub_8_p256_mod(sp_digit* r)
453{
454 sp_256_sub_8(r, r, p256_mod);
455}
Denys Vlasenko567eefc2021-10-06 14:25:10 +0200456#endif
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200457
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100458/* Multiply a and b into r. (r = a * b)
459 * r should be [16] array (512 bits).
460 */
461static void sp_256to512_mul_8(sp_digit* r, const sp_digit* a, const sp_digit* b)
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200462{
Denys Vlasenkobbd723e2021-10-05 23:19:18 +0200463#if ALLOW_ASM && defined(__GNUC__) && defined(__i386__)
464 sp_digit rr[15]; /* in case r coincides with a or b */
465 int k;
466 uint32_t accl;
467 uint32_t acch;
468
469 acch = accl = 0;
470 for (k = 0; k < 15; k++) {
471 int i, j;
472 uint32_t acc_hi;
473 i = k - 7;
474 if (i < 0)
475 i = 0;
476 j = k - i;
477 acc_hi = 0;
478 do {
479////////////////////////
480// uint64_t m = ((uint64_t)a[i]) * b[j];
481// acc_hi:acch:accl += m;
482 asm volatile (
483 // a[i] is already loaded in %%eax
484"\n mull %7"
485"\n addl %%eax, %0"
486"\n adcl %%edx, %1"
487"\n adcl $0, %2"
488 : "=rm" (accl), "=rm" (acch), "=rm" (acc_hi)
489 : "0" (accl), "1" (acch), "2" (acc_hi), "a" (a[i]), "m" (b[j])
490 : "cc", "dx"
491 );
492////////////////////////
493 j--;
494 i++;
495 } while (i != 8 && i <= k);
496 rr[k] = accl;
497 accl = acch;
498 acch = acc_hi;
499 }
500 r[15] = accl;
501 memcpy(r, rr, sizeof(rr));
Denys Vlasenko911344a2021-10-06 17:17:34 +0200502#elif ALLOW_ASM && defined(__GNUC__) && defined(__x86_64__)
Denys Vlasenko911344a2021-10-06 17:17:34 +0200503 const uint64_t* aa = (const void*)a;
504 const uint64_t* bb = (const void*)b;
505 uint64_t rr[8];
506 int k;
507 uint64_t accl;
508 uint64_t acch;
509
510 acch = accl = 0;
511 for (k = 0; k < 7; k++) {
512 int i, j;
513 uint64_t acc_hi;
514 i = k - 3;
515 if (i < 0)
516 i = 0;
517 j = k - i;
518 acc_hi = 0;
519 do {
520////////////////////////
521// uint128_t m = ((uint128_t)a[i]) * b[j];
522// acc_hi:acch:accl += m;
523 asm volatile (
524 // aa[i] is already loaded in %%rax
525"\n mulq %7"
526"\n addq %%rax, %0"
527"\n adcq %%rdx, %1"
528"\n adcq $0, %2"
529 : "=rm" (accl), "=rm" (acch), "=rm" (acc_hi)
530 : "0" (accl), "1" (acch), "2" (acc_hi), "a" (aa[i]), "m" (bb[j])
531 : "cc", "dx"
532 );
533////////////////////////
Denys Vlasenko17e6fb02021-10-06 21:22:36 +0200534 j--;
Denys Vlasenko911344a2021-10-06 17:17:34 +0200535 i++;
536 } while (i != 4 && i <= k);
537 rr[k] = accl;
538 accl = acch;
539 acch = acc_hi;
540 }
541 rr[7] = accl;
542 memcpy(r, rr, sizeof(rr));
Denys Vlasenkobbd723e2021-10-05 23:19:18 +0200543#elif 0
544 //TODO: arm assembly (untested)
545 sp_digit tmp[16];
546
547 asm volatile (
548"\n mov r5, #0"
549"\n mov r6, #0"
550"\n mov r7, #0"
551"\n mov r8, #0"
552"\n 1:"
553"\n subs r3, r5, #28"
554"\n movcc r3, #0"
555"\n sub r4, r5, r3"
Denys Vlasenko22fd8fd2021-10-06 16:10:49 +0200556"\n 2:"
Denys Vlasenkobbd723e2021-10-05 23:19:18 +0200557"\n ldr r14, [%[a], r3]"
558"\n ldr r12, [%[b], r4]"
559"\n umull r9, r10, r14, r12"
560"\n adds r6, r6, r9"
561"\n adcs r7, r7, r10"
562"\n adc r8, r8, #0"
563"\n add r3, r3, #4"
564"\n sub r4, r4, #4"
565"\n cmp r3, #32"
566"\n beq 3f"
567"\n cmp r3, r5"
568"\n ble 2b"
569"\n 3:"
570"\n str r6, [%[r], r5]"
571"\n mov r6, r7"
572"\n mov r7, r8"
573"\n mov r8, #0"
574"\n add r5, r5, #4"
575"\n cmp r5, #56"
576"\n ble 1b"
577"\n str r6, [%[r], r5]"
578 : [r] "r" (tmp), [a] "r" (a), [b] "r" (b)
Denys Vlasenko22fd8fd2021-10-06 16:10:49 +0200579 : "memory", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r12", "r14"
Denys Vlasenkobbd723e2021-10-05 23:19:18 +0200580 );
581 memcpy(r, tmp, sizeof(tmp));
582#else
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200583 sp_digit rr[15]; /* in case r coincides with a or b */
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200584 int i, j, k;
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200585 uint64_t acc;
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200586
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200587 acc = 0;
588 for (k = 0; k < 15; k++) {
589 uint32_t acc_hi;
590 i = k - 7;
591 if (i < 0)
592 i = 0;
593 j = k - i;
594 acc_hi = 0;
Denys Vlasenkobbd723e2021-10-05 23:19:18 +0200595 do {
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200596 uint64_t m = ((uint64_t)a[i]) * b[j];
597 acc += m;
598 if (acc < m)
599 acc_hi++;
600 j--;
601 i++;
Denys Vlasenkobbd723e2021-10-05 23:19:18 +0200602 } while (i != 8 && i <= k);
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200603 rr[k] = acc;
604 acc = (acc >> 32) | ((uint64_t)acc_hi << 32);
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200605 }
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200606 r[15] = acc;
607 memcpy(r, rr, sizeof(rr));
Denys Vlasenkobbd723e2021-10-05 23:19:18 +0200608#endif
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200609}
610
Denys Vlasenko389329e2021-10-05 13:39:33 +0200611/* Shift number right one bit. Bottom bit is lost. */
Denys Vlasenko4bc9da12021-11-27 11:28:11 +0100612#if UNALIGNED_LE_64BIT
613static void sp_256_rshift1_8(sp_digit* rr, uint64_t carry)
Denys Vlasenkoe7305052021-10-05 13:30:48 +0200614{
Denys Vlasenko4bc9da12021-11-27 11:28:11 +0100615 uint64_t *r = (void*)rr;
Denys Vlasenkoe7305052021-10-05 13:30:48 +0200616 int i;
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200617
Denys Vlasenko4bc9da12021-11-27 11:28:11 +0100618 carry = (((uint64_t)!!carry) << 63);
619 for (i = 3; i >= 0; i--) {
620 uint64_t c = r[i] << 63;
621 r[i] = (r[i] >> 1) | carry;
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200622 carry = c;
623 }
Denys Vlasenkoe7305052021-10-05 13:30:48 +0200624}
Denys Vlasenko4bc9da12021-11-27 11:28:11 +0100625#else
626static void sp_256_rshift1_8(sp_digit* r, sp_digit carry)
627{
628 int i;
629
630 carry = (((sp_digit)!!carry) << 31);
631 for (i = 7; i >= 0; i--) {
632 sp_digit c = r[i] << 31;
633 r[i] = (r[i] >> 1) | carry;
634 carry = c;
635 }
636}
637#endif
Denys Vlasenkoe7305052021-10-05 13:30:48 +0200638
Denys Vlasenkodcfd8d32021-11-27 16:07:42 +0100639/* Divide the number by 2 mod the modulus (prime). (r = (r / 2) % m) */
640static void sp_256_div2_8(sp_digit* r /*, const sp_digit* m*/)
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200641{
Denys Vlasenkodcfd8d32021-11-27 16:07:42 +0100642 const sp_digit* m = p256_mod;
643
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200644 int carry = 0;
Denys Vlasenkodcfd8d32021-11-27 16:07:42 +0100645 if (r[0] & 1)
646 carry = sp_256_add_8(r, r, m);
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200647 sp_256_norm_8(r);
Denys Vlasenko4bc9da12021-11-27 11:28:11 +0100648 sp_256_rshift1_8(r, carry);
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200649}
650
651/* Add two Montgomery form numbers (r = a + b % m) */
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200652static void sp_256_mont_add_8(sp_digit* r, const sp_digit* a, const sp_digit* b
653 /*, const sp_digit* m*/)
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200654{
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200655// const sp_digit* m = p256_mod;
656
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200657 int carry = sp_256_add_8(r, a, b);
658 sp_256_norm_8(r);
659 if (carry) {
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200660 sp_256_sub_8_p256_mod(r);
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200661 sp_256_norm_8(r);
Denys Vlasenko55578f22021-10-05 19:45:56 +0200662 }
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200663}
664
665/* Subtract two Montgomery form numbers (r = a - b % m) */
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200666static void sp_256_mont_sub_8(sp_digit* r, const sp_digit* a, const sp_digit* b
667 /*, const sp_digit* m*/)
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200668{
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200669 const sp_digit* m = p256_mod;
670
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200671 int borrow;
672 borrow = sp_256_sub_8(r, a, b);
673 sp_256_norm_8(r);
674 if (borrow) {
675 sp_256_add_8(r, r, m);
676 sp_256_norm_8(r);
Denys Vlasenko55578f22021-10-05 19:45:56 +0200677 }
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200678}
679
680/* Double a Montgomery form number (r = a + a % m) */
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200681static void sp_256_mont_dbl_8(sp_digit* r, const sp_digit* a /*, const sp_digit* m*/)
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200682{
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200683// const sp_digit* m = p256_mod;
684
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200685 int carry = sp_256_add_8(r, a, a);
686 sp_256_norm_8(r);
687 if (carry)
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200688 sp_256_sub_8_p256_mod(r);
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200689 sp_256_norm_8(r);
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200690}
691
692/* Triple a Montgomery form number (r = a + a + a % m) */
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200693static void sp_256_mont_tpl_8(sp_digit* r, const sp_digit* a /*, const sp_digit* m*/)
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200694{
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200695// const sp_digit* m = p256_mod;
696
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200697 int carry = sp_256_add_8(r, a, a);
698 sp_256_norm_8(r);
699 if (carry) {
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200700 sp_256_sub_8_p256_mod(r);
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200701 sp_256_norm_8(r);
Denys Vlasenko55578f22021-10-05 19:45:56 +0200702 }
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200703 carry = sp_256_add_8(r, r, a);
704 sp_256_norm_8(r);
705 if (carry) {
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200706 sp_256_sub_8_p256_mod(r);
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200707 sp_256_norm_8(r);
Denys Vlasenko55578f22021-10-05 19:45:56 +0200708 }
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200709}
710
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100711/* Shift the result in the high 256 bits down to the bottom.
712 * High half is cleared to zeros.
713 */
Denys Vlasenko4bc9da12021-11-27 11:28:11 +0100714#if BB_UNALIGNED_MEMACCESS_OK && ULONG_MAX > 0xffffffff
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100715static void sp_512to256_mont_shift_8(sp_digit* rr)
Denys Vlasenko4bc9da12021-11-27 11:28:11 +0100716{
717 uint64_t *r = (void*)rr;
718 int i;
719
720 for (i = 0; i < 4; i++) {
721 r[i] = r[i+4];
722 r[i+4] = 0;
723 }
724}
725#else
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100726static void sp_512to256_mont_shift_8(sp_digit* r)
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200727{
728 int i;
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200729
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200730 for (i = 0; i < 8; i++) {
Denys Vlasenko4bc9da12021-11-27 11:28:11 +0100731 r[i] = r[i+8];
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200732 r[i+8] = 0;
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200733 }
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200734}
Denys Vlasenko4bc9da12021-11-27 11:28:11 +0100735#endif
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200736
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100737/* Mul a by scalar b and add into r. (r += a * b)
738 * a = p256_mod
739 * b = r[0]
740 */
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200741static int sp_256_mul_add_8(sp_digit* r /*, const sp_digit* a, sp_digit b*/)
Denys Vlasenkoe7305052021-10-05 13:30:48 +0200742{
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200743// const sp_digit* a = p256_mod;
744//a[7..0] = ffffffff 00000001 00000000 00000000 00000000 ffffffff ffffffff ffffffff
745 sp_digit b = r[0];
Denys Vlasenkoe7305052021-10-05 13:30:48 +0200746
Denys Vlasenko00f2cce2021-10-06 10:15:29 +0200747 uint64_t t;
748
749// t = 0;
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200750// for (i = 0; i < 8; i++) {
751// uint32_t t_hi;
752// uint64_t m = ((uint64_t)b * a[i]) + r[i];
753// t += m;
754// t_hi = (t < m);
755// r[i] = (sp_digit)t;
756// t = (t >> 32) | ((uint64_t)t_hi << 32);
757// }
758// r[8] += (sp_digit)t;
759
760 // Unroll, then optimize the above loop:
761 //uint32_t t_hi;
762 uint64_t m;
Denys Vlasenko00f2cce2021-10-06 10:15:29 +0200763 uint32_t t32;
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200764
765 //m = ((uint64_t)b * a[0]) + r[0];
766 // Since b is r[0] and a[0] is ffffffff, the above optimizes to:
767 // m = r[0] * ffffffff + r[0] = (r[0] * 100000000 - r[0]) + r[0] = r[0] << 32;
768 //t += m;
Denys Vlasenko00f2cce2021-10-06 10:15:29 +0200769 // t = r[0] << 32 = b << 32;
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200770 //t_hi = (t < m);
771 // t_hi = 0;
772 //r[0] = (sp_digit)t;
773 r[0] = 0;
774 //t = (t >> 32) | ((uint64_t)t_hi << 32);
775 // t = b;
776
777 //m = ((uint64_t)b * a[1]) + r[1];
778 // Since a[1] is ffffffff, the above optimizes to:
779 // m = b * ffffffff + r[1] = (b * 100000000 - b) + r[1] = (b << 32) - b + r[1];
780 //t += m;
781 // t = b + (b << 32) - b + r[1] = (b << 32) + r[1];
782 //t_hi = (t < m);
783 // t_hi = 0;
784 //r[1] = (sp_digit)t;
785 // r[1] = r[1];
786 //t = (t >> 32) | ((uint64_t)t_hi << 32);
787 // t = b;
788
789 //m = ((uint64_t)b * a[2]) + r[2];
790 // Since a[2] is ffffffff, the above optimizes to:
791 // m = b * ffffffff + r[2] = (b * 100000000 - b) + r[2] = (b << 32) - b + r[2];
792 //t += m;
793 // t = b + (b << 32) - b + r[2] = (b << 32) + r[2]
794 //t_hi = (t < m);
795 // t_hi = 0;
796 //r[2] = (sp_digit)t;
797 // r[2] = r[2];
798 //t = (t >> 32) | ((uint64_t)t_hi << 32);
799 // t = b;
800
801 //m = ((uint64_t)b * a[3]) + r[3];
802 // Since a[3] is 00000000, the above optimizes to:
803 // m = b * 0 + r[3] = r[3];
804 //t += m;
Denys Vlasenko00f2cce2021-10-06 10:15:29 +0200805 // t = b + r[3];
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200806 //t_hi = (t < m);
807 // t_hi = 0;
808 //r[3] = (sp_digit)t;
809 r[3] = r[3] + b;
810 //t = (t >> 32) | ((uint64_t)t_hi << 32);
Denys Vlasenko00f2cce2021-10-06 10:15:29 +0200811 t32 = (r[3] < b); // 0 or 1
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200812
813 //m = ((uint64_t)b * a[4]) + r[4];
814 // Since a[4] is 00000000, the above optimizes to:
815 // m = b * 0 + r[4] = r[4];
816 //t += m;
Denys Vlasenko00f2cce2021-10-06 10:15:29 +0200817 // t = t32 + r[4];
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200818 //t_hi = (t < m);
819 // t_hi = 0;
Denys Vlasenko00f2cce2021-10-06 10:15:29 +0200820 //r[4] = (sp_digit)t;
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200821 //t = (t >> 32) | ((uint64_t)t_hi << 32);
Denys Vlasenko00f2cce2021-10-06 10:15:29 +0200822 if (t32 != 0) {
823 r[4]++;
824 t32 = (r[4] == 0); // 0 or 1
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200825
826 //m = ((uint64_t)b * a[5]) + r[5];
827 // Since a[5] is 00000000, the above optimizes to:
828 // m = b * 0 + r[5] = r[5];
829 //t += m;
Denys Vlasenko00f2cce2021-10-06 10:15:29 +0200830 // t = t32 + r[5]; (t32 is 0 or 1)
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200831 //t_hi = (t < m);
832 // t_hi = 0;
Denys Vlasenko00f2cce2021-10-06 10:15:29 +0200833 //r[5] = (sp_digit)t;
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200834 //t = (t >> 32) | ((uint64_t)t_hi << 32);
Denys Vlasenko00f2cce2021-10-06 10:15:29 +0200835 if (t32 != 0) {
836 r[5]++;
837 t32 = (r[5] == 0); // 0 or 1
838 }
839 }
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200840
841 //m = ((uint64_t)b * a[6]) + r[6];
842 // Since a[6] is 00000001, the above optimizes to:
Denys Vlasenko00f2cce2021-10-06 10:15:29 +0200843 // m = (uint64_t)b + r[6]; // 33 bits at most
844 //t += m;
845 t = t32 + (uint64_t)b + r[6];
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200846 //t_hi = (t < m);
Denys Vlasenko00f2cce2021-10-06 10:15:29 +0200847 // t_hi = 0;
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200848 r[6] = (sp_digit)t;
849 //t = (t >> 32) | ((uint64_t)t_hi << 32);
850 t = (t >> 32);
851
852 //m = ((uint64_t)b * a[7]) + r[7];
853 // Since a[7] is ffffffff, the above optimizes to:
854 // m = b * ffffffff + r[7] = (b * 100000000 - b) + r[7]
855 m = ((uint64_t)b << 32) - b + r[7];
856 t += m;
857 //t_hi = (t < m);
Denys Vlasenko00f2cce2021-10-06 10:15:29 +0200858 // t_hi in fact is always 0 here (256bit * 32bit can't have more than 32 bits of overflow)
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200859 r[7] = (sp_digit)t;
860 //t = (t >> 32) | ((uint64_t)t_hi << 32);
861 t = (t >> 32);
862
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200863 r[8] += (sp_digit)t;
864 return (r[8] < (sp_digit)t); /* 1 if addition overflowed */
Denys Vlasenkoe7305052021-10-05 13:30:48 +0200865}
866
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200867/* Reduce the number back to 256 bits using Montgomery reduction.
Denys Vlasenko9c671fe2021-11-27 18:42:27 +0100868 * Note: the result is NOT guaranteed to be less than p256_mod!
869 * (it is only guaranteed to fit into 256 bits).
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200870 *
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100871 * a Double-wide number to reduce in place.
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200872 * m The single precision number representing the modulus.
873 * mp The digit representing the negative inverse of m mod 2^n.
874 */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100875static void sp_512to256_mont_reduce_8(sp_digit* a/*, const sp_digit* m, sp_digit mp*/)
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200876{
Denys Vlasenkoc7842842021-10-06 01:09:37 +0200877// const sp_digit* m = p256_mod;
Denys Vlasenko389329e2021-10-05 13:39:33 +0200878 sp_digit mp = p256_mp_mod;
879
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200880 int i;
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200881// sp_digit mu;
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200882
883 if (mp != 1) {
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200884 sp_digit word16th = 0;
885 for (i = 0; i < 8; i++) {
886// mu = (sp_digit)(a[i] * mp);
887 if (sp_256_mul_add_8(a+i /*, m, mu*/)) {
888 int j = i + 8;
889 inc_next_word0:
890 if (++j > 15) { /* a[16] array has no more words? */
891 word16th++;
892 continue;
893 }
894 if (++a[j] == 0) /* did this overflow too? */
895 goto inc_next_word0;
896 }
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200897 }
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100898 sp_512to256_mont_shift_8(a);
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200899 if (word16th != 0)
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200900 sp_256_sub_8_p256_mod(a);
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200901 sp_256_norm_8(a);
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200902 }
Denys Vlasenko389329e2021-10-05 13:39:33 +0200903 else { /* Same code for explicit mp == 1 (which is always the case for P256) */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200904 sp_digit word16th = 0;
905 for (i = 0; i < 8; i++) {
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100906// mu = a[i];
Denys Vlasenko2430fcf2021-10-06 00:19:30 +0200907 if (sp_256_mul_add_8(a+i /*, m, mu*/)) {
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200908 int j = i + 8;
909 inc_next_word:
910 if (++j > 15) { /* a[16] array has no more words? */
911 word16th++;
912 continue;
913 }
914 if (++a[j] == 0) /* did this overflow too? */
915 goto inc_next_word;
916 }
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200917 }
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100918 sp_512to256_mont_shift_8(a);
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200919 if (word16th != 0)
Denys Vlasenko5e9c6172021-10-06 20:14:49 +0200920 sp_256_sub_8_p256_mod(a);
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200921 sp_256_norm_8(a);
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200922 }
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200923}
924
925/* Multiply two Montogmery form numbers mod the modulus (prime).
926 * (r = a * b mod m)
927 *
928 * r Result of multiplication.
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100929 * Should be [16] array (512 bits), but high half is cleared to zeros (used as scratch pad).
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200930 * a First number to multiply in Montogmery form.
931 * b Second number to multiply in Montogmery form.
932 * m Modulus (prime).
933 * mp Montogmery mulitplier.
934 */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100935static void sp_256to512z_mont_mul_8(sp_digit* r, const sp_digit* a, const sp_digit* b
Denys Vlasenko389329e2021-10-05 13:39:33 +0200936 /*, const sp_digit* m, sp_digit mp*/)
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200937{
Denys Vlasenko389329e2021-10-05 13:39:33 +0200938 //const sp_digit* m = p256_mod;
939 //sp_digit mp = p256_mp_mod;
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100940 sp_256to512_mul_8(r, a, b);
941 sp_512to256_mont_reduce_8(r /*, m, mp*/);
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200942}
943
944/* Square the Montgomery form number. (r = a * a mod m)
945 *
946 * r Result of squaring.
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100947 * Should be [16] array (512 bits), but high half is cleared to zeros (used as scratch pad).
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200948 * a Number to square in Montogmery form.
949 * m Modulus (prime).
950 * mp Montogmery mulitplier.
951 */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100952static void sp_256to512z_mont_sqr_8(sp_digit* r, const sp_digit* a
Denys Vlasenko389329e2021-10-05 13:39:33 +0200953 /*, const sp_digit* m, sp_digit mp*/)
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200954{
Denys Vlasenko389329e2021-10-05 13:39:33 +0200955 //const sp_digit* m = p256_mod;
956 //sp_digit mp = p256_mp_mod;
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100957 sp_256to512z_mont_mul_8(r, a, a /*, m, mp*/);
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200958}
959
960/* Invert the number, in Montgomery form, modulo the modulus (prime) of the
961 * P256 curve. (r = 1 / a mod m)
962 *
963 * r Inverse result.
964 * a Number to invert.
965 */
966#if 0
967/* Mod-2 for the P256 curve. */
968static const uint32_t p256_mod_2[8] = {
969 0xfffffffd,0xffffffff,0xffffffff,0x00000000,
970 0x00000000,0x00000000,0x00000001,0xffffffff,
971};
972//Bit pattern:
973//2 2 2 2 2 2 2 1...1
974//5 5 4 3 2 1 0 9...0 9...1
975//543210987654321098765432109876543210987654321098765432109876543210...09876543210...09876543210
976//111111111111111111111111111111110000000000000000000000000000000100...00000111111...11111111101
977#endif
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200978static void sp_256_mont_inv_8(sp_digit* r, sp_digit* a)
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200979{
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100980 sp_digit t[2*8];
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200981 int i;
982
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200983 memcpy(t, a, sizeof(sp_digit) * 8);
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200984 for (i = 254; i >= 0; i--) {
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100985 sp_256to512z_mont_sqr_8(t, t /*, p256_mod, p256_mp_mod*/);
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200986 /*if (p256_mod_2[i / 32] & ((sp_digit)1 << (i % 32)))*/
987 if (i >= 224 || i == 192 || (i <= 95 && i != 1))
Denys Vlasenko4415f7b2021-11-27 15:47:26 +0100988 sp_256to512z_mont_mul_8(t, t, a /*, p256_mod, p256_mp_mod*/);
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200989 }
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200990 memcpy(r, t, sizeof(sp_digit) * 8);
Denys Vlasenkoa2bc52d2021-04-27 01:21:26 +0200991}
992
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200993/* Multiply a number by Montogmery normalizer mod modulus (prime).
994 *
995 * r The resulting Montgomery form number.
996 * a The number to convert.
997 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +0200998static void sp_256_mod_mul_norm_8(sp_digit* r, const sp_digit* a)
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +0200999{
Denys Vlasenko12040122021-04-26 20:24:34 +02001000 int64_t t[8];
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001001 int32_t o;
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001002
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001003#define A(n) ((uint64_t)a[n])
Denys Vlasenko12040122021-04-26 20:24:34 +02001004 /* 1 1 0 -1 -1 -1 -1 0 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001005 t[0] = 0 + A(0) + A(1) - A(3) - A(4) - A(5) - A(6);
Denys Vlasenko12040122021-04-26 20:24:34 +02001006 /* 0 1 1 0 -1 -1 -1 -1 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001007 t[1] = 0 + A(1) + A(2) - A(4) - A(5) - A(6) - A(7);
Denys Vlasenko12040122021-04-26 20:24:34 +02001008 /* 0 0 1 1 0 -1 -1 -1 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001009 t[2] = 0 + A(2) + A(3) - A(5) - A(6) - A(7);
Denys Vlasenko12040122021-04-26 20:24:34 +02001010 /* -1 -1 0 2 2 1 0 -1 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001011 t[3] = 0 - A(0) - A(1) + 2 * A(3) + 2 * A(4) + A(5) - A(7);
Denys Vlasenko12040122021-04-26 20:24:34 +02001012 /* 0 -1 -1 0 2 2 1 0 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001013 t[4] = 0 - A(1) - A(2) + 2 * A(4) + 2 * A(5) + A(6);
Denys Vlasenko12040122021-04-26 20:24:34 +02001014 /* 0 0 -1 -1 0 2 2 1 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001015 t[5] = 0 - A(2) - A(3) + 2 * A(5) + 2 * A(6) + A(7);
Denys Vlasenko12040122021-04-26 20:24:34 +02001016 /* -1 -1 0 0 0 1 3 2 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001017 t[6] = 0 - A(0) - A(1) + A(5) + 3 * A(6) + 2 * A(7);
Denys Vlasenko12040122021-04-26 20:24:34 +02001018 /* 1 0 -1 -1 -1 -1 0 3 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001019 t[7] = 0 + A(0) - A(2) - A(3) - A(4) - A(5) + 3 * A(7);
1020#undef A
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001021
Denys Vlasenko12040122021-04-26 20:24:34 +02001022 t[1] += t[0] >> 32; t[0] &= 0xffffffff;
1023 t[2] += t[1] >> 32; t[1] &= 0xffffffff;
1024 t[3] += t[2] >> 32; t[2] &= 0xffffffff;
1025 t[4] += t[3] >> 32; t[3] &= 0xffffffff;
1026 t[5] += t[4] >> 32; t[4] &= 0xffffffff;
1027 t[6] += t[5] >> 32; t[5] &= 0xffffffff;
1028 t[7] += t[6] >> 32; t[6] &= 0xffffffff;
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001029 o = t[7] >> 32; //t[7] &= 0xffffffff;
Denys Vlasenko12040122021-04-26 20:24:34 +02001030 t[0] += o;
1031 t[3] -= o;
1032 t[6] -= o;
1033 t[7] += o;
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001034 r[0] = (sp_digit)t[0];
1035 t[1] += t[0] >> 32;
1036 r[1] = (sp_digit)t[1];
1037 t[2] += t[1] >> 32;
1038 r[2] = (sp_digit)t[2];
1039 t[3] += t[2] >> 32;
1040 r[3] = (sp_digit)t[3];
1041 t[4] += t[3] >> 32;
1042 r[4] = (sp_digit)t[4];
1043 t[5] += t[4] >> 32;
1044 r[5] = (sp_digit)t[5];
1045 t[6] += t[5] >> 32;
1046 r[6] = (sp_digit)t[6];
1047// t[7] += t[6] >> 32;
1048// r[7] = (sp_digit)t[7];
1049 r[7] = (sp_digit)t[7] + (sp_digit)(t[6] >> 32);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001050}
1051
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001052/* Map the Montgomery form projective co-ordinate point to an affine point.
1053 *
1054 * r Resulting affine co-ordinate point.
1055 * p Montgomery form projective co-ordinate point.
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001056 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001057static void sp_256_map_8(sp_point* r, sp_point* p)
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001058{
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001059 sp_digit t1[2*8];
1060 sp_digit t2[2*8];
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001061
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001062 sp_256_mont_inv_8(t1, p->z);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001063
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001064 sp_256to512z_mont_sqr_8(t2, t1 /*, p256_mod, p256_mp_mod*/);
1065 sp_256to512z_mont_mul_8(t1, t2, t1 /*, p256_mod, p256_mp_mod*/);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001066
Denys Vlasenko12040122021-04-26 20:24:34 +02001067 /* x /= z^2 */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001068 sp_256to512z_mont_mul_8(r->x, p->x, t2 /*, p256_mod, p256_mp_mod*/);
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001069 sp_512to256_mont_reduce_8(r->x /*, p256_mod, p256_mp_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001070 /* Reduce x to less than modulus */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001071 if (sp_256_cmp_8(r->x, p256_mod) >= 0)
Denys Vlasenko5e9c6172021-10-06 20:14:49 +02001072 sp_256_sub_8_p256_mod(r->x);
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001073 sp_256_norm_8(r->x);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001074
Denys Vlasenko12040122021-04-26 20:24:34 +02001075 /* y /= z^3 */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001076 sp_256to512z_mont_mul_8(r->y, p->y, t1 /*, p256_mod, p256_mp_mod*/);
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001077 sp_512to256_mont_reduce_8(r->y /*, p256_mod, p256_mp_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001078 /* Reduce y to less than modulus */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001079 if (sp_256_cmp_8(r->y, p256_mod) >= 0)
Denys Vlasenko5e9c6172021-10-06 20:14:49 +02001080 sp_256_sub_8_p256_mod(r->y);
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001081 sp_256_norm_8(r->y);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001082
Denys Vlasenko12040122021-04-26 20:24:34 +02001083 memset(r->z, 0, sizeof(r->z));
1084 r->z[0] = 1;
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001085}
1086
1087/* Double the Montgomery form projective point p.
1088 *
1089 * r Result of doubling point.
1090 * p Point to double.
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001091 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001092static void sp_256_proj_point_dbl_8(sp_point* r, sp_point* p)
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001093{
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001094 sp_digit t1[2*8];
1095 sp_digit t2[2*8];
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001096
Denys Vlasenko12040122021-04-26 20:24:34 +02001097 /* Put point to double into result */
1098 if (r != p)
1099 *r = *p; /* struct copy */
Denys Vlasenko4d3a5c12021-04-26 15:21:38 +02001100
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001101 if (r->infinity)
Denys Vlasenkoe7305052021-10-05 13:30:48 +02001102 return;
1103
Denys Vlasenko55578f22021-10-05 19:45:56 +02001104 if (SP_DEBUG) {
1105 /* unused part of t2, may result in spurios
1106 * differences in debug output. Clear it.
1107 */
1108 memset(t2, 0, sizeof(t2));
1109 }
1110
Denys Vlasenko12040122021-04-26 20:24:34 +02001111 /* T1 = Z * Z */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001112 sp_256to512z_mont_sqr_8(t1, r->z /*, p256_mod, p256_mp_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001113 /* Z = Y * Z */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001114 sp_256to512z_mont_mul_8(r->z, r->y, r->z /*, p256_mod, p256_mp_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001115 /* Z = 2Z */
Denys Vlasenkoc7842842021-10-06 01:09:37 +02001116 sp_256_mont_dbl_8(r->z, r->z /*, p256_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001117 /* T2 = X - T1 */
Denys Vlasenkoc7842842021-10-06 01:09:37 +02001118 sp_256_mont_sub_8(t2, r->x, t1 /*, p256_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001119 /* T1 = X + T1 */
Denys Vlasenkoc7842842021-10-06 01:09:37 +02001120 sp_256_mont_add_8(t1, r->x, t1 /*, p256_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001121 /* T2 = T1 * T2 */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001122 sp_256to512z_mont_mul_8(t2, t1, t2 /*, p256_mod, p256_mp_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001123 /* T1 = 3T2 */
Denys Vlasenkoc7842842021-10-06 01:09:37 +02001124 sp_256_mont_tpl_8(t1, t2 /*, p256_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001125 /* Y = 2Y */
Denys Vlasenkoc7842842021-10-06 01:09:37 +02001126 sp_256_mont_dbl_8(r->y, r->y /*, p256_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001127 /* Y = Y * Y */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001128 sp_256to512z_mont_sqr_8(r->y, r->y /*, p256_mod, p256_mp_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001129 /* T2 = Y * Y */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001130 sp_256to512z_mont_sqr_8(t2, r->y /*, p256_mod, p256_mp_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001131 /* T2 = T2/2 */
Denys Vlasenkodcfd8d32021-11-27 16:07:42 +01001132 sp_256_div2_8(t2 /*, p256_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001133 /* Y = Y * X */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001134 sp_256to512z_mont_mul_8(r->y, r->y, r->x /*, p256_mod, p256_mp_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001135 /* X = T1 * T1 */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001136 sp_256to512z_mont_mul_8(r->x, t1, t1 /*, p256_mod, p256_mp_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001137 /* X = X - Y */
Denys Vlasenkoc7842842021-10-06 01:09:37 +02001138 sp_256_mont_sub_8(r->x, r->x, r->y /*, p256_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001139 /* X = X - Y */
Denys Vlasenkoc7842842021-10-06 01:09:37 +02001140 sp_256_mont_sub_8(r->x, r->x, r->y /*, p256_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001141 /* Y = Y - X */
Denys Vlasenkoc7842842021-10-06 01:09:37 +02001142 sp_256_mont_sub_8(r->y, r->y, r->x /*, p256_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001143 /* Y = Y * T1 */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001144 sp_256to512z_mont_mul_8(r->y, r->y, t1 /*, p256_mod, p256_mp_mod*/);
Denys Vlasenko12040122021-04-26 20:24:34 +02001145 /* Y = Y - T2 */
Denys Vlasenkoc7842842021-10-06 01:09:37 +02001146 sp_256_mont_sub_8(r->y, r->y, t2 /*, p256_mod*/);
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001147 dump_512("y2 %s\n", r->y);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001148}
1149
1150/* Add two Montgomery form projective points.
1151 *
1152 * r Result of addition.
1153 * p Frist point to add.
1154 * q Second point to add.
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001155 */
Denys Vlasenko53b2fdc2021-10-10 13:50:53 +02001156static NOINLINE void sp_256_proj_point_add_8(sp_point* r, sp_point* p, sp_point* q)
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001157{
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001158 sp_digit t1[2*8];
1159 sp_digit t2[2*8];
1160 sp_digit t3[2*8];
1161 sp_digit t4[2*8];
1162 sp_digit t5[2*8];
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001163
Denys Vlasenko12040122021-04-26 20:24:34 +02001164 /* Ensure only the first point is the same as the result. */
1165 if (q == r) {
1166 sp_point* a = p;
1167 p = q;
1168 q = a;
1169 }
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001170
Denys Vlasenko12040122021-04-26 20:24:34 +02001171 /* Check double */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001172 sp_256_sub_8(t1, p256_mod, q->y);
1173 sp_256_norm_8(t1);
1174 if (sp_256_cmp_equal_8(p->x, q->x)
1175 && sp_256_cmp_equal_8(p->z, q->z)
1176 && (sp_256_cmp_equal_8(p->y, q->y) || sp_256_cmp_equal_8(p->y, t1))
Denys Vlasenko12040122021-04-26 20:24:34 +02001177 ) {
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001178 sp_256_proj_point_dbl_8(r, p);
Denys Vlasenkobbda85c2021-11-27 15:06:57 +01001179 return;
Denys Vlasenko12040122021-04-26 20:24:34 +02001180 }
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001181
Denys Vlasenko772e1872021-04-26 17:25:27 +02001182
Denys Vlasenkobbda85c2021-11-27 15:06:57 +01001183 if (p->infinity || q->infinity) {
Denys Vlasenko12040122021-04-26 20:24:34 +02001184 *r = p->infinity ? *q : *p; /* struct copy */
Denys Vlasenkobbda85c2021-11-27 15:06:57 +01001185 return;
Denys Vlasenko12040122021-04-26 20:24:34 +02001186 }
Denys Vlasenkobbda85c2021-11-27 15:06:57 +01001187
1188 /* U1 = X1*Z2^2 */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001189 sp_256to512z_mont_sqr_8(t1, q->z /*, p256_mod, p256_mp_mod*/);
1190 sp_256to512z_mont_mul_8(t3, t1, q->z /*, p256_mod, p256_mp_mod*/);
1191 sp_256to512z_mont_mul_8(t1, t1, r->x /*, p256_mod, p256_mp_mod*/);
Denys Vlasenkobbda85c2021-11-27 15:06:57 +01001192 /* U2 = X2*Z1^2 */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001193 sp_256to512z_mont_sqr_8(t2, r->z /*, p256_mod, p256_mp_mod*/);
1194 sp_256to512z_mont_mul_8(t4, t2, r->z /*, p256_mod, p256_mp_mod*/);
1195 sp_256to512z_mont_mul_8(t2, t2, q->x /*, p256_mod, p256_mp_mod*/);
Denys Vlasenkobbda85c2021-11-27 15:06:57 +01001196 /* S1 = Y1*Z2^3 */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001197 sp_256to512z_mont_mul_8(t3, t3, r->y /*, p256_mod, p256_mp_mod*/);
Denys Vlasenkobbda85c2021-11-27 15:06:57 +01001198 /* S2 = Y2*Z1^3 */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001199 sp_256to512z_mont_mul_8(t4, t4, q->y /*, p256_mod, p256_mp_mod*/);
Denys Vlasenkobbda85c2021-11-27 15:06:57 +01001200 /* H = U2 - U1 */
1201 sp_256_mont_sub_8(t2, t2, t1 /*, p256_mod*/);
1202 /* R = S2 - S1 */
1203 sp_256_mont_sub_8(t4, t4, t3 /*, p256_mod*/);
1204 /* Z3 = H*Z1*Z2 */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001205 sp_256to512z_mont_mul_8(r->z, r->z, q->z /*, p256_mod, p256_mp_mod*/);
1206 sp_256to512z_mont_mul_8(r->z, r->z, t2 /*, p256_mod, p256_mp_mod*/);
Denys Vlasenkobbda85c2021-11-27 15:06:57 +01001207 /* X3 = R^2 - H^3 - 2*U1*H^2 */
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001208 sp_256to512z_mont_sqr_8(r->x, t4 /*, p256_mod, p256_mp_mod*/);
1209 sp_256to512z_mont_sqr_8(t5, t2 /*, p256_mod, p256_mp_mod*/);
1210 sp_256to512z_mont_mul_8(r->y, t1, t5 /*, p256_mod, p256_mp_mod*/);
1211 sp_256to512z_mont_mul_8(t5, t5, t2 /*, p256_mod, p256_mp_mod*/);
Denys Vlasenkobbda85c2021-11-27 15:06:57 +01001212 sp_256_mont_sub_8(r->x, r->x, t5 /*, p256_mod*/);
1213 sp_256_mont_dbl_8(t1, r->y /*, p256_mod*/);
1214 sp_256_mont_sub_8(r->x, r->x, t1 /*, p256_mod*/);
1215 /* Y3 = R*(U1*H^2 - X3) - S1*H^3 */
1216 sp_256_mont_sub_8(r->y, r->y, r->x /*, p256_mod*/);
Denys Vlasenko4415f7b2021-11-27 15:47:26 +01001217 sp_256to512z_mont_mul_8(r->y, r->y, t4 /*, p256_mod, p256_mp_mod*/);
1218 sp_256to512z_mont_mul_8(t5, t5, t3 /*, p256_mod, p256_mp_mod*/);
Denys Vlasenkobbda85c2021-11-27 15:06:57 +01001219 sp_256_mont_sub_8(r->y, r->y, t5 /*, p256_mod*/);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001220}
1221
1222/* Multiply the point by the scalar and return the result.
1223 * If map is true then convert result to affine co-ordinates.
1224 *
1225 * r Resulting point.
1226 * g Point to multiply.
1227 * k Scalar to multiply by.
Denys Vlasenko03ab2a92021-04-26 14:55:46 +02001228 * map Indicates whether to convert result to affine.
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001229 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001230static void sp_256_ecc_mulmod_8(sp_point* r, const sp_point* g, const sp_digit* k /*, int map*/)
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001231{
Denys Vlasenko12040122021-04-26 20:24:34 +02001232 enum { map = 1 }; /* we always convert result to affine coordinates */
1233 sp_point t[3];
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001234 sp_digit n = n; /* for compiler */
Denys Vlasenko12040122021-04-26 20:24:34 +02001235 int c, y;
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001236
Denys Vlasenko12040122021-04-26 20:24:34 +02001237 memset(t, 0, sizeof(t));
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001238
Denys Vlasenko12040122021-04-26 20:24:34 +02001239 /* t[0] = {0, 0, 1} * norm */
1240 t[0].infinity = 1;
1241 /* t[1] = {g->x, g->y, g->z} * norm */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001242 sp_256_mod_mul_norm_8(t[1].x, g->x);
1243 sp_256_mod_mul_norm_8(t[1].y, g->y);
1244 sp_256_mod_mul_norm_8(t[1].z, g->z);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001245
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001246 /* For every bit, starting from most significant... */
1247 k += 7;
1248 c = 256;
1249 for (;;) {
1250 if ((c & 0x1f) == 0) {
1251 if (c == 0)
Denys Vlasenko12040122021-04-26 20:24:34 +02001252 break;
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001253 n = *k--;
Denys Vlasenko12040122021-04-26 20:24:34 +02001254 }
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001255
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001256 y = (n >> 31);
1257 dbg("y:%d t[%d] = t[0]+t[1]\n", y, y^1);
1258 sp_256_proj_point_add_8(&t[y^1], &t[0], &t[1]);
1259 dump_512("t[0].x %s\n", t[0].x);
1260 dump_512("t[0].y %s\n", t[0].y);
1261 dump_512("t[0].z %s\n", t[0].z);
1262 dump_512("t[1].x %s\n", t[1].x);
1263 dump_512("t[1].y %s\n", t[1].y);
1264 dump_512("t[1].z %s\n", t[1].z);
1265 dbg("t[2] = t[%d]\n", y);
Denys Vlasenko26c85222021-11-27 15:00:14 +01001266 t[2] = t[y]; /* struct copy */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001267 dbg("t[2] *= 2\n");
1268 sp_256_proj_point_dbl_8(&t[2], &t[2]);
1269 dump_512("t[2].x %s\n", t[2].x);
1270 dump_512("t[2].y %s\n", t[2].y);
1271 dump_512("t[2].z %s\n", t[2].z);
Denys Vlasenko26c85222021-11-27 15:00:14 +01001272 t[y] = t[2]; /* struct copy */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001273
1274 n <<= 1;
1275 c--;
Denys Vlasenko12040122021-04-26 20:24:34 +02001276 }
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001277
Denys Vlasenko12040122021-04-26 20:24:34 +02001278 if (map)
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001279 sp_256_map_8(r, &t[0]);
Denys Vlasenko12040122021-04-26 20:24:34 +02001280 else
Denys Vlasenko9c671fe2021-11-27 18:42:27 +01001281 *r = t[0]; /* struct copy */
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001282
Denys Vlasenko12040122021-04-26 20:24:34 +02001283 memset(t, 0, sizeof(t)); //paranoia
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001284}
1285
1286/* Multiply the base point of P256 by the scalar and return the result.
1287 * If map is true then convert result to affine co-ordinates.
1288 *
1289 * r Resulting point.
1290 * k Scalar to multiply by.
Denys Vlasenko03ab2a92021-04-26 14:55:46 +02001291 * map Indicates whether to convert result to affine.
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001292 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001293static void sp_256_ecc_mulmod_base_8(sp_point* r, sp_digit* k /*, int map*/)
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001294{
Denys Vlasenko39a3ef52021-04-27 01:31:51 +02001295 /* Since this function is called only once, save space:
1296 * don't have "static const sp_point p256_base = {...}",
1297 * it would have more zeros than data.
1298 */
Denys Vlasenko48a18d12021-04-27 12:24:21 +02001299 static const uint8_t p256_base_bin[] = {
1300 /* x (big-endian) */
1301 0x6b,0x17,0xd1,0xf2,0xe1,0x2c,0x42,0x47,0xf8,0xbc,0xe6,0xe5,0x63,0xa4,0x40,0xf2,0x77,0x03,0x7d,0x81,0x2d,0xeb,0x33,0xa0,0xf4,0xa1,0x39,0x45,0xd8,0x98,0xc2,0x96,
1302 /* y */
1303 0x4f,0xe3,0x42,0xe2,0xfe,0x1a,0x7f,0x9b,0x8e,0xe7,0xeb,0x4a,0x7c,0x0f,0x9e,0x16,0x2b,0xce,0x33,0x57,0x6b,0x31,0x5e,0xce,0xcb,0xb6,0x40,0x68,0x37,0xbf,0x51,0xf5,
Denys Vlasenko646e8562021-04-27 13:09:44 +02001304 /* z will be set to 1, infinity flag to "false" */
Denys Vlasenko39a3ef52021-04-27 01:31:51 +02001305 };
1306 sp_point p256_base;
1307
Denys Vlasenko48a18d12021-04-27 12:24:21 +02001308 sp_256_point_from_bin2x32(&p256_base, p256_base_bin);
Denys Vlasenko39a3ef52021-04-27 01:31:51 +02001309
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001310 sp_256_ecc_mulmod_8(r, &p256_base, k /*, map*/);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001311}
1312
1313/* Multiply the point by the scalar and serialize the X ordinate.
1314 * The number is 0 padded to maximum size on output.
1315 *
1316 * priv Scalar to multiply the point by.
Denys Vlasenko074b33b2021-04-26 14:33:38 +02001317 * pub2x32 Point to multiply.
1318 * out32 Buffer to hold X ordinate.
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001319 */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001320static void sp_ecc_secret_gen_256(const sp_digit priv[8], const uint8_t *pub2x32, uint8_t* out32)
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001321{
Denys Vlasenko12040122021-04-26 20:24:34 +02001322 sp_point point[1];
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001323
1324#if FIXED_PEER_PUBKEY
Denys Vlasenko12040122021-04-26 20:24:34 +02001325 memset((void*)pub2x32, 0x55, 64);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001326#endif
Denys Vlasenko12040122021-04-26 20:24:34 +02001327 dump_hex("peerkey %s\n", pub2x32, 32); /* in TLS, this is peer's public key */
1328 dump_hex(" %s\n", pub2x32 + 32, 32);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001329
Denys Vlasenko12040122021-04-26 20:24:34 +02001330 sp_256_point_from_bin2x32(point, pub2x32);
Denys Vlasenko81d8af12021-10-05 17:31:33 +02001331 dump_512("point->x %s\n", point->x);
1332 dump_512("point->y %s\n", point->y);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001333
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001334 sp_256_ecc_mulmod_8(point, point, priv);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001335
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001336 sp_256_to_bin_8(point->x, out32);
Denys Vlasenko12040122021-04-26 20:24:34 +02001337 dump_hex("out32: %s\n", out32, 32);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001338}
1339
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001340/* Generates a random scalar in [1..order-1] range. */
1341static void sp_256_ecc_gen_k_8(sp_digit k[8])
Denys Vlasenko074b33b2021-04-26 14:33:38 +02001342{
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001343 /* Since 32-bit words are "dense", no need to use
1344 * sp_256_from_bin_8(k, buf) to convert random stream
1345 * to sp_digit array - just store random bits there directly.
1346 */
1347 tls_get_random(k, 8 * sizeof(k[0]));
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001348#if FIXED_SECRET
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001349 memset(k, 0x77, 8 * sizeof(k[0]));
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001350#endif
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001351
1352// If scalar is too large, try again (pseudo-code)
1353// if (k >= 0xffffffff00000000ffffffffffffffffbce6faada7179e84f3b9cac2fc632551 - 1) // order of P256
1354// goto pick_another_random;
1355// k++; // ensure non-zero
1356 /* Simpler alternative, at the cost of not choosing some valid
1357 * random values, and slightly non-uniform distribution */
1358 if (k[0] == 0)
1359 k[0] = 1;
1360 if (k[7] >= 0xffffffff)
1361 k[7] = 0xfffffffe;
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001362}
1363
Denys Vlasenko074b33b2021-04-26 14:33:38 +02001364/* Makes a random EC key pair. */
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001365static void sp_ecc_make_key_256(sp_digit privkey[8], uint8_t *pubkey)
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001366{
1367 sp_point point[1];
1368
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001369 sp_256_ecc_gen_k_8(privkey);
Denys Vlasenko137864f2021-10-05 13:47:42 +02001370 dump_256("privkey %s\n", privkey);
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001371 sp_256_ecc_mulmod_base_8(point, privkey);
Denys Vlasenko137864f2021-10-05 13:47:42 +02001372 dump_512("point->x %s\n", point->x);
1373 dump_512("point->y %s\n", point->y);
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001374 sp_256_to_bin_8(point->x, pubkey);
1375 sp_256_to_bin_8(point->y, pubkey + 32);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001376
1377 memset(point, 0, sizeof(point)); //paranoia
1378}
1379
1380void FAST_FUNC curve_P256_compute_pubkey_and_premaster(
Denys Vlasenko074b33b2021-04-26 14:33:38 +02001381 uint8_t *pubkey2x32, uint8_t *premaster32,
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001382 const uint8_t *peerkey2x32)
1383{
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001384 sp_digit privkey[8];
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001385
Denys Vlasenko3b411eb2021-10-05 20:00:50 +02001386 dump_hex("peerkey2x32: %s\n", peerkey2x32, 64);
Denys Vlasenko074b33b2021-04-26 14:33:38 +02001387 sp_ecc_make_key_256(privkey, pubkey2x32);
1388 dump_hex("pubkey: %s\n", pubkey2x32, 32);
1389 dump_hex(" %s\n", pubkey2x32 + 32, 32);
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001390
Denys Vlasenko074b33b2021-04-26 14:33:38 +02001391 /* Combine our privkey and peer's public key to generate premaster */
Denys Vlasenkof18a1fd2021-04-26 13:25:56 +02001392 sp_ecc_secret_gen_256(privkey, /*x,y:*/peerkey2x32, premaster32);
1393 dump_hex("premaster: %s\n", premaster32, 32);
1394}