net/wireguard/crypto/zinc/chacha20/chacha20-unrolled-arm.S - codeaurora/cp-linux - Gitiles

 /* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Copyright (C) 2018 Google, Inc.
  */

 #include <linux/linkage.h>
 #include <asm/assembler.h>

 /*
  * Design notes:
  *
  * 16 registers would be needed to hold the state matrix, but only 14 are
  * available because 'sp' and 'pc' cannot be used.  So we spill the elements
  * (x8, x9) to the stack and swap them out with (x10, x11).  This adds one
  * 'ldrd' and one 'strd' instruction per round.
  *
  * All rotates are performed using the implicit rotate operand accepted by the
  * 'add' and 'eor' instructions.  This is faster than using explicit rotate
  * instructions.  To make this work, we allow the values in the second and last
  * rows of the ChaCha state matrix (rows 'b' and 'd') to temporarily have the
  * wrong rotation amount.  The rotation amount is then fixed up just in time
  * when the values are used.  'brot' is the number of bits the values in row 'b'
  * need to be rotated right to arrive at the correct values, and 'drot'
  * similarly for row 'd'.  (brot, drot) start out as (0, 0) but we make it such
  * that they end up as (25, 24) after every round.
  */

 	// ChaCha state registers
 	X0	.req	r0
 	X1	.req	r1
 	X2	.req	r2
 	X3	.req	r3
 	X4	.req	r4
 	X5	.req	r5
 	X6	.req	r6
 	X7	.req	r7
 	X8_X10	.req	r8	// shared by x8 and x10
 	X9_X11	.req	r9	// shared by x9 and x11
 	X12	.req	r10
 	X13	.req	r11
 	X14	.req	r12
 	X15	.req	r14

 .Lexpand_32byte_k:
 	// "expand 32-byte k"
 	.word	0x61707865, 0x3320646e, 0x79622d32, 0x6b206574

 #ifdef __thumb2__
 #  define adrl adr
 #endif

 .macro __rev		out, in,  t0, t1, t2
 .if __LINUX_ARM_ARCH__ >= 6
 	rev		\out, \in
 .else
 	lsl		\t0, \in, #24
 	and		\t1, \in, #0xff00
 	and		\t2, \in, #0xff0000
 	orr		\out, \t0, \in, lsr #24
 	orr		\out, \out, \t1, lsl #8
 	orr		\out, \out, \t2, lsr #8
 .endif
 .endm

 .macro _le32_bswap	x,  t0, t1, t2
 #ifdef __ARMEB__
 	__rev		\x, \x,  \t0, \t1, \t2
 #endif
 .endm

 .macro _le32_bswap_4x	a, b, c, d,  t0, t1, t2
 	_le32_bswap	\a,  \t0, \t1, \t2
 	_le32_bswap	\b,  \t0, \t1, \t2
 	_le32_bswap	\c,  \t0, \t1, \t2
 	_le32_bswap	\d,  \t0, \t1, \t2
 .endm

 .macro __ldrd		a, b, src, offset
 #if __LINUX_ARM_ARCH__ >= 6
 	ldrd		\a, \b, [\src, #\offset]
 #else
 	ldr		\a, [\src, #\offset]
 	ldr		\b, [\src, #\offset + 4]
 #endif
 .endm

 .macro __strd		a, b, dst, offset
 #if __LINUX_ARM_ARCH__ >= 6
 	strd		\a, \b, [\dst, #\offset]
 #else
 	str		\a, [\dst, #\offset]
 	str		\b, [\dst, #\offset + 4]
 #endif
 .endm

 .macro _halfround	a1, b1, c1, d1,  a2, b2, c2, d2

 	// a += b; d ^= a; d = rol(d, 16);
 	add		\a1, \a1, \b1, ror #brot
 	add		\a2, \a2, \b2, ror #brot
 	eor		\d1, \a1, \d1, ror #drot
 	eor		\d2, \a2, \d2, ror #drot
 	// drot == 32 - 16 == 16

 	// c += d; b ^= c; b = rol(b, 12);
 	add		\c1, \c1, \d1, ror #16
 	add		\c2, \c2, \d2, ror #16
 	eor		\b1, \c1, \b1, ror #brot
 	eor		\b2, \c2, \b2, ror #brot
 	// brot == 32 - 12 == 20

 	// a += b; d ^= a; d = rol(d, 8);
 	add		\a1, \a1, \b1, ror #20
 	add		\a2, \a2, \b2, ror #20
 	eor		\d1, \a1, \d1, ror #16
 	eor		\d2, \a2, \d2, ror #16
 	// drot == 32 - 8 == 24

 	// c += d; b ^= c; b = rol(b, 7);
 	add		\c1, \c1, \d1, ror #24
 	add		\c2, \c2, \d2, ror #24
 	eor		\b1, \c1, \b1, ror #20
 	eor		\b2, \c2, \b2, ror #20
 	// brot == 32 - 7 == 25
 .endm

 .macro _doubleround

 	// column round

 	// quarterrounds: (x0, x4, x8, x12) and (x1, x5, x9, x13)
 	_halfround	X0, X4, X8_X10, X12,  X1, X5, X9_X11, X13

 	// save (x8, x9); restore (x10, x11)
 	__strd		X8_X10, X9_X11, sp, 0
 	__ldrd		X8_X10, X9_X11, sp, 8

 	// quarterrounds: (x2, x6, x10, x14) and (x3, x7, x11, x15)
 	_halfround	X2, X6, X8_X10, X14,  X3, X7, X9_X11, X15

 	.set brot, 25
 	.set drot, 24

 	// diagonal round

 	// quarterrounds: (x0, x5, x10, x15) and (x1, x6, x11, x12)
 	_halfround	X0, X5, X8_X10, X15,  X1, X6, X9_X11, X12

 	// save (x10, x11); restore (x8, x9)
 	__strd		X8_X10, X9_X11, sp, 8
 	__ldrd		X8_X10, X9_X11, sp, 0

 	// quarterrounds: (x2, x7, x8, x13) and (x3, x4, x9, x14)
 	_halfround	X2, X7, X8_X10, X13,  X3, X4, X9_X11, X14
 .endm

 .macro _chacha_permute	nrounds
 	.set brot, 0
 	.set drot, 0
 	.rept \nrounds / 2
 	 _doubleround
 	.endr
 .endm

 .macro _chacha		nrounds

 .Lnext_block\@:
 	// Stack: unused0-unused1 x10-x11 x0-x15 OUT IN LEN
 	// Registers contain x0-x9,x12-x15.

 	// Do the core ChaCha permutation to update x0-x15.
 	_chacha_permute	\nrounds

 	add		sp, #8
 	// Stack: x10-x11 orig_x0-orig_x15 OUT IN LEN
 	// Registers contain x0-x9,x12-x15.
 	// x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'.

 	// Free up some registers (r8-r12,r14) by pushing (x8-x9,x12-x15).
 	push		{X8_X10, X9_X11, X12, X13, X14, X15}

 	// Load (OUT, IN, LEN).
 	ldr		r14, [sp, #96]
 	ldr		r12, [sp, #100]
 	ldr		r11, [sp, #104]

 	orr		r10, r14, r12

 	// Use slow path if fewer than 64 bytes remain.
 	cmp		r11, #64
 	blt		.Lxor_slowpath\@

 	// Use slow path if IN and/or OUT isn't 4-byte aligned.  Needed even on
 	// ARMv6+, since ldmia and stmia (used below) still require alignment.
 	tst		r10, #3
 	bne		.Lxor_slowpath\@

 	// Fast path: XOR 64 bytes of aligned data.

 	// Stack: x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN
 	// Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is OUT.
 	// x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'.

 	// x0-x3
 	__ldrd		r8, r9, sp, 32
 	__ldrd		r10, r11, sp, 40
 	add		X0, X0, r8
 	add		X1, X1, r9
 	add		X2, X2, r10
 	add		X3, X3, r11
 	_le32_bswap_4x	X0, X1, X2, X3,  r8, r9, r10
 	ldmia		r12!, {r8-r11}
 	eor		X0, X0, r8
 	eor		X1, X1, r9
 	eor		X2, X2, r10
 	eor		X3, X3, r11
 	stmia		r14!, {X0-X3}

 	// x4-x7
 	__ldrd		r8, r9, sp, 48
 	__ldrd		r10, r11, sp, 56
 	add		X4, r8, X4, ror #brot
 	add		X5, r9, X5, ror #brot
 	ldmia		r12!, {X0-X3}
 	add		X6, r10, X6, ror #brot
 	add		X7, r11, X7, ror #brot
 	_le32_bswap_4x	X4, X5, X6, X7,  r8, r9, r10
 	eor		X4, X4, X0
 	eor		X5, X5, X1
 	eor		X6, X6, X2
 	eor		X7, X7, X3
 	stmia		r14!, {X4-X7}

 	// x8-x15
 	pop		{r0-r7}			// (x8-x9,x12-x15,x10-x11)
 	__ldrd		r8, r9, sp, 32
 	__ldrd		r10, r11, sp, 40
 	add		r0, r0, r8		// x8
 	add		r1, r1, r9		// x9
 	add		r6, r6, r10		// x10
 	add		r7, r7, r11		// x11
 	_le32_bswap_4x	r0, r1, r6, r7,  r8, r9, r10
 	ldmia		r12!, {r8-r11}
 	eor		r0, r0, r8		// x8
 	eor		r1, r1, r9		// x9
 	eor		r6, r6, r10		// x10
 	eor		r7, r7, r11		// x11
 	stmia		r14!, {r0,r1,r6,r7}
 	ldmia		r12!, {r0,r1,r6,r7}
 	__ldrd		r8, r9, sp, 48
 	__ldrd		r10, r11, sp, 56
 	add		r2, r8, r2, ror #drot	// x12
 	add		r3, r9, r3, ror #drot	// x13
 	add		r4, r10, r4, ror #drot	// x14
 	add		r5, r11, r5, ror #drot	// x15
 	_le32_bswap_4x	r2, r3, r4, r5,  r9, r10, r11
 	  ldr		r9, [sp, #72]		// load LEN
 	eor		r2, r2, r0		// x12
 	eor		r3, r3, r1		// x13
 	eor		r4, r4, r6		// x14
 	eor		r5, r5, r7		// x15
 	  subs		r9, #64			// decrement and check LEN
 	stmia		r14!, {r2-r5}

 	beq		.Ldone\@

 .Lprepare_for_next_block\@:

 	// Stack: x0-x15 OUT IN LEN

 	// Increment block counter (x12)
 	add		r8, #1

 	// Store updated (OUT, IN, LEN)
 	str		r14, [sp, #64]
 	str		r12, [sp, #68]
 	str		r9, [sp, #72]

 	  mov		r14, sp

 	// Store updated block counter (x12)
 	str		r8, [sp, #48]

 	  sub		sp, #16

 	// Reload state and do next block
 	ldmia		r14!, {r0-r11}		// load x0-x11
 	__strd		r10, r11, sp, 8		// store x10-x11 before state
 	ldmia		r14, {r10-r12,r14}	// load x12-x15
 	b		.Lnext_block\@

 .Lxor_slowpath\@:
 	// Slow path: < 64 bytes remaining, or unaligned input or output buffer.
 	// We handle it by storing the 64 bytes of keystream to the stack, then
 	// XOR-ing the needed portion with the data.

 	// Allocate keystream buffer
 	sub		sp, #64
 	mov		r14, sp

 	// Stack: ks0-ks15 x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN
 	// Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is &ks0.
 	// x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'.

 	// Save keystream for x0-x3
 	__ldrd		r8, r9, sp, 96
 	__ldrd		r10, r11, sp, 104
 	add		X0, X0, r8
 	add		X1, X1, r9
 	add		X2, X2, r10
 	add		X3, X3, r11
 	_le32_bswap_4x	X0, X1, X2, X3,  r8, r9, r10
 	stmia		r14!, {X0-X3}

 	// Save keystream for x4-x7
 	__ldrd		r8, r9, sp, 112
 	__ldrd		r10, r11, sp, 120
 	add		X4, r8, X4, ror #brot
 	add		X5, r9, X5, ror #brot
 	add		X6, r10, X6, ror #brot
 	add		X7, r11, X7, ror #brot
 	_le32_bswap_4x	X4, X5, X6, X7,  r8, r9, r10
 	  add		r8, sp, #64
 	stmia		r14!, {X4-X7}

 	// Save keystream for x8-x15
 	ldm		r8, {r0-r7}		// (x8-x9,x12-x15,x10-x11)
 	__ldrd		r8, r9, sp, 128
 	__ldrd		r10, r11, sp, 136
 	add		r0, r0, r8		// x8
 	add		r1, r1, r9		// x9
 	add		r6, r6, r10		// x10
 	add		r7, r7, r11		// x11
 	_le32_bswap_4x	r0, r1, r6, r7,  r8, r9, r10
 	stmia		r14!, {r0,r1,r6,r7}
 	__ldrd		r8, r9, sp, 144
 	__ldrd		r10, r11, sp, 152
 	add		r2, r8, r2, ror #drot	// x12
 	add		r3, r9, r3, ror #drot	// x13
 	add		r4, r10, r4, ror #drot	// x14
 	add		r5, r11, r5, ror #drot	// x15
 	_le32_bswap_4x	r2, r3, r4, r5,  r9, r10, r11
 	stmia		r14, {r2-r5}

 	// Stack: ks0-ks15 unused0-unused7 x0-x15 OUT IN LEN
 	// Registers: r8 is block counter, r12 is IN.

 	ldr		r9, [sp, #168]		// LEN
 	ldr		r14, [sp, #160]		// OUT
 	cmp		r9, #64
 	  mov		r0, sp
 	movle		r1, r9
 	movgt		r1, #64
 	// r1 is number of bytes to XOR, in range [1, 64]

 .if __LINUX_ARM_ARCH__ < 6
 	orr		r2, r12, r14
 	tst		r2, #3			// IN or OUT misaligned?
 	bne		.Lxor_next_byte\@
 .endif

 	// XOR a word at a time
 .rept 16
 	subs		r1, #4
 	blt		.Lxor_words_done\@
 	ldr		r2, [r12], #4
 	ldr		r3, [r0], #4
 	eor		r2, r2, r3
 	str		r2, [r14], #4
 .endr
 	b		.Lxor_slowpath_done\@
 .Lxor_words_done\@:
 	ands		r1, r1, #3
 	beq		.Lxor_slowpath_done\@

 	// XOR a byte at a time
 .Lxor_next_byte\@:
 	ldrb		r2, [r12], #1
 	ldrb		r3, [r0], #1
 	eor		r2, r2, r3
 	strb		r2, [r14], #1
 	subs		r1, #1
 	bne		.Lxor_next_byte\@

 .Lxor_slowpath_done\@:
 	subs		r9, #64
 	add		sp, #96
 	bgt		.Lprepare_for_next_block\@

 .Ldone\@:
 .endm	// _chacha

 /*
  * void chacha20_arm(u8 *out, const u8 *in, size_t len, const u32 key[8],
  *		     const u32 iv[4]);
  */
 SYM_FUNC_START(chacha20_arm)
 	cmp		r2, #0			// len == 0?
 	reteq		lr

 	push		{r0-r2,r4-r11,lr}

 	// Push state x0-x15 onto stack.
 	// Also store an extra copy of x10-x11 just before the state.

 	ldr		r4, [sp, #48]		// iv
 	mov		r0, sp
 	sub		sp, #80

 	// iv: x12-x15
 	ldm		r4, {X12,X13,X14,X15}
 	stmdb		r0!, {X12,X13,X14,X15}

 	// key: x4-x11
 	__ldrd		X8_X10, X9_X11, r3, 24
 	__strd		X8_X10, X9_X11, sp, 8
 	stmdb		r0!, {X8_X10, X9_X11}
 	ldm		r3, {X4-X9_X11}
 	stmdb		r0!, {X4-X9_X11}

 	// constants: x0-x3
 	adrl		X3, .Lexpand_32byte_k
 	ldm		X3, {X0-X3}
 	__strd		X0, X1, sp, 16
 	__strd		X2, X3, sp, 24

 	_chacha		20

 	add		sp, #76
 	pop		{r4-r11, pc}
 SYM_FUNC_END(chacha20_arm)

 /*
  * void hchacha20_arm(const u32 state[16], u32 out[8]);
  */
 SYM_FUNC_START(hchacha20_arm)
 	push		{r1,r4-r11,lr}

 	mov		r14, r0
 	ldmia		r14!, {r0-r11}		// load x0-x11
 	push		{r10-r11}		// store x10-x11 to stack
 	ldm		r14, {r10-r12,r14}	// load x12-x15
 	sub		sp, #8

 	_chacha_permute	20

 	// Skip over (unused0-unused1, x10-x11)
 	add		sp, #16

 	// Fix up rotations of x12-x15
 	ror		X12, X12, #drot
 	ror		X13, X13, #drot
 	  pop		{r4}			// load 'out'
 	ror		X14, X14, #drot
 	ror		X15, X15, #drot

 	// Store (x0-x3,x12-x15) to 'out'
 	stm		r4, {X0,X1,X2,X3,X12,X13,X14,X15}

 	pop		{r4-r11,pc}
 SYM_FUNC_END(hchacha20_arm)
	/* SPDX-License-Identifier: GPL-2.0 */
	/*
	* Copyright (C) 2018 Google, Inc.
	*/

	#include <linux/linkage.h>
	#include <asm/assembler.h>

	/*
	* Design notes:
	*
	* 16 registers would be needed to hold the state matrix, but only 14 are
	* available because 'sp' and 'pc' cannot be used. So we spill the elements
	* (x8, x9) to the stack and swap them out with (x10, x11). This adds one
	* 'ldrd' and one 'strd' instruction per round.
	*
	* All rotates are performed using the implicit rotate operand accepted by the
	* 'add' and 'eor' instructions. This is faster than using explicit rotate
	* instructions. To make this work, we allow the values in the second and last
	* rows of the ChaCha state matrix (rows 'b' and 'd') to temporarily have the
	* wrong rotation amount. The rotation amount is then fixed up just in time
	* when the values are used. 'brot' is the number of bits the values in row 'b'
	* need to be rotated right to arrive at the correct values, and 'drot'
	* similarly for row 'd'. (brot, drot) start out as (0, 0) but we make it such
	* that they end up as (25, 24) after every round.
	*/

	// ChaCha state registers
	X0 .req r0
	X1 .req r1
	X2 .req r2
	X3 .req r3
	X4 .req r4
	X5 .req r5
	X6 .req r6
	X7 .req r7
	X8_X10 .req r8 // shared by x8 and x10
	X9_X11 .req r9 // shared by x9 and x11
	X12 .req r10
	X13 .req r11
	X14 .req r12
	X15 .req r14

	.Lexpand_32byte_k:
	// "expand 32-byte k"
	.word 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574

	#ifdef __thumb2__
	# define adrl adr
	#endif

	.macro __rev out, in, t0, t1, t2
	.if __LINUX_ARM_ARCH__ >= 6
	rev \out, \in
	.else
	lsl \t0, \in, #24
	and \t1, \in, #0xff00
	and \t2, \in, #0xff0000
	orr \out, \t0, \in, lsr #24
	orr \out, \out, \t1, lsl #8
	orr \out, \out, \t2, lsr #8
	.endif
	.endm

	.macro _le32_bswap x, t0, t1, t2
	#ifdef __ARMEB__
	__rev \x, \x, \t0, \t1, \t2
	#endif
	.endm

	.macro _le32_bswap_4x a, b, c, d, t0, t1, t2
	_le32_bswap \a, \t0, \t1, \t2
	_le32_bswap \b, \t0, \t1, \t2
	_le32_bswap \c, \t0, \t1, \t2
	_le32_bswap \d, \t0, \t1, \t2
	.endm

	.macro __ldrd a, b, src, offset
	#if __LINUX_ARM_ARCH__ >= 6
	ldrd \a, \b, [\src, #\offset]
	#else
	ldr \a, [\src, #\offset]
	ldr \b, [\src, #\offset + 4]
	#endif
	.endm

	.macro __strd a, b, dst, offset
	#if __LINUX_ARM_ARCH__ >= 6
	strd \a, \b, [\dst, #\offset]
	#else
	str \a, [\dst, #\offset]
	str \b, [\dst, #\offset + 4]
	#endif
	.endm

	.macro _halfround a1, b1, c1, d1, a2, b2, c2, d2

	// a += b; d ^= a; d = rol(d, 16);
	add \a1, \a1, \b1, ror #brot
	add \a2, \a2, \b2, ror #brot
	eor \d1, \a1, \d1, ror #drot
	eor \d2, \a2, \d2, ror #drot
	// drot == 32 - 16 == 16

	// c += d; b ^= c; b = rol(b, 12);
	add \c1, \c1, \d1, ror #16
	add \c2, \c2, \d2, ror #16
	eor \b1, \c1, \b1, ror #brot
	eor \b2, \c2, \b2, ror #brot
	// brot == 32 - 12 == 20

	// a += b; d ^= a; d = rol(d, 8);
	add \a1, \a1, \b1, ror #20
	add \a2, \a2, \b2, ror #20
	eor \d1, \a1, \d1, ror #16
	eor \d2, \a2, \d2, ror #16
	// drot == 32 - 8 == 24

	// c += d; b ^= c; b = rol(b, 7);
	add \c1, \c1, \d1, ror #24
	add \c2, \c2, \d2, ror #24
	eor \b1, \c1, \b1, ror #20
	eor \b2, \c2, \b2, ror #20
	// brot == 32 - 7 == 25
	.endm

	.macro _doubleround

	// column round

	// quarterrounds: (x0, x4, x8, x12) and (x1, x5, x9, x13)
	_halfround X0, X4, X8_X10, X12, X1, X5, X9_X11, X13

	// save (x8, x9); restore (x10, x11)
	__strd X8_X10, X9_X11, sp, 0
	__ldrd X8_X10, X9_X11, sp, 8

	// quarterrounds: (x2, x6, x10, x14) and (x3, x7, x11, x15)
	_halfround X2, X6, X8_X10, X14, X3, X7, X9_X11, X15

	.set brot, 25
	.set drot, 24

	// diagonal round

	// quarterrounds: (x0, x5, x10, x15) and (x1, x6, x11, x12)
	_halfround X0, X5, X8_X10, X15, X1, X6, X9_X11, X12

	// save (x10, x11); restore (x8, x9)
	__strd X8_X10, X9_X11, sp, 8
	__ldrd X8_X10, X9_X11, sp, 0

	// quarterrounds: (x2, x7, x8, x13) and (x3, x4, x9, x14)
	_halfround X2, X7, X8_X10, X13, X3, X4, X9_X11, X14
	.endm

	.macro _chacha_permute nrounds
	.set brot, 0
	.set drot, 0
	.rept \nrounds / 2
	_doubleround
	.endr
	.endm

	.macro _chacha nrounds

	.Lnext_block\@:
	// Stack: unused0-unused1 x10-x11 x0-x15 OUT IN LEN
	// Registers contain x0-x9,x12-x15.

	// Do the core ChaCha permutation to update x0-x15.
	_chacha_permute \nrounds

	add sp, #8
	// Stack: x10-x11 orig_x0-orig_x15 OUT IN LEN
	// Registers contain x0-x9,x12-x15.
	// x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'.

	// Free up some registers (r8-r12,r14) by pushing (x8-x9,x12-x15).
	push {X8_X10, X9_X11, X12, X13, X14, X15}

	// Load (OUT, IN, LEN).
	ldr r14, [sp, #96]
	ldr r12, [sp, #100]
	ldr r11, [sp, #104]

	orr r10, r14, r12

	// Use slow path if fewer than 64 bytes remain.
	cmp r11, #64
	blt .Lxor_slowpath\@

	// Use slow path if IN and/or OUT isn't 4-byte aligned. Needed even on
	// ARMv6+, since ldmia and stmia (used below) still require alignment.
	tst r10, #3
	bne .Lxor_slowpath\@

	// Fast path: XOR 64 bytes of aligned data.

	// Stack: x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN
	// Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is OUT.
	// x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'.

	// x0-x3
	__ldrd r8, r9, sp, 32
	__ldrd r10, r11, sp, 40
	add X0, X0, r8
	add X1, X1, r9
	add X2, X2, r10
	add X3, X3, r11
	_le32_bswap_4x X0, X1, X2, X3, r8, r9, r10
	ldmia r12!, {r8-r11}
	eor X0, X0, r8
	eor X1, X1, r9
	eor X2, X2, r10
	eor X3, X3, r11
	stmia r14!, {X0-X3}

	// x4-x7
	__ldrd r8, r9, sp, 48
	__ldrd r10, r11, sp, 56
	add X4, r8, X4, ror #brot
	add X5, r9, X5, ror #brot
	ldmia r12!, {X0-X3}
	add X6, r10, X6, ror #brot
	add X7, r11, X7, ror #brot
	_le32_bswap_4x X4, X5, X6, X7, r8, r9, r10
	eor X4, X4, X0
	eor X5, X5, X1
	eor X6, X6, X2
	eor X7, X7, X3
	stmia r14!, {X4-X7}

	// x8-x15
	pop {r0-r7} // (x8-x9,x12-x15,x10-x11)
	__ldrd r8, r9, sp, 32
	__ldrd r10, r11, sp, 40
	add r0, r0, r8 // x8
	add r1, r1, r9 // x9
	add r6, r6, r10 // x10
	add r7, r7, r11 // x11
	_le32_bswap_4x r0, r1, r6, r7, r8, r9, r10
	ldmia r12!, {r8-r11}
	eor r0, r0, r8 // x8
	eor r1, r1, r9 // x9
	eor r6, r6, r10 // x10
	eor r7, r7, r11 // x11
	stmia r14!, {r0,r1,r6,r7}
	ldmia r12!, {r0,r1,r6,r7}
	__ldrd r8, r9, sp, 48
	__ldrd r10, r11, sp, 56
	add r2, r8, r2, ror #drot // x12
	add r3, r9, r3, ror #drot // x13
	add r4, r10, r4, ror #drot // x14
	add r5, r11, r5, ror #drot // x15
	_le32_bswap_4x r2, r3, r4, r5, r9, r10, r11
	ldr r9, [sp, #72] // load LEN
	eor r2, r2, r0 // x12
	eor r3, r3, r1 // x13
	eor r4, r4, r6 // x14
	eor r5, r5, r7 // x15
	subs r9, #64 // decrement and check LEN
	stmia r14!, {r2-r5}

	beq .Ldone\@

	.Lprepare_for_next_block\@:

	// Stack: x0-x15 OUT IN LEN

	// Increment block counter (x12)
	add r8, #1

	// Store updated (OUT, IN, LEN)
	str r14, [sp, #64]
	str r12, [sp, #68]
	str r9, [sp, #72]

	mov r14, sp

	// Store updated block counter (x12)
	str r8, [sp, #48]

	sub sp, #16

	// Reload state and do next block
	ldmia r14!, {r0-r11} // load x0-x11
	__strd r10, r11, sp, 8 // store x10-x11 before state
	ldmia r14, {r10-r12,r14} // load x12-x15
	b .Lnext_block\@

	.Lxor_slowpath\@:
	// Slow path: < 64 bytes remaining, or unaligned input or output buffer.
	// We handle it by storing the 64 bytes of keystream to the stack, then
	// XOR-ing the needed portion with the data.

	// Allocate keystream buffer
	sub sp, #64
	mov r14, sp

	// Stack: ks0-ks15 x8-x9 x12-x15 x10-x11 orig_x0-orig_x15 OUT IN LEN
	// Registers: r0-r7 are x0-x7; r8-r11 are free; r12 is IN; r14 is &ks0.
	// x4-x7 are rotated by 'brot'; x12-x15 are rotated by 'drot'.

	// Save keystream for x0-x3
	__ldrd r8, r9, sp, 96
	__ldrd r10, r11, sp, 104
	add X0, X0, r8
	add X1, X1, r9
	add X2, X2, r10
	add X3, X3, r11
	_le32_bswap_4x X0, X1, X2, X3, r8, r9, r10
	stmia r14!, {X0-X3}

	// Save keystream for x4-x7
	__ldrd r8, r9, sp, 112
	__ldrd r10, r11, sp, 120
	add X4, r8, X4, ror #brot
	add X5, r9, X5, ror #brot
	add X6, r10, X6, ror #brot
	add X7, r11, X7, ror #brot
	_le32_bswap_4x X4, X5, X6, X7, r8, r9, r10
	add r8, sp, #64
	stmia r14!, {X4-X7}

	// Save keystream for x8-x15
	ldm r8, {r0-r7} // (x8-x9,x12-x15,x10-x11)
	__ldrd r8, r9, sp, 128
	__ldrd r10, r11, sp, 136
	add r0, r0, r8 // x8
	add r1, r1, r9 // x9
	add r6, r6, r10 // x10
	add r7, r7, r11 // x11
	_le32_bswap_4x r0, r1, r6, r7, r8, r9, r10
	stmia r14!, {r0,r1,r6,r7}
	__ldrd r8, r9, sp, 144
	__ldrd r10, r11, sp, 152
	add r2, r8, r2, ror #drot // x12
	add r3, r9, r3, ror #drot // x13
	add r4, r10, r4, ror #drot // x14
	add r5, r11, r5, ror #drot // x15
	_le32_bswap_4x r2, r3, r4, r5, r9, r10, r11
	stmia r14, {r2-r5}

	// Stack: ks0-ks15 unused0-unused7 x0-x15 OUT IN LEN
	// Registers: r8 is block counter, r12 is IN.

	ldr r9, [sp, #168] // LEN
	ldr r14, [sp, #160] // OUT
	cmp r9, #64
	mov r0, sp
	movle r1, r9
	movgt r1, #64
	// r1 is number of bytes to XOR, in range [1, 64]

	.if __LINUX_ARM_ARCH__ < 6
	orr r2, r12, r14
	tst r2, #3 // IN or OUT misaligned?
	bne .Lxor_next_byte\@
	.endif

	// XOR a word at a time
	.rept 16
	subs r1, #4
	blt .Lxor_words_done\@
	ldr r2, [r12], #4
	ldr r3, [r0], #4
	eor r2, r2, r3
	str r2, [r14], #4
	.endr
	b .Lxor_slowpath_done\@
	.Lxor_words_done\@:
	ands r1, r1, #3
	beq .Lxor_slowpath_done\@

	// XOR a byte at a time
	.Lxor_next_byte\@:
	ldrb r2, [r12], #1
	ldrb r3, [r0], #1
	eor r2, r2, r3
	strb r2, [r14], #1
	subs r1, #1
	bne .Lxor_next_byte\@

	.Lxor_slowpath_done\@:
	subs r9, #64
	add sp, #96
	bgt .Lprepare_for_next_block\@

	.Ldone\@:
	.endm // _chacha

	/*
	* void chacha20_arm(u8 out, const u8 in, size_t len, const u32 key[8],
	* const u32 iv[4]);
	*/
	SYM_FUNC_START(chacha20_arm)
	cmp r2, #0 // len == 0?
	reteq lr

	push {r0-r2,r4-r11,lr}

	// Push state x0-x15 onto stack.
	// Also store an extra copy of x10-x11 just before the state.

	ldr r4, [sp, #48] // iv
	mov r0, sp
	sub sp, #80

	// iv: x12-x15
	ldm r4, {X12,X13,X14,X15}
	stmdb r0!, {X12,X13,X14,X15}

	// key: x4-x11
	__ldrd X8_X10, X9_X11, r3, 24
	__strd X8_X10, X9_X11, sp, 8
	stmdb r0!, {X8_X10, X9_X11}
	ldm r3, {X4-X9_X11}
	stmdb r0!, {X4-X9_X11}

	// constants: x0-x3
	adrl X3, .Lexpand_32byte_k
	ldm X3, {X0-X3}
	__strd X0, X1, sp, 16
	__strd X2, X3, sp, 24

	_chacha 20

	add sp, #76
	pop {r4-r11, pc}
	SYM_FUNC_END(chacha20_arm)

	/*
	* void hchacha20_arm(const u32 state[16], u32 out[8]);
	*/
	SYM_FUNC_START(hchacha20_arm)
	push {r1,r4-r11,lr}

	mov r14, r0
	ldmia r14!, {r0-r11} // load x0-x11
	push {r10-r11} // store x10-x11 to stack
	ldm r14, {r10-r12,r14} // load x12-x15
	sub sp, #8

	_chacha_permute 20

	// Skip over (unused0-unused1, x10-x11)
	add sp, #16

	// Fix up rotations of x12-x15
	ror X12, X12, #drot
	ror X13, X13, #drot
	pop {r4} // load 'out'
	ror X14, X14, #drot
	ror X15, X15, #drot

	// Store (x0-x3,x12-x15) to 'out'
	stm r4, {X0,X1,X2,X3,X12,X13,X14,X15}

	pop {r4-r11,pc}
	SYM_FUNC_END(hchacha20_arm)