99 * /
1010
1111#include <linux/linkage.h>
12+ #include <asm/assembler.h>
1213
1314 .text
1415 .arch armv8 - a + crypto
1920 * /
2021ENTRY(ce_aes_ccm_auth_data)
2122 ldr w8 , [ x3 ] / * leftover from prev round? * /
22- ld1 {v0.2d } , [ x0 ] / * load mac * /
23+ ld1 {v0.16b } , [ x0 ] / * load mac * /
2324 cbz w8 , 1f
2425 sub w8 , w8 , # 16
2526 eor v1.16b , v1.16b , v1.16b
@@ -31,7 +32,7 @@ ENTRY(ce_aes_ccm_auth_data)
3132 beq 8f / * out of input? * /
3233 cbnz w8 , 0b
3334 eor v0.16b , v0.16b , v1.16b
34- 1 : ld1 {v3.2d } , [ x4 ] / * load first round key * /
35+ 1 : ld1 {v3.16b } , [ x4 ] / * load first round key * /
3536 prfm pldl1strm , [ x1 ]
3637 cmp w5 , # 12 / * which key size? * /
3738 add x6 , x4 , # 16
@@ -41,17 +42,17 @@ ENTRY(ce_aes_ccm_auth_data)
4142 mov v5.16b , v3.16b
4243 b 4f
43442 : mov v4.16b , v3.16b
44- ld1 {v5.2d } , [ x6 ], # 16 / * load 2nd round key * /
45+ ld1 {v5.16b } , [ x6 ], # 16 / * load 2nd round key * /
45463 : aese v0.16b , v4.16b
4647 aesmc v0.16b , v0.16b
47- 4 : ld1 {v3.2d } , [ x6 ], # 16 / * load next round key * /
48+ 4 : ld1 {v3.16b } , [ x6 ], # 16 / * load next round key * /
4849 aese v0.16b , v5.16b
4950 aesmc v0.16b , v0.16b
50- 5 : ld1 {v4.2d } , [ x6 ], # 16 / * load next round key * /
51+ 5 : ld1 {v4.16b } , [ x6 ], # 16 / * load next round key * /
5152 subs w7 , w7 , # 3
5253 aese v0.16b , v3.16b
5354 aesmc v0.16b , v0.16b
54- ld1 {v5.2d } , [ x6 ], # 16 / * load next round key * /
55+ ld1 {v5.16b } , [ x6 ], # 16 / * load next round key * /
5556 bpl 3b
5657 aese v0.16b , v4.16b
5758 subs w2 , w2 , # 16 / * last data? * /
@@ -60,7 +61,7 @@ ENTRY(ce_aes_ccm_auth_data)
6061 ld1 {v1.16b} , [ x1 ], # 16 / * load next input block * /
6162 eor v0.16b , v0.16b , v1.16b / * xor with mac * /
6263 bne 1b
63- 6 : st1 {v0.2d } , [ x0 ] / * store mac * /
64+ 6 : st1 {v0.16b } , [ x0 ] / * store mac * /
6465 beq 10f
6566 adds w2 , w2 , # 16
6667 beq 10f
@@ -79,7 +80,7 @@ ENTRY(ce_aes_ccm_auth_data)
7980 adds w7 , w7 , # 1
8081 bne 9b
8182 eor v0.16b , v0.16b , v1.16b
82- st1 {v0.2d } , [ x0 ]
83+ st1 {v0.16b } , [ x0 ]
838410 : str w8 , [ x3 ]
8485 ret
8586ENDPROC(ce_aes_ccm_auth_data)
@@ -89,27 +90,27 @@ ENDPROC(ce_aes_ccm_auth_data)
8990 * u32 rounds) ;
9091 * /
9192ENTRY(ce_aes_ccm_final)
92- ld1 {v3.2d } , [ x2 ], # 16 / * load first round key * /
93- ld1 {v0.2d } , [ x0 ] / * load mac * /
93+ ld1 {v3.16b } , [ x2 ], # 16 / * load first round key * /
94+ ld1 {v0.16b } , [ x0 ] / * load mac * /
9495 cmp w3 , # 12 / * which key size? * /
9596 sub w3 , w3 , # 2 / * modified # of rounds * /
96- ld1 {v1.2d } , [ x1 ] / * load 1st ctriv * /
97+ ld1 {v1.16b } , [ x1 ] / * load 1st ctriv * /
9798 bmi 0f
9899 bne 3f
99100 mov v5.16b , v3.16b
100101 b 2f
1011020 : mov v4.16b , v3.16b
102- 1 : ld1 {v5.2d } , [ x2 ], # 16 / * load next round key * /
103+ 1 : ld1 {v5.16b } , [ x2 ], # 16 / * load next round key * /
103104 aese v0.16b , v4.16b
104105 aesmc v0.16b , v0.16b
105106 aese v1.16b , v4.16b
106107 aesmc v1.16b , v1.16b
107- 2 : ld1 {v3.2d } , [ x2 ], # 16 / * load next round key * /
108+ 2 : ld1 {v3.16b } , [ x2 ], # 16 / * load next round key * /
108109 aese v0.16b , v5.16b
109110 aesmc v0.16b , v0.16b
110111 aese v1.16b , v5.16b
111112 aesmc v1.16b , v1.16b
112- 3 : ld1 {v4.2d } , [ x2 ], # 16 / * load next round key * /
113+ 3 : ld1 {v4.16b } , [ x2 ], # 16 / * load next round key * /
113114 subs w3 , w3 , # 3
114115 aese v0.16b , v3.16b
115116 aesmc v0.16b , v0.16b
@@ -120,47 +121,47 @@ ENTRY(ce_aes_ccm_final)
120121 aese v1.16b , v4.16b
121122 / * final round key cancels out * /
122123 eor v0.16b , v0.16b , v1.16b / * en - /decrypt the mac * /
123- st1 {v0.2d } , [ x0 ] / * store result * /
124+ st1 {v0.16b } , [ x0 ] / * store result * /
124125 ret
125126ENDPROC(ce_aes_ccm_final)
126127
127128 .macro aes_ccm_do_crypt , enc
128129 ldr x8 , [ x6 , # 8 ] / * load lower ctr * /
129- ld1 {v0.2d } , [ x5 ] / * load mac * /
130- rev x8 , x8 / * keep swabbed ctr in reg * /
130+ ld1 {v0.16b } , [ x5 ] / * load mac * /
131+ CPU_LE( rev x8, x8 ) / * keep swabbed ctr in reg * /
1311320 : / * outer loop * /
132- ld1 {v1.1d } , [ x6 ] / * load upper ctr * /
133+ ld1 {v1.8b } , [ x6 ] / * load upper ctr * /
133134 prfm pldl1strm , [ x1 ]
134135 add x8 , x8 , # 1
135136 rev x9 , x8
136137 cmp w4 , # 12 / * which key size? * /
137138 sub w7 , w4 , # 2 / * get modified # of rounds * /
138139 ins v1.d [ 1 ], x9 / * no carry in lower ctr * /
139- ld1 {v3.2d } , [ x3 ] / * load first round key * /
140+ ld1 {v3.16b } , [ x3 ] / * load first round key * /
140141 add x10 , x3 , # 16
141142 bmi 1f
142143 bne 4f
143144 mov v5.16b , v3.16b
144145 b 3f
1451461 : mov v4.16b , v3.16b
146- ld1 {v5.2d } , [ x10 ], # 16 / * load 2nd round key * /
147+ ld1 {v5.16b } , [ x10 ], # 16 / * load 2nd round key * /
1471482 : / * inner loop : 3 rounds , 2x interleaved * /
148149 aese v0.16b , v4.16b
149150 aesmc v0.16b , v0.16b
150151 aese v1.16b , v4.16b
151152 aesmc v1.16b , v1.16b
152- 3 : ld1 {v3.2d } , [ x10 ], # 16 / * load next round key * /
153+ 3 : ld1 {v3.16b } , [ x10 ], # 16 / * load next round key * /
153154 aese v0.16b , v5.16b
154155 aesmc v0.16b , v0.16b
155156 aese v1.16b , v5.16b
156157 aesmc v1.16b , v1.16b
157- 4 : ld1 {v4.2d } , [ x10 ], # 16 / * load next round key * /
158+ 4 : ld1 {v4.16b } , [ x10 ], # 16 / * load next round key * /
158159 subs w7 , w7 , # 3
159160 aese v0.16b , v3.16b
160161 aesmc v0.16b , v0.16b
161162 aese v1.16b , v3.16b
162163 aesmc v1.16b , v1.16b
163- ld1 {v5.2d } , [ x10 ], # 16 / * load next round key * /
164+ ld1 {v5.16b } , [ x10 ], # 16 / * load next round key * /
164165 bpl 2b
165166 aese v0.16b , v4.16b
166167 aese v1.16b , v4.16b
@@ -177,14 +178,14 @@ ENDPROC(ce_aes_ccm_final)
177178 eor v0.16b , v0.16b , v2.16b / * xor mac with pt ^ rk [ last ] * /
178179 st1 {v1.16b} , [ x0 ], # 16 / * write output block * /
179180 bne 0b
180- rev x8 , x8
181- st1 {v0.2d } , [ x5 ] / * store mac * /
181+ CPU_LE( rev x8, x8 )
182+ st1 {v0.16b } , [ x5 ] / * store mac * /
182183 str x8 , [ x6 , # 8 ] / * store lsb end of ctr (BE) * /
1831845 : ret
184185
1851866 : eor v0.16b , v0.16b , v5.16b / * final round mac * /
186187 eor v1.16b , v1.16b , v5.16b / * final round enc * /
187- st1 {v0.2d } , [ x5 ] / * store mac * /
188+ st1 {v0.16b } , [ x5 ] / * store mac * /
188189 add w2 , w2 , # 16 / * process partial tail block * /
1891907 : ldrb w9 , [ x1 ], # 1 / * get 1 byte of input * /
190191 umov w6 , v1.b [ 0 ] / * get top crypted ctr byte * /
0 commit comments