1238405Sjkim	# $FreeBSD$
2238405Sjkim.text
3238405Sjkim
4238405Sjkim.globl	gcm_gmult_4bit
5238405Sjkim.type	gcm_gmult_4bit,@function
6238405Sjkim.align	16
7238405Sjkimgcm_gmult_4bit:
8238405Sjkim	pushq	%rbx
9238405Sjkim	pushq	%rbp
10238405Sjkim	pushq	%r12
11238405Sjkim.Lgmult_prologue:
12238405Sjkim
13238405Sjkim	movzbq	15(%rdi),%r8
14238405Sjkim	leaq	.Lrem_4bit(%rip),%r11
15238405Sjkim	xorq	%rax,%rax
16238405Sjkim	xorq	%rbx,%rbx
17238405Sjkim	movb	%r8b,%al
18238405Sjkim	movb	%r8b,%bl
19238405Sjkim	shlb	$4,%al
20238405Sjkim	movq	$14,%rcx
21238405Sjkim	movq	8(%rsi,%rax,1),%r8
22238405Sjkim	movq	(%rsi,%rax,1),%r9
23238405Sjkim	andb	$240,%bl
24238405Sjkim	movq	%r8,%rdx
25238405Sjkim	jmp	.Loop1
26238405Sjkim
27238405Sjkim.align	16
28238405Sjkim.Loop1:
29238405Sjkim	shrq	$4,%r8
30238405Sjkim	andq	$15,%rdx
31238405Sjkim	movq	%r9,%r10
32238405Sjkim	movb	(%rdi,%rcx,1),%al
33238405Sjkim	shrq	$4,%r9
34238405Sjkim	xorq	8(%rsi,%rbx,1),%r8
35238405Sjkim	shlq	$60,%r10
36238405Sjkim	xorq	(%rsi,%rbx,1),%r9
37238405Sjkim	movb	%al,%bl
38238405Sjkim	xorq	(%r11,%rdx,8),%r9
39238405Sjkim	movq	%r8,%rdx
40238405Sjkim	shlb	$4,%al
41238405Sjkim	xorq	%r10,%r8
42238405Sjkim	decq	%rcx
43238405Sjkim	js	.Lbreak1
44238405Sjkim
45238405Sjkim	shrq	$4,%r8
46238405Sjkim	andq	$15,%rdx
47238405Sjkim	movq	%r9,%r10
48238405Sjkim	shrq	$4,%r9
49238405Sjkim	xorq	8(%rsi,%rax,1),%r8
50238405Sjkim	shlq	$60,%r10
51238405Sjkim	xorq	(%rsi,%rax,1),%r9
52238405Sjkim	andb	$240,%bl
53238405Sjkim	xorq	(%r11,%rdx,8),%r9
54238405Sjkim	movq	%r8,%rdx
55238405Sjkim	xorq	%r10,%r8
56238405Sjkim	jmp	.Loop1
57238405Sjkim
58238405Sjkim.align	16
59238405Sjkim.Lbreak1:
60238405Sjkim	shrq	$4,%r8
61238405Sjkim	andq	$15,%rdx
62238405Sjkim	movq	%r9,%r10
63238405Sjkim	shrq	$4,%r9
64238405Sjkim	xorq	8(%rsi,%rax,1),%r8
65238405Sjkim	shlq	$60,%r10
66238405Sjkim	xorq	(%rsi,%rax,1),%r9
67238405Sjkim	andb	$240,%bl
68238405Sjkim	xorq	(%r11,%rdx,8),%r9
69238405Sjkim	movq	%r8,%rdx
70238405Sjkim	xorq	%r10,%r8
71238405Sjkim
72238405Sjkim	shrq	$4,%r8
73238405Sjkim	andq	$15,%rdx
74238405Sjkim	movq	%r9,%r10
75238405Sjkim	shrq	$4,%r9
76238405Sjkim	xorq	8(%rsi,%rbx,1),%r8
77238405Sjkim	shlq	$60,%r10
78238405Sjkim	xorq	(%rsi,%rbx,1),%r9
79238405Sjkim	xorq	%r10,%r8
80238405Sjkim	xorq	(%r11,%rdx,8),%r9
81238405Sjkim
82238405Sjkim	bswapq	%r8
83238405Sjkim	bswapq	%r9
84238405Sjkim	movq	%r8,8(%rdi)
85238405Sjkim	movq	%r9,(%rdi)
86238405Sjkim
87238405Sjkim	movq	16(%rsp),%rbx
88238405Sjkim	leaq	24(%rsp),%rsp
89238405Sjkim.Lgmult_epilogue:
90238405Sjkim	.byte	0xf3,0xc3
91238405Sjkim.size	gcm_gmult_4bit,.-gcm_gmult_4bit
92238405Sjkim.globl	gcm_ghash_4bit
93238405Sjkim.type	gcm_ghash_4bit,@function
94238405Sjkim.align	16
95238405Sjkimgcm_ghash_4bit:
96238405Sjkim	pushq	%rbx
97238405Sjkim	pushq	%rbp
98238405Sjkim	pushq	%r12
99238405Sjkim	pushq	%r13
100238405Sjkim	pushq	%r14
101238405Sjkim	pushq	%r15
102238405Sjkim	subq	$280,%rsp
103238405Sjkim.Lghash_prologue:
104238405Sjkim	movq	%rdx,%r14
105238405Sjkim	movq	%rcx,%r15
106238405Sjkim	subq	$-128,%rsi
107238405Sjkim	leaq	16+128(%rsp),%rbp
108238405Sjkim	xorl	%edx,%edx
109238405Sjkim	movq	0+0-128(%rsi),%r8
110238405Sjkim	movq	0+8-128(%rsi),%rax
111238405Sjkim	movb	%al,%dl
112238405Sjkim	shrq	$4,%rax
113238405Sjkim	movq	%r8,%r10
114238405Sjkim	shrq	$4,%r8
115238405Sjkim	movq	16+0-128(%rsi),%r9
116238405Sjkim	shlb	$4,%dl
117238405Sjkim	movq	16+8-128(%rsi),%rbx
118238405Sjkim	shlq	$60,%r10
119238405Sjkim	movb	%dl,0(%rsp)
120238405Sjkim	orq	%r10,%rax
121238405Sjkim	movb	%bl,%dl
122238405Sjkim	shrq	$4,%rbx
123238405Sjkim	movq	%r9,%r10
124238405Sjkim	shrq	$4,%r9
125238405Sjkim	movq	%r8,0(%rbp)
126238405Sjkim	movq	32+0-128(%rsi),%r8
127238405Sjkim	shlb	$4,%dl
128238405Sjkim	movq	%rax,0-128(%rbp)
129238405Sjkim	movq	32+8-128(%rsi),%rax
130238405Sjkim	shlq	$60,%r10
131238405Sjkim	movb	%dl,1(%rsp)
132238405Sjkim	orq	%r10,%rbx
133238405Sjkim	movb	%al,%dl
134238405Sjkim	shrq	$4,%rax
135238405Sjkim	movq	%r8,%r10
136238405Sjkim	shrq	$4,%r8
137238405Sjkim	movq	%r9,8(%rbp)
138238405Sjkim	movq	48+0-128(%rsi),%r9
139238405Sjkim	shlb	$4,%dl
140238405Sjkim	movq	%rbx,8-128(%rbp)
141238405Sjkim	movq	48+8-128(%rsi),%rbx
142238405Sjkim	shlq	$60,%r10
143238405Sjkim	movb	%dl,2(%rsp)
144238405Sjkim	orq	%r10,%rax
145238405Sjkim	movb	%bl,%dl
146238405Sjkim	shrq	$4,%rbx
147238405Sjkim	movq	%r9,%r10
148238405Sjkim	shrq	$4,%r9
149238405Sjkim	movq	%r8,16(%rbp)
150238405Sjkim	movq	64+0-128(%rsi),%r8
151238405Sjkim	shlb	$4,%dl
152238405Sjkim	movq	%rax,16-128(%rbp)
153238405Sjkim	movq	64+8-128(%rsi),%rax
154238405Sjkim	shlq	$60,%r10
155238405Sjkim	movb	%dl,3(%rsp)
156238405Sjkim	orq	%r10,%rbx
157238405Sjkim	movb	%al,%dl
158238405Sjkim	shrq	$4,%rax
159238405Sjkim	movq	%r8,%r10
160238405Sjkim	shrq	$4,%r8
161238405Sjkim	movq	%r9,24(%rbp)
162238405Sjkim	movq	80+0-128(%rsi),%r9
163238405Sjkim	shlb	$4,%dl
164238405Sjkim	movq	%rbx,24-128(%rbp)
165238405Sjkim	movq	80+8-128(%rsi),%rbx
166238405Sjkim	shlq	$60,%r10
167238405Sjkim	movb	%dl,4(%rsp)
168238405Sjkim	orq	%r10,%rax
169238405Sjkim	movb	%bl,%dl
170238405Sjkim	shrq	$4,%rbx
171238405Sjkim	movq	%r9,%r10
172238405Sjkim	shrq	$4,%r9
173238405Sjkim	movq	%r8,32(%rbp)
174238405Sjkim	movq	96+0-128(%rsi),%r8
175238405Sjkim	shlb	$4,%dl
176238405Sjkim	movq	%rax,32-128(%rbp)
177238405Sjkim	movq	96+8-128(%rsi),%rax
178238405Sjkim	shlq	$60,%r10
179238405Sjkim	movb	%dl,5(%rsp)
180238405Sjkim	orq	%r10,%rbx
181238405Sjkim	movb	%al,%dl
182238405Sjkim	shrq	$4,%rax
183238405Sjkim	movq	%r8,%r10
184238405Sjkim	shrq	$4,%r8
185238405Sjkim	movq	%r9,40(%rbp)
186238405Sjkim	movq	112+0-128(%rsi),%r9
187238405Sjkim	shlb	$4,%dl
188238405Sjkim	movq	%rbx,40-128(%rbp)
189238405Sjkim	movq	112+8-128(%rsi),%rbx
190238405Sjkim	shlq	$60,%r10
191238405Sjkim	movb	%dl,6(%rsp)
192238405Sjkim	orq	%r10,%rax
193238405Sjkim	movb	%bl,%dl
194238405Sjkim	shrq	$4,%rbx
195238405Sjkim	movq	%r9,%r10
196238405Sjkim	shrq	$4,%r9
197238405Sjkim	movq	%r8,48(%rbp)
198238405Sjkim	movq	128+0-128(%rsi),%r8
199238405Sjkim	shlb	$4,%dl
200238405Sjkim	movq	%rax,48-128(%rbp)
201238405Sjkim	movq	128+8-128(%rsi),%rax
202238405Sjkim	shlq	$60,%r10
203238405Sjkim	movb	%dl,7(%rsp)
204238405Sjkim	orq	%r10,%rbx
205238405Sjkim	movb	%al,%dl
206238405Sjkim	shrq	$4,%rax
207238405Sjkim	movq	%r8,%r10
208238405Sjkim	shrq	$4,%r8
209238405Sjkim	movq	%r9,56(%rbp)
210238405Sjkim	movq	144+0-128(%rsi),%r9
211238405Sjkim	shlb	$4,%dl
212238405Sjkim	movq	%rbx,56-128(%rbp)
213238405Sjkim	movq	144+8-128(%rsi),%rbx
214238405Sjkim	shlq	$60,%r10
215238405Sjkim	movb	%dl,8(%rsp)
216238405Sjkim	orq	%r10,%rax
217238405Sjkim	movb	%bl,%dl
218238405Sjkim	shrq	$4,%rbx
219238405Sjkim	movq	%r9,%r10
220238405Sjkim	shrq	$4,%r9
221238405Sjkim	movq	%r8,64(%rbp)
222238405Sjkim	movq	160+0-128(%rsi),%r8
223238405Sjkim	shlb	$4,%dl
224238405Sjkim	movq	%rax,64-128(%rbp)
225238405Sjkim	movq	160+8-128(%rsi),%rax
226238405Sjkim	shlq	$60,%r10
227238405Sjkim	movb	%dl,9(%rsp)
228238405Sjkim	orq	%r10,%rbx
229238405Sjkim	movb	%al,%dl
230238405Sjkim	shrq	$4,%rax
231238405Sjkim	movq	%r8,%r10
232238405Sjkim	shrq	$4,%r8
233238405Sjkim	movq	%r9,72(%rbp)
234238405Sjkim	movq	176+0-128(%rsi),%r9
235238405Sjkim	shlb	$4,%dl
236238405Sjkim	movq	%rbx,72-128(%rbp)
237238405Sjkim	movq	176+8-128(%rsi),%rbx
238238405Sjkim	shlq	$60,%r10
239238405Sjkim	movb	%dl,10(%rsp)
240238405Sjkim	orq	%r10,%rax
241238405Sjkim	movb	%bl,%dl
242238405Sjkim	shrq	$4,%rbx
243238405Sjkim	movq	%r9,%r10
244238405Sjkim	shrq	$4,%r9
245238405Sjkim	movq	%r8,80(%rbp)
246238405Sjkim	movq	192+0-128(%rsi),%r8
247238405Sjkim	shlb	$4,%dl
248238405Sjkim	movq	%rax,80-128(%rbp)
249238405Sjkim	movq	192+8-128(%rsi),%rax
250238405Sjkim	shlq	$60,%r10
251238405Sjkim	movb	%dl,11(%rsp)
252238405Sjkim	orq	%r10,%rbx
253238405Sjkim	movb	%al,%dl
254238405Sjkim	shrq	$4,%rax
255238405Sjkim	movq	%r8,%r10
256238405Sjkim	shrq	$4,%r8
257238405Sjkim	movq	%r9,88(%rbp)
258238405Sjkim	movq	208+0-128(%rsi),%r9
259238405Sjkim	shlb	$4,%dl
260238405Sjkim	movq	%rbx,88-128(%rbp)
261238405Sjkim	movq	208+8-128(%rsi),%rbx
262238405Sjkim	shlq	$60,%r10
263238405Sjkim	movb	%dl,12(%rsp)
264238405Sjkim	orq	%r10,%rax
265238405Sjkim	movb	%bl,%dl
266238405Sjkim	shrq	$4,%rbx
267238405Sjkim	movq	%r9,%r10
268238405Sjkim	shrq	$4,%r9
269238405Sjkim	movq	%r8,96(%rbp)
270238405Sjkim	movq	224+0-128(%rsi),%r8
271238405Sjkim	shlb	$4,%dl
272238405Sjkim	movq	%rax,96-128(%rbp)
273238405Sjkim	movq	224+8-128(%rsi),%rax
274238405Sjkim	shlq	$60,%r10
275238405Sjkim	movb	%dl,13(%rsp)
276238405Sjkim	orq	%r10,%rbx
277238405Sjkim	movb	%al,%dl
278238405Sjkim	shrq	$4,%rax
279238405Sjkim	movq	%r8,%r10
280238405Sjkim	shrq	$4,%r8
281238405Sjkim	movq	%r9,104(%rbp)
282238405Sjkim	movq	240+0-128(%rsi),%r9
283238405Sjkim	shlb	$4,%dl
284238405Sjkim	movq	%rbx,104-128(%rbp)
285238405Sjkim	movq	240+8-128(%rsi),%rbx
286238405Sjkim	shlq	$60,%r10
287238405Sjkim	movb	%dl,14(%rsp)
288238405Sjkim	orq	%r10,%rax
289238405Sjkim	movb	%bl,%dl
290238405Sjkim	shrq	$4,%rbx
291238405Sjkim	movq	%r9,%r10
292238405Sjkim	shrq	$4,%r9
293238405Sjkim	movq	%r8,112(%rbp)
294238405Sjkim	shlb	$4,%dl
295238405Sjkim	movq	%rax,112-128(%rbp)
296238405Sjkim	shlq	$60,%r10
297238405Sjkim	movb	%dl,15(%rsp)
298238405Sjkim	orq	%r10,%rbx
299238405Sjkim	movq	%r9,120(%rbp)
300238405Sjkim	movq	%rbx,120-128(%rbp)
301238405Sjkim	addq	$-128,%rsi
302238405Sjkim	movq	8(%rdi),%r8
303238405Sjkim	movq	0(%rdi),%r9
304238405Sjkim	addq	%r14,%r15
305238405Sjkim	leaq	.Lrem_8bit(%rip),%r11
306238405Sjkim	jmp	.Louter_loop
307238405Sjkim.align	16
308238405Sjkim.Louter_loop:
309238405Sjkim	xorq	(%r14),%r9
310238405Sjkim	movq	8(%r14),%rdx
311238405Sjkim	leaq	16(%r14),%r14
312238405Sjkim	xorq	%r8,%rdx
313238405Sjkim	movq	%r9,(%rdi)
314238405Sjkim	movq	%rdx,8(%rdi)
315238405Sjkim	shrq	$32,%rdx
316238405Sjkim	xorq	%rax,%rax
317238405Sjkim	roll	$8,%edx
318238405Sjkim	movb	%dl,%al
319238405Sjkim	movzbl	%dl,%ebx
320238405Sjkim	shlb	$4,%al
321238405Sjkim	shrl	$4,%ebx
322238405Sjkim	roll	$8,%edx
323238405Sjkim	movq	8(%rsi,%rax,1),%r8
324238405Sjkim	movq	(%rsi,%rax,1),%r9
325238405Sjkim	movb	%dl,%al
326238405Sjkim	movzbl	%dl,%ecx
327238405Sjkim	shlb	$4,%al
328238405Sjkim	movzbq	(%rsp,%rbx,1),%r12
329238405Sjkim	shrl	$4,%ecx
330238405Sjkim	xorq	%r8,%r12
331238405Sjkim	movq	%r9,%r10
332238405Sjkim	shrq	$8,%r8
333238405Sjkim	movzbq	%r12b,%r12
334238405Sjkim	shrq	$8,%r9
335238405Sjkim	xorq	-128(%rbp,%rbx,8),%r8
336238405Sjkim	shlq	$56,%r10
337238405Sjkim	xorq	(%rbp,%rbx,8),%r9
338238405Sjkim	roll	$8,%edx
339238405Sjkim	xorq	8(%rsi,%rax,1),%r8
340238405Sjkim	xorq	(%rsi,%rax,1),%r9
341238405Sjkim	movb	%dl,%al
342238405Sjkim	xorq	%r10,%r8
343238405Sjkim	movzwq	(%r11,%r12,2),%r12
344238405Sjkim	movzbl	%dl,%ebx
345238405Sjkim	shlb	$4,%al
346238405Sjkim	movzbq	(%rsp,%rcx,1),%r13
347238405Sjkim	shrl	$4,%ebx
348238405Sjkim	shlq	$48,%r12
349238405Sjkim	xorq	%r8,%r13
350238405Sjkim	movq	%r9,%r10
351238405Sjkim	xorq	%r12,%r9
352238405Sjkim	shrq	$8,%r8
353238405Sjkim	movzbq	%r13b,%r13
354238405Sjkim	shrq	$8,%r9
355238405Sjkim	xorq	-128(%rbp,%rcx,8),%r8
356238405Sjkim	shlq	$56,%r10
357238405Sjkim	xorq	(%rbp,%rcx,8),%r9
358238405Sjkim	roll	$8,%edx
359238405Sjkim	xorq	8(%rsi,%rax,1),%r8
360238405Sjkim	xorq	(%rsi,%rax,1),%r9
361238405Sjkim	movb	%dl,%al
362238405Sjkim	xorq	%r10,%r8
363238405Sjkim	movzwq	(%r11,%r13,2),%r13
364238405Sjkim	movzbl	%dl,%ecx
365238405Sjkim	shlb	$4,%al
366238405Sjkim	movzbq	(%rsp,%rbx,1),%r12
367238405Sjkim	shrl	$4,%ecx
368238405Sjkim	shlq	$48,%r13
369238405Sjkim	xorq	%r8,%r12
370238405Sjkim	movq	%r9,%r10
371238405Sjkim	xorq	%r13,%r9
372238405Sjkim	shrq	$8,%r8
373238405Sjkim	movzbq	%r12b,%r12
374238405Sjkim	movl	8(%rdi),%edx
375238405Sjkim	shrq	$8,%r9
376238405Sjkim	xorq	-128(%rbp,%rbx,8),%r8
377238405Sjkim	shlq	$56,%r10
378238405Sjkim	xorq	(%rbp,%rbx,8),%r9
379238405Sjkim	roll	$8,%edx
380238405Sjkim	xorq	8(%rsi,%rax,1),%r8
381238405Sjkim	xorq	(%rsi,%rax,1),%r9
382238405Sjkim	movb	%dl,%al
383238405Sjkim	xorq	%r10,%r8
384238405Sjkim	movzwq	(%r11,%r12,2),%r12
385238405Sjkim	movzbl	%dl,%ebx
386238405Sjkim	shlb	$4,%al
387238405Sjkim	movzbq	(%rsp,%rcx,1),%r13
388238405Sjkim	shrl	$4,%ebx
389238405Sjkim	shlq	$48,%r12
390238405Sjkim	xorq	%r8,%r13
391238405Sjkim	movq	%r9,%r10
392238405Sjkim	xorq	%r12,%r9
393238405Sjkim	shrq	$8,%r8
394238405Sjkim	movzbq	%r13b,%r13
395238405Sjkim	shrq	$8,%r9
396238405Sjkim	xorq	-128(%rbp,%rcx,8),%r8
397238405Sjkim	shlq	$56,%r10
398238405Sjkim	xorq	(%rbp,%rcx,8),%r9
399238405Sjkim	roll	$8,%edx
400238405Sjkim	xorq	8(%rsi,%rax,1),%r8
401238405Sjkim	xorq	(%rsi,%rax,1),%r9
402238405Sjkim	movb	%dl,%al
403238405Sjkim	xorq	%r10,%r8
404238405Sjkim	movzwq	(%r11,%r13,2),%r13
405238405Sjkim	movzbl	%dl,%ecx
406238405Sjkim	shlb	$4,%al
407238405Sjkim	movzbq	(%rsp,%rbx,1),%r12
408238405Sjkim	shrl	$4,%ecx
409238405Sjkim	shlq	$48,%r13
410238405Sjkim	xorq	%r8,%r12
411238405Sjkim	movq	%r9,%r10
412238405Sjkim	xorq	%r13,%r9
413238405Sjkim	shrq	$8,%r8
414238405Sjkim	movzbq	%r12b,%r12
415238405Sjkim	shrq	$8,%r9
416238405Sjkim	xorq	-128(%rbp,%rbx,8),%r8
417238405Sjkim	shlq	$56,%r10
418238405Sjkim	xorq	(%rbp,%rbx,8),%r9
419238405Sjkim	roll	$8,%edx
420238405Sjkim	xorq	8(%rsi,%rax,1),%r8
421238405Sjkim	xorq	(%rsi,%rax,1),%r9
422238405Sjkim	movb	%dl,%al
423238405Sjkim	xorq	%r10,%r8
424238405Sjkim	movzwq	(%r11,%r12,2),%r12
425238405Sjkim	movzbl	%dl,%ebx
426238405Sjkim	shlb	$4,%al
427238405Sjkim	movzbq	(%rsp,%rcx,1),%r13
428238405Sjkim	shrl	$4,%ebx
429238405Sjkim	shlq	$48,%r12
430238405Sjkim	xorq	%r8,%r13
431238405Sjkim	movq	%r9,%r10
432238405Sjkim	xorq	%r12,%r9
433238405Sjkim	shrq	$8,%r8
434238405Sjkim	movzbq	%r13b,%r13
435238405Sjkim	shrq	$8,%r9
436238405Sjkim	xorq	-128(%rbp,%rcx,8),%r8
437238405Sjkim	shlq	$56,%r10
438238405Sjkim	xorq	(%rbp,%rcx,8),%r9
439238405Sjkim	roll	$8,%edx
440238405Sjkim	xorq	8(%rsi,%rax,1),%r8
441238405Sjkim	xorq	(%rsi,%rax,1),%r9
442238405Sjkim	movb	%dl,%al
443238405Sjkim	xorq	%r10,%r8
444238405Sjkim	movzwq	(%r11,%r13,2),%r13
445238405Sjkim	movzbl	%dl,%ecx
446238405Sjkim	shlb	$4,%al
447238405Sjkim	movzbq	(%rsp,%rbx,1),%r12
448238405Sjkim	shrl	$4,%ecx
449238405Sjkim	shlq	$48,%r13
450238405Sjkim	xorq	%r8,%r12
451238405Sjkim	movq	%r9,%r10
452238405Sjkim	xorq	%r13,%r9
453238405Sjkim	shrq	$8,%r8
454238405Sjkim	movzbq	%r12b,%r12
455238405Sjkim	movl	4(%rdi),%edx
456238405Sjkim	shrq	$8,%r9
457238405Sjkim	xorq	-128(%rbp,%rbx,8),%r8
458238405Sjkim	shlq	$56,%r10
459238405Sjkim	xorq	(%rbp,%rbx,8),%r9
460238405Sjkim	roll	$8,%edx
461238405Sjkim	xorq	8(%rsi,%rax,1),%r8
462238405Sjkim	xorq	(%rsi,%rax,1),%r9
463238405Sjkim	movb	%dl,%al
464238405Sjkim	xorq	%r10,%r8
465238405Sjkim	movzwq	(%r11,%r12,2),%r12
466238405Sjkim	movzbl	%dl,%ebx
467238405Sjkim	shlb	$4,%al
468238405Sjkim	movzbq	(%rsp,%rcx,1),%r13
469238405Sjkim	shrl	$4,%ebx
470238405Sjkim	shlq	$48,%r12
471238405Sjkim	xorq	%r8,%r13
472238405Sjkim	movq	%r9,%r10
473238405Sjkim	xorq	%r12,%r9
474238405Sjkim	shrq	$8,%r8
475238405Sjkim	movzbq	%r13b,%r13
476238405Sjkim	shrq	$8,%r9
477238405Sjkim	xorq	-128(%rbp,%rcx,8),%r8
478238405Sjkim	shlq	$56,%r10
479238405Sjkim	xorq	(%rbp,%rcx,8),%r9
480238405Sjkim	roll	$8,%edx
481238405Sjkim	xorq	8(%rsi,%rax,1),%r8
482238405Sjkim	xorq	(%rsi,%rax,1),%r9
483238405Sjkim	movb	%dl,%al
484238405Sjkim	xorq	%r10,%r8
485238405Sjkim	movzwq	(%r11,%r13,2),%r13
486238405Sjkim	movzbl	%dl,%ecx
487238405Sjkim	shlb	$4,%al
488238405Sjkim	movzbq	(%rsp,%rbx,1),%r12
489238405Sjkim	shrl	$4,%ecx
490238405Sjkim	shlq	$48,%r13
491238405Sjkim	xorq	%r8,%r12
492238405Sjkim	movq	%r9,%r10
493238405Sjkim	xorq	%r13,%r9
494238405Sjkim	shrq	$8,%r8
495238405Sjkim	movzbq	%r12b,%r12
496238405Sjkim	shrq	$8,%r9
497238405Sjkim	xorq	-128(%rbp,%rbx,8),%r8
498238405Sjkim	shlq	$56,%r10
499238405Sjkim	xorq	(%rbp,%rbx,8),%r9
500238405Sjkim	roll	$8,%edx
501238405Sjkim	xorq	8(%rsi,%rax,1),%r8
502238405Sjkim	xorq	(%rsi,%rax,1),%r9
503238405Sjkim	movb	%dl,%al
504238405Sjkim	xorq	%r10,%r8
505238405Sjkim	movzwq	(%r11,%r12,2),%r12
506238405Sjkim	movzbl	%dl,%ebx
507238405Sjkim	shlb	$4,%al
508238405Sjkim	movzbq	(%rsp,%rcx,1),%r13
509238405Sjkim	shrl	$4,%ebx
510238405Sjkim	shlq	$48,%r12
511238405Sjkim	xorq	%r8,%r13
512238405Sjkim	movq	%r9,%r10
513238405Sjkim	xorq	%r12,%r9
514238405Sjkim	shrq	$8,%r8
515238405Sjkim	movzbq	%r13b,%r13
516238405Sjkim	shrq	$8,%r9
517238405Sjkim	xorq	-128(%rbp,%rcx,8),%r8
518238405Sjkim	shlq	$56,%r10
519238405Sjkim	xorq	(%rbp,%rcx,8),%r9
520238405Sjkim	roll	$8,%edx
521238405Sjkim	xorq	8(%rsi,%rax,1),%r8
522238405Sjkim	xorq	(%rsi,%rax,1),%r9
523238405Sjkim	movb	%dl,%al
524238405Sjkim	xorq	%r10,%r8
525238405Sjkim	movzwq	(%r11,%r13,2),%r13
526238405Sjkim	movzbl	%dl,%ecx
527238405Sjkim	shlb	$4,%al
528238405Sjkim	movzbq	(%rsp,%rbx,1),%r12
529238405Sjkim	shrl	$4,%ecx
530238405Sjkim	shlq	$48,%r13
531238405Sjkim	xorq	%r8,%r12
532238405Sjkim	movq	%r9,%r10
533238405Sjkim	xorq	%r13,%r9
534238405Sjkim	shrq	$8,%r8
535238405Sjkim	movzbq	%r12b,%r12
536238405Sjkim	movl	0(%rdi),%edx
537238405Sjkim	shrq	$8,%r9
538238405Sjkim	xorq	-128(%rbp,%rbx,8),%r8
539238405Sjkim	shlq	$56,%r10
540238405Sjkim	xorq	(%rbp,%rbx,8),%r9
541238405Sjkim	roll	$8,%edx
542238405Sjkim	xorq	8(%rsi,%rax,1),%r8
543238405Sjkim	xorq	(%rsi,%rax,1),%r9
544238405Sjkim	movb	%dl,%al
545238405Sjkim	xorq	%r10,%r8
546238405Sjkim	movzwq	(%r11,%r12,2),%r12
547238405Sjkim	movzbl	%dl,%ebx
548238405Sjkim	shlb	$4,%al
549238405Sjkim	movzbq	(%rsp,%rcx,1),%r13
550238405Sjkim	shrl	$4,%ebx
551238405Sjkim	shlq	$48,%r12
552238405Sjkim	xorq	%r8,%r13
553238405Sjkim	movq	%r9,%r10
554238405Sjkim	xorq	%r12,%r9
555238405Sjkim	shrq	$8,%r8
556238405Sjkim	movzbq	%r13b,%r13
557238405Sjkim	shrq	$8,%r9
558238405Sjkim	xorq	-128(%rbp,%rcx,8),%r8
559238405Sjkim	shlq	$56,%r10
560238405Sjkim	xorq	(%rbp,%rcx,8),%r9
561238405Sjkim	roll	$8,%edx
562238405Sjkim	xorq	8(%rsi,%rax,1),%r8
563238405Sjkim	xorq	(%rsi,%rax,1),%r9
564238405Sjkim	movb	%dl,%al
565238405Sjkim	xorq	%r10,%r8
566238405Sjkim	movzwq	(%r11,%r13,2),%r13
567238405Sjkim	movzbl	%dl,%ecx
568238405Sjkim	shlb	$4,%al
569238405Sjkim	movzbq	(%rsp,%rbx,1),%r12
570238405Sjkim	shrl	$4,%ecx
571238405Sjkim	shlq	$48,%r13
572238405Sjkim	xorq	%r8,%r12
573238405Sjkim	movq	%r9,%r10
574238405Sjkim	xorq	%r13,%r9
575238405Sjkim	shrq	$8,%r8
576238405Sjkim	movzbq	%r12b,%r12
577238405Sjkim	shrq	$8,%r9
578238405Sjkim	xorq	-128(%rbp,%rbx,8),%r8
579238405Sjkim	shlq	$56,%r10
580238405Sjkim	xorq	(%rbp,%rbx,8),%r9
581238405Sjkim	roll	$8,%edx
582238405Sjkim	xorq	8(%rsi,%rax,1),%r8
583238405Sjkim	xorq	(%rsi,%rax,1),%r9
584238405Sjkim	movb	%dl,%al
585238405Sjkim	xorq	%r10,%r8
586238405Sjkim	movzwq	(%r11,%r12,2),%r12
587238405Sjkim	movzbl	%dl,%ebx
588238405Sjkim	shlb	$4,%al
589238405Sjkim	movzbq	(%rsp,%rcx,1),%r13
590238405Sjkim	shrl	$4,%ebx
591238405Sjkim	shlq	$48,%r12
592238405Sjkim	xorq	%r8,%r13
593238405Sjkim	movq	%r9,%r10
594238405Sjkim	xorq	%r12,%r9
595238405Sjkim	shrq	$8,%r8
596238405Sjkim	movzbq	%r13b,%r13
597238405Sjkim	shrq	$8,%r9
598238405Sjkim	xorq	-128(%rbp,%rcx,8),%r8
599238405Sjkim	shlq	$56,%r10
600238405Sjkim	xorq	(%rbp,%rcx,8),%r9
601238405Sjkim	roll	$8,%edx
602238405Sjkim	xorq	8(%rsi,%rax,1),%r8
603238405Sjkim	xorq	(%rsi,%rax,1),%r9
604238405Sjkim	movb	%dl,%al
605238405Sjkim	xorq	%r10,%r8
606238405Sjkim	movzwq	(%r11,%r13,2),%r13
607238405Sjkim	movzbl	%dl,%ecx
608238405Sjkim	shlb	$4,%al
609238405Sjkim	movzbq	(%rsp,%rbx,1),%r12
610238405Sjkim	andl	$240,%ecx
611238405Sjkim	shlq	$48,%r13
612238405Sjkim	xorq	%r8,%r12
613238405Sjkim	movq	%r9,%r10
614238405Sjkim	xorq	%r13,%r9
615238405Sjkim	shrq	$8,%r8
616238405Sjkim	movzbq	%r12b,%r12
617238405Sjkim	movl	-4(%rdi),%edx
618238405Sjkim	shrq	$8,%r9
619238405Sjkim	xorq	-128(%rbp,%rbx,8),%r8
620238405Sjkim	shlq	$56,%r10
621238405Sjkim	xorq	(%rbp,%rbx,8),%r9
622238405Sjkim	movzwq	(%r11,%r12,2),%r12
623238405Sjkim	xorq	8(%rsi,%rax,1),%r8
624238405Sjkim	xorq	(%rsi,%rax,1),%r9
625238405Sjkim	shlq	$48,%r12
626238405Sjkim	xorq	%r10,%r8
627238405Sjkim	xorq	%r12,%r9
628238405Sjkim	movzbq	%r8b,%r13
629238405Sjkim	shrq	$4,%r8
630238405Sjkim	movq	%r9,%r10
631238405Sjkim	shlb	$4,%r13b
632238405Sjkim	shrq	$4,%r9
633238405Sjkim	xorq	8(%rsi,%rcx,1),%r8
634238405Sjkim	movzwq	(%r11,%r13,2),%r13
635238405Sjkim	shlq	$60,%r10
636238405Sjkim	xorq	(%rsi,%rcx,1),%r9
637238405Sjkim	xorq	%r10,%r8
638238405Sjkim	shlq	$48,%r13
639238405Sjkim	bswapq	%r8
640238405Sjkim	xorq	%r13,%r9
641238405Sjkim	bswapq	%r9
642238405Sjkim	cmpq	%r15,%r14
643238405Sjkim	jb	.Louter_loop
644238405Sjkim	movq	%r8,8(%rdi)
645238405Sjkim	movq	%r9,(%rdi)
646238405Sjkim
647238405Sjkim	leaq	280(%rsp),%rsi
648238405Sjkim	movq	0(%rsi),%r15
649238405Sjkim	movq	8(%rsi),%r14
650238405Sjkim	movq	16(%rsi),%r13
651238405Sjkim	movq	24(%rsi),%r12
652238405Sjkim	movq	32(%rsi),%rbp
653238405Sjkim	movq	40(%rsi),%rbx
654238405Sjkim	leaq	48(%rsi),%rsp
655238405Sjkim.Lghash_epilogue:
656238405Sjkim	.byte	0xf3,0xc3
657238405Sjkim.size	gcm_ghash_4bit,.-gcm_ghash_4bit
658238405Sjkim.globl	gcm_init_clmul
659238405Sjkim.type	gcm_init_clmul,@function
660238405Sjkim.align	16
661238405Sjkimgcm_init_clmul:
662238405Sjkim	movdqu	(%rsi),%xmm2
663238405Sjkim	pshufd	$78,%xmm2,%xmm2
664238405Sjkim
665238405Sjkim
666238405Sjkim	pshufd	$255,%xmm2,%xmm4
667238405Sjkim	movdqa	%xmm2,%xmm3
668238405Sjkim	psllq	$1,%xmm2
669238405Sjkim	pxor	%xmm5,%xmm5
670238405Sjkim	psrlq	$63,%xmm3
671238405Sjkim	pcmpgtd	%xmm4,%xmm5
672238405Sjkim	pslldq	$8,%xmm3
673238405Sjkim	por	%xmm3,%xmm2
674238405Sjkim
675238405Sjkim
676238405Sjkim	pand	.L0x1c2_polynomial(%rip),%xmm5
677238405Sjkim	pxor	%xmm5,%xmm2
678238405Sjkim
679238405Sjkim
680238405Sjkim	movdqa	%xmm2,%xmm0
681238405Sjkim	movdqa	%xmm0,%xmm1
682238405Sjkim	pshufd	$78,%xmm0,%xmm3
683238405Sjkim	pshufd	$78,%xmm2,%xmm4
684238405Sjkim	pxor	%xmm0,%xmm3
685238405Sjkim	pxor	%xmm2,%xmm4
686238405Sjkim.byte	102,15,58,68,194,0
687238405Sjkim.byte	102,15,58,68,202,17
688238405Sjkim.byte	102,15,58,68,220,0
689238405Sjkim	pxor	%xmm0,%xmm3
690238405Sjkim	pxor	%xmm1,%xmm3
691238405Sjkim
692238405Sjkim	movdqa	%xmm3,%xmm4
693238405Sjkim	psrldq	$8,%xmm3
694238405Sjkim	pslldq	$8,%xmm4
695238405Sjkim	pxor	%xmm3,%xmm1
696238405Sjkim	pxor	%xmm4,%xmm0
697238405Sjkim
698238405Sjkim	movdqa	%xmm0,%xmm3
699238405Sjkim	psllq	$1,%xmm0
700238405Sjkim	pxor	%xmm3,%xmm0
701238405Sjkim	psllq	$5,%xmm0
702238405Sjkim	pxor	%xmm3,%xmm0
703238405Sjkim	psllq	$57,%xmm0
704238405Sjkim	movdqa	%xmm0,%xmm4
705238405Sjkim	pslldq	$8,%xmm0
706238405Sjkim	psrldq	$8,%xmm4
707238405Sjkim	pxor	%xmm3,%xmm0
708238405Sjkim	pxor	%xmm4,%xmm1
709238405Sjkim
710238405Sjkim
711238405Sjkim	movdqa	%xmm0,%xmm4
712238405Sjkim	psrlq	$5,%xmm0
713238405Sjkim	pxor	%xmm4,%xmm0
714238405Sjkim	psrlq	$1,%xmm0
715238405Sjkim	pxor	%xmm4,%xmm0
716238405Sjkim	pxor	%xmm1,%xmm4
717238405Sjkim	psrlq	$1,%xmm0
718238405Sjkim	pxor	%xmm4,%xmm0
719238405Sjkim	movdqu	%xmm2,(%rdi)
720238405Sjkim	movdqu	%xmm0,16(%rdi)
721238405Sjkim	.byte	0xf3,0xc3
722238405Sjkim.size	gcm_init_clmul,.-gcm_init_clmul
723238405Sjkim.globl	gcm_gmult_clmul
724238405Sjkim.type	gcm_gmult_clmul,@function
725238405Sjkim.align	16
726238405Sjkimgcm_gmult_clmul:
727238405Sjkim	movdqu	(%rdi),%xmm0
728238405Sjkim	movdqa	.Lbswap_mask(%rip),%xmm5
729238405Sjkim	movdqu	(%rsi),%xmm2
730238405Sjkim.byte	102,15,56,0,197
731238405Sjkim	movdqa	%xmm0,%xmm1
732238405Sjkim	pshufd	$78,%xmm0,%xmm3
733238405Sjkim	pshufd	$78,%xmm2,%xmm4
734238405Sjkim	pxor	%xmm0,%xmm3
735238405Sjkim	pxor	%xmm2,%xmm4
736238405Sjkim.byte	102,15,58,68,194,0
737238405Sjkim.byte	102,15,58,68,202,17
738238405Sjkim.byte	102,15,58,68,220,0
739238405Sjkim	pxor	%xmm0,%xmm3
740238405Sjkim	pxor	%xmm1,%xmm3
741238405Sjkim
742238405Sjkim	movdqa	%xmm3,%xmm4
743238405Sjkim	psrldq	$8,%xmm3
744238405Sjkim	pslldq	$8,%xmm4
745238405Sjkim	pxor	%xmm3,%xmm1
746238405Sjkim	pxor	%xmm4,%xmm0
747238405Sjkim
748238405Sjkim	movdqa	%xmm0,%xmm3
749238405Sjkim	psllq	$1,%xmm0
750238405Sjkim	pxor	%xmm3,%xmm0
751238405Sjkim	psllq	$5,%xmm0
752238405Sjkim	pxor	%xmm3,%xmm0
753238405Sjkim	psllq	$57,%xmm0
754238405Sjkim	movdqa	%xmm0,%xmm4
755238405Sjkim	pslldq	$8,%xmm0
756238405Sjkim	psrldq	$8,%xmm4
757238405Sjkim	pxor	%xmm3,%xmm0
758238405Sjkim	pxor	%xmm4,%xmm1
759238405Sjkim
760238405Sjkim
761238405Sjkim	movdqa	%xmm0,%xmm4
762238405Sjkim	psrlq	$5,%xmm0
763238405Sjkim	pxor	%xmm4,%xmm0
764238405Sjkim	psrlq	$1,%xmm0
765238405Sjkim	pxor	%xmm4,%xmm0
766238405Sjkim	pxor	%xmm1,%xmm4
767238405Sjkim	psrlq	$1,%xmm0
768238405Sjkim	pxor	%xmm4,%xmm0
769238405Sjkim.byte	102,15,56,0,197
770238405Sjkim	movdqu	%xmm0,(%rdi)
771238405Sjkim	.byte	0xf3,0xc3
772238405Sjkim.size	gcm_gmult_clmul,.-gcm_gmult_clmul
773238405Sjkim.globl	gcm_ghash_clmul
774238405Sjkim.type	gcm_ghash_clmul,@function
775238405Sjkim.align	16
776238405Sjkimgcm_ghash_clmul:
777238405Sjkim	movdqa	.Lbswap_mask(%rip),%xmm5
778238405Sjkim
779238405Sjkim	movdqu	(%rdi),%xmm0
780238405Sjkim	movdqu	(%rsi),%xmm2
781238405Sjkim.byte	102,15,56,0,197
782238405Sjkim
783238405Sjkim	subq	$16,%rcx
784238405Sjkim	jz	.Lodd_tail
785238405Sjkim
786238405Sjkim	movdqu	16(%rsi),%xmm8
787238405Sjkim
788238405Sjkim
789238405Sjkim
790238405Sjkim
791238405Sjkim
792238405Sjkim	movdqu	(%rdx),%xmm3
793238405Sjkim	movdqu	16(%rdx),%xmm6
794238405Sjkim.byte	102,15,56,0,221
795238405Sjkim.byte	102,15,56,0,245
796238405Sjkim	pxor	%xmm3,%xmm0
797238405Sjkim	movdqa	%xmm6,%xmm7
798238405Sjkim	pshufd	$78,%xmm6,%xmm3
799238405Sjkim	pshufd	$78,%xmm2,%xmm4
800238405Sjkim	pxor	%xmm6,%xmm3
801238405Sjkim	pxor	%xmm2,%xmm4
802238405Sjkim.byte	102,15,58,68,242,0
803238405Sjkim.byte	102,15,58,68,250,17
804238405Sjkim.byte	102,15,58,68,220,0
805238405Sjkim	pxor	%xmm6,%xmm3
806238405Sjkim	pxor	%xmm7,%xmm3
807238405Sjkim
808238405Sjkim	movdqa	%xmm3,%xmm4
809238405Sjkim	psrldq	$8,%xmm3
810238405Sjkim	pslldq	$8,%xmm4
811238405Sjkim	pxor	%xmm3,%xmm7
812238405Sjkim	pxor	%xmm4,%xmm6
813238405Sjkim	movdqa	%xmm0,%xmm1
814238405Sjkim	pshufd	$78,%xmm0,%xmm3
815238405Sjkim	pshufd	$78,%xmm8,%xmm4
816238405Sjkim	pxor	%xmm0,%xmm3
817238405Sjkim	pxor	%xmm8,%xmm4
818238405Sjkim
819238405Sjkim	leaq	32(%rdx),%rdx
820238405Sjkim	subq	$32,%rcx
821238405Sjkim	jbe	.Leven_tail
822238405Sjkim
823238405Sjkim.Lmod_loop:
824238405Sjkim.byte	102,65,15,58,68,192,0
825238405Sjkim.byte	102,65,15,58,68,200,17
826238405Sjkim.byte	102,15,58,68,220,0
827238405Sjkim	pxor	%xmm0,%xmm3
828238405Sjkim	pxor	%xmm1,%xmm3
829238405Sjkim
830238405Sjkim	movdqa	%xmm3,%xmm4
831238405Sjkim	psrldq	$8,%xmm3
832238405Sjkim	pslldq	$8,%xmm4
833238405Sjkim	pxor	%xmm3,%xmm1
834238405Sjkim	pxor	%xmm4,%xmm0
835238405Sjkim	movdqu	(%rdx),%xmm3
836238405Sjkim	pxor	%xmm6,%xmm0
837238405Sjkim	pxor	%xmm7,%xmm1
838238405Sjkim
839238405Sjkim	movdqu	16(%rdx),%xmm6
840238405Sjkim.byte	102,15,56,0,221
841238405Sjkim.byte	102,15,56,0,245
842238405Sjkim
843238405Sjkim	movdqa	%xmm6,%xmm7
844238405Sjkim	pshufd	$78,%xmm6,%xmm9
845238405Sjkim	pshufd	$78,%xmm2,%xmm10
846238405Sjkim	pxor	%xmm6,%xmm9
847238405Sjkim	pxor	%xmm2,%xmm10
848238405Sjkim	pxor	%xmm3,%xmm1
849238405Sjkim
850238405Sjkim	movdqa	%xmm0,%xmm3
851238405Sjkim	psllq	$1,%xmm0
852238405Sjkim	pxor	%xmm3,%xmm0
853238405Sjkim	psllq	$5,%xmm0
854238405Sjkim	pxor	%xmm3,%xmm0
855238405Sjkim.byte	102,15,58,68,242,0
856238405Sjkim	psllq	$57,%xmm0
857238405Sjkim	movdqa	%xmm0,%xmm4
858238405Sjkim	pslldq	$8,%xmm0
859238405Sjkim	psrldq	$8,%xmm4
860238405Sjkim	pxor	%xmm3,%xmm0
861238405Sjkim	pxor	%xmm4,%xmm1
862238405Sjkim
863238405Sjkim.byte	102,15,58,68,250,17
864238405Sjkim	movdqa	%xmm0,%xmm4
865238405Sjkim	psrlq	$5,%xmm0
866238405Sjkim	pxor	%xmm4,%xmm0
867238405Sjkim	psrlq	$1,%xmm0
868238405Sjkim	pxor	%xmm4,%xmm0
869238405Sjkim	pxor	%xmm1,%xmm4
870238405Sjkim	psrlq	$1,%xmm0
871238405Sjkim	pxor	%xmm4,%xmm0
872238405Sjkim
873238405Sjkim.byte	102,69,15,58,68,202,0
874238405Sjkim	movdqa	%xmm0,%xmm1
875238405Sjkim	pshufd	$78,%xmm0,%xmm3
876238405Sjkim	pshufd	$78,%xmm8,%xmm4
877238405Sjkim	pxor	%xmm0,%xmm3
878238405Sjkim	pxor	%xmm8,%xmm4
879238405Sjkim
880238405Sjkim	pxor	%xmm6,%xmm9
881238405Sjkim	pxor	%xmm7,%xmm9
882238405Sjkim	movdqa	%xmm9,%xmm10
883238405Sjkim	psrldq	$8,%xmm9
884238405Sjkim	pslldq	$8,%xmm10
885238405Sjkim	pxor	%xmm9,%xmm7
886238405Sjkim	pxor	%xmm10,%xmm6
887238405Sjkim
888238405Sjkim	leaq	32(%rdx),%rdx
889238405Sjkim	subq	$32,%rcx
890238405Sjkim	ja	.Lmod_loop
891238405Sjkim
892238405Sjkim.Leven_tail:
893238405Sjkim.byte	102,65,15,58,68,192,0
894238405Sjkim.byte	102,65,15,58,68,200,17
895238405Sjkim.byte	102,15,58,68,220,0
896238405Sjkim	pxor	%xmm0,%xmm3
897238405Sjkim	pxor	%xmm1,%xmm3
898238405Sjkim
899238405Sjkim	movdqa	%xmm3,%xmm4
900238405Sjkim	psrldq	$8,%xmm3
901238405Sjkim	pslldq	$8,%xmm4
902238405Sjkim	pxor	%xmm3,%xmm1
903238405Sjkim	pxor	%xmm4,%xmm0
904238405Sjkim	pxor	%xmm6,%xmm0
905238405Sjkim	pxor	%xmm7,%xmm1
906238405Sjkim
907238405Sjkim	movdqa	%xmm0,%xmm3
908238405Sjkim	psllq	$1,%xmm0
909238405Sjkim	pxor	%xmm3,%xmm0
910238405Sjkim	psllq	$5,%xmm0
911238405Sjkim	pxor	%xmm3,%xmm0
912238405Sjkim	psllq	$57,%xmm0
913238405Sjkim	movdqa	%xmm0,%xmm4
914238405Sjkim	pslldq	$8,%xmm0
915238405Sjkim	psrldq	$8,%xmm4
916238405Sjkim	pxor	%xmm3,%xmm0
917238405Sjkim	pxor	%xmm4,%xmm1
918238405Sjkim
919238405Sjkim
920238405Sjkim	movdqa	%xmm0,%xmm4
921238405Sjkim	psrlq	$5,%xmm0
922238405Sjkim	pxor	%xmm4,%xmm0
923238405Sjkim	psrlq	$1,%xmm0
924238405Sjkim	pxor	%xmm4,%xmm0
925238405Sjkim	pxor	%xmm1,%xmm4
926238405Sjkim	psrlq	$1,%xmm0
927238405Sjkim	pxor	%xmm4,%xmm0
928238405Sjkim	testq	%rcx,%rcx
929238405Sjkim	jnz	.Ldone
930238405Sjkim
931238405Sjkim.Lodd_tail:
932238405Sjkim	movdqu	(%rdx),%xmm3
933238405Sjkim.byte	102,15,56,0,221
934238405Sjkim	pxor	%xmm3,%xmm0
935238405Sjkim	movdqa	%xmm0,%xmm1
936238405Sjkim	pshufd	$78,%xmm0,%xmm3
937238405Sjkim	pshufd	$78,%xmm2,%xmm4
938238405Sjkim	pxor	%xmm0,%xmm3
939238405Sjkim	pxor	%xmm2,%xmm4
940238405Sjkim.byte	102,15,58,68,194,0
941238405Sjkim.byte	102,15,58,68,202,17
942238405Sjkim.byte	102,15,58,68,220,0
943238405Sjkim	pxor	%xmm0,%xmm3
944238405Sjkim	pxor	%xmm1,%xmm3
945238405Sjkim
946238405Sjkim	movdqa	%xmm3,%xmm4
947238405Sjkim	psrldq	$8,%xmm3
948238405Sjkim	pslldq	$8,%xmm4
949238405Sjkim	pxor	%xmm3,%xmm1
950238405Sjkim	pxor	%xmm4,%xmm0
951238405Sjkim
952238405Sjkim	movdqa	%xmm0,%xmm3
953238405Sjkim	psllq	$1,%xmm0
954238405Sjkim	pxor	%xmm3,%xmm0
955238405Sjkim	psllq	$5,%xmm0
956238405Sjkim	pxor	%xmm3,%xmm0
957238405Sjkim	psllq	$57,%xmm0
958238405Sjkim	movdqa	%xmm0,%xmm4
959238405Sjkim	pslldq	$8,%xmm0
960238405Sjkim	psrldq	$8,%xmm4
961238405Sjkim	pxor	%xmm3,%xmm0
962238405Sjkim	pxor	%xmm4,%xmm1
963238405Sjkim
964238405Sjkim
965238405Sjkim	movdqa	%xmm0,%xmm4
966238405Sjkim	psrlq	$5,%xmm0
967238405Sjkim	pxor	%xmm4,%xmm0
968238405Sjkim	psrlq	$1,%xmm0
969238405Sjkim	pxor	%xmm4,%xmm0
970238405Sjkim	pxor	%xmm1,%xmm4
971238405Sjkim	psrlq	$1,%xmm0
972238405Sjkim	pxor	%xmm4,%xmm0
973238405Sjkim.Ldone:
974238405Sjkim.byte	102,15,56,0,197
975238405Sjkim	movdqu	%xmm0,(%rdi)
976238405Sjkim	.byte	0xf3,0xc3
977238405Sjkim.LSEH_end_gcm_ghash_clmul:
978238405Sjkim.size	gcm_ghash_clmul,.-gcm_ghash_clmul
979238405Sjkim.align	64
980238405Sjkim.Lbswap_mask:
981238405Sjkim.byte	15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0
982238405Sjkim.L0x1c2_polynomial:
983238405Sjkim.byte	1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2
984238405Sjkim.align	64
985238405Sjkim.type	.Lrem_4bit,@object
986238405Sjkim.Lrem_4bit:
987238405Sjkim.long	0,0,0,471859200,0,943718400,0,610271232
988238405Sjkim.long	0,1887436800,0,1822425088,0,1220542464,0,1423966208
989238405Sjkim.long	0,3774873600,0,4246732800,0,3644850176,0,3311403008
990238405Sjkim.long	0,2441084928,0,2376073216,0,2847932416,0,3051356160
991238405Sjkim.type	.Lrem_8bit,@object
992238405Sjkim.Lrem_8bit:
993238405Sjkim.value	0x0000,0x01C2,0x0384,0x0246,0x0708,0x06CA,0x048C,0x054E
994238405Sjkim.value	0x0E10,0x0FD2,0x0D94,0x0C56,0x0918,0x08DA,0x0A9C,0x0B5E
995238405Sjkim.value	0x1C20,0x1DE2,0x1FA4,0x1E66,0x1B28,0x1AEA,0x18AC,0x196E
996238405Sjkim.value	0x1230,0x13F2,0x11B4,0x1076,0x1538,0x14FA,0x16BC,0x177E
997238405Sjkim.value	0x3840,0x3982,0x3BC4,0x3A06,0x3F48,0x3E8A,0x3CCC,0x3D0E
998238405Sjkim.value	0x3650,0x3792,0x35D4,0x3416,0x3158,0x309A,0x32DC,0x331E
999238405Sjkim.value	0x2460,0x25A2,0x27E4,0x2626,0x2368,0x22AA,0x20EC,0x212E
1000238405Sjkim.value	0x2A70,0x2BB2,0x29F4,0x2836,0x2D78,0x2CBA,0x2EFC,0x2F3E
1001238405Sjkim.value	0x7080,0x7142,0x7304,0x72C6,0x7788,0x764A,0x740C,0x75CE
1002238405Sjkim.value	0x7E90,0x7F52,0x7D14,0x7CD6,0x7998,0x785A,0x7A1C,0x7BDE
1003238405Sjkim.value	0x6CA0,0x6D62,0x6F24,0x6EE6,0x6BA8,0x6A6A,0x682C,0x69EE
1004238405Sjkim.value	0x62B0,0x6372,0x6134,0x60F6,0x65B8,0x647A,0x663C,0x67FE
1005238405Sjkim.value	0x48C0,0x4902,0x4B44,0x4A86,0x4FC8,0x4E0A,0x4C4C,0x4D8E
1006238405Sjkim.value	0x46D0,0x4712,0x4554,0x4496,0x41D8,0x401A,0x425C,0x439E
1007238405Sjkim.value	0x54E0,0x5522,0x5764,0x56A6,0x53E8,0x522A,0x506C,0x51AE
1008238405Sjkim.value	0x5AF0,0x5B32,0x5974,0x58B6,0x5DF8,0x5C3A,0x5E7C,0x5FBE
1009238405Sjkim.value	0xE100,0xE0C2,0xE284,0xE346,0xE608,0xE7CA,0xE58C,0xE44E
1010238405Sjkim.value	0xEF10,0xEED2,0xEC94,0xED56,0xE818,0xE9DA,0xEB9C,0xEA5E
1011238405Sjkim.value	0xFD20,0xFCE2,0xFEA4,0xFF66,0xFA28,0xFBEA,0xF9AC,0xF86E
1012238405Sjkim.value	0xF330,0xF2F2,0xF0B4,0xF176,0xF438,0xF5FA,0xF7BC,0xF67E
1013238405Sjkim.value	0xD940,0xD882,0xDAC4,0xDB06,0xDE48,0xDF8A,0xDDCC,0xDC0E
1014238405Sjkim.value	0xD750,0xD692,0xD4D4,0xD516,0xD058,0xD19A,0xD3DC,0xD21E
1015238405Sjkim.value	0xC560,0xC4A2,0xC6E4,0xC726,0xC268,0xC3AA,0xC1EC,0xC02E
1016238405Sjkim.value	0xCB70,0xCAB2,0xC8F4,0xC936,0xCC78,0xCDBA,0xCFFC,0xCE3E
1017238405Sjkim.value	0x9180,0x9042,0x9204,0x93C6,0x9688,0x974A,0x950C,0x94CE
1018238405Sjkim.value	0x9F90,0x9E52,0x9C14,0x9DD6,0x9898,0x995A,0x9B1C,0x9ADE
1019238405Sjkim.value	0x8DA0,0x8C62,0x8E24,0x8FE6,0x8AA8,0x8B6A,0x892C,0x88EE
1020238405Sjkim.value	0x83B0,0x8272,0x8034,0x81F6,0x84B8,0x857A,0x873C,0x86FE
1021238405Sjkim.value	0xA9C0,0xA802,0xAA44,0xAB86,0xAEC8,0xAF0A,0xAD4C,0xAC8E
1022238405Sjkim.value	0xA7D0,0xA612,0xA454,0xA596,0xA0D8,0xA11A,0xA35C,0xA29E
1023238405Sjkim.value	0xB5E0,0xB422,0xB664,0xB7A6,0xB2E8,0xB32A,0xB16C,0xB0AE
1024238405Sjkim.value	0xBBF0,0xBA32,0xB874,0xB9B6,0xBCF8,0xBD3A,0xBF7C,0xBEBE
1025238405Sjkim
1026238405Sjkim.byte	71,72,65,83,72,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0
1027238405Sjkim.align	64
1028