1//  z_Linux_asm.S:  - microtasking routines specifically
2//                    written for Intel platforms running Linux* OS
3
4//
5////===----------------------------------------------------------------------===//
6////
7//// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8//// See https://llvm.org/LICENSE.txt for license information.
9//// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10////
11////===----------------------------------------------------------------------===//
12//
13
14// -----------------------------------------------------------------------
15// macros
16// -----------------------------------------------------------------------
17
18#include "kmp_config.h"
19
20#if KMP_ARCH_X86 || KMP_ARCH_X86_64
21
22# if KMP_MIC
23// the 'delay r16/r32/r64' should be used instead of the 'pause'.
24// The delay operation has the effect of removing the current thread from
25// the round-robin HT mechanism, and therefore speeds up the issue rate of
26// the other threads on the same core.
27//
28// A value of 0 works fine for <= 2 threads per core, but causes the EPCC
29// barrier time to increase greatly for 3 or more threads per core.
30//
31// A value of 100 works pretty well for up to 4 threads per core, but isn't
32// quite as fast as 0 for 2 threads per core.
33//
34// We need to check what happens for oversubscription / > 4 threads per core.
35// It is possible that we need to pass the delay value in as a parameter
36// that the caller determines based on the total # threads / # cores.
37//
38//.macro pause_op
39//	mov    $100, %rax
40//	delay  %rax
41//.endm
42# else
43#  define pause_op   .byte 0xf3,0x90
44# endif // KMP_MIC
45
46# if KMP_OS_DARWIN
47#  define KMP_PREFIX_UNDERSCORE(x) _##x  // extra underscore for OS X* symbols
48#  define KMP_LABEL(x) L_##x             // form the name of label
49.macro KMP_CFI_DEF_OFFSET
50.endmacro
51.macro KMP_CFI_OFFSET
52.endmacro
53.macro KMP_CFI_REGISTER
54.endmacro
55.macro KMP_CFI_DEF
56.endmacro
57.macro ALIGN
58	.align $0
59.endmacro
60.macro DEBUG_INFO
61/* Not sure what .size does in icc, not sure if we need to do something
62   similar for OS X*.
63*/
64.endmacro
65.macro PROC
66	ALIGN  4
67	.globl KMP_PREFIX_UNDERSCORE($0)
68KMP_PREFIX_UNDERSCORE($0):
69.endmacro
70# else // KMP_OS_DARWIN
71#  define KMP_PREFIX_UNDERSCORE(x) x //no extra underscore for Linux* OS symbols
72// Format labels so that they don't override function names in gdb's backtraces
73// MIC assembler doesn't accept .L syntax, the L works fine there (as well as
74// on OS X*)
75# if KMP_MIC
76#  define KMP_LABEL(x) L_##x          // local label
77# else
78#  define KMP_LABEL(x) .L_##x         // local label hidden from backtraces
79# endif // KMP_MIC
80.macro ALIGN size
81	.align 1<<(\size)
82.endm
83.macro DEBUG_INFO proc
84	.cfi_endproc
85// Not sure why we need .type and .size for the functions
86	.align 16
87	.type  \proc,@function
88        .size  \proc,.-\proc
89.endm
90.macro PROC proc
91	ALIGN  4
92        .globl KMP_PREFIX_UNDERSCORE(\proc)
93KMP_PREFIX_UNDERSCORE(\proc):
94	.cfi_startproc
95.endm
96.macro KMP_CFI_DEF_OFFSET sz
97	.cfi_def_cfa_offset	\sz
98.endm
99.macro KMP_CFI_OFFSET reg, sz
100	.cfi_offset	\reg,\sz
101.endm
102.macro KMP_CFI_REGISTER reg
103	.cfi_def_cfa_register	\reg
104.endm
105.macro KMP_CFI_DEF reg, sz
106	.cfi_def_cfa	\reg,\sz
107.endm
108# endif // KMP_OS_DARWIN
109#endif // KMP_ARCH_X86 || KMP_ARCH_x86_64
110
111#if (KMP_OS_LINUX || KMP_OS_DARWIN || KMP_OS_WINDOWS) && (KMP_ARCH_AARCH64 || KMP_ARCH_ARM)
112
113# if KMP_OS_DARWIN
114#  define KMP_PREFIX_UNDERSCORE(x) _##x  // extra underscore for OS X* symbols
115#  define KMP_LABEL(x) L_##x             // form the name of label
116
117.macro ALIGN
118	.align $0
119.endmacro
120
121.macro DEBUG_INFO
122/* Not sure what .size does in icc, not sure if we need to do something
123   similar for OS X*.
124*/
125.endmacro
126
127.macro PROC
128	ALIGN  4
129	.globl KMP_PREFIX_UNDERSCORE($0)
130KMP_PREFIX_UNDERSCORE($0):
131.endmacro
132# elif KMP_OS_WINDOWS
133#  define KMP_PREFIX_UNDERSCORE(x) x  // no extra underscore for Windows/ARM64 symbols
134// Format labels so that they don't override function names in gdb's backtraces
135#  define KMP_LABEL(x) .L_##x         // local label hidden from backtraces
136
137.macro ALIGN size
138	.align 1<<(\size)
139.endm
140
141.macro DEBUG_INFO proc
142	ALIGN 2
143.endm
144
145.macro PROC proc
146	ALIGN 2
147	.globl KMP_PREFIX_UNDERSCORE(\proc)
148KMP_PREFIX_UNDERSCORE(\proc):
149.endm
150# else // KMP_OS_DARWIN || KMP_OS_WINDOWS
151#  define KMP_PREFIX_UNDERSCORE(x) x  // no extra underscore for Linux* OS symbols
152// Format labels so that they don't override function names in gdb's backtraces
153#  define KMP_LABEL(x) .L_##x         // local label hidden from backtraces
154
155.macro ALIGN size
156	.align 1<<(\size)
157.endm
158
159.macro DEBUG_INFO proc
160	.cfi_endproc
161// Not sure why we need .type and .size for the functions
162	ALIGN 2
163#if KMP_ARCH_ARM
164	.type  \proc,%function
165#else
166	.type  \proc,@function
167#endif
168	.size  \proc,.-\proc
169.endm
170
171.macro PROC proc
172	ALIGN 2
173	.globl KMP_PREFIX_UNDERSCORE(\proc)
174KMP_PREFIX_UNDERSCORE(\proc):
175	.cfi_startproc
176.endm
177# endif // KMP_OS_DARWIN
178
179#endif // (KMP_OS_LINUX || KMP_OS_DARWIN || KMP_OS_WINDOWS) && (KMP_ARCH_AARCH64 || KMP_ARCH_ARM)
180
181.macro COMMON name, size, align_power
182#if KMP_OS_DARWIN
183	.comm \name, \size
184#elif KMP_OS_WINDOWS
185	.comm \name, \size, \align_power
186#else // !KMP_OS_DARWIN && !KMP_OS_WINDOWS
187	.comm \name, \size, (1<<(\align_power))
188#endif
189.endm
190
191// -----------------------------------------------------------------------
192// data
193// -----------------------------------------------------------------------
194
195#ifdef KMP_GOMP_COMPAT
196
197// Support for unnamed common blocks.
198//
199// Because the symbol ".gomp_critical_user_" contains a ".", we have to
200// put this stuff in assembly.
201
202# if KMP_ARCH_X86
203#  if KMP_OS_DARWIN
204        .data
205        .comm .gomp_critical_user_,32
206        .data
207        .globl ___kmp_unnamed_critical_addr
208___kmp_unnamed_critical_addr:
209        .long .gomp_critical_user_
210#  else /* Linux* OS */
211        .data
212        .comm .gomp_critical_user_,32,8
213        .data
214	ALIGN 4
215        .global __kmp_unnamed_critical_addr
216__kmp_unnamed_critical_addr:
217        .4byte .gomp_critical_user_
218        .type __kmp_unnamed_critical_addr,@object
219        .size __kmp_unnamed_critical_addr,4
220#  endif /* KMP_OS_DARWIN */
221# endif /* KMP_ARCH_X86 */
222
223# if KMP_ARCH_X86_64
224#  if KMP_OS_DARWIN
225        .data
226        .comm .gomp_critical_user_,32
227        .data
228        .globl ___kmp_unnamed_critical_addr
229___kmp_unnamed_critical_addr:
230        .quad .gomp_critical_user_
231#  else /* Linux* OS */
232        .data
233        .comm .gomp_critical_user_,32,8
234        .data
235	ALIGN 8
236        .global __kmp_unnamed_critical_addr
237__kmp_unnamed_critical_addr:
238        .8byte .gomp_critical_user_
239        .type __kmp_unnamed_critical_addr,@object
240        .size __kmp_unnamed_critical_addr,8
241#  endif /* KMP_OS_DARWIN */
242# endif /* KMP_ARCH_X86_64 */
243
244#endif /* KMP_GOMP_COMPAT */
245
246
247#if KMP_ARCH_X86 && !KMP_ARCH_PPC64
248
249// -----------------------------------------------------------------------
250// microtasking routines specifically written for IA-32 architecture
251// running Linux* OS
252// -----------------------------------------------------------------------
253
254	.ident "Intel Corporation"
255	.data
256	ALIGN 4
257// void
258// __kmp_x86_pause( void );
259
260        .text
261	PROC  __kmp_x86_pause
262
263        pause_op
264        ret
265
266	DEBUG_INFO __kmp_x86_pause
267
268# if !KMP_ASM_INTRINS
269
270//------------------------------------------------------------------------
271// kmp_int32
272// __kmp_test_then_add32( volatile kmp_int32 *p, kmp_int32 d );
273
274        PROC      __kmp_test_then_add32
275
276        movl      4(%esp), %ecx
277        movl      8(%esp), %eax
278        lock
279        xaddl     %eax,(%ecx)
280        ret
281
282	DEBUG_INFO __kmp_test_then_add32
283
284//------------------------------------------------------------------------
285// FUNCTION __kmp_xchg_fixed8
286//
287// kmp_int32
288// __kmp_xchg_fixed8( volatile kmp_int8 *p, kmp_int8 d );
289//
290// parameters:
291// 	p:	4(%esp)
292// 	d:	8(%esp)
293//
294// return:	%al
295        PROC  __kmp_xchg_fixed8
296
297        movl      4(%esp), %ecx    // "p"
298        movb      8(%esp), %al	// "d"
299
300        lock
301        xchgb     %al,(%ecx)
302        ret
303
304        DEBUG_INFO __kmp_xchg_fixed8
305
306
307//------------------------------------------------------------------------
308// FUNCTION __kmp_xchg_fixed16
309//
310// kmp_int16
311// __kmp_xchg_fixed16( volatile kmp_int16 *p, kmp_int16 d );
312//
313// parameters:
314// 	p:	4(%esp)
315// 	d:	8(%esp)
316// return:     %ax
317        PROC  __kmp_xchg_fixed16
318
319        movl      4(%esp), %ecx    // "p"
320        movw      8(%esp), %ax	// "d"
321
322        lock
323        xchgw     %ax,(%ecx)
324        ret
325
326        DEBUG_INFO __kmp_xchg_fixed16
327
328
329//------------------------------------------------------------------------
330// FUNCTION __kmp_xchg_fixed32
331//
332// kmp_int32
333// __kmp_xchg_fixed32( volatile kmp_int32 *p, kmp_int32 d );
334//
335// parameters:
336// 	p:	4(%esp)
337// 	d:	8(%esp)
338//
339// return:	%eax
340        PROC  __kmp_xchg_fixed32
341
342        movl      4(%esp), %ecx    // "p"
343        movl      8(%esp), %eax	// "d"
344
345        lock
346        xchgl     %eax,(%ecx)
347        ret
348
349        DEBUG_INFO __kmp_xchg_fixed32
350
351
352// kmp_int8
353// __kmp_compare_and_store8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
354        PROC  __kmp_compare_and_store8
355
356        movl      4(%esp), %ecx
357        movb      8(%esp), %al
358        movb      12(%esp), %dl
359        lock
360        cmpxchgb  %dl,(%ecx)
361        sete      %al           // if %al == (%ecx) set %al = 1 else set %al = 0
362        and       $1, %eax      // sign extend previous instruction
363        ret
364
365        DEBUG_INFO __kmp_compare_and_store8
366
367// kmp_int16
368// __kmp_compare_and_store16(volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv);
369        PROC  __kmp_compare_and_store16
370
371        movl      4(%esp), %ecx
372        movw      8(%esp), %ax
373        movw      12(%esp), %dx
374        lock
375        cmpxchgw  %dx,(%ecx)
376        sete      %al           // if %ax == (%ecx) set %al = 1 else set %al = 0
377        and       $1, %eax      // sign extend previous instruction
378        ret
379
380        DEBUG_INFO __kmp_compare_and_store16
381
382// kmp_int32
383// __kmp_compare_and_store32(volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv);
384        PROC  __kmp_compare_and_store32
385
386        movl      4(%esp), %ecx
387        movl      8(%esp), %eax
388        movl      12(%esp), %edx
389        lock
390        cmpxchgl  %edx,(%ecx)
391        sete      %al          // if %eax == (%ecx) set %al = 1 else set %al = 0
392        and       $1, %eax     // sign extend previous instruction
393        ret
394
395        DEBUG_INFO __kmp_compare_and_store32
396
397// kmp_int32
398// __kmp_compare_and_store64(volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 s );
399        PROC  __kmp_compare_and_store64
400
401        pushl     %ebp
402        movl      %esp, %ebp
403        pushl     %ebx
404        pushl     %edi
405        movl      8(%ebp), %edi
406        movl      12(%ebp), %eax        // "cv" low order word
407        movl      16(%ebp), %edx        // "cv" high order word
408        movl      20(%ebp), %ebx        // "sv" low order word
409        movl      24(%ebp), %ecx        // "sv" high order word
410        lock
411        cmpxchg8b (%edi)
412        sete      %al      // if %edx:eax == (%edi) set %al = 1 else set %al = 0
413        and       $1, %eax // sign extend previous instruction
414        popl      %edi
415        popl      %ebx
416        movl      %ebp, %esp
417        popl      %ebp
418        ret
419
420        DEBUG_INFO __kmp_compare_and_store64
421
422// kmp_int8
423// __kmp_compare_and_store_ret8(volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv);
424        PROC  __kmp_compare_and_store_ret8
425
426        movl      4(%esp), %ecx
427        movb      8(%esp), %al
428        movb      12(%esp), %dl
429        lock
430        cmpxchgb  %dl,(%ecx)
431        ret
432
433        DEBUG_INFO __kmp_compare_and_store_ret8
434
435// kmp_int16
436// __kmp_compare_and_store_ret16(volatile kmp_int16 *p, kmp_int16 cv,
437//                               kmp_int16 sv);
438        PROC  __kmp_compare_and_store_ret16
439
440        movl      4(%esp), %ecx
441        movw      8(%esp), %ax
442        movw      12(%esp), %dx
443        lock
444        cmpxchgw  %dx,(%ecx)
445        ret
446
447        DEBUG_INFO __kmp_compare_and_store_ret16
448
449// kmp_int32
450// __kmp_compare_and_store_ret32(volatile kmp_int32 *p, kmp_int32 cv,
451//                               kmp_int32 sv);
452        PROC  __kmp_compare_and_store_ret32
453
454        movl      4(%esp), %ecx
455        movl      8(%esp), %eax
456        movl      12(%esp), %edx
457        lock
458        cmpxchgl  %edx,(%ecx)
459        ret
460
461        DEBUG_INFO __kmp_compare_and_store_ret32
462
463// kmp_int64
464// __kmp_compare_and_store_ret64(volatile kmp_int64 *p, kmp_int64 cv,
465//                               kmp_int64 sv);
466        PROC  __kmp_compare_and_store_ret64
467
468        pushl     %ebp
469        movl      %esp, %ebp
470        pushl     %ebx
471        pushl     %edi
472        movl      8(%ebp), %edi
473        movl      12(%ebp), %eax        // "cv" low order word
474        movl      16(%ebp), %edx        // "cv" high order word
475        movl      20(%ebp), %ebx        // "sv" low order word
476        movl      24(%ebp), %ecx        // "sv" high order word
477        lock
478        cmpxchg8b (%edi)
479        popl      %edi
480        popl      %ebx
481        movl      %ebp, %esp
482        popl      %ebp
483        ret
484
485        DEBUG_INFO __kmp_compare_and_store_ret64
486
487
488//------------------------------------------------------------------------
489// FUNCTION __kmp_xchg_real32
490//
491// kmp_real32
492// __kmp_xchg_real32( volatile kmp_real32 *addr, kmp_real32 data );
493//
494// parameters:
495// 	addr:	4(%esp)
496// 	data:	8(%esp)
497//
498// return:	%eax
499        PROC  __kmp_xchg_real32
500
501        pushl   %ebp
502        movl    %esp, %ebp
503        subl    $4, %esp
504        pushl   %esi
505
506        movl    4(%ebp), %esi
507        flds    (%esi)
508                        // load <addr>
509        fsts    -4(%ebp)
510                        // store old value
511
512        movl    8(%ebp), %eax
513
514        lock
515        xchgl   %eax, (%esi)
516
517        flds    -4(%ebp)
518                        // return old value
519
520        popl    %esi
521        movl    %ebp, %esp
522        popl    %ebp
523        ret
524
525        DEBUG_INFO __kmp_xchg_real32
526
527# endif /* !KMP_ASM_INTRINS */
528
529//------------------------------------------------------------------------
530// int
531// __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),
532//                         int gtid, int tid,
533//                         int argc, void *p_argv[]
534// #if OMPT_SUPPORT
535//                         ,
536//                         void **exit_frame_ptr
537// #endif
538//                       ) {
539// #if OMPT_SUPPORT
540//   *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
541// #endif
542//
543//   (*pkfn)( & gtid, & tid, argv[0], ... );
544//   return 1;
545// }
546
547// -- Begin __kmp_invoke_microtask
548// mark_begin;
549	PROC  __kmp_invoke_microtask
550
551	pushl %ebp
552	KMP_CFI_DEF_OFFSET 8
553	KMP_CFI_OFFSET ebp,-8
554	movl %esp,%ebp		// establish the base pointer for this routine.
555	KMP_CFI_REGISTER ebp
556	subl $8,%esp		// allocate space for two local variables.
557				// These varibales are:
558				//	argv: -4(%ebp)
559				//	temp: -8(%ebp)
560				//
561	pushl %ebx		// save %ebx to use during this routine
562				//
563#if OMPT_SUPPORT
564	movl 28(%ebp),%ebx	// get exit_frame address
565	movl %ebp,(%ebx)	// save exit_frame
566#endif
567
568	movl 20(%ebp),%ebx	// Stack alignment - # args
569	addl $2,%ebx		// #args +2  Always pass at least 2 args (gtid and tid)
570	shll $2,%ebx		// Number of bytes used on stack: (#args+2)*4
571	movl %esp,%eax		//
572	subl %ebx,%eax		// %esp-((#args+2)*4) -> %eax -- without mods, stack ptr would be this
573	movl %eax,%ebx		// Save to %ebx
574	andl $0xFFFFFF80,%eax	// mask off 7 bits
575	subl %eax,%ebx		// Amount to subtract from %esp
576	subl %ebx,%esp		// Prepare the stack ptr --
577				//   now it will be aligned on 128-byte boundary at the call
578
579	movl 24(%ebp),%eax	// copy from p_argv[]
580	movl %eax,-4(%ebp)	// into the local variable *argv.
581
582	movl 20(%ebp),%ebx	// argc is 20(%ebp)
583	shll $2,%ebx
584
585KMP_LABEL(invoke_2):
586	cmpl $0,%ebx
587	jg  KMP_LABEL(invoke_4)
588	jmp KMP_LABEL(invoke_3)
589	ALIGN 2
590KMP_LABEL(invoke_4):
591	movl -4(%ebp),%eax
592	subl $4,%ebx			// decrement argc.
593	addl %ebx,%eax			// index into argv.
594	movl (%eax),%edx
595	pushl %edx
596
597	jmp KMP_LABEL(invoke_2)
598	ALIGN 2
599KMP_LABEL(invoke_3):
600	leal 16(%ebp),%eax		// push & tid
601	pushl %eax
602
603	leal 12(%ebp),%eax		// push & gtid
604	pushl %eax
605
606	movl 8(%ebp),%ebx
607	call *%ebx			// call (*pkfn)();
608
609	movl $1,%eax			// return 1;
610
611	movl -12(%ebp),%ebx		// restore %ebx
612	leave
613	KMP_CFI_DEF esp,4
614	ret
615
616	DEBUG_INFO __kmp_invoke_microtask
617// -- End  __kmp_invoke_microtask
618
619
620// kmp_uint64
621// __kmp_hardware_timestamp(void)
622	PROC  __kmp_hardware_timestamp
623	rdtsc
624	ret
625
626	DEBUG_INFO __kmp_hardware_timestamp
627// -- End  __kmp_hardware_timestamp
628
629#endif /* KMP_ARCH_X86 */
630
631
632#if KMP_ARCH_X86_64
633
634// -----------------------------------------------------------------------
635// microtasking routines specifically written for IA-32 architecture and
636// Intel(R) 64 running Linux* OS
637// -----------------------------------------------------------------------
638
639// -- Machine type P
640// mark_description "Intel Corporation";
641	.ident "Intel Corporation"
642// --	.file "z_Linux_asm.S"
643	.data
644	ALIGN 4
645
646// To prevent getting our code into .data section .text added to every routine
647// definition for x86_64.
648//------------------------------------------------------------------------
649# if !KMP_ASM_INTRINS
650
651//------------------------------------------------------------------------
652// FUNCTION __kmp_test_then_add32
653//
654// kmp_int32
655// __kmp_test_then_add32( volatile kmp_int32 *p, kmp_int32 d );
656//
657// parameters:
658// 	p:	%rdi
659// 	d:	%esi
660//
661// return:	%eax
662        .text
663        PROC  __kmp_test_then_add32
664
665        movl      %esi, %eax	// "d"
666        lock
667        xaddl     %eax,(%rdi)
668        ret
669
670        DEBUG_INFO __kmp_test_then_add32
671
672
673//------------------------------------------------------------------------
674// FUNCTION __kmp_test_then_add64
675//
676// kmp_int64
677// __kmp_test_then_add64( volatile kmp_int64 *p, kmp_int64 d );
678//
679// parameters:
680// 	p:	%rdi
681// 	d:	%rsi
682//	return:	%rax
683        .text
684        PROC  __kmp_test_then_add64
685
686        movq      %rsi, %rax	// "d"
687        lock
688        xaddq     %rax,(%rdi)
689        ret
690
691        DEBUG_INFO __kmp_test_then_add64
692
693
694//------------------------------------------------------------------------
695// FUNCTION __kmp_xchg_fixed8
696//
697// kmp_int32
698// __kmp_xchg_fixed8( volatile kmp_int8 *p, kmp_int8 d );
699//
700// parameters:
701// 	p:	%rdi
702// 	d:	%sil
703//
704// return:	%al
705        .text
706        PROC  __kmp_xchg_fixed8
707
708        movb      %sil, %al	// "d"
709
710        lock
711        xchgb     %al,(%rdi)
712        ret
713
714        DEBUG_INFO __kmp_xchg_fixed8
715
716
717//------------------------------------------------------------------------
718// FUNCTION __kmp_xchg_fixed16
719//
720// kmp_int16
721// __kmp_xchg_fixed16( volatile kmp_int16 *p, kmp_int16 d );
722//
723// parameters:
724// 	p:	%rdi
725// 	d:	%si
726// return:     %ax
727        .text
728        PROC  __kmp_xchg_fixed16
729
730        movw      %si, %ax	// "d"
731
732        lock
733        xchgw     %ax,(%rdi)
734        ret
735
736        DEBUG_INFO __kmp_xchg_fixed16
737
738
739//------------------------------------------------------------------------
740// FUNCTION __kmp_xchg_fixed32
741//
742// kmp_int32
743// __kmp_xchg_fixed32( volatile kmp_int32 *p, kmp_int32 d );
744//
745// parameters:
746// 	p:	%rdi
747// 	d:	%esi
748//
749// return:	%eax
750        .text
751        PROC  __kmp_xchg_fixed32
752
753        movl      %esi, %eax	// "d"
754
755        lock
756        xchgl     %eax,(%rdi)
757        ret
758
759        DEBUG_INFO __kmp_xchg_fixed32
760
761
762//------------------------------------------------------------------------
763// FUNCTION __kmp_xchg_fixed64
764//
765// kmp_int64
766// __kmp_xchg_fixed64( volatile kmp_int64 *p, kmp_int64 d );
767//
768// parameters:
769// 	p:	%rdi
770// 	d:	%rsi
771// return:	%rax
772        .text
773        PROC  __kmp_xchg_fixed64
774
775        movq      %rsi, %rax	// "d"
776
777        lock
778        xchgq     %rax,(%rdi)
779        ret
780
781        DEBUG_INFO __kmp_xchg_fixed64
782
783
784//------------------------------------------------------------------------
785// FUNCTION __kmp_compare_and_store8
786//
787// kmp_int8
788// __kmp_compare_and_store8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
789//
790// parameters:
791// 	p:	%rdi
792// 	cv:	%esi
793//	sv:	%edx
794//
795// return:	%eax
796        .text
797        PROC  __kmp_compare_and_store8
798
799        movb      %sil, %al	// "cv"
800        lock
801        cmpxchgb  %dl,(%rdi)
802        sete      %al           // if %al == (%rdi) set %al = 1 else set %al = 0
803        andq      $1, %rax      // sign extend previous instruction for return value
804        ret
805
806        DEBUG_INFO __kmp_compare_and_store8
807
808
809//------------------------------------------------------------------------
810// FUNCTION __kmp_compare_and_store16
811//
812// kmp_int16
813// __kmp_compare_and_store16( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
814//
815// parameters:
816// 	p:	%rdi
817// 	cv:	%si
818//	sv:	%dx
819//
820// return:	%eax
821        .text
822        PROC  __kmp_compare_and_store16
823
824        movw      %si, %ax	// "cv"
825        lock
826        cmpxchgw  %dx,(%rdi)
827        sete      %al           // if %ax == (%rdi) set %al = 1 else set %al = 0
828        andq      $1, %rax      // sign extend previous instruction for return value
829        ret
830
831        DEBUG_INFO __kmp_compare_and_store16
832
833
834//------------------------------------------------------------------------
835// FUNCTION __kmp_compare_and_store32
836//
837// kmp_int32
838// __kmp_compare_and_store32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
839//
840// parameters:
841// 	p:	%rdi
842// 	cv:	%esi
843//	sv:	%edx
844//
845// return:	%eax
846        .text
847        PROC  __kmp_compare_and_store32
848
849        movl      %esi, %eax	// "cv"
850        lock
851        cmpxchgl  %edx,(%rdi)
852        sete      %al           // if %eax == (%rdi) set %al = 1 else set %al = 0
853        andq      $1, %rax      // sign extend previous instruction for return value
854        ret
855
856        DEBUG_INFO __kmp_compare_and_store32
857
858
859//------------------------------------------------------------------------
860// FUNCTION __kmp_compare_and_store64
861//
862// kmp_int32
863// __kmp_compare_and_store64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
864//
865// parameters:
866// 	p:	%rdi
867// 	cv:	%rsi
868//	sv:	%rdx
869//	return:	%eax
870        .text
871        PROC  __kmp_compare_and_store64
872
873        movq      %rsi, %rax    // "cv"
874        lock
875        cmpxchgq  %rdx,(%rdi)
876        sete      %al           // if %rax == (%rdi) set %al = 1 else set %al = 0
877        andq      $1, %rax      // sign extend previous instruction for return value
878        ret
879
880        DEBUG_INFO __kmp_compare_and_store64
881
882//------------------------------------------------------------------------
883// FUNCTION __kmp_compare_and_store_ret8
884//
885// kmp_int8
886// __kmp_compare_and_store_ret8( volatile kmp_int8 *p, kmp_int8 cv, kmp_int8 sv );
887//
888// parameters:
889// 	p:	%rdi
890// 	cv:	%esi
891//	sv:	%edx
892//
893// return:	%eax
894        .text
895        PROC  __kmp_compare_and_store_ret8
896
897        movb      %sil, %al	// "cv"
898        lock
899        cmpxchgb  %dl,(%rdi)
900        ret
901
902        DEBUG_INFO __kmp_compare_and_store_ret8
903
904
905//------------------------------------------------------------------------
906// FUNCTION __kmp_compare_and_store_ret16
907//
908// kmp_int16
909// __kmp_compare_and_store16_ret( volatile kmp_int16 *p, kmp_int16 cv, kmp_int16 sv );
910//
911// parameters:
912// 	p:	%rdi
913// 	cv:	%si
914//	sv:	%dx
915//
916// return:	%eax
917        .text
918        PROC  __kmp_compare_and_store_ret16
919
920        movw      %si, %ax	// "cv"
921        lock
922        cmpxchgw  %dx,(%rdi)
923        ret
924
925        DEBUG_INFO __kmp_compare_and_store_ret16
926
927
928//------------------------------------------------------------------------
929// FUNCTION __kmp_compare_and_store_ret32
930//
931// kmp_int32
932// __kmp_compare_and_store_ret32( volatile kmp_int32 *p, kmp_int32 cv, kmp_int32 sv );
933//
934// parameters:
935// 	p:	%rdi
936// 	cv:	%esi
937//	sv:	%edx
938//
939// return:	%eax
940        .text
941        PROC  __kmp_compare_and_store_ret32
942
943        movl      %esi, %eax	// "cv"
944        lock
945        cmpxchgl  %edx,(%rdi)
946        ret
947
948        DEBUG_INFO __kmp_compare_and_store_ret32
949
950
951//------------------------------------------------------------------------
952// FUNCTION __kmp_compare_and_store_ret64
953//
954// kmp_int64
955// __kmp_compare_and_store_ret64( volatile kmp_int64 *p, kmp_int64 cv, kmp_int64 sv );
956//
957// parameters:
958// 	p:	%rdi
959// 	cv:	%rsi
960//	sv:	%rdx
961//	return:	%eax
962        .text
963        PROC  __kmp_compare_and_store_ret64
964
965        movq      %rsi, %rax    // "cv"
966        lock
967        cmpxchgq  %rdx,(%rdi)
968        ret
969
970        DEBUG_INFO __kmp_compare_and_store_ret64
971
972# endif /* !KMP_ASM_INTRINS */
973
974
975# if !KMP_MIC
976
977# if !KMP_ASM_INTRINS
978
979//------------------------------------------------------------------------
980// FUNCTION __kmp_xchg_real32
981//
982// kmp_real32
983// __kmp_xchg_real32( volatile kmp_real32 *addr, kmp_real32 data );
984//
985// parameters:
986// 	addr:	%rdi
987// 	data:	%xmm0 (lower 4 bytes)
988//
989// return:	%xmm0 (lower 4 bytes)
990        .text
991        PROC  __kmp_xchg_real32
992
993	movd	%xmm0, %eax	// load "data" to eax
994
995         lock
996         xchgl %eax, (%rdi)
997
998	movd	%eax, %xmm0	// load old value into return register
999
1000        ret
1001
1002        DEBUG_INFO __kmp_xchg_real32
1003
1004
1005//------------------------------------------------------------------------
1006// FUNCTION __kmp_xchg_real64
1007//
1008// kmp_real64
1009// __kmp_xchg_real64( volatile kmp_real64 *addr, kmp_real64 data );
1010//
1011// parameters:
1012//      addr:   %rdi
1013//      data:   %xmm0 (lower 8 bytes)
1014//      return: %xmm0 (lower 8 bytes)
1015        .text
1016        PROC  __kmp_xchg_real64
1017
1018	movd	%xmm0, %rax	// load "data" to rax
1019
1020         lock
1021	xchgq  %rax, (%rdi)
1022
1023	movd	%rax, %xmm0	// load old value into return register
1024        ret
1025
1026        DEBUG_INFO __kmp_xchg_real64
1027
1028
1029# endif /* !KMP_MIC */
1030
1031# endif /* !KMP_ASM_INTRINS */
1032
1033//------------------------------------------------------------------------
1034// int
1035// __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),
1036//                         int gtid, int tid,
1037//                         int argc, void *p_argv[]
1038// #if OMPT_SUPPORT
1039//                         ,
1040//                         void **exit_frame_ptr
1041// #endif
1042//                       ) {
1043// #if OMPT_SUPPORT
1044//   *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
1045// #endif
1046//
1047//   (*pkfn)( & gtid, & tid, argv[0], ... );
1048//   return 1;
1049// }
1050//
1051// note: at call to pkfn must have %rsp 128-byte aligned for compiler
1052//
1053// parameters:
1054//      %rdi:  	pkfn
1055//	%esi:	gtid
1056//	%edx:	tid
1057//	%ecx:	argc
1058//	%r8:	p_argv
1059//	%r9:	&exit_frame
1060//
1061// locals:
1062//	__gtid:	gtid parm pushed on stack so can pass &gtid to pkfn
1063//	__tid:	tid parm pushed on stack so can pass &tid to pkfn
1064//
1065// reg temps:
1066//	%rax:	used all over the place
1067//	%rdx:	used in stack pointer alignment calculation
1068//	%r11:	used to traverse p_argv array
1069//	%rsi:	used as temporary for stack parameters
1070//		used as temporary for number of pkfn parms to push
1071//	%rbx:	used to hold pkfn address, and zero constant, callee-save
1072//
1073// return:	%eax 	(always 1/TRUE)
1074__gtid = -16
1075__tid = -24
1076
1077// -- Begin __kmp_invoke_microtask
1078// mark_begin;
1079        .text
1080	PROC  __kmp_invoke_microtask
1081
1082	pushq 	%rbp		// save base pointer
1083	KMP_CFI_DEF_OFFSET 16
1084	KMP_CFI_OFFSET rbp,-16
1085	movq 	%rsp,%rbp	// establish the base pointer for this routine.
1086	KMP_CFI_REGISTER rbp
1087
1088#if OMPT_SUPPORT
1089	movq	%rbp, (%r9)	// save exit_frame
1090#endif
1091
1092	pushq 	%rbx		// %rbx is callee-saved register
1093	pushq	%rsi		// Put gtid on stack so can pass &tgid to pkfn
1094	pushq	%rdx		// Put tid on stack so can pass &tid to pkfn
1095
1096	movq	%rcx, %rax	// Stack alignment calculation begins; argc -> %rax
1097	movq	$0, %rbx	// constant for cmovs later
1098	subq	$4, %rax	// subtract four args passed in registers to pkfn
1099#if KMP_MIC
1100	js	KMP_LABEL(kmp_0)	// jump to movq
1101	jmp	KMP_LABEL(kmp_0_exit)	// jump ahead
1102KMP_LABEL(kmp_0):
1103	movq	%rbx, %rax	// zero negative value in %rax <- max(0, argc-4)
1104KMP_LABEL(kmp_0_exit):
1105#else
1106	cmovsq	%rbx, %rax	// zero negative value in %rax <- max(0, argc-4)
1107#endif // KMP_MIC
1108
1109	movq	%rax, %rsi	// save max(0, argc-4) -> %rsi for later
1110	shlq 	$3, %rax	// Number of bytes used on stack: max(0, argc-4)*8
1111
1112	movq 	%rsp, %rdx	//
1113	subq 	%rax, %rdx	// %rsp-(max(0,argc-4)*8) -> %rdx --
1114				// without align, stack ptr would be this
1115	movq 	%rdx, %rax	// Save to %rax
1116
1117	andq 	$0xFFFFFFFFFFFFFF80, %rax  // mask off lower 7 bits (128 bytes align)
1118	subq 	%rax, %rdx	// Amount to subtract from %rsp
1119	subq 	%rdx, %rsp	// Prepare the stack ptr --
1120				// now %rsp will align to 128-byte boundary at call site
1121
1122				// setup pkfn parameter reg and stack
1123	movq	%rcx, %rax	// argc -> %rax
1124	cmpq	$0, %rsi
1125	je	KMP_LABEL(kmp_invoke_pass_parms)	// jump ahead if no parms to push
1126	shlq	$3, %rcx	// argc*8 -> %rcx
1127	movq 	%r8, %rdx	// p_argv -> %rdx
1128	addq	%rcx, %rdx	// &p_argv[argc] -> %rdx
1129
1130	movq	%rsi, %rcx	// max (0, argc-4) -> %rcx
1131
1132KMP_LABEL(kmp_invoke_push_parms):
1133	// push nth - 7th parms to pkfn on stack
1134	subq	$8, %rdx	// decrement p_argv pointer to previous parm
1135	movq	(%rdx), %rsi	// p_argv[%rcx-1] -> %rsi
1136	pushq	%rsi		// push p_argv[%rcx-1] onto stack (reverse order)
1137	subl	$1, %ecx
1138
1139// C69570: "X86_64_RELOC_BRANCH not supported" error at linking on mac_32e
1140//		if the name of the label that is an operand of this jecxz starts with a dot (".");
1141//	   Apple's linker does not support 1-byte length relocation;
1142//         Resolution: replace all .labelX entries with L_labelX.
1143
1144	jecxz   KMP_LABEL(kmp_invoke_pass_parms)  // stop when four p_argv[] parms left
1145	jmp	KMP_LABEL(kmp_invoke_push_parms)
1146	ALIGN 3
1147KMP_LABEL(kmp_invoke_pass_parms):	// put 1st - 6th parms to pkfn in registers.
1148				// order here is important to avoid trashing
1149				// registers used for both input and output parms!
1150	movq	%rdi, %rbx	// pkfn -> %rbx
1151	leaq	__gtid(%rbp), %rdi // &gtid -> %rdi (store 1st parm to pkfn)
1152	leaq	__tid(%rbp), %rsi  // &tid -> %rsi (store 2nd parm to pkfn)
1153
1154	movq	%r8, %r11	// p_argv -> %r11
1155
1156#if KMP_MIC
1157	cmpq	$4, %rax	// argc >= 4?
1158	jns	KMP_LABEL(kmp_4)	// jump to movq
1159	jmp	KMP_LABEL(kmp_4_exit)	// jump ahead
1160KMP_LABEL(kmp_4):
1161	movq	24(%r11), %r9	// p_argv[3] -> %r9 (store 6th parm to pkfn)
1162KMP_LABEL(kmp_4_exit):
1163
1164	cmpq	$3, %rax	// argc >= 3?
1165	jns	KMP_LABEL(kmp_3)	// jump to movq
1166	jmp	KMP_LABEL(kmp_3_exit)	// jump ahead
1167KMP_LABEL(kmp_3):
1168	movq	16(%r11), %r8	// p_argv[2] -> %r8 (store 5th parm to pkfn)
1169KMP_LABEL(kmp_3_exit):
1170
1171	cmpq	$2, %rax	// argc >= 2?
1172	jns	KMP_LABEL(kmp_2)	// jump to movq
1173	jmp	KMP_LABEL(kmp_2_exit)	// jump ahead
1174KMP_LABEL(kmp_2):
1175	movq	8(%r11), %rcx	// p_argv[1] -> %rcx (store 4th parm to pkfn)
1176KMP_LABEL(kmp_2_exit):
1177
1178	cmpq	$1, %rax	// argc >= 1?
1179	jns	KMP_LABEL(kmp_1)	// jump to movq
1180	jmp	KMP_LABEL(kmp_1_exit)	// jump ahead
1181KMP_LABEL(kmp_1):
1182	movq	(%r11), %rdx	// p_argv[0] -> %rdx (store 3rd parm to pkfn)
1183KMP_LABEL(kmp_1_exit):
1184#else
1185	cmpq	$4, %rax	// argc >= 4?
1186	cmovnsq	24(%r11), %r9	// p_argv[3] -> %r9 (store 6th parm to pkfn)
1187
1188	cmpq	$3, %rax	// argc >= 3?
1189	cmovnsq	16(%r11), %r8	// p_argv[2] -> %r8 (store 5th parm to pkfn)
1190
1191	cmpq	$2, %rax	// argc >= 2?
1192	cmovnsq	8(%r11), %rcx	// p_argv[1] -> %rcx (store 4th parm to pkfn)
1193
1194	cmpq	$1, %rax	// argc >= 1?
1195	cmovnsq	(%r11), %rdx	// p_argv[0] -> %rdx (store 3rd parm to pkfn)
1196#endif // KMP_MIC
1197
1198	call	*%rbx		// call (*pkfn)();
1199	movq	$1, %rax	// move 1 into return register;
1200
1201	movq	-8(%rbp), %rbx	// restore %rbx	using %rbp since %rsp was modified
1202	movq 	%rbp, %rsp	// restore stack pointer
1203	popq 	%rbp		// restore frame pointer
1204	KMP_CFI_DEF rsp,8
1205	ret
1206
1207	DEBUG_INFO __kmp_invoke_microtask
1208// -- End  __kmp_invoke_microtask
1209
1210// kmp_uint64
1211// __kmp_hardware_timestamp(void)
1212        .text
1213	PROC  __kmp_hardware_timestamp
1214	rdtsc
1215	shlq    $32, %rdx
1216	orq     %rdx, %rax
1217	ret
1218
1219	DEBUG_INFO __kmp_hardware_timestamp
1220// -- End  __kmp_hardware_timestamp
1221
1222//------------------------------------------------------------------------
1223// FUNCTION __kmp_bsr32
1224//
1225// int
1226// __kmp_bsr32( int );
1227        .text
1228        PROC  __kmp_bsr32
1229
1230        bsr    %edi,%eax
1231        ret
1232
1233        DEBUG_INFO __kmp_bsr32
1234
1235// -----------------------------------------------------------------------
1236#endif /* KMP_ARCH_X86_64 */
1237
1238// '
1239#if (KMP_OS_LINUX || KMP_OS_DARWIN || KMP_OS_WINDOWS) && KMP_ARCH_AARCH64
1240
1241//------------------------------------------------------------------------
1242// int
1243// __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),
1244//                         int gtid, int tid,
1245//                         int argc, void *p_argv[]
1246// #if OMPT_SUPPORT
1247//                         ,
1248//                         void **exit_frame_ptr
1249// #endif
1250//                       ) {
1251// #if OMPT_SUPPORT
1252//   *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
1253// #endif
1254//
1255//   (*pkfn)( & gtid, & tid, argv[0], ... );
1256//
1257// // FIXME: This is done at call-site and can be removed here.
1258// #if OMPT_SUPPORT
1259//   *exit_frame_ptr = 0;
1260// #endif
1261//
1262//   return 1;
1263// }
1264//
1265// parameters:
1266//	x0:	pkfn
1267//	w1:	gtid
1268//	w2:	tid
1269//	w3:	argc
1270//	x4:	p_argv
1271//	x5:	&exit_frame
1272//
1273// locals:
1274//	__gtid:	gtid parm pushed on stack so can pass &gtid to pkfn
1275//	__tid:	tid parm pushed on stack so can pass &tid to pkfn
1276//
1277// reg temps:
1278//	 x8:	used to hold pkfn address
1279//	 w9:	used as temporary for number of pkfn parms
1280//	x10:	used to traverse p_argv array
1281//	x11:	used as temporary for stack placement calculation
1282//	x12:	used as temporary for stack parameters
1283//	x19:	used to preserve exit_frame_ptr, callee-save
1284//
1285// return:	w0	(always 1/TRUE)
1286//
1287
1288__gtid = 4
1289__tid = 8
1290
1291// -- Begin __kmp_invoke_microtask
1292// mark_begin;
1293	.text
1294	PROC __kmp_invoke_microtask
1295
1296	stp	x29, x30, [sp, #-16]!
1297# if OMPT_SUPPORT
1298	stp	x19, x20, [sp, #-16]!
1299# endif
1300	mov	x29, sp
1301
1302	orr	w9, wzr, #1
1303	add	w9, w9, w3, lsr #1
1304	sub	sp, sp, w9, uxtw #4
1305	mov	x11, sp
1306
1307	mov	x8, x0
1308	str	w1, [x29, #-__gtid]
1309	str	w2, [x29, #-__tid]
1310	mov	w9, w3
1311	mov	x10, x4
1312# if OMPT_SUPPORT
1313	mov	x19, x5
1314	str	x29, [x19]
1315# endif
1316
1317	sub	x0, x29, #__gtid
1318	sub	x1, x29, #__tid
1319
1320	cbz	w9, KMP_LABEL(kmp_1)
1321	ldr	x2, [x10]
1322
1323	sub	w9, w9, #1
1324	cbz	w9, KMP_LABEL(kmp_1)
1325	ldr	x3, [x10, #8]!
1326
1327	sub	w9, w9, #1
1328	cbz	w9, KMP_LABEL(kmp_1)
1329	ldr	x4, [x10, #8]!
1330
1331	sub	w9, w9, #1
1332	cbz	w9, KMP_LABEL(kmp_1)
1333	ldr	x5, [x10, #8]!
1334
1335	sub	w9, w9, #1
1336	cbz	w9, KMP_LABEL(kmp_1)
1337	ldr	x6, [x10, #8]!
1338
1339	sub	w9, w9, #1
1340	cbz	w9, KMP_LABEL(kmp_1)
1341	ldr	x7, [x10, #8]!
1342
1343KMP_LABEL(kmp_0):
1344	sub	w9, w9, #1
1345	cbz	w9, KMP_LABEL(kmp_1)
1346	ldr	x12, [x10, #8]!
1347	str	x12, [x11], #8
1348	b	KMP_LABEL(kmp_0)
1349KMP_LABEL(kmp_1):
1350	blr	x8
1351	orr	w0, wzr, #1
1352	mov	sp, x29
1353# if OMPT_SUPPORT
1354	str	xzr, [x19]
1355	ldp	x19, x20, [sp], #16
1356# endif
1357	ldp	x29, x30, [sp], #16
1358	ret
1359
1360	DEBUG_INFO __kmp_invoke_microtask
1361// -- End  __kmp_invoke_microtask
1362
1363#endif /* (KMP_OS_LINUX || KMP_OS_DARWIN || KMP_OS_WINDOWS) && KMP_ARCH_AARCH64 */
1364
1365#if (KMP_OS_LINUX || KMP_OS_DARWIN || KMP_OS_WINDOWS) && KMP_ARCH_ARM
1366
1367//------------------------------------------------------------------------
1368// int
1369// __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),
1370//                         int gtid, int tid,
1371//                         int argc, void *p_argv[]
1372// #if OMPT_SUPPORT
1373//                         ,
1374//                         void **exit_frame_ptr
1375// #endif
1376//                       ) {
1377// #if OMPT_SUPPORT
1378//   *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
1379// #endif
1380//
1381//   (*pkfn)( & gtid, & tid, argv[0], ... );
1382//
1383// // FIXME: This is done at call-site and can be removed here.
1384// #if OMPT_SUPPORT
1385//   *exit_frame_ptr = 0;
1386// #endif
1387//
1388//   return 1;
1389// }
1390//
1391// parameters:
1392//	r0:	pkfn
1393//	r1:	gtid
1394//	r2:	tid
1395//	r3:	argc
1396//	r4(stack):	p_argv
1397//	r5(stack):	&exit_frame
1398//
1399// locals:
1400//	__gtid:	gtid parm pushed on stack so can pass &gtid to pkfn
1401//	__tid:	tid parm pushed on stack so can pass &tid to pkfn
1402//
1403// reg temps:
1404//	 r4:	used to hold pkfn address
1405//	 r5:	used as temporary for number of pkfn parms
1406//	 r6:	used to traverse p_argv array
1407//	 r7:	frame pointer (in some configurations)
1408//	 r8:	used as temporary for stack placement calculation
1409//	 	and as pointer to base of callee saved area
1410//	 r9:	used as temporary for stack parameters
1411//	r10:	used to preserve exit_frame_ptr, callee-save
1412//	r11:	frame pointer (in some configurations)
1413//
1414// return:	r0	(always 1/TRUE)
1415//
1416
1417__gtid = 4
1418__tid = 8
1419
1420// -- Begin __kmp_invoke_microtask
1421// mark_begin;
1422	.text
1423	PROC __kmp_invoke_microtask
1424
1425	// Pushing one extra register (r3) to keep the stack aligned
1426	// for when we call pkfn below
1427	push	{r3-r11,lr}
1428	// Load p_argv and &exit_frame
1429	ldr	r4, [sp, #10*4]
1430# if OMPT_SUPPORT
1431	ldr	r5, [sp, #11*4]
1432# endif
1433
1434# if KMP_OS_DARWIN || (defined(__thumb__) && !KMP_OS_WINDOWS)
1435# define FP r7
1436# define FPOFF 4*4
1437#else
1438# define FP r11
1439# define FPOFF 8*4
1440#endif
1441	add	FP, sp, #FPOFF
1442# if OMPT_SUPPORT
1443	mov	r10, r5
1444	str	FP, [r10]
1445# endif
1446	mov	r8, sp
1447
1448	// Calculate how much stack to allocate, in increments of 8 bytes.
1449	// We strictly need 4*(argc-2) bytes (2 arguments are passed in
1450	// registers) but allocate 4*argc for simplicity (to avoid needing
1451	// to handle the argc<2 cases). We align the number of bytes
1452	// allocated to 8 bytes, to keep the stack aligned. (Since we
1453	// already allocate more than enough, it's ok to round down
1454	// instead of up for the alignment.) We allocate another extra
1455	// 8 bytes for gtid and tid.
1456	mov	r5, #1
1457	add	r5, r5, r3, lsr #1
1458	sub	sp, sp, r5, lsl #3
1459
1460	str	r1, [r8, #-__gtid]
1461	str	r2, [r8, #-__tid]
1462	mov	r5, r3
1463	mov	r6, r4
1464	mov	r4, r0
1465
1466	// Prepare the first 2 parameters to pkfn - pointers to gtid and tid
1467	// in our stack frame.
1468	sub	r0, r8, #__gtid
1469	sub	r1, r8, #__tid
1470
1471	mov	r8, sp
1472
1473	// Load p_argv[0] and p_argv[1] into r2 and r3, if argc >= 1/2
1474	cmp	r5, #0
1475	beq	KMP_LABEL(kmp_1)
1476	ldr	r2, [r6]
1477
1478	subs	r5, r5, #1
1479	beq	KMP_LABEL(kmp_1)
1480	ldr	r3, [r6, #4]!
1481
1482	// Loop, loading the rest of p_argv and writing the elements on the
1483	// stack.
1484KMP_LABEL(kmp_0):
1485	subs	r5, r5, #1
1486	beq	KMP_LABEL(kmp_1)
1487	ldr	r12, [r6, #4]!
1488	str	r12, [r8], #4
1489	b	KMP_LABEL(kmp_0)
1490KMP_LABEL(kmp_1):
1491	blx	r4
1492	mov	r0, #1
1493
1494	sub	r4, FP, #FPOFF
1495	mov	sp, r4
1496# undef FP
1497# undef FPOFF
1498
1499# if OMPT_SUPPORT
1500	mov	r1, #0
1501	str	r1, [r10]
1502# endif
1503	pop	{r3-r11,pc}
1504
1505	DEBUG_INFO __kmp_invoke_microtask
1506// -- End  __kmp_invoke_microtask
1507
1508#endif /* (KMP_OS_LINUX || KMP_OS_DARWIN || KMP_OS_WINDOWS) && KMP_ARCH_AARCH64 */
1509
1510#if KMP_ARCH_PPC64
1511
1512//------------------------------------------------------------------------
1513// int
1514// __kmp_invoke_microtask( void (*pkfn) (int gtid, int tid, ...),
1515//                         int gtid, int tid,
1516//                         int argc, void *p_argv[]
1517// #if OMPT_SUPPORT
1518//                         ,
1519//                         void **exit_frame_ptr
1520// #endif
1521//                       ) {
1522// #if OMPT_SUPPORT
1523//   *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
1524// #endif
1525//
1526//   (*pkfn)( & gtid, & tid, argv[0], ... );
1527//
1528// // FIXME: This is done at call-site and can be removed here.
1529// #if OMPT_SUPPORT
1530//   *exit_frame_ptr = 0;
1531// #endif
1532//
1533//   return 1;
1534// }
1535//
1536// parameters:
1537//	r3:	pkfn
1538//	r4:	gtid
1539//	r5:	tid
1540//	r6:	argc
1541//	r7:	p_argv
1542//	r8:	&exit_frame
1543//
1544// return:	r3	(always 1/TRUE)
1545//
1546	.text
1547# if KMP_ARCH_PPC64_ELFv2
1548	.abiversion 2
1549# endif
1550	.globl	__kmp_invoke_microtask
1551
1552# if KMP_ARCH_PPC64_ELFv2
1553	.p2align	4
1554# else
1555	.p2align	2
1556# endif
1557
1558	.type	__kmp_invoke_microtask,@function
1559
1560# if KMP_ARCH_PPC64_ELFv2
1561__kmp_invoke_microtask:
1562.Lfunc_begin0:
1563.Lfunc_gep0:
1564	addis 2, 12, .TOC.-.Lfunc_gep0@ha
1565	addi 2, 2, .TOC.-.Lfunc_gep0@l
1566.Lfunc_lep0:
1567	.localentry	__kmp_invoke_microtask, .Lfunc_lep0-.Lfunc_gep0
1568# else
1569	.section	.opd,"aw",@progbits
1570__kmp_invoke_microtask:
1571	.p2align	3
1572	.quad	.Lfunc_begin0
1573	.quad	.TOC.@tocbase
1574	.quad	0
1575	.text
1576.Lfunc_begin0:
1577# endif
1578
1579// -- Begin __kmp_invoke_microtask
1580// mark_begin;
1581
1582// We need to allocate a stack frame large enough to hold all of the parameters
1583// on the stack for the microtask plus what this function needs. That's 48
1584// bytes under the ELFv1 ABI (32 bytes under ELFv2), plus 8*(2 + argc) for the
1585// parameters to the microtask, plus 8 bytes to store the values of r4 and r5,
1586// and 8 bytes to store r31. With OMP-T support, we need an additional 8 bytes
1587// to save r30 to hold a copy of r8.
1588
1589	.cfi_startproc
1590	mflr 0
1591	std 31, -8(1)
1592	std 0, 16(1)
1593
1594// This is unusual because normally we'd set r31 equal to r1 after the stack
1595// frame is established. In this case, however, we need to dynamically compute
1596// the stack frame size, and so we keep a direct copy of r1 to access our
1597// register save areas and restore the r1 value before returning.
1598	mr 31, 1
1599	.cfi_def_cfa_register r31
1600	.cfi_offset r31, -8
1601	.cfi_offset lr, 16
1602
1603// Compute the size necessary for the local stack frame.
1604# if KMP_ARCH_PPC64_ELFv2
1605	li 12, 72
1606# else
1607	li 12, 88
1608# endif
1609	sldi 0, 6, 3
1610	add 12, 0, 12
1611	neg 12, 12
1612
1613// We need to make sure that the stack frame stays aligned (to 16 bytes).
1614	li 0, -16
1615	and 12, 0, 12
1616
1617// Establish the local stack frame.
1618	stdux 1, 1, 12
1619
1620# if OMPT_SUPPORT
1621	.cfi_offset r30, -16
1622	std 30, -16(31)
1623	std 1, 0(8)
1624	mr 30, 8
1625# endif
1626
1627// Store gtid and tid to the stack because they're passed by reference to the microtask.
1628	stw 4, -20(31)
1629	stw 5, -24(31)
1630
1631	mr 12, 6
1632	mr 4, 7
1633
1634	cmpwi 0, 12, 1
1635	blt	 0, .Lcall
1636
1637	ld 5, 0(4)
1638
1639	cmpwi 0, 12, 2
1640	blt	 0, .Lcall
1641
1642	ld 6, 8(4)
1643
1644	cmpwi 0, 12, 3
1645	blt	 0, .Lcall
1646
1647	ld 7, 16(4)
1648
1649	cmpwi 0, 12, 4
1650	blt	 0, .Lcall
1651
1652	ld 8, 24(4)
1653
1654	cmpwi 0, 12, 5
1655	blt	 0, .Lcall
1656
1657	ld 9, 32(4)
1658
1659	cmpwi 0, 12, 6
1660	blt	 0, .Lcall
1661
1662	ld 10, 40(4)
1663
1664	cmpwi 0, 12, 7
1665	blt	 0, .Lcall
1666
1667// There are more than 6 microtask parameters, so we need to store the
1668// remainder to the stack.
1669	addi 12, 12, -6
1670	mtctr 12
1671
1672// These are set to 8 bytes before the first desired store address (we're using
1673// pre-increment loads and stores in the loop below). The parameter save area
1674// for the microtask begins 48 + 8*8 == 112 bytes above r1 for ELFv1 and
1675// 32 + 8*8 == 96 bytes above r1 for ELFv2.
1676	addi 4, 4, 40
1677# if KMP_ARCH_PPC64_ELFv2
1678	addi 12, 1, 88
1679# else
1680	addi 12, 1, 104
1681# endif
1682
1683.Lnext:
1684	ldu 0, 8(4)
1685	stdu 0, 8(12)
1686	bdnz .Lnext
1687
1688.Lcall:
1689# if KMP_ARCH_PPC64_ELFv2
1690	std 2, 24(1)
1691	mr 12, 3
1692#else
1693	std 2, 40(1)
1694// For ELFv1, we need to load the actual function address from the function descriptor.
1695	ld 12, 0(3)
1696	ld 2, 8(3)
1697	ld 11, 16(3)
1698#endif
1699
1700	addi 3, 31, -20
1701	addi 4, 31, -24
1702
1703	mtctr 12
1704	bctrl
1705# if KMP_ARCH_PPC64_ELFv2
1706	ld 2, 24(1)
1707# else
1708	ld 2, 40(1)
1709# endif
1710
1711# if OMPT_SUPPORT
1712	li 3, 0
1713	std 3, 0(30)
1714# endif
1715
1716	li 3, 1
1717
1718# if OMPT_SUPPORT
1719	ld 30, -16(31)
1720# endif
1721
1722	mr 1, 31
1723	ld 0, 16(1)
1724	ld 31, -8(1)
1725	mtlr 0
1726	blr
1727
1728	.long	0
1729	.quad	0
1730.Lfunc_end0:
1731	.size	__kmp_invoke_microtask, .Lfunc_end0-.Lfunc_begin0
1732	.cfi_endproc
1733
1734// -- End  __kmp_invoke_microtask
1735
1736#endif /* KMP_ARCH_PPC64 */
1737
1738#if KMP_ARCH_RISCV64
1739
1740//------------------------------------------------------------------------
1741//
1742// typedef void (*microtask_t)(int *gtid, int *tid, ...);
1743//
1744// int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
1745//                            void *p_argv[]
1746// #if OMPT_SUPPORT
1747//                            ,
1748//                            void **exit_frame_ptr
1749// #endif
1750//                            ) {
1751// #if OMPT_SUPPORT
1752//   *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
1753// #endif
1754//
1755//   (*pkfn)(&gtid, &tid, argv[0], ...);
1756//
1757//   return 1;
1758// }
1759//
1760// Parameters:
1761//   a0: pkfn
1762//   a1: gtid
1763//   a2: tid
1764//   a3: argc
1765//   a4: p_argv
1766//   a5: exit_frame_ptr
1767//
1768// Locals:
1769//   __gtid: gtid param pushed on stack so can pass &gtid to pkfn
1770//   __tid: tid param pushed on stack so can pass &tid to pkfn
1771//
1772// Temp. registers:
1773//
1774//  t0: used to calculate the dynamic stack size / used to hold pkfn address
1775//  t1: used as temporary for stack placement calculation
1776//  t2: used as temporary for stack arguments
1777//  t3: used as temporary for number of remaining pkfn parms
1778//  t4: used to traverse p_argv array
1779//
1780// return: a0 (always 1/TRUE)
1781//
1782
1783__gtid = -20
1784__tid = -24
1785
1786// -- Begin __kmp_invoke_microtask
1787// mark_begin;
1788	.text
1789	.globl	__kmp_invoke_microtask
1790	.p2align	1
1791	.type	__kmp_invoke_microtask,@function
1792__kmp_invoke_microtask:
1793	.cfi_startproc
1794
1795	// First, save ra and fp
1796	addi	sp, sp, -16
1797	sd	ra, 8(sp)
1798	sd	fp, 0(sp)
1799	addi	fp, sp, 16
1800	.cfi_def_cfa	fp, 0
1801	.cfi_offset	ra, -8
1802	.cfi_offset	fp, -16
1803
1804	// Compute the dynamic stack size:
1805	//
1806	// - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them by
1807	//   reference
1808	// - We need 8 bytes for each argument that cannot be passed to the 'pkfn'
1809	//   function by register. Given that we have 8 of such registers (a[0-7])
1810	//   and two + 'argc' arguments (consider &gtid and &tid), we need to
1811	//   reserve max(0, argc - 6)*8 extra bytes
1812	//
1813	// The total number of bytes is then max(0, argc - 6)*8 + 8
1814
1815	// Compute max(0, argc - 6) using the following bithack:
1816	// max(0, x) = x - (x & (x >> 31)), where x := argc - 6
1817	// Source: http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax
1818	addi	t0, a3, -6
1819	srai	t1, t0, 31
1820	and	t1, t0, t1
1821	sub	t0, t0, t1
1822
1823	addi	t0, t0, 1
1824
1825	slli	t0, t0, 3
1826	sub	sp, sp, t0
1827
1828	// Align the stack to 16 bytes
1829	andi	sp, sp, -16
1830
1831	mv	t0, a0
1832	mv	t3, a3
1833	mv	t4, a4
1834
1835#if OMPT_SUPPORT
1836	// Save frame pointer into exit_frame
1837	sd	fp, 0(a5)
1838#endif
1839
1840	// Prepare arguments for the pkfn function (first 8 using a0-a7 registers)
1841
1842	sw	a1, __gtid(fp)
1843	sw	a2, __tid(fp)
1844
1845	addi	a0, fp, __gtid
1846	addi	a1, fp, __tid
1847
1848	beqz	t3, .L_kmp_3
1849	ld	a2, 0(t4)
1850
1851	addi	t3, t3, -1
1852	beqz	t3, .L_kmp_3
1853	ld	a3, 8(t4)
1854
1855	addi	t3, t3, -1
1856	beqz	t3, .L_kmp_3
1857	ld	a4, 16(t4)
1858
1859	addi	t3, t3, -1
1860	beqz	t3, .L_kmp_3
1861	ld	a5, 24(t4)
1862
1863	addi	t3, t3, -1
1864	beqz	t3, .L_kmp_3
1865	ld	a6, 32(t4)
1866
1867	addi	t3, t3, -1
1868	beqz	t3, .L_kmp_3
1869	ld	a7, 40(t4)
1870
1871	// Prepare any additional argument passed through the stack
1872	addi	t4, t4, 48
1873	mv	t1, sp
1874	j .L_kmp_2
1875.L_kmp_1:
1876	ld	t2, 0(t4)
1877	sd	t2, 0(t1)
1878	addi	t4, t4, 8
1879	addi	t1, t1, 8
1880.L_kmp_2:
1881	addi	t3, t3, -1
1882	bnez	t3, .L_kmp_1
1883
1884.L_kmp_3:
1885	// Call pkfn function
1886	jalr	t0
1887
1888	// Restore stack and return
1889
1890	addi	a0, zero, 1
1891
1892	addi	sp, fp, -16
1893	ld	fp, 0(sp)
1894	ld	ra, 8(sp)
1895	addi	sp, sp, 16
1896	ret
1897.Lfunc_end0:
1898	.size	__kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask
1899	.cfi_endproc
1900
1901// -- End  __kmp_invoke_microtask
1902
1903#endif /* KMP_ARCH_RISCV64 */
1904
1905#if KMP_ARCH_LOONGARCH64
1906
1907//------------------------------------------------------------------------
1908//
1909// typedef void (*microtask_t)(int *gtid, int *tid, ...);
1910//
1911// int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
1912//                            void *p_argv[]
1913// #if OMPT_SUPPORT
1914//                            ,
1915//                            void **exit_frame_ptr
1916// #endif
1917//                            ) {
1918// #if OMPT_SUPPORT
1919//   *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
1920// #endif
1921//
1922//   (*pkfn)(&gtid, &tid, argv[0], ...);
1923//
1924//   return 1;
1925// }
1926//
1927// Parameters:
1928//   a0: pkfn
1929//   a1: gtid
1930//   a2: tid
1931//   a3: argc
1932//   a4: p_argv
1933//   a5: exit_frame_ptr
1934//
1935// Locals:
1936//   __gtid: gtid param pushed on stack so can pass &gtid to pkfn
1937//   __tid: tid param pushed on stack so can pass &tid to pkfn
1938//
1939// Temp registers:
1940//
1941//  t0: used to calculate the dynamic stack size / used to hold pkfn address
1942//  t1: used as temporary for stack placement calculation
1943//  t2: used as temporary for stack arguments
1944//  t3: used as temporary for number of remaining pkfn parms
1945//  t4: used to traverse p_argv array
1946//
1947// return: a0 (always 1/TRUE)
1948//
1949
1950// -- Begin __kmp_invoke_microtask
1951// mark_begin;
1952	.text
1953	.globl	__kmp_invoke_microtask
1954	.p2align	2
1955	.type	__kmp_invoke_microtask,@function
1956__kmp_invoke_microtask:
1957	.cfi_startproc
1958
1959	// First, save ra and fp
1960	addi.d	$sp, $sp, -16
1961	st.d	$ra, $sp, 8
1962	st.d	$fp, $sp, 0
1963	addi.d	$fp, $sp, 16
1964	.cfi_def_cfa	22, 0
1965	.cfi_offset	1, -8
1966	.cfi_offset	22, -16
1967
1968	// Compute the dynamic stack size:
1969	//
1970	// - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them by
1971	//   reference
1972	// - We need 8 bytes for each argument that cannot be passed to the 'pkfn'
1973	//   function by register. Given that we have 8 of such registers (a[0-7])
1974	//   and two + 'argc' arguments (consider &gtid and &tid), we need to
1975	//   reserve max(0, argc - 6)*8 extra bytes
1976	//
1977	// The total number of bytes is then max(0, argc - 6)*8 + 8
1978
1979	addi.d  $t0, $a3, -6
1980	slt  $t1, $t0, $zero
1981	masknez  $t0, $t0, $t1
1982	addi.d  $t0, $t0, 1
1983	slli.d	$t0, $t0, 3
1984	sub.d	$sp, $sp, $t0
1985
1986	// Align the stack to 16 bytes
1987	bstrins.d $sp, $zero, 3, 0
1988
1989	move	$t0, $a0
1990	move	$t3, $a3
1991	move	$t4, $a4
1992
1993#if OMPT_SUPPORT
1994	// Save frame pointer into exit_frame
1995	st.d	$fp, $a5, 0
1996#endif
1997
1998	// Prepare arguments for the pkfn function (first 8 using a0-a7 registers)
1999
2000	st.w	$a1, $fp, -20
2001	st.w	$a2, $fp, -24
2002
2003	addi.d	$a0, $fp, -20
2004	addi.d	$a1, $fp, -24
2005
2006	beqz	$t3, .L_kmp_3
2007	ld.d	$a2, $t4, 0
2008
2009	addi.d	$t3, $t3, -1
2010	beqz	$t3, .L_kmp_3
2011	ld.d	$a3, $t4, 8
2012
2013	addi.d	$t3, $t3, -1
2014	beqz	$t3, .L_kmp_3
2015	ld.d	$a4, $t4, 16
2016
2017	addi.d	$t3, $t3, -1
2018	beqz	$t3, .L_kmp_3
2019	ld.d	$a5, $t4, 24
2020
2021	addi.d	$t3, $t3, -1
2022	beqz	$t3, .L_kmp_3
2023	ld.d	$a6, $t4, 32
2024
2025	addi.d	$t3, $t3, -1
2026	beqz	$t3, .L_kmp_3
2027	ld.d	$a7, $t4, 40
2028
2029	// Prepare any additional argument passed through the stack
2030	addi.d	$t4, $t4, 48
2031	move	$t1, $sp
2032	b .L_kmp_2
2033.L_kmp_1:
2034	ld.d	$t2, $t4, 0
2035	st.d	$t2, $t1, 0
2036	addi.d	$t4, $t4, 8
2037	addi.d	$t1, $t1, 8
2038.L_kmp_2:
2039	addi.d	$t3, $t3, -1
2040	bnez	$t3, .L_kmp_1
2041
2042.L_kmp_3:
2043	// Call pkfn function
2044	jirl	$ra, $t0, 0
2045
2046	// Restore stack and return
2047
2048	addi.d	$a0, $zero, 1
2049
2050	addi.d	$sp, $fp, -16
2051	ld.d	$fp, $sp, 0
2052	ld.d	$ra, $sp, 8
2053	addi.d	$sp, $sp, 16
2054	jr $ra
2055.Lfunc_end0:
2056	.size	__kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask
2057	.cfi_endproc
2058
2059// -- End  __kmp_invoke_microtask
2060
2061#endif /* KMP_ARCH_LOONGARCH64 */
2062
2063#if KMP_ARCH_VE
2064
2065//------------------------------------------------------------------------
2066//
2067// typedef void (*microtask_t)(int *gtid, int *tid, ...);
2068//
2069// int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
2070//                            void *p_argv[]
2071// #if OMPT_SUPPORT
2072//                            ,
2073//                            void **exit_frame_ptr
2074// #endif
2075//                            ) {
2076// #if OMPT_SUPPORT
2077//   *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
2078// #endif
2079//
2080//   (*pkfn)(&gtid, &tid, argv[0], ...);
2081//
2082//   return 1;
2083// }
2084//
2085// Parameters:
2086//   s0: pkfn
2087//   s1: gtid
2088//   s2: tid
2089//   s3: argc
2090//   s4: p_argv
2091//   s5: exit_frame_ptr
2092//
2093// Locals:
2094//   __gtid: gtid param pushed on stack so can pass &gtid to pkfn
2095//   __tid: tid param pushed on stack so can pass &tid to pkfn
2096//
2097// Temp. registers:
2098//
2099//  s34: used to calculate the dynamic stack size
2100//  s35: used as temporary for stack placement calculation
2101//  s36: used as temporary for stack arguments
2102//  s37: used as temporary for number of remaining pkfn parms
2103//  s38: used to traverse p_argv array
2104//
2105// return: s0 (always 1/TRUE)
2106//
2107
2108__gtid = -4
2109__tid = -8
2110
2111// -- Begin __kmp_invoke_microtask
2112// mark_begin;
2113	.text
2114	.globl	__kmp_invoke_microtask
2115	// A function requires 8 bytes align.
2116	.p2align	3
2117	.type	__kmp_invoke_microtask,@function
2118__kmp_invoke_microtask:
2119	.cfi_startproc
2120
2121	// First, save fp and lr.  VE stores them at caller stack frame.
2122	st	%fp, 0(, %sp)
2123	st	%lr, 8(, %sp)
2124	or	%fp, 0, %sp
2125	.cfi_def_cfa	%fp, 0
2126	.cfi_offset	%lr, 8
2127	.cfi_offset	%fp, 0
2128
2129	// Compute the dynamic stack size:
2130	//
2131	// - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them
2132	//   by reference
2133	// - We need 8 bytes for whole arguments.  We have two + 'argc'
2134	//   arguments (condider &gtid and &tid).  We need to reserve
2135	//   (argc + 2) * 8 bytes.
2136	// - We need 176 bytes for RSA and others
2137	//
2138	// The total number of bytes is then (argc + 2) * 8 + 8 + 176.
2139	//
2140	// |------------------------------|
2141	// | return address of callee     | 8(%fp)
2142	// |------------------------------|
2143	// | frame pointer of callee      | 0(%fp)
2144	// |------------------------------| <------------------ %fp
2145	// | __tid / __gtid               | -8(%fp) / -4(%fp)
2146	// |------------------------------|
2147	// | argc+2 for arguments         | 176(%sp)
2148	// |------------------------------|
2149	// | RSA                          |
2150	// |------------------------------|
2151	// | return address               |
2152	// |------------------------------|
2153	// | frame pointer                |
2154	// |------------------------------| <------------------ %sp
2155
2156	adds.w.sx	%s34, 2, %s3
2157	sll	%s34, %s34, 3
2158	lea	%s34, 184(, %s34)
2159	subs.l	%sp, %sp, %s34
2160
2161	// Align the stack to 16 bytes.
2162	and	%sp, -16, %sp
2163
2164	// Save pkfn.
2165	or	%s12, 0, %s0
2166
2167	// Call host to allocate stack if it is necessary.
2168	brge.l	%sp, %sl, .L_kmp_pass
2169	ld	%s61, 24(, %tp)
2170	lea	%s63, 0x13b
2171	shm.l	%s63, 0(%s61)
2172	shm.l	%sl, 8(%s61)
2173	shm.l	%sp, 16(%s61)
2174	monc
2175
2176.L_kmp_pass:
2177	lea	%s35, 176(, %sp)
2178	adds.w.sx	%s37, 0, %s3
2179	or	%s38, 0, %s4
2180
2181#if OMPT_SUPPORT
2182	// Save frame pointer into exit_frame.
2183	st	%fp, 0(%s5)
2184#endif
2185
2186	// Prepare arguments for the pkfn function (first 8 using s0-s7
2187	// registers, but need to store stack also because of varargs).
2188
2189	stl	%s1, __gtid(%fp)
2190	stl	%s2, __tid(%fp)
2191
2192	adds.l	%s0, __gtid, %fp
2193	st	%s0, 0(, %s35)
2194	adds.l	%s1, __tid, %fp
2195	st	%s1, 8(, %s35)
2196
2197	breq.l	0, %s37, .L_kmp_call
2198	ld	%s2, 0(, %s38)
2199	st	%s2, 16(, %s35)
2200
2201	breq.l	1, %s37, .L_kmp_call
2202	ld	%s3, 8(, %s38)
2203	st	%s3, 24(, %s35)
2204
2205	breq.l	2, %s37, .L_kmp_call
2206	ld	%s4, 16(, %s38)
2207	st	%s4, 32(, %s35)
2208
2209	breq.l	3, %s37, .L_kmp_call
2210	ld	%s5, 24(, %s38)
2211	st	%s5, 40(, %s35)
2212
2213	breq.l	4, %s37, .L_kmp_call
2214	ld	%s6, 32(, %s38)
2215	st	%s6, 48(, %s35)
2216
2217	breq.l	5, %s37, .L_kmp_call
2218	ld	%s7, 40(, %s38)
2219	st	%s7, 56(, %s35)
2220
2221	breq.l	6, %s37, .L_kmp_call
2222
2223	// Prepare any additional argument passed through the stack.
2224	adds.l	%s37, -6, %s37
2225	lea	%s38, 48(, %s38)
2226	lea	%s35, 64(, %s35)
2227.L_kmp_loop:
2228	ld	%s36, 0(, %s38)
2229	st	%s36, 0(, %s35)
2230	adds.l	%s37, -1, %s37
2231	adds.l	%s38, 8, %s38
2232	adds.l	%s35, 8, %s35
2233	brne.l	0, %s37, .L_kmp_loop
2234
2235.L_kmp_call:
2236	// Call pkfn function.
2237	bsic	%lr, (, %s12)
2238
2239	// Return value.
2240	lea	%s0, 1
2241
2242	// Restore stack and return.
2243	or	%sp, 0, %fp
2244	ld	%lr, 8(, %sp)
2245	ld	%fp, 0(, %sp)
2246	b.l.t	(, %lr)
2247.Lfunc_end0:
2248	.size	__kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask
2249	.cfi_endproc
2250
2251// -- End  __kmp_invoke_microtask
2252
2253#endif /* KMP_ARCH_VE */
2254
2255#if KMP_ARCH_S390X
2256
2257//------------------------------------------------------------------------
2258//
2259// typedef void (*microtask_t)(int *gtid, int *tid, ...);
2260//
2261// int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
2262//                            void *p_argv[]
2263// #if OMPT_SUPPORT
2264//                            ,
2265//                            void **exit_frame_ptr
2266// #endif
2267//                            ) {
2268// #if OMPT_SUPPORT
2269//   *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
2270// #endif
2271//
2272//   (*pkfn)(&gtid, &tid, argv[0], ...);
2273//
2274//   return 1;
2275// }
2276//
2277// Parameters:
2278//   r2: pkfn
2279//   r3: gtid
2280//   r4: tid
2281//   r5: argc
2282//   r6: p_argv
2283//   SP+160: exit_frame_ptr
2284//
2285// Locals:
2286//   __gtid: gtid param pushed on stack so can pass &gtid to pkfn
2287//   __tid: tid param pushed on stack so can pass &tid to pkfn
2288//
2289// Temp. registers:
2290//
2291//  r0: used to fetch argv slots
2292//  r7: used as temporary for number of remaining pkfn parms
2293//  r8: argv
2294//  r9: pkfn
2295//  r10: stack size
2296//  r11: previous fp
2297//  r12: stack parameter area
2298//  r13: argv slot
2299//
2300// return: r2 (always 1/TRUE)
2301//
2302
2303// -- Begin __kmp_invoke_microtask
2304// mark_begin;
2305	.text
2306	.globl	__kmp_invoke_microtask
2307	.p2align	1
2308	.type	__kmp_invoke_microtask,@function
2309__kmp_invoke_microtask:
2310	.cfi_startproc
2311
2312	stmg	%r6,%r14,48(%r15)
2313        .cfi_offset %r6, -112
2314        .cfi_offset %r7, -104
2315        .cfi_offset %r8, -96
2316        .cfi_offset %r9, -88
2317        .cfi_offset %r10, -80
2318        .cfi_offset %r11, -72
2319        .cfi_offset %r12, -64
2320        .cfi_offset %r13, -56
2321        .cfi_offset %r14, -48
2322        .cfi_offset %r15, -40
2323	lgr	%r11,%r15
2324	.cfi_def_cfa %r11, 160
2325
2326	// Compute the dynamic stack size:
2327	//
2328	// - We need 8 bytes for storing 'gtid' and 'tid', so we can pass them by
2329	//   reference
2330	// - We need 8 bytes for each argument that cannot be passed to the 'pkfn'
2331	//   function by register. Given that we have 5 of such registers (r[2-6])
2332	//   and two + 'argc' arguments (consider &gtid and &tid), we need to
2333	//   reserve max(0, argc - 3)*8 extra bytes
2334	//
2335	// The total number of bytes is then max(0, argc - 3)*8 + 8
2336
2337	lgr	%r10,%r5
2338	aghi	%r10,-2
2339	jnm	0f
2340	lghi	%r10,0
23410:
2342	sllg	%r10,%r10,3
2343	lgr	%r12,%r10
2344	aghi	%r10,176
2345	sgr 	%r15,%r10
2346	agr	%r12,%r15
2347	stg	%r11,0(%r15)
2348
2349	lgr	%r9,%r2			// pkfn
2350
2351#if OMPT_SUPPORT
2352	// Save frame pointer into exit_frame
2353	lg	%r8,160(%r11)
2354	stg	%r11,0(%r8)
2355#endif
2356
2357	// Prepare arguments for the pkfn function (first 5 using r2-r6 registers)
2358
2359	stg     %r3,160(%r12)
2360	la	%r2,164(%r12)		// gid
2361	stg	%r4,168(%r12)
2362	la	%r3,172(%r12)		// tid
2363	lgr	%r8,%r6			// argv
2364
2365	// If argc > 0
2366	ltgr	%r7,%r5
2367	jz	1f
2368
2369	lg	%r4,0(%r8)		// argv[0]
2370	aghi	%r7,-1
2371	jz	1f
2372
2373	// If argc > 1
2374	lg	%r5,8(%r8)		// argv[1]
2375	aghi	%r7,-1
2376	jz	1f
2377
2378	// If argc > 2
2379	lg	%r6,16(%r8)		// argv[2]
2380	aghi	%r7,-1
2381	jz	1f
2382
2383	lghi	%r13,0			// Index [n]
23842:
2385	lg	%r0,24(%r13,%r8)	// argv[2+n]
2386	stg	%r0,160(%r13,%r15)	// parm[2+n]
2387	aghi	%r13,8			// Next
2388	aghi	%r7,-1
2389	jnz	2b
2390
23911:
2392	basr	%r14,%r9		// Call pkfn
2393
2394	// Restore stack and return
2395
2396	lgr	%r15,%r11
2397	lmg	%r6,%r14,48(%r15)
2398	lghi	%r2,1
2399	br	%r14
2400.Lfunc_end0:
2401	.size	__kmp_invoke_microtask, .Lfunc_end0-__kmp_invoke_microtask
2402	.cfi_endproc
2403
2404// -- End  __kmp_invoke_microtask
2405
2406#endif /* KMP_ARCH_S390X */
2407
2408#if KMP_ARCH_ARM || KMP_ARCH_MIPS
2409    .data
2410    COMMON .gomp_critical_user_, 32, 3
2411    .data
2412    .align 4
2413    .global __kmp_unnamed_critical_addr
2414__kmp_unnamed_critical_addr:
2415    .4byte .gomp_critical_user_
2416#ifdef __ELF__
2417    .size __kmp_unnamed_critical_addr,4
2418#endif
2419#endif /* KMP_ARCH_ARM */
2420
2421#if KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64 ||                   \
2422    KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || KMP_ARCH_VE ||                 \
2423    KMP_ARCH_S390X
2424#ifndef KMP_PREFIX_UNDERSCORE
2425# define KMP_PREFIX_UNDERSCORE(x) x
2426#endif
2427    .data
2428    COMMON .gomp_critical_user_, 32, 3
2429    .data
2430    .align 8
2431    .global KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr)
2432KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr):
2433    .8byte .gomp_critical_user_
2434#ifdef __ELF__
2435    .size KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr),8
2436#endif
2437#endif /* KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64 ||
2438          KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || KMP_ARCH_VE ||
2439          KMP_ARCH_S390X */
2440
2441#if KMP_OS_LINUX
2442# if KMP_ARCH_ARM || KMP_ARCH_AARCH64
2443.section .note.GNU-stack,"",%progbits
2444# elif !KMP_ARCH_WASM
2445.section .note.GNU-stack,"",@progbits
2446# endif
2447#endif
2448
2449#if KMP_ARCH_WASM
2450.data
2451.global .gomp_critical_user_
2452.global .gomp_critical_user_.var
2453.global .gomp_critical_user_.reduction.var
2454.global __kmp_unnamed_critical_addr
2455.gomp_critical_user_:
2456.zero 4
2457.size .gomp_critical_user_, 4
2458.gomp_critical_user_.var:
2459.zero 4
2460.size .gomp_critical_user_.var, 4
2461.gomp_critical_user_.reduction.var:
2462.zero 4
2463.size .gomp_critical_user_.reduction.var, 4
2464__kmp_unnamed_critical_addr:
2465    .4byte .gomp_critical_user_
2466    .size __kmp_unnamed_critical_addr, 4
2467#endif
2468