1238384Sjkim#!/usr/bin/env perl 2238384Sjkim# 3238384Sjkim# Copyright (c) 2010-2011 Intel Corp. 4238384Sjkim# Author: Vinodh.Gopal@intel.com 5238384Sjkim# Jim Guilford 6238384Sjkim# Erdinc.Ozturk@intel.com 7238384Sjkim# Maxim.Perminov@intel.com 8238384Sjkim# 9238384Sjkim# More information about algorithm used can be found at: 10238384Sjkim# http://www.cse.buffalo.edu/srds2009/escs2009_submission_Gopal.pdf 11238384Sjkim# 12238384Sjkim# ==================================================================== 13238384Sjkim# Copyright (c) 2011 The OpenSSL Project. All rights reserved. 14238384Sjkim# 15238384Sjkim# Redistribution and use in source and binary forms, with or without 16238384Sjkim# modification, are permitted provided that the following conditions 17238384Sjkim# are met: 18238384Sjkim# 19238384Sjkim# 1. Redistributions of source code must retain the above copyright 20238384Sjkim# notice, this list of conditions and the following disclaimer. 21238384Sjkim# 22238384Sjkim# 2. Redistributions in binary form must reproduce the above copyright 23238384Sjkim# notice, this list of conditions and the following disclaimer in 24238384Sjkim# the documentation and/or other materials provided with the 25238384Sjkim# distribution. 26238384Sjkim# 27238384Sjkim# 3. All advertising materials mentioning features or use of this 28238384Sjkim# software must display the following acknowledgment: 29238384Sjkim# "This product includes software developed by the OpenSSL Project 30238384Sjkim# for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)" 31238384Sjkim# 32238384Sjkim# 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to 33238384Sjkim# endorse or promote products derived from this software without 34238384Sjkim# prior written permission. For written permission, please contact 35238384Sjkim# licensing@OpenSSL.org. 36238384Sjkim# 37238384Sjkim# 5. Products derived from this software may not be called "OpenSSL" 38238384Sjkim# nor may "OpenSSL" appear in their names without prior written 39238384Sjkim# permission of the OpenSSL Project. 40238384Sjkim# 41238384Sjkim# 6. Redistributions of any form whatsoever must retain the following 42238384Sjkim# acknowledgment: 43238384Sjkim# "This product includes software developed by the OpenSSL Project 44238384Sjkim# for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)" 45238384Sjkim# 46238384Sjkim# THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY 47238384Sjkim# EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 48238384Sjkim# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 49238384Sjkim# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR 50238384Sjkim# ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 51238384Sjkim# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 52238384Sjkim# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 53238384Sjkim# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 54238384Sjkim# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 55238384Sjkim# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 56238384Sjkim# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 57238384Sjkim# OF THE POSSIBILITY OF SUCH DAMAGE. 58238384Sjkim# ==================================================================== 59238384Sjkim 60238384Sjkim$flavour = shift; 61238384Sjkim$output = shift; 62238384Sjkimif ($flavour =~ /\./) { $output = $flavour; undef $flavour; } 63238384Sjkim 64238384Sjkimmy $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); 65238384Sjkim 66238384Sjkim$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 67238384Sjkim( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or 68238384Sjkim( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or 69238384Sjkimdie "can't locate x86_64-xlate.pl"; 70238384Sjkim 71246772Sjkimopen OUT,"| \"$^X\" $xlate $flavour $output"; 72246772Sjkim*STDOUT=*OUT; 73238384Sjkim 74238384Sjkimuse strict; 75238384Sjkimmy $code=".text\n\n"; 76238384Sjkimmy $m=0; 77238384Sjkim 78238384Sjkim# 79238384Sjkim# Define x512 macros 80238384Sjkim# 81238384Sjkim 82238384Sjkim#MULSTEP_512_ADD MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src1, src2, add_src, tmp1, tmp2 83238384Sjkim# 84238384Sjkim# uses rax, rdx, and args 85238384Sjkimsub MULSTEP_512_ADD 86238384Sjkim{ 87238384Sjkim my ($x, $DST, $SRC2, $ASRC, $OP, $TMP)=@_; 88238384Sjkim my @X=@$x; # make a copy 89238384Sjkim$code.=<<___; 90238384Sjkim mov (+8*0)($SRC2), %rax 91238384Sjkim mul $OP # rdx:rax = %OP * [0] 92238384Sjkim mov ($ASRC), $X[0] 93238384Sjkim add %rax, $X[0] 94238384Sjkim adc \$0, %rdx 95238384Sjkim mov $X[0], $DST 96238384Sjkim___ 97238384Sjkimfor(my $i=1;$i<8;$i++) { 98238384Sjkim$code.=<<___; 99238384Sjkim mov %rdx, $TMP 100238384Sjkim 101238384Sjkim mov (+8*$i)($SRC2), %rax 102238384Sjkim mul $OP # rdx:rax = %OP * [$i] 103238384Sjkim mov (+8*$i)($ASRC), $X[$i] 104238384Sjkim add %rax, $X[$i] 105238384Sjkim adc \$0, %rdx 106238384Sjkim add $TMP, $X[$i] 107238384Sjkim adc \$0, %rdx 108238384Sjkim___ 109238384Sjkim} 110238384Sjkim$code.=<<___; 111238384Sjkim mov %rdx, $X[0] 112238384Sjkim___ 113238384Sjkim} 114238384Sjkim 115238384Sjkim#MULSTEP_512 MACRO x7, x6, x5, x4, x3, x2, x1, x0, dst, src2, src1_val, tmp 116238384Sjkim# 117238384Sjkim# uses rax, rdx, and args 118238384Sjkimsub MULSTEP_512 119238384Sjkim{ 120238384Sjkim my ($x, $DST, $SRC2, $OP, $TMP)=@_; 121238384Sjkim my @X=@$x; # make a copy 122238384Sjkim$code.=<<___; 123238384Sjkim mov (+8*0)($SRC2), %rax 124238384Sjkim mul $OP # rdx:rax = %OP * [0] 125238384Sjkim add %rax, $X[0] 126238384Sjkim adc \$0, %rdx 127238384Sjkim mov $X[0], $DST 128238384Sjkim___ 129238384Sjkimfor(my $i=1;$i<8;$i++) { 130238384Sjkim$code.=<<___; 131238384Sjkim mov %rdx, $TMP 132238384Sjkim 133238384Sjkim mov (+8*$i)($SRC2), %rax 134238384Sjkim mul $OP # rdx:rax = %OP * [$i] 135238384Sjkim add %rax, $X[$i] 136238384Sjkim adc \$0, %rdx 137238384Sjkim add $TMP, $X[$i] 138238384Sjkim adc \$0, %rdx 139238384Sjkim___ 140238384Sjkim} 141238384Sjkim$code.=<<___; 142238384Sjkim mov %rdx, $X[0] 143238384Sjkim___ 144238384Sjkim} 145238384Sjkim 146238384Sjkim# 147238384Sjkim# Swizzle Macros 148238384Sjkim# 149238384Sjkim 150238384Sjkim# macro to copy data from flat space to swizzled table 151238384Sjkim#MACRO swizzle pDst, pSrc, tmp1, tmp2 152238384Sjkim# pDst and pSrc are modified 153238384Sjkimsub swizzle 154238384Sjkim{ 155238384Sjkim my ($pDst, $pSrc, $cnt, $d0)=@_; 156238384Sjkim$code.=<<___; 157238384Sjkim mov \$8, $cnt 158238384Sjkimloop_$m: 159238384Sjkim mov ($pSrc), $d0 160238384Sjkim mov $d0#w, ($pDst) 161238384Sjkim shr \$16, $d0 162238384Sjkim mov $d0#w, (+64*1)($pDst) 163238384Sjkim shr \$16, $d0 164238384Sjkim mov $d0#w, (+64*2)($pDst) 165238384Sjkim shr \$16, $d0 166238384Sjkim mov $d0#w, (+64*3)($pDst) 167238384Sjkim lea 8($pSrc), $pSrc 168238384Sjkim lea 64*4($pDst), $pDst 169238384Sjkim dec $cnt 170238384Sjkim jnz loop_$m 171238384Sjkim___ 172238384Sjkim 173238384Sjkim $m++; 174238384Sjkim} 175238384Sjkim 176238384Sjkim# macro to copy data from swizzled table to flat space 177238384Sjkim#MACRO unswizzle pDst, pSrc, tmp*3 178238384Sjkimsub unswizzle 179238384Sjkim{ 180238384Sjkim my ($pDst, $pSrc, $cnt, $d0, $d1)=@_; 181238384Sjkim$code.=<<___; 182238384Sjkim mov \$4, $cnt 183238384Sjkimloop_$m: 184238384Sjkim movzxw (+64*3+256*0)($pSrc), $d0 185238384Sjkim movzxw (+64*3+256*1)($pSrc), $d1 186238384Sjkim shl \$16, $d0 187238384Sjkim shl \$16, $d1 188238384Sjkim mov (+64*2+256*0)($pSrc), $d0#w 189238384Sjkim mov (+64*2+256*1)($pSrc), $d1#w 190238384Sjkim shl \$16, $d0 191238384Sjkim shl \$16, $d1 192238384Sjkim mov (+64*1+256*0)($pSrc), $d0#w 193238384Sjkim mov (+64*1+256*1)($pSrc), $d1#w 194238384Sjkim shl \$16, $d0 195238384Sjkim shl \$16, $d1 196238384Sjkim mov (+64*0+256*0)($pSrc), $d0#w 197238384Sjkim mov (+64*0+256*1)($pSrc), $d1#w 198238384Sjkim mov $d0, (+8*0)($pDst) 199238384Sjkim mov $d1, (+8*1)($pDst) 200238384Sjkim lea 256*2($pSrc), $pSrc 201238384Sjkim lea 8*2($pDst), $pDst 202238384Sjkim sub \$1, $cnt 203238384Sjkim jnz loop_$m 204238384Sjkim___ 205238384Sjkim 206238384Sjkim $m++; 207238384Sjkim} 208238384Sjkim 209238384Sjkim# 210238384Sjkim# Data Structures 211238384Sjkim# 212238384Sjkim 213238384Sjkim# Reduce Data 214238384Sjkim# 215238384Sjkim# 216238384Sjkim# Offset Value 217238384Sjkim# 0C0 Carries 218238384Sjkim# 0B8 X2[10] 219238384Sjkim# 0B0 X2[9] 220238384Sjkim# 0A8 X2[8] 221238384Sjkim# 0A0 X2[7] 222238384Sjkim# 098 X2[6] 223238384Sjkim# 090 X2[5] 224238384Sjkim# 088 X2[4] 225238384Sjkim# 080 X2[3] 226238384Sjkim# 078 X2[2] 227238384Sjkim# 070 X2[1] 228238384Sjkim# 068 X2[0] 229238384Sjkim# 060 X1[12] P[10] 230238384Sjkim# 058 X1[11] P[9] Z[8] 231238384Sjkim# 050 X1[10] P[8] Z[7] 232238384Sjkim# 048 X1[9] P[7] Z[6] 233238384Sjkim# 040 X1[8] P[6] Z[5] 234238384Sjkim# 038 X1[7] P[5] Z[4] 235238384Sjkim# 030 X1[6] P[4] Z[3] 236238384Sjkim# 028 X1[5] P[3] Z[2] 237238384Sjkim# 020 X1[4] P[2] Z[1] 238238384Sjkim# 018 X1[3] P[1] Z[0] 239238384Sjkim# 010 X1[2] P[0] Y[2] 240238384Sjkim# 008 X1[1] Q[1] Y[1] 241238384Sjkim# 000 X1[0] Q[0] Y[0] 242238384Sjkim 243238384Sjkimmy $X1_offset = 0; # 13 qwords 244238384Sjkimmy $X2_offset = $X1_offset + 13*8; # 11 qwords 245238384Sjkimmy $Carries_offset = $X2_offset + 11*8; # 1 qword 246238384Sjkimmy $Q_offset = 0; # 2 qwords 247238384Sjkimmy $P_offset = $Q_offset + 2*8; # 11 qwords 248238384Sjkimmy $Y_offset = 0; # 3 qwords 249238384Sjkimmy $Z_offset = $Y_offset + 3*8; # 9 qwords 250238384Sjkim 251238384Sjkimmy $Red_Data_Size = $Carries_offset + 1*8; # (25 qwords) 252238384Sjkim 253238384Sjkim# 254238384Sjkim# Stack Frame 255238384Sjkim# 256238384Sjkim# 257238384Sjkim# offset value 258238384Sjkim# ... <old stack contents> 259238384Sjkim# ... 260238384Sjkim# 280 Garray 261238384Sjkim 262238384Sjkim# 278 tmp16[15] 263238384Sjkim# ... ... 264238384Sjkim# 200 tmp16[0] 265238384Sjkim 266238384Sjkim# 1F8 tmp[7] 267238384Sjkim# ... ... 268238384Sjkim# 1C0 tmp[0] 269238384Sjkim 270238384Sjkim# 1B8 GT[7] 271238384Sjkim# ... ... 272238384Sjkim# 180 GT[0] 273238384Sjkim 274238384Sjkim# 178 Reduce Data 275238384Sjkim# ... ... 276238384Sjkim# 0B8 Reduce Data 277238384Sjkim# 0B0 reserved 278238384Sjkim# 0A8 reserved 279238384Sjkim# 0A0 reserved 280238384Sjkim# 098 reserved 281238384Sjkim# 090 reserved 282238384Sjkim# 088 reduce result addr 283238384Sjkim# 080 exp[8] 284238384Sjkim 285238384Sjkim# ... 286238384Sjkim# 048 exp[1] 287238384Sjkim# 040 exp[0] 288238384Sjkim 289238384Sjkim# 038 reserved 290238384Sjkim# 030 loop_idx 291238384Sjkim# 028 pg 292238384Sjkim# 020 i 293238384Sjkim# 018 pData ; arg 4 294238384Sjkim# 010 pG ; arg 2 295238384Sjkim# 008 pResult ; arg 1 296238384Sjkim# 000 rsp ; stack pointer before subtract 297238384Sjkim 298238384Sjkimmy $rsp_offset = 0; 299238384Sjkimmy $pResult_offset = 8*1 + $rsp_offset; 300238384Sjkimmy $pG_offset = 8*1 + $pResult_offset; 301238384Sjkimmy $pData_offset = 8*1 + $pG_offset; 302238384Sjkimmy $i_offset = 8*1 + $pData_offset; 303238384Sjkimmy $pg_offset = 8*1 + $i_offset; 304238384Sjkimmy $loop_idx_offset = 8*1 + $pg_offset; 305238384Sjkimmy $reserved1_offset = 8*1 + $loop_idx_offset; 306238384Sjkimmy $exp_offset = 8*1 + $reserved1_offset; 307238384Sjkimmy $red_result_addr_offset= 8*9 + $exp_offset; 308238384Sjkimmy $reserved2_offset = 8*1 + $red_result_addr_offset; 309238384Sjkimmy $Reduce_Data_offset = 8*5 + $reserved2_offset; 310238384Sjkimmy $GT_offset = $Red_Data_Size + $Reduce_Data_offset; 311238384Sjkimmy $tmp_offset = 8*8 + $GT_offset; 312238384Sjkimmy $tmp16_offset = 8*8 + $tmp_offset; 313238384Sjkimmy $garray_offset = 8*16 + $tmp16_offset; 314238384Sjkimmy $mem_size = 8*8*32 + $garray_offset; 315238384Sjkim 316238384Sjkim# 317238384Sjkim# Offsets within Reduce Data 318238384Sjkim# 319238384Sjkim# 320238384Sjkim# struct MODF_2FOLD_MONT_512_C1_DATA { 321238384Sjkim# UINT64 t[8][8]; 322238384Sjkim# UINT64 m[8]; 323238384Sjkim# UINT64 m1[8]; /* 2^768 % m */ 324238384Sjkim# UINT64 m2[8]; /* 2^640 % m */ 325238384Sjkim# UINT64 k1[2]; /* (- 1/m) % 2^128 */ 326238384Sjkim# }; 327238384Sjkim 328238384Sjkimmy $T = 0; 329238384Sjkimmy $M = 512; # = 8 * 8 * 8 330238384Sjkimmy $M1 = 576; # = 8 * 8 * 9 /* += 8 * 8 */ 331238384Sjkimmy $M2 = 640; # = 8 * 8 * 10 /* += 8 * 8 */ 332238384Sjkimmy $K1 = 704; # = 8 * 8 * 11 /* += 8 * 8 */ 333238384Sjkim 334238384Sjkim# 335238384Sjkim# FUNCTIONS 336238384Sjkim# 337238384Sjkim 338238384Sjkim{{{ 339238384Sjkim# 340238384Sjkim# MULADD_128x512 : Function to multiply 128-bits (2 qwords) by 512-bits (8 qwords) 341238384Sjkim# and add 512-bits (8 qwords) 342238384Sjkim# to get 640 bits (10 qwords) 343238384Sjkim# Input: 128-bit mul source: [rdi+8*1], rbp 344238384Sjkim# 512-bit mul source: [rsi+8*n] 345238384Sjkim# 512-bit add source: r15, r14, ..., r9, r8 346238384Sjkim# Output: r9, r8, r15, r14, r13, r12, r11, r10, [rcx+8*1], [rcx+8*0] 347238384Sjkim# Clobbers all regs except: rcx, rsi, rdi 348238384Sjkim$code.=<<___; 349238384Sjkim.type MULADD_128x512,\@abi-omnipotent 350238384Sjkim.align 16 351238384SjkimMULADD_128x512: 352238384Sjkim___ 353238384Sjkim &MULSTEP_512([map("%r$_",(8..15))], "(+8*0)(%rcx)", "%rsi", "%rbp", "%rbx"); 354238384Sjkim$code.=<<___; 355238384Sjkim mov (+8*1)(%rdi), %rbp 356238384Sjkim___ 357238384Sjkim &MULSTEP_512([map("%r$_",(9..15,8))], "(+8*1)(%rcx)", "%rsi", "%rbp", "%rbx"); 358238384Sjkim$code.=<<___; 359238384Sjkim ret 360238384Sjkim.size MULADD_128x512,.-MULADD_128x512 361238384Sjkim___ 362238384Sjkim}}} 363238384Sjkim 364238384Sjkim{{{ 365238384Sjkim#MULADD_256x512 MACRO pDst, pA, pB, OP, TMP, X7, X6, X5, X4, X3, X2, X1, X0 366238384Sjkim# 367238384Sjkim# Inputs: pDst: Destination (768 bits, 12 qwords) 368238384Sjkim# pA: Multiplicand (1024 bits, 16 qwords) 369238384Sjkim# pB: Multiplicand (512 bits, 8 qwords) 370238384Sjkim# Dst = Ah * B + Al 371238384Sjkim# where Ah is (in qwords) A[15:12] (256 bits) and Al is A[7:0] (512 bits) 372238384Sjkim# Results in X3 X2 X1 X0 X7 X6 X5 X4 Dst[3:0] 373238384Sjkim# Uses registers: arguments, RAX, RDX 374238384Sjkimsub MULADD_256x512 375238384Sjkim{ 376238384Sjkim my ($pDst, $pA, $pB, $OP, $TMP, $X)=@_; 377238384Sjkim$code.=<<___; 378238384Sjkim mov (+8*12)($pA), $OP 379238384Sjkim___ 380238384Sjkim &MULSTEP_512_ADD($X, "(+8*0)($pDst)", $pB, $pA, $OP, $TMP); 381238384Sjkim push(@$X,shift(@$X)); 382238384Sjkim 383238384Sjkim$code.=<<___; 384238384Sjkim mov (+8*13)($pA), $OP 385238384Sjkim___ 386238384Sjkim &MULSTEP_512($X, "(+8*1)($pDst)", $pB, $OP, $TMP); 387238384Sjkim push(@$X,shift(@$X)); 388238384Sjkim 389238384Sjkim$code.=<<___; 390238384Sjkim mov (+8*14)($pA), $OP 391238384Sjkim___ 392238384Sjkim &MULSTEP_512($X, "(+8*2)($pDst)", $pB, $OP, $TMP); 393238384Sjkim push(@$X,shift(@$X)); 394238384Sjkim 395238384Sjkim$code.=<<___; 396238384Sjkim mov (+8*15)($pA), $OP 397238384Sjkim___ 398238384Sjkim &MULSTEP_512($X, "(+8*3)($pDst)", $pB, $OP, $TMP); 399238384Sjkim push(@$X,shift(@$X)); 400238384Sjkim} 401238384Sjkim 402238384Sjkim# 403238384Sjkim# mont_reduce(UINT64 *x, /* 1024 bits, 16 qwords */ 404238384Sjkim# UINT64 *m, /* 512 bits, 8 qwords */ 405238384Sjkim# MODF_2FOLD_MONT_512_C1_DATA *data, 406238384Sjkim# UINT64 *r) /* 512 bits, 8 qwords */ 407238384Sjkim# Input: x (number to be reduced): tmp16 (Implicit) 408238384Sjkim# m (modulus): [pM] (Implicit) 409238384Sjkim# data (reduce data): [pData] (Implicit) 410238384Sjkim# Output: r (result): Address in [red_res_addr] 411238384Sjkim# result also in: r9, r8, r15, r14, r13, r12, r11, r10 412238384Sjkim 413238384Sjkimmy @X=map("%r$_",(8..15)); 414238384Sjkim 415238384Sjkim$code.=<<___; 416238384Sjkim.type mont_reduce,\@abi-omnipotent 417238384Sjkim.align 16 418238384Sjkimmont_reduce: 419238384Sjkim___ 420238384Sjkim 421238384Sjkimmy $STACK_DEPTH = 8; 422238384Sjkim # 423238384Sjkim # X1 = Xh * M1 + Xl 424238384Sjkim$code.=<<___; 425238384Sjkim lea (+$Reduce_Data_offset+$X1_offset+$STACK_DEPTH)(%rsp), %rdi # pX1 (Dst) 769 bits, 13 qwords 426238384Sjkim mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rsi # pM1 (Bsrc) 512 bits, 8 qwords 427238384Sjkim add \$$M1, %rsi 428238384Sjkim lea (+$tmp16_offset+$STACK_DEPTH)(%rsp), %rcx # X (Asrc) 1024 bits, 16 qwords 429238384Sjkim 430238384Sjkim___ 431238384Sjkim 432238384Sjkim &MULADD_256x512("%rdi", "%rcx", "%rsi", "%rbp", "%rbx", \@X); # rotates @X 4 times 433238384Sjkim # results in r11, r10, r9, r8, r15, r14, r13, r12, X1[3:0] 434238384Sjkim 435238384Sjkim$code.=<<___; 436238384Sjkim xor %rax, %rax 437238384Sjkim # X1 += xl 438238384Sjkim add (+8*8)(%rcx), $X[4] 439238384Sjkim adc (+8*9)(%rcx), $X[5] 440238384Sjkim adc (+8*10)(%rcx), $X[6] 441238384Sjkim adc (+8*11)(%rcx), $X[7] 442238384Sjkim adc \$0, %rax 443238384Sjkim # X1 is now rax, r11-r8, r15-r12, tmp16[3:0] 444238384Sjkim 445238384Sjkim # 446238384Sjkim # check for carry ;; carry stored in rax 447238384Sjkim mov $X[4], (+8*8)(%rdi) # rdi points to X1 448238384Sjkim mov $X[5], (+8*9)(%rdi) 449238384Sjkim mov $X[6], %rbp 450238384Sjkim mov $X[7], (+8*11)(%rdi) 451238384Sjkim 452238384Sjkim mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp) 453238384Sjkim 454238384Sjkim mov (+8*0)(%rdi), $X[4] 455238384Sjkim mov (+8*1)(%rdi), $X[5] 456238384Sjkim mov (+8*2)(%rdi), $X[6] 457238384Sjkim mov (+8*3)(%rdi), $X[7] 458238384Sjkim 459238384Sjkim # X1 is now stored in: X1[11], rbp, X1[9:8], r15-r8 460238384Sjkim # rdi -> X1 461238384Sjkim # rsi -> M1 462238384Sjkim 463238384Sjkim # 464238384Sjkim # X2 = Xh * M2 + Xl 465238384Sjkim # do first part (X2 = Xh * M2) 466238384Sjkim add \$8*10, %rdi # rdi -> pXh ; 128 bits, 2 qwords 467238384Sjkim # Xh is actually { [rdi+8*1], rbp } 468238384Sjkim add \$`$M2-$M1`, %rsi # rsi -> M2 469238384Sjkim lea (+$Reduce_Data_offset+$X2_offset+$STACK_DEPTH)(%rsp), %rcx # rcx -> pX2 ; 641 bits, 11 qwords 470238384Sjkim___ 471238384Sjkim unshift(@X,pop(@X)); unshift(@X,pop(@X)); 472238384Sjkim$code.=<<___; 473238384Sjkim 474238384Sjkim call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8 475238384Sjkim # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0] 476238384Sjkim mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rax 477238384Sjkim 478238384Sjkim # X2 += Xl 479238384Sjkim add (+8*8-8*10)(%rdi), $X[6] # (-8*10) is to adjust rdi -> Xh to Xl 480238384Sjkim adc (+8*9-8*10)(%rdi), $X[7] 481238384Sjkim mov $X[6], (+8*8)(%rcx) 482238384Sjkim mov $X[7], (+8*9)(%rcx) 483238384Sjkim 484238384Sjkim adc %rax, %rax 485238384Sjkim mov %rax, (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp) 486238384Sjkim 487238384Sjkim lea (+$Reduce_Data_offset+$Q_offset+$STACK_DEPTH)(%rsp), %rdi # rdi -> pQ ; 128 bits, 2 qwords 488238384Sjkim add \$`$K1-$M2`, %rsi # rsi -> pK1 ; 128 bits, 2 qwords 489238384Sjkim 490238384Sjkim # MUL_128x128t128 rdi, rcx, rsi ; Q = X2 * K1 (bottom half) 491238384Sjkim # B1:B0 = rsi[1:0] = K1[1:0] 492238384Sjkim # A1:A0 = rcx[1:0] = X2[1:0] 493238384Sjkim # Result = rdi[1],rbp = Q[1],rbp 494238384Sjkim mov (%rsi), %r8 # B0 495238384Sjkim mov (+8*1)(%rsi), %rbx # B1 496238384Sjkim 497238384Sjkim mov (%rcx), %rax # A0 498238384Sjkim mul %r8 # B0 499238384Sjkim mov %rax, %rbp 500238384Sjkim mov %rdx, %r9 501238384Sjkim 502238384Sjkim mov (+8*1)(%rcx), %rax # A1 503238384Sjkim mul %r8 # B0 504238384Sjkim add %rax, %r9 505238384Sjkim 506238384Sjkim mov (%rcx), %rax # A0 507238384Sjkim mul %rbx # B1 508238384Sjkim add %rax, %r9 509238384Sjkim 510238384Sjkim mov %r9, (+8*1)(%rdi) 511238384Sjkim # end MUL_128x128t128 512238384Sjkim 513238384Sjkim sub \$`$K1-$M`, %rsi 514238384Sjkim 515238384Sjkim mov (%rcx), $X[6] 516238384Sjkim mov (+8*1)(%rcx), $X[7] # r9:r8 = X2[1:0] 517238384Sjkim 518238384Sjkim call MULADD_128x512 # args in rcx, rdi / rbp, rsi, r15-r8 519238384Sjkim # result in r9, r8, r15, r14, r13, r12, r11, r10, X2[1:0] 520238384Sjkim 521238384Sjkim # load first half of m to rdx, rdi, rbx, rax 522238384Sjkim # moved this here for efficiency 523238384Sjkim mov (+8*0)(%rsi), %rax 524238384Sjkim mov (+8*1)(%rsi), %rbx 525238384Sjkim mov (+8*2)(%rsi), %rdi 526238384Sjkim mov (+8*3)(%rsi), %rdx 527238384Sjkim 528238384Sjkim # continue with reduction 529238384Sjkim mov (+$Reduce_Data_offset+$Carries_offset+$STACK_DEPTH)(%rsp), %rbp 530238384Sjkim 531238384Sjkim add (+8*8)(%rcx), $X[6] 532238384Sjkim adc (+8*9)(%rcx), $X[7] 533238384Sjkim 534238384Sjkim #accumulate the final carry to rbp 535238384Sjkim adc %rbp, %rbp 536238384Sjkim 537238384Sjkim # Add in overflow corrections: R = (X2>>128) += T[overflow] 538238384Sjkim # R = {r9, r8, r15, r14, ..., r10} 539238384Sjkim shl \$3, %rbp 540238384Sjkim mov (+$pData_offset+$STACK_DEPTH)(%rsp), %rcx # rsi -> Data (and points to T) 541238384Sjkim add %rcx, %rbp # pT ; 512 bits, 8 qwords, spread out 542238384Sjkim 543238384Sjkim # rsi will be used to generate a mask after the addition 544238384Sjkim xor %rsi, %rsi 545238384Sjkim 546238384Sjkim add (+8*8*0)(%rbp), $X[0] 547238384Sjkim adc (+8*8*1)(%rbp), $X[1] 548238384Sjkim adc (+8*8*2)(%rbp), $X[2] 549238384Sjkim adc (+8*8*3)(%rbp), $X[3] 550238384Sjkim adc (+8*8*4)(%rbp), $X[4] 551238384Sjkim adc (+8*8*5)(%rbp), $X[5] 552238384Sjkim adc (+8*8*6)(%rbp), $X[6] 553238384Sjkim adc (+8*8*7)(%rbp), $X[7] 554238384Sjkim 555238384Sjkim # if there is a carry: rsi = 0xFFFFFFFFFFFFFFFF 556238384Sjkim # if carry is clear: rsi = 0x0000000000000000 557238384Sjkim sbb \$0, %rsi 558238384Sjkim 559238384Sjkim # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m 560238384Sjkim and %rsi, %rax 561238384Sjkim and %rsi, %rbx 562238384Sjkim and %rsi, %rdi 563238384Sjkim and %rsi, %rdx 564238384Sjkim 565238384Sjkim mov \$1, %rbp 566238384Sjkim sub %rax, $X[0] 567238384Sjkim sbb %rbx, $X[1] 568238384Sjkim sbb %rdi, $X[2] 569238384Sjkim sbb %rdx, $X[3] 570238384Sjkim 571238384Sjkim # if there is a borrow: rbp = 0 572238384Sjkim # if there is no borrow: rbp = 1 573238384Sjkim # this is used to save the borrows in between the first half and the 2nd half of the subtraction of m 574238384Sjkim sbb \$0, %rbp 575238384Sjkim 576238384Sjkim #load second half of m to rdx, rdi, rbx, rax 577238384Sjkim 578238384Sjkim add \$$M, %rcx 579238384Sjkim mov (+8*4)(%rcx), %rax 580238384Sjkim mov (+8*5)(%rcx), %rbx 581238384Sjkim mov (+8*6)(%rcx), %rdi 582238384Sjkim mov (+8*7)(%rcx), %rdx 583238384Sjkim 584238384Sjkim # use the rsi mask as before 585238384Sjkim # if carry is clear, subtract 0. Otherwise, subtract 256 bits of m 586238384Sjkim and %rsi, %rax 587238384Sjkim and %rsi, %rbx 588238384Sjkim and %rsi, %rdi 589238384Sjkim and %rsi, %rdx 590238384Sjkim 591238384Sjkim # if rbp = 0, there was a borrow before, it is moved to the carry flag 592238384Sjkim # if rbp = 1, there was not a borrow before, carry flag is cleared 593238384Sjkim sub \$1, %rbp 594238384Sjkim 595238384Sjkim sbb %rax, $X[4] 596238384Sjkim sbb %rbx, $X[5] 597238384Sjkim sbb %rdi, $X[6] 598238384Sjkim sbb %rdx, $X[7] 599238384Sjkim 600238384Sjkim # write R back to memory 601238384Sjkim 602238384Sjkim mov (+$red_result_addr_offset+$STACK_DEPTH)(%rsp), %rsi 603238384Sjkim mov $X[0], (+8*0)(%rsi) 604238384Sjkim mov $X[1], (+8*1)(%rsi) 605238384Sjkim mov $X[2], (+8*2)(%rsi) 606238384Sjkim mov $X[3], (+8*3)(%rsi) 607238384Sjkim mov $X[4], (+8*4)(%rsi) 608238384Sjkim mov $X[5], (+8*5)(%rsi) 609238384Sjkim mov $X[6], (+8*6)(%rsi) 610238384Sjkim mov $X[7], (+8*7)(%rsi) 611238384Sjkim 612238384Sjkim ret 613238384Sjkim.size mont_reduce,.-mont_reduce 614238384Sjkim___ 615238384Sjkim}}} 616238384Sjkim 617238384Sjkim{{{ 618238384Sjkim#MUL_512x512 MACRO pDst, pA, pB, x7, x6, x5, x4, x3, x2, x1, x0, tmp*2 619238384Sjkim# 620238384Sjkim# Inputs: pDst: Destination (1024 bits, 16 qwords) 621238384Sjkim# pA: Multiplicand (512 bits, 8 qwords) 622238384Sjkim# pB: Multiplicand (512 bits, 8 qwords) 623238384Sjkim# Uses registers rax, rdx, args 624238384Sjkim# B operand in [pB] and also in x7...x0 625238384Sjkimsub MUL_512x512 626238384Sjkim{ 627238384Sjkim my ($pDst, $pA, $pB, $x, $OP, $TMP, $pDst_o)=@_; 628238384Sjkim my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/); 629238384Sjkim my @X=@$x; # make a copy 630238384Sjkim 631238384Sjkim$code.=<<___; 632238384Sjkim mov (+8*0)($pA), $OP 633238384Sjkim 634238384Sjkim mov $X[0], %rax 635238384Sjkim mul $OP # rdx:rax = %OP * [0] 636238384Sjkim mov %rax, (+$pDst_o+8*0)($pDst) 637238384Sjkim mov %rdx, $X[0] 638238384Sjkim___ 639238384Sjkimfor(my $i=1;$i<8;$i++) { 640238384Sjkim$code.=<<___; 641238384Sjkim mov $X[$i], %rax 642238384Sjkim mul $OP # rdx:rax = %OP * [$i] 643238384Sjkim add %rax, $X[$i-1] 644238384Sjkim adc \$0, %rdx 645238384Sjkim mov %rdx, $X[$i] 646238384Sjkim___ 647238384Sjkim} 648238384Sjkim 649238384Sjkimfor(my $i=1;$i<8;$i++) { 650238384Sjkim$code.=<<___; 651238384Sjkim mov (+8*$i)($pA), $OP 652238384Sjkim___ 653238384Sjkim 654238384Sjkim &MULSTEP_512(\@X, "(+$pDst_o+8*$i)($pDst)", $pB, $OP, $TMP); 655238384Sjkim push(@X,shift(@X)); 656238384Sjkim} 657238384Sjkim 658238384Sjkim$code.=<<___; 659238384Sjkim mov $X[0], (+$pDst_o+8*8)($pDst) 660238384Sjkim mov $X[1], (+$pDst_o+8*9)($pDst) 661238384Sjkim mov $X[2], (+$pDst_o+8*10)($pDst) 662238384Sjkim mov $X[3], (+$pDst_o+8*11)($pDst) 663238384Sjkim mov $X[4], (+$pDst_o+8*12)($pDst) 664238384Sjkim mov $X[5], (+$pDst_o+8*13)($pDst) 665238384Sjkim mov $X[6], (+$pDst_o+8*14)($pDst) 666238384Sjkim mov $X[7], (+$pDst_o+8*15)($pDst) 667238384Sjkim___ 668238384Sjkim} 669238384Sjkim 670238384Sjkim# 671238384Sjkim# mont_mul_a3b : subroutine to compute (Src1 * Src2) % M (all 512-bits) 672238384Sjkim# Input: src1: Address of source 1: rdi 673238384Sjkim# src2: Address of source 2: rsi 674238384Sjkim# Output: dst: Address of destination: [red_res_addr] 675238384Sjkim# src2 and result also in: r9, r8, r15, r14, r13, r12, r11, r10 676238384Sjkim# Temp: Clobbers [tmp16], all registers 677238384Sjkim$code.=<<___; 678238384Sjkim.type mont_mul_a3b,\@abi-omnipotent 679238384Sjkim.align 16 680238384Sjkimmont_mul_a3b: 681238384Sjkim # 682238384Sjkim # multiply tmp = src1 * src2 683238384Sjkim # For multiply: dst = rcx, src1 = rdi, src2 = rsi 684238384Sjkim # stack depth is extra 8 from call 685238384Sjkim___ 686238384Sjkim &MUL_512x512("%rsp+$tmp16_offset+8", "%rdi", "%rsi", [map("%r$_",(10..15,8..9))], "%rbp", "%rbx"); 687238384Sjkim$code.=<<___; 688238384Sjkim # 689238384Sjkim # Dst = tmp % m 690238384Sjkim # Call reduce(tmp, m, data, dst) 691238384Sjkim 692238384Sjkim # tail recursion optimization: jmp to mont_reduce and return from there 693238384Sjkim jmp mont_reduce 694238384Sjkim # call mont_reduce 695238384Sjkim # ret 696238384Sjkim.size mont_mul_a3b,.-mont_mul_a3b 697238384Sjkim___ 698238384Sjkim}}} 699238384Sjkim 700238384Sjkim{{{ 701238384Sjkim#SQR_512 MACRO pDest, pA, x7, x6, x5, x4, x3, x2, x1, x0, tmp*4 702238384Sjkim# 703238384Sjkim# Input in memory [pA] and also in x7...x0 704238384Sjkim# Uses all argument registers plus rax and rdx 705238384Sjkim# 706238384Sjkim# This version computes all of the off-diagonal terms into memory, 707238384Sjkim# and then it adds in the diagonal terms 708238384Sjkim 709238384Sjkimsub SQR_512 710238384Sjkim{ 711238384Sjkim my ($pDst, $pA, $x, $A, $tmp, $x7, $x6, $pDst_o)=@_; 712238384Sjkim my ($pDst, $pDst_o) = ($pDst =~ m/([^+]*)\+?(.*)?/); 713238384Sjkim my @X=@$x; # make a copy 714238384Sjkim$code.=<<___; 715238384Sjkim # ------------------ 716238384Sjkim # first pass 01...07 717238384Sjkim # ------------------ 718238384Sjkim mov $X[0], $A 719238384Sjkim 720238384Sjkim mov $X[1],%rax 721238384Sjkim mul $A 722238384Sjkim mov %rax, (+$pDst_o+8*1)($pDst) 723238384Sjkim___ 724238384Sjkimfor(my $i=2;$i<8;$i++) { 725238384Sjkim$code.=<<___; 726238384Sjkim mov %rdx, $X[$i-2] 727238384Sjkim mov $X[$i],%rax 728238384Sjkim mul $A 729238384Sjkim add %rax, $X[$i-2] 730238384Sjkim adc \$0, %rdx 731238384Sjkim___ 732238384Sjkim} 733238384Sjkim$code.=<<___; 734238384Sjkim mov %rdx, $x7 735238384Sjkim 736238384Sjkim mov $X[0], (+$pDst_o+8*2)($pDst) 737238384Sjkim 738238384Sjkim # ------------------ 739238384Sjkim # second pass 12...17 740238384Sjkim # ------------------ 741238384Sjkim 742238384Sjkim mov (+8*1)($pA), $A 743238384Sjkim 744238384Sjkim mov (+8*2)($pA),%rax 745238384Sjkim mul $A 746238384Sjkim add %rax, $X[1] 747238384Sjkim adc \$0, %rdx 748238384Sjkim mov $X[1], (+$pDst_o+8*3)($pDst) 749238384Sjkim 750238384Sjkim mov %rdx, $X[0] 751238384Sjkim mov (+8*3)($pA),%rax 752238384Sjkim mul $A 753238384Sjkim add %rax, $X[2] 754238384Sjkim adc \$0, %rdx 755238384Sjkim add $X[0], $X[2] 756238384Sjkim adc \$0, %rdx 757238384Sjkim mov $X[2], (+$pDst_o+8*4)($pDst) 758238384Sjkim 759238384Sjkim mov %rdx, $X[0] 760238384Sjkim mov (+8*4)($pA),%rax 761238384Sjkim mul $A 762238384Sjkim add %rax, $X[3] 763238384Sjkim adc \$0, %rdx 764238384Sjkim add $X[0], $X[3] 765238384Sjkim adc \$0, %rdx 766238384Sjkim 767238384Sjkim mov %rdx, $X[0] 768238384Sjkim mov (+8*5)($pA),%rax 769238384Sjkim mul $A 770238384Sjkim add %rax, $X[4] 771238384Sjkim adc \$0, %rdx 772238384Sjkim add $X[0], $X[4] 773238384Sjkim adc \$0, %rdx 774238384Sjkim 775238384Sjkim mov %rdx, $X[0] 776238384Sjkim mov $X[6],%rax 777238384Sjkim mul $A 778238384Sjkim add %rax, $X[5] 779238384Sjkim adc \$0, %rdx 780238384Sjkim add $X[0], $X[5] 781238384Sjkim adc \$0, %rdx 782238384Sjkim 783238384Sjkim mov %rdx, $X[0] 784238384Sjkim mov $X[7],%rax 785238384Sjkim mul $A 786238384Sjkim add %rax, $x7 787238384Sjkim adc \$0, %rdx 788238384Sjkim add $X[0], $x7 789238384Sjkim adc \$0, %rdx 790238384Sjkim 791238384Sjkim mov %rdx, $X[1] 792238384Sjkim 793238384Sjkim # ------------------ 794238384Sjkim # third pass 23...27 795238384Sjkim # ------------------ 796238384Sjkim mov (+8*2)($pA), $A 797238384Sjkim 798238384Sjkim mov (+8*3)($pA),%rax 799238384Sjkim mul $A 800238384Sjkim add %rax, $X[3] 801238384Sjkim adc \$0, %rdx 802238384Sjkim mov $X[3], (+$pDst_o+8*5)($pDst) 803238384Sjkim 804238384Sjkim mov %rdx, $X[0] 805238384Sjkim mov (+8*4)($pA),%rax 806238384Sjkim mul $A 807238384Sjkim add %rax, $X[4] 808238384Sjkim adc \$0, %rdx 809238384Sjkim add $X[0], $X[4] 810238384Sjkim adc \$0, %rdx 811238384Sjkim mov $X[4], (+$pDst_o+8*6)($pDst) 812238384Sjkim 813238384Sjkim mov %rdx, $X[0] 814238384Sjkim mov (+8*5)($pA),%rax 815238384Sjkim mul $A 816238384Sjkim add %rax, $X[5] 817238384Sjkim adc \$0, %rdx 818238384Sjkim add $X[0], $X[5] 819238384Sjkim adc \$0, %rdx 820238384Sjkim 821238384Sjkim mov %rdx, $X[0] 822238384Sjkim mov $X[6],%rax 823238384Sjkim mul $A 824238384Sjkim add %rax, $x7 825238384Sjkim adc \$0, %rdx 826238384Sjkim add $X[0], $x7 827238384Sjkim adc \$0, %rdx 828238384Sjkim 829238384Sjkim mov %rdx, $X[0] 830238384Sjkim mov $X[7],%rax 831238384Sjkim mul $A 832238384Sjkim add %rax, $X[1] 833238384Sjkim adc \$0, %rdx 834238384Sjkim add $X[0], $X[1] 835238384Sjkim adc \$0, %rdx 836238384Sjkim 837238384Sjkim mov %rdx, $X[2] 838238384Sjkim 839238384Sjkim # ------------------ 840238384Sjkim # fourth pass 34...37 841238384Sjkim # ------------------ 842238384Sjkim 843238384Sjkim mov (+8*3)($pA), $A 844238384Sjkim 845238384Sjkim mov (+8*4)($pA),%rax 846238384Sjkim mul $A 847238384Sjkim add %rax, $X[5] 848238384Sjkim adc \$0, %rdx 849238384Sjkim mov $X[5], (+$pDst_o+8*7)($pDst) 850238384Sjkim 851238384Sjkim mov %rdx, $X[0] 852238384Sjkim mov (+8*5)($pA),%rax 853238384Sjkim mul $A 854238384Sjkim add %rax, $x7 855238384Sjkim adc \$0, %rdx 856238384Sjkim add $X[0], $x7 857238384Sjkim adc \$0, %rdx 858238384Sjkim mov $x7, (+$pDst_o+8*8)($pDst) 859238384Sjkim 860238384Sjkim mov %rdx, $X[0] 861238384Sjkim mov $X[6],%rax 862238384Sjkim mul $A 863238384Sjkim add %rax, $X[1] 864238384Sjkim adc \$0, %rdx 865238384Sjkim add $X[0], $X[1] 866238384Sjkim adc \$0, %rdx 867238384Sjkim 868238384Sjkim mov %rdx, $X[0] 869238384Sjkim mov $X[7],%rax 870238384Sjkim mul $A 871238384Sjkim add %rax, $X[2] 872238384Sjkim adc \$0, %rdx 873238384Sjkim add $X[0], $X[2] 874238384Sjkim adc \$0, %rdx 875238384Sjkim 876238384Sjkim mov %rdx, $X[5] 877238384Sjkim 878238384Sjkim # ------------------ 879238384Sjkim # fifth pass 45...47 880238384Sjkim # ------------------ 881238384Sjkim mov (+8*4)($pA), $A 882238384Sjkim 883238384Sjkim mov (+8*5)($pA),%rax 884238384Sjkim mul $A 885238384Sjkim add %rax, $X[1] 886238384Sjkim adc \$0, %rdx 887238384Sjkim mov $X[1], (+$pDst_o+8*9)($pDst) 888238384Sjkim 889238384Sjkim mov %rdx, $X[0] 890238384Sjkim mov $X[6],%rax 891238384Sjkim mul $A 892238384Sjkim add %rax, $X[2] 893238384Sjkim adc \$0, %rdx 894238384Sjkim add $X[0], $X[2] 895238384Sjkim adc \$0, %rdx 896238384Sjkim mov $X[2], (+$pDst_o+8*10)($pDst) 897238384Sjkim 898238384Sjkim mov %rdx, $X[0] 899238384Sjkim mov $X[7],%rax 900238384Sjkim mul $A 901238384Sjkim add %rax, $X[5] 902238384Sjkim adc \$0, %rdx 903238384Sjkim add $X[0], $X[5] 904238384Sjkim adc \$0, %rdx 905238384Sjkim 906238384Sjkim mov %rdx, $X[1] 907238384Sjkim 908238384Sjkim # ------------------ 909238384Sjkim # sixth pass 56...57 910238384Sjkim # ------------------ 911238384Sjkim mov (+8*5)($pA), $A 912238384Sjkim 913238384Sjkim mov $X[6],%rax 914238384Sjkim mul $A 915238384Sjkim add %rax, $X[5] 916238384Sjkim adc \$0, %rdx 917238384Sjkim mov $X[5], (+$pDst_o+8*11)($pDst) 918238384Sjkim 919238384Sjkim mov %rdx, $X[0] 920238384Sjkim mov $X[7],%rax 921238384Sjkim mul $A 922238384Sjkim add %rax, $X[1] 923238384Sjkim adc \$0, %rdx 924238384Sjkim add $X[0], $X[1] 925238384Sjkim adc \$0, %rdx 926238384Sjkim mov $X[1], (+$pDst_o+8*12)($pDst) 927238384Sjkim 928238384Sjkim mov %rdx, $X[2] 929238384Sjkim 930238384Sjkim # ------------------ 931238384Sjkim # seventh pass 67 932238384Sjkim # ------------------ 933238384Sjkim mov $X[6], $A 934238384Sjkim 935238384Sjkim mov $X[7],%rax 936238384Sjkim mul $A 937238384Sjkim add %rax, $X[2] 938238384Sjkim adc \$0, %rdx 939238384Sjkim mov $X[2], (+$pDst_o+8*13)($pDst) 940238384Sjkim 941238384Sjkim mov %rdx, (+$pDst_o+8*14)($pDst) 942238384Sjkim 943238384Sjkim # start finalize (add in squares, and double off-terms) 944238384Sjkim mov (+$pDst_o+8*1)($pDst), $X[0] 945238384Sjkim mov (+$pDst_o+8*2)($pDst), $X[1] 946238384Sjkim mov (+$pDst_o+8*3)($pDst), $X[2] 947238384Sjkim mov (+$pDst_o+8*4)($pDst), $X[3] 948238384Sjkim mov (+$pDst_o+8*5)($pDst), $X[4] 949238384Sjkim mov (+$pDst_o+8*6)($pDst), $X[5] 950238384Sjkim 951238384Sjkim mov (+8*3)($pA), %rax 952238384Sjkim mul %rax 953238384Sjkim mov %rax, $x6 954238384Sjkim mov %rdx, $X[6] 955238384Sjkim 956238384Sjkim add $X[0], $X[0] 957238384Sjkim adc $X[1], $X[1] 958238384Sjkim adc $X[2], $X[2] 959238384Sjkim adc $X[3], $X[3] 960238384Sjkim adc $X[4], $X[4] 961238384Sjkim adc $X[5], $X[5] 962238384Sjkim adc \$0, $X[6] 963238384Sjkim 964238384Sjkim mov (+8*0)($pA), %rax 965238384Sjkim mul %rax 966238384Sjkim mov %rax, (+$pDst_o+8*0)($pDst) 967238384Sjkim mov %rdx, $A 968238384Sjkim 969238384Sjkim mov (+8*1)($pA), %rax 970238384Sjkim mul %rax 971238384Sjkim 972238384Sjkim add $A, $X[0] 973238384Sjkim adc %rax, $X[1] 974238384Sjkim adc \$0, %rdx 975238384Sjkim 976238384Sjkim mov %rdx, $A 977238384Sjkim mov $X[0], (+$pDst_o+8*1)($pDst) 978238384Sjkim mov $X[1], (+$pDst_o+8*2)($pDst) 979238384Sjkim 980238384Sjkim mov (+8*2)($pA), %rax 981238384Sjkim mul %rax 982238384Sjkim 983238384Sjkim add $A, $X[2] 984238384Sjkim adc %rax, $X[3] 985238384Sjkim adc \$0, %rdx 986238384Sjkim 987238384Sjkim mov %rdx, $A 988238384Sjkim 989238384Sjkim mov $X[2], (+$pDst_o+8*3)($pDst) 990238384Sjkim mov $X[3], (+$pDst_o+8*4)($pDst) 991238384Sjkim 992238384Sjkim xor $tmp, $tmp 993238384Sjkim add $A, $X[4] 994238384Sjkim adc $x6, $X[5] 995238384Sjkim adc \$0, $tmp 996238384Sjkim 997238384Sjkim mov $X[4], (+$pDst_o+8*5)($pDst) 998238384Sjkim mov $X[5], (+$pDst_o+8*6)($pDst) 999238384Sjkim 1000238384Sjkim # %%tmp has 0/1 in column 7 1001238384Sjkim # %%A6 has a full value in column 7 1002238384Sjkim 1003238384Sjkim mov (+$pDst_o+8*7)($pDst), $X[0] 1004238384Sjkim mov (+$pDst_o+8*8)($pDst), $X[1] 1005238384Sjkim mov (+$pDst_o+8*9)($pDst), $X[2] 1006238384Sjkim mov (+$pDst_o+8*10)($pDst), $X[3] 1007238384Sjkim mov (+$pDst_o+8*11)($pDst), $X[4] 1008238384Sjkim mov (+$pDst_o+8*12)($pDst), $X[5] 1009238384Sjkim mov (+$pDst_o+8*13)($pDst), $x6 1010238384Sjkim mov (+$pDst_o+8*14)($pDst), $x7 1011238384Sjkim 1012238384Sjkim mov $X[7], %rax 1013238384Sjkim mul %rax 1014238384Sjkim mov %rax, $X[7] 1015238384Sjkim mov %rdx, $A 1016238384Sjkim 1017238384Sjkim add $X[0], $X[0] 1018238384Sjkim adc $X[1], $X[1] 1019238384Sjkim adc $X[2], $X[2] 1020238384Sjkim adc $X[3], $X[3] 1021238384Sjkim adc $X[4], $X[4] 1022238384Sjkim adc $X[5], $X[5] 1023238384Sjkim adc $x6, $x6 1024238384Sjkim adc $x7, $x7 1025238384Sjkim adc \$0, $A 1026238384Sjkim 1027238384Sjkim add $tmp, $X[0] 1028238384Sjkim 1029238384Sjkim mov (+8*4)($pA), %rax 1030238384Sjkim mul %rax 1031238384Sjkim 1032238384Sjkim add $X[6], $X[0] 1033238384Sjkim adc %rax, $X[1] 1034238384Sjkim adc \$0, %rdx 1035238384Sjkim 1036238384Sjkim mov %rdx, $tmp 1037238384Sjkim 1038238384Sjkim mov $X[0], (+$pDst_o+8*7)($pDst) 1039238384Sjkim mov $X[1], (+$pDst_o+8*8)($pDst) 1040238384Sjkim 1041238384Sjkim mov (+8*5)($pA), %rax 1042238384Sjkim mul %rax 1043238384Sjkim 1044238384Sjkim add $tmp, $X[2] 1045238384Sjkim adc %rax, $X[3] 1046238384Sjkim adc \$0, %rdx 1047238384Sjkim 1048238384Sjkim mov %rdx, $tmp 1049238384Sjkim 1050238384Sjkim mov $X[2], (+$pDst_o+8*9)($pDst) 1051238384Sjkim mov $X[3], (+$pDst_o+8*10)($pDst) 1052238384Sjkim 1053238384Sjkim mov (+8*6)($pA), %rax 1054238384Sjkim mul %rax 1055238384Sjkim 1056238384Sjkim add $tmp, $X[4] 1057238384Sjkim adc %rax, $X[5] 1058238384Sjkim adc \$0, %rdx 1059238384Sjkim 1060238384Sjkim mov $X[4], (+$pDst_o+8*11)($pDst) 1061238384Sjkim mov $X[5], (+$pDst_o+8*12)($pDst) 1062238384Sjkim 1063238384Sjkim add %rdx, $x6 1064238384Sjkim adc $X[7], $x7 1065238384Sjkim adc \$0, $A 1066238384Sjkim 1067238384Sjkim mov $x6, (+$pDst_o+8*13)($pDst) 1068238384Sjkim mov $x7, (+$pDst_o+8*14)($pDst) 1069238384Sjkim mov $A, (+$pDst_o+8*15)($pDst) 1070238384Sjkim___ 1071238384Sjkim} 1072238384Sjkim 1073238384Sjkim# 1074238384Sjkim# sqr_reduce: subroutine to compute Result = reduce(Result * Result) 1075238384Sjkim# 1076238384Sjkim# input and result also in: r9, r8, r15, r14, r13, r12, r11, r10 1077238384Sjkim# 1078238384Sjkim$code.=<<___; 1079238384Sjkim.type sqr_reduce,\@abi-omnipotent 1080238384Sjkim.align 16 1081238384Sjkimsqr_reduce: 1082238384Sjkim mov (+$pResult_offset+8)(%rsp), %rcx 1083238384Sjkim___ 1084238384Sjkim &SQR_512("%rsp+$tmp16_offset+8", "%rcx", [map("%r$_",(10..15,8..9))], "%rbx", "%rbp", "%rsi", "%rdi"); 1085238384Sjkim$code.=<<___; 1086238384Sjkim # tail recursion optimization: jmp to mont_reduce and return from there 1087238384Sjkim jmp mont_reduce 1088238384Sjkim # call mont_reduce 1089238384Sjkim # ret 1090238384Sjkim.size sqr_reduce,.-sqr_reduce 1091238384Sjkim___ 1092238384Sjkim}}} 1093238384Sjkim 1094238384Sjkim# 1095238384Sjkim# MAIN FUNCTION 1096238384Sjkim# 1097238384Sjkim 1098238384Sjkim#mod_exp_512(UINT64 *result, /* 512 bits, 8 qwords */ 1099238384Sjkim# UINT64 *g, /* 512 bits, 8 qwords */ 1100238384Sjkim# UINT64 *exp, /* 512 bits, 8 qwords */ 1101238384Sjkim# struct mod_ctx_512 *data) 1102238384Sjkim 1103238384Sjkim# window size = 5 1104238384Sjkim# table size = 2^5 = 32 1105238384Sjkim#table_entries equ 32 1106238384Sjkim#table_size equ table_entries * 8 1107238384Sjkim$code.=<<___; 1108238384Sjkim.globl mod_exp_512 1109238384Sjkim.type mod_exp_512,\@function,4 1110238384Sjkimmod_exp_512: 1111238384Sjkim push %rbp 1112238384Sjkim push %rbx 1113238384Sjkim push %r12 1114238384Sjkim push %r13 1115238384Sjkim push %r14 1116238384Sjkim push %r15 1117238384Sjkim 1118238384Sjkim # adjust stack down and then align it with cache boundary 1119238384Sjkim mov %rsp, %r8 1120238384Sjkim sub \$$mem_size, %rsp 1121238384Sjkim and \$-64, %rsp 1122238384Sjkim 1123238384Sjkim # store previous stack pointer and arguments 1124238384Sjkim mov %r8, (+$rsp_offset)(%rsp) 1125238384Sjkim mov %rdi, (+$pResult_offset)(%rsp) 1126238384Sjkim mov %rsi, (+$pG_offset)(%rsp) 1127238384Sjkim mov %rcx, (+$pData_offset)(%rsp) 1128238384Sjkim.Lbody: 1129238384Sjkim # transform g into montgomery space 1130238384Sjkim # GT = reduce(g * C2) = reduce(g * (2^256)) 1131238384Sjkim # reduce expects to have the input in [tmp16] 1132238384Sjkim pxor %xmm4, %xmm4 1133238384Sjkim movdqu (+16*0)(%rsi), %xmm0 1134238384Sjkim movdqu (+16*1)(%rsi), %xmm1 1135238384Sjkim movdqu (+16*2)(%rsi), %xmm2 1136238384Sjkim movdqu (+16*3)(%rsi), %xmm3 1137238384Sjkim movdqa %xmm4, (+$tmp16_offset+16*0)(%rsp) 1138238384Sjkim movdqa %xmm4, (+$tmp16_offset+16*1)(%rsp) 1139238384Sjkim movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp) 1140238384Sjkim movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp) 1141238384Sjkim movdqa %xmm0, (+$tmp16_offset+16*2)(%rsp) 1142238384Sjkim movdqa %xmm1, (+$tmp16_offset+16*3)(%rsp) 1143238384Sjkim movdqa %xmm2, (+$tmp16_offset+16*4)(%rsp) 1144238384Sjkim movdqa %xmm3, (+$tmp16_offset+16*5)(%rsp) 1145238384Sjkim 1146238384Sjkim # load pExp before rdx gets blown away 1147238384Sjkim movdqu (+16*0)(%rdx), %xmm0 1148238384Sjkim movdqu (+16*1)(%rdx), %xmm1 1149238384Sjkim movdqu (+16*2)(%rdx), %xmm2 1150238384Sjkim movdqu (+16*3)(%rdx), %xmm3 1151238384Sjkim 1152238384Sjkim lea (+$GT_offset)(%rsp), %rbx 1153238384Sjkim mov %rbx, (+$red_result_addr_offset)(%rsp) 1154238384Sjkim call mont_reduce 1155238384Sjkim 1156238384Sjkim # Initialize tmp = C 1157238384Sjkim lea (+$tmp_offset)(%rsp), %rcx 1158238384Sjkim xor %rax, %rax 1159238384Sjkim mov %rax, (+8*0)(%rcx) 1160238384Sjkim mov %rax, (+8*1)(%rcx) 1161238384Sjkim mov %rax, (+8*3)(%rcx) 1162238384Sjkim mov %rax, (+8*4)(%rcx) 1163238384Sjkim mov %rax, (+8*5)(%rcx) 1164238384Sjkim mov %rax, (+8*6)(%rcx) 1165238384Sjkim mov %rax, (+8*7)(%rcx) 1166238384Sjkim mov %rax, (+$exp_offset+8*8)(%rsp) 1167238384Sjkim movq \$1, (+8*2)(%rcx) 1168238384Sjkim 1169238384Sjkim lea (+$garray_offset)(%rsp), %rbp 1170238384Sjkim mov %rcx, %rsi # pTmp 1171238384Sjkim mov %rbp, %rdi # Garray[][0] 1172238384Sjkim___ 1173238384Sjkim 1174238384Sjkim &swizzle("%rdi", "%rcx", "%rax", "%rbx"); 1175238384Sjkim 1176238384Sjkim # for (rax = 31; rax != 0; rax--) { 1177238384Sjkim # tmp = reduce(tmp * G) 1178238384Sjkim # swizzle(pg, tmp); 1179238384Sjkim # pg += 2; } 1180238384Sjkim$code.=<<___; 1181238384Sjkim mov \$31, %rax 1182238384Sjkim mov %rax, (+$i_offset)(%rsp) 1183238384Sjkim mov %rbp, (+$pg_offset)(%rsp) 1184238384Sjkim # rsi -> pTmp 1185238384Sjkim mov %rsi, (+$red_result_addr_offset)(%rsp) 1186238384Sjkim mov (+8*0)(%rsi), %r10 1187238384Sjkim mov (+8*1)(%rsi), %r11 1188238384Sjkim mov (+8*2)(%rsi), %r12 1189238384Sjkim mov (+8*3)(%rsi), %r13 1190238384Sjkim mov (+8*4)(%rsi), %r14 1191238384Sjkim mov (+8*5)(%rsi), %r15 1192238384Sjkim mov (+8*6)(%rsi), %r8 1193238384Sjkim mov (+8*7)(%rsi), %r9 1194238384Sjkiminit_loop: 1195238384Sjkim lea (+$GT_offset)(%rsp), %rdi 1196238384Sjkim call mont_mul_a3b 1197238384Sjkim lea (+$tmp_offset)(%rsp), %rsi 1198238384Sjkim mov (+$pg_offset)(%rsp), %rbp 1199238384Sjkim add \$2, %rbp 1200238384Sjkim mov %rbp, (+$pg_offset)(%rsp) 1201238384Sjkim mov %rsi, %rcx # rcx = rsi = addr of tmp 1202238384Sjkim___ 1203238384Sjkim 1204238384Sjkim &swizzle("%rbp", "%rcx", "%rax", "%rbx"); 1205238384Sjkim$code.=<<___; 1206238384Sjkim mov (+$i_offset)(%rsp), %rax 1207238384Sjkim sub \$1, %rax 1208238384Sjkim mov %rax, (+$i_offset)(%rsp) 1209238384Sjkim jne init_loop 1210238384Sjkim 1211238384Sjkim # 1212238384Sjkim # Copy exponent onto stack 1213238384Sjkim movdqa %xmm0, (+$exp_offset+16*0)(%rsp) 1214238384Sjkim movdqa %xmm1, (+$exp_offset+16*1)(%rsp) 1215238384Sjkim movdqa %xmm2, (+$exp_offset+16*2)(%rsp) 1216238384Sjkim movdqa %xmm3, (+$exp_offset+16*3)(%rsp) 1217238384Sjkim 1218238384Sjkim 1219238384Sjkim # 1220238384Sjkim # Do exponentiation 1221238384Sjkim # Initialize result to G[exp{511:507}] 1222238384Sjkim mov (+$exp_offset+62)(%rsp), %eax 1223238384Sjkim mov %rax, %rdx 1224238384Sjkim shr \$11, %rax 1225238384Sjkim and \$0x07FF, %edx 1226238384Sjkim mov %edx, (+$exp_offset+62)(%rsp) 1227238384Sjkim lea (+$garray_offset)(%rsp,%rax,2), %rsi 1228238384Sjkim mov (+$pResult_offset)(%rsp), %rdx 1229238384Sjkim___ 1230238384Sjkim 1231238384Sjkim &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax"); 1232238384Sjkim 1233238384Sjkim # 1234238384Sjkim # Loop variables 1235238384Sjkim # rcx = [loop_idx] = index: 510-5 to 0 by 5 1236238384Sjkim$code.=<<___; 1237238384Sjkim movq \$505, (+$loop_idx_offset)(%rsp) 1238238384Sjkim 1239238384Sjkim mov (+$pResult_offset)(%rsp), %rcx 1240238384Sjkim mov %rcx, (+$red_result_addr_offset)(%rsp) 1241238384Sjkim mov (+8*0)(%rcx), %r10 1242238384Sjkim mov (+8*1)(%rcx), %r11 1243238384Sjkim mov (+8*2)(%rcx), %r12 1244238384Sjkim mov (+8*3)(%rcx), %r13 1245238384Sjkim mov (+8*4)(%rcx), %r14 1246238384Sjkim mov (+8*5)(%rcx), %r15 1247238384Sjkim mov (+8*6)(%rcx), %r8 1248238384Sjkim mov (+8*7)(%rcx), %r9 1249238384Sjkim jmp sqr_2 1250238384Sjkim 1251238384Sjkimmain_loop_a3b: 1252238384Sjkim call sqr_reduce 1253238384Sjkim call sqr_reduce 1254238384Sjkim call sqr_reduce 1255238384Sjkimsqr_2: 1256238384Sjkim call sqr_reduce 1257238384Sjkim call sqr_reduce 1258238384Sjkim 1259238384Sjkim # 1260238384Sjkim # Do multiply, first look up proper value in Garray 1261238384Sjkim mov (+$loop_idx_offset)(%rsp), %rcx # bit index 1262238384Sjkim mov %rcx, %rax 1263238384Sjkim shr \$4, %rax # rax is word pointer 1264238384Sjkim mov (+$exp_offset)(%rsp,%rax,2), %edx 1265238384Sjkim and \$15, %rcx 1266238384Sjkim shrq %cl, %rdx 1267238384Sjkim and \$0x1F, %rdx 1268238384Sjkim 1269238384Sjkim lea (+$garray_offset)(%rsp,%rdx,2), %rsi 1270238384Sjkim lea (+$tmp_offset)(%rsp), %rdx 1271238384Sjkim mov %rdx, %rdi 1272238384Sjkim___ 1273238384Sjkim 1274238384Sjkim &unswizzle("%rdx", "%rsi", "%rbp", "%rbx", "%rax"); 1275238384Sjkim # rdi = tmp = pG 1276238384Sjkim 1277238384Sjkim # 1278238384Sjkim # Call mod_mul_a1(pDst, pSrc1, pSrc2, pM, pData) 1279238384Sjkim # result result pG M Data 1280238384Sjkim$code.=<<___; 1281238384Sjkim mov (+$pResult_offset)(%rsp), %rsi 1282238384Sjkim call mont_mul_a3b 1283238384Sjkim 1284238384Sjkim # 1285238384Sjkim # finish loop 1286238384Sjkim mov (+$loop_idx_offset)(%rsp), %rcx 1287238384Sjkim sub \$5, %rcx 1288238384Sjkim mov %rcx, (+$loop_idx_offset)(%rsp) 1289238384Sjkim jge main_loop_a3b 1290238384Sjkim 1291238384Sjkim # 1292238384Sjkim 1293238384Sjkimend_main_loop_a3b: 1294238384Sjkim # transform result out of Montgomery space 1295238384Sjkim # result = reduce(result) 1296238384Sjkim mov (+$pResult_offset)(%rsp), %rdx 1297238384Sjkim pxor %xmm4, %xmm4 1298238384Sjkim movdqu (+16*0)(%rdx), %xmm0 1299238384Sjkim movdqu (+16*1)(%rdx), %xmm1 1300238384Sjkim movdqu (+16*2)(%rdx), %xmm2 1301238384Sjkim movdqu (+16*3)(%rdx), %xmm3 1302238384Sjkim movdqa %xmm4, (+$tmp16_offset+16*4)(%rsp) 1303238384Sjkim movdqa %xmm4, (+$tmp16_offset+16*5)(%rsp) 1304238384Sjkim movdqa %xmm4, (+$tmp16_offset+16*6)(%rsp) 1305238384Sjkim movdqa %xmm4, (+$tmp16_offset+16*7)(%rsp) 1306238384Sjkim movdqa %xmm0, (+$tmp16_offset+16*0)(%rsp) 1307238384Sjkim movdqa %xmm1, (+$tmp16_offset+16*1)(%rsp) 1308238384Sjkim movdqa %xmm2, (+$tmp16_offset+16*2)(%rsp) 1309238384Sjkim movdqa %xmm3, (+$tmp16_offset+16*3)(%rsp) 1310238384Sjkim call mont_reduce 1311238384Sjkim 1312238384Sjkim # If result > m, subract m 1313238384Sjkim # load result into r15:r8 1314238384Sjkim mov (+$pResult_offset)(%rsp), %rax 1315238384Sjkim mov (+8*0)(%rax), %r8 1316238384Sjkim mov (+8*1)(%rax), %r9 1317238384Sjkim mov (+8*2)(%rax), %r10 1318238384Sjkim mov (+8*3)(%rax), %r11 1319238384Sjkim mov (+8*4)(%rax), %r12 1320238384Sjkim mov (+8*5)(%rax), %r13 1321238384Sjkim mov (+8*6)(%rax), %r14 1322238384Sjkim mov (+8*7)(%rax), %r15 1323238384Sjkim 1324238384Sjkim # subtract m 1325238384Sjkim mov (+$pData_offset)(%rsp), %rbx 1326238384Sjkim add \$$M, %rbx 1327238384Sjkim 1328238384Sjkim sub (+8*0)(%rbx), %r8 1329238384Sjkim sbb (+8*1)(%rbx), %r9 1330238384Sjkim sbb (+8*2)(%rbx), %r10 1331238384Sjkim sbb (+8*3)(%rbx), %r11 1332238384Sjkim sbb (+8*4)(%rbx), %r12 1333238384Sjkim sbb (+8*5)(%rbx), %r13 1334238384Sjkim sbb (+8*6)(%rbx), %r14 1335238384Sjkim sbb (+8*7)(%rbx), %r15 1336238384Sjkim 1337238384Sjkim # if Carry is clear, replace result with difference 1338238384Sjkim mov (+8*0)(%rax), %rsi 1339238384Sjkim mov (+8*1)(%rax), %rdi 1340238384Sjkim mov (+8*2)(%rax), %rcx 1341238384Sjkim mov (+8*3)(%rax), %rdx 1342238384Sjkim cmovnc %r8, %rsi 1343238384Sjkim cmovnc %r9, %rdi 1344238384Sjkim cmovnc %r10, %rcx 1345238384Sjkim cmovnc %r11, %rdx 1346238384Sjkim mov %rsi, (+8*0)(%rax) 1347238384Sjkim mov %rdi, (+8*1)(%rax) 1348238384Sjkim mov %rcx, (+8*2)(%rax) 1349238384Sjkim mov %rdx, (+8*3)(%rax) 1350238384Sjkim 1351238384Sjkim mov (+8*4)(%rax), %rsi 1352238384Sjkim mov (+8*5)(%rax), %rdi 1353238384Sjkim mov (+8*6)(%rax), %rcx 1354238384Sjkim mov (+8*7)(%rax), %rdx 1355238384Sjkim cmovnc %r12, %rsi 1356238384Sjkim cmovnc %r13, %rdi 1357238384Sjkim cmovnc %r14, %rcx 1358238384Sjkim cmovnc %r15, %rdx 1359238384Sjkim mov %rsi, (+8*4)(%rax) 1360238384Sjkim mov %rdi, (+8*5)(%rax) 1361238384Sjkim mov %rcx, (+8*6)(%rax) 1362238384Sjkim mov %rdx, (+8*7)(%rax) 1363238384Sjkim 1364238384Sjkim mov (+$rsp_offset)(%rsp), %rsi 1365238384Sjkim mov 0(%rsi),%r15 1366238384Sjkim mov 8(%rsi),%r14 1367238384Sjkim mov 16(%rsi),%r13 1368238384Sjkim mov 24(%rsi),%r12 1369238384Sjkim mov 32(%rsi),%rbx 1370238384Sjkim mov 40(%rsi),%rbp 1371238384Sjkim lea 48(%rsi),%rsp 1372238384Sjkim.Lepilogue: 1373238384Sjkim ret 1374238384Sjkim.size mod_exp_512, . - mod_exp_512 1375238384Sjkim___ 1376238384Sjkim 1377238384Sjkimif ($win64) { 1378238384Sjkim# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, 1379238384Sjkim# CONTEXT *context,DISPATCHER_CONTEXT *disp) 1380238384Sjkimmy $rec="%rcx"; 1381238384Sjkimmy $frame="%rdx"; 1382238384Sjkimmy $context="%r8"; 1383238384Sjkimmy $disp="%r9"; 1384238384Sjkim 1385238384Sjkim$code.=<<___; 1386238384Sjkim.extern __imp_RtlVirtualUnwind 1387238384Sjkim.type mod_exp_512_se_handler,\@abi-omnipotent 1388238384Sjkim.align 16 1389238384Sjkimmod_exp_512_se_handler: 1390238384Sjkim push %rsi 1391238384Sjkim push %rdi 1392238384Sjkim push %rbx 1393238384Sjkim push %rbp 1394238384Sjkim push %r12 1395238384Sjkim push %r13 1396238384Sjkim push %r14 1397238384Sjkim push %r15 1398238384Sjkim pushfq 1399238384Sjkim sub \$64,%rsp 1400238384Sjkim 1401238384Sjkim mov 120($context),%rax # pull context->Rax 1402238384Sjkim mov 248($context),%rbx # pull context->Rip 1403238384Sjkim 1404238384Sjkim lea .Lbody(%rip),%r10 1405238384Sjkim cmp %r10,%rbx # context->Rip<prologue label 1406238384Sjkim jb .Lin_prologue 1407238384Sjkim 1408238384Sjkim mov 152($context),%rax # pull context->Rsp 1409238384Sjkim 1410238384Sjkim lea .Lepilogue(%rip),%r10 1411238384Sjkim cmp %r10,%rbx # context->Rip>=epilogue label 1412238384Sjkim jae .Lin_prologue 1413238384Sjkim 1414238384Sjkim mov $rsp_offset(%rax),%rax # pull saved Rsp 1415238384Sjkim 1416238384Sjkim mov 32(%rax),%rbx 1417238384Sjkim mov 40(%rax),%rbp 1418238384Sjkim mov 24(%rax),%r12 1419238384Sjkim mov 16(%rax),%r13 1420238384Sjkim mov 8(%rax),%r14 1421238384Sjkim mov 0(%rax),%r15 1422238384Sjkim lea 48(%rax),%rax 1423238384Sjkim mov %rbx,144($context) # restore context->Rbx 1424238384Sjkim mov %rbp,160($context) # restore context->Rbp 1425238384Sjkim mov %r12,216($context) # restore context->R12 1426238384Sjkim mov %r13,224($context) # restore context->R13 1427238384Sjkim mov %r14,232($context) # restore context->R14 1428238384Sjkim mov %r15,240($context) # restore context->R15 1429238384Sjkim 1430238384Sjkim.Lin_prologue: 1431238384Sjkim mov 8(%rax),%rdi 1432238384Sjkim mov 16(%rax),%rsi 1433238384Sjkim mov %rax,152($context) # restore context->Rsp 1434238384Sjkim mov %rsi,168($context) # restore context->Rsi 1435238384Sjkim mov %rdi,176($context) # restore context->Rdi 1436238384Sjkim 1437238384Sjkim mov 40($disp),%rdi # disp->ContextRecord 1438238384Sjkim mov $context,%rsi # context 1439238384Sjkim mov \$154,%ecx # sizeof(CONTEXT) 1440238384Sjkim .long 0xa548f3fc # cld; rep movsq 1441238384Sjkim 1442238384Sjkim mov $disp,%rsi 1443238384Sjkim xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER 1444238384Sjkim mov 8(%rsi),%rdx # arg2, disp->ImageBase 1445238384Sjkim mov 0(%rsi),%r8 # arg3, disp->ControlPc 1446238384Sjkim mov 16(%rsi),%r9 # arg4, disp->FunctionEntry 1447238384Sjkim mov 40(%rsi),%r10 # disp->ContextRecord 1448238384Sjkim lea 56(%rsi),%r11 # &disp->HandlerData 1449238384Sjkim lea 24(%rsi),%r12 # &disp->EstablisherFrame 1450238384Sjkim mov %r10,32(%rsp) # arg5 1451238384Sjkim mov %r11,40(%rsp) # arg6 1452238384Sjkim mov %r12,48(%rsp) # arg7 1453238384Sjkim mov %rcx,56(%rsp) # arg8, (NULL) 1454238384Sjkim call *__imp_RtlVirtualUnwind(%rip) 1455238384Sjkim 1456238384Sjkim mov \$1,%eax # ExceptionContinueSearch 1457238384Sjkim add \$64,%rsp 1458238384Sjkim popfq 1459238384Sjkim pop %r15 1460238384Sjkim pop %r14 1461238384Sjkim pop %r13 1462238384Sjkim pop %r12 1463238384Sjkim pop %rbp 1464238384Sjkim pop %rbx 1465238384Sjkim pop %rdi 1466238384Sjkim pop %rsi 1467238384Sjkim ret 1468238384Sjkim.size mod_exp_512_se_handler,.-mod_exp_512_se_handler 1469238384Sjkim 1470238384Sjkim.section .pdata 1471238384Sjkim.align 4 1472238384Sjkim .rva .LSEH_begin_mod_exp_512 1473238384Sjkim .rva .LSEH_end_mod_exp_512 1474238384Sjkim .rva .LSEH_info_mod_exp_512 1475238384Sjkim 1476238384Sjkim.section .xdata 1477238384Sjkim.align 8 1478238384Sjkim.LSEH_info_mod_exp_512: 1479238384Sjkim .byte 9,0,0,0 1480238384Sjkim .rva mod_exp_512_se_handler 1481238384Sjkim___ 1482238384Sjkim} 1483238384Sjkim 1484238384Sjkimsub reg_part { 1485238384Sjkimmy ($reg,$conv)=@_; 1486238384Sjkim if ($reg =~ /%r[0-9]+/) { $reg .= $conv; } 1487238384Sjkim elsif ($conv eq "b") { $reg =~ s/%[er]([^x]+)x?/%$1l/; } 1488238384Sjkim elsif ($conv eq "w") { $reg =~ s/%[er](.+)/%$1/; } 1489238384Sjkim elsif ($conv eq "d") { $reg =~ s/%[er](.+)/%e$1/; } 1490238384Sjkim return $reg; 1491238384Sjkim} 1492238384Sjkim 1493238384Sjkim$code =~ s/(%[a-z0-9]+)#([bwd])/reg_part($1,$2)/gem; 1494238384Sjkim$code =~ s/\`([^\`]*)\`/eval $1/gem; 1495238384Sjkim$code =~ s/(\(\+[^)]+\))/eval $1/gem; 1496238384Sjkimprint $code; 1497238384Sjkimclose STDOUT; 1498