1/*
2 * Double-precision vector e^x function.
3 *
4 * Copyright (c) 2019-2023, Arm Limited.
5 * SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
6 */
7
8#include "mathlib.h"
9#include "v_math.h"
10
11#define N (1 << V_EXP_TABLE_BITS)
12#define IndexMask (N - 1)
13
14const static volatile struct
15{
16  float64x2_t poly[3];
17  float64x2_t inv_ln2, ln2_hi, ln2_lo, shift;
18#if !WANT_SIMD_EXCEPT
19  float64x2_t special_bound, scale_thresh;
20#endif
21} data = {
22  /* maxerr: 1.88 +0.5 ulp
23     rel error: 1.4337*2^-53
24     abs error: 1.4299*2^-53 in [ -ln2/256, ln2/256 ].  */
25  .poly = { V2 (0x1.ffffffffffd43p-2), V2 (0x1.55555c75adbb2p-3),
26	    V2 (0x1.55555da646206p-5) },
27#if !WANT_SIMD_EXCEPT
28  .scale_thresh = V2 (163840.0), /* 1280.0 * N.  */
29  .special_bound = V2 (704.0),
30#endif
31  .inv_ln2 = V2 (0x1.71547652b82fep7), /* N/ln2.  */
32  .ln2_hi = V2 (0x1.62e42fefa39efp-8), /* ln2/N.  */
33  .ln2_lo = V2 (0x1.abc9e3b39803f3p-63),
34  .shift = V2 (0x1.8p+52)
35};
36
37#define C(i) data.poly[i]
38#define Tab __v_exp_data
39
40#if WANT_SIMD_EXCEPT
41
42# define TinyBound v_u64 (0x2000000000000000) /* asuint64 (0x1p-511).  */
43# define BigBound v_u64 (0x4080000000000000) /* asuint64 (0x1p9).  */
44# define SpecialBound v_u64 (0x2080000000000000) /* BigBound - TinyBound.  */
45
46static float64x2_t VPCS_ATTR NOINLINE
47special_case (float64x2_t x, float64x2_t y, uint64x2_t cmp)
48{
49  /* If fenv exceptions are to be triggered correctly, fall back to the scalar
50     routine to special lanes.  */
51  return v_call_f64 (exp, x, y, cmp);
52}
53
54#else
55
56# define SpecialOffset v_u64 (0x6000000000000000) /* 0x1p513.  */
57/* SpecialBias1 + SpecialBias1 = asuint(1.0).  */
58# define SpecialBias1 v_u64 (0x7000000000000000) /* 0x1p769.  */
59# define SpecialBias2 v_u64 (0x3010000000000000) /* 0x1p-254.  */
60
61static inline float64x2_t VPCS_ATTR
62special_case (float64x2_t s, float64x2_t y, float64x2_t n)
63{
64  /* 2^(n/N) may overflow, break it up into s1*s2.  */
65  uint64x2_t b = vandq_u64 (vcltzq_f64 (n), SpecialOffset);
66  float64x2_t s1 = vreinterpretq_f64_u64 (vsubq_u64 (SpecialBias1, b));
67  float64x2_t s2 = vreinterpretq_f64_u64 (
68      vaddq_u64 (vsubq_u64 (vreinterpretq_u64_f64 (s), SpecialBias2), b));
69  uint64x2_t cmp = vcagtq_f64 (n, data.scale_thresh);
70  float64x2_t r1 = vmulq_f64 (s1, s1);
71  float64x2_t r0 = vmulq_f64 (vfmaq_f64 (s2, y, s2), s1);
72  return vbslq_f64 (cmp, r1, r0);
73}
74
75#endif
76
77float64x2_t VPCS_ATTR V_NAME_D1 (exp) (float64x2_t x)
78{
79  float64x2_t n, r, r2, s, y, z;
80  uint64x2_t cmp, u, e;
81
82#if WANT_SIMD_EXCEPT
83  /* If any lanes are special, mask them with 1 and retain a copy of x to allow
84     special_case to fix special lanes later. This is only necessary if fenv
85     exceptions are to be triggered correctly.  */
86  float64x2_t xm = x;
87  uint64x2_t iax = vreinterpretq_u64_f64 (vabsq_f64 (x));
88  cmp = vcgeq_u64 (vsubq_u64 (iax, TinyBound), SpecialBound);
89  if (unlikely (v_any_u64 (cmp)))
90    x = vbslq_f64 (cmp, v_f64 (1), x);
91#else
92  cmp = vcagtq_f64 (x, data.special_bound);
93#endif
94
95  /* n = round(x/(ln2/N)).  */
96  z = vfmaq_f64 (data.shift, x, data.inv_ln2);
97  u = vreinterpretq_u64_f64 (z);
98  n = vsubq_f64 (z, data.shift);
99
100  /* r = x - n*ln2/N.  */
101  r = x;
102  r = vfmsq_f64 (r, data.ln2_hi, n);
103  r = vfmsq_f64 (r, data.ln2_lo, n);
104
105  e = vshlq_n_u64 (u, 52 - V_EXP_TABLE_BITS);
106
107  /* y = exp(r) - 1 ~= r + C0 r^2 + C1 r^3 + C2 r^4.  */
108  r2 = vmulq_f64 (r, r);
109  y = vfmaq_f64 (C (0), C (1), r);
110  y = vfmaq_f64 (y, C (2), r2);
111  y = vfmaq_f64 (r, y, r2);
112
113  /* s = 2^(n/N).  */
114  u = (uint64x2_t){ Tab[u[0] & IndexMask], Tab[u[1] & IndexMask] };
115  s = vreinterpretq_f64_u64 (vaddq_u64 (u, e));
116
117  if (unlikely (v_any_u64 (cmp)))
118#if WANT_SIMD_EXCEPT
119    return special_case (xm, vfmaq_f64 (s, y, s), cmp);
120#else
121    return special_case (s, y, n);
122#endif
123
124  return vfmaq_f64 (s, y, s);
125}
126