1/* Builtins' description for AArch64 SIMD architecture. 2 Copyright (C) 2011-2015 Free Software Foundation, Inc. 3 Contributed by ARM Ltd. 4 5 This file is part of GCC. 6 7 GCC is free software; you can redistribute it and/or modify it 8 under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 3, or (at your option) 10 any later version. 11 12 GCC is distributed in the hope that it will be useful, but 13 WITHOUT ANY WARRANTY; without even the implied warranty of 14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 General Public License for more details. 16 17 You should have received a copy of the GNU General Public License 18 along with GCC; see the file COPYING3. If not see 19 <http://www.gnu.org/licenses/>. */ 20 21#include "config.h" 22#include "system.h" 23#include "coretypes.h" 24#include "tm.h" 25#include "rtl.h" 26#include "hash-set.h" 27#include "machmode.h" 28#include "vec.h" 29#include "double-int.h" 30#include "input.h" 31#include "alias.h" 32#include "symtab.h" 33#include "wide-int.h" 34#include "inchash.h" 35#include "tree.h" 36#include "fold-const.h" 37#include "stor-layout.h" 38#include "stringpool.h" 39#include "calls.h" 40#include "hashtab.h" 41#include "hard-reg-set.h" 42#include "function.h" 43#include "flags.h" 44#include "statistics.h" 45#include "real.h" 46#include "fixed-value.h" 47#include "insn-config.h" 48#include "expmed.h" 49#include "dojump.h" 50#include "explow.h" 51#include "emit-rtl.h" 52#include "varasm.h" 53#include "stmt.h" 54#include "expr.h" 55#include "tm_p.h" 56#include "recog.h" 57#include "langhooks.h" 58#include "diagnostic-core.h" 59#include "insn-codes.h" 60#include "optabs.h" 61#include "hash-table.h" 62#include "ggc.h" 63#include "predict.h" 64#include "dominance.h" 65#include "cfg.h" 66#include "cfgrtl.h" 67#include "cfganal.h" 68#include "lcm.h" 69#include "cfgbuild.h" 70#include "cfgcleanup.h" 71#include "basic-block.h" 72#include "tree-ssa-alias.h" 73#include "internal-fn.h" 74#include "gimple-fold.h" 75#include "tree-eh.h" 76#include "gimple-expr.h" 77#include "is-a.h" 78#include "gimple.h" 79#include "gimple-iterator.h" 80 81#define v8qi_UP V8QImode 82#define v4hi_UP V4HImode 83#define v2si_UP V2SImode 84#define v2sf_UP V2SFmode 85#define v1df_UP V1DFmode 86#define di_UP DImode 87#define df_UP DFmode 88#define v16qi_UP V16QImode 89#define v8hi_UP V8HImode 90#define v4si_UP V4SImode 91#define v4sf_UP V4SFmode 92#define v2di_UP V2DImode 93#define v2df_UP V2DFmode 94#define ti_UP TImode 95#define ei_UP EImode 96#define oi_UP OImode 97#define ci_UP CImode 98#define xi_UP XImode 99#define si_UP SImode 100#define sf_UP SFmode 101#define hi_UP HImode 102#define qi_UP QImode 103#define UP(X) X##_UP 104 105#define SIMD_MAX_BUILTIN_ARGS 5 106 107enum aarch64_type_qualifiers 108{ 109 /* T foo. */ 110 qualifier_none = 0x0, 111 /* unsigned T foo. */ 112 qualifier_unsigned = 0x1, /* 1 << 0 */ 113 /* const T foo. */ 114 qualifier_const = 0x2, /* 1 << 1 */ 115 /* T *foo. */ 116 qualifier_pointer = 0x4, /* 1 << 2 */ 117 /* Used when expanding arguments if an operand could 118 be an immediate. */ 119 qualifier_immediate = 0x8, /* 1 << 3 */ 120 qualifier_maybe_immediate = 0x10, /* 1 << 4 */ 121 /* void foo (...). */ 122 qualifier_void = 0x20, /* 1 << 5 */ 123 /* Some patterns may have internal operands, this qualifier is an 124 instruction to the initialisation code to skip this operand. */ 125 qualifier_internal = 0x40, /* 1 << 6 */ 126 /* Some builtins should use the T_*mode* encoded in a simd_builtin_datum 127 rather than using the type of the operand. */ 128 qualifier_map_mode = 0x80, /* 1 << 7 */ 129 /* qualifier_pointer | qualifier_map_mode */ 130 qualifier_pointer_map_mode = 0x84, 131 /* qualifier_const | qualifier_pointer | qualifier_map_mode */ 132 qualifier_const_pointer_map_mode = 0x86, 133 /* Polynomial types. */ 134 qualifier_poly = 0x100, 135 /* Lane indices - must be in range, and flipped for bigendian. */ 136 qualifier_lane_index = 0x200 137}; 138 139typedef struct 140{ 141 const char *name; 142 machine_mode mode; 143 const enum insn_code code; 144 unsigned int fcode; 145 enum aarch64_type_qualifiers *qualifiers; 146} aarch64_simd_builtin_datum; 147 148static enum aarch64_type_qualifiers 149aarch64_types_unop_qualifiers[SIMD_MAX_BUILTIN_ARGS] 150 = { qualifier_none, qualifier_none }; 151#define TYPES_UNOP (aarch64_types_unop_qualifiers) 152static enum aarch64_type_qualifiers 153aarch64_types_unopu_qualifiers[SIMD_MAX_BUILTIN_ARGS] 154 = { qualifier_unsigned, qualifier_unsigned }; 155#define TYPES_UNOPU (aarch64_types_unopu_qualifiers) 156static enum aarch64_type_qualifiers 157aarch64_types_binop_qualifiers[SIMD_MAX_BUILTIN_ARGS] 158 = { qualifier_none, qualifier_none, qualifier_maybe_immediate }; 159#define TYPES_BINOP (aarch64_types_binop_qualifiers) 160static enum aarch64_type_qualifiers 161aarch64_types_binopu_qualifiers[SIMD_MAX_BUILTIN_ARGS] 162 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned }; 163#define TYPES_BINOPU (aarch64_types_binopu_qualifiers) 164static enum aarch64_type_qualifiers 165aarch64_types_binop_uus_qualifiers[SIMD_MAX_BUILTIN_ARGS] 166 = { qualifier_unsigned, qualifier_unsigned, qualifier_none }; 167#define TYPES_BINOP_UUS (aarch64_types_binop_uus_qualifiers) 168static enum aarch64_type_qualifiers 169aarch64_types_binop_ssu_qualifiers[SIMD_MAX_BUILTIN_ARGS] 170 = { qualifier_none, qualifier_none, qualifier_unsigned }; 171#define TYPES_BINOP_SSU (aarch64_types_binop_ssu_qualifiers) 172static enum aarch64_type_qualifiers 173aarch64_types_binopp_qualifiers[SIMD_MAX_BUILTIN_ARGS] 174 = { qualifier_poly, qualifier_poly, qualifier_poly }; 175#define TYPES_BINOPP (aarch64_types_binopp_qualifiers) 176 177static enum aarch64_type_qualifiers 178aarch64_types_ternop_qualifiers[SIMD_MAX_BUILTIN_ARGS] 179 = { qualifier_none, qualifier_none, qualifier_none, qualifier_none }; 180#define TYPES_TERNOP (aarch64_types_ternop_qualifiers) 181static enum aarch64_type_qualifiers 182aarch64_types_ternop_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS] 183 = { qualifier_none, qualifier_none, qualifier_none, qualifier_lane_index }; 184#define TYPES_TERNOP_LANE (aarch64_types_ternop_lane_qualifiers) 185static enum aarch64_type_qualifiers 186aarch64_types_ternopu_qualifiers[SIMD_MAX_BUILTIN_ARGS] 187 = { qualifier_unsigned, qualifier_unsigned, 188 qualifier_unsigned, qualifier_unsigned }; 189#define TYPES_TERNOPU (aarch64_types_ternopu_qualifiers) 190 191static enum aarch64_type_qualifiers 192aarch64_types_quadop_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS] 193 = { qualifier_none, qualifier_none, qualifier_none, 194 qualifier_none, qualifier_lane_index }; 195#define TYPES_QUADOP_LANE (aarch64_types_quadop_lane_qualifiers) 196 197static enum aarch64_type_qualifiers 198aarch64_types_binop_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS] 199 = { qualifier_none, qualifier_none, qualifier_immediate }; 200#define TYPES_GETREG (aarch64_types_binop_imm_qualifiers) 201#define TYPES_SHIFTIMM (aarch64_types_binop_imm_qualifiers) 202static enum aarch64_type_qualifiers 203aarch64_types_shift_to_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS] 204 = { qualifier_unsigned, qualifier_none, qualifier_immediate }; 205#define TYPES_SHIFTIMM_USS (aarch64_types_shift_to_unsigned_qualifiers) 206static enum aarch64_type_qualifiers 207aarch64_types_unsigned_shift_qualifiers[SIMD_MAX_BUILTIN_ARGS] 208 = { qualifier_unsigned, qualifier_unsigned, qualifier_immediate }; 209#define TYPES_USHIFTIMM (aarch64_types_unsigned_shift_qualifiers) 210 211static enum aarch64_type_qualifiers 212aarch64_types_ternop_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS] 213 = { qualifier_none, qualifier_none, qualifier_none, qualifier_immediate }; 214#define TYPES_SETREG (aarch64_types_ternop_imm_qualifiers) 215#define TYPES_SHIFTINSERT (aarch64_types_ternop_imm_qualifiers) 216#define TYPES_SHIFTACC (aarch64_types_ternop_imm_qualifiers) 217 218static enum aarch64_type_qualifiers 219aarch64_types_unsigned_shiftacc_qualifiers[SIMD_MAX_BUILTIN_ARGS] 220 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned, 221 qualifier_immediate }; 222#define TYPES_USHIFTACC (aarch64_types_unsigned_shiftacc_qualifiers) 223 224 225static enum aarch64_type_qualifiers 226aarch64_types_combine_qualifiers[SIMD_MAX_BUILTIN_ARGS] 227 = { qualifier_none, qualifier_none, qualifier_none }; 228#define TYPES_COMBINE (aarch64_types_combine_qualifiers) 229 230static enum aarch64_type_qualifiers 231aarch64_types_load1_qualifiers[SIMD_MAX_BUILTIN_ARGS] 232 = { qualifier_none, qualifier_const_pointer_map_mode }; 233#define TYPES_LOAD1 (aarch64_types_load1_qualifiers) 234#define TYPES_LOADSTRUCT (aarch64_types_load1_qualifiers) 235static enum aarch64_type_qualifiers 236aarch64_types_loadstruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS] 237 = { qualifier_none, qualifier_const_pointer_map_mode, 238 qualifier_none, qualifier_none }; 239#define TYPES_LOADSTRUCT_LANE (aarch64_types_loadstruct_lane_qualifiers) 240 241static enum aarch64_type_qualifiers 242aarch64_types_bsl_p_qualifiers[SIMD_MAX_BUILTIN_ARGS] 243 = { qualifier_poly, qualifier_unsigned, 244 qualifier_poly, qualifier_poly }; 245#define TYPES_BSL_P (aarch64_types_bsl_p_qualifiers) 246static enum aarch64_type_qualifiers 247aarch64_types_bsl_s_qualifiers[SIMD_MAX_BUILTIN_ARGS] 248 = { qualifier_none, qualifier_unsigned, 249 qualifier_none, qualifier_none }; 250#define TYPES_BSL_S (aarch64_types_bsl_s_qualifiers) 251static enum aarch64_type_qualifiers 252aarch64_types_bsl_u_qualifiers[SIMD_MAX_BUILTIN_ARGS] 253 = { qualifier_unsigned, qualifier_unsigned, 254 qualifier_unsigned, qualifier_unsigned }; 255#define TYPES_BSL_U (aarch64_types_bsl_u_qualifiers) 256 257/* The first argument (return type) of a store should be void type, 258 which we represent with qualifier_void. Their first operand will be 259 a DImode pointer to the location to store to, so we must use 260 qualifier_map_mode | qualifier_pointer to build a pointer to the 261 element type of the vector. */ 262static enum aarch64_type_qualifiers 263aarch64_types_store1_qualifiers[SIMD_MAX_BUILTIN_ARGS] 264 = { qualifier_void, qualifier_pointer_map_mode, qualifier_none }; 265#define TYPES_STORE1 (aarch64_types_store1_qualifiers) 266#define TYPES_STORESTRUCT (aarch64_types_store1_qualifiers) 267static enum aarch64_type_qualifiers 268aarch64_types_storestruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS] 269 = { qualifier_void, qualifier_pointer_map_mode, 270 qualifier_none, qualifier_none }; 271#define TYPES_STORESTRUCT_LANE (aarch64_types_storestruct_lane_qualifiers) 272 273#define CF0(N, X) CODE_FOR_aarch64_##N##X 274#define CF1(N, X) CODE_FOR_##N##X##1 275#define CF2(N, X) CODE_FOR_##N##X##2 276#define CF3(N, X) CODE_FOR_##N##X##3 277#define CF4(N, X) CODE_FOR_##N##X##4 278#define CF10(N, X) CODE_FOR_##N##X 279 280#define VAR1(T, N, MAP, A) \ 281 {#N #A, UP (A), CF##MAP (N, A), 0, TYPES_##T}, 282#define VAR2(T, N, MAP, A, B) \ 283 VAR1 (T, N, MAP, A) \ 284 VAR1 (T, N, MAP, B) 285#define VAR3(T, N, MAP, A, B, C) \ 286 VAR2 (T, N, MAP, A, B) \ 287 VAR1 (T, N, MAP, C) 288#define VAR4(T, N, MAP, A, B, C, D) \ 289 VAR3 (T, N, MAP, A, B, C) \ 290 VAR1 (T, N, MAP, D) 291#define VAR5(T, N, MAP, A, B, C, D, E) \ 292 VAR4 (T, N, MAP, A, B, C, D) \ 293 VAR1 (T, N, MAP, E) 294#define VAR6(T, N, MAP, A, B, C, D, E, F) \ 295 VAR5 (T, N, MAP, A, B, C, D, E) \ 296 VAR1 (T, N, MAP, F) 297#define VAR7(T, N, MAP, A, B, C, D, E, F, G) \ 298 VAR6 (T, N, MAP, A, B, C, D, E, F) \ 299 VAR1 (T, N, MAP, G) 300#define VAR8(T, N, MAP, A, B, C, D, E, F, G, H) \ 301 VAR7 (T, N, MAP, A, B, C, D, E, F, G) \ 302 VAR1 (T, N, MAP, H) 303#define VAR9(T, N, MAP, A, B, C, D, E, F, G, H, I) \ 304 VAR8 (T, N, MAP, A, B, C, D, E, F, G, H) \ 305 VAR1 (T, N, MAP, I) 306#define VAR10(T, N, MAP, A, B, C, D, E, F, G, H, I, J) \ 307 VAR9 (T, N, MAP, A, B, C, D, E, F, G, H, I) \ 308 VAR1 (T, N, MAP, J) 309#define VAR11(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K) \ 310 VAR10 (T, N, MAP, A, B, C, D, E, F, G, H, I, J) \ 311 VAR1 (T, N, MAP, K) 312#define VAR12(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K, L) \ 313 VAR11 (T, N, MAP, A, B, C, D, E, F, G, H, I, J, K) \ 314 VAR1 (T, N, MAP, L) 315 316#include "aarch64-builtin-iterators.h" 317 318static aarch64_simd_builtin_datum aarch64_simd_builtin_data[] = { 319#include "aarch64-simd-builtins.def" 320}; 321 322/* There's only 8 CRC32 builtins. Probably not worth their own .def file. */ 323#define AARCH64_CRC32_BUILTINS \ 324 CRC32_BUILTIN (crc32b, QI) \ 325 CRC32_BUILTIN (crc32h, HI) \ 326 CRC32_BUILTIN (crc32w, SI) \ 327 CRC32_BUILTIN (crc32x, DI) \ 328 CRC32_BUILTIN (crc32cb, QI) \ 329 CRC32_BUILTIN (crc32ch, HI) \ 330 CRC32_BUILTIN (crc32cw, SI) \ 331 CRC32_BUILTIN (crc32cx, DI) 332 333typedef struct 334{ 335 const char *name; 336 machine_mode mode; 337 const enum insn_code icode; 338 unsigned int fcode; 339} aarch64_crc_builtin_datum; 340 341#define CRC32_BUILTIN(N, M) \ 342 AARCH64_BUILTIN_##N, 343 344#undef VAR1 345#define VAR1(T, N, MAP, A) \ 346 AARCH64_SIMD_BUILTIN_##T##_##N##A, 347 348enum aarch64_builtins 349{ 350 AARCH64_BUILTIN_MIN, 351 352 AARCH64_BUILTIN_GET_FPCR, 353 AARCH64_BUILTIN_SET_FPCR, 354 AARCH64_BUILTIN_GET_FPSR, 355 AARCH64_BUILTIN_SET_FPSR, 356 357 AARCH64_SIMD_BUILTIN_BASE, 358 AARCH64_SIMD_BUILTIN_LANE_CHECK, 359#include "aarch64-simd-builtins.def" 360 /* The first enum element which is based on an insn_data pattern. */ 361 AARCH64_SIMD_PATTERN_START = AARCH64_SIMD_BUILTIN_LANE_CHECK + 1, 362 AARCH64_SIMD_BUILTIN_MAX = AARCH64_SIMD_PATTERN_START 363 + ARRAY_SIZE (aarch64_simd_builtin_data) - 1, 364 AARCH64_CRC32_BUILTIN_BASE, 365 AARCH64_CRC32_BUILTINS 366 AARCH64_CRC32_BUILTIN_MAX, 367 AARCH64_BUILTIN_MAX 368}; 369 370#undef CRC32_BUILTIN 371#define CRC32_BUILTIN(N, M) \ 372 {"__builtin_aarch64_"#N, M##mode, CODE_FOR_aarch64_##N, AARCH64_BUILTIN_##N}, 373 374static aarch64_crc_builtin_datum aarch64_crc_builtin_data[] = { 375 AARCH64_CRC32_BUILTINS 376}; 377 378#undef CRC32_BUILTIN 379 380static GTY(()) tree aarch64_builtin_decls[AARCH64_BUILTIN_MAX]; 381 382#define NUM_DREG_TYPES 6 383#define NUM_QREG_TYPES 6 384 385/* Internal scalar builtin types. These types are used to support 386 neon intrinsic builtins. They are _not_ user-visible types. Therefore 387 the mangling for these types are implementation defined. */ 388const char *aarch64_scalar_builtin_types[] = { 389 "__builtin_aarch64_simd_qi", 390 "__builtin_aarch64_simd_hi", 391 "__builtin_aarch64_simd_si", 392 "__builtin_aarch64_simd_sf", 393 "__builtin_aarch64_simd_di", 394 "__builtin_aarch64_simd_df", 395 "__builtin_aarch64_simd_poly8", 396 "__builtin_aarch64_simd_poly16", 397 "__builtin_aarch64_simd_poly64", 398 "__builtin_aarch64_simd_poly128", 399 "__builtin_aarch64_simd_ti", 400 "__builtin_aarch64_simd_uqi", 401 "__builtin_aarch64_simd_uhi", 402 "__builtin_aarch64_simd_usi", 403 "__builtin_aarch64_simd_udi", 404 "__builtin_aarch64_simd_ei", 405 "__builtin_aarch64_simd_oi", 406 "__builtin_aarch64_simd_ci", 407 "__builtin_aarch64_simd_xi", 408 NULL 409}; 410 411#define ENTRY(E, M, Q, G) E, 412enum aarch64_simd_type 413{ 414#include "aarch64-simd-builtin-types.def" 415 ARM_NEON_H_TYPES_LAST 416}; 417#undef ENTRY 418 419struct aarch64_simd_type_info 420{ 421 enum aarch64_simd_type type; 422 423 /* Internal type name. */ 424 const char *name; 425 426 /* Internal type name(mangled). The mangled names conform to the 427 AAPCS64 (see "Procedure Call Standard for the ARM 64-bit Architecture", 428 Appendix A). To qualify for emission with the mangled names defined in 429 that document, a vector type must not only be of the correct mode but also 430 be of the correct internal AdvSIMD vector type (e.g. __Int8x8_t); these 431 types are registered by aarch64_init_simd_builtin_types (). In other 432 words, vector types defined in other ways e.g. via vector_size attribute 433 will get default mangled names. */ 434 const char *mangle; 435 436 /* Internal type. */ 437 tree itype; 438 439 /* Element type. */ 440 tree eltype; 441 442 /* Machine mode the internal type maps to. */ 443 enum machine_mode mode; 444 445 /* Qualifiers. */ 446 enum aarch64_type_qualifiers q; 447}; 448 449#define ENTRY(E, M, Q, G) \ 450 {E, "__" #E, #G "__" #E, NULL_TREE, NULL_TREE, M##mode, qualifier_##Q}, 451static struct aarch64_simd_type_info aarch64_simd_types [] = { 452#include "aarch64-simd-builtin-types.def" 453}; 454#undef ENTRY 455 456static tree aarch64_simd_intOI_type_node = NULL_TREE; 457static tree aarch64_simd_intEI_type_node = NULL_TREE; 458static tree aarch64_simd_intCI_type_node = NULL_TREE; 459static tree aarch64_simd_intXI_type_node = NULL_TREE; 460 461static const char * 462aarch64_mangle_builtin_scalar_type (const_tree type) 463{ 464 int i = 0; 465 466 while (aarch64_scalar_builtin_types[i] != NULL) 467 { 468 const char *name = aarch64_scalar_builtin_types[i]; 469 470 if (TREE_CODE (TYPE_NAME (type)) == TYPE_DECL 471 && DECL_NAME (TYPE_NAME (type)) 472 && !strcmp (IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type))), name)) 473 return aarch64_scalar_builtin_types[i]; 474 i++; 475 } 476 return NULL; 477} 478 479static const char * 480aarch64_mangle_builtin_vector_type (const_tree type) 481{ 482 int i; 483 int nelts = sizeof (aarch64_simd_types) / sizeof (aarch64_simd_types[0]); 484 485 for (i = 0; i < nelts; i++) 486 if (aarch64_simd_types[i].mode == TYPE_MODE (type) 487 && TYPE_NAME (type) 488 && TREE_CODE (TYPE_NAME (type)) == TYPE_DECL 489 && DECL_NAME (TYPE_NAME (type)) 490 && !strcmp 491 (IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type))), 492 aarch64_simd_types[i].name)) 493 return aarch64_simd_types[i].mangle; 494 495 return NULL; 496} 497 498const char * 499aarch64_mangle_builtin_type (const_tree type) 500{ 501 const char *mangle; 502 /* Walk through all the AArch64 builtins types tables to filter out the 503 incoming type. */ 504 if ((mangle = aarch64_mangle_builtin_vector_type (type)) 505 || (mangle = aarch64_mangle_builtin_scalar_type (type))) 506 return mangle; 507 508 return NULL; 509} 510 511static tree 512aarch64_simd_builtin_std_type (enum machine_mode mode, 513 enum aarch64_type_qualifiers q) 514{ 515#define QUAL_TYPE(M) \ 516 ((q == qualifier_none) ? int##M##_type_node : unsigned_int##M##_type_node); 517 switch (mode) 518 { 519 case QImode: 520 return QUAL_TYPE (QI); 521 case HImode: 522 return QUAL_TYPE (HI); 523 case SImode: 524 return QUAL_TYPE (SI); 525 case DImode: 526 return QUAL_TYPE (DI); 527 case TImode: 528 return QUAL_TYPE (TI); 529 case OImode: 530 return aarch64_simd_intOI_type_node; 531 case EImode: 532 return aarch64_simd_intEI_type_node; 533 case CImode: 534 return aarch64_simd_intCI_type_node; 535 case XImode: 536 return aarch64_simd_intXI_type_node; 537 case SFmode: 538 return float_type_node; 539 case DFmode: 540 return double_type_node; 541 default: 542 gcc_unreachable (); 543 } 544#undef QUAL_TYPE 545} 546 547static tree 548aarch64_lookup_simd_builtin_type (enum machine_mode mode, 549 enum aarch64_type_qualifiers q) 550{ 551 int i; 552 int nelts = sizeof (aarch64_simd_types) / sizeof (aarch64_simd_types[0]); 553 554 /* Non-poly scalar modes map to standard types not in the table. */ 555 if (q != qualifier_poly && !VECTOR_MODE_P (mode)) 556 return aarch64_simd_builtin_std_type (mode, q); 557 558 for (i = 0; i < nelts; i++) 559 if (aarch64_simd_types[i].mode == mode 560 && aarch64_simd_types[i].q == q) 561 return aarch64_simd_types[i].itype; 562 563 return NULL_TREE; 564} 565 566static tree 567aarch64_simd_builtin_type (enum machine_mode mode, 568 bool unsigned_p, bool poly_p) 569{ 570 if (poly_p) 571 return aarch64_lookup_simd_builtin_type (mode, qualifier_poly); 572 else if (unsigned_p) 573 return aarch64_lookup_simd_builtin_type (mode, qualifier_unsigned); 574 else 575 return aarch64_lookup_simd_builtin_type (mode, qualifier_none); 576} 577 578static void 579aarch64_init_simd_builtin_types (void) 580{ 581 int i; 582 int nelts = sizeof (aarch64_simd_types) / sizeof (aarch64_simd_types[0]); 583 tree tdecl; 584 585 /* Init all the element types built by the front-end. */ 586 aarch64_simd_types[Int8x8_t].eltype = intQI_type_node; 587 aarch64_simd_types[Int8x16_t].eltype = intQI_type_node; 588 aarch64_simd_types[Int16x4_t].eltype = intHI_type_node; 589 aarch64_simd_types[Int16x8_t].eltype = intHI_type_node; 590 aarch64_simd_types[Int32x2_t].eltype = intSI_type_node; 591 aarch64_simd_types[Int32x4_t].eltype = intSI_type_node; 592 aarch64_simd_types[Int64x1_t].eltype = intDI_type_node; 593 aarch64_simd_types[Int64x2_t].eltype = intDI_type_node; 594 aarch64_simd_types[Uint8x8_t].eltype = unsigned_intQI_type_node; 595 aarch64_simd_types[Uint8x16_t].eltype = unsigned_intQI_type_node; 596 aarch64_simd_types[Uint16x4_t].eltype = unsigned_intHI_type_node; 597 aarch64_simd_types[Uint16x8_t].eltype = unsigned_intHI_type_node; 598 aarch64_simd_types[Uint32x2_t].eltype = unsigned_intSI_type_node; 599 aarch64_simd_types[Uint32x4_t].eltype = unsigned_intSI_type_node; 600 aarch64_simd_types[Uint64x1_t].eltype = unsigned_intDI_type_node; 601 aarch64_simd_types[Uint64x2_t].eltype = unsigned_intDI_type_node; 602 603 /* Poly types are a world of their own. */ 604 aarch64_simd_types[Poly8_t].eltype = aarch64_simd_types[Poly8_t].itype = 605 build_distinct_type_copy (unsigned_intQI_type_node); 606 aarch64_simd_types[Poly16_t].eltype = aarch64_simd_types[Poly16_t].itype = 607 build_distinct_type_copy (unsigned_intHI_type_node); 608 aarch64_simd_types[Poly64_t].eltype = aarch64_simd_types[Poly64_t].itype = 609 build_distinct_type_copy (unsigned_intDI_type_node); 610 aarch64_simd_types[Poly128_t].eltype = aarch64_simd_types[Poly128_t].itype = 611 build_distinct_type_copy (unsigned_intTI_type_node); 612 /* Init poly vector element types with scalar poly types. */ 613 aarch64_simd_types[Poly8x8_t].eltype = aarch64_simd_types[Poly8_t].itype; 614 aarch64_simd_types[Poly8x16_t].eltype = aarch64_simd_types[Poly8_t].itype; 615 aarch64_simd_types[Poly16x4_t].eltype = aarch64_simd_types[Poly16_t].itype; 616 aarch64_simd_types[Poly16x8_t].eltype = aarch64_simd_types[Poly16_t].itype; 617 aarch64_simd_types[Poly64x1_t].eltype = aarch64_simd_types[Poly64_t].itype; 618 aarch64_simd_types[Poly64x2_t].eltype = aarch64_simd_types[Poly64_t].itype; 619 620 /* Continue with standard types. */ 621 aarch64_simd_types[Float32x2_t].eltype = float_type_node; 622 aarch64_simd_types[Float32x4_t].eltype = float_type_node; 623 aarch64_simd_types[Float64x1_t].eltype = double_type_node; 624 aarch64_simd_types[Float64x2_t].eltype = double_type_node; 625 626 for (i = 0; i < nelts; i++) 627 { 628 tree eltype = aarch64_simd_types[i].eltype; 629 enum machine_mode mode = aarch64_simd_types[i].mode; 630 631 if (aarch64_simd_types[i].itype == NULL) 632 { 633 aarch64_simd_types[i].itype 634 = build_distinct_type_copy 635 (build_vector_type (eltype, GET_MODE_NUNITS (mode))); 636 SET_TYPE_STRUCTURAL_EQUALITY (aarch64_simd_types[i].itype); 637 } 638 639 tdecl = add_builtin_type (aarch64_simd_types[i].name, 640 aarch64_simd_types[i].itype); 641 TYPE_NAME (aarch64_simd_types[i].itype) = tdecl; 642 } 643 644#define AARCH64_BUILD_SIGNED_TYPE(mode) \ 645 make_signed_type (GET_MODE_PRECISION (mode)); 646 aarch64_simd_intOI_type_node = AARCH64_BUILD_SIGNED_TYPE (OImode); 647 aarch64_simd_intEI_type_node = AARCH64_BUILD_SIGNED_TYPE (EImode); 648 aarch64_simd_intCI_type_node = AARCH64_BUILD_SIGNED_TYPE (CImode); 649 aarch64_simd_intXI_type_node = AARCH64_BUILD_SIGNED_TYPE (XImode); 650#undef AARCH64_BUILD_SIGNED_TYPE 651 652 tdecl = add_builtin_type 653 ("__builtin_aarch64_simd_ei" , aarch64_simd_intEI_type_node); 654 TYPE_NAME (aarch64_simd_intEI_type_node) = tdecl; 655 tdecl = add_builtin_type 656 ("__builtin_aarch64_simd_oi" , aarch64_simd_intOI_type_node); 657 TYPE_NAME (aarch64_simd_intOI_type_node) = tdecl; 658 tdecl = add_builtin_type 659 ("__builtin_aarch64_simd_ci" , aarch64_simd_intCI_type_node); 660 TYPE_NAME (aarch64_simd_intCI_type_node) = tdecl; 661 tdecl = add_builtin_type 662 ("__builtin_aarch64_simd_xi" , aarch64_simd_intXI_type_node); 663 TYPE_NAME (aarch64_simd_intXI_type_node) = tdecl; 664} 665 666static void 667aarch64_init_simd_builtin_scalar_types (void) 668{ 669 /* Define typedefs for all the standard scalar types. */ 670 (*lang_hooks.types.register_builtin_type) (intQI_type_node, 671 "__builtin_aarch64_simd_qi"); 672 (*lang_hooks.types.register_builtin_type) (intHI_type_node, 673 "__builtin_aarch64_simd_hi"); 674 (*lang_hooks.types.register_builtin_type) (intSI_type_node, 675 "__builtin_aarch64_simd_si"); 676 (*lang_hooks.types.register_builtin_type) (float_type_node, 677 "__builtin_aarch64_simd_sf"); 678 (*lang_hooks.types.register_builtin_type) (intDI_type_node, 679 "__builtin_aarch64_simd_di"); 680 (*lang_hooks.types.register_builtin_type) (double_type_node, 681 "__builtin_aarch64_simd_df"); 682 (*lang_hooks.types.register_builtin_type) (unsigned_intQI_type_node, 683 "__builtin_aarch64_simd_poly8"); 684 (*lang_hooks.types.register_builtin_type) (unsigned_intHI_type_node, 685 "__builtin_aarch64_simd_poly16"); 686 (*lang_hooks.types.register_builtin_type) (unsigned_intDI_type_node, 687 "__builtin_aarch64_simd_poly64"); 688 (*lang_hooks.types.register_builtin_type) (unsigned_intTI_type_node, 689 "__builtin_aarch64_simd_poly128"); 690 (*lang_hooks.types.register_builtin_type) (intTI_type_node, 691 "__builtin_aarch64_simd_ti"); 692 /* Unsigned integer types for various mode sizes. */ 693 (*lang_hooks.types.register_builtin_type) (unsigned_intQI_type_node, 694 "__builtin_aarch64_simd_uqi"); 695 (*lang_hooks.types.register_builtin_type) (unsigned_intHI_type_node, 696 "__builtin_aarch64_simd_uhi"); 697 (*lang_hooks.types.register_builtin_type) (unsigned_intSI_type_node, 698 "__builtin_aarch64_simd_usi"); 699 (*lang_hooks.types.register_builtin_type) (unsigned_intDI_type_node, 700 "__builtin_aarch64_simd_udi"); 701} 702 703static void 704aarch64_init_simd_builtins (void) 705{ 706 unsigned int i, fcode = AARCH64_SIMD_PATTERN_START; 707 708 aarch64_init_simd_builtin_types (); 709 710 /* Strong-typing hasn't been implemented for all AdvSIMD builtin intrinsics. 711 Therefore we need to preserve the old __builtin scalar types. It can be 712 removed once all the intrinsics become strongly typed using the qualifier 713 system. */ 714 aarch64_init_simd_builtin_scalar_types (); 715 716 tree lane_check_fpr = build_function_type_list (void_type_node, 717 size_type_node, 718 size_type_node, 719 intSI_type_node, 720 NULL); 721 aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_LANE_CHECK] = 722 add_builtin_function ("__builtin_aarch64_im_lane_boundsi", lane_check_fpr, 723 AARCH64_SIMD_BUILTIN_LANE_CHECK, BUILT_IN_MD, 724 NULL, NULL_TREE); 725 726 for (i = 0; i < ARRAY_SIZE (aarch64_simd_builtin_data); i++, fcode++) 727 { 728 bool print_type_signature_p = false; 729 char type_signature[SIMD_MAX_BUILTIN_ARGS] = { 0 }; 730 aarch64_simd_builtin_datum *d = &aarch64_simd_builtin_data[i]; 731 char namebuf[60]; 732 tree ftype = NULL; 733 tree fndecl = NULL; 734 735 d->fcode = fcode; 736 737 /* We must track two variables here. op_num is 738 the operand number as in the RTL pattern. This is 739 required to access the mode (e.g. V4SF mode) of the 740 argument, from which the base type can be derived. 741 arg_num is an index in to the qualifiers data, which 742 gives qualifiers to the type (e.g. const unsigned). 743 The reason these two variables may differ by one is the 744 void return type. While all return types take the 0th entry 745 in the qualifiers array, there is no operand for them in the 746 RTL pattern. */ 747 int op_num = insn_data[d->code].n_operands - 1; 748 int arg_num = d->qualifiers[0] & qualifier_void 749 ? op_num + 1 750 : op_num; 751 tree return_type = void_type_node, args = void_list_node; 752 tree eltype; 753 754 /* Build a function type directly from the insn_data for this 755 builtin. The build_function_type () function takes care of 756 removing duplicates for us. */ 757 for (; op_num >= 0; arg_num--, op_num--) 758 { 759 machine_mode op_mode = insn_data[d->code].operand[op_num].mode; 760 enum aarch64_type_qualifiers qualifiers = d->qualifiers[arg_num]; 761 762 if (qualifiers & qualifier_unsigned) 763 { 764 type_signature[arg_num] = 'u'; 765 print_type_signature_p = true; 766 } 767 else if (qualifiers & qualifier_poly) 768 { 769 type_signature[arg_num] = 'p'; 770 print_type_signature_p = true; 771 } 772 else 773 type_signature[arg_num] = 's'; 774 775 /* Skip an internal operand for vget_{low, high}. */ 776 if (qualifiers & qualifier_internal) 777 continue; 778 779 /* Some builtins have different user-facing types 780 for certain arguments, encoded in d->mode. */ 781 if (qualifiers & qualifier_map_mode) 782 op_mode = d->mode; 783 784 /* For pointers, we want a pointer to the basic type 785 of the vector. */ 786 if (qualifiers & qualifier_pointer && VECTOR_MODE_P (op_mode)) 787 op_mode = GET_MODE_INNER (op_mode); 788 789 eltype = aarch64_simd_builtin_type 790 (op_mode, 791 (qualifiers & qualifier_unsigned) != 0, 792 (qualifiers & qualifier_poly) != 0); 793 gcc_assert (eltype != NULL); 794 795 /* Add qualifiers. */ 796 if (qualifiers & qualifier_const) 797 eltype = build_qualified_type (eltype, TYPE_QUAL_CONST); 798 799 if (qualifiers & qualifier_pointer) 800 eltype = build_pointer_type (eltype); 801 802 /* If we have reached arg_num == 0, we are at a non-void 803 return type. Otherwise, we are still processing 804 arguments. */ 805 if (arg_num == 0) 806 return_type = eltype; 807 else 808 args = tree_cons (NULL_TREE, eltype, args); 809 } 810 811 ftype = build_function_type (return_type, args); 812 813 gcc_assert (ftype != NULL); 814 815 if (print_type_signature_p) 816 snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s_%s", 817 d->name, type_signature); 818 else 819 snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s", 820 d->name); 821 822 fndecl = add_builtin_function (namebuf, ftype, fcode, BUILT_IN_MD, 823 NULL, NULL_TREE); 824 aarch64_builtin_decls[fcode] = fndecl; 825 } 826} 827 828static void 829aarch64_init_crc32_builtins () 830{ 831 tree usi_type = aarch64_simd_builtin_std_type (SImode, qualifier_unsigned); 832 unsigned int i = 0; 833 834 for (i = 0; i < ARRAY_SIZE (aarch64_crc_builtin_data); ++i) 835 { 836 aarch64_crc_builtin_datum* d = &aarch64_crc_builtin_data[i]; 837 tree argtype = aarch64_simd_builtin_std_type (d->mode, 838 qualifier_unsigned); 839 tree ftype = build_function_type_list (usi_type, usi_type, argtype, NULL_TREE); 840 tree fndecl = add_builtin_function (d->name, ftype, d->fcode, 841 BUILT_IN_MD, NULL, NULL_TREE); 842 843 aarch64_builtin_decls[d->fcode] = fndecl; 844 } 845} 846 847void 848aarch64_init_builtins (void) 849{ 850 tree ftype_set_fpr 851 = build_function_type_list (void_type_node, unsigned_type_node, NULL); 852 tree ftype_get_fpr 853 = build_function_type_list (unsigned_type_node, NULL); 854 855 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR] 856 = add_builtin_function ("__builtin_aarch64_get_fpcr", ftype_get_fpr, 857 AARCH64_BUILTIN_GET_FPCR, BUILT_IN_MD, NULL, NULL_TREE); 858 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR] 859 = add_builtin_function ("__builtin_aarch64_set_fpcr", ftype_set_fpr, 860 AARCH64_BUILTIN_SET_FPCR, BUILT_IN_MD, NULL, NULL_TREE); 861 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR] 862 = add_builtin_function ("__builtin_aarch64_get_fpsr", ftype_get_fpr, 863 AARCH64_BUILTIN_GET_FPSR, BUILT_IN_MD, NULL, NULL_TREE); 864 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR] 865 = add_builtin_function ("__builtin_aarch64_set_fpsr", ftype_set_fpr, 866 AARCH64_BUILTIN_SET_FPSR, BUILT_IN_MD, NULL, NULL_TREE); 867 868 if (TARGET_SIMD) 869 aarch64_init_simd_builtins (); 870 if (TARGET_CRC32) 871 aarch64_init_crc32_builtins (); 872} 873 874tree 875aarch64_builtin_decl (unsigned code, bool initialize_p ATTRIBUTE_UNUSED) 876{ 877 if (code >= AARCH64_BUILTIN_MAX) 878 return error_mark_node; 879 880 return aarch64_builtin_decls[code]; 881} 882 883typedef enum 884{ 885 SIMD_ARG_COPY_TO_REG, 886 SIMD_ARG_CONSTANT, 887 SIMD_ARG_LANE_INDEX, 888 SIMD_ARG_STOP 889} builtin_simd_arg; 890 891static rtx 892aarch64_simd_expand_args (rtx target, int icode, int have_retval, 893 tree exp, builtin_simd_arg *args) 894{ 895 rtx pat; 896 rtx op[SIMD_MAX_BUILTIN_ARGS + 1]; /* First element for result operand. */ 897 int opc = 0; 898 899 if (have_retval) 900 { 901 machine_mode tmode = insn_data[icode].operand[0].mode; 902 if (!target 903 || GET_MODE (target) != tmode 904 || !(*insn_data[icode].operand[0].predicate) (target, tmode)) 905 target = gen_reg_rtx (tmode); 906 op[opc++] = target; 907 } 908 909 for (;;) 910 { 911 builtin_simd_arg thisarg = args[opc - have_retval]; 912 913 if (thisarg == SIMD_ARG_STOP) 914 break; 915 else 916 { 917 tree arg = CALL_EXPR_ARG (exp, opc - have_retval); 918 enum machine_mode mode = insn_data[icode].operand[opc].mode; 919 op[opc] = expand_normal (arg); 920 921 switch (thisarg) 922 { 923 case SIMD_ARG_COPY_TO_REG: 924 if (POINTER_TYPE_P (TREE_TYPE (arg))) 925 op[opc] = convert_memory_address (Pmode, op[opc]); 926 /*gcc_assert (GET_MODE (op[opc]) == mode); */ 927 if (!(*insn_data[icode].operand[opc].predicate) 928 (op[opc], mode)) 929 op[opc] = copy_to_mode_reg (mode, op[opc]); 930 break; 931 932 case SIMD_ARG_LANE_INDEX: 933 /* Must be a previous operand into which this is an index. */ 934 gcc_assert (opc > 0); 935 if (CONST_INT_P (op[opc])) 936 { 937 machine_mode vmode = insn_data[icode].operand[opc - 1].mode; 938 aarch64_simd_lane_bounds (op[opc], 939 0, GET_MODE_NUNITS (vmode), exp); 940 /* Keep to GCC-vector-extension lane indices in the RTL. */ 941 op[opc] = GEN_INT (ENDIAN_LANE_N (vmode, INTVAL (op[opc]))); 942 } 943 /* Fall through - if the lane index isn't a constant then 944 the next case will error. */ 945 case SIMD_ARG_CONSTANT: 946 if (!(*insn_data[icode].operand[opc].predicate) 947 (op[opc], mode)) 948 { 949 error ("%Kargument %d must be a constant immediate", 950 exp, opc + 1 - have_retval); 951 return const0_rtx; 952 } 953 break; 954 955 case SIMD_ARG_STOP: 956 gcc_unreachable (); 957 } 958 959 opc++; 960 } 961 } 962 963 switch (opc) 964 { 965 case 1: 966 pat = GEN_FCN (icode) (op[0]); 967 break; 968 969 case 2: 970 pat = GEN_FCN (icode) (op[0], op[1]); 971 break; 972 973 case 3: 974 pat = GEN_FCN (icode) (op[0], op[1], op[2]); 975 break; 976 977 case 4: 978 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3]); 979 break; 980 981 case 5: 982 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4]); 983 break; 984 985 case 6: 986 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5]); 987 break; 988 989 default: 990 gcc_unreachable (); 991 } 992 993 if (!pat) 994 return NULL_RTX; 995 996 emit_insn (pat); 997 998 return target; 999} 1000 1001/* Expand an AArch64 AdvSIMD builtin(intrinsic). */ 1002rtx 1003aarch64_simd_expand_builtin (int fcode, tree exp, rtx target) 1004{ 1005 if (fcode == AARCH64_SIMD_BUILTIN_LANE_CHECK) 1006 { 1007 rtx totalsize = expand_normal (CALL_EXPR_ARG (exp, 0)); 1008 rtx elementsize = expand_normal (CALL_EXPR_ARG (exp, 1)); 1009 if (CONST_INT_P (totalsize) && CONST_INT_P (elementsize) 1010 && UINTVAL (elementsize) != 0 1011 && UINTVAL (totalsize) != 0) 1012 { 1013 rtx lane_idx = expand_normal (CALL_EXPR_ARG (exp, 2)); 1014 if (CONST_INT_P (lane_idx)) 1015 aarch64_simd_lane_bounds (lane_idx, 0, 1016 UINTVAL (totalsize) 1017 / UINTVAL (elementsize), 1018 exp); 1019 else 1020 error ("%Klane index must be a constant immediate", exp); 1021 } 1022 else 1023 error ("%Ktotal size and element size must be a non-zero constant immediate", exp); 1024 /* Don't generate any RTL. */ 1025 return const0_rtx; 1026 } 1027 aarch64_simd_builtin_datum *d = 1028 &aarch64_simd_builtin_data[fcode - AARCH64_SIMD_PATTERN_START]; 1029 enum insn_code icode = d->code; 1030 builtin_simd_arg args[SIMD_MAX_BUILTIN_ARGS + 1]; 1031 int num_args = insn_data[d->code].n_operands; 1032 int is_void = 0; 1033 int k; 1034 1035 is_void = !!(d->qualifiers[0] & qualifier_void); 1036 1037 num_args += is_void; 1038 1039 for (k = 1; k < num_args; k++) 1040 { 1041 /* We have four arrays of data, each indexed in a different fashion. 1042 qualifiers - element 0 always describes the function return type. 1043 operands - element 0 is either the operand for return value (if 1044 the function has a non-void return type) or the operand for the 1045 first argument. 1046 expr_args - element 0 always holds the first argument. 1047 args - element 0 is always used for the return type. */ 1048 int qualifiers_k = k; 1049 int operands_k = k - is_void; 1050 int expr_args_k = k - 1; 1051 1052 if (d->qualifiers[qualifiers_k] & qualifier_lane_index) 1053 args[k] = SIMD_ARG_LANE_INDEX; 1054 else if (d->qualifiers[qualifiers_k] & qualifier_immediate) 1055 args[k] = SIMD_ARG_CONSTANT; 1056 else if (d->qualifiers[qualifiers_k] & qualifier_maybe_immediate) 1057 { 1058 rtx arg 1059 = expand_normal (CALL_EXPR_ARG (exp, 1060 (expr_args_k))); 1061 /* Handle constants only if the predicate allows it. */ 1062 bool op_const_int_p = 1063 (CONST_INT_P (arg) 1064 && (*insn_data[icode].operand[operands_k].predicate) 1065 (arg, insn_data[icode].operand[operands_k].mode)); 1066 args[k] = op_const_int_p ? SIMD_ARG_CONSTANT : SIMD_ARG_COPY_TO_REG; 1067 } 1068 else 1069 args[k] = SIMD_ARG_COPY_TO_REG; 1070 1071 } 1072 args[k] = SIMD_ARG_STOP; 1073 1074 /* The interface to aarch64_simd_expand_args expects a 0 if 1075 the function is void, and a 1 if it is not. */ 1076 return aarch64_simd_expand_args 1077 (target, icode, !is_void, exp, &args[1]); 1078} 1079 1080rtx 1081aarch64_crc32_expand_builtin (int fcode, tree exp, rtx target) 1082{ 1083 rtx pat; 1084 aarch64_crc_builtin_datum *d 1085 = &aarch64_crc_builtin_data[fcode - (AARCH64_CRC32_BUILTIN_BASE + 1)]; 1086 enum insn_code icode = d->icode; 1087 tree arg0 = CALL_EXPR_ARG (exp, 0); 1088 tree arg1 = CALL_EXPR_ARG (exp, 1); 1089 rtx op0 = expand_normal (arg0); 1090 rtx op1 = expand_normal (arg1); 1091 machine_mode tmode = insn_data[icode].operand[0].mode; 1092 machine_mode mode0 = insn_data[icode].operand[1].mode; 1093 machine_mode mode1 = insn_data[icode].operand[2].mode; 1094 1095 if (! target 1096 || GET_MODE (target) != tmode 1097 || ! (*insn_data[icode].operand[0].predicate) (target, tmode)) 1098 target = gen_reg_rtx (tmode); 1099 1100 gcc_assert ((GET_MODE (op0) == mode0 || GET_MODE (op0) == VOIDmode) 1101 && (GET_MODE (op1) == mode1 || GET_MODE (op1) == VOIDmode)); 1102 1103 if (! (*insn_data[icode].operand[1].predicate) (op0, mode0)) 1104 op0 = copy_to_mode_reg (mode0, op0); 1105 if (! (*insn_data[icode].operand[2].predicate) (op1, mode1)) 1106 op1 = copy_to_mode_reg (mode1, op1); 1107 1108 pat = GEN_FCN (icode) (target, op0, op1); 1109 if (!pat) 1110 return NULL_RTX; 1111 1112 emit_insn (pat); 1113 return target; 1114} 1115 1116/* Expand an expression EXP that calls a built-in function, 1117 with result going to TARGET if that's convenient. */ 1118rtx 1119aarch64_expand_builtin (tree exp, 1120 rtx target, 1121 rtx subtarget ATTRIBUTE_UNUSED, 1122 machine_mode mode ATTRIBUTE_UNUSED, 1123 int ignore ATTRIBUTE_UNUSED) 1124{ 1125 tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0); 1126 int fcode = DECL_FUNCTION_CODE (fndecl); 1127 int icode; 1128 rtx pat, op0; 1129 tree arg0; 1130 1131 switch (fcode) 1132 { 1133 case AARCH64_BUILTIN_GET_FPCR: 1134 case AARCH64_BUILTIN_SET_FPCR: 1135 case AARCH64_BUILTIN_GET_FPSR: 1136 case AARCH64_BUILTIN_SET_FPSR: 1137 if ((fcode == AARCH64_BUILTIN_GET_FPCR) 1138 || (fcode == AARCH64_BUILTIN_GET_FPSR)) 1139 { 1140 icode = (fcode == AARCH64_BUILTIN_GET_FPSR) ? 1141 CODE_FOR_get_fpsr : CODE_FOR_get_fpcr; 1142 target = gen_reg_rtx (SImode); 1143 pat = GEN_FCN (icode) (target); 1144 } 1145 else 1146 { 1147 target = NULL_RTX; 1148 icode = (fcode == AARCH64_BUILTIN_SET_FPSR) ? 1149 CODE_FOR_set_fpsr : CODE_FOR_set_fpcr; 1150 arg0 = CALL_EXPR_ARG (exp, 0); 1151 op0 = expand_normal (arg0); 1152 pat = GEN_FCN (icode) (op0); 1153 } 1154 emit_insn (pat); 1155 return target; 1156 } 1157 1158 if (fcode >= AARCH64_SIMD_BUILTIN_BASE && fcode <= AARCH64_SIMD_BUILTIN_MAX) 1159 return aarch64_simd_expand_builtin (fcode, exp, target); 1160 else if (fcode >= AARCH64_CRC32_BUILTIN_BASE && fcode <= AARCH64_CRC32_BUILTIN_MAX) 1161 return aarch64_crc32_expand_builtin (fcode, exp, target); 1162 1163 gcc_unreachable (); 1164} 1165 1166tree 1167aarch64_builtin_vectorized_function (tree fndecl, tree type_out, tree type_in) 1168{ 1169 machine_mode in_mode, out_mode; 1170 int in_n, out_n; 1171 1172 if (TREE_CODE (type_out) != VECTOR_TYPE 1173 || TREE_CODE (type_in) != VECTOR_TYPE) 1174 return NULL_TREE; 1175 1176 out_mode = TYPE_MODE (TREE_TYPE (type_out)); 1177 out_n = TYPE_VECTOR_SUBPARTS (type_out); 1178 in_mode = TYPE_MODE (TREE_TYPE (type_in)); 1179 in_n = TYPE_VECTOR_SUBPARTS (type_in); 1180 1181#undef AARCH64_CHECK_BUILTIN_MODE 1182#define AARCH64_CHECK_BUILTIN_MODE(C, N) 1 1183#define AARCH64_FIND_FRINT_VARIANT(N) \ 1184 (AARCH64_CHECK_BUILTIN_MODE (2, D) \ 1185 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v2df] \ 1186 : (AARCH64_CHECK_BUILTIN_MODE (4, S) \ 1187 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v4sf] \ 1188 : (AARCH64_CHECK_BUILTIN_MODE (2, S) \ 1189 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v2sf] \ 1190 : NULL_TREE))) 1191 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) 1192 { 1193 enum built_in_function fn = DECL_FUNCTION_CODE (fndecl); 1194 switch (fn) 1195 { 1196#undef AARCH64_CHECK_BUILTIN_MODE 1197#define AARCH64_CHECK_BUILTIN_MODE(C, N) \ 1198 (out_mode == N##Fmode && out_n == C \ 1199 && in_mode == N##Fmode && in_n == C) 1200 case BUILT_IN_FLOOR: 1201 case BUILT_IN_FLOORF: 1202 return AARCH64_FIND_FRINT_VARIANT (floor); 1203 case BUILT_IN_CEIL: 1204 case BUILT_IN_CEILF: 1205 return AARCH64_FIND_FRINT_VARIANT (ceil); 1206 case BUILT_IN_TRUNC: 1207 case BUILT_IN_TRUNCF: 1208 return AARCH64_FIND_FRINT_VARIANT (btrunc); 1209 case BUILT_IN_ROUND: 1210 case BUILT_IN_ROUNDF: 1211 return AARCH64_FIND_FRINT_VARIANT (round); 1212 case BUILT_IN_NEARBYINT: 1213 case BUILT_IN_NEARBYINTF: 1214 return AARCH64_FIND_FRINT_VARIANT (nearbyint); 1215 case BUILT_IN_SQRT: 1216 case BUILT_IN_SQRTF: 1217 return AARCH64_FIND_FRINT_VARIANT (sqrt); 1218#undef AARCH64_CHECK_BUILTIN_MODE 1219#define AARCH64_CHECK_BUILTIN_MODE(C, N) \ 1220 (out_mode == SImode && out_n == C \ 1221 && in_mode == N##Imode && in_n == C) 1222 case BUILT_IN_CLZ: 1223 { 1224 if (AARCH64_CHECK_BUILTIN_MODE (4, S)) 1225 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_clzv4si]; 1226 return NULL_TREE; 1227 } 1228 case BUILT_IN_CTZ: 1229 { 1230 if (AARCH64_CHECK_BUILTIN_MODE (2, S)) 1231 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_ctzv2si]; 1232 else if (AARCH64_CHECK_BUILTIN_MODE (4, S)) 1233 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_ctzv4si]; 1234 return NULL_TREE; 1235 } 1236#undef AARCH64_CHECK_BUILTIN_MODE 1237#define AARCH64_CHECK_BUILTIN_MODE(C, N) \ 1238 (out_mode == N##Imode && out_n == C \ 1239 && in_mode == N##Fmode && in_n == C) 1240 case BUILT_IN_LFLOOR: 1241 case BUILT_IN_LFLOORF: 1242 case BUILT_IN_LLFLOOR: 1243 case BUILT_IN_IFLOORF: 1244 { 1245 enum aarch64_builtins builtin; 1246 if (AARCH64_CHECK_BUILTIN_MODE (2, D)) 1247 builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv2dfv2di; 1248 else if (AARCH64_CHECK_BUILTIN_MODE (4, S)) 1249 builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv4sfv4si; 1250 else if (AARCH64_CHECK_BUILTIN_MODE (2, S)) 1251 builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv2sfv2si; 1252 else 1253 return NULL_TREE; 1254 1255 return aarch64_builtin_decls[builtin]; 1256 } 1257 case BUILT_IN_LCEIL: 1258 case BUILT_IN_LCEILF: 1259 case BUILT_IN_LLCEIL: 1260 case BUILT_IN_ICEILF: 1261 { 1262 enum aarch64_builtins builtin; 1263 if (AARCH64_CHECK_BUILTIN_MODE (2, D)) 1264 builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv2dfv2di; 1265 else if (AARCH64_CHECK_BUILTIN_MODE (4, S)) 1266 builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv4sfv4si; 1267 else if (AARCH64_CHECK_BUILTIN_MODE (2, S)) 1268 builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv2sfv2si; 1269 else 1270 return NULL_TREE; 1271 1272 return aarch64_builtin_decls[builtin]; 1273 } 1274 case BUILT_IN_LROUND: 1275 case BUILT_IN_IROUNDF: 1276 { 1277 enum aarch64_builtins builtin; 1278 if (AARCH64_CHECK_BUILTIN_MODE (2, D)) 1279 builtin = AARCH64_SIMD_BUILTIN_UNOP_lroundv2dfv2di; 1280 else if (AARCH64_CHECK_BUILTIN_MODE (4, S)) 1281 builtin = AARCH64_SIMD_BUILTIN_UNOP_lroundv4sfv4si; 1282 else if (AARCH64_CHECK_BUILTIN_MODE (2, S)) 1283 builtin = AARCH64_SIMD_BUILTIN_UNOP_lroundv2sfv2si; 1284 else 1285 return NULL_TREE; 1286 1287 return aarch64_builtin_decls[builtin]; 1288 } 1289 case BUILT_IN_BSWAP16: 1290#undef AARCH64_CHECK_BUILTIN_MODE 1291#define AARCH64_CHECK_BUILTIN_MODE(C, N) \ 1292 (out_mode == N##Imode && out_n == C \ 1293 && in_mode == N##Imode && in_n == C) 1294 if (AARCH64_CHECK_BUILTIN_MODE (4, H)) 1295 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOPU_bswapv4hi]; 1296 else if (AARCH64_CHECK_BUILTIN_MODE (8, H)) 1297 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOPU_bswapv8hi]; 1298 else 1299 return NULL_TREE; 1300 case BUILT_IN_BSWAP32: 1301 if (AARCH64_CHECK_BUILTIN_MODE (2, S)) 1302 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOPU_bswapv2si]; 1303 else if (AARCH64_CHECK_BUILTIN_MODE (4, S)) 1304 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOPU_bswapv4si]; 1305 else 1306 return NULL_TREE; 1307 case BUILT_IN_BSWAP64: 1308 if (AARCH64_CHECK_BUILTIN_MODE (2, D)) 1309 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOPU_bswapv2di]; 1310 else 1311 return NULL_TREE; 1312 default: 1313 return NULL_TREE; 1314 } 1315 } 1316 1317 return NULL_TREE; 1318} 1319 1320#undef VAR1 1321#define VAR1(T, N, MAP, A) \ 1322 case AARCH64_SIMD_BUILTIN_##T##_##N##A: 1323 1324tree 1325aarch64_fold_builtin (tree fndecl, int n_args ATTRIBUTE_UNUSED, tree *args, 1326 bool ignore ATTRIBUTE_UNUSED) 1327{ 1328 int fcode = DECL_FUNCTION_CODE (fndecl); 1329 tree type = TREE_TYPE (TREE_TYPE (fndecl)); 1330 1331 switch (fcode) 1332 { 1333 BUILTIN_VDQF (UNOP, abs, 2) 1334 return fold_build1 (ABS_EXPR, type, args[0]); 1335 break; 1336 VAR1 (UNOP, floatv2si, 2, v2sf) 1337 VAR1 (UNOP, floatv4si, 2, v4sf) 1338 VAR1 (UNOP, floatv2di, 2, v2df) 1339 return fold_build1 (FLOAT_EXPR, type, args[0]); 1340 default: 1341 break; 1342 } 1343 1344 return NULL_TREE; 1345} 1346 1347bool 1348aarch64_gimple_fold_builtin (gimple_stmt_iterator *gsi) 1349{ 1350 bool changed = false; 1351 gimple stmt = gsi_stmt (*gsi); 1352 tree call = gimple_call_fn (stmt); 1353 tree fndecl; 1354 gimple new_stmt = NULL; 1355 1356 if (call) 1357 { 1358 fndecl = gimple_call_fndecl (stmt); 1359 if (fndecl) 1360 { 1361 int fcode = DECL_FUNCTION_CODE (fndecl); 1362 int nargs = gimple_call_num_args (stmt); 1363 tree *args = (nargs > 0 1364 ? gimple_call_arg_ptr (stmt, 0) 1365 : &error_mark_node); 1366 1367 /* We use gimple's REDUC_(PLUS|MIN|MAX)_EXPRs for float, signed int 1368 and unsigned int; it will distinguish according to the types of 1369 the arguments to the __builtin. */ 1370 switch (fcode) 1371 { 1372 BUILTIN_VALL (UNOP, reduc_plus_scal_, 10) 1373 new_stmt = gimple_build_assign (gimple_call_lhs (stmt), 1374 REDUC_PLUS_EXPR, args[0]); 1375 break; 1376 BUILTIN_VDQIF (UNOP, reduc_smax_scal_, 10) 1377 BUILTIN_VDQ_BHSI (UNOPU, reduc_umax_scal_, 10) 1378 new_stmt = gimple_build_assign (gimple_call_lhs (stmt), 1379 REDUC_MAX_EXPR, args[0]); 1380 break; 1381 BUILTIN_VDQIF (UNOP, reduc_smin_scal_, 10) 1382 BUILTIN_VDQ_BHSI (UNOPU, reduc_umin_scal_, 10) 1383 new_stmt = gimple_build_assign (gimple_call_lhs (stmt), 1384 REDUC_MIN_EXPR, args[0]); 1385 break; 1386 1387 default: 1388 break; 1389 } 1390 } 1391 } 1392 1393 if (new_stmt) 1394 { 1395 gsi_replace (gsi, new_stmt, true); 1396 changed = true; 1397 } 1398 1399 return changed; 1400} 1401 1402void 1403aarch64_atomic_assign_expand_fenv (tree *hold, tree *clear, tree *update) 1404{ 1405 const unsigned AARCH64_FE_INVALID = 1; 1406 const unsigned AARCH64_FE_DIVBYZERO = 2; 1407 const unsigned AARCH64_FE_OVERFLOW = 4; 1408 const unsigned AARCH64_FE_UNDERFLOW = 8; 1409 const unsigned AARCH64_FE_INEXACT = 16; 1410 const unsigned HOST_WIDE_INT AARCH64_FE_ALL_EXCEPT = (AARCH64_FE_INVALID 1411 | AARCH64_FE_DIVBYZERO 1412 | AARCH64_FE_OVERFLOW 1413 | AARCH64_FE_UNDERFLOW 1414 | AARCH64_FE_INEXACT); 1415 const unsigned HOST_WIDE_INT AARCH64_FE_EXCEPT_SHIFT = 8; 1416 tree fenv_cr, fenv_sr, get_fpcr, set_fpcr, mask_cr, mask_sr; 1417 tree ld_fenv_cr, ld_fenv_sr, masked_fenv_cr, masked_fenv_sr, hold_fnclex_cr; 1418 tree hold_fnclex_sr, new_fenv_var, reload_fenv, restore_fnenv, get_fpsr, set_fpsr; 1419 tree update_call, atomic_feraiseexcept, hold_fnclex, masked_fenv, ld_fenv; 1420 1421 /* Generate the equivalence of : 1422 unsigned int fenv_cr; 1423 fenv_cr = __builtin_aarch64_get_fpcr (); 1424 1425 unsigned int fenv_sr; 1426 fenv_sr = __builtin_aarch64_get_fpsr (); 1427 1428 Now set all exceptions to non-stop 1429 unsigned int mask_cr 1430 = ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT); 1431 unsigned int masked_cr; 1432 masked_cr = fenv_cr & mask_cr; 1433 1434 And clear all exception flags 1435 unsigned int maske_sr = ~AARCH64_FE_ALL_EXCEPT; 1436 unsigned int masked_cr; 1437 masked_sr = fenv_sr & mask_sr; 1438 1439 __builtin_aarch64_set_cr (masked_cr); 1440 __builtin_aarch64_set_sr (masked_sr); */ 1441 1442 fenv_cr = create_tmp_var (unsigned_type_node); 1443 fenv_sr = create_tmp_var (unsigned_type_node); 1444 1445 get_fpcr = aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR]; 1446 set_fpcr = aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR]; 1447 get_fpsr = aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR]; 1448 set_fpsr = aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR]; 1449 1450 mask_cr = build_int_cst (unsigned_type_node, 1451 ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT)); 1452 mask_sr = build_int_cst (unsigned_type_node, 1453 ~(AARCH64_FE_ALL_EXCEPT)); 1454 1455 ld_fenv_cr = build2 (MODIFY_EXPR, unsigned_type_node, 1456 fenv_cr, build_call_expr (get_fpcr, 0)); 1457 ld_fenv_sr = build2 (MODIFY_EXPR, unsigned_type_node, 1458 fenv_sr, build_call_expr (get_fpsr, 0)); 1459 1460 masked_fenv_cr = build2 (BIT_AND_EXPR, unsigned_type_node, fenv_cr, mask_cr); 1461 masked_fenv_sr = build2 (BIT_AND_EXPR, unsigned_type_node, fenv_sr, mask_sr); 1462 1463 hold_fnclex_cr = build_call_expr (set_fpcr, 1, masked_fenv_cr); 1464 hold_fnclex_sr = build_call_expr (set_fpsr, 1, masked_fenv_sr); 1465 1466 hold_fnclex = build2 (COMPOUND_EXPR, void_type_node, hold_fnclex_cr, 1467 hold_fnclex_sr); 1468 masked_fenv = build2 (COMPOUND_EXPR, void_type_node, masked_fenv_cr, 1469 masked_fenv_sr); 1470 ld_fenv = build2 (COMPOUND_EXPR, void_type_node, ld_fenv_cr, ld_fenv_sr); 1471 1472 *hold = build2 (COMPOUND_EXPR, void_type_node, 1473 build2 (COMPOUND_EXPR, void_type_node, masked_fenv, ld_fenv), 1474 hold_fnclex); 1475 1476 /* Store the value of masked_fenv to clear the exceptions: 1477 __builtin_aarch64_set_fpsr (masked_fenv_sr); */ 1478 1479 *clear = build_call_expr (set_fpsr, 1, masked_fenv_sr); 1480 1481 /* Generate the equivalent of : 1482 unsigned int new_fenv_var; 1483 new_fenv_var = __builtin_aarch64_get_fpsr (); 1484 1485 __builtin_aarch64_set_fpsr (fenv_sr); 1486 1487 __atomic_feraiseexcept (new_fenv_var); */ 1488 1489 new_fenv_var = create_tmp_var (unsigned_type_node); 1490 reload_fenv = build2 (MODIFY_EXPR, unsigned_type_node, 1491 new_fenv_var, build_call_expr (get_fpsr, 0)); 1492 restore_fnenv = build_call_expr (set_fpsr, 1, fenv_sr); 1493 atomic_feraiseexcept = builtin_decl_implicit (BUILT_IN_ATOMIC_FERAISEEXCEPT); 1494 update_call = build_call_expr (atomic_feraiseexcept, 1, 1495 fold_convert (integer_type_node, new_fenv_var)); 1496 *update = build2 (COMPOUND_EXPR, void_type_node, 1497 build2 (COMPOUND_EXPR, void_type_node, 1498 reload_fenv, restore_fnenv), update_call); 1499} 1500 1501 1502#undef AARCH64_CHECK_BUILTIN_MODE 1503#undef AARCH64_FIND_FRINT_VARIANT 1504#undef CF0 1505#undef CF1 1506#undef CF2 1507#undef CF3 1508#undef CF4 1509#undef CF10 1510#undef VAR1 1511#undef VAR2 1512#undef VAR3 1513#undef VAR4 1514#undef VAR5 1515#undef VAR6 1516#undef VAR7 1517#undef VAR8 1518#undef VAR9 1519#undef VAR10 1520#undef VAR11 1521 1522#include "gt-aarch64-builtins.h" 1523