1/* Passes for transactional memory support. 2 Copyright (C) 2008-2015 Free Software Foundation, Inc. 3 4 This file is part of GCC. 5 6 GCC is free software; you can redistribute it and/or modify it under 7 the terms of the GNU General Public License as published by the Free 8 Software Foundation; either version 3, or (at your option) any later 9 version. 10 11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY 12 WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 for more details. 15 16 You should have received a copy of the GNU General Public License 17 along with GCC; see the file COPYING3. If not see 18 <http://www.gnu.org/licenses/>. */ 19 20#include "config.h" 21#include "system.h" 22#include "coretypes.h" 23#include "hash-table.h" 24#include "hash-set.h" 25#include "machmode.h" 26#include "vec.h" 27#include "double-int.h" 28#include "input.h" 29#include "alias.h" 30#include "symtab.h" 31#include "options.h" 32#include "wide-int.h" 33#include "inchash.h" 34#include "tree.h" 35#include "fold-const.h" 36#include "predict.h" 37#include "tm.h" 38#include "hard-reg-set.h" 39#include "input.h" 40#include "function.h" 41#include "dominance.h" 42#include "cfg.h" 43#include "basic-block.h" 44#include "tree-ssa-alias.h" 45#include "internal-fn.h" 46#include "tree-eh.h" 47#include "gimple-expr.h" 48#include "is-a.h" 49#include "gimple.h" 50#include "calls.h" 51#include "rtl.h" 52#include "emit-rtl.h" 53#include "gimplify.h" 54#include "gimple-iterator.h" 55#include "gimplify-me.h" 56#include "gimple-walk.h" 57#include "gimple-ssa.h" 58#include "hash-map.h" 59#include "plugin-api.h" 60#include "ipa-ref.h" 61#include "cgraph.h" 62#include "tree-cfg.h" 63#include "stringpool.h" 64#include "tree-ssanames.h" 65#include "tree-into-ssa.h" 66#include "tree-pass.h" 67#include "tree-inline.h" 68#include "diagnostic-core.h" 69#include "demangle.h" 70#include "output.h" 71#include "trans-mem.h" 72#include "params.h" 73#include "target.h" 74#include "langhooks.h" 75#include "gimple-pretty-print.h" 76#include "cfgloop.h" 77#include "tree-ssa-address.h" 78 79 80#define A_RUNINSTRUMENTEDCODE 0x0001 81#define A_RUNUNINSTRUMENTEDCODE 0x0002 82#define A_SAVELIVEVARIABLES 0x0004 83#define A_RESTORELIVEVARIABLES 0x0008 84#define A_ABORTTRANSACTION 0x0010 85 86#define AR_USERABORT 0x0001 87#define AR_USERRETRY 0x0002 88#define AR_TMCONFLICT 0x0004 89#define AR_EXCEPTIONBLOCKABORT 0x0008 90#define AR_OUTERABORT 0x0010 91 92#define MODE_SERIALIRREVOCABLE 0x0000 93 94 95/* The representation of a transaction changes several times during the 96 lowering process. In the beginning, in the front-end we have the 97 GENERIC tree TRANSACTION_EXPR. For example, 98 99 __transaction { 100 local++; 101 if (++global == 10) 102 __tm_abort; 103 } 104 105 During initial gimplification (gimplify.c) the TRANSACTION_EXPR node is 106 trivially replaced with a GIMPLE_TRANSACTION node. 107 108 During pass_lower_tm, we examine the body of transactions looking 109 for aborts. Transactions that do not contain an abort may be 110 merged into an outer transaction. We also add a TRY-FINALLY node 111 to arrange for the transaction to be committed on any exit. 112 113 [??? Think about how this arrangement affects throw-with-commit 114 and throw-with-abort operations. In this case we want the TRY to 115 handle gotos, but not to catch any exceptions because the transaction 116 will already be closed.] 117 118 GIMPLE_TRANSACTION [label=NULL] { 119 try { 120 local = local + 1; 121 t0 = global; 122 t1 = t0 + 1; 123 global = t1; 124 if (t1 == 10) 125 __builtin___tm_abort (); 126 } finally { 127 __builtin___tm_commit (); 128 } 129 } 130 131 During pass_lower_eh, we create EH regions for the transactions, 132 intermixed with the regular EH stuff. This gives us a nice persistent 133 mapping (all the way through rtl) from transactional memory operation 134 back to the transaction, which allows us to get the abnormal edges 135 correct to model transaction aborts and restarts: 136 137 GIMPLE_TRANSACTION [label=over] 138 local = local + 1; 139 t0 = global; 140 t1 = t0 + 1; 141 global = t1; 142 if (t1 == 10) 143 __builtin___tm_abort (); 144 __builtin___tm_commit (); 145 over: 146 147 This is the end of all_lowering_passes, and so is what is present 148 during the IPA passes, and through all of the optimization passes. 149 150 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all 151 functions and mark functions for cloning. 152 153 At the end of gimple optimization, before exiting SSA form, 154 pass_tm_edges replaces statements that perform transactional 155 memory operations with the appropriate TM builtins, and swap 156 out function calls with their transactional clones. At this 157 point we introduce the abnormal transaction restart edges and 158 complete lowering of the GIMPLE_TRANSACTION node. 159 160 x = __builtin___tm_start (MAY_ABORT); 161 eh_label: 162 if (x & abort_transaction) 163 goto over; 164 local = local + 1; 165 t0 = __builtin___tm_load (global); 166 t1 = t0 + 1; 167 __builtin___tm_store (&global, t1); 168 if (t1 == 10) 169 __builtin___tm_abort (); 170 __builtin___tm_commit (); 171 over: 172*/ 173 174static void *expand_regions (struct tm_region *, 175 void *(*callback)(struct tm_region *, void *), 176 void *, bool); 177 178 179/* Return the attributes we want to examine for X, or NULL if it's not 180 something we examine. We look at function types, but allow pointers 181 to function types and function decls and peek through. */ 182 183static tree 184get_attrs_for (const_tree x) 185{ 186 if (x == NULL_TREE) 187 return NULL_TREE; 188 189 switch (TREE_CODE (x)) 190 { 191 case FUNCTION_DECL: 192 return TYPE_ATTRIBUTES (TREE_TYPE (x)); 193 break; 194 195 default: 196 if (TYPE_P (x)) 197 return NULL_TREE; 198 x = TREE_TYPE (x); 199 if (TREE_CODE (x) != POINTER_TYPE) 200 return NULL_TREE; 201 /* FALLTHRU */ 202 203 case POINTER_TYPE: 204 x = TREE_TYPE (x); 205 if (TREE_CODE (x) != FUNCTION_TYPE && TREE_CODE (x) != METHOD_TYPE) 206 return NULL_TREE; 207 /* FALLTHRU */ 208 209 case FUNCTION_TYPE: 210 case METHOD_TYPE: 211 return TYPE_ATTRIBUTES (x); 212 } 213} 214 215/* Return true if X has been marked TM_PURE. */ 216 217bool 218is_tm_pure (const_tree x) 219{ 220 unsigned flags; 221 222 switch (TREE_CODE (x)) 223 { 224 case FUNCTION_DECL: 225 case FUNCTION_TYPE: 226 case METHOD_TYPE: 227 break; 228 229 default: 230 if (TYPE_P (x)) 231 return false; 232 x = TREE_TYPE (x); 233 if (TREE_CODE (x) != POINTER_TYPE) 234 return false; 235 /* FALLTHRU */ 236 237 case POINTER_TYPE: 238 x = TREE_TYPE (x); 239 if (TREE_CODE (x) != FUNCTION_TYPE && TREE_CODE (x) != METHOD_TYPE) 240 return false; 241 break; 242 } 243 244 flags = flags_from_decl_or_type (x); 245 return (flags & ECF_TM_PURE) != 0; 246} 247 248/* Return true if X has been marked TM_IRREVOCABLE. */ 249 250static bool 251is_tm_irrevocable (tree x) 252{ 253 tree attrs = get_attrs_for (x); 254 255 if (attrs && lookup_attribute ("transaction_unsafe", attrs)) 256 return true; 257 258 /* A call to the irrevocable builtin is by definition, 259 irrevocable. */ 260 if (TREE_CODE (x) == ADDR_EXPR) 261 x = TREE_OPERAND (x, 0); 262 if (TREE_CODE (x) == FUNCTION_DECL 263 && DECL_BUILT_IN_CLASS (x) == BUILT_IN_NORMAL 264 && DECL_FUNCTION_CODE (x) == BUILT_IN_TM_IRREVOCABLE) 265 return true; 266 267 return false; 268} 269 270/* Return true if X has been marked TM_SAFE. */ 271 272bool 273is_tm_safe (const_tree x) 274{ 275 if (flag_tm) 276 { 277 tree attrs = get_attrs_for (x); 278 if (attrs) 279 { 280 if (lookup_attribute ("transaction_safe", attrs)) 281 return true; 282 if (lookup_attribute ("transaction_may_cancel_outer", attrs)) 283 return true; 284 } 285 } 286 return false; 287} 288 289/* Return true if CALL is const, or tm_pure. */ 290 291static bool 292is_tm_pure_call (gimple call) 293{ 294 tree fn = gimple_call_fn (call); 295 296 if (TREE_CODE (fn) == ADDR_EXPR) 297 { 298 fn = TREE_OPERAND (fn, 0); 299 gcc_assert (TREE_CODE (fn) == FUNCTION_DECL); 300 } 301 else 302 fn = TREE_TYPE (fn); 303 304 return is_tm_pure (fn); 305} 306 307/* Return true if X has been marked TM_CALLABLE. */ 308 309static bool 310is_tm_callable (tree x) 311{ 312 tree attrs = get_attrs_for (x); 313 if (attrs) 314 { 315 if (lookup_attribute ("transaction_callable", attrs)) 316 return true; 317 if (lookup_attribute ("transaction_safe", attrs)) 318 return true; 319 if (lookup_attribute ("transaction_may_cancel_outer", attrs)) 320 return true; 321 } 322 return false; 323} 324 325/* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */ 326 327bool 328is_tm_may_cancel_outer (tree x) 329{ 330 tree attrs = get_attrs_for (x); 331 if (attrs) 332 return lookup_attribute ("transaction_may_cancel_outer", attrs) != NULL; 333 return false; 334} 335 336/* Return true for built in functions that "end" a transaction. */ 337 338bool 339is_tm_ending_fndecl (tree fndecl) 340{ 341 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) 342 switch (DECL_FUNCTION_CODE (fndecl)) 343 { 344 case BUILT_IN_TM_COMMIT: 345 case BUILT_IN_TM_COMMIT_EH: 346 case BUILT_IN_TM_ABORT: 347 case BUILT_IN_TM_IRREVOCABLE: 348 return true; 349 default: 350 break; 351 } 352 353 return false; 354} 355 356/* Return true if STMT is a built in function call that "ends" a 357 transaction. */ 358 359bool 360is_tm_ending (gimple stmt) 361{ 362 tree fndecl; 363 364 if (gimple_code (stmt) != GIMPLE_CALL) 365 return false; 366 367 fndecl = gimple_call_fndecl (stmt); 368 return (fndecl != NULL_TREE 369 && is_tm_ending_fndecl (fndecl)); 370} 371 372/* Return true if STMT is a TM load. */ 373 374static bool 375is_tm_load (gimple stmt) 376{ 377 tree fndecl; 378 379 if (gimple_code (stmt) != GIMPLE_CALL) 380 return false; 381 382 fndecl = gimple_call_fndecl (stmt); 383 return (fndecl && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL 384 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl))); 385} 386 387/* Same as above, but for simple TM loads, that is, not the 388 after-write, after-read, etc optimized variants. */ 389 390static bool 391is_tm_simple_load (gimple stmt) 392{ 393 tree fndecl; 394 395 if (gimple_code (stmt) != GIMPLE_CALL) 396 return false; 397 398 fndecl = gimple_call_fndecl (stmt); 399 if (fndecl && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) 400 { 401 enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl); 402 return (fcode == BUILT_IN_TM_LOAD_1 403 || fcode == BUILT_IN_TM_LOAD_2 404 || fcode == BUILT_IN_TM_LOAD_4 405 || fcode == BUILT_IN_TM_LOAD_8 406 || fcode == BUILT_IN_TM_LOAD_FLOAT 407 || fcode == BUILT_IN_TM_LOAD_DOUBLE 408 || fcode == BUILT_IN_TM_LOAD_LDOUBLE 409 || fcode == BUILT_IN_TM_LOAD_M64 410 || fcode == BUILT_IN_TM_LOAD_M128 411 || fcode == BUILT_IN_TM_LOAD_M256); 412 } 413 return false; 414} 415 416/* Return true if STMT is a TM store. */ 417 418static bool 419is_tm_store (gimple stmt) 420{ 421 tree fndecl; 422 423 if (gimple_code (stmt) != GIMPLE_CALL) 424 return false; 425 426 fndecl = gimple_call_fndecl (stmt); 427 return (fndecl && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL 428 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl))); 429} 430 431/* Same as above, but for simple TM stores, that is, not the 432 after-write, after-read, etc optimized variants. */ 433 434static bool 435is_tm_simple_store (gimple stmt) 436{ 437 tree fndecl; 438 439 if (gimple_code (stmt) != GIMPLE_CALL) 440 return false; 441 442 fndecl = gimple_call_fndecl (stmt); 443 if (fndecl && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) 444 { 445 enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl); 446 return (fcode == BUILT_IN_TM_STORE_1 447 || fcode == BUILT_IN_TM_STORE_2 448 || fcode == BUILT_IN_TM_STORE_4 449 || fcode == BUILT_IN_TM_STORE_8 450 || fcode == BUILT_IN_TM_STORE_FLOAT 451 || fcode == BUILT_IN_TM_STORE_DOUBLE 452 || fcode == BUILT_IN_TM_STORE_LDOUBLE 453 || fcode == BUILT_IN_TM_STORE_M64 454 || fcode == BUILT_IN_TM_STORE_M128 455 || fcode == BUILT_IN_TM_STORE_M256); 456 } 457 return false; 458} 459 460/* Return true if FNDECL is BUILT_IN_TM_ABORT. */ 461 462static bool 463is_tm_abort (tree fndecl) 464{ 465 return (fndecl 466 && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL 467 && DECL_FUNCTION_CODE (fndecl) == BUILT_IN_TM_ABORT); 468} 469 470/* Build a GENERIC tree for a user abort. This is called by front ends 471 while transforming the __tm_abort statement. */ 472 473tree 474build_tm_abort_call (location_t loc, bool is_outer) 475{ 476 return build_call_expr_loc (loc, builtin_decl_explicit (BUILT_IN_TM_ABORT), 1, 477 build_int_cst (integer_type_node, 478 AR_USERABORT 479 | (is_outer ? AR_OUTERABORT : 0))); 480} 481 482/* Map for aribtrary function replacement under TM, as created 483 by the tm_wrap attribute. */ 484 485struct tm_wrapper_hasher : ggc_cache_hasher<tree_map *> 486{ 487 static inline hashval_t hash (tree_map *m) { return m->hash; } 488 static inline bool 489 equal (tree_map *a, tree_map *b) 490 { 491 return a->base.from == b->base.from; 492 } 493 494 static void 495 handle_cache_entry (tree_map *&m) 496 { 497 extern void gt_ggc_mx (tree_map *&); 498 if (m == HTAB_EMPTY_ENTRY || m == HTAB_DELETED_ENTRY) 499 return; 500 else if (ggc_marked_p (m->base.from)) 501 gt_ggc_mx (m); 502 else 503 m = static_cast<tree_map *> (HTAB_DELETED_ENTRY); 504 } 505}; 506 507static GTY((cache)) hash_table<tm_wrapper_hasher> *tm_wrap_map; 508 509void 510record_tm_replacement (tree from, tree to) 511{ 512 struct tree_map **slot, *h; 513 514 /* Do not inline wrapper functions that will get replaced in the TM 515 pass. 516 517 Suppose you have foo() that will get replaced into tmfoo(). Make 518 sure the inliner doesn't try to outsmart us and inline foo() 519 before we get a chance to do the TM replacement. */ 520 DECL_UNINLINABLE (from) = 1; 521 522 if (tm_wrap_map == NULL) 523 tm_wrap_map = hash_table<tm_wrapper_hasher>::create_ggc (32); 524 525 h = ggc_alloc<tree_map> (); 526 h->hash = htab_hash_pointer (from); 527 h->base.from = from; 528 h->to = to; 529 530 slot = tm_wrap_map->find_slot_with_hash (h, h->hash, INSERT); 531 *slot = h; 532} 533 534/* Return a TM-aware replacement function for DECL. */ 535 536static tree 537find_tm_replacement_function (tree fndecl) 538{ 539 if (tm_wrap_map) 540 { 541 struct tree_map *h, in; 542 543 in.base.from = fndecl; 544 in.hash = htab_hash_pointer (fndecl); 545 h = tm_wrap_map->find_with_hash (&in, in.hash); 546 if (h) 547 return h->to; 548 } 549 550 /* ??? We may well want TM versions of most of the common <string.h> 551 functions. For now, we've already these two defined. */ 552 /* Adjust expand_call_tm() attributes as necessary for the cases 553 handled here: */ 554 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) 555 switch (DECL_FUNCTION_CODE (fndecl)) 556 { 557 case BUILT_IN_MEMCPY: 558 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY); 559 case BUILT_IN_MEMMOVE: 560 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE); 561 case BUILT_IN_MEMSET: 562 return builtin_decl_explicit (BUILT_IN_TM_MEMSET); 563 default: 564 return NULL; 565 } 566 567 return NULL; 568} 569 570/* When appropriate, record TM replacement for memory allocation functions. 571 572 FROM is the FNDECL to wrap. */ 573void 574tm_malloc_replacement (tree from) 575{ 576 const char *str; 577 tree to; 578 579 if (TREE_CODE (from) != FUNCTION_DECL) 580 return; 581 582 /* If we have a previous replacement, the user must be explicitly 583 wrapping malloc/calloc/free. They better know what they're 584 doing... */ 585 if (find_tm_replacement_function (from)) 586 return; 587 588 str = IDENTIFIER_POINTER (DECL_NAME (from)); 589 590 if (!strcmp (str, "malloc")) 591 to = builtin_decl_explicit (BUILT_IN_TM_MALLOC); 592 else if (!strcmp (str, "calloc")) 593 to = builtin_decl_explicit (BUILT_IN_TM_CALLOC); 594 else if (!strcmp (str, "free")) 595 to = builtin_decl_explicit (BUILT_IN_TM_FREE); 596 else 597 return; 598 599 TREE_NOTHROW (to) = 0; 600 601 record_tm_replacement (from, to); 602} 603 604/* Diagnostics for tm_safe functions/regions. Called by the front end 605 once we've lowered the function to high-gimple. */ 606 607/* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq. 608 Process exactly one statement. WI->INFO is set to non-null when in 609 the context of a tm_safe function, and null for a __transaction block. */ 610 611#define DIAG_TM_OUTER 1 612#define DIAG_TM_SAFE 2 613#define DIAG_TM_RELAXED 4 614 615struct diagnose_tm 616{ 617 unsigned int summary_flags : 8; 618 unsigned int block_flags : 8; 619 unsigned int func_flags : 8; 620 unsigned int saw_volatile : 1; 621 gimple stmt; 622}; 623 624/* Return true if T is a volatile variable of some kind. */ 625 626static bool 627volatile_var_p (tree t) 628{ 629 return (SSA_VAR_P (t) 630 && TREE_THIS_VOLATILE (TREE_TYPE (t))); 631} 632 633/* Tree callback function for diagnose_tm pass. */ 634 635static tree 636diagnose_tm_1_op (tree *tp, int *walk_subtrees ATTRIBUTE_UNUSED, 637 void *data) 638{ 639 struct walk_stmt_info *wi = (struct walk_stmt_info *) data; 640 struct diagnose_tm *d = (struct diagnose_tm *) wi->info; 641 642 if (volatile_var_p (*tp) 643 && d->block_flags & DIAG_TM_SAFE 644 && !d->saw_volatile) 645 { 646 d->saw_volatile = 1; 647 error_at (gimple_location (d->stmt), 648 "invalid volatile use of %qD inside transaction", 649 *tp); 650 } 651 652 return NULL_TREE; 653} 654 655static inline bool 656is_tm_safe_or_pure (const_tree x) 657{ 658 return is_tm_safe (x) || is_tm_pure (x); 659} 660 661static tree 662diagnose_tm_1 (gimple_stmt_iterator *gsi, bool *handled_ops_p, 663 struct walk_stmt_info *wi) 664{ 665 gimple stmt = gsi_stmt (*gsi); 666 struct diagnose_tm *d = (struct diagnose_tm *) wi->info; 667 668 /* Save stmt for use in leaf analysis. */ 669 d->stmt = stmt; 670 671 switch (gimple_code (stmt)) 672 { 673 case GIMPLE_CALL: 674 { 675 tree fn = gimple_call_fn (stmt); 676 677 if ((d->summary_flags & DIAG_TM_OUTER) == 0 678 && is_tm_may_cancel_outer (fn)) 679 error_at (gimple_location (stmt), 680 "%<transaction_may_cancel_outer%> function call not within" 681 " outer transaction or %<transaction_may_cancel_outer%>"); 682 683 if (d->summary_flags & DIAG_TM_SAFE) 684 { 685 bool is_safe, direct_call_p; 686 tree replacement; 687 688 if (TREE_CODE (fn) == ADDR_EXPR 689 && TREE_CODE (TREE_OPERAND (fn, 0)) == FUNCTION_DECL) 690 { 691 direct_call_p = true; 692 replacement = TREE_OPERAND (fn, 0); 693 replacement = find_tm_replacement_function (replacement); 694 if (replacement) 695 fn = replacement; 696 } 697 else 698 { 699 direct_call_p = false; 700 replacement = NULL_TREE; 701 } 702 703 if (is_tm_safe_or_pure (fn)) 704 is_safe = true; 705 else if (is_tm_callable (fn) || is_tm_irrevocable (fn)) 706 { 707 /* A function explicitly marked transaction_callable as 708 opposed to transaction_safe is being defined to be 709 unsafe as part of its ABI, regardless of its contents. */ 710 is_safe = false; 711 } 712 else if (direct_call_p) 713 { 714 if (IS_TYPE_OR_DECL_P (fn) 715 && flags_from_decl_or_type (fn) & ECF_TM_BUILTIN) 716 is_safe = true; 717 else if (replacement) 718 { 719 /* ??? At present we've been considering replacements 720 merely transaction_callable, and therefore might 721 enter irrevocable. The tm_wrap attribute has not 722 yet made it into the new language spec. */ 723 is_safe = false; 724 } 725 else 726 { 727 /* ??? Diagnostics for unmarked direct calls moved into 728 the IPA pass. Section 3.2 of the spec details how 729 functions not marked should be considered "implicitly 730 safe" based on having examined the function body. */ 731 is_safe = true; 732 } 733 } 734 else 735 { 736 /* An unmarked indirect call. Consider it unsafe even 737 though optimization may yet figure out how to inline. */ 738 is_safe = false; 739 } 740 741 if (!is_safe) 742 { 743 if (TREE_CODE (fn) == ADDR_EXPR) 744 fn = TREE_OPERAND (fn, 0); 745 if (d->block_flags & DIAG_TM_SAFE) 746 { 747 if (direct_call_p) 748 error_at (gimple_location (stmt), 749 "unsafe function call %qD within " 750 "atomic transaction", fn); 751 else 752 { 753 if (!DECL_P (fn) || DECL_NAME (fn)) 754 error_at (gimple_location (stmt), 755 "unsafe function call %qE within " 756 "atomic transaction", fn); 757 else 758 error_at (gimple_location (stmt), 759 "unsafe indirect function call within " 760 "atomic transaction"); 761 } 762 } 763 else 764 { 765 if (direct_call_p) 766 error_at (gimple_location (stmt), 767 "unsafe function call %qD within " 768 "%<transaction_safe%> function", fn); 769 else 770 { 771 if (!DECL_P (fn) || DECL_NAME (fn)) 772 error_at (gimple_location (stmt), 773 "unsafe function call %qE within " 774 "%<transaction_safe%> function", fn); 775 else 776 error_at (gimple_location (stmt), 777 "unsafe indirect function call within " 778 "%<transaction_safe%> function"); 779 } 780 } 781 } 782 } 783 } 784 break; 785 786 case GIMPLE_ASM: 787 /* ??? We ought to come up with a way to add attributes to 788 asm statements, and then add "transaction_safe" to it. 789 Either that or get the language spec to resurrect __tm_waiver. */ 790 if (d->block_flags & DIAG_TM_SAFE) 791 error_at (gimple_location (stmt), 792 "asm not allowed in atomic transaction"); 793 else if (d->func_flags & DIAG_TM_SAFE) 794 error_at (gimple_location (stmt), 795 "asm not allowed in %<transaction_safe%> function"); 796 break; 797 798 case GIMPLE_TRANSACTION: 799 { 800 gtransaction *trans_stmt = as_a <gtransaction *> (stmt); 801 unsigned char inner_flags = DIAG_TM_SAFE; 802 803 if (gimple_transaction_subcode (trans_stmt) & GTMA_IS_RELAXED) 804 { 805 if (d->block_flags & DIAG_TM_SAFE) 806 error_at (gimple_location (stmt), 807 "relaxed transaction in atomic transaction"); 808 else if (d->func_flags & DIAG_TM_SAFE) 809 error_at (gimple_location (stmt), 810 "relaxed transaction in %<transaction_safe%> function"); 811 inner_flags = DIAG_TM_RELAXED; 812 } 813 else if (gimple_transaction_subcode (trans_stmt) & GTMA_IS_OUTER) 814 { 815 if (d->block_flags) 816 error_at (gimple_location (stmt), 817 "outer transaction in transaction"); 818 else if (d->func_flags & DIAG_TM_OUTER) 819 error_at (gimple_location (stmt), 820 "outer transaction in " 821 "%<transaction_may_cancel_outer%> function"); 822 else if (d->func_flags & DIAG_TM_SAFE) 823 error_at (gimple_location (stmt), 824 "outer transaction in %<transaction_safe%> function"); 825 inner_flags |= DIAG_TM_OUTER; 826 } 827 828 *handled_ops_p = true; 829 if (gimple_transaction_body (trans_stmt)) 830 { 831 struct walk_stmt_info wi_inner; 832 struct diagnose_tm d_inner; 833 834 memset (&d_inner, 0, sizeof (d_inner)); 835 d_inner.func_flags = d->func_flags; 836 d_inner.block_flags = d->block_flags | inner_flags; 837 d_inner.summary_flags = d_inner.func_flags | d_inner.block_flags; 838 839 memset (&wi_inner, 0, sizeof (wi_inner)); 840 wi_inner.info = &d_inner; 841 842 walk_gimple_seq (gimple_transaction_body (trans_stmt), 843 diagnose_tm_1, diagnose_tm_1_op, &wi_inner); 844 } 845 } 846 break; 847 848 default: 849 break; 850 } 851 852 return NULL_TREE; 853} 854 855static unsigned int 856diagnose_tm_blocks (void) 857{ 858 struct walk_stmt_info wi; 859 struct diagnose_tm d; 860 861 memset (&d, 0, sizeof (d)); 862 if (is_tm_may_cancel_outer (current_function_decl)) 863 d.func_flags = DIAG_TM_OUTER | DIAG_TM_SAFE; 864 else if (is_tm_safe (current_function_decl)) 865 d.func_flags = DIAG_TM_SAFE; 866 d.summary_flags = d.func_flags; 867 868 memset (&wi, 0, sizeof (wi)); 869 wi.info = &d; 870 871 walk_gimple_seq (gimple_body (current_function_decl), 872 diagnose_tm_1, diagnose_tm_1_op, &wi); 873 874 return 0; 875} 876 877namespace { 878 879const pass_data pass_data_diagnose_tm_blocks = 880{ 881 GIMPLE_PASS, /* type */ 882 "*diagnose_tm_blocks", /* name */ 883 OPTGROUP_NONE, /* optinfo_flags */ 884 TV_TRANS_MEM, /* tv_id */ 885 PROP_gimple_any, /* properties_required */ 886 0, /* properties_provided */ 887 0, /* properties_destroyed */ 888 0, /* todo_flags_start */ 889 0, /* todo_flags_finish */ 890}; 891 892class pass_diagnose_tm_blocks : public gimple_opt_pass 893{ 894public: 895 pass_diagnose_tm_blocks (gcc::context *ctxt) 896 : gimple_opt_pass (pass_data_diagnose_tm_blocks, ctxt) 897 {} 898 899 /* opt_pass methods: */ 900 virtual bool gate (function *) { return flag_tm; } 901 virtual unsigned int execute (function *) { return diagnose_tm_blocks (); } 902 903}; // class pass_diagnose_tm_blocks 904 905} // anon namespace 906 907gimple_opt_pass * 908make_pass_diagnose_tm_blocks (gcc::context *ctxt) 909{ 910 return new pass_diagnose_tm_blocks (ctxt); 911} 912 913/* Instead of instrumenting thread private memory, we save the 914 addresses in a log which we later use to save/restore the addresses 915 upon transaction start/restart. 916 917 The log is keyed by address, where each element contains individual 918 statements among different code paths that perform the store. 919 920 This log is later used to generate either plain save/restore of the 921 addresses upon transaction start/restart, or calls to the ITM_L* 922 logging functions. 923 924 So for something like: 925 926 struct large { int x[1000]; }; 927 struct large lala = { 0 }; 928 __transaction { 929 lala.x[i] = 123; 930 ... 931 } 932 933 We can either save/restore: 934 935 lala = { 0 }; 936 trxn = _ITM_startTransaction (); 937 if (trxn & a_saveLiveVariables) 938 tmp_lala1 = lala.x[i]; 939 else if (a & a_restoreLiveVariables) 940 lala.x[i] = tmp_lala1; 941 942 or use the logging functions: 943 944 lala = { 0 }; 945 trxn = _ITM_startTransaction (); 946 _ITM_LU4 (&lala.x[i]); 947 948 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as 949 far up the dominator tree to shadow all of the writes to a given 950 location (thus reducing the total number of logging calls), but not 951 so high as to be called on a path that does not perform a 952 write. */ 953 954/* One individual log entry. We may have multiple statements for the 955 same location if neither dominate each other (on different 956 execution paths). */ 957typedef struct tm_log_entry 958{ 959 /* Address to save. */ 960 tree addr; 961 /* Entry block for the transaction this address occurs in. */ 962 basic_block entry_block; 963 /* Dominating statements the store occurs in. */ 964 vec<gimple> stmts; 965 /* Initially, while we are building the log, we place a nonzero 966 value here to mean that this address *will* be saved with a 967 save/restore sequence. Later, when generating the save sequence 968 we place the SSA temp generated here. */ 969 tree save_var; 970} *tm_log_entry_t; 971 972 973/* Log entry hashtable helpers. */ 974 975struct log_entry_hasher 976{ 977 typedef tm_log_entry value_type; 978 typedef tm_log_entry compare_type; 979 static inline hashval_t hash (const value_type *); 980 static inline bool equal (const value_type *, const compare_type *); 981 static inline void remove (value_type *); 982}; 983 984/* Htab support. Return hash value for a `tm_log_entry'. */ 985inline hashval_t 986log_entry_hasher::hash (const value_type *log) 987{ 988 return iterative_hash_expr (log->addr, 0); 989} 990 991/* Htab support. Return true if two log entries are the same. */ 992inline bool 993log_entry_hasher::equal (const value_type *log1, const compare_type *log2) 994{ 995 /* FIXME: 996 997 rth: I suggest that we get rid of the component refs etc. 998 I.e. resolve the reference to base + offset. 999 1000 We may need to actually finish a merge with mainline for this, 1001 since we'd like to be presented with Richi's MEM_REF_EXPRs more 1002 often than not. But in the meantime your tm_log_entry could save 1003 the results of get_inner_reference. 1004 1005 See: g++.dg/tm/pr46653.C 1006 */ 1007 1008 /* Special case plain equality because operand_equal_p() below will 1009 return FALSE if the addresses are equal but they have 1010 side-effects (e.g. a volatile address). */ 1011 if (log1->addr == log2->addr) 1012 return true; 1013 1014 return operand_equal_p (log1->addr, log2->addr, 0); 1015} 1016 1017/* Htab support. Free one tm_log_entry. */ 1018inline void 1019log_entry_hasher::remove (value_type *lp) 1020{ 1021 lp->stmts.release (); 1022 free (lp); 1023} 1024 1025 1026/* The actual log. */ 1027static hash_table<log_entry_hasher> *tm_log; 1028 1029/* Addresses to log with a save/restore sequence. These should be in 1030 dominator order. */ 1031static vec<tree> tm_log_save_addresses; 1032 1033enum thread_memory_type 1034 { 1035 mem_non_local = 0, 1036 mem_thread_local, 1037 mem_transaction_local, 1038 mem_max 1039 }; 1040 1041typedef struct tm_new_mem_map 1042{ 1043 /* SSA_NAME being dereferenced. */ 1044 tree val; 1045 enum thread_memory_type local_new_memory; 1046} tm_new_mem_map_t; 1047 1048/* Hashtable helpers. */ 1049 1050struct tm_mem_map_hasher : typed_free_remove <tm_new_mem_map_t> 1051{ 1052 typedef tm_new_mem_map_t value_type; 1053 typedef tm_new_mem_map_t compare_type; 1054 static inline hashval_t hash (const value_type *); 1055 static inline bool equal (const value_type *, const compare_type *); 1056}; 1057 1058inline hashval_t 1059tm_mem_map_hasher::hash (const value_type *v) 1060{ 1061 return (intptr_t)v->val >> 4; 1062} 1063 1064inline bool 1065tm_mem_map_hasher::equal (const value_type *v, const compare_type *c) 1066{ 1067 return v->val == c->val; 1068} 1069 1070/* Map for an SSA_NAME originally pointing to a non aliased new piece 1071 of memory (malloc, alloc, etc). */ 1072static hash_table<tm_mem_map_hasher> *tm_new_mem_hash; 1073 1074/* Initialize logging data structures. */ 1075static void 1076tm_log_init (void) 1077{ 1078 tm_log = new hash_table<log_entry_hasher> (10); 1079 tm_new_mem_hash = new hash_table<tm_mem_map_hasher> (5); 1080 tm_log_save_addresses.create (5); 1081} 1082 1083/* Free logging data structures. */ 1084static void 1085tm_log_delete (void) 1086{ 1087 delete tm_log; 1088 tm_log = NULL; 1089 delete tm_new_mem_hash; 1090 tm_new_mem_hash = NULL; 1091 tm_log_save_addresses.release (); 1092} 1093 1094/* Return true if MEM is a transaction invariant memory for the TM 1095 region starting at REGION_ENTRY_BLOCK. */ 1096static bool 1097transaction_invariant_address_p (const_tree mem, basic_block region_entry_block) 1098{ 1099 if ((TREE_CODE (mem) == INDIRECT_REF || TREE_CODE (mem) == MEM_REF) 1100 && TREE_CODE (TREE_OPERAND (mem, 0)) == SSA_NAME) 1101 { 1102 basic_block def_bb; 1103 1104 def_bb = gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem, 0))); 1105 return def_bb != region_entry_block 1106 && dominated_by_p (CDI_DOMINATORS, region_entry_block, def_bb); 1107 } 1108 1109 mem = strip_invariant_refs (mem); 1110 return mem && (CONSTANT_CLASS_P (mem) || decl_address_invariant_p (mem)); 1111} 1112 1113/* Given an address ADDR in STMT, find it in the memory log or add it, 1114 making sure to keep only the addresses highest in the dominator 1115 tree. 1116 1117 ENTRY_BLOCK is the entry_block for the transaction. 1118 1119 If we find the address in the log, make sure it's either the same 1120 address, or an equivalent one that dominates ADDR. 1121 1122 If we find the address, but neither ADDR dominates the found 1123 address, nor the found one dominates ADDR, we're on different 1124 execution paths. Add it. 1125 1126 If known, ENTRY_BLOCK is the entry block for the region, otherwise 1127 NULL. */ 1128static void 1129tm_log_add (basic_block entry_block, tree addr, gimple stmt) 1130{ 1131 tm_log_entry **slot; 1132 struct tm_log_entry l, *lp; 1133 1134 l.addr = addr; 1135 slot = tm_log->find_slot (&l, INSERT); 1136 if (!*slot) 1137 { 1138 tree type = TREE_TYPE (addr); 1139 1140 lp = XNEW (struct tm_log_entry); 1141 lp->addr = addr; 1142 *slot = lp; 1143 1144 /* Small invariant addresses can be handled as save/restores. */ 1145 if (entry_block 1146 && transaction_invariant_address_p (lp->addr, entry_block) 1147 && TYPE_SIZE_UNIT (type) != NULL 1148 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type)) 1149 && ((HOST_WIDE_INT) tree_to_uhwi (TYPE_SIZE_UNIT (type)) 1150 < PARAM_VALUE (PARAM_TM_MAX_AGGREGATE_SIZE)) 1151 /* We must be able to copy this type normally. I.e., no 1152 special constructors and the like. */ 1153 && !TREE_ADDRESSABLE (type)) 1154 { 1155 lp->save_var = create_tmp_reg (TREE_TYPE (lp->addr), "tm_save"); 1156 lp->stmts.create (0); 1157 lp->entry_block = entry_block; 1158 /* Save addresses separately in dominator order so we don't 1159 get confused by overlapping addresses in the save/restore 1160 sequence. */ 1161 tm_log_save_addresses.safe_push (lp->addr); 1162 } 1163 else 1164 { 1165 /* Use the logging functions. */ 1166 lp->stmts.create (5); 1167 lp->stmts.quick_push (stmt); 1168 lp->save_var = NULL; 1169 } 1170 } 1171 else 1172 { 1173 size_t i; 1174 gimple oldstmt; 1175 1176 lp = *slot; 1177 1178 /* If we're generating a save/restore sequence, we don't care 1179 about statements. */ 1180 if (lp->save_var) 1181 return; 1182 1183 for (i = 0; lp->stmts.iterate (i, &oldstmt); ++i) 1184 { 1185 if (stmt == oldstmt) 1186 return; 1187 /* We already have a store to the same address, higher up the 1188 dominator tree. Nothing to do. */ 1189 if (dominated_by_p (CDI_DOMINATORS, 1190 gimple_bb (stmt), gimple_bb (oldstmt))) 1191 return; 1192 /* We should be processing blocks in dominator tree order. */ 1193 gcc_assert (!dominated_by_p (CDI_DOMINATORS, 1194 gimple_bb (oldstmt), gimple_bb (stmt))); 1195 } 1196 /* Store is on a different code path. */ 1197 lp->stmts.safe_push (stmt); 1198 } 1199} 1200 1201/* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME 1202 result, insert the new statements before GSI. */ 1203 1204static tree 1205gimplify_addr (gimple_stmt_iterator *gsi, tree x) 1206{ 1207 if (TREE_CODE (x) == TARGET_MEM_REF) 1208 x = tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x)), x); 1209 else 1210 x = build_fold_addr_expr (x); 1211 return force_gimple_operand_gsi (gsi, x, true, NULL, true, GSI_SAME_STMT); 1212} 1213 1214/* Instrument one address with the logging functions. 1215 ADDR is the address to save. 1216 STMT is the statement before which to place it. */ 1217static void 1218tm_log_emit_stmt (tree addr, gimple stmt) 1219{ 1220 tree type = TREE_TYPE (addr); 1221 tree size = TYPE_SIZE_UNIT (type); 1222 gimple_stmt_iterator gsi = gsi_for_stmt (stmt); 1223 gimple log; 1224 enum built_in_function code = BUILT_IN_TM_LOG; 1225 1226 if (type == float_type_node) 1227 code = BUILT_IN_TM_LOG_FLOAT; 1228 else if (type == double_type_node) 1229 code = BUILT_IN_TM_LOG_DOUBLE; 1230 else if (type == long_double_type_node) 1231 code = BUILT_IN_TM_LOG_LDOUBLE; 1232 else if (tree_fits_uhwi_p (size)) 1233 { 1234 unsigned int n = tree_to_uhwi (size); 1235 switch (n) 1236 { 1237 case 1: 1238 code = BUILT_IN_TM_LOG_1; 1239 break; 1240 case 2: 1241 code = BUILT_IN_TM_LOG_2; 1242 break; 1243 case 4: 1244 code = BUILT_IN_TM_LOG_4; 1245 break; 1246 case 8: 1247 code = BUILT_IN_TM_LOG_8; 1248 break; 1249 default: 1250 code = BUILT_IN_TM_LOG; 1251 if (TREE_CODE (type) == VECTOR_TYPE) 1252 { 1253 if (n == 8 && builtin_decl_explicit (BUILT_IN_TM_LOG_M64)) 1254 code = BUILT_IN_TM_LOG_M64; 1255 else if (n == 16 && builtin_decl_explicit (BUILT_IN_TM_LOG_M128)) 1256 code = BUILT_IN_TM_LOG_M128; 1257 else if (n == 32 && builtin_decl_explicit (BUILT_IN_TM_LOG_M256)) 1258 code = BUILT_IN_TM_LOG_M256; 1259 } 1260 break; 1261 } 1262 } 1263 1264 addr = gimplify_addr (&gsi, addr); 1265 if (code == BUILT_IN_TM_LOG) 1266 log = gimple_build_call (builtin_decl_explicit (code), 2, addr, size); 1267 else 1268 log = gimple_build_call (builtin_decl_explicit (code), 1, addr); 1269 gsi_insert_before (&gsi, log, GSI_SAME_STMT); 1270} 1271 1272/* Go through the log and instrument address that must be instrumented 1273 with the logging functions. Leave the save/restore addresses for 1274 later. */ 1275static void 1276tm_log_emit (void) 1277{ 1278 hash_table<log_entry_hasher>::iterator hi; 1279 struct tm_log_entry *lp; 1280 1281 FOR_EACH_HASH_TABLE_ELEMENT (*tm_log, lp, tm_log_entry_t, hi) 1282 { 1283 size_t i; 1284 gimple stmt; 1285 1286 if (dump_file) 1287 { 1288 fprintf (dump_file, "TM thread private mem logging: "); 1289 print_generic_expr (dump_file, lp->addr, 0); 1290 fprintf (dump_file, "\n"); 1291 } 1292 1293 if (lp->save_var) 1294 { 1295 if (dump_file) 1296 fprintf (dump_file, "DUMPING to variable\n"); 1297 continue; 1298 } 1299 else 1300 { 1301 if (dump_file) 1302 fprintf (dump_file, "DUMPING with logging functions\n"); 1303 for (i = 0; lp->stmts.iterate (i, &stmt); ++i) 1304 tm_log_emit_stmt (lp->addr, stmt); 1305 } 1306 } 1307} 1308 1309/* Emit the save sequence for the corresponding addresses in the log. 1310 ENTRY_BLOCK is the entry block for the transaction. 1311 BB is the basic block to insert the code in. */ 1312static void 1313tm_log_emit_saves (basic_block entry_block, basic_block bb) 1314{ 1315 size_t i; 1316 gimple_stmt_iterator gsi = gsi_last_bb (bb); 1317 gimple stmt; 1318 struct tm_log_entry l, *lp; 1319 1320 for (i = 0; i < tm_log_save_addresses.length (); ++i) 1321 { 1322 l.addr = tm_log_save_addresses[i]; 1323 lp = *(tm_log->find_slot (&l, NO_INSERT)); 1324 gcc_assert (lp->save_var != NULL); 1325 1326 /* We only care about variables in the current transaction. */ 1327 if (lp->entry_block != entry_block) 1328 continue; 1329 1330 stmt = gimple_build_assign (lp->save_var, unshare_expr (lp->addr)); 1331 1332 /* Make sure we can create an SSA_NAME for this type. For 1333 instance, aggregates aren't allowed, in which case the system 1334 will create a VOP for us and everything will just work. */ 1335 if (is_gimple_reg_type (TREE_TYPE (lp->save_var))) 1336 { 1337 lp->save_var = make_ssa_name (lp->save_var, stmt); 1338 gimple_assign_set_lhs (stmt, lp->save_var); 1339 } 1340 1341 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); 1342 } 1343} 1344 1345/* Emit the restore sequence for the corresponding addresses in the log. 1346 ENTRY_BLOCK is the entry block for the transaction. 1347 BB is the basic block to insert the code in. */ 1348static void 1349tm_log_emit_restores (basic_block entry_block, basic_block bb) 1350{ 1351 int i; 1352 struct tm_log_entry l, *lp; 1353 gimple_stmt_iterator gsi; 1354 gimple stmt; 1355 1356 for (i = tm_log_save_addresses.length () - 1; i >= 0; i--) 1357 { 1358 l.addr = tm_log_save_addresses[i]; 1359 lp = *(tm_log->find_slot (&l, NO_INSERT)); 1360 gcc_assert (lp->save_var != NULL); 1361 1362 /* We only care about variables in the current transaction. */ 1363 if (lp->entry_block != entry_block) 1364 continue; 1365 1366 /* Restores are in LIFO order from the saves in case we have 1367 overlaps. */ 1368 gsi = gsi_start_bb (bb); 1369 1370 stmt = gimple_build_assign (unshare_expr (lp->addr), lp->save_var); 1371 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 1372 } 1373} 1374 1375 1376static tree lower_sequence_tm (gimple_stmt_iterator *, bool *, 1377 struct walk_stmt_info *); 1378static tree lower_sequence_no_tm (gimple_stmt_iterator *, bool *, 1379 struct walk_stmt_info *); 1380 1381/* Evaluate an address X being dereferenced and determine if it 1382 originally points to a non aliased new chunk of memory (malloc, 1383 alloca, etc). 1384 1385 Return MEM_THREAD_LOCAL if it points to a thread-local address. 1386 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address. 1387 Return MEM_NON_LOCAL otherwise. 1388 1389 ENTRY_BLOCK is the entry block to the transaction containing the 1390 dereference of X. */ 1391static enum thread_memory_type 1392thread_private_new_memory (basic_block entry_block, tree x) 1393{ 1394 gimple stmt = NULL; 1395 enum tree_code code; 1396 tm_new_mem_map_t **slot; 1397 tm_new_mem_map_t elt, *elt_p; 1398 tree val = x; 1399 enum thread_memory_type retval = mem_transaction_local; 1400 1401 if (!entry_block 1402 || TREE_CODE (x) != SSA_NAME 1403 /* Possible uninitialized use, or a function argument. In 1404 either case, we don't care. */ 1405 || SSA_NAME_IS_DEFAULT_DEF (x)) 1406 return mem_non_local; 1407 1408 /* Look in cache first. */ 1409 elt.val = x; 1410 slot = tm_new_mem_hash->find_slot (&elt, INSERT); 1411 elt_p = *slot; 1412 if (elt_p) 1413 return elt_p->local_new_memory; 1414 1415 /* Optimistically assume the memory is transaction local during 1416 processing. This catches recursion into this variable. */ 1417 *slot = elt_p = XNEW (tm_new_mem_map_t); 1418 elt_p->val = val; 1419 elt_p->local_new_memory = mem_transaction_local; 1420 1421 /* Search DEF chain to find the original definition of this address. */ 1422 do 1423 { 1424 if (ptr_deref_may_alias_global_p (x)) 1425 { 1426 /* Address escapes. This is not thread-private. */ 1427 retval = mem_non_local; 1428 goto new_memory_ret; 1429 } 1430 1431 stmt = SSA_NAME_DEF_STMT (x); 1432 1433 /* If the malloc call is outside the transaction, this is 1434 thread-local. */ 1435 if (retval != mem_thread_local 1436 && !dominated_by_p (CDI_DOMINATORS, gimple_bb (stmt), entry_block)) 1437 retval = mem_thread_local; 1438 1439 if (is_gimple_assign (stmt)) 1440 { 1441 code = gimple_assign_rhs_code (stmt); 1442 /* x = foo ==> foo */ 1443 if (code == SSA_NAME) 1444 x = gimple_assign_rhs1 (stmt); 1445 /* x = foo + n ==> foo */ 1446 else if (code == POINTER_PLUS_EXPR) 1447 x = gimple_assign_rhs1 (stmt); 1448 /* x = (cast*) foo ==> foo */ 1449 else if (code == VIEW_CONVERT_EXPR || CONVERT_EXPR_CODE_P (code)) 1450 x = gimple_assign_rhs1 (stmt); 1451 /* x = c ? op1 : op2 == > op1 or op2 just like a PHI */ 1452 else if (code == COND_EXPR) 1453 { 1454 tree op1 = gimple_assign_rhs2 (stmt); 1455 tree op2 = gimple_assign_rhs3 (stmt); 1456 enum thread_memory_type mem; 1457 retval = thread_private_new_memory (entry_block, op1); 1458 if (retval == mem_non_local) 1459 goto new_memory_ret; 1460 mem = thread_private_new_memory (entry_block, op2); 1461 retval = MIN (retval, mem); 1462 goto new_memory_ret; 1463 } 1464 else 1465 { 1466 retval = mem_non_local; 1467 goto new_memory_ret; 1468 } 1469 } 1470 else 1471 { 1472 if (gimple_code (stmt) == GIMPLE_PHI) 1473 { 1474 unsigned int i; 1475 enum thread_memory_type mem; 1476 tree phi_result = gimple_phi_result (stmt); 1477 1478 /* If any of the ancestors are non-local, we are sure to 1479 be non-local. Otherwise we can avoid doing anything 1480 and inherit what has already been generated. */ 1481 retval = mem_max; 1482 for (i = 0; i < gimple_phi_num_args (stmt); ++i) 1483 { 1484 tree op = PHI_ARG_DEF (stmt, i); 1485 1486 /* Exclude self-assignment. */ 1487 if (phi_result == op) 1488 continue; 1489 1490 mem = thread_private_new_memory (entry_block, op); 1491 if (mem == mem_non_local) 1492 { 1493 retval = mem; 1494 goto new_memory_ret; 1495 } 1496 retval = MIN (retval, mem); 1497 } 1498 goto new_memory_ret; 1499 } 1500 break; 1501 } 1502 } 1503 while (TREE_CODE (x) == SSA_NAME); 1504 1505 if (stmt && is_gimple_call (stmt) && gimple_call_flags (stmt) & ECF_MALLOC) 1506 /* Thread-local or transaction-local. */ 1507 ; 1508 else 1509 retval = mem_non_local; 1510 1511 new_memory_ret: 1512 elt_p->local_new_memory = retval; 1513 return retval; 1514} 1515 1516/* Determine whether X has to be instrumented using a read 1517 or write barrier. 1518 1519 ENTRY_BLOCK is the entry block for the region where stmt resides 1520 in. NULL if unknown. 1521 1522 STMT is the statement in which X occurs in. It is used for thread 1523 private memory instrumentation. If no TPM instrumentation is 1524 desired, STMT should be null. */ 1525static bool 1526requires_barrier (basic_block entry_block, tree x, gimple stmt) 1527{ 1528 tree orig = x; 1529 while (handled_component_p (x)) 1530 x = TREE_OPERAND (x, 0); 1531 1532 switch (TREE_CODE (x)) 1533 { 1534 case INDIRECT_REF: 1535 case MEM_REF: 1536 { 1537 enum thread_memory_type ret; 1538 1539 ret = thread_private_new_memory (entry_block, TREE_OPERAND (x, 0)); 1540 if (ret == mem_non_local) 1541 return true; 1542 if (stmt && ret == mem_thread_local) 1543 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */ 1544 tm_log_add (entry_block, orig, stmt); 1545 1546 /* Transaction-locals require nothing at all. For malloc, a 1547 transaction restart frees the memory and we reallocate. 1548 For alloca, the stack pointer gets reset by the retry and 1549 we reallocate. */ 1550 return false; 1551 } 1552 1553 case TARGET_MEM_REF: 1554 if (TREE_CODE (TMR_BASE (x)) != ADDR_EXPR) 1555 return true; 1556 x = TREE_OPERAND (TMR_BASE (x), 0); 1557 if (TREE_CODE (x) == PARM_DECL) 1558 return false; 1559 gcc_assert (TREE_CODE (x) == VAR_DECL); 1560 /* FALLTHRU */ 1561 1562 case PARM_DECL: 1563 case RESULT_DECL: 1564 case VAR_DECL: 1565 if (DECL_BY_REFERENCE (x)) 1566 { 1567 /* ??? This value is a pointer, but aggregate_value_p has been 1568 jigged to return true which confuses needs_to_live_in_memory. 1569 This ought to be cleaned up generically. 1570 1571 FIXME: Verify this still happens after the next mainline 1572 merge. Testcase ie g++.dg/tm/pr47554.C. 1573 */ 1574 return false; 1575 } 1576 1577 if (is_global_var (x)) 1578 return !TREE_READONLY (x); 1579 if (/* FIXME: This condition should actually go below in the 1580 tm_log_add() call, however is_call_clobbered() depends on 1581 aliasing info which is not available during 1582 gimplification. Since requires_barrier() gets called 1583 during lower_sequence_tm/gimplification, leave the call 1584 to needs_to_live_in_memory until we eliminate 1585 lower_sequence_tm altogether. */ 1586 needs_to_live_in_memory (x)) 1587 return true; 1588 else 1589 { 1590 /* For local memory that doesn't escape (aka thread private 1591 memory), we can either save the value at the beginning of 1592 the transaction and restore on restart, or call a tm 1593 function to dynamically save and restore on restart 1594 (ITM_L*). */ 1595 if (stmt) 1596 tm_log_add (entry_block, orig, stmt); 1597 return false; 1598 } 1599 1600 default: 1601 return false; 1602 } 1603} 1604 1605/* Mark the GIMPLE_ASSIGN statement as appropriate for being inside 1606 a transaction region. */ 1607 1608static void 1609examine_assign_tm (unsigned *state, gimple_stmt_iterator *gsi) 1610{ 1611 gimple stmt = gsi_stmt (*gsi); 1612 1613 if (requires_barrier (/*entry_block=*/NULL, gimple_assign_rhs1 (stmt), NULL)) 1614 *state |= GTMA_HAVE_LOAD; 1615 if (requires_barrier (/*entry_block=*/NULL, gimple_assign_lhs (stmt), NULL)) 1616 *state |= GTMA_HAVE_STORE; 1617} 1618 1619/* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */ 1620 1621static void 1622examine_call_tm (unsigned *state, gimple_stmt_iterator *gsi) 1623{ 1624 gimple stmt = gsi_stmt (*gsi); 1625 tree fn; 1626 1627 if (is_tm_pure_call (stmt)) 1628 return; 1629 1630 /* Check if this call is a transaction abort. */ 1631 fn = gimple_call_fndecl (stmt); 1632 if (is_tm_abort (fn)) 1633 *state |= GTMA_HAVE_ABORT; 1634 1635 /* Note that something may happen. */ 1636 *state |= GTMA_HAVE_LOAD | GTMA_HAVE_STORE; 1637} 1638 1639/* Lower a GIMPLE_TRANSACTION statement. */ 1640 1641static void 1642lower_transaction (gimple_stmt_iterator *gsi, struct walk_stmt_info *wi) 1643{ 1644 gimple g; 1645 gtransaction *stmt = as_a <gtransaction *> (gsi_stmt (*gsi)); 1646 unsigned int *outer_state = (unsigned int *) wi->info; 1647 unsigned int this_state = 0; 1648 struct walk_stmt_info this_wi; 1649 1650 /* First, lower the body. The scanning that we do inside gives 1651 us some idea of what we're dealing with. */ 1652 memset (&this_wi, 0, sizeof (this_wi)); 1653 this_wi.info = (void *) &this_state; 1654 walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt), 1655 lower_sequence_tm, NULL, &this_wi); 1656 1657 /* If there was absolutely nothing transaction related inside the 1658 transaction, we may elide it. Likewise if this is a nested 1659 transaction and does not contain an abort. */ 1660 if (this_state == 0 1661 || (!(this_state & GTMA_HAVE_ABORT) && outer_state != NULL)) 1662 { 1663 if (outer_state) 1664 *outer_state |= this_state; 1665 1666 gsi_insert_seq_before (gsi, gimple_transaction_body (stmt), 1667 GSI_SAME_STMT); 1668 gimple_transaction_set_body (stmt, NULL); 1669 1670 gsi_remove (gsi, true); 1671 wi->removed_stmt = true; 1672 return; 1673 } 1674 1675 /* Wrap the body of the transaction in a try-finally node so that 1676 the commit call is always properly called. */ 1677 g = gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT), 0); 1678 if (flag_exceptions) 1679 { 1680 tree ptr; 1681 gimple_seq n_seq, e_seq; 1682 1683 n_seq = gimple_seq_alloc_with_stmt (g); 1684 e_seq = NULL; 1685 1686 g = gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER), 1687 1, integer_zero_node); 1688 ptr = create_tmp_var (ptr_type_node); 1689 gimple_call_set_lhs (g, ptr); 1690 gimple_seq_add_stmt (&e_seq, g); 1691 1692 g = gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH), 1693 1, ptr); 1694 gimple_seq_add_stmt (&e_seq, g); 1695 1696 g = gimple_build_eh_else (n_seq, e_seq); 1697 } 1698 1699 g = gimple_build_try (gimple_transaction_body (stmt), 1700 gimple_seq_alloc_with_stmt (g), GIMPLE_TRY_FINALLY); 1701 gsi_insert_after (gsi, g, GSI_CONTINUE_LINKING); 1702 1703 gimple_transaction_set_body (stmt, NULL); 1704 1705 /* If the transaction calls abort or if this is an outer transaction, 1706 add an "over" label afterwards. */ 1707 if ((this_state & (GTMA_HAVE_ABORT)) 1708 || (gimple_transaction_subcode (stmt) & GTMA_IS_OUTER)) 1709 { 1710 tree label = create_artificial_label (UNKNOWN_LOCATION); 1711 gimple_transaction_set_label (stmt, label); 1712 gsi_insert_after (gsi, gimple_build_label (label), GSI_CONTINUE_LINKING); 1713 } 1714 1715 /* Record the set of operations found for use later. */ 1716 this_state |= gimple_transaction_subcode (stmt) & GTMA_DECLARATION_MASK; 1717 gimple_transaction_set_subcode (stmt, this_state); 1718} 1719 1720/* Iterate through the statements in the sequence, lowering them all 1721 as appropriate for being in a transaction. */ 1722 1723static tree 1724lower_sequence_tm (gimple_stmt_iterator *gsi, bool *handled_ops_p, 1725 struct walk_stmt_info *wi) 1726{ 1727 unsigned int *state = (unsigned int *) wi->info; 1728 gimple stmt = gsi_stmt (*gsi); 1729 1730 *handled_ops_p = true; 1731 switch (gimple_code (stmt)) 1732 { 1733 case GIMPLE_ASSIGN: 1734 /* Only memory reads/writes need to be instrumented. */ 1735 if (gimple_assign_single_p (stmt)) 1736 examine_assign_tm (state, gsi); 1737 break; 1738 1739 case GIMPLE_CALL: 1740 examine_call_tm (state, gsi); 1741 break; 1742 1743 case GIMPLE_ASM: 1744 *state |= GTMA_MAY_ENTER_IRREVOCABLE; 1745 break; 1746 1747 case GIMPLE_TRANSACTION: 1748 lower_transaction (gsi, wi); 1749 break; 1750 1751 default: 1752 *handled_ops_p = !gimple_has_substatements (stmt); 1753 break; 1754 } 1755 1756 return NULL_TREE; 1757} 1758 1759/* Iterate through the statements in the sequence, lowering them all 1760 as appropriate for being outside of a transaction. */ 1761 1762static tree 1763lower_sequence_no_tm (gimple_stmt_iterator *gsi, bool *handled_ops_p, 1764 struct walk_stmt_info * wi) 1765{ 1766 gimple stmt = gsi_stmt (*gsi); 1767 1768 if (gimple_code (stmt) == GIMPLE_TRANSACTION) 1769 { 1770 *handled_ops_p = true; 1771 lower_transaction (gsi, wi); 1772 } 1773 else 1774 *handled_ops_p = !gimple_has_substatements (stmt); 1775 1776 return NULL_TREE; 1777} 1778 1779/* Main entry point for flattening GIMPLE_TRANSACTION constructs. After 1780 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has 1781 been moved out, and all the data required for constructing a proper 1782 CFG has been recorded. */ 1783 1784static unsigned int 1785execute_lower_tm (void) 1786{ 1787 struct walk_stmt_info wi; 1788 gimple_seq body; 1789 1790 /* Transactional clones aren't created until a later pass. */ 1791 gcc_assert (!decl_is_tm_clone (current_function_decl)); 1792 1793 body = gimple_body (current_function_decl); 1794 memset (&wi, 0, sizeof (wi)); 1795 walk_gimple_seq_mod (&body, lower_sequence_no_tm, NULL, &wi); 1796 gimple_set_body (current_function_decl, body); 1797 1798 return 0; 1799} 1800 1801namespace { 1802 1803const pass_data pass_data_lower_tm = 1804{ 1805 GIMPLE_PASS, /* type */ 1806 "tmlower", /* name */ 1807 OPTGROUP_NONE, /* optinfo_flags */ 1808 TV_TRANS_MEM, /* tv_id */ 1809 PROP_gimple_lcf, /* properties_required */ 1810 0, /* properties_provided */ 1811 0, /* properties_destroyed */ 1812 0, /* todo_flags_start */ 1813 0, /* todo_flags_finish */ 1814}; 1815 1816class pass_lower_tm : public gimple_opt_pass 1817{ 1818public: 1819 pass_lower_tm (gcc::context *ctxt) 1820 : gimple_opt_pass (pass_data_lower_tm, ctxt) 1821 {} 1822 1823 /* opt_pass methods: */ 1824 virtual bool gate (function *) { return flag_tm; } 1825 virtual unsigned int execute (function *) { return execute_lower_tm (); } 1826 1827}; // class pass_lower_tm 1828 1829} // anon namespace 1830 1831gimple_opt_pass * 1832make_pass_lower_tm (gcc::context *ctxt) 1833{ 1834 return new pass_lower_tm (ctxt); 1835} 1836 1837/* Collect region information for each transaction. */ 1838 1839struct tm_region 1840{ 1841public: 1842 1843 /* The field "transaction_stmt" is initially a gtransaction *, 1844 but eventually gets lowered to a gcall *(to BUILT_IN_TM_START). 1845 1846 Helper method to get it as a gtransaction *, with code-checking 1847 in a checked-build. */ 1848 1849 gtransaction * 1850 get_transaction_stmt () const 1851 { 1852 return as_a <gtransaction *> (transaction_stmt); 1853 } 1854 1855public: 1856 1857 /* Link to the next unnested transaction. */ 1858 struct tm_region *next; 1859 1860 /* Link to the next inner transaction. */ 1861 struct tm_region *inner; 1862 1863 /* Link to the next outer transaction. */ 1864 struct tm_region *outer; 1865 1866 /* The GIMPLE_TRANSACTION statement beginning this transaction. 1867 After TM_MARK, this gets replaced by a call to 1868 BUILT_IN_TM_START. 1869 Hence this will be either a gtransaction *or a gcall *. */ 1870 gimple transaction_stmt; 1871 1872 /* After TM_MARK expands the GIMPLE_TRANSACTION into a call to 1873 BUILT_IN_TM_START, this field is true if the transaction is an 1874 outer transaction. */ 1875 bool original_transaction_was_outer; 1876 1877 /* Return value from BUILT_IN_TM_START. */ 1878 tree tm_state; 1879 1880 /* The entry block to this region. This will always be the first 1881 block of the body of the transaction. */ 1882 basic_block entry_block; 1883 1884 /* The first block after an expanded call to _ITM_beginTransaction. */ 1885 basic_block restart_block; 1886 1887 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK. 1888 These blocks are still a part of the region (i.e., the border is 1889 inclusive). Note that this set is only complete for paths in the CFG 1890 starting at ENTRY_BLOCK, and that there is no exit block recorded for 1891 the edge to the "over" label. */ 1892 bitmap exit_blocks; 1893 1894 /* The set of all blocks that have an TM_IRREVOCABLE call. */ 1895 bitmap irr_blocks; 1896}; 1897 1898typedef struct tm_region *tm_region_p; 1899 1900/* True if there are pending edge statements to be committed for the 1901 current function being scanned in the tmmark pass. */ 1902bool pending_edge_inserts_p; 1903 1904static struct tm_region *all_tm_regions; 1905static bitmap_obstack tm_obstack; 1906 1907 1908/* A subroutine of tm_region_init. Record the existence of the 1909 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */ 1910 1911static struct tm_region * 1912tm_region_init_0 (struct tm_region *outer, basic_block bb, 1913 gtransaction *stmt) 1914{ 1915 struct tm_region *region; 1916 1917 region = (struct tm_region *) 1918 obstack_alloc (&tm_obstack.obstack, sizeof (struct tm_region)); 1919 1920 if (outer) 1921 { 1922 region->next = outer->inner; 1923 outer->inner = region; 1924 } 1925 else 1926 { 1927 region->next = all_tm_regions; 1928 all_tm_regions = region; 1929 } 1930 region->inner = NULL; 1931 region->outer = outer; 1932 1933 region->transaction_stmt = stmt; 1934 region->original_transaction_was_outer = false; 1935 region->tm_state = NULL; 1936 1937 /* There are either one or two edges out of the block containing 1938 the GIMPLE_TRANSACTION, one to the actual region and one to the 1939 "over" label if the region contains an abort. The former will 1940 always be the one marked FALLTHRU. */ 1941 region->entry_block = FALLTHRU_EDGE (bb)->dest; 1942 1943 region->exit_blocks = BITMAP_ALLOC (&tm_obstack); 1944 region->irr_blocks = BITMAP_ALLOC (&tm_obstack); 1945 1946 return region; 1947} 1948 1949/* A subroutine of tm_region_init. Record all the exit and 1950 irrevocable blocks in BB into the region's exit_blocks and 1951 irr_blocks bitmaps. Returns the new region being scanned. */ 1952 1953static struct tm_region * 1954tm_region_init_1 (struct tm_region *region, basic_block bb) 1955{ 1956 gimple_stmt_iterator gsi; 1957 gimple g; 1958 1959 if (!region 1960 || (!region->irr_blocks && !region->exit_blocks)) 1961 return region; 1962 1963 /* Check to see if this is the end of a region by seeing if it 1964 contains a call to __builtin_tm_commit{,_eh}. Note that the 1965 outermost region for DECL_IS_TM_CLONE need not collect this. */ 1966 for (gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi)) 1967 { 1968 g = gsi_stmt (gsi); 1969 if (gimple_code (g) == GIMPLE_CALL) 1970 { 1971 tree fn = gimple_call_fndecl (g); 1972 if (fn && DECL_BUILT_IN_CLASS (fn) == BUILT_IN_NORMAL) 1973 { 1974 if ((DECL_FUNCTION_CODE (fn) == BUILT_IN_TM_COMMIT 1975 || DECL_FUNCTION_CODE (fn) == BUILT_IN_TM_COMMIT_EH) 1976 && region->exit_blocks) 1977 { 1978 bitmap_set_bit (region->exit_blocks, bb->index); 1979 region = region->outer; 1980 break; 1981 } 1982 if (DECL_FUNCTION_CODE (fn) == BUILT_IN_TM_IRREVOCABLE) 1983 bitmap_set_bit (region->irr_blocks, bb->index); 1984 } 1985 } 1986 } 1987 return region; 1988} 1989 1990/* Collect all of the transaction regions within the current function 1991 and record them in ALL_TM_REGIONS. The REGION parameter may specify 1992 an "outermost" region for use by tm clones. */ 1993 1994static void 1995tm_region_init (struct tm_region *region) 1996{ 1997 gimple g; 1998 edge_iterator ei; 1999 edge e; 2000 basic_block bb; 2001 auto_vec<basic_block> queue; 2002 bitmap visited_blocks = BITMAP_ALLOC (NULL); 2003 struct tm_region *old_region; 2004 auto_vec<tm_region_p> bb_regions; 2005 2006 all_tm_regions = region; 2007 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 2008 2009 /* We could store this information in bb->aux, but we may get called 2010 through get_all_tm_blocks() from another pass that may be already 2011 using bb->aux. */ 2012 bb_regions.safe_grow_cleared (last_basic_block_for_fn (cfun)); 2013 2014 queue.safe_push (bb); 2015 bb_regions[bb->index] = region; 2016 do 2017 { 2018 bb = queue.pop (); 2019 region = bb_regions[bb->index]; 2020 bb_regions[bb->index] = NULL; 2021 2022 /* Record exit and irrevocable blocks. */ 2023 region = tm_region_init_1 (region, bb); 2024 2025 /* Check for the last statement in the block beginning a new region. */ 2026 g = last_stmt (bb); 2027 old_region = region; 2028 if (g) 2029 if (gtransaction *trans_stmt = dyn_cast <gtransaction *> (g)) 2030 region = tm_region_init_0 (region, bb, trans_stmt); 2031 2032 /* Process subsequent blocks. */ 2033 FOR_EACH_EDGE (e, ei, bb->succs) 2034 if (!bitmap_bit_p (visited_blocks, e->dest->index)) 2035 { 2036 bitmap_set_bit (visited_blocks, e->dest->index); 2037 queue.safe_push (e->dest); 2038 2039 /* If the current block started a new region, make sure that only 2040 the entry block of the new region is associated with this region. 2041 Other successors are still part of the old region. */ 2042 if (old_region != region && e->dest != region->entry_block) 2043 bb_regions[e->dest->index] = old_region; 2044 else 2045 bb_regions[e->dest->index] = region; 2046 } 2047 } 2048 while (!queue.is_empty ()); 2049 BITMAP_FREE (visited_blocks); 2050} 2051 2052/* The "gate" function for all transactional memory expansion and optimization 2053 passes. We collect region information for each top-level transaction, and 2054 if we don't find any, we skip all of the TM passes. Each region will have 2055 all of the exit blocks recorded, and the originating statement. */ 2056 2057static bool 2058gate_tm_init (void) 2059{ 2060 if (!flag_tm) 2061 return false; 2062 2063 calculate_dominance_info (CDI_DOMINATORS); 2064 bitmap_obstack_initialize (&tm_obstack); 2065 2066 /* If the function is a TM_CLONE, then the entire function is the region. */ 2067 if (decl_is_tm_clone (current_function_decl)) 2068 { 2069 struct tm_region *region = (struct tm_region *) 2070 obstack_alloc (&tm_obstack.obstack, sizeof (struct tm_region)); 2071 memset (region, 0, sizeof (*region)); 2072 region->entry_block = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 2073 /* For a clone, the entire function is the region. But even if 2074 we don't need to record any exit blocks, we may need to 2075 record irrevocable blocks. */ 2076 region->irr_blocks = BITMAP_ALLOC (&tm_obstack); 2077 2078 tm_region_init (region); 2079 } 2080 else 2081 { 2082 tm_region_init (NULL); 2083 2084 /* If we didn't find any regions, cleanup and skip the whole tree 2085 of tm-related optimizations. */ 2086 if (all_tm_regions == NULL) 2087 { 2088 bitmap_obstack_release (&tm_obstack); 2089 return false; 2090 } 2091 } 2092 2093 return true; 2094} 2095 2096namespace { 2097 2098const pass_data pass_data_tm_init = 2099{ 2100 GIMPLE_PASS, /* type */ 2101 "*tminit", /* name */ 2102 OPTGROUP_NONE, /* optinfo_flags */ 2103 TV_TRANS_MEM, /* tv_id */ 2104 ( PROP_ssa | PROP_cfg ), /* properties_required */ 2105 0, /* properties_provided */ 2106 0, /* properties_destroyed */ 2107 0, /* todo_flags_start */ 2108 0, /* todo_flags_finish */ 2109}; 2110 2111class pass_tm_init : public gimple_opt_pass 2112{ 2113public: 2114 pass_tm_init (gcc::context *ctxt) 2115 : gimple_opt_pass (pass_data_tm_init, ctxt) 2116 {} 2117 2118 /* opt_pass methods: */ 2119 virtual bool gate (function *) { return gate_tm_init (); } 2120 2121}; // class pass_tm_init 2122 2123} // anon namespace 2124 2125gimple_opt_pass * 2126make_pass_tm_init (gcc::context *ctxt) 2127{ 2128 return new pass_tm_init (ctxt); 2129} 2130 2131/* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region 2132 represented by STATE. */ 2133 2134static inline void 2135transaction_subcode_ior (struct tm_region *region, unsigned flags) 2136{ 2137 if (region && region->transaction_stmt) 2138 { 2139 gtransaction *transaction_stmt = region->get_transaction_stmt (); 2140 flags |= gimple_transaction_subcode (transaction_stmt); 2141 gimple_transaction_set_subcode (transaction_stmt, flags); 2142 } 2143} 2144 2145/* Construct a memory load in a transactional context. Return the 2146 gimple statement performing the load, or NULL if there is no 2147 TM_LOAD builtin of the appropriate size to do the load. 2148 2149 LOC is the location to use for the new statement(s). */ 2150 2151static gcall * 2152build_tm_load (location_t loc, tree lhs, tree rhs, gimple_stmt_iterator *gsi) 2153{ 2154 enum built_in_function code = END_BUILTINS; 2155 tree t, type = TREE_TYPE (rhs), decl; 2156 gcall *gcall; 2157 2158 if (type == float_type_node) 2159 code = BUILT_IN_TM_LOAD_FLOAT; 2160 else if (type == double_type_node) 2161 code = BUILT_IN_TM_LOAD_DOUBLE; 2162 else if (type == long_double_type_node) 2163 code = BUILT_IN_TM_LOAD_LDOUBLE; 2164 else if (TYPE_SIZE_UNIT (type) != NULL 2165 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type))) 2166 { 2167 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type))) 2168 { 2169 case 1: 2170 code = BUILT_IN_TM_LOAD_1; 2171 break; 2172 case 2: 2173 code = BUILT_IN_TM_LOAD_2; 2174 break; 2175 case 4: 2176 code = BUILT_IN_TM_LOAD_4; 2177 break; 2178 case 8: 2179 code = BUILT_IN_TM_LOAD_8; 2180 break; 2181 } 2182 } 2183 2184 if (code == END_BUILTINS) 2185 { 2186 decl = targetm.vectorize.builtin_tm_load (type); 2187 if (!decl) 2188 return NULL; 2189 } 2190 else 2191 decl = builtin_decl_explicit (code); 2192 2193 t = gimplify_addr (gsi, rhs); 2194 gcall = gimple_build_call (decl, 1, t); 2195 gimple_set_location (gcall, loc); 2196 2197 t = TREE_TYPE (TREE_TYPE (decl)); 2198 if (useless_type_conversion_p (type, t)) 2199 { 2200 gimple_call_set_lhs (gcall, lhs); 2201 gsi_insert_before (gsi, gcall, GSI_SAME_STMT); 2202 } 2203 else 2204 { 2205 gimple g; 2206 tree temp; 2207 2208 temp = create_tmp_reg (t); 2209 gimple_call_set_lhs (gcall, temp); 2210 gsi_insert_before (gsi, gcall, GSI_SAME_STMT); 2211 2212 t = fold_build1 (VIEW_CONVERT_EXPR, type, temp); 2213 g = gimple_build_assign (lhs, t); 2214 gsi_insert_before (gsi, g, GSI_SAME_STMT); 2215 } 2216 2217 return gcall; 2218} 2219 2220 2221/* Similarly for storing TYPE in a transactional context. */ 2222 2223static gcall * 2224build_tm_store (location_t loc, tree lhs, tree rhs, gimple_stmt_iterator *gsi) 2225{ 2226 enum built_in_function code = END_BUILTINS; 2227 tree t, fn, type = TREE_TYPE (rhs), simple_type; 2228 gcall *gcall; 2229 2230 if (type == float_type_node) 2231 code = BUILT_IN_TM_STORE_FLOAT; 2232 else if (type == double_type_node) 2233 code = BUILT_IN_TM_STORE_DOUBLE; 2234 else if (type == long_double_type_node) 2235 code = BUILT_IN_TM_STORE_LDOUBLE; 2236 else if (TYPE_SIZE_UNIT (type) != NULL 2237 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type))) 2238 { 2239 switch (tree_to_uhwi (TYPE_SIZE_UNIT (type))) 2240 { 2241 case 1: 2242 code = BUILT_IN_TM_STORE_1; 2243 break; 2244 case 2: 2245 code = BUILT_IN_TM_STORE_2; 2246 break; 2247 case 4: 2248 code = BUILT_IN_TM_STORE_4; 2249 break; 2250 case 8: 2251 code = BUILT_IN_TM_STORE_8; 2252 break; 2253 } 2254 } 2255 2256 if (code == END_BUILTINS) 2257 { 2258 fn = targetm.vectorize.builtin_tm_store (type); 2259 if (!fn) 2260 return NULL; 2261 } 2262 else 2263 fn = builtin_decl_explicit (code); 2264 2265 simple_type = TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn)))); 2266 2267 if (TREE_CODE (rhs) == CONSTRUCTOR) 2268 { 2269 /* Handle the easy initialization to zero. */ 2270 if (!CONSTRUCTOR_ELTS (rhs)) 2271 rhs = build_int_cst (simple_type, 0); 2272 else 2273 { 2274 /* ...otherwise punt to the caller and probably use 2275 BUILT_IN_TM_MEMMOVE, because we can't wrap a 2276 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce 2277 valid gimple. */ 2278 return NULL; 2279 } 2280 } 2281 else if (!useless_type_conversion_p (simple_type, type)) 2282 { 2283 gimple g; 2284 tree temp; 2285 2286 temp = create_tmp_reg (simple_type); 2287 t = fold_build1 (VIEW_CONVERT_EXPR, simple_type, rhs); 2288 g = gimple_build_assign (temp, t); 2289 gimple_set_location (g, loc); 2290 gsi_insert_before (gsi, g, GSI_SAME_STMT); 2291 2292 rhs = temp; 2293 } 2294 2295 t = gimplify_addr (gsi, lhs); 2296 gcall = gimple_build_call (fn, 2, t, rhs); 2297 gimple_set_location (gcall, loc); 2298 gsi_insert_before (gsi, gcall, GSI_SAME_STMT); 2299 2300 return gcall; 2301} 2302 2303 2304/* Expand an assignment statement into transactional builtins. */ 2305 2306static void 2307expand_assign_tm (struct tm_region *region, gimple_stmt_iterator *gsi) 2308{ 2309 gimple stmt = gsi_stmt (*gsi); 2310 location_t loc = gimple_location (stmt); 2311 tree lhs = gimple_assign_lhs (stmt); 2312 tree rhs = gimple_assign_rhs1 (stmt); 2313 bool store_p = requires_barrier (region->entry_block, lhs, NULL); 2314 bool load_p = requires_barrier (region->entry_block, rhs, NULL); 2315 gimple gcall = NULL; 2316 2317 if (!load_p && !store_p) 2318 { 2319 /* Add thread private addresses to log if applicable. */ 2320 requires_barrier (region->entry_block, lhs, stmt); 2321 gsi_next (gsi); 2322 return; 2323 } 2324 2325 // Remove original load/store statement. 2326 gsi_remove (gsi, true); 2327 2328 if (load_p && !store_p) 2329 { 2330 transaction_subcode_ior (region, GTMA_HAVE_LOAD); 2331 gcall = build_tm_load (loc, lhs, rhs, gsi); 2332 } 2333 else if (store_p && !load_p) 2334 { 2335 transaction_subcode_ior (region, GTMA_HAVE_STORE); 2336 gcall = build_tm_store (loc, lhs, rhs, gsi); 2337 } 2338 if (!gcall) 2339 { 2340 tree lhs_addr, rhs_addr, tmp; 2341 2342 if (load_p) 2343 transaction_subcode_ior (region, GTMA_HAVE_LOAD); 2344 if (store_p) 2345 transaction_subcode_ior (region, GTMA_HAVE_STORE); 2346 2347 /* ??? Figure out if there's any possible overlap between the LHS 2348 and the RHS and if not, use MEMCPY. */ 2349 2350 if (load_p && is_gimple_reg (lhs)) 2351 { 2352 tmp = create_tmp_var (TREE_TYPE (lhs)); 2353 lhs_addr = build_fold_addr_expr (tmp); 2354 } 2355 else 2356 { 2357 tmp = NULL_TREE; 2358 lhs_addr = gimplify_addr (gsi, lhs); 2359 } 2360 rhs_addr = gimplify_addr (gsi, rhs); 2361 gcall = gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_MEMMOVE), 2362 3, lhs_addr, rhs_addr, 2363 TYPE_SIZE_UNIT (TREE_TYPE (lhs))); 2364 gimple_set_location (gcall, loc); 2365 gsi_insert_before (gsi, gcall, GSI_SAME_STMT); 2366 2367 if (tmp) 2368 { 2369 gcall = gimple_build_assign (lhs, tmp); 2370 gsi_insert_before (gsi, gcall, GSI_SAME_STMT); 2371 } 2372 } 2373 2374 /* Now that we have the load/store in its instrumented form, add 2375 thread private addresses to the log if applicable. */ 2376 if (!store_p) 2377 requires_barrier (region->entry_block, lhs, gcall); 2378 2379 // The calls to build_tm_{store,load} above inserted the instrumented 2380 // call into the stream. 2381 // gsi_insert_before (gsi, gcall, GSI_SAME_STMT); 2382} 2383 2384 2385/* Expand a call statement as appropriate for a transaction. That is, 2386 either verify that the call does not affect the transaction, or 2387 redirect the call to a clone that handles transactions, or change 2388 the transaction state to IRREVOCABLE. Return true if the call is 2389 one of the builtins that end a transaction. */ 2390 2391static bool 2392expand_call_tm (struct tm_region *region, 2393 gimple_stmt_iterator *gsi) 2394{ 2395 gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi)); 2396 tree lhs = gimple_call_lhs (stmt); 2397 tree fn_decl; 2398 struct cgraph_node *node; 2399 bool retval = false; 2400 2401 fn_decl = gimple_call_fndecl (stmt); 2402 2403 if (fn_decl == builtin_decl_explicit (BUILT_IN_TM_MEMCPY) 2404 || fn_decl == builtin_decl_explicit (BUILT_IN_TM_MEMMOVE)) 2405 transaction_subcode_ior (region, GTMA_HAVE_STORE | GTMA_HAVE_LOAD); 2406 if (fn_decl == builtin_decl_explicit (BUILT_IN_TM_MEMSET)) 2407 transaction_subcode_ior (region, GTMA_HAVE_STORE); 2408 2409 if (is_tm_pure_call (stmt)) 2410 return false; 2411 2412 if (fn_decl) 2413 retval = is_tm_ending_fndecl (fn_decl); 2414 if (!retval) 2415 { 2416 /* Assume all non-const/pure calls write to memory, except 2417 transaction ending builtins. */ 2418 transaction_subcode_ior (region, GTMA_HAVE_STORE); 2419 } 2420 2421 /* For indirect calls, we already generated a call into the runtime. */ 2422 if (!fn_decl) 2423 { 2424 tree fn = gimple_call_fn (stmt); 2425 2426 /* We are guaranteed never to go irrevocable on a safe or pure 2427 call, and the pure call was handled above. */ 2428 if (is_tm_safe (fn)) 2429 return false; 2430 else 2431 transaction_subcode_ior (region, GTMA_MAY_ENTER_IRREVOCABLE); 2432 2433 return false; 2434 } 2435 2436 node = cgraph_node::get (fn_decl); 2437 /* All calls should have cgraph here. */ 2438 if (!node) 2439 { 2440 /* We can have a nodeless call here if some pass after IPA-tm 2441 added uninstrumented calls. For example, loop distribution 2442 can transform certain loop constructs into __builtin_mem* 2443 calls. In this case, see if we have a suitable TM 2444 replacement and fill in the gaps. */ 2445 gcc_assert (DECL_BUILT_IN_CLASS (fn_decl) == BUILT_IN_NORMAL); 2446 enum built_in_function code = DECL_FUNCTION_CODE (fn_decl); 2447 gcc_assert (code == BUILT_IN_MEMCPY 2448 || code == BUILT_IN_MEMMOVE 2449 || code == BUILT_IN_MEMSET); 2450 2451 tree repl = find_tm_replacement_function (fn_decl); 2452 if (repl) 2453 { 2454 gimple_call_set_fndecl (stmt, repl); 2455 update_stmt (stmt); 2456 node = cgraph_node::create (repl); 2457 node->local.tm_may_enter_irr = false; 2458 return expand_call_tm (region, gsi); 2459 } 2460 gcc_unreachable (); 2461 } 2462 if (node->local.tm_may_enter_irr) 2463 transaction_subcode_ior (region, GTMA_MAY_ENTER_IRREVOCABLE); 2464 2465 if (is_tm_abort (fn_decl)) 2466 { 2467 transaction_subcode_ior (region, GTMA_HAVE_ABORT); 2468 return true; 2469 } 2470 2471 /* Instrument the store if needed. 2472 2473 If the assignment happens inside the function call (return slot 2474 optimization), there is no instrumentation to be done, since 2475 the callee should have done the right thing. */ 2476 if (lhs && requires_barrier (region->entry_block, lhs, stmt) 2477 && !gimple_call_return_slot_opt_p (stmt)) 2478 { 2479 tree tmp = create_tmp_reg (TREE_TYPE (lhs)); 2480 location_t loc = gimple_location (stmt); 2481 edge fallthru_edge = NULL; 2482 gassign *assign_stmt; 2483 2484 /* Remember if the call was going to throw. */ 2485 if (stmt_can_throw_internal (stmt)) 2486 { 2487 edge_iterator ei; 2488 edge e; 2489 basic_block bb = gimple_bb (stmt); 2490 2491 FOR_EACH_EDGE (e, ei, bb->succs) 2492 if (e->flags & EDGE_FALLTHRU) 2493 { 2494 fallthru_edge = e; 2495 break; 2496 } 2497 } 2498 2499 gimple_call_set_lhs (stmt, tmp); 2500 update_stmt (stmt); 2501 assign_stmt = gimple_build_assign (lhs, tmp); 2502 gimple_set_location (assign_stmt, loc); 2503 2504 /* We cannot throw in the middle of a BB. If the call was going 2505 to throw, place the instrumentation on the fallthru edge, so 2506 the call remains the last statement in the block. */ 2507 if (fallthru_edge) 2508 { 2509 gimple_seq fallthru_seq = gimple_seq_alloc_with_stmt (assign_stmt); 2510 gimple_stmt_iterator fallthru_gsi = gsi_start (fallthru_seq); 2511 expand_assign_tm (region, &fallthru_gsi); 2512 gsi_insert_seq_on_edge (fallthru_edge, fallthru_seq); 2513 pending_edge_inserts_p = true; 2514 } 2515 else 2516 { 2517 gsi_insert_after (gsi, assign_stmt, GSI_CONTINUE_LINKING); 2518 expand_assign_tm (region, gsi); 2519 } 2520 2521 transaction_subcode_ior (region, GTMA_HAVE_STORE); 2522 } 2523 2524 return retval; 2525} 2526 2527 2528/* Expand all statements in BB as appropriate for being inside 2529 a transaction. */ 2530 2531static void 2532expand_block_tm (struct tm_region *region, basic_block bb) 2533{ 2534 gimple_stmt_iterator gsi; 2535 2536 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); ) 2537 { 2538 gimple stmt = gsi_stmt (gsi); 2539 switch (gimple_code (stmt)) 2540 { 2541 case GIMPLE_ASSIGN: 2542 /* Only memory reads/writes need to be instrumented. */ 2543 if (gimple_assign_single_p (stmt) 2544 && !gimple_clobber_p (stmt)) 2545 { 2546 expand_assign_tm (region, &gsi); 2547 continue; 2548 } 2549 break; 2550 2551 case GIMPLE_CALL: 2552 if (expand_call_tm (region, &gsi)) 2553 return; 2554 break; 2555 2556 case GIMPLE_ASM: 2557 gcc_unreachable (); 2558 2559 default: 2560 break; 2561 } 2562 if (!gsi_end_p (gsi)) 2563 gsi_next (&gsi); 2564 } 2565} 2566 2567/* Return the list of basic-blocks in REGION. 2568 2569 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks 2570 following a TM_IRREVOCABLE call. 2571 2572 INCLUDE_UNINSTRUMENTED_P is TRUE if we should include the 2573 uninstrumented code path blocks in the list of basic blocks 2574 returned, false otherwise. */ 2575 2576static vec<basic_block> 2577get_tm_region_blocks (basic_block entry_block, 2578 bitmap exit_blocks, 2579 bitmap irr_blocks, 2580 bitmap all_region_blocks, 2581 bool stop_at_irrevocable_p, 2582 bool include_uninstrumented_p = true) 2583{ 2584 vec<basic_block> bbs = vNULL; 2585 unsigned i; 2586 edge e; 2587 edge_iterator ei; 2588 bitmap visited_blocks = BITMAP_ALLOC (NULL); 2589 2590 i = 0; 2591 bbs.safe_push (entry_block); 2592 bitmap_set_bit (visited_blocks, entry_block->index); 2593 2594 do 2595 { 2596 basic_block bb = bbs[i++]; 2597 2598 if (exit_blocks && 2599 bitmap_bit_p (exit_blocks, bb->index)) 2600 continue; 2601 2602 if (stop_at_irrevocable_p 2603 && irr_blocks 2604 && bitmap_bit_p (irr_blocks, bb->index)) 2605 continue; 2606 2607 FOR_EACH_EDGE (e, ei, bb->succs) 2608 if ((include_uninstrumented_p 2609 || !(e->flags & EDGE_TM_UNINSTRUMENTED)) 2610 && !bitmap_bit_p (visited_blocks, e->dest->index)) 2611 { 2612 bitmap_set_bit (visited_blocks, e->dest->index); 2613 bbs.safe_push (e->dest); 2614 } 2615 } 2616 while (i < bbs.length ()); 2617 2618 if (all_region_blocks) 2619 bitmap_ior_into (all_region_blocks, visited_blocks); 2620 2621 BITMAP_FREE (visited_blocks); 2622 return bbs; 2623} 2624 2625// Callback data for collect_bb2reg. 2626struct bb2reg_stuff 2627{ 2628 vec<tm_region_p> *bb2reg; 2629 bool include_uninstrumented_p; 2630}; 2631 2632// Callback for expand_regions, collect innermost region data for each bb. 2633static void * 2634collect_bb2reg (struct tm_region *region, void *data) 2635{ 2636 struct bb2reg_stuff *stuff = (struct bb2reg_stuff *)data; 2637 vec<tm_region_p> *bb2reg = stuff->bb2reg; 2638 vec<basic_block> queue; 2639 unsigned int i; 2640 basic_block bb; 2641 2642 queue = get_tm_region_blocks (region->entry_block, 2643 region->exit_blocks, 2644 region->irr_blocks, 2645 NULL, 2646 /*stop_at_irr_p=*/true, 2647 stuff->include_uninstrumented_p); 2648 2649 // We expect expand_region to perform a post-order traversal of the region 2650 // tree. Therefore the last region seen for any bb is the innermost. 2651 FOR_EACH_VEC_ELT (queue, i, bb) 2652 (*bb2reg)[bb->index] = region; 2653 2654 queue.release (); 2655 return NULL; 2656} 2657 2658// Returns a vector, indexed by BB->INDEX, of the innermost tm_region to 2659// which a basic block belongs. Note that we only consider the instrumented 2660// code paths for the region; the uninstrumented code paths are ignored if 2661// INCLUDE_UNINSTRUMENTED_P is false. 2662// 2663// ??? This data is very similar to the bb_regions array that is collected 2664// during tm_region_init. Or, rather, this data is similar to what could 2665// be used within tm_region_init. The actual computation in tm_region_init 2666// begins and ends with bb_regions entirely full of NULL pointers, due to 2667// the way in which pointers are swapped in and out of the array. 2668// 2669// ??? Our callers expect that blocks are not shared between transactions. 2670// When the optimizers get too smart, and blocks are shared, then during 2671// the tm_mark phase we'll add log entries to only one of the two transactions, 2672// and in the tm_edge phase we'll add edges to the CFG that create invalid 2673// cycles. The symptom being SSA defs that do not dominate their uses. 2674// Note that the optimizers were locally correct with their transformation, 2675// as we have no info within the program that suggests that the blocks cannot 2676// be shared. 2677// 2678// ??? There is currently a hack inside tree-ssa-pre.c to work around the 2679// only known instance of this block sharing. 2680 2681static vec<tm_region_p> 2682get_bb_regions_instrumented (bool traverse_clones, 2683 bool include_uninstrumented_p) 2684{ 2685 unsigned n = last_basic_block_for_fn (cfun); 2686 struct bb2reg_stuff stuff; 2687 vec<tm_region_p> ret; 2688 2689 ret.create (n); 2690 ret.safe_grow_cleared (n); 2691 stuff.bb2reg = &ret; 2692 stuff.include_uninstrumented_p = include_uninstrumented_p; 2693 expand_regions (all_tm_regions, collect_bb2reg, &stuff, traverse_clones); 2694 2695 return ret; 2696} 2697 2698/* Set the IN_TRANSACTION for all gimple statements that appear in a 2699 transaction. */ 2700 2701void 2702compute_transaction_bits (void) 2703{ 2704 struct tm_region *region; 2705 vec<basic_block> queue; 2706 unsigned int i; 2707 basic_block bb; 2708 2709 /* ?? Perhaps we need to abstract gate_tm_init further, because we 2710 certainly don't need it to calculate CDI_DOMINATOR info. */ 2711 gate_tm_init (); 2712 2713 FOR_EACH_BB_FN (bb, cfun) 2714 bb->flags &= ~BB_IN_TRANSACTION; 2715 2716 for (region = all_tm_regions; region; region = region->next) 2717 { 2718 queue = get_tm_region_blocks (region->entry_block, 2719 region->exit_blocks, 2720 region->irr_blocks, 2721 NULL, 2722 /*stop_at_irr_p=*/true); 2723 for (i = 0; queue.iterate (i, &bb); ++i) 2724 bb->flags |= BB_IN_TRANSACTION; 2725 queue.release (); 2726 } 2727 2728 if (all_tm_regions) 2729 bitmap_obstack_release (&tm_obstack); 2730} 2731 2732/* Replace the GIMPLE_TRANSACTION in this region with the corresponding 2733 call to BUILT_IN_TM_START. */ 2734 2735static void * 2736expand_transaction (struct tm_region *region, void *data ATTRIBUTE_UNUSED) 2737{ 2738 tree tm_start = builtin_decl_explicit (BUILT_IN_TM_START); 2739 basic_block transaction_bb = gimple_bb (region->transaction_stmt); 2740 tree tm_state = region->tm_state; 2741 tree tm_state_type = TREE_TYPE (tm_state); 2742 edge abort_edge = NULL; 2743 edge inst_edge = NULL; 2744 edge uninst_edge = NULL; 2745 edge fallthru_edge = NULL; 2746 2747 // Identify the various successors of the transaction start. 2748 { 2749 edge_iterator i; 2750 edge e; 2751 FOR_EACH_EDGE (e, i, transaction_bb->succs) 2752 { 2753 if (e->flags & EDGE_TM_ABORT) 2754 abort_edge = e; 2755 else if (e->flags & EDGE_TM_UNINSTRUMENTED) 2756 uninst_edge = e; 2757 else 2758 inst_edge = e; 2759 if (e->flags & EDGE_FALLTHRU) 2760 fallthru_edge = e; 2761 } 2762 } 2763 2764 /* ??? There are plenty of bits here we're not computing. */ 2765 { 2766 int subcode = gimple_transaction_subcode (region->get_transaction_stmt ()); 2767 int flags = 0; 2768 if (subcode & GTMA_DOES_GO_IRREVOCABLE) 2769 flags |= PR_DOESGOIRREVOCABLE; 2770 if ((subcode & GTMA_MAY_ENTER_IRREVOCABLE) == 0) 2771 flags |= PR_HASNOIRREVOCABLE; 2772 /* If the transaction does not have an abort in lexical scope and is not 2773 marked as an outer transaction, then it will never abort. */ 2774 if ((subcode & GTMA_HAVE_ABORT) == 0 && (subcode & GTMA_IS_OUTER) == 0) 2775 flags |= PR_HASNOABORT; 2776 if ((subcode & GTMA_HAVE_STORE) == 0) 2777 flags |= PR_READONLY; 2778 if (inst_edge && !(subcode & GTMA_HAS_NO_INSTRUMENTATION)) 2779 flags |= PR_INSTRUMENTEDCODE; 2780 if (uninst_edge) 2781 flags |= PR_UNINSTRUMENTEDCODE; 2782 if (subcode & GTMA_IS_OUTER) 2783 region->original_transaction_was_outer = true; 2784 tree t = build_int_cst (tm_state_type, flags); 2785 gcall *call = gimple_build_call (tm_start, 1, t); 2786 gimple_call_set_lhs (call, tm_state); 2787 gimple_set_location (call, gimple_location (region->transaction_stmt)); 2788 2789 // Replace the GIMPLE_TRANSACTION with the call to BUILT_IN_TM_START. 2790 gimple_stmt_iterator gsi = gsi_last_bb (transaction_bb); 2791 gcc_assert (gsi_stmt (gsi) == region->transaction_stmt); 2792 gsi_insert_before (&gsi, call, GSI_SAME_STMT); 2793 gsi_remove (&gsi, true); 2794 region->transaction_stmt = call; 2795 } 2796 2797 // Generate log saves. 2798 if (!tm_log_save_addresses.is_empty ()) 2799 tm_log_emit_saves (region->entry_block, transaction_bb); 2800 2801 // In the beginning, we've no tests to perform on transaction restart. 2802 // Note that after this point, transaction_bb becomes the "most recent 2803 // block containing tests for the transaction". 2804 region->restart_block = region->entry_block; 2805 2806 // Generate log restores. 2807 if (!tm_log_save_addresses.is_empty ()) 2808 { 2809 basic_block test_bb = create_empty_bb (transaction_bb); 2810 basic_block code_bb = create_empty_bb (test_bb); 2811 basic_block join_bb = create_empty_bb (code_bb); 2812 add_bb_to_loop (test_bb, transaction_bb->loop_father); 2813 add_bb_to_loop (code_bb, transaction_bb->loop_father); 2814 add_bb_to_loop (join_bb, transaction_bb->loop_father); 2815 if (region->restart_block == region->entry_block) 2816 region->restart_block = test_bb; 2817 2818 tree t1 = create_tmp_reg (tm_state_type); 2819 tree t2 = build_int_cst (tm_state_type, A_RESTORELIVEVARIABLES); 2820 gimple stmt = gimple_build_assign (t1, BIT_AND_EXPR, tm_state, t2); 2821 gimple_stmt_iterator gsi = gsi_last_bb (test_bb); 2822 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 2823 2824 t2 = build_int_cst (tm_state_type, 0); 2825 stmt = gimple_build_cond (NE_EXPR, t1, t2, NULL, NULL); 2826 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 2827 2828 tm_log_emit_restores (region->entry_block, code_bb); 2829 2830 edge ei = make_edge (transaction_bb, test_bb, EDGE_FALLTHRU); 2831 edge et = make_edge (test_bb, code_bb, EDGE_TRUE_VALUE); 2832 edge ef = make_edge (test_bb, join_bb, EDGE_FALSE_VALUE); 2833 redirect_edge_pred (fallthru_edge, join_bb); 2834 2835 join_bb->frequency = test_bb->frequency = transaction_bb->frequency; 2836 join_bb->count = test_bb->count = transaction_bb->count; 2837 2838 ei->probability = PROB_ALWAYS; 2839 et->probability = PROB_LIKELY; 2840 ef->probability = PROB_UNLIKELY; 2841 et->count = apply_probability (test_bb->count, et->probability); 2842 ef->count = apply_probability (test_bb->count, ef->probability); 2843 2844 code_bb->count = et->count; 2845 code_bb->frequency = EDGE_FREQUENCY (et); 2846 2847 transaction_bb = join_bb; 2848 } 2849 2850 // If we have an ABORT edge, create a test to perform the abort. 2851 if (abort_edge) 2852 { 2853 basic_block test_bb = create_empty_bb (transaction_bb); 2854 add_bb_to_loop (test_bb, transaction_bb->loop_father); 2855 if (region->restart_block == region->entry_block) 2856 region->restart_block = test_bb; 2857 2858 tree t1 = create_tmp_reg (tm_state_type); 2859 tree t2 = build_int_cst (tm_state_type, A_ABORTTRANSACTION); 2860 gimple stmt = gimple_build_assign (t1, BIT_AND_EXPR, tm_state, t2); 2861 gimple_stmt_iterator gsi = gsi_last_bb (test_bb); 2862 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 2863 2864 t2 = build_int_cst (tm_state_type, 0); 2865 stmt = gimple_build_cond (NE_EXPR, t1, t2, NULL, NULL); 2866 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 2867 2868 edge ei = make_edge (transaction_bb, test_bb, EDGE_FALLTHRU); 2869 test_bb->frequency = transaction_bb->frequency; 2870 test_bb->count = transaction_bb->count; 2871 ei->probability = PROB_ALWAYS; 2872 2873 // Not abort edge. If both are live, chose one at random as we'll 2874 // we'll be fixing that up below. 2875 redirect_edge_pred (fallthru_edge, test_bb); 2876 fallthru_edge->flags = EDGE_FALSE_VALUE; 2877 fallthru_edge->probability = PROB_VERY_LIKELY; 2878 fallthru_edge->count 2879 = apply_probability (test_bb->count, fallthru_edge->probability); 2880 2881 // Abort/over edge. 2882 redirect_edge_pred (abort_edge, test_bb); 2883 abort_edge->flags = EDGE_TRUE_VALUE; 2884 abort_edge->probability = PROB_VERY_UNLIKELY; 2885 abort_edge->count 2886 = apply_probability (test_bb->count, abort_edge->probability); 2887 2888 transaction_bb = test_bb; 2889 } 2890 2891 // If we have both instrumented and uninstrumented code paths, select one. 2892 if (inst_edge && uninst_edge) 2893 { 2894 basic_block test_bb = create_empty_bb (transaction_bb); 2895 add_bb_to_loop (test_bb, transaction_bb->loop_father); 2896 if (region->restart_block == region->entry_block) 2897 region->restart_block = test_bb; 2898 2899 tree t1 = create_tmp_reg (tm_state_type); 2900 tree t2 = build_int_cst (tm_state_type, A_RUNUNINSTRUMENTEDCODE); 2901 2902 gimple stmt = gimple_build_assign (t1, BIT_AND_EXPR, tm_state, t2); 2903 gimple_stmt_iterator gsi = gsi_last_bb (test_bb); 2904 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 2905 2906 t2 = build_int_cst (tm_state_type, 0); 2907 stmt = gimple_build_cond (NE_EXPR, t1, t2, NULL, NULL); 2908 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 2909 2910 // Create the edge into test_bb first, as we want to copy values 2911 // out of the fallthru edge. 2912 edge e = make_edge (transaction_bb, test_bb, fallthru_edge->flags); 2913 e->probability = fallthru_edge->probability; 2914 test_bb->count = e->count = fallthru_edge->count; 2915 test_bb->frequency = EDGE_FREQUENCY (e); 2916 2917 // Now update the edges to the inst/uninist implementations. 2918 // For now assume that the paths are equally likely. When using HTM, 2919 // we'll try the uninst path first and fallback to inst path if htm 2920 // buffers are exceeded. Without HTM we start with the inst path and 2921 // use the uninst path when falling back to serial mode. 2922 redirect_edge_pred (inst_edge, test_bb); 2923 inst_edge->flags = EDGE_FALSE_VALUE; 2924 inst_edge->probability = REG_BR_PROB_BASE / 2; 2925 inst_edge->count 2926 = apply_probability (test_bb->count, inst_edge->probability); 2927 2928 redirect_edge_pred (uninst_edge, test_bb); 2929 uninst_edge->flags = EDGE_TRUE_VALUE; 2930 uninst_edge->probability = REG_BR_PROB_BASE / 2; 2931 uninst_edge->count 2932 = apply_probability (test_bb->count, uninst_edge->probability); 2933 } 2934 2935 // If we have no previous special cases, and we have PHIs at the beginning 2936 // of the atomic region, this means we have a loop at the beginning of the 2937 // atomic region that shares the first block. This can cause problems with 2938 // the transaction restart abnormal edges to be added in the tm_edges pass. 2939 // Solve this by adding a new empty block to receive the abnormal edges. 2940 if (region->restart_block == region->entry_block 2941 && phi_nodes (region->entry_block)) 2942 { 2943 basic_block empty_bb = create_empty_bb (transaction_bb); 2944 region->restart_block = empty_bb; 2945 add_bb_to_loop (empty_bb, transaction_bb->loop_father); 2946 2947 redirect_edge_pred (fallthru_edge, empty_bb); 2948 make_edge (transaction_bb, empty_bb, EDGE_FALLTHRU); 2949 } 2950 2951 return NULL; 2952} 2953 2954/* Generate the temporary to be used for the return value of 2955 BUILT_IN_TM_START. */ 2956 2957static void * 2958generate_tm_state (struct tm_region *region, void *data ATTRIBUTE_UNUSED) 2959{ 2960 tree tm_start = builtin_decl_explicit (BUILT_IN_TM_START); 2961 region->tm_state = 2962 create_tmp_reg (TREE_TYPE (TREE_TYPE (tm_start)), "tm_state"); 2963 2964 // Reset the subcode, post optimizations. We'll fill this in 2965 // again as we process blocks. 2966 if (region->exit_blocks) 2967 { 2968 gtransaction *transaction_stmt = region->get_transaction_stmt (); 2969 unsigned int subcode = gimple_transaction_subcode (transaction_stmt); 2970 2971 if (subcode & GTMA_DOES_GO_IRREVOCABLE) 2972 subcode &= (GTMA_DECLARATION_MASK | GTMA_DOES_GO_IRREVOCABLE 2973 | GTMA_MAY_ENTER_IRREVOCABLE 2974 | GTMA_HAS_NO_INSTRUMENTATION); 2975 else 2976 subcode &= GTMA_DECLARATION_MASK; 2977 gimple_transaction_set_subcode (transaction_stmt, subcode); 2978 } 2979 2980 return NULL; 2981} 2982 2983// Propagate flags from inner transactions outwards. 2984static void 2985propagate_tm_flags_out (struct tm_region *region) 2986{ 2987 if (region == NULL) 2988 return; 2989 propagate_tm_flags_out (region->inner); 2990 2991 if (region->outer && region->outer->transaction_stmt) 2992 { 2993 unsigned s 2994 = gimple_transaction_subcode (region->get_transaction_stmt ()); 2995 s &= (GTMA_HAVE_ABORT | GTMA_HAVE_LOAD | GTMA_HAVE_STORE 2996 | GTMA_MAY_ENTER_IRREVOCABLE); 2997 s |= gimple_transaction_subcode (region->outer->get_transaction_stmt ()); 2998 gimple_transaction_set_subcode (region->outer->get_transaction_stmt (), 2999 s); 3000 } 3001 3002 propagate_tm_flags_out (region->next); 3003} 3004 3005/* Entry point to the MARK phase of TM expansion. Here we replace 3006 transactional memory statements with calls to builtins, and function 3007 calls with their transactional clones (if available). But we don't 3008 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */ 3009 3010static unsigned int 3011execute_tm_mark (void) 3012{ 3013 pending_edge_inserts_p = false; 3014 3015 expand_regions (all_tm_regions, generate_tm_state, NULL, 3016 /*traverse_clones=*/true); 3017 3018 tm_log_init (); 3019 3020 vec<tm_region_p> bb_regions 3021 = get_bb_regions_instrumented (/*traverse_clones=*/true, 3022 /*include_uninstrumented_p=*/false); 3023 struct tm_region *r; 3024 unsigned i; 3025 3026 // Expand memory operations into calls into the runtime. 3027 // This collects log entries as well. 3028 FOR_EACH_VEC_ELT (bb_regions, i, r) 3029 { 3030 if (r != NULL) 3031 { 3032 if (r->transaction_stmt) 3033 { 3034 unsigned sub 3035 = gimple_transaction_subcode (r->get_transaction_stmt ()); 3036 3037 /* If we're sure to go irrevocable, there won't be 3038 anything to expand, since the run-time will go 3039 irrevocable right away. */ 3040 if (sub & GTMA_DOES_GO_IRREVOCABLE 3041 && sub & GTMA_MAY_ENTER_IRREVOCABLE) 3042 continue; 3043 } 3044 expand_block_tm (r, BASIC_BLOCK_FOR_FN (cfun, i)); 3045 } 3046 } 3047 3048 bb_regions.release (); 3049 3050 // Propagate flags from inner transactions outwards. 3051 propagate_tm_flags_out (all_tm_regions); 3052 3053 // Expand GIMPLE_TRANSACTIONs into calls into the runtime. 3054 expand_regions (all_tm_regions, expand_transaction, NULL, 3055 /*traverse_clones=*/false); 3056 3057 tm_log_emit (); 3058 tm_log_delete (); 3059 3060 if (pending_edge_inserts_p) 3061 gsi_commit_edge_inserts (); 3062 free_dominance_info (CDI_DOMINATORS); 3063 return 0; 3064} 3065 3066namespace { 3067 3068const pass_data pass_data_tm_mark = 3069{ 3070 GIMPLE_PASS, /* type */ 3071 "tmmark", /* name */ 3072 OPTGROUP_NONE, /* optinfo_flags */ 3073 TV_TRANS_MEM, /* tv_id */ 3074 ( PROP_ssa | PROP_cfg ), /* properties_required */ 3075 0, /* properties_provided */ 3076 0, /* properties_destroyed */ 3077 0, /* todo_flags_start */ 3078 TODO_update_ssa, /* todo_flags_finish */ 3079}; 3080 3081class pass_tm_mark : public gimple_opt_pass 3082{ 3083public: 3084 pass_tm_mark (gcc::context *ctxt) 3085 : gimple_opt_pass (pass_data_tm_mark, ctxt) 3086 {} 3087 3088 /* opt_pass methods: */ 3089 virtual unsigned int execute (function *) { return execute_tm_mark (); } 3090 3091}; // class pass_tm_mark 3092 3093} // anon namespace 3094 3095gimple_opt_pass * 3096make_pass_tm_mark (gcc::context *ctxt) 3097{ 3098 return new pass_tm_mark (ctxt); 3099} 3100 3101 3102/* Create an abnormal edge from STMT at iter, splitting the block 3103 as necessary. Adjust *PNEXT as needed for the split block. */ 3104 3105static inline void 3106split_bb_make_tm_edge (gimple stmt, basic_block dest_bb, 3107 gimple_stmt_iterator iter, gimple_stmt_iterator *pnext) 3108{ 3109 basic_block bb = gimple_bb (stmt); 3110 if (!gsi_one_before_end_p (iter)) 3111 { 3112 edge e = split_block (bb, stmt); 3113 *pnext = gsi_start_bb (e->dest); 3114 } 3115 make_edge (bb, dest_bb, EDGE_ABNORMAL); 3116 3117 // Record the need for the edge for the benefit of the rtl passes. 3118 if (cfun->gimple_df->tm_restart == NULL) 3119 cfun->gimple_df->tm_restart 3120 = hash_table<tm_restart_hasher>::create_ggc (31); 3121 3122 struct tm_restart_node dummy; 3123 dummy.stmt = stmt; 3124 dummy.label_or_list = gimple_block_label (dest_bb); 3125 3126 tm_restart_node **slot = cfun->gimple_df->tm_restart->find_slot (&dummy, 3127 INSERT); 3128 struct tm_restart_node *n = *slot; 3129 if (n == NULL) 3130 { 3131 n = ggc_alloc<tm_restart_node> (); 3132 *n = dummy; 3133 } 3134 else 3135 { 3136 tree old = n->label_or_list; 3137 if (TREE_CODE (old) == LABEL_DECL) 3138 old = tree_cons (NULL, old, NULL); 3139 n->label_or_list = tree_cons (NULL, dummy.label_or_list, old); 3140 } 3141} 3142 3143/* Split block BB as necessary for every builtin function we added, and 3144 wire up the abnormal back edges implied by the transaction restart. */ 3145 3146static void 3147expand_block_edges (struct tm_region *const region, basic_block bb) 3148{ 3149 gimple_stmt_iterator gsi, next_gsi; 3150 3151 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi = next_gsi) 3152 { 3153 gimple stmt = gsi_stmt (gsi); 3154 gcall *call_stmt; 3155 3156 next_gsi = gsi; 3157 gsi_next (&next_gsi); 3158 3159 // ??? Shouldn't we split for any non-pure, non-irrevocable function? 3160 call_stmt = dyn_cast <gcall *> (stmt); 3161 if ((!call_stmt) 3162 || (gimple_call_flags (call_stmt) & ECF_TM_BUILTIN) == 0) 3163 continue; 3164 3165 if (DECL_FUNCTION_CODE (gimple_call_fndecl (call_stmt)) 3166 == BUILT_IN_TM_ABORT) 3167 { 3168 // If we have a ``_transaction_cancel [[outer]]'', there is only 3169 // one abnormal edge: to the transaction marked OUTER. 3170 // All compiler-generated instances of BUILT_IN_TM_ABORT have a 3171 // constant argument, which we can examine here. Users invoking 3172 // TM_ABORT directly get what they deserve. 3173 tree arg = gimple_call_arg (call_stmt, 0); 3174 if (TREE_CODE (arg) == INTEGER_CST 3175 && (TREE_INT_CST_LOW (arg) & AR_OUTERABORT) != 0 3176 && !decl_is_tm_clone (current_function_decl)) 3177 { 3178 // Find the GTMA_IS_OUTER transaction. 3179 for (struct tm_region *o = region; o; o = o->outer) 3180 if (o->original_transaction_was_outer) 3181 { 3182 split_bb_make_tm_edge (call_stmt, o->restart_block, 3183 gsi, &next_gsi); 3184 break; 3185 } 3186 3187 // Otherwise, the front-end should have semantically checked 3188 // outer aborts, but in either case the target region is not 3189 // within this function. 3190 continue; 3191 } 3192 3193 // Non-outer, TM aborts have an abnormal edge to the inner-most 3194 // transaction, the one being aborted; 3195 split_bb_make_tm_edge (call_stmt, region->restart_block, gsi, 3196 &next_gsi); 3197 } 3198 3199 // All TM builtins have an abnormal edge to the outer-most transaction. 3200 // We never restart inner transactions. For tm clones, we know a-priori 3201 // that the outer-most transaction is outside the function. 3202 if (decl_is_tm_clone (current_function_decl)) 3203 continue; 3204 3205 if (cfun->gimple_df->tm_restart == NULL) 3206 cfun->gimple_df->tm_restart 3207 = hash_table<tm_restart_hasher>::create_ggc (31); 3208 3209 // All TM builtins have an abnormal edge to the outer-most transaction. 3210 // We never restart inner transactions. 3211 for (struct tm_region *o = region; o; o = o->outer) 3212 if (!o->outer) 3213 { 3214 split_bb_make_tm_edge (call_stmt, o->restart_block, gsi, &next_gsi); 3215 break; 3216 } 3217 3218 // Delete any tail-call annotation that may have been added. 3219 // The tail-call pass may have mis-identified the commit as being 3220 // a candidate because we had not yet added this restart edge. 3221 gimple_call_set_tail (call_stmt, false); 3222 } 3223} 3224 3225/* Entry point to the final expansion of transactional nodes. */ 3226 3227namespace { 3228 3229const pass_data pass_data_tm_edges = 3230{ 3231 GIMPLE_PASS, /* type */ 3232 "tmedge", /* name */ 3233 OPTGROUP_NONE, /* optinfo_flags */ 3234 TV_TRANS_MEM, /* tv_id */ 3235 ( PROP_ssa | PROP_cfg ), /* properties_required */ 3236 0, /* properties_provided */ 3237 0, /* properties_destroyed */ 3238 0, /* todo_flags_start */ 3239 TODO_update_ssa, /* todo_flags_finish */ 3240}; 3241 3242class pass_tm_edges : public gimple_opt_pass 3243{ 3244public: 3245 pass_tm_edges (gcc::context *ctxt) 3246 : gimple_opt_pass (pass_data_tm_edges, ctxt) 3247 {} 3248 3249 /* opt_pass methods: */ 3250 virtual unsigned int execute (function *); 3251 3252}; // class pass_tm_edges 3253 3254unsigned int 3255pass_tm_edges::execute (function *fun) 3256{ 3257 vec<tm_region_p> bb_regions 3258 = get_bb_regions_instrumented (/*traverse_clones=*/false, 3259 /*include_uninstrumented_p=*/true); 3260 struct tm_region *r; 3261 unsigned i; 3262 3263 FOR_EACH_VEC_ELT (bb_regions, i, r) 3264 if (r != NULL) 3265 expand_block_edges (r, BASIC_BLOCK_FOR_FN (fun, i)); 3266 3267 bb_regions.release (); 3268 3269 /* We've got to release the dominance info now, to indicate that it 3270 must be rebuilt completely. Otherwise we'll crash trying to update 3271 the SSA web in the TODO section following this pass. */ 3272 free_dominance_info (CDI_DOMINATORS); 3273 bitmap_obstack_release (&tm_obstack); 3274 all_tm_regions = NULL; 3275 3276 return 0; 3277} 3278 3279} // anon namespace 3280 3281gimple_opt_pass * 3282make_pass_tm_edges (gcc::context *ctxt) 3283{ 3284 return new pass_tm_edges (ctxt); 3285} 3286 3287/* Helper function for expand_regions. Expand REGION and recurse to 3288 the inner region. Call CALLBACK on each region. CALLBACK returns 3289 NULL to continue the traversal, otherwise a non-null value which 3290 this function will return as well. TRAVERSE_CLONES is true if we 3291 should traverse transactional clones. */ 3292 3293static void * 3294expand_regions_1 (struct tm_region *region, 3295 void *(*callback)(struct tm_region *, void *), 3296 void *data, 3297 bool traverse_clones) 3298{ 3299 void *retval = NULL; 3300 if (region->exit_blocks 3301 || (traverse_clones && decl_is_tm_clone (current_function_decl))) 3302 { 3303 retval = callback (region, data); 3304 if (retval) 3305 return retval; 3306 } 3307 if (region->inner) 3308 { 3309 retval = expand_regions (region->inner, callback, data, traverse_clones); 3310 if (retval) 3311 return retval; 3312 } 3313 return retval; 3314} 3315 3316/* Traverse the regions enclosed and including REGION. Execute 3317 CALLBACK for each region, passing DATA. CALLBACK returns NULL to 3318 continue the traversal, otherwise a non-null value which this 3319 function will return as well. TRAVERSE_CLONES is true if we should 3320 traverse transactional clones. */ 3321 3322static void * 3323expand_regions (struct tm_region *region, 3324 void *(*callback)(struct tm_region *, void *), 3325 void *data, 3326 bool traverse_clones) 3327{ 3328 void *retval = NULL; 3329 while (region) 3330 { 3331 retval = expand_regions_1 (region, callback, data, traverse_clones); 3332 if (retval) 3333 return retval; 3334 region = region->next; 3335 } 3336 return retval; 3337} 3338 3339 3340/* A unique TM memory operation. */ 3341typedef struct tm_memop 3342{ 3343 /* Unique ID that all memory operations to the same location have. */ 3344 unsigned int value_id; 3345 /* Address of load/store. */ 3346 tree addr; 3347} *tm_memop_t; 3348 3349/* TM memory operation hashtable helpers. */ 3350 3351struct tm_memop_hasher : typed_free_remove <tm_memop> 3352{ 3353 typedef tm_memop value_type; 3354 typedef tm_memop compare_type; 3355 static inline hashval_t hash (const value_type *); 3356 static inline bool equal (const value_type *, const compare_type *); 3357}; 3358 3359/* Htab support. Return a hash value for a `tm_memop'. */ 3360inline hashval_t 3361tm_memop_hasher::hash (const value_type *mem) 3362{ 3363 tree addr = mem->addr; 3364 /* We drill down to the SSA_NAME/DECL for the hash, but equality is 3365 actually done with operand_equal_p (see tm_memop_eq). */ 3366 if (TREE_CODE (addr) == ADDR_EXPR) 3367 addr = TREE_OPERAND (addr, 0); 3368 return iterative_hash_expr (addr, 0); 3369} 3370 3371/* Htab support. Return true if two tm_memop's are the same. */ 3372inline bool 3373tm_memop_hasher::equal (const value_type *mem1, const compare_type *mem2) 3374{ 3375 return operand_equal_p (mem1->addr, mem2->addr, 0); 3376} 3377 3378/* Sets for solving data flow equations in the memory optimization pass. */ 3379struct tm_memopt_bitmaps 3380{ 3381 /* Stores available to this BB upon entry. Basically, stores that 3382 dominate this BB. */ 3383 bitmap store_avail_in; 3384 /* Stores available at the end of this BB. */ 3385 bitmap store_avail_out; 3386 bitmap store_antic_in; 3387 bitmap store_antic_out; 3388 /* Reads available to this BB upon entry. Basically, reads that 3389 dominate this BB. */ 3390 bitmap read_avail_in; 3391 /* Reads available at the end of this BB. */ 3392 bitmap read_avail_out; 3393 /* Reads performed in this BB. */ 3394 bitmap read_local; 3395 /* Writes performed in this BB. */ 3396 bitmap store_local; 3397 3398 /* Temporary storage for pass. */ 3399 /* Is the current BB in the worklist? */ 3400 bool avail_in_worklist_p; 3401 /* Have we visited this BB? */ 3402 bool visited_p; 3403}; 3404 3405static bitmap_obstack tm_memopt_obstack; 3406 3407/* Unique counter for TM loads and stores. Loads and stores of the 3408 same address get the same ID. */ 3409static unsigned int tm_memopt_value_id; 3410static hash_table<tm_memop_hasher> *tm_memopt_value_numbers; 3411 3412#define STORE_AVAIL_IN(BB) \ 3413 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in 3414#define STORE_AVAIL_OUT(BB) \ 3415 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out 3416#define STORE_ANTIC_IN(BB) \ 3417 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in 3418#define STORE_ANTIC_OUT(BB) \ 3419 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out 3420#define READ_AVAIL_IN(BB) \ 3421 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in 3422#define READ_AVAIL_OUT(BB) \ 3423 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out 3424#define READ_LOCAL(BB) \ 3425 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local 3426#define STORE_LOCAL(BB) \ 3427 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local 3428#define AVAIL_IN_WORKLIST_P(BB) \ 3429 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p 3430#define BB_VISITED_P(BB) \ 3431 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p 3432 3433/* Given a TM load/store in STMT, return the value number for the address 3434 it accesses. */ 3435 3436static unsigned int 3437tm_memopt_value_number (gimple stmt, enum insert_option op) 3438{ 3439 struct tm_memop tmpmem, *mem; 3440 tm_memop **slot; 3441 3442 gcc_assert (is_tm_load (stmt) || is_tm_store (stmt)); 3443 tmpmem.addr = gimple_call_arg (stmt, 0); 3444 slot = tm_memopt_value_numbers->find_slot (&tmpmem, op); 3445 if (*slot) 3446 mem = *slot; 3447 else if (op == INSERT) 3448 { 3449 mem = XNEW (struct tm_memop); 3450 *slot = mem; 3451 mem->value_id = tm_memopt_value_id++; 3452 mem->addr = tmpmem.addr; 3453 } 3454 else 3455 gcc_unreachable (); 3456 return mem->value_id; 3457} 3458 3459/* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */ 3460 3461static void 3462tm_memopt_accumulate_memops (basic_block bb) 3463{ 3464 gimple_stmt_iterator gsi; 3465 3466 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 3467 { 3468 gimple stmt = gsi_stmt (gsi); 3469 bitmap bits; 3470 unsigned int loc; 3471 3472 if (is_tm_store (stmt)) 3473 bits = STORE_LOCAL (bb); 3474 else if (is_tm_load (stmt)) 3475 bits = READ_LOCAL (bb); 3476 else 3477 continue; 3478 3479 loc = tm_memopt_value_number (stmt, INSERT); 3480 bitmap_set_bit (bits, loc); 3481 if (dump_file) 3482 { 3483 fprintf (dump_file, "TM memopt (%s): value num=%d, BB=%d, addr=", 3484 is_tm_load (stmt) ? "LOAD" : "STORE", loc, 3485 gimple_bb (stmt)->index); 3486 print_generic_expr (dump_file, gimple_call_arg (stmt, 0), 0); 3487 fprintf (dump_file, "\n"); 3488 } 3489 } 3490} 3491 3492/* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */ 3493 3494static void 3495dump_tm_memopt_set (const char *set_name, bitmap bits) 3496{ 3497 unsigned i; 3498 bitmap_iterator bi; 3499 const char *comma = ""; 3500 3501 fprintf (dump_file, "TM memopt: %s: [", set_name); 3502 EXECUTE_IF_SET_IN_BITMAP (bits, 0, i, bi) 3503 { 3504 hash_table<tm_memop_hasher>::iterator hi; 3505 struct tm_memop *mem = NULL; 3506 3507 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */ 3508 FOR_EACH_HASH_TABLE_ELEMENT (*tm_memopt_value_numbers, mem, tm_memop_t, hi) 3509 if (mem->value_id == i) 3510 break; 3511 gcc_assert (mem->value_id == i); 3512 fprintf (dump_file, "%s", comma); 3513 comma = ", "; 3514 print_generic_expr (dump_file, mem->addr, 0); 3515 } 3516 fprintf (dump_file, "]\n"); 3517} 3518 3519/* Prettily dump all of the memopt sets in BLOCKS. */ 3520 3521static void 3522dump_tm_memopt_sets (vec<basic_block> blocks) 3523{ 3524 size_t i; 3525 basic_block bb; 3526 3527 for (i = 0; blocks.iterate (i, &bb); ++i) 3528 { 3529 fprintf (dump_file, "------------BB %d---------\n", bb->index); 3530 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb)); 3531 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb)); 3532 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb)); 3533 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb)); 3534 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb)); 3535 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb)); 3536 } 3537} 3538 3539/* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */ 3540 3541static void 3542tm_memopt_compute_avin (basic_block bb) 3543{ 3544 edge e; 3545 unsigned ix; 3546 3547 /* Seed with the AVOUT of any predecessor. */ 3548 for (ix = 0; ix < EDGE_COUNT (bb->preds); ix++) 3549 { 3550 e = EDGE_PRED (bb, ix); 3551 /* Make sure we have already visited this BB, and is thus 3552 initialized. 3553 3554 If e->src->aux is NULL, this predecessor is actually on an 3555 enclosing transaction. We only care about the current 3556 transaction, so ignore it. */ 3557 if (e->src->aux && BB_VISITED_P (e->src)) 3558 { 3559 bitmap_copy (STORE_AVAIL_IN (bb), STORE_AVAIL_OUT (e->src)); 3560 bitmap_copy (READ_AVAIL_IN (bb), READ_AVAIL_OUT (e->src)); 3561 break; 3562 } 3563 } 3564 3565 for (; ix < EDGE_COUNT (bb->preds); ix++) 3566 { 3567 e = EDGE_PRED (bb, ix); 3568 if (e->src->aux && BB_VISITED_P (e->src)) 3569 { 3570 bitmap_and_into (STORE_AVAIL_IN (bb), STORE_AVAIL_OUT (e->src)); 3571 bitmap_and_into (READ_AVAIL_IN (bb), READ_AVAIL_OUT (e->src)); 3572 } 3573 } 3574 3575 BB_VISITED_P (bb) = true; 3576} 3577 3578/* Compute the STORE_ANTIC_IN for the basic block BB. */ 3579 3580static void 3581tm_memopt_compute_antin (basic_block bb) 3582{ 3583 edge e; 3584 unsigned ix; 3585 3586 /* Seed with the ANTIC_OUT of any successor. */ 3587 for (ix = 0; ix < EDGE_COUNT (bb->succs); ix++) 3588 { 3589 e = EDGE_SUCC (bb, ix); 3590 /* Make sure we have already visited this BB, and is thus 3591 initialized. */ 3592 if (BB_VISITED_P (e->dest)) 3593 { 3594 bitmap_copy (STORE_ANTIC_IN (bb), STORE_ANTIC_OUT (e->dest)); 3595 break; 3596 } 3597 } 3598 3599 for (; ix < EDGE_COUNT (bb->succs); ix++) 3600 { 3601 e = EDGE_SUCC (bb, ix); 3602 if (BB_VISITED_P (e->dest)) 3603 bitmap_and_into (STORE_ANTIC_IN (bb), STORE_ANTIC_OUT (e->dest)); 3604 } 3605 3606 BB_VISITED_P (bb) = true; 3607} 3608 3609/* Compute the AVAIL sets for every basic block in BLOCKS. 3610 3611 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows: 3612 3613 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb]) 3614 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors]) 3615 3616 This is basically what we do in lcm's compute_available(), but here 3617 we calculate two sets of sets (one for STOREs and one for READs), 3618 and we work on a region instead of the entire CFG. 3619 3620 REGION is the TM region. 3621 BLOCKS are the basic blocks in the region. */ 3622 3623static void 3624tm_memopt_compute_available (struct tm_region *region, 3625 vec<basic_block> blocks) 3626{ 3627 edge e; 3628 basic_block *worklist, *qin, *qout, *qend, bb; 3629 unsigned int qlen, i; 3630 edge_iterator ei; 3631 bool changed; 3632 3633 /* Allocate a worklist array/queue. Entries are only added to the 3634 list if they were not already on the list. So the size is 3635 bounded by the number of basic blocks in the region. */ 3636 qlen = blocks.length () - 1; 3637 qin = qout = worklist = 3638 XNEWVEC (basic_block, qlen); 3639 3640 /* Put every block in the region on the worklist. */ 3641 for (i = 0; blocks.iterate (i, &bb); ++i) 3642 { 3643 /* Seed AVAIL_OUT with the LOCAL set. */ 3644 bitmap_ior_into (STORE_AVAIL_OUT (bb), STORE_LOCAL (bb)); 3645 bitmap_ior_into (READ_AVAIL_OUT (bb), READ_LOCAL (bb)); 3646 3647 AVAIL_IN_WORKLIST_P (bb) = true; 3648 /* No need to insert the entry block, since it has an AVIN of 3649 null, and an AVOUT that has already been seeded in. */ 3650 if (bb != region->entry_block) 3651 *qin++ = bb; 3652 } 3653 3654 /* The entry block has been initialized with the local sets. */ 3655 BB_VISITED_P (region->entry_block) = true; 3656 3657 qin = worklist; 3658 qend = &worklist[qlen]; 3659 3660 /* Iterate until the worklist is empty. */ 3661 while (qlen) 3662 { 3663 /* Take the first entry off the worklist. */ 3664 bb = *qout++; 3665 qlen--; 3666 3667 if (qout >= qend) 3668 qout = worklist; 3669 3670 /* This block can be added to the worklist again if necessary. */ 3671 AVAIL_IN_WORKLIST_P (bb) = false; 3672 tm_memopt_compute_avin (bb); 3673 3674 /* Note: We do not add the LOCAL sets here because we already 3675 seeded the AVAIL_OUT sets with them. */ 3676 changed = bitmap_ior_into (STORE_AVAIL_OUT (bb), STORE_AVAIL_IN (bb)); 3677 changed |= bitmap_ior_into (READ_AVAIL_OUT (bb), READ_AVAIL_IN (bb)); 3678 if (changed 3679 && (region->exit_blocks == NULL 3680 || !bitmap_bit_p (region->exit_blocks, bb->index))) 3681 /* If the out state of this block changed, then we need to add 3682 its successors to the worklist if they are not already in. */ 3683 FOR_EACH_EDGE (e, ei, bb->succs) 3684 if (!AVAIL_IN_WORKLIST_P (e->dest) 3685 && e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun)) 3686 { 3687 *qin++ = e->dest; 3688 AVAIL_IN_WORKLIST_P (e->dest) = true; 3689 qlen++; 3690 3691 if (qin >= qend) 3692 qin = worklist; 3693 } 3694 } 3695 3696 free (worklist); 3697 3698 if (dump_file) 3699 dump_tm_memopt_sets (blocks); 3700} 3701 3702/* Compute ANTIC sets for every basic block in BLOCKS. 3703 3704 We compute STORE_ANTIC_OUT as follows: 3705 3706 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb]) 3707 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors]) 3708 3709 REGION is the TM region. 3710 BLOCKS are the basic blocks in the region. */ 3711 3712static void 3713tm_memopt_compute_antic (struct tm_region *region, 3714 vec<basic_block> blocks) 3715{ 3716 edge e; 3717 basic_block *worklist, *qin, *qout, *qend, bb; 3718 unsigned int qlen; 3719 int i; 3720 edge_iterator ei; 3721 3722 /* Allocate a worklist array/queue. Entries are only added to the 3723 list if they were not already on the list. So the size is 3724 bounded by the number of basic blocks in the region. */ 3725 qin = qout = worklist = XNEWVEC (basic_block, blocks.length ()); 3726 3727 for (qlen = 0, i = blocks.length () - 1; i >= 0; --i) 3728 { 3729 bb = blocks[i]; 3730 3731 /* Seed ANTIC_OUT with the LOCAL set. */ 3732 bitmap_ior_into (STORE_ANTIC_OUT (bb), STORE_LOCAL (bb)); 3733 3734 /* Put every block in the region on the worklist. */ 3735 AVAIL_IN_WORKLIST_P (bb) = true; 3736 /* No need to insert exit blocks, since their ANTIC_IN is NULL, 3737 and their ANTIC_OUT has already been seeded in. */ 3738 if (region->exit_blocks 3739 && !bitmap_bit_p (region->exit_blocks, bb->index)) 3740 { 3741 qlen++; 3742 *qin++ = bb; 3743 } 3744 } 3745 3746 /* The exit blocks have been initialized with the local sets. */ 3747 if (region->exit_blocks) 3748 { 3749 unsigned int i; 3750 bitmap_iterator bi; 3751 EXECUTE_IF_SET_IN_BITMAP (region->exit_blocks, 0, i, bi) 3752 BB_VISITED_P (BASIC_BLOCK_FOR_FN (cfun, i)) = true; 3753 } 3754 3755 qin = worklist; 3756 qend = &worklist[qlen]; 3757 3758 /* Iterate until the worklist is empty. */ 3759 while (qlen) 3760 { 3761 /* Take the first entry off the worklist. */ 3762 bb = *qout++; 3763 qlen--; 3764 3765 if (qout >= qend) 3766 qout = worklist; 3767 3768 /* This block can be added to the worklist again if necessary. */ 3769 AVAIL_IN_WORKLIST_P (bb) = false; 3770 tm_memopt_compute_antin (bb); 3771 3772 /* Note: We do not add the LOCAL sets here because we already 3773 seeded the ANTIC_OUT sets with them. */ 3774 if (bitmap_ior_into (STORE_ANTIC_OUT (bb), STORE_ANTIC_IN (bb)) 3775 && bb != region->entry_block) 3776 /* If the out state of this block changed, then we need to add 3777 its predecessors to the worklist if they are not already in. */ 3778 FOR_EACH_EDGE (e, ei, bb->preds) 3779 if (!AVAIL_IN_WORKLIST_P (e->src)) 3780 { 3781 *qin++ = e->src; 3782 AVAIL_IN_WORKLIST_P (e->src) = true; 3783 qlen++; 3784 3785 if (qin >= qend) 3786 qin = worklist; 3787 } 3788 } 3789 3790 free (worklist); 3791 3792 if (dump_file) 3793 dump_tm_memopt_sets (blocks); 3794} 3795 3796/* Offsets of load variants from TM_LOAD. For example, 3797 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*. 3798 See gtm-builtins.def. */ 3799#define TRANSFORM_RAR 1 3800#define TRANSFORM_RAW 2 3801#define TRANSFORM_RFW 3 3802/* Offsets of store variants from TM_STORE. */ 3803#define TRANSFORM_WAR 1 3804#define TRANSFORM_WAW 2 3805 3806/* Inform about a load/store optimization. */ 3807 3808static void 3809dump_tm_memopt_transform (gimple stmt) 3810{ 3811 if (dump_file) 3812 { 3813 fprintf (dump_file, "TM memopt: transforming: "); 3814 print_gimple_stmt (dump_file, stmt, 0, 0); 3815 fprintf (dump_file, "\n"); 3816 } 3817} 3818 3819/* Perform a read/write optimization. Replaces the TM builtin in STMT 3820 by a builtin that is OFFSET entries down in the builtins table in 3821 gtm-builtins.def. */ 3822 3823static void 3824tm_memopt_transform_stmt (unsigned int offset, 3825 gcall *stmt, 3826 gimple_stmt_iterator *gsi) 3827{ 3828 tree fn = gimple_call_fn (stmt); 3829 gcc_assert (TREE_CODE (fn) == ADDR_EXPR); 3830 TREE_OPERAND (fn, 0) 3831 = builtin_decl_explicit ((enum built_in_function) 3832 (DECL_FUNCTION_CODE (TREE_OPERAND (fn, 0)) 3833 + offset)); 3834 gimple_call_set_fn (stmt, fn); 3835 gsi_replace (gsi, stmt, true); 3836 dump_tm_memopt_transform (stmt); 3837} 3838 3839/* Perform the actual TM memory optimization transformations in the 3840 basic blocks in BLOCKS. */ 3841 3842static void 3843tm_memopt_transform_blocks (vec<basic_block> blocks) 3844{ 3845 size_t i; 3846 basic_block bb; 3847 gimple_stmt_iterator gsi; 3848 3849 for (i = 0; blocks.iterate (i, &bb); ++i) 3850 { 3851 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 3852 { 3853 gimple stmt = gsi_stmt (gsi); 3854 bitmap read_avail = READ_AVAIL_IN (bb); 3855 bitmap store_avail = STORE_AVAIL_IN (bb); 3856 bitmap store_antic = STORE_ANTIC_OUT (bb); 3857 unsigned int loc; 3858 3859 if (is_tm_simple_load (stmt)) 3860 { 3861 gcall *call_stmt = as_a <gcall *> (stmt); 3862 loc = tm_memopt_value_number (stmt, NO_INSERT); 3863 if (store_avail && bitmap_bit_p (store_avail, loc)) 3864 tm_memopt_transform_stmt (TRANSFORM_RAW, call_stmt, &gsi); 3865 else if (store_antic && bitmap_bit_p (store_antic, loc)) 3866 { 3867 tm_memopt_transform_stmt (TRANSFORM_RFW, call_stmt, &gsi); 3868 bitmap_set_bit (store_avail, loc); 3869 } 3870 else if (read_avail && bitmap_bit_p (read_avail, loc)) 3871 tm_memopt_transform_stmt (TRANSFORM_RAR, call_stmt, &gsi); 3872 else 3873 bitmap_set_bit (read_avail, loc); 3874 } 3875 else if (is_tm_simple_store (stmt)) 3876 { 3877 gcall *call_stmt = as_a <gcall *> (stmt); 3878 loc = tm_memopt_value_number (stmt, NO_INSERT); 3879 if (store_avail && bitmap_bit_p (store_avail, loc)) 3880 tm_memopt_transform_stmt (TRANSFORM_WAW, call_stmt, &gsi); 3881 else 3882 { 3883 if (read_avail && bitmap_bit_p (read_avail, loc)) 3884 tm_memopt_transform_stmt (TRANSFORM_WAR, call_stmt, &gsi); 3885 bitmap_set_bit (store_avail, loc); 3886 } 3887 } 3888 } 3889 } 3890} 3891 3892/* Return a new set of bitmaps for a BB. */ 3893 3894static struct tm_memopt_bitmaps * 3895tm_memopt_init_sets (void) 3896{ 3897 struct tm_memopt_bitmaps *b 3898 = XOBNEW (&tm_memopt_obstack.obstack, struct tm_memopt_bitmaps); 3899 b->store_avail_in = BITMAP_ALLOC (&tm_memopt_obstack); 3900 b->store_avail_out = BITMAP_ALLOC (&tm_memopt_obstack); 3901 b->store_antic_in = BITMAP_ALLOC (&tm_memopt_obstack); 3902 b->store_antic_out = BITMAP_ALLOC (&tm_memopt_obstack); 3903 b->store_avail_out = BITMAP_ALLOC (&tm_memopt_obstack); 3904 b->read_avail_in = BITMAP_ALLOC (&tm_memopt_obstack); 3905 b->read_avail_out = BITMAP_ALLOC (&tm_memopt_obstack); 3906 b->read_local = BITMAP_ALLOC (&tm_memopt_obstack); 3907 b->store_local = BITMAP_ALLOC (&tm_memopt_obstack); 3908 return b; 3909} 3910 3911/* Free sets computed for each BB. */ 3912 3913static void 3914tm_memopt_free_sets (vec<basic_block> blocks) 3915{ 3916 size_t i; 3917 basic_block bb; 3918 3919 for (i = 0; blocks.iterate (i, &bb); ++i) 3920 bb->aux = NULL; 3921} 3922 3923/* Clear the visited bit for every basic block in BLOCKS. */ 3924 3925static void 3926tm_memopt_clear_visited (vec<basic_block> blocks) 3927{ 3928 size_t i; 3929 basic_block bb; 3930 3931 for (i = 0; blocks.iterate (i, &bb); ++i) 3932 BB_VISITED_P (bb) = false; 3933} 3934 3935/* Replace TM load/stores with hints for the runtime. We handle 3936 things like read-after-write, write-after-read, read-after-read, 3937 read-for-write, etc. */ 3938 3939static unsigned int 3940execute_tm_memopt (void) 3941{ 3942 struct tm_region *region; 3943 vec<basic_block> bbs; 3944 3945 tm_memopt_value_id = 0; 3946 tm_memopt_value_numbers = new hash_table<tm_memop_hasher> (10); 3947 3948 for (region = all_tm_regions; region; region = region->next) 3949 { 3950 /* All the TM stores/loads in the current region. */ 3951 size_t i; 3952 basic_block bb; 3953 3954 bitmap_obstack_initialize (&tm_memopt_obstack); 3955 3956 /* Save all BBs for the current region. */ 3957 bbs = get_tm_region_blocks (region->entry_block, 3958 region->exit_blocks, 3959 region->irr_blocks, 3960 NULL, 3961 false); 3962 3963 /* Collect all the memory operations. */ 3964 for (i = 0; bbs.iterate (i, &bb); ++i) 3965 { 3966 bb->aux = tm_memopt_init_sets (); 3967 tm_memopt_accumulate_memops (bb); 3968 } 3969 3970 /* Solve data flow equations and transform each block accordingly. */ 3971 tm_memopt_clear_visited (bbs); 3972 tm_memopt_compute_available (region, bbs); 3973 tm_memopt_clear_visited (bbs); 3974 tm_memopt_compute_antic (region, bbs); 3975 tm_memopt_transform_blocks (bbs); 3976 3977 tm_memopt_free_sets (bbs); 3978 bbs.release (); 3979 bitmap_obstack_release (&tm_memopt_obstack); 3980 tm_memopt_value_numbers->empty (); 3981 } 3982 3983 delete tm_memopt_value_numbers; 3984 tm_memopt_value_numbers = NULL; 3985 return 0; 3986} 3987 3988namespace { 3989 3990const pass_data pass_data_tm_memopt = 3991{ 3992 GIMPLE_PASS, /* type */ 3993 "tmmemopt", /* name */ 3994 OPTGROUP_NONE, /* optinfo_flags */ 3995 TV_TRANS_MEM, /* tv_id */ 3996 ( PROP_ssa | PROP_cfg ), /* properties_required */ 3997 0, /* properties_provided */ 3998 0, /* properties_destroyed */ 3999 0, /* todo_flags_start */ 4000 0, /* todo_flags_finish */ 4001}; 4002 4003class pass_tm_memopt : public gimple_opt_pass 4004{ 4005public: 4006 pass_tm_memopt (gcc::context *ctxt) 4007 : gimple_opt_pass (pass_data_tm_memopt, ctxt) 4008 {} 4009 4010 /* opt_pass methods: */ 4011 virtual bool gate (function *) { return flag_tm && optimize > 0; } 4012 virtual unsigned int execute (function *) { return execute_tm_memopt (); } 4013 4014}; // class pass_tm_memopt 4015 4016} // anon namespace 4017 4018gimple_opt_pass * 4019make_pass_tm_memopt (gcc::context *ctxt) 4020{ 4021 return new pass_tm_memopt (ctxt); 4022} 4023 4024 4025/* Interprocedual analysis for the creation of transactional clones. 4026 The aim of this pass is to find which functions are referenced in 4027 a non-irrevocable transaction context, and for those over which 4028 we have control (or user directive), create a version of the 4029 function which uses only the transactional interface to reference 4030 protected memories. This analysis proceeds in several steps: 4031 4032 (1) Collect the set of all possible transactional clones: 4033 4034 (a) For all local public functions marked tm_callable, push 4035 it onto the tm_callee queue. 4036 4037 (b) For all local functions, scan for calls in transaction blocks. 4038 Push the caller and callee onto the tm_caller and tm_callee 4039 queues. Count the number of callers for each callee. 4040 4041 (c) For each local function on the callee list, assume we will 4042 create a transactional clone. Push *all* calls onto the 4043 callee queues; count the number of clone callers separately 4044 to the number of original callers. 4045 4046 (2) Propagate irrevocable status up the dominator tree: 4047 4048 (a) Any external function on the callee list that is not marked 4049 tm_callable is irrevocable. Push all callers of such onto 4050 a worklist. 4051 4052 (b) For each function on the worklist, mark each block that 4053 contains an irrevocable call. Use the AND operator to 4054 propagate that mark up the dominator tree. 4055 4056 (c) If we reach the entry block for a possible transactional 4057 clone, then the transactional clone is irrevocable, and 4058 we should not create the clone after all. Push all 4059 callers onto the worklist. 4060 4061 (d) Place tm_irrevocable calls at the beginning of the relevant 4062 blocks. Special case here is the entry block for the entire 4063 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for 4064 the library to begin the region in serial mode. Decrement 4065 the call count for all callees in the irrevocable region. 4066 4067 (3) Create the transactional clones: 4068 4069 Any tm_callee that still has a non-zero call count is cloned. 4070*/ 4071 4072/* This structure is stored in the AUX field of each cgraph_node. */ 4073struct tm_ipa_cg_data 4074{ 4075 /* The clone of the function that got created. */ 4076 struct cgraph_node *clone; 4077 4078 /* The tm regions in the normal function. */ 4079 struct tm_region *all_tm_regions; 4080 4081 /* The blocks of the normal/clone functions that contain irrevocable 4082 calls, or blocks that are post-dominated by irrevocable calls. */ 4083 bitmap irrevocable_blocks_normal; 4084 bitmap irrevocable_blocks_clone; 4085 4086 /* The blocks of the normal function that are involved in transactions. */ 4087 bitmap transaction_blocks_normal; 4088 4089 /* The number of callers to the transactional clone of this function 4090 from normal and transactional clones respectively. */ 4091 unsigned tm_callers_normal; 4092 unsigned tm_callers_clone; 4093 4094 /* True if all calls to this function's transactional clone 4095 are irrevocable. Also automatically true if the function 4096 has no transactional clone. */ 4097 bool is_irrevocable; 4098 4099 /* Flags indicating the presence of this function in various queues. */ 4100 bool in_callee_queue; 4101 bool in_worklist; 4102 4103 /* Flags indicating the kind of scan desired while in the worklist. */ 4104 bool want_irr_scan_normal; 4105}; 4106 4107typedef vec<cgraph_node *> cgraph_node_queue; 4108 4109/* Return the ipa data associated with NODE, allocating zeroed memory 4110 if necessary. TRAVERSE_ALIASES is true if we must traverse aliases 4111 and set *NODE accordingly. */ 4112 4113static struct tm_ipa_cg_data * 4114get_cg_data (struct cgraph_node **node, bool traverse_aliases) 4115{ 4116 struct tm_ipa_cg_data *d; 4117 4118 if (traverse_aliases && (*node)->alias) 4119 *node = (*node)->get_alias_target (); 4120 4121 d = (struct tm_ipa_cg_data *) (*node)->aux; 4122 4123 if (d == NULL) 4124 { 4125 d = (struct tm_ipa_cg_data *) 4126 obstack_alloc (&tm_obstack.obstack, sizeof (*d)); 4127 (*node)->aux = (void *) d; 4128 memset (d, 0, sizeof (*d)); 4129 } 4130 4131 return d; 4132} 4133 4134/* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that 4135 it is already present. */ 4136 4137static void 4138maybe_push_queue (struct cgraph_node *node, 4139 cgraph_node_queue *queue_p, bool *in_queue_p) 4140{ 4141 if (!*in_queue_p) 4142 { 4143 *in_queue_p = true; 4144 queue_p->safe_push (node); 4145 } 4146} 4147 4148/* Duplicate the basic blocks in QUEUE for use in the uninstrumented 4149 code path. QUEUE are the basic blocks inside the transaction 4150 represented in REGION. 4151 4152 Later in split_code_paths() we will add the conditional to choose 4153 between the two alternatives. */ 4154 4155static void 4156ipa_uninstrument_transaction (struct tm_region *region, 4157 vec<basic_block> queue) 4158{ 4159 gimple transaction = region->transaction_stmt; 4160 basic_block transaction_bb = gimple_bb (transaction); 4161 int n = queue.length (); 4162 basic_block *new_bbs = XNEWVEC (basic_block, n); 4163 4164 copy_bbs (queue.address (), n, new_bbs, NULL, 0, NULL, NULL, transaction_bb, 4165 true); 4166 edge e = make_edge (transaction_bb, new_bbs[0], EDGE_TM_UNINSTRUMENTED); 4167 add_phi_args_after_copy (new_bbs, n, e); 4168 4169 // Now we will have a GIMPLE_ATOMIC with 3 possible edges out of it. 4170 // a) EDGE_FALLTHRU into the transaction 4171 // b) EDGE_TM_ABORT out of the transaction 4172 // c) EDGE_TM_UNINSTRUMENTED into the uninstrumented blocks. 4173 4174 free (new_bbs); 4175} 4176 4177/* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone. 4178 Queue all callees within block BB. */ 4179 4180static void 4181ipa_tm_scan_calls_block (cgraph_node_queue *callees_p, 4182 basic_block bb, bool for_clone) 4183{ 4184 gimple_stmt_iterator gsi; 4185 4186 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 4187 { 4188 gimple stmt = gsi_stmt (gsi); 4189 if (is_gimple_call (stmt) && !is_tm_pure_call (stmt)) 4190 { 4191 tree fndecl = gimple_call_fndecl (stmt); 4192 if (fndecl) 4193 { 4194 struct tm_ipa_cg_data *d; 4195 unsigned *pcallers; 4196 struct cgraph_node *node; 4197 4198 if (is_tm_ending_fndecl (fndecl)) 4199 continue; 4200 if (find_tm_replacement_function (fndecl)) 4201 continue; 4202 4203 node = cgraph_node::get (fndecl); 4204 gcc_assert (node != NULL); 4205 d = get_cg_data (&node, true); 4206 4207 pcallers = (for_clone ? &d->tm_callers_clone 4208 : &d->tm_callers_normal); 4209 *pcallers += 1; 4210 4211 maybe_push_queue (node, callees_p, &d->in_callee_queue); 4212 } 4213 } 4214 } 4215} 4216 4217/* Scan all calls in NODE that are within a transaction region, 4218 and push the resulting nodes into the callee queue. */ 4219 4220static void 4221ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data *d, 4222 cgraph_node_queue *callees_p) 4223{ 4224 struct tm_region *r; 4225 4226 d->transaction_blocks_normal = BITMAP_ALLOC (&tm_obstack); 4227 d->all_tm_regions = all_tm_regions; 4228 4229 for (r = all_tm_regions; r; r = r->next) 4230 { 4231 vec<basic_block> bbs; 4232 basic_block bb; 4233 unsigned i; 4234 4235 bbs = get_tm_region_blocks (r->entry_block, r->exit_blocks, NULL, 4236 d->transaction_blocks_normal, false); 4237 4238 // Generate the uninstrumented code path for this transaction. 4239 ipa_uninstrument_transaction (r, bbs); 4240 4241 FOR_EACH_VEC_ELT (bbs, i, bb) 4242 ipa_tm_scan_calls_block (callees_p, bb, false); 4243 4244 bbs.release (); 4245 } 4246 4247 // ??? copy_bbs should maintain cgraph edges for the blocks as it is 4248 // copying them, rather than forcing us to do this externally. 4249 cgraph_edge::rebuild_edges (); 4250 4251 // ??? In ipa_uninstrument_transaction we don't try to update dominators 4252 // because copy_bbs doesn't return a VEC like iterate_fix_dominators expects. 4253 // Instead, just release dominators here so update_ssa recomputes them. 4254 free_dominance_info (CDI_DOMINATORS); 4255 4256 // When building the uninstrumented code path, copy_bbs will have invoked 4257 // create_new_def_for starting an "ssa update context". There is only one 4258 // instance of this context, so resolve ssa updates before moving on to 4259 // the next function. 4260 update_ssa (TODO_update_ssa); 4261} 4262 4263/* Scan all calls in NODE as if this is the transactional clone, 4264 and push the destinations into the callee queue. */ 4265 4266static void 4267ipa_tm_scan_calls_clone (struct cgraph_node *node, 4268 cgraph_node_queue *callees_p) 4269{ 4270 struct function *fn = DECL_STRUCT_FUNCTION (node->decl); 4271 basic_block bb; 4272 4273 FOR_EACH_BB_FN (bb, fn) 4274 ipa_tm_scan_calls_block (callees_p, bb, true); 4275} 4276 4277/* The function NODE has been detected to be irrevocable. Push all 4278 of its callers onto WORKLIST for the purpose of re-scanning them. */ 4279 4280static void 4281ipa_tm_note_irrevocable (struct cgraph_node *node, 4282 cgraph_node_queue *worklist_p) 4283{ 4284 struct tm_ipa_cg_data *d = get_cg_data (&node, true); 4285 struct cgraph_edge *e; 4286 4287 d->is_irrevocable = true; 4288 4289 for (e = node->callers; e ; e = e->next_caller) 4290 { 4291 basic_block bb; 4292 struct cgraph_node *caller; 4293 4294 /* Don't examine recursive calls. */ 4295 if (e->caller == node) 4296 continue; 4297 /* Even if we think we can go irrevocable, believe the user 4298 above all. */ 4299 if (is_tm_safe_or_pure (e->caller->decl)) 4300 continue; 4301 4302 caller = e->caller; 4303 d = get_cg_data (&caller, true); 4304 4305 /* Check if the callee is in a transactional region. If so, 4306 schedule the function for normal re-scan as well. */ 4307 bb = gimple_bb (e->call_stmt); 4308 gcc_assert (bb != NULL); 4309 if (d->transaction_blocks_normal 4310 && bitmap_bit_p (d->transaction_blocks_normal, bb->index)) 4311 d->want_irr_scan_normal = true; 4312 4313 maybe_push_queue (caller, worklist_p, &d->in_worklist); 4314 } 4315} 4316 4317/* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement 4318 within the block is irrevocable. */ 4319 4320static bool 4321ipa_tm_scan_irr_block (basic_block bb) 4322{ 4323 gimple_stmt_iterator gsi; 4324 tree fn; 4325 4326 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 4327 { 4328 gimple stmt = gsi_stmt (gsi); 4329 switch (gimple_code (stmt)) 4330 { 4331 case GIMPLE_ASSIGN: 4332 if (gimple_assign_single_p (stmt)) 4333 { 4334 tree lhs = gimple_assign_lhs (stmt); 4335 tree rhs = gimple_assign_rhs1 (stmt); 4336 if (volatile_var_p (lhs) || volatile_var_p (rhs)) 4337 return true; 4338 } 4339 break; 4340 4341 case GIMPLE_CALL: 4342 { 4343 tree lhs = gimple_call_lhs (stmt); 4344 if (lhs && volatile_var_p (lhs)) 4345 return true; 4346 4347 if (is_tm_pure_call (stmt)) 4348 break; 4349 4350 fn = gimple_call_fn (stmt); 4351 4352 /* Functions with the attribute are by definition irrevocable. */ 4353 if (is_tm_irrevocable (fn)) 4354 return true; 4355 4356 /* For direct function calls, go ahead and check for replacement 4357 functions, or transitive irrevocable functions. For indirect 4358 functions, we'll ask the runtime. */ 4359 if (TREE_CODE (fn) == ADDR_EXPR) 4360 { 4361 struct tm_ipa_cg_data *d; 4362 struct cgraph_node *node; 4363 4364 fn = TREE_OPERAND (fn, 0); 4365 if (is_tm_ending_fndecl (fn)) 4366 break; 4367 if (find_tm_replacement_function (fn)) 4368 break; 4369 4370 node = cgraph_node::get (fn); 4371 d = get_cg_data (&node, true); 4372 4373 /* Return true if irrevocable, but above all, believe 4374 the user. */ 4375 if (d->is_irrevocable 4376 && !is_tm_safe_or_pure (fn)) 4377 return true; 4378 } 4379 break; 4380 } 4381 4382 case GIMPLE_ASM: 4383 /* ??? The Approved Method of indicating that an inline 4384 assembly statement is not relevant to the transaction 4385 is to wrap it in a __tm_waiver block. This is not 4386 yet implemented, so we can't check for it. */ 4387 if (is_tm_safe (current_function_decl)) 4388 { 4389 tree t = build1 (NOP_EXPR, void_type_node, size_zero_node); 4390 SET_EXPR_LOCATION (t, gimple_location (stmt)); 4391 error ("%Kasm not allowed in %<transaction_safe%> function", t); 4392 } 4393 return true; 4394 4395 default: 4396 break; 4397 } 4398 } 4399 4400 return false; 4401} 4402 4403/* For each of the blocks seeded witin PQUEUE, walk the CFG looking 4404 for new irrevocable blocks, marking them in NEW_IRR. Don't bother 4405 scanning past OLD_IRR or EXIT_BLOCKS. */ 4406 4407static bool 4408ipa_tm_scan_irr_blocks (vec<basic_block> *pqueue, bitmap new_irr, 4409 bitmap old_irr, bitmap exit_blocks) 4410{ 4411 bool any_new_irr = false; 4412 edge e; 4413 edge_iterator ei; 4414 bitmap visited_blocks = BITMAP_ALLOC (NULL); 4415 4416 do 4417 { 4418 basic_block bb = pqueue->pop (); 4419 4420 /* Don't re-scan blocks we know already are irrevocable. */ 4421 if (old_irr && bitmap_bit_p (old_irr, bb->index)) 4422 continue; 4423 4424 if (ipa_tm_scan_irr_block (bb)) 4425 { 4426 bitmap_set_bit (new_irr, bb->index); 4427 any_new_irr = true; 4428 } 4429 else if (exit_blocks == NULL || !bitmap_bit_p (exit_blocks, bb->index)) 4430 { 4431 FOR_EACH_EDGE (e, ei, bb->succs) 4432 if (!bitmap_bit_p (visited_blocks, e->dest->index)) 4433 { 4434 bitmap_set_bit (visited_blocks, e->dest->index); 4435 pqueue->safe_push (e->dest); 4436 } 4437 } 4438 } 4439 while (!pqueue->is_empty ()); 4440 4441 BITMAP_FREE (visited_blocks); 4442 4443 return any_new_irr; 4444} 4445 4446/* Propagate the irrevocable property both up and down the dominator tree. 4447 BB is the current block being scanned; EXIT_BLOCKS are the edges of the 4448 TM regions; OLD_IRR are the results of a previous scan of the dominator 4449 tree which has been fully propagated; NEW_IRR is the set of new blocks 4450 which are gaining the irrevocable property during the current scan. */ 4451 4452static void 4453ipa_tm_propagate_irr (basic_block entry_block, bitmap new_irr, 4454 bitmap old_irr, bitmap exit_blocks) 4455{ 4456 vec<basic_block> bbs; 4457 bitmap all_region_blocks; 4458 4459 /* If this block is in the old set, no need to rescan. */ 4460 if (old_irr && bitmap_bit_p (old_irr, entry_block->index)) 4461 return; 4462 4463 all_region_blocks = BITMAP_ALLOC (&tm_obstack); 4464 bbs = get_tm_region_blocks (entry_block, exit_blocks, NULL, 4465 all_region_blocks, false); 4466 do 4467 { 4468 basic_block bb = bbs.pop (); 4469 bool this_irr = bitmap_bit_p (new_irr, bb->index); 4470 bool all_son_irr = false; 4471 edge_iterator ei; 4472 edge e; 4473 4474 /* Propagate up. If my children are, I am too, but we must have 4475 at least one child that is. */ 4476 if (!this_irr) 4477 { 4478 FOR_EACH_EDGE (e, ei, bb->succs) 4479 { 4480 if (!bitmap_bit_p (new_irr, e->dest->index)) 4481 { 4482 all_son_irr = false; 4483 break; 4484 } 4485 else 4486 all_son_irr = true; 4487 } 4488 if (all_son_irr) 4489 { 4490 /* Add block to new_irr if it hasn't already been processed. */ 4491 if (!old_irr || !bitmap_bit_p (old_irr, bb->index)) 4492 { 4493 bitmap_set_bit (new_irr, bb->index); 4494 this_irr = true; 4495 } 4496 } 4497 } 4498 4499 /* Propagate down to everyone we immediately dominate. */ 4500 if (this_irr) 4501 { 4502 basic_block son; 4503 for (son = first_dom_son (CDI_DOMINATORS, bb); 4504 son; 4505 son = next_dom_son (CDI_DOMINATORS, son)) 4506 { 4507 /* Make sure block is actually in a TM region, and it 4508 isn't already in old_irr. */ 4509 if ((!old_irr || !bitmap_bit_p (old_irr, son->index)) 4510 && bitmap_bit_p (all_region_blocks, son->index)) 4511 bitmap_set_bit (new_irr, son->index); 4512 } 4513 } 4514 } 4515 while (!bbs.is_empty ()); 4516 4517 BITMAP_FREE (all_region_blocks); 4518 bbs.release (); 4519} 4520 4521static void 4522ipa_tm_decrement_clone_counts (basic_block bb, bool for_clone) 4523{ 4524 gimple_stmt_iterator gsi; 4525 4526 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 4527 { 4528 gimple stmt = gsi_stmt (gsi); 4529 if (is_gimple_call (stmt) && !is_tm_pure_call (stmt)) 4530 { 4531 tree fndecl = gimple_call_fndecl (stmt); 4532 if (fndecl) 4533 { 4534 struct tm_ipa_cg_data *d; 4535 unsigned *pcallers; 4536 struct cgraph_node *tnode; 4537 4538 if (is_tm_ending_fndecl (fndecl)) 4539 continue; 4540 if (find_tm_replacement_function (fndecl)) 4541 continue; 4542 4543 tnode = cgraph_node::get (fndecl); 4544 d = get_cg_data (&tnode, true); 4545 4546 pcallers = (for_clone ? &d->tm_callers_clone 4547 : &d->tm_callers_normal); 4548 4549 gcc_assert (*pcallers > 0); 4550 *pcallers -= 1; 4551 } 4552 } 4553 } 4554} 4555 4556/* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions, 4557 as well as other irrevocable actions such as inline assembly. Mark all 4558 such blocks as irrevocable and decrement the number of calls to 4559 transactional clones. Return true if, for the transactional clone, the 4560 entire function is irrevocable. */ 4561 4562static bool 4563ipa_tm_scan_irr_function (struct cgraph_node *node, bool for_clone) 4564{ 4565 struct tm_ipa_cg_data *d; 4566 bitmap new_irr, old_irr; 4567 bool ret = false; 4568 4569 /* Builtin operators (operator new, and such). */ 4570 if (DECL_STRUCT_FUNCTION (node->decl) == NULL 4571 || DECL_STRUCT_FUNCTION (node->decl)->cfg == NULL) 4572 return false; 4573 4574 push_cfun (DECL_STRUCT_FUNCTION (node->decl)); 4575 calculate_dominance_info (CDI_DOMINATORS); 4576 4577 d = get_cg_data (&node, true); 4578 auto_vec<basic_block, 10> queue; 4579 new_irr = BITMAP_ALLOC (&tm_obstack); 4580 4581 /* Scan each tm region, propagating irrevocable status through the tree. */ 4582 if (for_clone) 4583 { 4584 old_irr = d->irrevocable_blocks_clone; 4585 queue.quick_push (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun))); 4586 if (ipa_tm_scan_irr_blocks (&queue, new_irr, old_irr, NULL)) 4587 { 4588 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)), 4589 new_irr, 4590 old_irr, NULL); 4591 ret = bitmap_bit_p (new_irr, 4592 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun))->index); 4593 } 4594 } 4595 else 4596 { 4597 struct tm_region *region; 4598 4599 old_irr = d->irrevocable_blocks_normal; 4600 for (region = d->all_tm_regions; region; region = region->next) 4601 { 4602 queue.quick_push (region->entry_block); 4603 if (ipa_tm_scan_irr_blocks (&queue, new_irr, old_irr, 4604 region->exit_blocks)) 4605 ipa_tm_propagate_irr (region->entry_block, new_irr, old_irr, 4606 region->exit_blocks); 4607 } 4608 } 4609 4610 /* If we found any new irrevocable blocks, reduce the call count for 4611 transactional clones within the irrevocable blocks. Save the new 4612 set of irrevocable blocks for next time. */ 4613 if (!bitmap_empty_p (new_irr)) 4614 { 4615 bitmap_iterator bmi; 4616 unsigned i; 4617 4618 EXECUTE_IF_SET_IN_BITMAP (new_irr, 0, i, bmi) 4619 ipa_tm_decrement_clone_counts (BASIC_BLOCK_FOR_FN (cfun, i), 4620 for_clone); 4621 4622 if (old_irr) 4623 { 4624 bitmap_ior_into (old_irr, new_irr); 4625 BITMAP_FREE (new_irr); 4626 } 4627 else if (for_clone) 4628 d->irrevocable_blocks_clone = new_irr; 4629 else 4630 d->irrevocable_blocks_normal = new_irr; 4631 4632 if (dump_file && new_irr) 4633 { 4634 const char *dname; 4635 bitmap_iterator bmi; 4636 unsigned i; 4637 4638 dname = lang_hooks.decl_printable_name (current_function_decl, 2); 4639 EXECUTE_IF_SET_IN_BITMAP (new_irr, 0, i, bmi) 4640 fprintf (dump_file, "%s: bb %d goes irrevocable\n", dname, i); 4641 } 4642 } 4643 else 4644 BITMAP_FREE (new_irr); 4645 4646 pop_cfun (); 4647 4648 return ret; 4649} 4650 4651/* Return true if, for the transactional clone of NODE, any call 4652 may enter irrevocable mode. */ 4653 4654static bool 4655ipa_tm_mayenterirr_function (struct cgraph_node *node) 4656{ 4657 struct tm_ipa_cg_data *d; 4658 tree decl; 4659 unsigned flags; 4660 4661 d = get_cg_data (&node, true); 4662 decl = node->decl; 4663 flags = flags_from_decl_or_type (decl); 4664 4665 /* Handle some TM builtins. Ordinarily these aren't actually generated 4666 at this point, but handling these functions when written in by the 4667 user makes it easier to build unit tests. */ 4668 if (flags & ECF_TM_BUILTIN) 4669 return false; 4670 4671 /* Filter out all functions that are marked. */ 4672 if (flags & ECF_TM_PURE) 4673 return false; 4674 if (is_tm_safe (decl)) 4675 return false; 4676 if (is_tm_irrevocable (decl)) 4677 return true; 4678 if (is_tm_callable (decl)) 4679 return true; 4680 if (find_tm_replacement_function (decl)) 4681 return true; 4682 4683 /* If we aren't seeing the final version of the function we don't 4684 know what it will contain at runtime. */ 4685 if (node->get_availability () < AVAIL_AVAILABLE) 4686 return true; 4687 4688 /* If the function must go irrevocable, then of course true. */ 4689 if (d->is_irrevocable) 4690 return true; 4691 4692 /* If there are any blocks marked irrevocable, then the function 4693 as a whole may enter irrevocable. */ 4694 if (d->irrevocable_blocks_clone) 4695 return true; 4696 4697 /* We may have previously marked this function as tm_may_enter_irr; 4698 see pass_diagnose_tm_blocks. */ 4699 if (node->local.tm_may_enter_irr) 4700 return true; 4701 4702 /* Recurse on the main body for aliases. In general, this will 4703 result in one of the bits above being set so that we will not 4704 have to recurse next time. */ 4705 if (node->alias) 4706 return ipa_tm_mayenterirr_function (cgraph_node::get (node->thunk.alias)); 4707 4708 /* What remains is unmarked local functions without items that force 4709 the function to go irrevocable. */ 4710 return false; 4711} 4712 4713/* Diagnose calls from transaction_safe functions to unmarked 4714 functions that are determined to not be safe. */ 4715 4716static void 4717ipa_tm_diagnose_tm_safe (struct cgraph_node *node) 4718{ 4719 struct cgraph_edge *e; 4720 4721 for (e = node->callees; e ; e = e->next_callee) 4722 if (!is_tm_callable (e->callee->decl) 4723 && e->callee->local.tm_may_enter_irr) 4724 error_at (gimple_location (e->call_stmt), 4725 "unsafe function call %qD within " 4726 "%<transaction_safe%> function", e->callee->decl); 4727} 4728 4729/* Diagnose call from atomic transactions to unmarked functions 4730 that are determined to not be safe. */ 4731 4732static void 4733ipa_tm_diagnose_transaction (struct cgraph_node *node, 4734 struct tm_region *all_tm_regions) 4735{ 4736 struct tm_region *r; 4737 4738 for (r = all_tm_regions; r ; r = r->next) 4739 if (gimple_transaction_subcode (r->get_transaction_stmt ()) 4740 & GTMA_IS_RELAXED) 4741 { 4742 /* Atomic transactions can be nested inside relaxed. */ 4743 if (r->inner) 4744 ipa_tm_diagnose_transaction (node, r->inner); 4745 } 4746 else 4747 { 4748 vec<basic_block> bbs; 4749 gimple_stmt_iterator gsi; 4750 basic_block bb; 4751 size_t i; 4752 4753 bbs = get_tm_region_blocks (r->entry_block, r->exit_blocks, 4754 r->irr_blocks, NULL, false); 4755 4756 for (i = 0; bbs.iterate (i, &bb); ++i) 4757 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 4758 { 4759 gimple stmt = gsi_stmt (gsi); 4760 tree fndecl; 4761 4762 if (gimple_code (stmt) == GIMPLE_ASM) 4763 { 4764 error_at (gimple_location (stmt), 4765 "asm not allowed in atomic transaction"); 4766 continue; 4767 } 4768 4769 if (!is_gimple_call (stmt)) 4770 continue; 4771 fndecl = gimple_call_fndecl (stmt); 4772 4773 /* Indirect function calls have been diagnosed already. */ 4774 if (!fndecl) 4775 continue; 4776 4777 /* Stop at the end of the transaction. */ 4778 if (is_tm_ending_fndecl (fndecl)) 4779 { 4780 if (bitmap_bit_p (r->exit_blocks, bb->index)) 4781 break; 4782 continue; 4783 } 4784 4785 /* Marked functions have been diagnosed already. */ 4786 if (is_tm_pure_call (stmt)) 4787 continue; 4788 if (is_tm_callable (fndecl)) 4789 continue; 4790 4791 if (cgraph_node::local_info (fndecl)->tm_may_enter_irr) 4792 error_at (gimple_location (stmt), 4793 "unsafe function call %qD within " 4794 "atomic transaction", fndecl); 4795 } 4796 4797 bbs.release (); 4798 } 4799} 4800 4801/* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in 4802 OLD_DECL. The returned value is a freshly malloced pointer that 4803 should be freed by the caller. */ 4804 4805static tree 4806tm_mangle (tree old_asm_id) 4807{ 4808 const char *old_asm_name; 4809 char *tm_name; 4810 void *alloc = NULL; 4811 struct demangle_component *dc; 4812 tree new_asm_id; 4813 4814 /* Determine if the symbol is already a valid C++ mangled name. Do this 4815 even for C, which might be interfacing with C++ code via appropriately 4816 ugly identifiers. */ 4817 /* ??? We could probably do just as well checking for "_Z" and be done. */ 4818 old_asm_name = IDENTIFIER_POINTER (old_asm_id); 4819 dc = cplus_demangle_v3_components (old_asm_name, DMGL_NO_OPTS, &alloc); 4820 4821 if (dc == NULL) 4822 { 4823 char length[8]; 4824 4825 do_unencoded: 4826 sprintf (length, "%u", IDENTIFIER_LENGTH (old_asm_id)); 4827 tm_name = concat ("_ZGTt", length, old_asm_name, NULL); 4828 } 4829 else 4830 { 4831 old_asm_name += 2; /* Skip _Z */ 4832 4833 switch (dc->type) 4834 { 4835 case DEMANGLE_COMPONENT_TRANSACTION_CLONE: 4836 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE: 4837 /* Don't play silly games, you! */ 4838 goto do_unencoded; 4839 4840 case DEMANGLE_COMPONENT_HIDDEN_ALIAS: 4841 /* I'd really like to know if we can ever be passed one of 4842 these from the C++ front end. The Logical Thing would 4843 seem that hidden-alias should be outer-most, so that we 4844 get hidden-alias of a transaction-clone and not vice-versa. */ 4845 old_asm_name += 2; 4846 break; 4847 4848 default: 4849 break; 4850 } 4851 4852 tm_name = concat ("_ZGTt", old_asm_name, NULL); 4853 } 4854 free (alloc); 4855 4856 new_asm_id = get_identifier (tm_name); 4857 free (tm_name); 4858 4859 return new_asm_id; 4860} 4861 4862static inline void 4863ipa_tm_mark_force_output_node (struct cgraph_node *node) 4864{ 4865 node->mark_force_output (); 4866 node->analyzed = true; 4867} 4868 4869static inline void 4870ipa_tm_mark_forced_by_abi_node (struct cgraph_node *node) 4871{ 4872 node->forced_by_abi = true; 4873 node->analyzed = true; 4874} 4875 4876/* Callback data for ipa_tm_create_version_alias. */ 4877struct create_version_alias_info 4878{ 4879 struct cgraph_node *old_node; 4880 tree new_decl; 4881}; 4882 4883/* A subroutine of ipa_tm_create_version, called via 4884 cgraph_for_node_and_aliases. Create new tm clones for each of 4885 the existing aliases. */ 4886static bool 4887ipa_tm_create_version_alias (struct cgraph_node *node, void *data) 4888{ 4889 struct create_version_alias_info *info 4890 = (struct create_version_alias_info *)data; 4891 tree old_decl, new_decl, tm_name; 4892 struct cgraph_node *new_node; 4893 4894 if (!node->cpp_implicit_alias) 4895 return false; 4896 4897 old_decl = node->decl; 4898 tm_name = tm_mangle (DECL_ASSEMBLER_NAME (old_decl)); 4899 new_decl = build_decl (DECL_SOURCE_LOCATION (old_decl), 4900 TREE_CODE (old_decl), tm_name, 4901 TREE_TYPE (old_decl)); 4902 4903 SET_DECL_ASSEMBLER_NAME (new_decl, tm_name); 4904 SET_DECL_RTL (new_decl, NULL); 4905 4906 /* Based loosely on C++'s make_alias_for(). */ 4907 TREE_PUBLIC (new_decl) = TREE_PUBLIC (old_decl); 4908 DECL_CONTEXT (new_decl) = DECL_CONTEXT (old_decl); 4909 DECL_LANG_SPECIFIC (new_decl) = DECL_LANG_SPECIFIC (old_decl); 4910 TREE_READONLY (new_decl) = TREE_READONLY (old_decl); 4911 DECL_EXTERNAL (new_decl) = 0; 4912 DECL_ARTIFICIAL (new_decl) = 1; 4913 TREE_ADDRESSABLE (new_decl) = 1; 4914 TREE_USED (new_decl) = 1; 4915 TREE_SYMBOL_REFERENCED (tm_name) = 1; 4916 4917 /* Perform the same remapping to the comdat group. */ 4918 if (DECL_ONE_ONLY (new_decl)) 4919 varpool_node::get (new_decl)->set_comdat_group 4920 (tm_mangle (decl_comdat_group_id (old_decl))); 4921 4922 new_node = cgraph_node::create_same_body_alias (new_decl, info->new_decl); 4923 new_node->tm_clone = true; 4924 new_node->externally_visible = info->old_node->externally_visible; 4925 new_node->no_reorder = info->old_node->no_reorder; 4926 /* ?? Do not traverse aliases here. */ 4927 get_cg_data (&node, false)->clone = new_node; 4928 4929 record_tm_clone_pair (old_decl, new_decl); 4930 4931 if (info->old_node->force_output 4932 || info->old_node->ref_list.first_referring ()) 4933 ipa_tm_mark_force_output_node (new_node); 4934 if (info->old_node->forced_by_abi) 4935 ipa_tm_mark_forced_by_abi_node (new_node); 4936 return false; 4937} 4938 4939/* Create a copy of the function (possibly declaration only) of OLD_NODE, 4940 appropriate for the transactional clone. */ 4941 4942static void 4943ipa_tm_create_version (struct cgraph_node *old_node) 4944{ 4945 tree new_decl, old_decl, tm_name; 4946 struct cgraph_node *new_node; 4947 4948 old_decl = old_node->decl; 4949 new_decl = copy_node (old_decl); 4950 4951 /* DECL_ASSEMBLER_NAME needs to be set before we call 4952 cgraph_copy_node_for_versioning below, because cgraph_node will 4953 fill the assembler_name_hash. */ 4954 tm_name = tm_mangle (DECL_ASSEMBLER_NAME (old_decl)); 4955 SET_DECL_ASSEMBLER_NAME (new_decl, tm_name); 4956 SET_DECL_RTL (new_decl, NULL); 4957 TREE_SYMBOL_REFERENCED (tm_name) = 1; 4958 4959 /* Perform the same remapping to the comdat group. */ 4960 if (DECL_ONE_ONLY (new_decl)) 4961 varpool_node::get (new_decl)->set_comdat_group 4962 (tm_mangle (DECL_COMDAT_GROUP (old_decl))); 4963 4964 gcc_assert (!old_node->ipa_transforms_to_apply.exists ()); 4965 new_node = old_node->create_version_clone (new_decl, vNULL, NULL); 4966 new_node->local.local = false; 4967 new_node->externally_visible = old_node->externally_visible; 4968 new_node->lowered = true; 4969 new_node->tm_clone = 1; 4970 if (!old_node->implicit_section) 4971 new_node->set_section (old_node->get_section ()); 4972 get_cg_data (&old_node, true)->clone = new_node; 4973 4974 if (old_node->get_availability () >= AVAIL_INTERPOSABLE) 4975 { 4976 /* Remap extern inline to static inline. */ 4977 /* ??? Is it worth trying to use make_decl_one_only? */ 4978 if (DECL_DECLARED_INLINE_P (new_decl) && DECL_EXTERNAL (new_decl)) 4979 { 4980 DECL_EXTERNAL (new_decl) = 0; 4981 TREE_PUBLIC (new_decl) = 0; 4982 DECL_WEAK (new_decl) = 0; 4983 } 4984 4985 tree_function_versioning (old_decl, new_decl, 4986 NULL, false, NULL, 4987 false, NULL, NULL); 4988 } 4989 4990 record_tm_clone_pair (old_decl, new_decl); 4991 4992 symtab->call_cgraph_insertion_hooks (new_node); 4993 if (old_node->force_output 4994 || old_node->ref_list.first_referring ()) 4995 ipa_tm_mark_force_output_node (new_node); 4996 if (old_node->forced_by_abi) 4997 ipa_tm_mark_forced_by_abi_node (new_node); 4998 4999 /* Do the same thing, but for any aliases of the original node. */ 5000 { 5001 struct create_version_alias_info data; 5002 data.old_node = old_node; 5003 data.new_decl = new_decl; 5004 old_node->call_for_symbol_thunks_and_aliases (ipa_tm_create_version_alias, 5005 &data, true); 5006 } 5007} 5008 5009/* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */ 5010 5011static void 5012ipa_tm_insert_irr_call (struct cgraph_node *node, struct tm_region *region, 5013 basic_block bb) 5014{ 5015 gimple_stmt_iterator gsi; 5016 gcall *g; 5017 5018 transaction_subcode_ior (region, GTMA_MAY_ENTER_IRREVOCABLE); 5019 5020 g = gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE), 5021 1, build_int_cst (NULL_TREE, MODE_SERIALIRREVOCABLE)); 5022 5023 split_block_after_labels (bb); 5024 gsi = gsi_after_labels (bb); 5025 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 5026 5027 node->create_edge (cgraph_node::get_create 5028 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE)), 5029 g, 0, 5030 compute_call_stmt_bb_frequency (node->decl, 5031 gimple_bb (g))); 5032} 5033 5034/* Construct a call to TM_GETTMCLONE and insert it before GSI. */ 5035 5036static bool 5037ipa_tm_insert_gettmclone_call (struct cgraph_node *node, 5038 struct tm_region *region, 5039 gimple_stmt_iterator *gsi, gcall *stmt) 5040{ 5041 tree gettm_fn, ret, old_fn, callfn; 5042 gcall *g; 5043 gassign *g2; 5044 bool safe; 5045 5046 old_fn = gimple_call_fn (stmt); 5047 5048 if (TREE_CODE (old_fn) == ADDR_EXPR) 5049 { 5050 tree fndecl = TREE_OPERAND (old_fn, 0); 5051 tree clone = get_tm_clone_pair (fndecl); 5052 5053 /* By transforming the call into a TM_GETTMCLONE, we are 5054 technically taking the address of the original function and 5055 its clone. Explain this so inlining will know this function 5056 is needed. */ 5057 cgraph_node::get (fndecl)->mark_address_taken () ; 5058 if (clone) 5059 cgraph_node::get (clone)->mark_address_taken (); 5060 } 5061 5062 safe = is_tm_safe (TREE_TYPE (old_fn)); 5063 gettm_fn = builtin_decl_explicit (safe ? BUILT_IN_TM_GETTMCLONE_SAFE 5064 : BUILT_IN_TM_GETTMCLONE_IRR); 5065 ret = create_tmp_var (ptr_type_node); 5066 5067 if (!safe) 5068 transaction_subcode_ior (region, GTMA_MAY_ENTER_IRREVOCABLE); 5069 5070 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */ 5071 if (TREE_CODE (old_fn) == OBJ_TYPE_REF) 5072 old_fn = OBJ_TYPE_REF_EXPR (old_fn); 5073 5074 g = gimple_build_call (gettm_fn, 1, old_fn); 5075 ret = make_ssa_name (ret, g); 5076 gimple_call_set_lhs (g, ret); 5077 5078 gsi_insert_before (gsi, g, GSI_SAME_STMT); 5079 5080 node->create_edge (cgraph_node::get_create (gettm_fn), g, 0, 5081 compute_call_stmt_bb_frequency (node->decl, 5082 gimple_bb (g))); 5083 5084 /* Cast return value from tm_gettmclone* into appropriate function 5085 pointer. */ 5086 callfn = create_tmp_var (TREE_TYPE (old_fn)); 5087 g2 = gimple_build_assign (callfn, 5088 fold_build1 (NOP_EXPR, TREE_TYPE (callfn), ret)); 5089 callfn = make_ssa_name (callfn, g2); 5090 gimple_assign_set_lhs (g2, callfn); 5091 gsi_insert_before (gsi, g2, GSI_SAME_STMT); 5092 5093 /* ??? This is a hack to preserve the NOTHROW bit on the call, 5094 which we would have derived from the decl. Failure to save 5095 this bit means we might have to split the basic block. */ 5096 if (gimple_call_nothrow_p (stmt)) 5097 gimple_call_set_nothrow (stmt, true); 5098 5099 gimple_call_set_fn (stmt, callfn); 5100 5101 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS 5102 for a call statement. Fix it. */ 5103 { 5104 tree lhs = gimple_call_lhs (stmt); 5105 tree rettype = TREE_TYPE (gimple_call_fntype (stmt)); 5106 if (lhs 5107 && !useless_type_conversion_p (TREE_TYPE (lhs), rettype)) 5108 { 5109 tree temp; 5110 5111 temp = create_tmp_reg (rettype); 5112 gimple_call_set_lhs (stmt, temp); 5113 5114 g2 = gimple_build_assign (lhs, 5115 fold_build1 (VIEW_CONVERT_EXPR, 5116 TREE_TYPE (lhs), temp)); 5117 gsi_insert_after (gsi, g2, GSI_SAME_STMT); 5118 } 5119 } 5120 5121 update_stmt (stmt); 5122 cgraph_edge *e = cgraph_node::get (current_function_decl)->get_edge (stmt); 5123 if (e && e->indirect_info) 5124 e->indirect_info->polymorphic = false; 5125 5126 return true; 5127} 5128 5129/* Helper function for ipa_tm_transform_calls*. Given a call 5130 statement in GSI which resides inside transaction REGION, redirect 5131 the call to either its wrapper function, or its clone. */ 5132 5133static void 5134ipa_tm_transform_calls_redirect (struct cgraph_node *node, 5135 struct tm_region *region, 5136 gimple_stmt_iterator *gsi, 5137 bool *need_ssa_rename_p) 5138{ 5139 gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi)); 5140 struct cgraph_node *new_node; 5141 struct cgraph_edge *e = node->get_edge (stmt); 5142 tree fndecl = gimple_call_fndecl (stmt); 5143 5144 /* For indirect calls, pass the address through the runtime. */ 5145 if (fndecl == NULL) 5146 { 5147 *need_ssa_rename_p |= 5148 ipa_tm_insert_gettmclone_call (node, region, gsi, stmt); 5149 return; 5150 } 5151 5152 /* Handle some TM builtins. Ordinarily these aren't actually generated 5153 at this point, but handling these functions when written in by the 5154 user makes it easier to build unit tests. */ 5155 if (flags_from_decl_or_type (fndecl) & ECF_TM_BUILTIN) 5156 return; 5157 5158 /* Fixup recursive calls inside clones. */ 5159 /* ??? Why did cgraph_copy_node_for_versioning update the call edges 5160 for recursion but not update the call statements themselves? */ 5161 if (e->caller == e->callee && decl_is_tm_clone (current_function_decl)) 5162 { 5163 gimple_call_set_fndecl (stmt, current_function_decl); 5164 return; 5165 } 5166 5167 /* If there is a replacement, use it. */ 5168 fndecl = find_tm_replacement_function (fndecl); 5169 if (fndecl) 5170 { 5171 new_node = cgraph_node::get_create (fndecl); 5172 5173 /* ??? Mark all transaction_wrap functions tm_may_enter_irr. 5174 5175 We can't do this earlier in record_tm_replacement because 5176 cgraph_remove_unreachable_nodes is called before we inject 5177 references to the node. Further, we can't do this in some 5178 nice central place in ipa_tm_execute because we don't have 5179 the exact list of wrapper functions that would be used. 5180 Marking more wrappers than necessary results in the creation 5181 of unnecessary cgraph_nodes, which can cause some of the 5182 other IPA passes to crash. 5183 5184 We do need to mark these nodes so that we get the proper 5185 result in expand_call_tm. */ 5186 /* ??? This seems broken. How is it that we're marking the 5187 CALLEE as may_enter_irr? Surely we should be marking the 5188 CALLER. Also note that find_tm_replacement_function also 5189 contains mappings into the TM runtime, e.g. memcpy. These 5190 we know won't go irrevocable. */ 5191 new_node->local.tm_may_enter_irr = 1; 5192 } 5193 else 5194 { 5195 struct tm_ipa_cg_data *d; 5196 struct cgraph_node *tnode = e->callee; 5197 5198 d = get_cg_data (&tnode, true); 5199 new_node = d->clone; 5200 5201 /* As we've already skipped pure calls and appropriate builtins, 5202 and we've already marked irrevocable blocks, if we can't come 5203 up with a static replacement, then ask the runtime. */ 5204 if (new_node == NULL) 5205 { 5206 *need_ssa_rename_p |= 5207 ipa_tm_insert_gettmclone_call (node, region, gsi, stmt); 5208 return; 5209 } 5210 5211 fndecl = new_node->decl; 5212 } 5213 5214 e->redirect_callee (new_node); 5215 gimple_call_set_fndecl (stmt, fndecl); 5216} 5217 5218/* Helper function for ipa_tm_transform_calls. For a given BB, 5219 install calls to tm_irrevocable when IRR_BLOCKS are reached, 5220 redirect other calls to the generated transactional clone. */ 5221 5222static bool 5223ipa_tm_transform_calls_1 (struct cgraph_node *node, struct tm_region *region, 5224 basic_block bb, bitmap irr_blocks) 5225{ 5226 gimple_stmt_iterator gsi; 5227 bool need_ssa_rename = false; 5228 5229 if (irr_blocks && bitmap_bit_p (irr_blocks, bb->index)) 5230 { 5231 ipa_tm_insert_irr_call (node, region, bb); 5232 return true; 5233 } 5234 5235 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 5236 { 5237 gimple stmt = gsi_stmt (gsi); 5238 5239 if (!is_gimple_call (stmt)) 5240 continue; 5241 if (is_tm_pure_call (stmt)) 5242 continue; 5243 5244 /* Redirect edges to the appropriate replacement or clone. */ 5245 ipa_tm_transform_calls_redirect (node, region, &gsi, &need_ssa_rename); 5246 } 5247 5248 return need_ssa_rename; 5249} 5250 5251/* Walk the CFG for REGION, beginning at BB. Install calls to 5252 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to 5253 the generated transactional clone. */ 5254 5255static bool 5256ipa_tm_transform_calls (struct cgraph_node *node, struct tm_region *region, 5257 basic_block bb, bitmap irr_blocks) 5258{ 5259 bool need_ssa_rename = false; 5260 edge e; 5261 edge_iterator ei; 5262 auto_vec<basic_block> queue; 5263 bitmap visited_blocks = BITMAP_ALLOC (NULL); 5264 5265 queue.safe_push (bb); 5266 do 5267 { 5268 bb = queue.pop (); 5269 5270 need_ssa_rename |= 5271 ipa_tm_transform_calls_1 (node, region, bb, irr_blocks); 5272 5273 if (irr_blocks && bitmap_bit_p (irr_blocks, bb->index)) 5274 continue; 5275 5276 if (region && bitmap_bit_p (region->exit_blocks, bb->index)) 5277 continue; 5278 5279 FOR_EACH_EDGE (e, ei, bb->succs) 5280 if (!bitmap_bit_p (visited_blocks, e->dest->index)) 5281 { 5282 bitmap_set_bit (visited_blocks, e->dest->index); 5283 queue.safe_push (e->dest); 5284 } 5285 } 5286 while (!queue.is_empty ()); 5287 5288 BITMAP_FREE (visited_blocks); 5289 5290 return need_ssa_rename; 5291} 5292 5293/* Transform the calls within the TM regions within NODE. */ 5294 5295static void 5296ipa_tm_transform_transaction (struct cgraph_node *node) 5297{ 5298 struct tm_ipa_cg_data *d; 5299 struct tm_region *region; 5300 bool need_ssa_rename = false; 5301 5302 d = get_cg_data (&node, true); 5303 5304 push_cfun (DECL_STRUCT_FUNCTION (node->decl)); 5305 calculate_dominance_info (CDI_DOMINATORS); 5306 5307 for (region = d->all_tm_regions; region; region = region->next) 5308 { 5309 /* If we're sure to go irrevocable, don't transform anything. */ 5310 if (d->irrevocable_blocks_normal 5311 && bitmap_bit_p (d->irrevocable_blocks_normal, 5312 region->entry_block->index)) 5313 { 5314 transaction_subcode_ior (region, GTMA_DOES_GO_IRREVOCABLE 5315 | GTMA_MAY_ENTER_IRREVOCABLE 5316 | GTMA_HAS_NO_INSTRUMENTATION); 5317 continue; 5318 } 5319 5320 need_ssa_rename |= 5321 ipa_tm_transform_calls (node, region, region->entry_block, 5322 d->irrevocable_blocks_normal); 5323 } 5324 5325 if (need_ssa_rename) 5326 update_ssa (TODO_update_ssa_only_virtuals); 5327 5328 pop_cfun (); 5329} 5330 5331/* Transform the calls within the transactional clone of NODE. */ 5332 5333static void 5334ipa_tm_transform_clone (struct cgraph_node *node) 5335{ 5336 struct tm_ipa_cg_data *d; 5337 bool need_ssa_rename; 5338 5339 d = get_cg_data (&node, true); 5340 5341 /* If this function makes no calls and has no irrevocable blocks, 5342 then there's nothing to do. */ 5343 /* ??? Remove non-aborting top-level transactions. */ 5344 if (!node->callees && !node->indirect_calls && !d->irrevocable_blocks_clone) 5345 return; 5346 5347 push_cfun (DECL_STRUCT_FUNCTION (d->clone->decl)); 5348 calculate_dominance_info (CDI_DOMINATORS); 5349 5350 need_ssa_rename = 5351 ipa_tm_transform_calls (d->clone, NULL, 5352 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)), 5353 d->irrevocable_blocks_clone); 5354 5355 if (need_ssa_rename) 5356 update_ssa (TODO_update_ssa_only_virtuals); 5357 5358 pop_cfun (); 5359} 5360 5361/* Main entry point for the transactional memory IPA pass. */ 5362 5363static unsigned int 5364ipa_tm_execute (void) 5365{ 5366 cgraph_node_queue tm_callees = cgraph_node_queue (); 5367 /* List of functions that will go irrevocable. */ 5368 cgraph_node_queue irr_worklist = cgraph_node_queue (); 5369 5370 struct cgraph_node *node; 5371 struct tm_ipa_cg_data *d; 5372 enum availability a; 5373 unsigned int i; 5374 5375#ifdef ENABLE_CHECKING 5376 cgraph_node::verify_cgraph_nodes (); 5377#endif 5378 5379 bitmap_obstack_initialize (&tm_obstack); 5380 initialize_original_copy_tables (); 5381 5382 /* For all local functions marked tm_callable, queue them. */ 5383 FOR_EACH_DEFINED_FUNCTION (node) 5384 if (is_tm_callable (node->decl) 5385 && node->get_availability () >= AVAIL_INTERPOSABLE) 5386 { 5387 d = get_cg_data (&node, true); 5388 maybe_push_queue (node, &tm_callees, &d->in_callee_queue); 5389 } 5390 5391 /* For all local reachable functions... */ 5392 FOR_EACH_DEFINED_FUNCTION (node) 5393 if (node->lowered 5394 && node->get_availability () >= AVAIL_INTERPOSABLE) 5395 { 5396 /* ... marked tm_pure, record that fact for the runtime by 5397 indicating that the pure function is its own tm_callable. 5398 No need to do this if the function's address can't be taken. */ 5399 if (is_tm_pure (node->decl)) 5400 { 5401 if (!node->local.local) 5402 record_tm_clone_pair (node->decl, node->decl); 5403 continue; 5404 } 5405 5406 push_cfun (DECL_STRUCT_FUNCTION (node->decl)); 5407 calculate_dominance_info (CDI_DOMINATORS); 5408 5409 tm_region_init (NULL); 5410 if (all_tm_regions) 5411 { 5412 d = get_cg_data (&node, true); 5413 5414 /* Scan for calls that are in each transaction, and 5415 generate the uninstrumented code path. */ 5416 ipa_tm_scan_calls_transaction (d, &tm_callees); 5417 5418 /* Put it in the worklist so we can scan the function 5419 later (ipa_tm_scan_irr_function) and mark the 5420 irrevocable blocks. */ 5421 maybe_push_queue (node, &irr_worklist, &d->in_worklist); 5422 d->want_irr_scan_normal = true; 5423 } 5424 5425 pop_cfun (); 5426 } 5427 5428 /* For every local function on the callee list, scan as if we will be 5429 creating a transactional clone, queueing all new functions we find 5430 along the way. */ 5431 for (i = 0; i < tm_callees.length (); ++i) 5432 { 5433 node = tm_callees[i]; 5434 a = node->get_availability (); 5435 d = get_cg_data (&node, true); 5436 5437 /* Put it in the worklist so we can scan the function later 5438 (ipa_tm_scan_irr_function) and mark the irrevocable 5439 blocks. */ 5440 maybe_push_queue (node, &irr_worklist, &d->in_worklist); 5441 5442 /* Some callees cannot be arbitrarily cloned. These will always be 5443 irrevocable. Mark these now, so that we need not scan them. */ 5444 if (is_tm_irrevocable (node->decl)) 5445 ipa_tm_note_irrevocable (node, &irr_worklist); 5446 else if (a <= AVAIL_NOT_AVAILABLE 5447 && !is_tm_safe_or_pure (node->decl)) 5448 ipa_tm_note_irrevocable (node, &irr_worklist); 5449 else if (a >= AVAIL_INTERPOSABLE) 5450 { 5451 if (!tree_versionable_function_p (node->decl)) 5452 ipa_tm_note_irrevocable (node, &irr_worklist); 5453 else if (!d->is_irrevocable) 5454 { 5455 /* If this is an alias, make sure its base is queued as well. 5456 we need not scan the callees now, as the base will do. */ 5457 if (node->alias) 5458 { 5459 node = cgraph_node::get (node->thunk.alias); 5460 d = get_cg_data (&node, true); 5461 maybe_push_queue (node, &tm_callees, &d->in_callee_queue); 5462 continue; 5463 } 5464 5465 /* Add all nodes called by this function into 5466 tm_callees as well. */ 5467 ipa_tm_scan_calls_clone (node, &tm_callees); 5468 } 5469 } 5470 } 5471 5472 /* Iterate scans until no more work to be done. Prefer not to use 5473 vec::pop because the worklist tends to follow a breadth-first 5474 search of the callgraph, which should allow convergance with a 5475 minimum number of scans. But we also don't want the worklist 5476 array to grow without bound, so we shift the array up periodically. */ 5477 for (i = 0; i < irr_worklist.length (); ++i) 5478 { 5479 if (i > 256 && i == irr_worklist.length () / 8) 5480 { 5481 irr_worklist.block_remove (0, i); 5482 i = 0; 5483 } 5484 5485 node = irr_worklist[i]; 5486 d = get_cg_data (&node, true); 5487 d->in_worklist = false; 5488 5489 if (d->want_irr_scan_normal) 5490 { 5491 d->want_irr_scan_normal = false; 5492 ipa_tm_scan_irr_function (node, false); 5493 } 5494 if (d->in_callee_queue && ipa_tm_scan_irr_function (node, true)) 5495 ipa_tm_note_irrevocable (node, &irr_worklist); 5496 } 5497 5498 /* For every function on the callee list, collect the tm_may_enter_irr 5499 bit on the node. */ 5500 irr_worklist.truncate (0); 5501 for (i = 0; i < tm_callees.length (); ++i) 5502 { 5503 node = tm_callees[i]; 5504 if (ipa_tm_mayenterirr_function (node)) 5505 { 5506 d = get_cg_data (&node, true); 5507 gcc_assert (d->in_worklist == false); 5508 maybe_push_queue (node, &irr_worklist, &d->in_worklist); 5509 } 5510 } 5511 5512 /* Propagate the tm_may_enter_irr bit to callers until stable. */ 5513 for (i = 0; i < irr_worklist.length (); ++i) 5514 { 5515 struct cgraph_node *caller; 5516 struct cgraph_edge *e; 5517 struct ipa_ref *ref; 5518 5519 if (i > 256 && i == irr_worklist.length () / 8) 5520 { 5521 irr_worklist.block_remove (0, i); 5522 i = 0; 5523 } 5524 5525 node = irr_worklist[i]; 5526 d = get_cg_data (&node, true); 5527 d->in_worklist = false; 5528 node->local.tm_may_enter_irr = true; 5529 5530 /* Propagate back to normal callers. */ 5531 for (e = node->callers; e ; e = e->next_caller) 5532 { 5533 caller = e->caller; 5534 if (!is_tm_safe_or_pure (caller->decl) 5535 && !caller->local.tm_may_enter_irr) 5536 { 5537 d = get_cg_data (&caller, true); 5538 maybe_push_queue (caller, &irr_worklist, &d->in_worklist); 5539 } 5540 } 5541 5542 /* Propagate back to referring aliases as well. */ 5543 FOR_EACH_ALIAS (node, ref) 5544 { 5545 caller = dyn_cast<cgraph_node *> (ref->referring); 5546 if (!caller->local.tm_may_enter_irr) 5547 { 5548 /* ?? Do not traverse aliases here. */ 5549 d = get_cg_data (&caller, false); 5550 maybe_push_queue (caller, &irr_worklist, &d->in_worklist); 5551 } 5552 } 5553 } 5554 5555 /* Now validate all tm_safe functions, and all atomic regions in 5556 other functions. */ 5557 FOR_EACH_DEFINED_FUNCTION (node) 5558 if (node->lowered 5559 && node->get_availability () >= AVAIL_INTERPOSABLE) 5560 { 5561 d = get_cg_data (&node, true); 5562 if (is_tm_safe (node->decl)) 5563 ipa_tm_diagnose_tm_safe (node); 5564 else if (d->all_tm_regions) 5565 ipa_tm_diagnose_transaction (node, d->all_tm_regions); 5566 } 5567 5568 /* Create clones. Do those that are not irrevocable and have a 5569 positive call count. Do those publicly visible functions that 5570 the user directed us to clone. */ 5571 for (i = 0; i < tm_callees.length (); ++i) 5572 { 5573 bool doit = false; 5574 5575 node = tm_callees[i]; 5576 if (node->cpp_implicit_alias) 5577 continue; 5578 5579 a = node->get_availability (); 5580 d = get_cg_data (&node, true); 5581 5582 if (a <= AVAIL_NOT_AVAILABLE) 5583 doit = is_tm_callable (node->decl); 5584 else if (a <= AVAIL_AVAILABLE && is_tm_callable (node->decl)) 5585 doit = true; 5586 else if (!d->is_irrevocable 5587 && d->tm_callers_normal + d->tm_callers_clone > 0) 5588 doit = true; 5589 5590 if (doit) 5591 ipa_tm_create_version (node); 5592 } 5593 5594 /* Redirect calls to the new clones, and insert irrevocable marks. */ 5595 for (i = 0; i < tm_callees.length (); ++i) 5596 { 5597 node = tm_callees[i]; 5598 if (node->analyzed) 5599 { 5600 d = get_cg_data (&node, true); 5601 if (d->clone) 5602 ipa_tm_transform_clone (node); 5603 } 5604 } 5605 FOR_EACH_DEFINED_FUNCTION (node) 5606 if (node->lowered 5607 && node->get_availability () >= AVAIL_INTERPOSABLE) 5608 { 5609 d = get_cg_data (&node, true); 5610 if (d->all_tm_regions) 5611 ipa_tm_transform_transaction (node); 5612 } 5613 5614 /* Free and clear all data structures. */ 5615 tm_callees.release (); 5616 irr_worklist.release (); 5617 bitmap_obstack_release (&tm_obstack); 5618 free_original_copy_tables (); 5619 5620 FOR_EACH_FUNCTION (node) 5621 node->aux = NULL; 5622 5623#ifdef ENABLE_CHECKING 5624 cgraph_node::verify_cgraph_nodes (); 5625#endif 5626 5627 return 0; 5628} 5629 5630namespace { 5631 5632const pass_data pass_data_ipa_tm = 5633{ 5634 SIMPLE_IPA_PASS, /* type */ 5635 "tmipa", /* name */ 5636 OPTGROUP_NONE, /* optinfo_flags */ 5637 TV_TRANS_MEM, /* tv_id */ 5638 ( PROP_ssa | PROP_cfg ), /* properties_required */ 5639 0, /* properties_provided */ 5640 0, /* properties_destroyed */ 5641 0, /* todo_flags_start */ 5642 0, /* todo_flags_finish */ 5643}; 5644 5645class pass_ipa_tm : public simple_ipa_opt_pass 5646{ 5647public: 5648 pass_ipa_tm (gcc::context *ctxt) 5649 : simple_ipa_opt_pass (pass_data_ipa_tm, ctxt) 5650 {} 5651 5652 /* opt_pass methods: */ 5653 virtual bool gate (function *) { return flag_tm; } 5654 virtual unsigned int execute (function *) { return ipa_tm_execute (); } 5655 5656}; // class pass_ipa_tm 5657 5658} // anon namespace 5659 5660simple_ipa_opt_pass * 5661make_pass_ipa_tm (gcc::context *ctxt) 5662{ 5663 return new pass_ipa_tm (ctxt); 5664} 5665 5666#include "gt-trans-mem.h" 5667