InstCombineAtomicRMW.cpp revision 360784 
1//===- InstCombineAtomicRMW.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visit functions for atomic rmw instructions.
10//
11//===----------------------------------------------------------------------===//
12#include "InstCombineInternal.h"
13#include "llvm/IR/Instructions.h"
14
15using namespace llvm;
16
17namespace {
18/// Return true if and only if the given instruction does not modify the memory
19/// location referenced.  Note that an idemptent atomicrmw may still have
20/// ordering effects on nearby instructions, or be volatile.
21/// TODO: Common w/ the version in AtomicExpandPass, and change the term used.
22/// Idemptotent is confusing in this context.
23bool isIdempotentRMW(AtomicRMWInst& RMWI) {
24  if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
25    switch(RMWI.getOperation()) {
26    case AtomicRMWInst::FAdd: // -0.0
27      return CF->isZero() && CF->isNegative();
28    case AtomicRMWInst::FSub: // +0.0
29      return CF->isZero() && !CF->isNegative();
30    default:
31      return false;
32    };
33
34  auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
35  if(!C)
36    return false;
37
38  switch(RMWI.getOperation()) {
39    case AtomicRMWInst::Add:
40    case AtomicRMWInst::Sub:
41    case AtomicRMWInst::Or:
42    case AtomicRMWInst::Xor:
43      return C->isZero();
44    case AtomicRMWInst::And:
45      return C->isMinusOne();
46    case AtomicRMWInst::Min:
47      return C->isMaxValue(true);
48    case AtomicRMWInst::Max:
49      return C->isMinValue(true);
50    case AtomicRMWInst::UMin:
51      return C->isMaxValue(false);
52    case AtomicRMWInst::UMax:
53      return C->isMinValue(false);
54    default:
55      return false;
56  }
57}
58
59/// Return true if the given instruction always produces a value in memory
60/// equivalent to its value operand.
61bool isSaturating(AtomicRMWInst& RMWI) {
62  if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
63    switch(RMWI.getOperation()) {
64    case AtomicRMWInst::FAdd:
65    case AtomicRMWInst::FSub:
66      return CF->isNaN();
67    default:
68      return false;
69    };
70
71  auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
72  if(!C)
73    return false;
74
75  switch(RMWI.getOperation()) {
76  default:
77    return false;
78  case AtomicRMWInst::Xchg:
79    return true;
80  case AtomicRMWInst::Or:
81    return C->isAllOnesValue();
82  case AtomicRMWInst::And:
83    return C->isZero();
84  case AtomicRMWInst::Min:
85    return C->isMinValue(true);
86  case AtomicRMWInst::Max:
87    return C->isMaxValue(true);
88  case AtomicRMWInst::UMin:
89    return C->isMinValue(false);
90  case AtomicRMWInst::UMax:
91    return C->isMaxValue(false);
92  };
93}
94}
95
96Instruction *InstCombiner::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
97
98  // Volatile RMWs perform a load and a store, we cannot replace this by just a
99  // load or just a store. We chose not to canonicalize out of general paranoia
100  // about user expectations around volatile.
101  if (RMWI.isVolatile())
102    return nullptr;
103
104  // Any atomicrmw op which produces a known result in memory can be
105  // replaced w/an atomicrmw xchg.
106  if (isSaturating(RMWI) &&
107      RMWI.getOperation() != AtomicRMWInst::Xchg) {
108    RMWI.setOperation(AtomicRMWInst::Xchg);
109    return &RMWI;
110  }
111
112  AtomicOrdering Ordering = RMWI.getOrdering();
113  assert(Ordering != AtomicOrdering::NotAtomic &&
114         Ordering != AtomicOrdering::Unordered &&
115         "AtomicRMWs don't make sense with Unordered or NotAtomic");
116
117  // Any atomicrmw xchg with no uses can be converted to a atomic store if the
118  // ordering is compatible.
119  if (RMWI.getOperation() == AtomicRMWInst::Xchg &&
120      RMWI.use_empty()) {
121    if (Ordering != AtomicOrdering::Release &&
122        Ordering != AtomicOrdering::Monotonic)
123      return nullptr;
124    auto *SI = new StoreInst(RMWI.getValOperand(),
125                             RMWI.getPointerOperand(), &RMWI);
126    SI->setAtomic(Ordering, RMWI.getSyncScopeID());
127    SI->setAlignment(MaybeAlign(DL.getABITypeAlignment(RMWI.getType())));
128    return eraseInstFromFunction(RMWI);
129  }
130
131  if (!isIdempotentRMW(RMWI))
132    return nullptr;
133
134  // We chose to canonicalize all idempotent operations to an single
135  // operation code and constant.  This makes it easier for the rest of the
136  // optimizer to match easily.  The choices of or w/0 and fadd w/-0.0 are
137  // arbitrary.
138  if (RMWI.getType()->isIntegerTy() &&
139      RMWI.getOperation() != AtomicRMWInst::Or) {
140    RMWI.setOperation(AtomicRMWInst::Or);
141    RMWI.setOperand(1, ConstantInt::get(RMWI.getType(), 0));
142    return &RMWI;
143  } else if (RMWI.getType()->isFloatingPointTy() &&
144             RMWI.getOperation() != AtomicRMWInst::FAdd) {
145    RMWI.setOperation(AtomicRMWInst::FAdd);
146    RMWI.setOperand(1, ConstantFP::getNegativeZero(RMWI.getType()));
147    return &RMWI;
148  }
149
150  // Check if the required ordering is compatible with an atomic load.
151  if (Ordering != AtomicOrdering::Acquire &&
152      Ordering != AtomicOrdering::Monotonic)
153    return nullptr;
154
155  LoadInst *Load = new LoadInst(RMWI.getType(), RMWI.getPointerOperand());
156  Load->setAtomic(Ordering, RMWI.getSyncScopeID());
157  Load->setAlignment(MaybeAlign(DL.getABITypeAlignment(RMWI.getType())));
158  return Load;
159}
160