1/* Copyright (C) 2012-2015 Free Software Foundation, Inc. 2 Contributed by Richard Henderson <rth@redhat.com>. 3 4 This file is part of the GNU Atomic Library (libatomic). 5 6 Libatomic is free software; you can redistribute it and/or modify it 7 under the terms of the GNU General Public License as published by 8 the Free Software Foundation; either version 3 of the License, or 9 (at your option) any later version. 10 11 Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY 12 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 more details. 15 16 Under Section 7 of GPL version 3, you are granted additional 17 permissions described in the GCC Runtime Library Exception, version 18 3.1, as published by the Free Software Foundation. 19 20 You should have received a copy of the GNU General Public License and 21 a copy of the GCC Runtime Library Exception along with this program; 22 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 23 <http://www.gnu.org/licenses/>. */ 24 25#include <libatomic_i.h> 26 27 28/* This file is included multiple times with required defines: 29 NAME the name of the operation that we're implementing; 30 OP a two-operand functional macro the implements the operation. 31*/ 32 33 34/* If we support the builtin, just use it. */ 35#if !DONE && SIZE(HAVE_ATOMIC_FETCH_OP) 36UTYPE 37SIZE(C2(libat_fetch_,NAME)) (UTYPE *mptr, UTYPE opval, int smodel) 38{ 39 if (maybe_specialcase_relaxed(smodel)) 40 return C2(__atomic_fetch_,NAME) (mptr, opval, __ATOMIC_RELAXED); 41 else if (maybe_specialcase_acqrel(smodel)) 42 return C2(__atomic_fetch_,NAME) (mptr, opval, __ATOMIC_ACQ_REL); 43 else 44 return C2(__atomic_fetch_,NAME) (mptr, opval, __ATOMIC_SEQ_CST); 45} 46 47UTYPE 48SIZE(C3(libat_,NAME,_fetch)) (UTYPE *mptr, UTYPE opval, int smodel) 49{ 50 if (maybe_specialcase_relaxed(smodel)) 51 return C3(__atomic_,NAME,_fetch) (mptr, opval, __ATOMIC_RELAXED); 52 else if (maybe_specialcase_acqrel(smodel)) 53 return C3(__atomic_,NAME,_fetch) (mptr, opval, __ATOMIC_ACQ_REL); 54 else 55 return C3(__atomic_,NAME,_fetch) (mptr, opval, __ATOMIC_SEQ_CST); 56} 57 58#define DONE 1 59#endif /* HAVE_ATOMIC_FETCH_OP */ 60 61 62#if !DONE && defined(atomic_compare_exchange_n) 63UTYPE 64SIZE(C2(libat_fetch_,NAME)) (UTYPE *mptr, UTYPE opval, int smodel) 65{ 66 UTYPE oldval, t; 67 68 pre_barrier (smodel); 69 70 oldval = *mptr; 71 do 72 { 73 t = OP(oldval, opval); 74 } 75 while (!atomic_compare_exchange_n (mptr, &oldval, t, true, 76 __ATOMIC_RELAXED, __ATOMIC_RELAXED)); 77 78 post_barrier (smodel); 79 return oldval; 80} 81 82UTYPE 83SIZE(C3(libat_,NAME,_fetch)) (UTYPE *mptr, UTYPE opval, int smodel) 84{ 85 UTYPE oldval, t; 86 87 pre_barrier (smodel); 88 89 oldval = *mptr; 90 do 91 { 92 t = OP(oldval, opval); 93 } 94 while (!atomic_compare_exchange_n (mptr, &oldval, t, true, 95 __ATOMIC_RELAXED, __ATOMIC_RELAXED)); 96 97 post_barrier (smodel); 98 return t; 99} 100 101#define DONE 1 102#endif /* atomic_compare_exchange_n */ 103 104 105/* If this type is no larger than word-sized, fall back to a word-sized 106 compare-and-swap loop. */ 107#if !DONE && N < WORDSIZE && defined(atomic_compare_exchange_w) 108UTYPE 109SIZE(C2(libat_fetch_,NAME)) (UTYPE *mptr, UTYPE opval, int smodel) 110{ 111 UWORD mask, shift, woldval, wopval, t, *wptr; 112 113 pre_barrier (smodel); 114 115 wptr = (UWORD *)((uintptr_t)mptr & -WORDSIZE); 116 shift = (((uintptr_t)mptr % WORDSIZE) * CHAR_BIT) ^ SIZE(INVERT_MASK); 117 mask = SIZE(MASK) << shift; 118 119 wopval = (UWORD)opval << shift; 120 woldval = __atomic_load_n (wptr, __ATOMIC_RELAXED); 121 do 122 { 123 t = (woldval & ~mask) | (OP(woldval, wopval) & mask); 124 } 125 while (!atomic_compare_exchange_w (wptr, &woldval, t, true, 126 __ATOMIC_RELAXED, __ATOMIC_RELAXED)); 127 128 post_barrier (smodel); 129 return woldval >> shift; 130} 131 132UTYPE 133SIZE(C3(libat_,NAME,_fetch)) (UTYPE *mptr, UTYPE opval, int smodel) 134{ 135 UWORD mask, shift, woldval, wopval, t, *wptr; 136 137 pre_barrier (smodel); 138 139 wptr = (UWORD *)((uintptr_t)mptr & -WORDSIZE); 140 shift = (((uintptr_t)mptr % WORDSIZE) * CHAR_BIT) ^ SIZE(INVERT_MASK); 141 mask = SIZE(MASK) << shift; 142 143 wopval = (UWORD)opval << shift; 144 woldval = __atomic_load_n (wptr, __ATOMIC_RELAXED); 145 do 146 { 147 t = (woldval & ~mask) | (OP(woldval, wopval) & mask); 148 } 149 while (!atomic_compare_exchange_w (wptr, &woldval, t, true, 150 __ATOMIC_RELAXED, __ATOMIC_RELAXED)); 151 152 post_barrier (smodel); 153 return t >> shift; 154} 155 156#define DONE 1 157#endif /* atomic_compare_exchange_w */ 158 159 160/* Otherwise, fall back to some sort of protection mechanism. */ 161#if !DONE 162UTYPE 163SIZE(C2(libat_fetch_,NAME)) (UTYPE *mptr, UTYPE opval, int smodel UNUSED) 164{ 165 UTYPE ret; 166 UWORD magic; 167 168 pre_seq_barrier (smodel); 169 magic = protect_start (mptr); 170 171 ret = *mptr; 172 *mptr = OP(ret, opval); 173 174 protect_end (mptr, magic); 175 post_seq_barrier (smodel); 176 177 return ret; 178} 179 180UTYPE 181SIZE(C3(libat_,NAME,_fetch)) (UTYPE *mptr, UTYPE opval, int smodel UNUSED) 182{ 183 UTYPE ret; 184 UWORD magic; 185 186 pre_seq_barrier (smodel); 187 magic = protect_start (mptr); 188 189 ret = OP (*mptr, opval); 190 *mptr = ret; 191 192 protect_end (mptr, magic); 193 post_seq_barrier (smodel); 194 195 return ret; 196} 197#endif 198 199EXPORT_ALIAS (SIZE(C2(fetch_,NAME))); 200EXPORT_ALIAS (SIZE(C2(NAME,_fetch))); 201