1/* Copyright (C) 2012-2015 Free Software Foundation, Inc.
2   Contributed by Richard Henderson <rth@redhat.com>.
3
4   This file is part of the GNU Atomic Library (libatomic).
5
6   Libatomic is free software; you can redistribute it and/or modify it
7   under the terms of the GNU General Public License as published by
8   the Free Software Foundation; either version 3 of the License, or
9   (at your option) any later version.
10
11   Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY
12   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
13   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
14   more details.
15
16   Under Section 7 of GPL version 3, you are granted additional
17   permissions described in the GCC Runtime Library Exception, version
18   3.1, as published by the Free Software Foundation.
19
20   You should have received a copy of the GNU General Public License and
21   a copy of the GCC Runtime Library Exception along with this program;
22   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
23   <http://www.gnu.org/licenses/>.  */
24
25#include <libatomic_i.h>
26
27
28/* This file is included multiple times with required defines:
29     NAME	the name of the operation that we're implementing;
30     OP		a two-operand functional macro the implements the operation.
31*/
32
33
34/* If we support the builtin, just use it.  */
35#if !DONE && SIZE(HAVE_ATOMIC_FETCH_OP)
36UTYPE
37SIZE(C2(libat_fetch_,NAME)) (UTYPE *mptr, UTYPE opval, int smodel)
38{
39  if (maybe_specialcase_relaxed(smodel))
40    return C2(__atomic_fetch_,NAME) (mptr, opval, __ATOMIC_RELAXED);
41  else if (maybe_specialcase_acqrel(smodel))
42    return C2(__atomic_fetch_,NAME) (mptr, opval, __ATOMIC_ACQ_REL);
43  else
44    return C2(__atomic_fetch_,NAME) (mptr, opval, __ATOMIC_SEQ_CST);
45}
46
47UTYPE
48SIZE(C3(libat_,NAME,_fetch)) (UTYPE *mptr, UTYPE opval, int smodel)
49{
50  if (maybe_specialcase_relaxed(smodel))
51    return C3(__atomic_,NAME,_fetch) (mptr, opval, __ATOMIC_RELAXED);
52  else if (maybe_specialcase_acqrel(smodel))
53    return C3(__atomic_,NAME,_fetch) (mptr, opval, __ATOMIC_ACQ_REL);
54  else
55    return C3(__atomic_,NAME,_fetch) (mptr, opval, __ATOMIC_SEQ_CST);
56}
57
58#define DONE 1
59#endif /* HAVE_ATOMIC_FETCH_OP */
60
61
62#if !DONE && defined(atomic_compare_exchange_n)
63UTYPE
64SIZE(C2(libat_fetch_,NAME)) (UTYPE *mptr, UTYPE opval, int smodel)
65{
66  UTYPE oldval, t;
67
68  pre_barrier (smodel);
69
70  oldval = *mptr;
71  do
72    {
73      t = OP(oldval, opval);
74    }
75  while (!atomic_compare_exchange_n (mptr, &oldval, t, true,
76				     __ATOMIC_RELAXED, __ATOMIC_RELAXED));
77
78  post_barrier (smodel);
79  return oldval;
80}
81
82UTYPE
83SIZE(C3(libat_,NAME,_fetch)) (UTYPE *mptr, UTYPE opval, int smodel)
84{
85  UTYPE oldval, t;
86
87  pre_barrier (smodel);
88
89  oldval = *mptr;
90  do
91    {
92      t = OP(oldval, opval);
93    }
94  while (!atomic_compare_exchange_n (mptr, &oldval, t, true,
95				     __ATOMIC_RELAXED, __ATOMIC_RELAXED));
96
97  post_barrier (smodel);
98  return t;
99}
100
101#define DONE 1
102#endif /* atomic_compare_exchange_n */
103
104
105/* If this type is no larger than word-sized, fall back to a word-sized
106   compare-and-swap loop.  */
107#if !DONE && N < WORDSIZE && defined(atomic_compare_exchange_w)
108UTYPE
109SIZE(C2(libat_fetch_,NAME)) (UTYPE *mptr, UTYPE opval, int smodel)
110{
111  UWORD mask, shift, woldval, wopval, t, *wptr;
112
113  pre_barrier (smodel);
114
115  wptr = (UWORD *)((uintptr_t)mptr & -WORDSIZE);
116  shift = (((uintptr_t)mptr % WORDSIZE) * CHAR_BIT) ^ SIZE(INVERT_MASK);
117  mask = SIZE(MASK) << shift;
118
119  wopval = (UWORD)opval << shift;
120  woldval = __atomic_load_n (wptr, __ATOMIC_RELAXED);
121  do
122    {
123      t = (woldval & ~mask) | (OP(woldval, wopval) & mask);
124    }
125  while (!atomic_compare_exchange_w (wptr, &woldval, t, true,
126				     __ATOMIC_RELAXED, __ATOMIC_RELAXED));
127
128  post_barrier (smodel);
129  return woldval >> shift;
130}
131
132UTYPE
133SIZE(C3(libat_,NAME,_fetch)) (UTYPE *mptr, UTYPE opval, int smodel)
134{
135  UWORD mask, shift, woldval, wopval, t, *wptr;
136
137  pre_barrier (smodel);
138
139  wptr = (UWORD *)((uintptr_t)mptr & -WORDSIZE);
140  shift = (((uintptr_t)mptr % WORDSIZE) * CHAR_BIT) ^ SIZE(INVERT_MASK);
141  mask = SIZE(MASK) << shift;
142
143  wopval = (UWORD)opval << shift;
144  woldval = __atomic_load_n (wptr, __ATOMIC_RELAXED);
145  do
146    {
147      t = (woldval & ~mask) | (OP(woldval, wopval) & mask);
148    }
149  while (!atomic_compare_exchange_w (wptr, &woldval, t, true,
150				     __ATOMIC_RELAXED, __ATOMIC_RELAXED));
151
152  post_barrier (smodel);
153  return t >> shift;
154}
155
156#define DONE 1
157#endif /* atomic_compare_exchange_w */
158
159
160/* Otherwise, fall back to some sort of protection mechanism.  */
161#if !DONE
162UTYPE
163SIZE(C2(libat_fetch_,NAME)) (UTYPE *mptr, UTYPE opval, int smodel UNUSED)
164{
165  UTYPE ret;
166  UWORD magic;
167
168  pre_seq_barrier (smodel);
169  magic = protect_start (mptr);
170
171  ret = *mptr;
172  *mptr = OP(ret, opval);
173
174  protect_end (mptr, magic);
175  post_seq_barrier (smodel);
176
177  return ret;
178}
179
180UTYPE
181SIZE(C3(libat_,NAME,_fetch)) (UTYPE *mptr, UTYPE opval, int smodel UNUSED)
182{
183  UTYPE ret;
184  UWORD magic;
185
186  pre_seq_barrier (smodel);
187  magic = protect_start (mptr);
188
189  ret = OP (*mptr, opval);
190  *mptr = ret;
191
192  protect_end (mptr, magic);
193  post_seq_barrier (smodel);
194
195  return ret;
196}
197#endif
198
199EXPORT_ALIAS (SIZE(C2(fetch_,NAME)));
200EXPORT_ALIAS (SIZE(C2(NAME,_fetch)));
201