1 /*
2 * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "ci/ciUtilities.hpp"
27 #include "gc/shared/cardTable.hpp"
28 #include "gc/shared/cardTableBarrierSet.hpp"
29 #include "gc/shared/c2/cardTableBarrierSetC2.hpp"
30 #include "opto/arraycopynode.hpp"
31 #include "opto/graphKit.hpp"
32 #include "opto/idealKit.hpp"
33 #include "opto/macro.hpp"
34 #include "opto/rootnode.hpp"
35 #include "utilities/macros.hpp"
36
37 #define __ ideal.
38
39 Node* CardTableBarrierSetC2::byte_map_base_node(GraphKit* kit) const {
40 // Get base of card map
41 CardTable::CardValue* card_table_base = ci_card_table_address();
42 if (card_table_base != NULL) {
43 return kit->makecon(TypeRawPtr::make((address)card_table_base));
44 } else {
45 return kit->null();
46 }
47 }
48
49 // vanilla post barrier
50 // Insert a write-barrier store. This is to let generational GC work; we have
51 // to flag all oop-stores before the next GC point.
52 void CardTableBarrierSetC2::post_barrier(GraphKit* kit,
53 Node* ctl,
54 Node* oop_store,
55 Node* obj,
56 Node* adr,
57 uint adr_idx,
58 Node* val,
59 BasicType bt,
60 bool use_precise) const {
61 // No store check needed if we're storing a NULL or an old object
62 // (latter case is probably a string constant). The concurrent
63 // mark sweep garbage collector, however, needs to have all nonNull
64 // oop updates flagged via card-marks.
65 if (val != NULL && val->is_Con()) {
66 // must be either an oop or NULL
67 const Type* t = val->bottom_type();
68 if (t == TypePtr::NULL_PTR || t == Type::TOP)
69 // stores of null never (?) need barriers
70 return;
71 }
72
73 if (use_ReduceInitialCardMarks()
74 && obj == kit->just_allocated_object(kit->control())) {
75 // We can skip marks on a freshly-allocated object in Eden.
76 // Keep this code in sync with new_deferred_store_barrier() in runtime.cpp.
77 // That routine informs GC to take appropriate compensating steps,
78 // upon a slow-path allocation, so as to make this card-mark
79 // elision safe.
80 return;
81 }
82
83 if (!use_precise) {
84 // All card marks for a (non-array) instance are in one place:
85 adr = obj;
86 }
87 // (Else it's an array (or unknown), and we want more precise card marks.)
88 assert(adr != NULL, "");
89
90 IdealKit ideal(kit, true);
91
92 BarrierSet* bs = BarrierSet::barrier_set();
93 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
94 CardTable* ct = ctbs->card_table();
95
96 float likely = PROB_LIKELY_MAG(3);
97
98 // Convert the pointer to an int prior to doing math on it
99 Node* cast = __ CastPX(__ ctrl(), adr);
100
101 // Divide by card size
102 Node* card_offset = __ URShiftX( cast, __ ConI(CardTable::card_shift) );
103
104 // Combine card table base and card offset
105 Node* card_adr = __ AddP(__ top(), byte_map_base_node(kit), card_offset );
106
107 // Get the alias_index for raw card-mark memory
108 int adr_type = Compile::AliasIdxRaw;
109 Node* zero = __ ConI(0); // Dirty card value
110
111 if (kit->C->do_stack_allocation()) {
112 // Stack allocation: cache CastP2XNode for later processing
113 state()->add_enqueue_barrier(static_cast<CastP2XNode*>(cast));
114
115 Node* low_off = kit->longcon(ct->byte_map_bottom_offset());
116 Node* delta_off = kit->longcon(ct->byte_map_top_offset() - ct->byte_map_bottom_offset());
117 Node* sub_off = __ SubL(cast, low_off);
118
119 __ uif_then(sub_off, BoolTest::le, delta_off, likely); } {
120
121 if (UseCondCardMark) {
122 if (ct->scanned_concurrently()) {
123 kit->insert_mem_bar(Op_MemBarVolatile, oop_store);
124 __ sync_kit(kit);
125 }
126 // The classic GC reference write barrier is typically implemented
127 // as a store into the global card mark table. Unfortunately
128 // unconditional stores can result in false sharing and excessive
129 // coherence traffic as well as false transactional aborts.
130 // UseCondCardMark enables MP "polite" conditional card mark
131 // stores. In theory we could relax the load from ctrl() to
132 // no_ctrl, but that doesn't buy much latitude.
133 Node* card_val = __ load( __ ctrl(), card_adr, TypeInt::BYTE, T_BYTE, adr_type);
134 __ if_then(card_val, BoolTest::ne, zero);
135 }
136
137 // Smash zero into card
138 if(!ct->scanned_concurrently()) {
139 __ store(__ ctrl(), card_adr, zero, T_BYTE, adr_type, MemNode::unordered);
140 } else {
141 // Specialized path for CM store barrier
142 __ storeCM(__ ctrl(), card_adr, zero, oop_store, adr_idx, T_BYTE, adr_type);
143 }
144
145 if (UseCondCardMark) {
146 __ end_if();
147 }
148 } if (kit->C->do_stack_allocation()) {
149 __ end_if();
150 }
151
152 // Final sync IdealKit and GraphKit.
153 kit->final_sync(ideal);
154 }
155
156 void CardTableBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const {
157 BarrierSetC2::clone(kit, src, dst, size, is_array);
158 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
159
160 // If necessary, emit some card marks afterwards. (Non-arrays only.)
161 bool card_mark = !is_array && !use_ReduceInitialCardMarks();
162 if (card_mark) {
163 assert(!is_array, "");
164 // Put in store barrier for any and all oops we are sticking
165 // into this object. (We could avoid this if we could prove
166 // that the object type contains no oop fields at all.)
167 Node* no_particular_value = NULL;
168 Node* no_particular_field = NULL;
169 int raw_adr_idx = Compile::AliasIdxRaw;
170 post_barrier(kit, kit->control(),
171 kit->memory(raw_adr_type),
172 dst,
173 no_particular_field,
174 raw_adr_idx,
175 no_particular_value,
176 T_OBJECT,
177 false);
178 }
179 }
180
181 bool CardTableBarrierSetC2::use_ReduceInitialCardMarks() const {
182 return ReduceInitialCardMarks;
183 }
184
185 bool CardTableBarrierSetC2::is_gc_barrier_node(Node* node) const {
186 return ModRefBarrierSetC2::is_gc_barrier_node(node) || node->Opcode() == Op_StoreCM;
187 }
188
189 bool CardTableBarrierSetC2::process_barrier_node(Node* node, PhaseIterGVN& igvn) const {
190 assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required");
191
192 // Must have a control node
193 if (node->in(0) == NULL) {
194 return false;
195 }
196
197 Node *addx_node = node->find_out_with(Op_AddX);
198 if (addx_node == NULL) {
199 return false;
200 }
201
202 Node *addx_out = addx_node->unique_out();
203 if (addx_out == NULL) {
204 return false;
205 }
206
207 CmpNode* cmp_node = addx_out->as_Cmp();
208 // the input to the CMPX is the card_table_top_offset constant
209 Node* cmp_node_in_2_node = cmp_node->in(2);
210 if (!cmp_node_in_2_node->is_Con()) {
211 return false;
212 }
213
214 BarrierSet* bs = BarrierSet::barrier_set();
215 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
216 CardTable* ct = ctbs->card_table();
217 size_t constant = ct->byte_map_top_offset() - ct->byte_map_bottom_offset();
218
219 // Check that the input to this CMP node is the expected constant
220 const TypeX* otype = cmp_node_in_2_node->find_intptr_t_type();
221 if (otype != NULL && otype->is_con() &&
222 size_t(otype->get_con()) != constant) {
223 // Constant offset but not the card table size constant so just return
224 return false;
225 }
226
227 // we can't change the compare or the constant so create a new constant(0) and replace the variable
228 Node* cmp_node_in_1_node = cmp_node->in(1);
229 ConNode* zeroConstant_node = igvn.makecon(TypeX_ZERO);
230 if (cmp_node_in_1_node->_idx == zeroConstant_node->_idx) {
231 // we can get here via different nodes - but we only want to change the input once
232 return false;
233 }
234
235 igvn.rehash_node_delayed(cmp_node);
236 int numReplaced = cmp_node->replace_edge(cmp_node_in_1_node, zeroConstant_node);
237 assert(numReplaced == 1, "Failed to replace the card_offset with Conx(0)");
238 igvn.replace_node(addx_node, igvn.C->top());
239
240 return true;
241 }
242
243 void CardTableBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
244 assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required");
245 assert(node->outcnt() <= 2, "node->outcnt() <= 2");
246
247 // Certain loop optimisations may introduce a CastP2X node with
248 // ConvL2I in case of an AllocateArray op. Check for that case
249 // here and do not attempt to eliminate it as write barrier.
250 if (macro->C->do_stack_allocation() && !state()->is_a_barrier(static_cast<CastP2XNode*>(node))) {
251 return;
252 }
253
254 Node *shift = node->find_out_with(Op_URShiftX);
255 Node *addp = shift->unique_out();
256 for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
257 Node *mem = addp->last_out(j);
258 if (UseCondCardMark && mem->is_Load()) {
259 assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
260 // The load is checking if the card has been written so
261 // replace it with zero to fold the test.
262 macro->replace_node(mem, macro->intcon(0));
263 continue;
264 }
265 assert(mem->is_Store(), "store required");
266 macro->replace_node(mem, mem->in(MemNode::Memory));
267 }
268
269 if (macro->C->do_stack_allocation()) {
270 Node *addl_node = node->find_out_with(Op_AddL);
271 assert(addl_node != NULL, "stackallocation expects addl");
272
273 Node* cmp_node = addl_node->unique_out();
274 assert(cmp_node != NULL && cmp_node->is_Cmp(), "expected unique cmp node");
275
276 macro->replace_node(cmp_node, macro->makecon(TypeInt::CC_EQ));
277 }
278
279 // Stack allocation: remove this node from our cache so we don't process it later
280 state()->remove_enqueue_barrier(static_cast<CastP2XNode*>(node));
281 }
282
283 bool CardTableBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, ArrayCopyPhase phase) const {
284 bool is_oop = is_reference_type(type);
285 return is_oop && (!tightly_coupled_alloc || !use_ReduceInitialCardMarks());
286 }
287
288 bool CardTableBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
289 // We need to process write barriers for extra checks in case we have stack allocation on
290 if (C->do_stack_allocation()) {
291 BarrierSetC2State* set_state = state();
292
293 for (int i = 0; i < set_state->enqueue_barriers_count(); i++) {
294 Node* n = set_state->enqueue_barrier(i);
295 process_barrier_node(n, igvn);
296 }
297
298 if (set_state->enqueue_barriers_count()) {
299 // this kicks in the dead code elimination we need to remove the redundant check
300 igvn.optimize();
301 }
302 }
303
304 return false;
305 }
306
307 Node* CardTableBarrierSetC2::step_over_gc_barrier(Node* c) const {
308 if (Compile::current()->do_stack_allocation() &&
309 !use_ReduceInitialCardMarks() &&
310 c != NULL && c->is_Region() && c->req() == 3) {
311
312 // [Proj] <----------- step over to here and return
313 // |
314 // -----------
315 // / \
316 // / \
317 // / [CastP2X]
318 // | /
319 // | [AddL]
320 // | /
321 // | [CmpUL]
322 // | /
323 // \ [Bool]
324 // \ /
325 // [If]
326 // / \
327 // [IfFalse] [IfTrue]
328 // \ /
329 // [Region] <---------------- c node
330
331 Node* if_bool = c->in(1);
332 assert(if_bool->is_IfTrue() || if_bool->is_IfFalse(), "Invalid gc graph pattern");
333 Node* if_node = if_bool->in(0);
334 Node* proj_node = if_node->in(0);
335 assert(proj_node->is_Proj(), "Invalid gc graph pattern");
336 return proj_node;
337 }
338 return c;
339 }
340
341 void CardTableBarrierSetC2::register_potential_barrier_node(Node* node) const {
342 if (node->Opcode() == Op_CastP2X) {
343 state()->add_enqueue_barrier(static_cast<CastP2XNode*>(node));
344 }
345 }
346
347 void CardTableBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
348 if (node->Opcode() == Op_CastP2X) {
349 state()->remove_enqueue_barrier(static_cast<CastP2XNode*>(node));
350 }
351 }
352
353 BarrierSetC2State* CardTableBarrierSetC2::state() const {
354 BarrierSetC2State* ret = reinterpret_cast<BarrierSetC2State*>(Compile::current()->barrier_set_state());
355 assert(ret != NULL, "Sanity");
356 return ret;
357 }
358
359 void* CardTableBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
360 return new(comp_arena) BarrierSetC2State(comp_arena);
361 }
362
363 BarrierSetC2State::BarrierSetC2State(Arena* comp_arena)
364 : _enqueue_barriers(new (comp_arena) GrowableArray<CastP2XNode*>(comp_arena, 8, 0, NULL)) {
365 }
366
367 int BarrierSetC2State::enqueue_barriers_count() const {
368 return _enqueue_barriers->length();
369 }
370
371 CastP2XNode* BarrierSetC2State::enqueue_barrier(int idx) const {
372 return _enqueue_barriers->at(idx);
373 }
374
375 void BarrierSetC2State::add_enqueue_barrier(CastP2XNode* n) {
376 assert(!_enqueue_barriers->contains(n), "duplicate entry in barrier list");
377 _enqueue_barriers->append(n);
378 }
379
380 void BarrierSetC2State::remove_enqueue_barrier(CastP2XNode* n) {
381 if (_enqueue_barriers->contains(n)) {
382 _enqueue_barriers->remove(n);
383 }
384 }
385
386 bool BarrierSetC2State::is_a_barrier(CastP2XNode* n) {
387 return _enqueue_barriers->contains(n);
388 }