1 |
684 |
jeremybenn |
/* Analysis Utilities for Loop Vectorization.
|
2 |
|
|
Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012
|
3 |
|
|
Free Software Foundation, Inc.
|
4 |
|
|
Contributed by Dorit Nuzman <dorit@il.ibm.com>
|
5 |
|
|
|
6 |
|
|
This file is part of GCC.
|
7 |
|
|
|
8 |
|
|
GCC is free software; you can redistribute it and/or modify it under
|
9 |
|
|
the terms of the GNU General Public License as published by the Free
|
10 |
|
|
Software Foundation; either version 3, or (at your option) any later
|
11 |
|
|
version.
|
12 |
|
|
|
13 |
|
|
GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
14 |
|
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
15 |
|
|
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
16 |
|
|
for more details.
|
17 |
|
|
|
18 |
|
|
You should have received a copy of the GNU General Public License
|
19 |
|
|
along with GCC; see the file COPYING3. If not see
|
20 |
|
|
<http://www.gnu.org/licenses/>. */
|
21 |
|
|
|
22 |
|
|
#include "config.h"
|
23 |
|
|
#include "system.h"
|
24 |
|
|
#include "coretypes.h"
|
25 |
|
|
#include "tm.h"
|
26 |
|
|
#include "ggc.h"
|
27 |
|
|
#include "tree.h"
|
28 |
|
|
#include "target.h"
|
29 |
|
|
#include "basic-block.h"
|
30 |
|
|
#include "gimple-pretty-print.h"
|
31 |
|
|
#include "tree-flow.h"
|
32 |
|
|
#include "tree-dump.h"
|
33 |
|
|
#include "cfgloop.h"
|
34 |
|
|
#include "expr.h"
|
35 |
|
|
#include "optabs.h"
|
36 |
|
|
#include "params.h"
|
37 |
|
|
#include "tree-data-ref.h"
|
38 |
|
|
#include "tree-vectorizer.h"
|
39 |
|
|
#include "recog.h"
|
40 |
|
|
#include "diagnostic-core.h"
|
41 |
|
|
|
42 |
|
|
/* Pattern recognition functions */
|
43 |
|
|
static gimple vect_recog_widen_sum_pattern (VEC (gimple, heap) **, tree *,
|
44 |
|
|
tree *);
|
45 |
|
|
static gimple vect_recog_widen_mult_pattern (VEC (gimple, heap) **, tree *,
|
46 |
|
|
tree *);
|
47 |
|
|
static gimple vect_recog_dot_prod_pattern (VEC (gimple, heap) **, tree *,
|
48 |
|
|
tree *);
|
49 |
|
|
static gimple vect_recog_pow_pattern (VEC (gimple, heap) **, tree *, tree *);
|
50 |
|
|
static gimple vect_recog_over_widening_pattern (VEC (gimple, heap) **, tree *,
|
51 |
|
|
tree *);
|
52 |
|
|
static gimple vect_recog_widen_shift_pattern (VEC (gimple, heap) **,
|
53 |
|
|
tree *, tree *);
|
54 |
|
|
static gimple vect_recog_vector_vector_shift_pattern (VEC (gimple, heap) **,
|
55 |
|
|
tree *, tree *);
|
56 |
|
|
static gimple vect_recog_sdivmod_pow2_pattern (VEC (gimple, heap) **,
|
57 |
|
|
tree *, tree *);
|
58 |
|
|
static gimple vect_recog_mixed_size_cond_pattern (VEC (gimple, heap) **,
|
59 |
|
|
tree *, tree *);
|
60 |
|
|
static gimple vect_recog_bool_pattern (VEC (gimple, heap) **, tree *, tree *);
|
61 |
|
|
static vect_recog_func_ptr vect_vect_recog_func_ptrs[NUM_PATTERNS] = {
|
62 |
|
|
vect_recog_widen_mult_pattern,
|
63 |
|
|
vect_recog_widen_sum_pattern,
|
64 |
|
|
vect_recog_dot_prod_pattern,
|
65 |
|
|
vect_recog_pow_pattern,
|
66 |
|
|
vect_recog_over_widening_pattern,
|
67 |
|
|
vect_recog_widen_shift_pattern,
|
68 |
|
|
vect_recog_vector_vector_shift_pattern,
|
69 |
|
|
vect_recog_sdivmod_pow2_pattern,
|
70 |
|
|
vect_recog_mixed_size_cond_pattern,
|
71 |
|
|
vect_recog_bool_pattern};
|
72 |
|
|
|
73 |
|
|
static inline void
|
74 |
|
|
append_pattern_def_seq (stmt_vec_info stmt_info, gimple stmt)
|
75 |
|
|
{
|
76 |
|
|
gimple_seq_add_stmt_without_update (&STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
|
77 |
|
|
stmt);
|
78 |
|
|
}
|
79 |
|
|
|
80 |
|
|
static inline void
|
81 |
|
|
new_pattern_def_seq (stmt_vec_info stmt_info, gimple stmt)
|
82 |
|
|
{
|
83 |
|
|
STMT_VINFO_PATTERN_DEF_SEQ (stmt_info) = NULL;
|
84 |
|
|
append_pattern_def_seq (stmt_info, stmt);
|
85 |
|
|
}
|
86 |
|
|
|
87 |
|
|
/* Function widened_name_p
|
88 |
|
|
|
89 |
|
|
Check whether NAME, an ssa-name used in USE_STMT,
|
90 |
|
|
is a result of a type-promotion, such that:
|
91 |
|
|
DEF_STMT: NAME = NOP (name0)
|
92 |
|
|
where the type of name0 (HALF_TYPE) is smaller than the type of NAME.
|
93 |
|
|
If CHECK_SIGN is TRUE, check that either both types are signed or both are
|
94 |
|
|
unsigned. */
|
95 |
|
|
|
96 |
|
|
static bool
|
97 |
|
|
widened_name_p (tree name, gimple use_stmt, tree *half_type, gimple *def_stmt,
|
98 |
|
|
bool check_sign)
|
99 |
|
|
{
|
100 |
|
|
tree dummy;
|
101 |
|
|
gimple dummy_gimple;
|
102 |
|
|
loop_vec_info loop_vinfo;
|
103 |
|
|
stmt_vec_info stmt_vinfo;
|
104 |
|
|
tree type = TREE_TYPE (name);
|
105 |
|
|
tree oprnd0;
|
106 |
|
|
enum vect_def_type dt;
|
107 |
|
|
tree def;
|
108 |
|
|
|
109 |
|
|
stmt_vinfo = vinfo_for_stmt (use_stmt);
|
110 |
|
|
loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
|
111 |
|
|
|
112 |
|
|
if (!vect_is_simple_use (name, use_stmt, loop_vinfo, NULL, def_stmt, &def,
|
113 |
|
|
&dt))
|
114 |
|
|
return false;
|
115 |
|
|
|
116 |
|
|
if (dt != vect_internal_def
|
117 |
|
|
&& dt != vect_external_def && dt != vect_constant_def)
|
118 |
|
|
return false;
|
119 |
|
|
|
120 |
|
|
if (! *def_stmt)
|
121 |
|
|
return false;
|
122 |
|
|
|
123 |
|
|
if (!is_gimple_assign (*def_stmt))
|
124 |
|
|
return false;
|
125 |
|
|
|
126 |
|
|
if (gimple_assign_rhs_code (*def_stmt) != NOP_EXPR)
|
127 |
|
|
return false;
|
128 |
|
|
|
129 |
|
|
oprnd0 = gimple_assign_rhs1 (*def_stmt);
|
130 |
|
|
|
131 |
|
|
*half_type = TREE_TYPE (oprnd0);
|
132 |
|
|
if (!INTEGRAL_TYPE_P (type) || !INTEGRAL_TYPE_P (*half_type)
|
133 |
|
|
|| ((TYPE_UNSIGNED (type) != TYPE_UNSIGNED (*half_type)) && check_sign)
|
134 |
|
|
|| (TYPE_PRECISION (type) < (TYPE_PRECISION (*half_type) * 2)))
|
135 |
|
|
return false;
|
136 |
|
|
|
137 |
|
|
if (!vect_is_simple_use (oprnd0, *def_stmt, loop_vinfo,
|
138 |
|
|
NULL, &dummy_gimple, &dummy, &dt))
|
139 |
|
|
return false;
|
140 |
|
|
|
141 |
|
|
return true;
|
142 |
|
|
}
|
143 |
|
|
|
144 |
|
|
/* Helper to return a new temporary for pattern of TYPE for STMT. If STMT
|
145 |
|
|
is NULL, the caller must set SSA_NAME_DEF_STMT for the returned SSA var. */
|
146 |
|
|
|
147 |
|
|
static tree
|
148 |
|
|
vect_recog_temp_ssa_var (tree type, gimple stmt)
|
149 |
|
|
{
|
150 |
|
|
tree var = create_tmp_var (type, "patt");
|
151 |
|
|
|
152 |
|
|
add_referenced_var (var);
|
153 |
|
|
var = make_ssa_name (var, stmt);
|
154 |
|
|
return var;
|
155 |
|
|
}
|
156 |
|
|
|
157 |
|
|
/* Function vect_recog_dot_prod_pattern
|
158 |
|
|
|
159 |
|
|
Try to find the following pattern:
|
160 |
|
|
|
161 |
|
|
type x_t, y_t;
|
162 |
|
|
TYPE1 prod;
|
163 |
|
|
TYPE2 sum = init;
|
164 |
|
|
loop:
|
165 |
|
|
sum_0 = phi <init, sum_1>
|
166 |
|
|
S1 x_t = ...
|
167 |
|
|
S2 y_t = ...
|
168 |
|
|
S3 x_T = (TYPE1) x_t;
|
169 |
|
|
S4 y_T = (TYPE1) y_t;
|
170 |
|
|
S5 prod = x_T * y_T;
|
171 |
|
|
[S6 prod = (TYPE2) prod; #optional]
|
172 |
|
|
S7 sum_1 = prod + sum_0;
|
173 |
|
|
|
174 |
|
|
where 'TYPE1' is exactly double the size of type 'type', and 'TYPE2' is the
|
175 |
|
|
same size of 'TYPE1' or bigger. This is a special case of a reduction
|
176 |
|
|
computation.
|
177 |
|
|
|
178 |
|
|
Input:
|
179 |
|
|
|
180 |
|
|
* STMTS: Contains a stmt from which the pattern search begins. In the
|
181 |
|
|
example, when this function is called with S7, the pattern {S3,S4,S5,S6,S7}
|
182 |
|
|
will be detected.
|
183 |
|
|
|
184 |
|
|
Output:
|
185 |
|
|
|
186 |
|
|
* TYPE_IN: The type of the input arguments to the pattern.
|
187 |
|
|
|
188 |
|
|
* TYPE_OUT: The type of the output of this pattern.
|
189 |
|
|
|
190 |
|
|
* Return value: A new stmt that will be used to replace the sequence of
|
191 |
|
|
stmts that constitute the pattern. In this case it will be:
|
192 |
|
|
WIDEN_DOT_PRODUCT <x_t, y_t, sum_0>
|
193 |
|
|
|
194 |
|
|
Note: The dot-prod idiom is a widening reduction pattern that is
|
195 |
|
|
vectorized without preserving all the intermediate results. It
|
196 |
|
|
produces only N/2 (widened) results (by summing up pairs of
|
197 |
|
|
intermediate results) rather than all N results. Therefore, we
|
198 |
|
|
cannot allow this pattern when we want to get all the results and in
|
199 |
|
|
the correct order (as is the case when this computation is in an
|
200 |
|
|
inner-loop nested in an outer-loop that us being vectorized). */
|
201 |
|
|
|
202 |
|
|
static gimple
|
203 |
|
|
vect_recog_dot_prod_pattern (VEC (gimple, heap) **stmts, tree *type_in,
|
204 |
|
|
tree *type_out)
|
205 |
|
|
{
|
206 |
|
|
gimple stmt, last_stmt = VEC_index (gimple, *stmts, 0);
|
207 |
|
|
tree oprnd0, oprnd1;
|
208 |
|
|
tree oprnd00, oprnd01;
|
209 |
|
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
|
210 |
|
|
tree type, half_type;
|
211 |
|
|
gimple pattern_stmt;
|
212 |
|
|
tree prod_type;
|
213 |
|
|
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
|
214 |
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_info);
|
215 |
|
|
tree var;
|
216 |
|
|
|
217 |
|
|
if (!is_gimple_assign (last_stmt))
|
218 |
|
|
return NULL;
|
219 |
|
|
|
220 |
|
|
type = gimple_expr_type (last_stmt);
|
221 |
|
|
|
222 |
|
|
/* Look for the following pattern
|
223 |
|
|
DX = (TYPE1) X;
|
224 |
|
|
DY = (TYPE1) Y;
|
225 |
|
|
DPROD = DX * DY;
|
226 |
|
|
DDPROD = (TYPE2) DPROD;
|
227 |
|
|
sum_1 = DDPROD + sum_0;
|
228 |
|
|
In which
|
229 |
|
|
- DX is double the size of X
|
230 |
|
|
- DY is double the size of Y
|
231 |
|
|
- DX, DY, DPROD all have the same type
|
232 |
|
|
- sum is the same size of DPROD or bigger
|
233 |
|
|
- sum has been recognized as a reduction variable.
|
234 |
|
|
|
235 |
|
|
This is equivalent to:
|
236 |
|
|
DPROD = X w* Y; #widen mult
|
237 |
|
|
sum_1 = DPROD w+ sum_0; #widen summation
|
238 |
|
|
or
|
239 |
|
|
DPROD = X w* Y; #widen mult
|
240 |
|
|
sum_1 = DPROD + sum_0; #summation
|
241 |
|
|
*/
|
242 |
|
|
|
243 |
|
|
/* Starting from LAST_STMT, follow the defs of its uses in search
|
244 |
|
|
of the above pattern. */
|
245 |
|
|
|
246 |
|
|
if (gimple_assign_rhs_code (last_stmt) != PLUS_EXPR)
|
247 |
|
|
return NULL;
|
248 |
|
|
|
249 |
|
|
if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
|
250 |
|
|
{
|
251 |
|
|
/* Has been detected as widening-summation? */
|
252 |
|
|
|
253 |
|
|
stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo);
|
254 |
|
|
type = gimple_expr_type (stmt);
|
255 |
|
|
if (gimple_assign_rhs_code (stmt) != WIDEN_SUM_EXPR)
|
256 |
|
|
return NULL;
|
257 |
|
|
oprnd0 = gimple_assign_rhs1 (stmt);
|
258 |
|
|
oprnd1 = gimple_assign_rhs2 (stmt);
|
259 |
|
|
half_type = TREE_TYPE (oprnd0);
|
260 |
|
|
}
|
261 |
|
|
else
|
262 |
|
|
{
|
263 |
|
|
gimple def_stmt;
|
264 |
|
|
|
265 |
|
|
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def)
|
266 |
|
|
return NULL;
|
267 |
|
|
oprnd0 = gimple_assign_rhs1 (last_stmt);
|
268 |
|
|
oprnd1 = gimple_assign_rhs2 (last_stmt);
|
269 |
|
|
if (!types_compatible_p (TREE_TYPE (oprnd0), type)
|
270 |
|
|
|| !types_compatible_p (TREE_TYPE (oprnd1), type))
|
271 |
|
|
return NULL;
|
272 |
|
|
stmt = last_stmt;
|
273 |
|
|
|
274 |
|
|
if (widened_name_p (oprnd0, stmt, &half_type, &def_stmt, true))
|
275 |
|
|
{
|
276 |
|
|
stmt = def_stmt;
|
277 |
|
|
oprnd0 = gimple_assign_rhs1 (stmt);
|
278 |
|
|
}
|
279 |
|
|
else
|
280 |
|
|
half_type = type;
|
281 |
|
|
}
|
282 |
|
|
|
283 |
|
|
/* So far so good. Since last_stmt was detected as a (summation) reduction,
|
284 |
|
|
we know that oprnd1 is the reduction variable (defined by a loop-header
|
285 |
|
|
phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
|
286 |
|
|
Left to check that oprnd0 is defined by a (widen_)mult_expr */
|
287 |
|
|
if (TREE_CODE (oprnd0) != SSA_NAME)
|
288 |
|
|
return NULL;
|
289 |
|
|
|
290 |
|
|
prod_type = half_type;
|
291 |
|
|
stmt = SSA_NAME_DEF_STMT (oprnd0);
|
292 |
|
|
|
293 |
|
|
/* It could not be the dot_prod pattern if the stmt is outside the loop. */
|
294 |
|
|
if (!gimple_bb (stmt) || !flow_bb_inside_loop_p (loop, gimple_bb (stmt)))
|
295 |
|
|
return NULL;
|
296 |
|
|
|
297 |
|
|
/* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
|
298 |
|
|
inside the loop (in case we are analyzing an outer-loop). */
|
299 |
|
|
if (!is_gimple_assign (stmt))
|
300 |
|
|
return NULL;
|
301 |
|
|
stmt_vinfo = vinfo_for_stmt (stmt);
|
302 |
|
|
gcc_assert (stmt_vinfo);
|
303 |
|
|
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_internal_def)
|
304 |
|
|
return NULL;
|
305 |
|
|
if (gimple_assign_rhs_code (stmt) != MULT_EXPR)
|
306 |
|
|
return NULL;
|
307 |
|
|
if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
|
308 |
|
|
{
|
309 |
|
|
/* Has been detected as a widening multiplication? */
|
310 |
|
|
|
311 |
|
|
stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo);
|
312 |
|
|
if (gimple_assign_rhs_code (stmt) != WIDEN_MULT_EXPR)
|
313 |
|
|
return NULL;
|
314 |
|
|
stmt_vinfo = vinfo_for_stmt (stmt);
|
315 |
|
|
gcc_assert (stmt_vinfo);
|
316 |
|
|
gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_internal_def);
|
317 |
|
|
oprnd00 = gimple_assign_rhs1 (stmt);
|
318 |
|
|
oprnd01 = gimple_assign_rhs2 (stmt);
|
319 |
|
|
}
|
320 |
|
|
else
|
321 |
|
|
{
|
322 |
|
|
tree half_type0, half_type1;
|
323 |
|
|
gimple def_stmt;
|
324 |
|
|
tree oprnd0, oprnd1;
|
325 |
|
|
|
326 |
|
|
oprnd0 = gimple_assign_rhs1 (stmt);
|
327 |
|
|
oprnd1 = gimple_assign_rhs2 (stmt);
|
328 |
|
|
if (!types_compatible_p (TREE_TYPE (oprnd0), prod_type)
|
329 |
|
|
|| !types_compatible_p (TREE_TYPE (oprnd1), prod_type))
|
330 |
|
|
return NULL;
|
331 |
|
|
if (!widened_name_p (oprnd0, stmt, &half_type0, &def_stmt, true))
|
332 |
|
|
return NULL;
|
333 |
|
|
oprnd00 = gimple_assign_rhs1 (def_stmt);
|
334 |
|
|
if (!widened_name_p (oprnd1, stmt, &half_type1, &def_stmt, true))
|
335 |
|
|
return NULL;
|
336 |
|
|
oprnd01 = gimple_assign_rhs1 (def_stmt);
|
337 |
|
|
if (!types_compatible_p (half_type0, half_type1))
|
338 |
|
|
return NULL;
|
339 |
|
|
if (TYPE_PRECISION (prod_type) != TYPE_PRECISION (half_type0) * 2)
|
340 |
|
|
return NULL;
|
341 |
|
|
}
|
342 |
|
|
|
343 |
|
|
half_type = TREE_TYPE (oprnd00);
|
344 |
|
|
*type_in = half_type;
|
345 |
|
|
*type_out = type;
|
346 |
|
|
|
347 |
|
|
/* Pattern detected. Create a stmt to be used to replace the pattern: */
|
348 |
|
|
var = vect_recog_temp_ssa_var (type, NULL);
|
349 |
|
|
pattern_stmt = gimple_build_assign_with_ops3 (DOT_PROD_EXPR, var,
|
350 |
|
|
oprnd00, oprnd01, oprnd1);
|
351 |
|
|
|
352 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
353 |
|
|
{
|
354 |
|
|
fprintf (vect_dump, "vect_recog_dot_prod_pattern: detected: ");
|
355 |
|
|
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
|
356 |
|
|
}
|
357 |
|
|
|
358 |
|
|
/* We don't allow changing the order of the computation in the inner-loop
|
359 |
|
|
when doing outer-loop vectorization. */
|
360 |
|
|
gcc_assert (!nested_in_vect_loop_p (loop, last_stmt));
|
361 |
|
|
|
362 |
|
|
return pattern_stmt;
|
363 |
|
|
}
|
364 |
|
|
|
365 |
|
|
|
366 |
|
|
/* Handle widening operation by a constant. At the moment we support MULT_EXPR
|
367 |
|
|
and LSHIFT_EXPR.
|
368 |
|
|
|
369 |
|
|
For MULT_EXPR we check that CONST_OPRND fits HALF_TYPE, and for LSHIFT_EXPR
|
370 |
|
|
we check that CONST_OPRND is less or equal to the size of HALF_TYPE.
|
371 |
|
|
|
372 |
|
|
Otherwise, if the type of the result (TYPE) is at least 4 times bigger than
|
373 |
|
|
HALF_TYPE, and there is an intermediate type (2 times smaller than TYPE)
|
374 |
|
|
that satisfies the above restrictions, we can perform a widening opeartion
|
375 |
|
|
from the intermediate type to TYPE and replace a_T = (TYPE) a_t;
|
376 |
|
|
with a_it = (interm_type) a_t; */
|
377 |
|
|
|
378 |
|
|
static bool
|
379 |
|
|
vect_handle_widen_op_by_const (gimple stmt, enum tree_code code,
|
380 |
|
|
tree const_oprnd, tree *oprnd,
|
381 |
|
|
VEC (gimple, heap) **stmts, tree type,
|
382 |
|
|
tree *half_type, gimple def_stmt)
|
383 |
|
|
{
|
384 |
|
|
tree new_type, new_oprnd, tmp;
|
385 |
|
|
gimple new_stmt;
|
386 |
|
|
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (vinfo_for_stmt (stmt));
|
387 |
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_info);
|
388 |
|
|
|
389 |
|
|
if (code != MULT_EXPR && code != LSHIFT_EXPR)
|
390 |
|
|
return false;
|
391 |
|
|
|
392 |
|
|
if (((code == MULT_EXPR && int_fits_type_p (const_oprnd, *half_type))
|
393 |
|
|
|| (code == LSHIFT_EXPR
|
394 |
|
|
&& compare_tree_int (const_oprnd, TYPE_PRECISION (*half_type))
|
395 |
|
|
!= 1))
|
396 |
|
|
&& TYPE_PRECISION (type) == (TYPE_PRECISION (*half_type) * 2))
|
397 |
|
|
{
|
398 |
|
|
/* CONST_OPRND is a constant of HALF_TYPE. */
|
399 |
|
|
*oprnd = gimple_assign_rhs1 (def_stmt);
|
400 |
|
|
return true;
|
401 |
|
|
}
|
402 |
|
|
|
403 |
|
|
if (TYPE_PRECISION (type) < (TYPE_PRECISION (*half_type) * 4)
|
404 |
|
|
|| !gimple_bb (def_stmt)
|
405 |
|
|
|| !flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
|
406 |
|
|
|| !vinfo_for_stmt (def_stmt))
|
407 |
|
|
return false;
|
408 |
|
|
|
409 |
|
|
/* TYPE is 4 times bigger than HALF_TYPE, try widening operation for
|
410 |
|
|
a type 2 times bigger than HALF_TYPE. */
|
411 |
|
|
new_type = build_nonstandard_integer_type (TYPE_PRECISION (type) / 2,
|
412 |
|
|
TYPE_UNSIGNED (type));
|
413 |
|
|
if ((code == MULT_EXPR && !int_fits_type_p (const_oprnd, new_type))
|
414 |
|
|
|| (code == LSHIFT_EXPR
|
415 |
|
|
&& compare_tree_int (const_oprnd, TYPE_PRECISION (new_type)) == 1))
|
416 |
|
|
return false;
|
417 |
|
|
|
418 |
|
|
/* Use NEW_TYPE for widening operation. */
|
419 |
|
|
if (STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt)))
|
420 |
|
|
{
|
421 |
|
|
new_stmt = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt));
|
422 |
|
|
/* Check if the already created pattern stmt is what we need. */
|
423 |
|
|
if (!is_gimple_assign (new_stmt)
|
424 |
|
|
|| gimple_assign_rhs_code (new_stmt) != NOP_EXPR
|
425 |
|
|
|| TREE_TYPE (gimple_assign_lhs (new_stmt)) != new_type)
|
426 |
|
|
return false;
|
427 |
|
|
|
428 |
|
|
VEC_safe_push (gimple, heap, *stmts, def_stmt);
|
429 |
|
|
*oprnd = gimple_assign_lhs (new_stmt);
|
430 |
|
|
}
|
431 |
|
|
else
|
432 |
|
|
{
|
433 |
|
|
/* Create a_T = (NEW_TYPE) a_t; */
|
434 |
|
|
*oprnd = gimple_assign_rhs1 (def_stmt);
|
435 |
|
|
tmp = create_tmp_var (new_type, NULL);
|
436 |
|
|
add_referenced_var (tmp);
|
437 |
|
|
new_oprnd = make_ssa_name (tmp, NULL);
|
438 |
|
|
new_stmt = gimple_build_assign_with_ops (NOP_EXPR, new_oprnd, *oprnd,
|
439 |
|
|
NULL_TREE);
|
440 |
|
|
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt)) = new_stmt;
|
441 |
|
|
VEC_safe_push (gimple, heap, *stmts, def_stmt);
|
442 |
|
|
*oprnd = new_oprnd;
|
443 |
|
|
}
|
444 |
|
|
|
445 |
|
|
*half_type = new_type;
|
446 |
|
|
return true;
|
447 |
|
|
}
|
448 |
|
|
|
449 |
|
|
|
450 |
|
|
/* Function vect_recog_widen_mult_pattern
|
451 |
|
|
|
452 |
|
|
Try to find the following pattern:
|
453 |
|
|
|
454 |
|
|
type a_t, b_t;
|
455 |
|
|
TYPE a_T, b_T, prod_T;
|
456 |
|
|
|
457 |
|
|
S1 a_t = ;
|
458 |
|
|
S2 b_t = ;
|
459 |
|
|
S3 a_T = (TYPE) a_t;
|
460 |
|
|
S4 b_T = (TYPE) b_t;
|
461 |
|
|
S5 prod_T = a_T * b_T;
|
462 |
|
|
|
463 |
|
|
where type 'TYPE' is at least double the size of type 'type'.
|
464 |
|
|
|
465 |
|
|
Also detect unsgigned cases:
|
466 |
|
|
|
467 |
|
|
unsigned type a_t, b_t;
|
468 |
|
|
unsigned TYPE u_prod_T;
|
469 |
|
|
TYPE a_T, b_T, prod_T;
|
470 |
|
|
|
471 |
|
|
S1 a_t = ;
|
472 |
|
|
S2 b_t = ;
|
473 |
|
|
S3 a_T = (TYPE) a_t;
|
474 |
|
|
S4 b_T = (TYPE) b_t;
|
475 |
|
|
S5 prod_T = a_T * b_T;
|
476 |
|
|
S6 u_prod_T = (unsigned TYPE) prod_T;
|
477 |
|
|
|
478 |
|
|
and multiplication by constants:
|
479 |
|
|
|
480 |
|
|
type a_t;
|
481 |
|
|
TYPE a_T, prod_T;
|
482 |
|
|
|
483 |
|
|
S1 a_t = ;
|
484 |
|
|
S3 a_T = (TYPE) a_t;
|
485 |
|
|
S5 prod_T = a_T * CONST;
|
486 |
|
|
|
487 |
|
|
A special case of multiplication by constants is when 'TYPE' is 4 times
|
488 |
|
|
bigger than 'type', but CONST fits an intermediate type 2 times smaller
|
489 |
|
|
than 'TYPE'. In that case we create an additional pattern stmt for S3
|
490 |
|
|
to create a variable of the intermediate type, and perform widen-mult
|
491 |
|
|
on the intermediate type as well:
|
492 |
|
|
|
493 |
|
|
type a_t;
|
494 |
|
|
interm_type a_it;
|
495 |
|
|
TYPE a_T, prod_T, prod_T';
|
496 |
|
|
|
497 |
|
|
S1 a_t = ;
|
498 |
|
|
S3 a_T = (TYPE) a_t;
|
499 |
|
|
'--> a_it = (interm_type) a_t;
|
500 |
|
|
S5 prod_T = a_T * CONST;
|
501 |
|
|
'--> prod_T' = a_it w* CONST;
|
502 |
|
|
|
503 |
|
|
Input/Output:
|
504 |
|
|
|
505 |
|
|
* STMTS: Contains a stmt from which the pattern search begins. In the
|
506 |
|
|
example, when this function is called with S5, the pattern {S3,S4,S5,(S6)}
|
507 |
|
|
is detected. In case of unsigned widen-mult, the original stmt (S5) is
|
508 |
|
|
replaced with S6 in STMTS. In case of multiplication by a constant
|
509 |
|
|
of an intermediate type (the last case above), STMTS also contains S3
|
510 |
|
|
(inserted before S5).
|
511 |
|
|
|
512 |
|
|
Output:
|
513 |
|
|
|
514 |
|
|
* TYPE_IN: The type of the input arguments to the pattern.
|
515 |
|
|
|
516 |
|
|
* TYPE_OUT: The type of the output of this pattern.
|
517 |
|
|
|
518 |
|
|
* Return value: A new stmt that will be used to replace the sequence of
|
519 |
|
|
stmts that constitute the pattern. In this case it will be:
|
520 |
|
|
WIDEN_MULT <a_t, b_t>
|
521 |
|
|
*/
|
522 |
|
|
|
523 |
|
|
static gimple
|
524 |
|
|
vect_recog_widen_mult_pattern (VEC (gimple, heap) **stmts,
|
525 |
|
|
tree *type_in, tree *type_out)
|
526 |
|
|
{
|
527 |
|
|
gimple last_stmt = VEC_pop (gimple, *stmts);
|
528 |
|
|
gimple def_stmt0, def_stmt1;
|
529 |
|
|
tree oprnd0, oprnd1;
|
530 |
|
|
tree type, half_type0, half_type1;
|
531 |
|
|
gimple pattern_stmt;
|
532 |
|
|
tree vectype, vectype_out = NULL_TREE;
|
533 |
|
|
tree dummy;
|
534 |
|
|
tree var;
|
535 |
|
|
enum tree_code dummy_code;
|
536 |
|
|
int dummy_int;
|
537 |
|
|
VEC (tree, heap) *dummy_vec;
|
538 |
|
|
bool op1_ok;
|
539 |
|
|
|
540 |
|
|
if (!is_gimple_assign (last_stmt))
|
541 |
|
|
return NULL;
|
542 |
|
|
|
543 |
|
|
type = gimple_expr_type (last_stmt);
|
544 |
|
|
|
545 |
|
|
/* Starting from LAST_STMT, follow the defs of its uses in search
|
546 |
|
|
of the above pattern. */
|
547 |
|
|
|
548 |
|
|
if (gimple_assign_rhs_code (last_stmt) != MULT_EXPR)
|
549 |
|
|
return NULL;
|
550 |
|
|
|
551 |
|
|
oprnd0 = gimple_assign_rhs1 (last_stmt);
|
552 |
|
|
oprnd1 = gimple_assign_rhs2 (last_stmt);
|
553 |
|
|
if (!types_compatible_p (TREE_TYPE (oprnd0), type)
|
554 |
|
|
|| !types_compatible_p (TREE_TYPE (oprnd1), type))
|
555 |
|
|
return NULL;
|
556 |
|
|
|
557 |
|
|
/* Check argument 0. */
|
558 |
|
|
if (!widened_name_p (oprnd0, last_stmt, &half_type0, &def_stmt0, false))
|
559 |
|
|
return NULL;
|
560 |
|
|
/* Check argument 1. */
|
561 |
|
|
op1_ok = widened_name_p (oprnd1, last_stmt, &half_type1, &def_stmt1, false);
|
562 |
|
|
|
563 |
|
|
if (op1_ok)
|
564 |
|
|
{
|
565 |
|
|
oprnd0 = gimple_assign_rhs1 (def_stmt0);
|
566 |
|
|
oprnd1 = gimple_assign_rhs1 (def_stmt1);
|
567 |
|
|
}
|
568 |
|
|
else
|
569 |
|
|
{
|
570 |
|
|
if (TREE_CODE (oprnd1) == INTEGER_CST
|
571 |
|
|
&& TREE_CODE (half_type0) == INTEGER_TYPE
|
572 |
|
|
&& vect_handle_widen_op_by_const (last_stmt, MULT_EXPR, oprnd1,
|
573 |
|
|
&oprnd0, stmts, type,
|
574 |
|
|
&half_type0, def_stmt0))
|
575 |
|
|
half_type1 = half_type0;
|
576 |
|
|
else
|
577 |
|
|
return NULL;
|
578 |
|
|
}
|
579 |
|
|
|
580 |
|
|
/* Handle unsigned case. Look for
|
581 |
|
|
S6 u_prod_T = (unsigned TYPE) prod_T;
|
582 |
|
|
Use unsigned TYPE as the type for WIDEN_MULT_EXPR. */
|
583 |
|
|
if (TYPE_UNSIGNED (type) != TYPE_UNSIGNED (half_type0))
|
584 |
|
|
{
|
585 |
|
|
tree lhs = gimple_assign_lhs (last_stmt), use_lhs;
|
586 |
|
|
imm_use_iterator imm_iter;
|
587 |
|
|
use_operand_p use_p;
|
588 |
|
|
int nuses = 0;
|
589 |
|
|
gimple use_stmt = NULL;
|
590 |
|
|
tree use_type;
|
591 |
|
|
|
592 |
|
|
if (TYPE_UNSIGNED (type) == TYPE_UNSIGNED (half_type1))
|
593 |
|
|
return NULL;
|
594 |
|
|
|
595 |
|
|
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
|
596 |
|
|
{
|
597 |
|
|
if (is_gimple_debug (USE_STMT (use_p)))
|
598 |
|
|
continue;
|
599 |
|
|
use_stmt = USE_STMT (use_p);
|
600 |
|
|
nuses++;
|
601 |
|
|
}
|
602 |
|
|
|
603 |
|
|
if (nuses != 1 || !is_gimple_assign (use_stmt)
|
604 |
|
|
|| gimple_assign_rhs_code (use_stmt) != NOP_EXPR)
|
605 |
|
|
return NULL;
|
606 |
|
|
|
607 |
|
|
use_lhs = gimple_assign_lhs (use_stmt);
|
608 |
|
|
use_type = TREE_TYPE (use_lhs);
|
609 |
|
|
if (!INTEGRAL_TYPE_P (use_type)
|
610 |
|
|
|| (TYPE_UNSIGNED (type) == TYPE_UNSIGNED (use_type))
|
611 |
|
|
|| (TYPE_PRECISION (type) != TYPE_PRECISION (use_type)))
|
612 |
|
|
return NULL;
|
613 |
|
|
|
614 |
|
|
type = use_type;
|
615 |
|
|
last_stmt = use_stmt;
|
616 |
|
|
}
|
617 |
|
|
|
618 |
|
|
if (!types_compatible_p (half_type0, half_type1))
|
619 |
|
|
return NULL;
|
620 |
|
|
|
621 |
|
|
/* Pattern detected. */
|
622 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
623 |
|
|
fprintf (vect_dump, "vect_recog_widen_mult_pattern: detected: ");
|
624 |
|
|
|
625 |
|
|
/* Check target support */
|
626 |
|
|
vectype = get_vectype_for_scalar_type (half_type0);
|
627 |
|
|
vectype_out = get_vectype_for_scalar_type (type);
|
628 |
|
|
if (!vectype
|
629 |
|
|
|| !vectype_out
|
630 |
|
|
|| !supportable_widening_operation (WIDEN_MULT_EXPR, last_stmt,
|
631 |
|
|
vectype_out, vectype,
|
632 |
|
|
&dummy, &dummy, &dummy_code,
|
633 |
|
|
&dummy_code, &dummy_int, &dummy_vec))
|
634 |
|
|
return NULL;
|
635 |
|
|
|
636 |
|
|
*type_in = vectype;
|
637 |
|
|
*type_out = vectype_out;
|
638 |
|
|
|
639 |
|
|
/* Pattern supported. Create a stmt to be used to replace the pattern: */
|
640 |
|
|
var = vect_recog_temp_ssa_var (type, NULL);
|
641 |
|
|
pattern_stmt = gimple_build_assign_with_ops (WIDEN_MULT_EXPR, var, oprnd0,
|
642 |
|
|
oprnd1);
|
643 |
|
|
|
644 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
645 |
|
|
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
|
646 |
|
|
|
647 |
|
|
VEC_safe_push (gimple, heap, *stmts, last_stmt);
|
648 |
|
|
return pattern_stmt;
|
649 |
|
|
}
|
650 |
|
|
|
651 |
|
|
|
652 |
|
|
/* Function vect_recog_pow_pattern
|
653 |
|
|
|
654 |
|
|
Try to find the following pattern:
|
655 |
|
|
|
656 |
|
|
x = POW (y, N);
|
657 |
|
|
|
658 |
|
|
with POW being one of pow, powf, powi, powif and N being
|
659 |
|
|
either 2 or 0.5.
|
660 |
|
|
|
661 |
|
|
Input:
|
662 |
|
|
|
663 |
|
|
* LAST_STMT: A stmt from which the pattern search begins.
|
664 |
|
|
|
665 |
|
|
Output:
|
666 |
|
|
|
667 |
|
|
* TYPE_IN: The type of the input arguments to the pattern.
|
668 |
|
|
|
669 |
|
|
* TYPE_OUT: The type of the output of this pattern.
|
670 |
|
|
|
671 |
|
|
* Return value: A new stmt that will be used to replace the sequence of
|
672 |
|
|
stmts that constitute the pattern. In this case it will be:
|
673 |
|
|
x = x * x
|
674 |
|
|
or
|
675 |
|
|
x = sqrt (x)
|
676 |
|
|
*/
|
677 |
|
|
|
678 |
|
|
static gimple
|
679 |
|
|
vect_recog_pow_pattern (VEC (gimple, heap) **stmts, tree *type_in,
|
680 |
|
|
tree *type_out)
|
681 |
|
|
{
|
682 |
|
|
gimple last_stmt = VEC_index (gimple, *stmts, 0);
|
683 |
|
|
tree fn, base, exp = NULL;
|
684 |
|
|
gimple stmt;
|
685 |
|
|
tree var;
|
686 |
|
|
|
687 |
|
|
if (!is_gimple_call (last_stmt) || gimple_call_lhs (last_stmt) == NULL)
|
688 |
|
|
return NULL;
|
689 |
|
|
|
690 |
|
|
fn = gimple_call_fndecl (last_stmt);
|
691 |
|
|
if (fn == NULL_TREE || DECL_BUILT_IN_CLASS (fn) != BUILT_IN_NORMAL)
|
692 |
|
|
return NULL;
|
693 |
|
|
|
694 |
|
|
switch (DECL_FUNCTION_CODE (fn))
|
695 |
|
|
{
|
696 |
|
|
case BUILT_IN_POWIF:
|
697 |
|
|
case BUILT_IN_POWI:
|
698 |
|
|
case BUILT_IN_POWF:
|
699 |
|
|
case BUILT_IN_POW:
|
700 |
|
|
base = gimple_call_arg (last_stmt, 0);
|
701 |
|
|
exp = gimple_call_arg (last_stmt, 1);
|
702 |
|
|
if (TREE_CODE (exp) != REAL_CST
|
703 |
|
|
&& TREE_CODE (exp) != INTEGER_CST)
|
704 |
|
|
return NULL;
|
705 |
|
|
break;
|
706 |
|
|
|
707 |
|
|
default:
|
708 |
|
|
return NULL;
|
709 |
|
|
}
|
710 |
|
|
|
711 |
|
|
/* We now have a pow or powi builtin function call with a constant
|
712 |
|
|
exponent. */
|
713 |
|
|
|
714 |
|
|
*type_out = NULL_TREE;
|
715 |
|
|
|
716 |
|
|
/* Catch squaring. */
|
717 |
|
|
if ((host_integerp (exp, 0)
|
718 |
|
|
&& tree_low_cst (exp, 0) == 2)
|
719 |
|
|
|| (TREE_CODE (exp) == REAL_CST
|
720 |
|
|
&& REAL_VALUES_EQUAL (TREE_REAL_CST (exp), dconst2)))
|
721 |
|
|
{
|
722 |
|
|
*type_in = TREE_TYPE (base);
|
723 |
|
|
|
724 |
|
|
var = vect_recog_temp_ssa_var (TREE_TYPE (base), NULL);
|
725 |
|
|
stmt = gimple_build_assign_with_ops (MULT_EXPR, var, base, base);
|
726 |
|
|
return stmt;
|
727 |
|
|
}
|
728 |
|
|
|
729 |
|
|
/* Catch square root. */
|
730 |
|
|
if (TREE_CODE (exp) == REAL_CST
|
731 |
|
|
&& REAL_VALUES_EQUAL (TREE_REAL_CST (exp), dconsthalf))
|
732 |
|
|
{
|
733 |
|
|
tree newfn = mathfn_built_in (TREE_TYPE (base), BUILT_IN_SQRT);
|
734 |
|
|
*type_in = get_vectype_for_scalar_type (TREE_TYPE (base));
|
735 |
|
|
if (*type_in)
|
736 |
|
|
{
|
737 |
|
|
gimple stmt = gimple_build_call (newfn, 1, base);
|
738 |
|
|
if (vectorizable_function (stmt, *type_in, *type_in)
|
739 |
|
|
!= NULL_TREE)
|
740 |
|
|
{
|
741 |
|
|
var = vect_recog_temp_ssa_var (TREE_TYPE (base), stmt);
|
742 |
|
|
gimple_call_set_lhs (stmt, var);
|
743 |
|
|
return stmt;
|
744 |
|
|
}
|
745 |
|
|
}
|
746 |
|
|
}
|
747 |
|
|
|
748 |
|
|
return NULL;
|
749 |
|
|
}
|
750 |
|
|
|
751 |
|
|
|
752 |
|
|
/* Function vect_recog_widen_sum_pattern
|
753 |
|
|
|
754 |
|
|
Try to find the following pattern:
|
755 |
|
|
|
756 |
|
|
type x_t;
|
757 |
|
|
TYPE x_T, sum = init;
|
758 |
|
|
loop:
|
759 |
|
|
sum_0 = phi <init, sum_1>
|
760 |
|
|
S1 x_t = *p;
|
761 |
|
|
S2 x_T = (TYPE) x_t;
|
762 |
|
|
S3 sum_1 = x_T + sum_0;
|
763 |
|
|
|
764 |
|
|
where type 'TYPE' is at least double the size of type 'type', i.e - we're
|
765 |
|
|
summing elements of type 'type' into an accumulator of type 'TYPE'. This is
|
766 |
|
|
a special case of a reduction computation.
|
767 |
|
|
|
768 |
|
|
Input:
|
769 |
|
|
|
770 |
|
|
* LAST_STMT: A stmt from which the pattern search begins. In the example,
|
771 |
|
|
when this function is called with S3, the pattern {S2,S3} will be detected.
|
772 |
|
|
|
773 |
|
|
Output:
|
774 |
|
|
|
775 |
|
|
* TYPE_IN: The type of the input arguments to the pattern.
|
776 |
|
|
|
777 |
|
|
* TYPE_OUT: The type of the output of this pattern.
|
778 |
|
|
|
779 |
|
|
* Return value: A new stmt that will be used to replace the sequence of
|
780 |
|
|
stmts that constitute the pattern. In this case it will be:
|
781 |
|
|
WIDEN_SUM <x_t, sum_0>
|
782 |
|
|
|
783 |
|
|
Note: The widening-sum idiom is a widening reduction pattern that is
|
784 |
|
|
vectorized without preserving all the intermediate results. It
|
785 |
|
|
produces only N/2 (widened) results (by summing up pairs of
|
786 |
|
|
intermediate results) rather than all N results. Therefore, we
|
787 |
|
|
cannot allow this pattern when we want to get all the results and in
|
788 |
|
|
the correct order (as is the case when this computation is in an
|
789 |
|
|
inner-loop nested in an outer-loop that us being vectorized). */
|
790 |
|
|
|
791 |
|
|
static gimple
|
792 |
|
|
vect_recog_widen_sum_pattern (VEC (gimple, heap) **stmts, tree *type_in,
|
793 |
|
|
tree *type_out)
|
794 |
|
|
{
|
795 |
|
|
gimple stmt, last_stmt = VEC_index (gimple, *stmts, 0);
|
796 |
|
|
tree oprnd0, oprnd1;
|
797 |
|
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
|
798 |
|
|
tree type, half_type;
|
799 |
|
|
gimple pattern_stmt;
|
800 |
|
|
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
|
801 |
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_info);
|
802 |
|
|
tree var;
|
803 |
|
|
|
804 |
|
|
if (!is_gimple_assign (last_stmt))
|
805 |
|
|
return NULL;
|
806 |
|
|
|
807 |
|
|
type = gimple_expr_type (last_stmt);
|
808 |
|
|
|
809 |
|
|
/* Look for the following pattern
|
810 |
|
|
DX = (TYPE) X;
|
811 |
|
|
sum_1 = DX + sum_0;
|
812 |
|
|
In which DX is at least double the size of X, and sum_1 has been
|
813 |
|
|
recognized as a reduction variable.
|
814 |
|
|
*/
|
815 |
|
|
|
816 |
|
|
/* Starting from LAST_STMT, follow the defs of its uses in search
|
817 |
|
|
of the above pattern. */
|
818 |
|
|
|
819 |
|
|
if (gimple_assign_rhs_code (last_stmt) != PLUS_EXPR)
|
820 |
|
|
return NULL;
|
821 |
|
|
|
822 |
|
|
if (STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_reduction_def)
|
823 |
|
|
return NULL;
|
824 |
|
|
|
825 |
|
|
oprnd0 = gimple_assign_rhs1 (last_stmt);
|
826 |
|
|
oprnd1 = gimple_assign_rhs2 (last_stmt);
|
827 |
|
|
if (!types_compatible_p (TREE_TYPE (oprnd0), type)
|
828 |
|
|
|| !types_compatible_p (TREE_TYPE (oprnd1), type))
|
829 |
|
|
return NULL;
|
830 |
|
|
|
831 |
|
|
/* So far so good. Since last_stmt was detected as a (summation) reduction,
|
832 |
|
|
we know that oprnd1 is the reduction variable (defined by a loop-header
|
833 |
|
|
phi), and oprnd0 is an ssa-name defined by a stmt in the loop body.
|
834 |
|
|
Left to check that oprnd0 is defined by a cast from type 'type' to type
|
835 |
|
|
'TYPE'. */
|
836 |
|
|
|
837 |
|
|
if (!widened_name_p (oprnd0, last_stmt, &half_type, &stmt, true))
|
838 |
|
|
return NULL;
|
839 |
|
|
|
840 |
|
|
oprnd0 = gimple_assign_rhs1 (stmt);
|
841 |
|
|
*type_in = half_type;
|
842 |
|
|
*type_out = type;
|
843 |
|
|
|
844 |
|
|
/* Pattern detected. Create a stmt to be used to replace the pattern: */
|
845 |
|
|
var = vect_recog_temp_ssa_var (type, NULL);
|
846 |
|
|
pattern_stmt = gimple_build_assign_with_ops (WIDEN_SUM_EXPR, var,
|
847 |
|
|
oprnd0, oprnd1);
|
848 |
|
|
|
849 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
850 |
|
|
{
|
851 |
|
|
fprintf (vect_dump, "vect_recog_widen_sum_pattern: detected: ");
|
852 |
|
|
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
|
853 |
|
|
}
|
854 |
|
|
|
855 |
|
|
/* We don't allow changing the order of the computation in the inner-loop
|
856 |
|
|
when doing outer-loop vectorization. */
|
857 |
|
|
gcc_assert (!nested_in_vect_loop_p (loop, last_stmt));
|
858 |
|
|
|
859 |
|
|
return pattern_stmt;
|
860 |
|
|
}
|
861 |
|
|
|
862 |
|
|
|
863 |
|
|
/* Return TRUE if the operation in STMT can be performed on a smaller type.
|
864 |
|
|
|
865 |
|
|
Input:
|
866 |
|
|
STMT - a statement to check.
|
867 |
|
|
DEF - we support operations with two operands, one of which is constant.
|
868 |
|
|
The other operand can be defined by a demotion operation, or by a
|
869 |
|
|
previous statement in a sequence of over-promoted operations. In the
|
870 |
|
|
later case DEF is used to replace that operand. (It is defined by a
|
871 |
|
|
pattern statement we created for the previous statement in the
|
872 |
|
|
sequence).
|
873 |
|
|
|
874 |
|
|
Input/output:
|
875 |
|
|
NEW_TYPE - Output: a smaller type that we are trying to use. Input: if not
|
876 |
|
|
NULL, it's the type of DEF.
|
877 |
|
|
STMTS - additional pattern statements. If a pattern statement (type
|
878 |
|
|
conversion) is created in this function, its original statement is
|
879 |
|
|
added to STMTS.
|
880 |
|
|
|
881 |
|
|
Output:
|
882 |
|
|
OP0, OP1 - if the operation fits a smaller type, OP0 and OP1 are the new
|
883 |
|
|
operands to use in the new pattern statement for STMT (will be created
|
884 |
|
|
in vect_recog_over_widening_pattern ()).
|
885 |
|
|
NEW_DEF_STMT - in case DEF has to be promoted, we create two pattern
|
886 |
|
|
statements for STMT: the first one is a type promotion and the second
|
887 |
|
|
one is the operation itself. We return the type promotion statement
|
888 |
|
|
in NEW_DEF_STMT and further store it in STMT_VINFO_PATTERN_DEF_SEQ of
|
889 |
|
|
the second pattern statement. */
|
890 |
|
|
|
891 |
|
|
static bool
|
892 |
|
|
vect_operation_fits_smaller_type (gimple stmt, tree def, tree *new_type,
|
893 |
|
|
tree *op0, tree *op1, gimple *new_def_stmt,
|
894 |
|
|
VEC (gimple, heap) **stmts)
|
895 |
|
|
{
|
896 |
|
|
enum tree_code code;
|
897 |
|
|
tree const_oprnd, oprnd;
|
898 |
|
|
tree interm_type = NULL_TREE, half_type, tmp, new_oprnd, type;
|
899 |
|
|
gimple def_stmt, new_stmt;
|
900 |
|
|
bool first = false;
|
901 |
|
|
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (vinfo_for_stmt (stmt));
|
902 |
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_info);
|
903 |
|
|
|
904 |
|
|
*op0 = NULL_TREE;
|
905 |
|
|
*op1 = NULL_TREE;
|
906 |
|
|
*new_def_stmt = NULL;
|
907 |
|
|
|
908 |
|
|
if (!is_gimple_assign (stmt))
|
909 |
|
|
return false;
|
910 |
|
|
|
911 |
|
|
code = gimple_assign_rhs_code (stmt);
|
912 |
|
|
if (code != LSHIFT_EXPR && code != RSHIFT_EXPR
|
913 |
|
|
&& code != BIT_IOR_EXPR && code != BIT_XOR_EXPR && code != BIT_AND_EXPR)
|
914 |
|
|
return false;
|
915 |
|
|
|
916 |
|
|
oprnd = gimple_assign_rhs1 (stmt);
|
917 |
|
|
const_oprnd = gimple_assign_rhs2 (stmt);
|
918 |
|
|
type = gimple_expr_type (stmt);
|
919 |
|
|
|
920 |
|
|
if (TREE_CODE (oprnd) != SSA_NAME
|
921 |
|
|
|| TREE_CODE (const_oprnd) != INTEGER_CST)
|
922 |
|
|
return false;
|
923 |
|
|
|
924 |
|
|
/* If we are in the middle of a sequence, we use DEF from a previous
|
925 |
|
|
statement. Otherwise, OPRND has to be a result of type promotion. */
|
926 |
|
|
if (*new_type)
|
927 |
|
|
{
|
928 |
|
|
half_type = *new_type;
|
929 |
|
|
oprnd = def;
|
930 |
|
|
}
|
931 |
|
|
else
|
932 |
|
|
{
|
933 |
|
|
first = true;
|
934 |
|
|
if (!widened_name_p (oprnd, stmt, &half_type, &def_stmt, false)
|
935 |
|
|
|| !gimple_bb (def_stmt)
|
936 |
|
|
|| !flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
|
937 |
|
|
|| !vinfo_for_stmt (def_stmt))
|
938 |
|
|
return false;
|
939 |
|
|
}
|
940 |
|
|
|
941 |
|
|
/* Can we perform the operation on a smaller type? */
|
942 |
|
|
switch (code)
|
943 |
|
|
{
|
944 |
|
|
case BIT_IOR_EXPR:
|
945 |
|
|
case BIT_XOR_EXPR:
|
946 |
|
|
case BIT_AND_EXPR:
|
947 |
|
|
if (!int_fits_type_p (const_oprnd, half_type))
|
948 |
|
|
{
|
949 |
|
|
/* HALF_TYPE is not enough. Try a bigger type if possible. */
|
950 |
|
|
if (TYPE_PRECISION (type) < (TYPE_PRECISION (half_type) * 4))
|
951 |
|
|
return false;
|
952 |
|
|
|
953 |
|
|
interm_type = build_nonstandard_integer_type (
|
954 |
|
|
TYPE_PRECISION (half_type) * 2, TYPE_UNSIGNED (type));
|
955 |
|
|
if (!int_fits_type_p (const_oprnd, interm_type))
|
956 |
|
|
return false;
|
957 |
|
|
}
|
958 |
|
|
|
959 |
|
|
break;
|
960 |
|
|
|
961 |
|
|
case LSHIFT_EXPR:
|
962 |
|
|
/* Try intermediate type - HALF_TYPE is not enough for sure. */
|
963 |
|
|
if (TYPE_PRECISION (type) < (TYPE_PRECISION (half_type) * 4))
|
964 |
|
|
return false;
|
965 |
|
|
|
966 |
|
|
/* Check that HALF_TYPE size + shift amount <= INTERM_TYPE size.
|
967 |
|
|
(e.g., if the original value was char, the shift amount is at most 8
|
968 |
|
|
if we want to use short). */
|
969 |
|
|
if (compare_tree_int (const_oprnd, TYPE_PRECISION (half_type)) == 1)
|
970 |
|
|
return false;
|
971 |
|
|
|
972 |
|
|
interm_type = build_nonstandard_integer_type (
|
973 |
|
|
TYPE_PRECISION (half_type) * 2, TYPE_UNSIGNED (type));
|
974 |
|
|
|
975 |
|
|
if (!vect_supportable_shift (code, interm_type))
|
976 |
|
|
return false;
|
977 |
|
|
|
978 |
|
|
break;
|
979 |
|
|
|
980 |
|
|
case RSHIFT_EXPR:
|
981 |
|
|
if (vect_supportable_shift (code, half_type))
|
982 |
|
|
break;
|
983 |
|
|
|
984 |
|
|
/* Try intermediate type - HALF_TYPE is not supported. */
|
985 |
|
|
if (TYPE_PRECISION (type) < (TYPE_PRECISION (half_type) * 4))
|
986 |
|
|
return false;
|
987 |
|
|
|
988 |
|
|
interm_type = build_nonstandard_integer_type (
|
989 |
|
|
TYPE_PRECISION (half_type) * 2, TYPE_UNSIGNED (type));
|
990 |
|
|
|
991 |
|
|
if (!vect_supportable_shift (code, interm_type))
|
992 |
|
|
return false;
|
993 |
|
|
|
994 |
|
|
break;
|
995 |
|
|
|
996 |
|
|
default:
|
997 |
|
|
gcc_unreachable ();
|
998 |
|
|
}
|
999 |
|
|
|
1000 |
|
|
/* There are four possible cases:
|
1001 |
|
|
1. OPRND is defined by a type promotion (in that case FIRST is TRUE, it's
|
1002 |
|
|
the first statement in the sequence)
|
1003 |
|
|
a. The original, HALF_TYPE, is not enough - we replace the promotion
|
1004 |
|
|
from HALF_TYPE to TYPE with a promotion to INTERM_TYPE.
|
1005 |
|
|
b. HALF_TYPE is sufficient, OPRND is set as the RHS of the original
|
1006 |
|
|
promotion.
|
1007 |
|
|
2. OPRND is defined by a pattern statement we created.
|
1008 |
|
|
a. Its type is not sufficient for the operation, we create a new stmt:
|
1009 |
|
|
a type conversion for OPRND from HALF_TYPE to INTERM_TYPE. We store
|
1010 |
|
|
this statement in NEW_DEF_STMT, and it is later put in
|
1011 |
|
|
STMT_VINFO_PATTERN_DEF_SEQ of the pattern statement for STMT.
|
1012 |
|
|
b. OPRND is good to use in the new statement. */
|
1013 |
|
|
if (first)
|
1014 |
|
|
{
|
1015 |
|
|
if (interm_type)
|
1016 |
|
|
{
|
1017 |
|
|
/* Replace the original type conversion HALF_TYPE->TYPE with
|
1018 |
|
|
HALF_TYPE->INTERM_TYPE. */
|
1019 |
|
|
if (STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt)))
|
1020 |
|
|
{
|
1021 |
|
|
new_stmt = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt));
|
1022 |
|
|
/* Check if the already created pattern stmt is what we need. */
|
1023 |
|
|
if (!is_gimple_assign (new_stmt)
|
1024 |
|
|
|| gimple_assign_rhs_code (new_stmt) != NOP_EXPR
|
1025 |
|
|
|| TREE_TYPE (gimple_assign_lhs (new_stmt)) != interm_type)
|
1026 |
|
|
return false;
|
1027 |
|
|
|
1028 |
|
|
VEC_safe_push (gimple, heap, *stmts, def_stmt);
|
1029 |
|
|
oprnd = gimple_assign_lhs (new_stmt);
|
1030 |
|
|
}
|
1031 |
|
|
else
|
1032 |
|
|
{
|
1033 |
|
|
/* Create NEW_OPRND = (INTERM_TYPE) OPRND. */
|
1034 |
|
|
oprnd = gimple_assign_rhs1 (def_stmt);
|
1035 |
|
|
tmp = create_tmp_reg (interm_type, NULL);
|
1036 |
|
|
add_referenced_var (tmp);
|
1037 |
|
|
new_oprnd = make_ssa_name (tmp, NULL);
|
1038 |
|
|
new_stmt = gimple_build_assign_with_ops (NOP_EXPR, new_oprnd,
|
1039 |
|
|
oprnd, NULL_TREE);
|
1040 |
|
|
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt)) = new_stmt;
|
1041 |
|
|
VEC_safe_push (gimple, heap, *stmts, def_stmt);
|
1042 |
|
|
oprnd = new_oprnd;
|
1043 |
|
|
}
|
1044 |
|
|
}
|
1045 |
|
|
else
|
1046 |
|
|
{
|
1047 |
|
|
/* Retrieve the operand before the type promotion. */
|
1048 |
|
|
oprnd = gimple_assign_rhs1 (def_stmt);
|
1049 |
|
|
}
|
1050 |
|
|
}
|
1051 |
|
|
else
|
1052 |
|
|
{
|
1053 |
|
|
if (interm_type)
|
1054 |
|
|
{
|
1055 |
|
|
/* Create a type conversion HALF_TYPE->INTERM_TYPE. */
|
1056 |
|
|
tmp = create_tmp_reg (interm_type, NULL);
|
1057 |
|
|
add_referenced_var (tmp);
|
1058 |
|
|
new_oprnd = make_ssa_name (tmp, NULL);
|
1059 |
|
|
new_stmt = gimple_build_assign_with_ops (NOP_EXPR, new_oprnd,
|
1060 |
|
|
oprnd, NULL_TREE);
|
1061 |
|
|
oprnd = new_oprnd;
|
1062 |
|
|
*new_def_stmt = new_stmt;
|
1063 |
|
|
}
|
1064 |
|
|
|
1065 |
|
|
/* Otherwise, OPRND is already set. */
|
1066 |
|
|
}
|
1067 |
|
|
|
1068 |
|
|
if (interm_type)
|
1069 |
|
|
*new_type = interm_type;
|
1070 |
|
|
else
|
1071 |
|
|
*new_type = half_type;
|
1072 |
|
|
|
1073 |
|
|
*op0 = oprnd;
|
1074 |
|
|
*op1 = fold_convert (*new_type, const_oprnd);
|
1075 |
|
|
|
1076 |
|
|
return true;
|
1077 |
|
|
}
|
1078 |
|
|
|
1079 |
|
|
|
1080 |
|
|
/* Try to find a statement or a sequence of statements that can be performed
|
1081 |
|
|
on a smaller type:
|
1082 |
|
|
|
1083 |
|
|
type x_t;
|
1084 |
|
|
TYPE x_T, res0_T, res1_T;
|
1085 |
|
|
loop:
|
1086 |
|
|
S1 x_t = *p;
|
1087 |
|
|
S2 x_T = (TYPE) x_t;
|
1088 |
|
|
S3 res0_T = op (x_T, C0);
|
1089 |
|
|
S4 res1_T = op (res0_T, C1);
|
1090 |
|
|
S5 ... = () res1_T; - type demotion
|
1091 |
|
|
|
1092 |
|
|
where type 'TYPE' is at least double the size of type 'type', C0 and C1 are
|
1093 |
|
|
constants.
|
1094 |
|
|
Check if S3 and S4 can be done on a smaller type than 'TYPE', it can either
|
1095 |
|
|
be 'type' or some intermediate type. For now, we expect S5 to be a type
|
1096 |
|
|
demotion operation. We also check that S3 and S4 have only one use. */
|
1097 |
|
|
|
1098 |
|
|
static gimple
|
1099 |
|
|
vect_recog_over_widening_pattern (VEC (gimple, heap) **stmts,
|
1100 |
|
|
tree *type_in, tree *type_out)
|
1101 |
|
|
{
|
1102 |
|
|
gimple stmt = VEC_pop (gimple, *stmts);
|
1103 |
|
|
gimple pattern_stmt = NULL, new_def_stmt, prev_stmt = NULL, use_stmt = NULL;
|
1104 |
|
|
tree op0, op1, vectype = NULL_TREE, lhs, use_lhs, use_type;
|
1105 |
|
|
imm_use_iterator imm_iter;
|
1106 |
|
|
use_operand_p use_p;
|
1107 |
|
|
int nuses = 0;
|
1108 |
|
|
tree var = NULL_TREE, new_type = NULL_TREE, tmp, new_oprnd;
|
1109 |
|
|
bool first;
|
1110 |
|
|
struct loop *loop = (gimple_bb (stmt))->loop_father;
|
1111 |
|
|
tree type = NULL;
|
1112 |
|
|
|
1113 |
|
|
first = true;
|
1114 |
|
|
while (1)
|
1115 |
|
|
{
|
1116 |
|
|
if (!vinfo_for_stmt (stmt)
|
1117 |
|
|
|| STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (stmt)))
|
1118 |
|
|
return NULL;
|
1119 |
|
|
|
1120 |
|
|
new_def_stmt = NULL;
|
1121 |
|
|
if (!vect_operation_fits_smaller_type (stmt, var, &new_type,
|
1122 |
|
|
&op0, &op1, &new_def_stmt,
|
1123 |
|
|
stmts))
|
1124 |
|
|
{
|
1125 |
|
|
if (first)
|
1126 |
|
|
return NULL;
|
1127 |
|
|
else
|
1128 |
|
|
break;
|
1129 |
|
|
}
|
1130 |
|
|
|
1131 |
|
|
/* STMT can be performed on a smaller type. Check its uses. */
|
1132 |
|
|
lhs = gimple_assign_lhs (stmt);
|
1133 |
|
|
nuses = 0;
|
1134 |
|
|
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
|
1135 |
|
|
{
|
1136 |
|
|
if (is_gimple_debug (USE_STMT (use_p)))
|
1137 |
|
|
continue;
|
1138 |
|
|
use_stmt = USE_STMT (use_p);
|
1139 |
|
|
nuses++;
|
1140 |
|
|
}
|
1141 |
|
|
|
1142 |
|
|
if (nuses != 1 || !is_gimple_assign (use_stmt)
|
1143 |
|
|
|| !gimple_bb (use_stmt)
|
1144 |
|
|
|| !flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
|
1145 |
|
|
return NULL;
|
1146 |
|
|
|
1147 |
|
|
/* Create pattern statement for STMT. */
|
1148 |
|
|
vectype = get_vectype_for_scalar_type (new_type);
|
1149 |
|
|
if (!vectype)
|
1150 |
|
|
return NULL;
|
1151 |
|
|
|
1152 |
|
|
/* We want to collect all the statements for which we create pattern
|
1153 |
|
|
statetments, except for the case when the last statement in the
|
1154 |
|
|
sequence doesn't have a corresponding pattern statement. In such
|
1155 |
|
|
case we associate the last pattern statement with the last statement
|
1156 |
|
|
in the sequence. Therefore, we only add the original statement to
|
1157 |
|
|
the list if we know that it is not the last. */
|
1158 |
|
|
if (prev_stmt)
|
1159 |
|
|
VEC_safe_push (gimple, heap, *stmts, prev_stmt);
|
1160 |
|
|
|
1161 |
|
|
var = vect_recog_temp_ssa_var (new_type, NULL);
|
1162 |
|
|
pattern_stmt
|
1163 |
|
|
= gimple_build_assign_with_ops (gimple_assign_rhs_code (stmt), var,
|
1164 |
|
|
op0, op1);
|
1165 |
|
|
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt)) = pattern_stmt;
|
1166 |
|
|
new_pattern_def_seq (vinfo_for_stmt (stmt), new_def_stmt);
|
1167 |
|
|
|
1168 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
1169 |
|
|
{
|
1170 |
|
|
fprintf (vect_dump, "created pattern stmt: ");
|
1171 |
|
|
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
|
1172 |
|
|
}
|
1173 |
|
|
|
1174 |
|
|
type = gimple_expr_type (stmt);
|
1175 |
|
|
prev_stmt = stmt;
|
1176 |
|
|
stmt = use_stmt;
|
1177 |
|
|
|
1178 |
|
|
first = false;
|
1179 |
|
|
}
|
1180 |
|
|
|
1181 |
|
|
/* We got a sequence. We expect it to end with a type demotion operation.
|
1182 |
|
|
Otherwise, we quit (for now). There are three possible cases: the
|
1183 |
|
|
conversion is to NEW_TYPE (we don't do anything), the conversion is to
|
1184 |
|
|
a type bigger than NEW_TYPE and/or the signedness of USE_TYPE and
|
1185 |
|
|
NEW_TYPE differs (we create a new conversion statement). */
|
1186 |
|
|
if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (use_stmt)))
|
1187 |
|
|
{
|
1188 |
|
|
use_lhs = gimple_assign_lhs (use_stmt);
|
1189 |
|
|
use_type = TREE_TYPE (use_lhs);
|
1190 |
|
|
/* Support only type demotion or signedess change. */
|
1191 |
|
|
if (!INTEGRAL_TYPE_P (use_type)
|
1192 |
|
|
|| TYPE_PRECISION (type) <= TYPE_PRECISION (use_type))
|
1193 |
|
|
return NULL;
|
1194 |
|
|
|
1195 |
|
|
/* Check that NEW_TYPE is not bigger than the conversion result. */
|
1196 |
|
|
if (TYPE_PRECISION (new_type) > TYPE_PRECISION (use_type))
|
1197 |
|
|
return NULL;
|
1198 |
|
|
|
1199 |
|
|
if (TYPE_UNSIGNED (new_type) != TYPE_UNSIGNED (use_type)
|
1200 |
|
|
|| TYPE_PRECISION (new_type) != TYPE_PRECISION (use_type))
|
1201 |
|
|
{
|
1202 |
|
|
/* Create NEW_TYPE->USE_TYPE conversion. */
|
1203 |
|
|
tmp = create_tmp_reg (use_type, NULL);
|
1204 |
|
|
add_referenced_var (tmp);
|
1205 |
|
|
new_oprnd = make_ssa_name (tmp, NULL);
|
1206 |
|
|
pattern_stmt = gimple_build_assign_with_ops (NOP_EXPR, new_oprnd,
|
1207 |
|
|
var, NULL_TREE);
|
1208 |
|
|
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (use_stmt)) = pattern_stmt;
|
1209 |
|
|
|
1210 |
|
|
*type_in = get_vectype_for_scalar_type (new_type);
|
1211 |
|
|
*type_out = get_vectype_for_scalar_type (use_type);
|
1212 |
|
|
|
1213 |
|
|
/* We created a pattern statement for the last statement in the
|
1214 |
|
|
sequence, so we don't need to associate it with the pattern
|
1215 |
|
|
statement created for PREV_STMT. Therefore, we add PREV_STMT
|
1216 |
|
|
to the list in order to mark it later in vect_pattern_recog_1. */
|
1217 |
|
|
if (prev_stmt)
|
1218 |
|
|
VEC_safe_push (gimple, heap, *stmts, prev_stmt);
|
1219 |
|
|
}
|
1220 |
|
|
else
|
1221 |
|
|
{
|
1222 |
|
|
if (prev_stmt)
|
1223 |
|
|
STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (use_stmt))
|
1224 |
|
|
= STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (prev_stmt));
|
1225 |
|
|
|
1226 |
|
|
*type_in = vectype;
|
1227 |
|
|
*type_out = NULL_TREE;
|
1228 |
|
|
}
|
1229 |
|
|
|
1230 |
|
|
VEC_safe_push (gimple, heap, *stmts, use_stmt);
|
1231 |
|
|
}
|
1232 |
|
|
else
|
1233 |
|
|
/* TODO: support general case, create a conversion to the correct type. */
|
1234 |
|
|
return NULL;
|
1235 |
|
|
|
1236 |
|
|
/* Pattern detected. */
|
1237 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
1238 |
|
|
{
|
1239 |
|
|
fprintf (vect_dump, "vect_recog_over_widening_pattern: detected: ");
|
1240 |
|
|
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
|
1241 |
|
|
}
|
1242 |
|
|
|
1243 |
|
|
return pattern_stmt;
|
1244 |
|
|
}
|
1245 |
|
|
|
1246 |
|
|
/* Detect widening shift pattern:
|
1247 |
|
|
|
1248 |
|
|
type a_t;
|
1249 |
|
|
TYPE a_T, res_T;
|
1250 |
|
|
|
1251 |
|
|
S1 a_t = ;
|
1252 |
|
|
S2 a_T = (TYPE) a_t;
|
1253 |
|
|
S3 res_T = a_T << CONST;
|
1254 |
|
|
|
1255 |
|
|
where type 'TYPE' is at least double the size of type 'type'.
|
1256 |
|
|
|
1257 |
|
|
Also detect unsigned cases:
|
1258 |
|
|
|
1259 |
|
|
unsigned type a_t;
|
1260 |
|
|
unsigned TYPE u_res_T;
|
1261 |
|
|
TYPE a_T, res_T;
|
1262 |
|
|
|
1263 |
|
|
S1 a_t = ;
|
1264 |
|
|
S2 a_T = (TYPE) a_t;
|
1265 |
|
|
S3 res_T = a_T << CONST;
|
1266 |
|
|
S4 u_res_T = (unsigned TYPE) res_T;
|
1267 |
|
|
|
1268 |
|
|
And a case when 'TYPE' is 4 times bigger than 'type'. In that case we
|
1269 |
|
|
create an additional pattern stmt for S2 to create a variable of an
|
1270 |
|
|
intermediate type, and perform widen-shift on the intermediate type:
|
1271 |
|
|
|
1272 |
|
|
type a_t;
|
1273 |
|
|
interm_type a_it;
|
1274 |
|
|
TYPE a_T, res_T, res_T';
|
1275 |
|
|
|
1276 |
|
|
S1 a_t = ;
|
1277 |
|
|
S2 a_T = (TYPE) a_t;
|
1278 |
|
|
'--> a_it = (interm_type) a_t;
|
1279 |
|
|
S3 res_T = a_T << CONST;
|
1280 |
|
|
'--> res_T' = a_it <<* CONST;
|
1281 |
|
|
|
1282 |
|
|
Input/Output:
|
1283 |
|
|
|
1284 |
|
|
* STMTS: Contains a stmt from which the pattern search begins.
|
1285 |
|
|
In case of unsigned widen-shift, the original stmt (S3) is replaced with S4
|
1286 |
|
|
in STMTS. When an intermediate type is used and a pattern statement is
|
1287 |
|
|
created for S2, we also put S2 here (before S3).
|
1288 |
|
|
|
1289 |
|
|
Output:
|
1290 |
|
|
|
1291 |
|
|
* TYPE_IN: The type of the input arguments to the pattern.
|
1292 |
|
|
|
1293 |
|
|
* TYPE_OUT: The type of the output of this pattern.
|
1294 |
|
|
|
1295 |
|
|
* Return value: A new stmt that will be used to replace the sequence of
|
1296 |
|
|
stmts that constitute the pattern. In this case it will be:
|
1297 |
|
|
WIDEN_LSHIFT_EXPR <a_t, CONST>. */
|
1298 |
|
|
|
1299 |
|
|
static gimple
|
1300 |
|
|
vect_recog_widen_shift_pattern (VEC (gimple, heap) **stmts,
|
1301 |
|
|
tree *type_in, tree *type_out)
|
1302 |
|
|
{
|
1303 |
|
|
gimple last_stmt = VEC_pop (gimple, *stmts);
|
1304 |
|
|
gimple def_stmt0;
|
1305 |
|
|
tree oprnd0, oprnd1;
|
1306 |
|
|
tree type, half_type0;
|
1307 |
|
|
gimple pattern_stmt, orig_stmt = NULL;
|
1308 |
|
|
tree vectype, vectype_out = NULL_TREE;
|
1309 |
|
|
tree dummy;
|
1310 |
|
|
tree var;
|
1311 |
|
|
enum tree_code dummy_code;
|
1312 |
|
|
int dummy_int;
|
1313 |
|
|
VEC (tree, heap) * dummy_vec;
|
1314 |
|
|
gimple use_stmt = NULL;
|
1315 |
|
|
bool over_widen = false;
|
1316 |
|
|
|
1317 |
|
|
if (!is_gimple_assign (last_stmt) || !vinfo_for_stmt (last_stmt))
|
1318 |
|
|
return NULL;
|
1319 |
|
|
|
1320 |
|
|
orig_stmt = last_stmt;
|
1321 |
|
|
if (STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (last_stmt)))
|
1322 |
|
|
{
|
1323 |
|
|
/* This statement was also detected as over-widening operation (it can't
|
1324 |
|
|
be any other pattern, because only over-widening detects shifts).
|
1325 |
|
|
LAST_STMT is the final type demotion statement, but its related
|
1326 |
|
|
statement is shift. We analyze the related statement to catch cases:
|
1327 |
|
|
|
1328 |
|
|
orig code:
|
1329 |
|
|
type a_t;
|
1330 |
|
|
itype res;
|
1331 |
|
|
TYPE a_T, res_T;
|
1332 |
|
|
|
1333 |
|
|
S1 a_T = (TYPE) a_t;
|
1334 |
|
|
S2 res_T = a_T << CONST;
|
1335 |
|
|
S3 res = (itype)res_T;
|
1336 |
|
|
|
1337 |
|
|
(size of type * 2 <= size of itype
|
1338 |
|
|
and size of itype * 2 <= size of TYPE)
|
1339 |
|
|
|
1340 |
|
|
code after over-widening pattern detection:
|
1341 |
|
|
|
1342 |
|
|
S1 a_T = (TYPE) a_t;
|
1343 |
|
|
--> a_it = (itype) a_t;
|
1344 |
|
|
S2 res_T = a_T << CONST;
|
1345 |
|
|
S3 res = (itype)res_T; <--- LAST_STMT
|
1346 |
|
|
--> res = a_it << CONST;
|
1347 |
|
|
|
1348 |
|
|
after widen_shift:
|
1349 |
|
|
|
1350 |
|
|
S1 a_T = (TYPE) a_t;
|
1351 |
|
|
--> a_it = (itype) a_t; - redundant
|
1352 |
|
|
S2 res_T = a_T << CONST;
|
1353 |
|
|
S3 res = (itype)res_T;
|
1354 |
|
|
--> res = a_t w<< CONST;
|
1355 |
|
|
|
1356 |
|
|
i.e., we replace the three statements with res = a_t w<< CONST. */
|
1357 |
|
|
last_stmt = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (last_stmt));
|
1358 |
|
|
over_widen = true;
|
1359 |
|
|
}
|
1360 |
|
|
|
1361 |
|
|
if (gimple_assign_rhs_code (last_stmt) != LSHIFT_EXPR)
|
1362 |
|
|
return NULL;
|
1363 |
|
|
|
1364 |
|
|
oprnd0 = gimple_assign_rhs1 (last_stmt);
|
1365 |
|
|
oprnd1 = gimple_assign_rhs2 (last_stmt);
|
1366 |
|
|
if (TREE_CODE (oprnd0) != SSA_NAME || TREE_CODE (oprnd1) != INTEGER_CST)
|
1367 |
|
|
return NULL;
|
1368 |
|
|
|
1369 |
|
|
/* Check operand 0: it has to be defined by a type promotion. */
|
1370 |
|
|
if (!widened_name_p (oprnd0, last_stmt, &half_type0, &def_stmt0, false))
|
1371 |
|
|
return NULL;
|
1372 |
|
|
|
1373 |
|
|
/* Check operand 1: has to be positive. We check that it fits the type
|
1374 |
|
|
in vect_handle_widen_op_by_const (). */
|
1375 |
|
|
if (tree_int_cst_compare (oprnd1, size_zero_node) <= 0)
|
1376 |
|
|
return NULL;
|
1377 |
|
|
|
1378 |
|
|
oprnd0 = gimple_assign_rhs1 (def_stmt0);
|
1379 |
|
|
type = gimple_expr_type (last_stmt);
|
1380 |
|
|
|
1381 |
|
|
/* Check if this a widening operation. */
|
1382 |
|
|
if (!vect_handle_widen_op_by_const (last_stmt, LSHIFT_EXPR, oprnd1,
|
1383 |
|
|
&oprnd0, stmts,
|
1384 |
|
|
type, &half_type0, def_stmt0))
|
1385 |
|
|
return NULL;
|
1386 |
|
|
|
1387 |
|
|
/* Handle unsigned case. Look for
|
1388 |
|
|
S4 u_res_T = (unsigned TYPE) res_T;
|
1389 |
|
|
Use unsigned TYPE as the type for WIDEN_LSHIFT_EXPR. */
|
1390 |
|
|
if (TYPE_UNSIGNED (type) != TYPE_UNSIGNED (half_type0))
|
1391 |
|
|
{
|
1392 |
|
|
tree lhs = gimple_assign_lhs (last_stmt), use_lhs;
|
1393 |
|
|
imm_use_iterator imm_iter;
|
1394 |
|
|
use_operand_p use_p;
|
1395 |
|
|
int nuses = 0;
|
1396 |
|
|
tree use_type;
|
1397 |
|
|
|
1398 |
|
|
if (over_widen)
|
1399 |
|
|
{
|
1400 |
|
|
/* In case of over-widening pattern, S4 should be ORIG_STMT itself.
|
1401 |
|
|
We check here that TYPE is the correct type for the operation,
|
1402 |
|
|
i.e., it's the type of the original result. */
|
1403 |
|
|
tree orig_type = gimple_expr_type (orig_stmt);
|
1404 |
|
|
if ((TYPE_UNSIGNED (type) != TYPE_UNSIGNED (orig_type))
|
1405 |
|
|
|| (TYPE_PRECISION (type) != TYPE_PRECISION (orig_type)))
|
1406 |
|
|
return NULL;
|
1407 |
|
|
}
|
1408 |
|
|
else
|
1409 |
|
|
{
|
1410 |
|
|
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
|
1411 |
|
|
{
|
1412 |
|
|
if (is_gimple_debug (USE_STMT (use_p)))
|
1413 |
|
|
continue;
|
1414 |
|
|
use_stmt = USE_STMT (use_p);
|
1415 |
|
|
nuses++;
|
1416 |
|
|
}
|
1417 |
|
|
|
1418 |
|
|
if (nuses != 1 || !is_gimple_assign (use_stmt)
|
1419 |
|
|
|| !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (use_stmt)))
|
1420 |
|
|
return NULL;
|
1421 |
|
|
|
1422 |
|
|
use_lhs = gimple_assign_lhs (use_stmt);
|
1423 |
|
|
use_type = TREE_TYPE (use_lhs);
|
1424 |
|
|
|
1425 |
|
|
if (!INTEGRAL_TYPE_P (use_type)
|
1426 |
|
|
|| (TYPE_UNSIGNED (type) == TYPE_UNSIGNED (use_type))
|
1427 |
|
|
|| (TYPE_PRECISION (type) != TYPE_PRECISION (use_type)))
|
1428 |
|
|
return NULL;
|
1429 |
|
|
|
1430 |
|
|
type = use_type;
|
1431 |
|
|
}
|
1432 |
|
|
}
|
1433 |
|
|
|
1434 |
|
|
/* Pattern detected. */
|
1435 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
1436 |
|
|
fprintf (vect_dump, "vect_recog_widen_shift_pattern: detected: ");
|
1437 |
|
|
|
1438 |
|
|
/* Check target support. */
|
1439 |
|
|
vectype = get_vectype_for_scalar_type (half_type0);
|
1440 |
|
|
vectype_out = get_vectype_for_scalar_type (type);
|
1441 |
|
|
|
1442 |
|
|
if (!vectype
|
1443 |
|
|
|| !vectype_out
|
1444 |
|
|
|| !supportable_widening_operation (WIDEN_LSHIFT_EXPR, last_stmt,
|
1445 |
|
|
vectype_out, vectype,
|
1446 |
|
|
&dummy, &dummy, &dummy_code,
|
1447 |
|
|
&dummy_code, &dummy_int,
|
1448 |
|
|
&dummy_vec))
|
1449 |
|
|
return NULL;
|
1450 |
|
|
|
1451 |
|
|
*type_in = vectype;
|
1452 |
|
|
*type_out = vectype_out;
|
1453 |
|
|
|
1454 |
|
|
/* Pattern supported. Create a stmt to be used to replace the pattern. */
|
1455 |
|
|
var = vect_recog_temp_ssa_var (type, NULL);
|
1456 |
|
|
pattern_stmt =
|
1457 |
|
|
gimple_build_assign_with_ops (WIDEN_LSHIFT_EXPR, var, oprnd0, oprnd1);
|
1458 |
|
|
|
1459 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
1460 |
|
|
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
|
1461 |
|
|
|
1462 |
|
|
if (use_stmt)
|
1463 |
|
|
last_stmt = use_stmt;
|
1464 |
|
|
else
|
1465 |
|
|
last_stmt = orig_stmt;
|
1466 |
|
|
|
1467 |
|
|
VEC_safe_push (gimple, heap, *stmts, last_stmt);
|
1468 |
|
|
return pattern_stmt;
|
1469 |
|
|
}
|
1470 |
|
|
|
1471 |
|
|
/* Detect a vector by vector shift pattern that wouldn't be otherwise
|
1472 |
|
|
vectorized:
|
1473 |
|
|
|
1474 |
|
|
type a_t;
|
1475 |
|
|
TYPE b_T, res_T;
|
1476 |
|
|
|
1477 |
|
|
S1 a_t = ;
|
1478 |
|
|
S2 b_T = ;
|
1479 |
|
|
S3 res_T = b_T op a_t;
|
1480 |
|
|
|
1481 |
|
|
where type 'TYPE' is a type with different size than 'type',
|
1482 |
|
|
and op is <<, >> or rotate.
|
1483 |
|
|
|
1484 |
|
|
Also detect cases:
|
1485 |
|
|
|
1486 |
|
|
type a_t;
|
1487 |
|
|
TYPE b_T, c_T, res_T;
|
1488 |
|
|
|
1489 |
|
|
S0 c_T = ;
|
1490 |
|
|
S1 a_t = (type) c_T;
|
1491 |
|
|
S2 b_T = ;
|
1492 |
|
|
S3 res_T = b_T op a_t;
|
1493 |
|
|
|
1494 |
|
|
Input/Output:
|
1495 |
|
|
|
1496 |
|
|
* STMTS: Contains a stmt from which the pattern search begins,
|
1497 |
|
|
i.e. the shift/rotate stmt. The original stmt (S3) is replaced
|
1498 |
|
|
with a shift/rotate which has same type on both operands, in the
|
1499 |
|
|
second case just b_T op c_T, in the first case with added cast
|
1500 |
|
|
from a_t to c_T in STMT_VINFO_PATTERN_DEF_SEQ.
|
1501 |
|
|
|
1502 |
|
|
Output:
|
1503 |
|
|
|
1504 |
|
|
* TYPE_IN: The type of the input arguments to the pattern.
|
1505 |
|
|
|
1506 |
|
|
* TYPE_OUT: The type of the output of this pattern.
|
1507 |
|
|
|
1508 |
|
|
* Return value: A new stmt that will be used to replace the shift/rotate
|
1509 |
|
|
S3 stmt. */
|
1510 |
|
|
|
1511 |
|
|
static gimple
|
1512 |
|
|
vect_recog_vector_vector_shift_pattern (VEC (gimple, heap) **stmts,
|
1513 |
|
|
tree *type_in, tree *type_out)
|
1514 |
|
|
{
|
1515 |
|
|
gimple last_stmt = VEC_pop (gimple, *stmts);
|
1516 |
|
|
tree oprnd0, oprnd1, lhs, var;
|
1517 |
|
|
gimple pattern_stmt, def_stmt;
|
1518 |
|
|
enum tree_code rhs_code;
|
1519 |
|
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
|
1520 |
|
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
|
1521 |
|
|
enum vect_def_type dt;
|
1522 |
|
|
tree def;
|
1523 |
|
|
|
1524 |
|
|
if (!is_gimple_assign (last_stmt))
|
1525 |
|
|
return NULL;
|
1526 |
|
|
|
1527 |
|
|
rhs_code = gimple_assign_rhs_code (last_stmt);
|
1528 |
|
|
switch (rhs_code)
|
1529 |
|
|
{
|
1530 |
|
|
case LSHIFT_EXPR:
|
1531 |
|
|
case RSHIFT_EXPR:
|
1532 |
|
|
case LROTATE_EXPR:
|
1533 |
|
|
case RROTATE_EXPR:
|
1534 |
|
|
break;
|
1535 |
|
|
default:
|
1536 |
|
|
return NULL;
|
1537 |
|
|
}
|
1538 |
|
|
|
1539 |
|
|
if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
|
1540 |
|
|
return NULL;
|
1541 |
|
|
|
1542 |
|
|
lhs = gimple_assign_lhs (last_stmt);
|
1543 |
|
|
oprnd0 = gimple_assign_rhs1 (last_stmt);
|
1544 |
|
|
oprnd1 = gimple_assign_rhs2 (last_stmt);
|
1545 |
|
|
if (TREE_CODE (oprnd0) != SSA_NAME
|
1546 |
|
|
|| TREE_CODE (oprnd1) != SSA_NAME
|
1547 |
|
|
|| TYPE_MODE (TREE_TYPE (oprnd0)) == TYPE_MODE (TREE_TYPE (oprnd1))
|
1548 |
|
|
|| TYPE_PRECISION (TREE_TYPE (oprnd1))
|
1549 |
|
|
!= GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (oprnd1)))
|
1550 |
|
|
|| TYPE_PRECISION (TREE_TYPE (lhs))
|
1551 |
|
|
!= TYPE_PRECISION (TREE_TYPE (oprnd0)))
|
1552 |
|
|
return NULL;
|
1553 |
|
|
|
1554 |
|
|
if (!vect_is_simple_use (oprnd1, last_stmt, loop_vinfo, NULL, &def_stmt,
|
1555 |
|
|
&def, &dt))
|
1556 |
|
|
return NULL;
|
1557 |
|
|
|
1558 |
|
|
if (dt != vect_internal_def)
|
1559 |
|
|
return NULL;
|
1560 |
|
|
|
1561 |
|
|
*type_in = get_vectype_for_scalar_type (TREE_TYPE (oprnd0));
|
1562 |
|
|
*type_out = *type_in;
|
1563 |
|
|
if (*type_in == NULL_TREE)
|
1564 |
|
|
return NULL;
|
1565 |
|
|
|
1566 |
|
|
def = NULL_TREE;
|
1567 |
|
|
if (gimple_assign_cast_p (def_stmt))
|
1568 |
|
|
{
|
1569 |
|
|
tree rhs1 = gimple_assign_rhs1 (def_stmt);
|
1570 |
|
|
if (TYPE_MODE (TREE_TYPE (rhs1)) == TYPE_MODE (TREE_TYPE (oprnd0))
|
1571 |
|
|
&& TYPE_PRECISION (TREE_TYPE (rhs1))
|
1572 |
|
|
== TYPE_PRECISION (TREE_TYPE (oprnd0)))
|
1573 |
|
|
def = rhs1;
|
1574 |
|
|
}
|
1575 |
|
|
|
1576 |
|
|
if (def == NULL_TREE)
|
1577 |
|
|
{
|
1578 |
|
|
def = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
|
1579 |
|
|
def_stmt = gimple_build_assign_with_ops (NOP_EXPR, def, oprnd1,
|
1580 |
|
|
NULL_TREE);
|
1581 |
|
|
new_pattern_def_seq (stmt_vinfo, def_stmt);
|
1582 |
|
|
}
|
1583 |
|
|
|
1584 |
|
|
/* Pattern detected. */
|
1585 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
1586 |
|
|
fprintf (vect_dump, "vect_recog_vector_vector_shift_pattern: detected: ");
|
1587 |
|
|
|
1588 |
|
|
/* Pattern supported. Create a stmt to be used to replace the pattern. */
|
1589 |
|
|
var = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
|
1590 |
|
|
pattern_stmt = gimple_build_assign_with_ops (rhs_code, var, oprnd0, def);
|
1591 |
|
|
|
1592 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
1593 |
|
|
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
|
1594 |
|
|
|
1595 |
|
|
VEC_safe_push (gimple, heap, *stmts, last_stmt);
|
1596 |
|
|
return pattern_stmt;
|
1597 |
|
|
}
|
1598 |
|
|
|
1599 |
|
|
/* Detect a signed division by power of two constant that wouldn't be
|
1600 |
|
|
otherwise vectorized:
|
1601 |
|
|
|
1602 |
|
|
type a_t, b_t;
|
1603 |
|
|
|
1604 |
|
|
S1 a_t = b_t / N;
|
1605 |
|
|
|
1606 |
|
|
where type 'type' is a signed integral type and N is a constant positive
|
1607 |
|
|
power of two.
|
1608 |
|
|
|
1609 |
|
|
Similarly handle signed modulo by power of two constant:
|
1610 |
|
|
|
1611 |
|
|
S4 a_t = b_t % N;
|
1612 |
|
|
|
1613 |
|
|
Input/Output:
|
1614 |
|
|
|
1615 |
|
|
* STMTS: Contains a stmt from which the pattern search begins,
|
1616 |
|
|
i.e. the division stmt. S1 is replaced by:
|
1617 |
|
|
S3 y_t = b_t < 0 ? N - 1 : 0;
|
1618 |
|
|
S2 x_t = b_t + y_t;
|
1619 |
|
|
S1' a_t = x_t >> log2 (N);
|
1620 |
|
|
|
1621 |
|
|
S4 is replaced by (where *_T temporaries have unsigned type):
|
1622 |
|
|
S9 y_T = b_t < 0 ? -1U : 0U;
|
1623 |
|
|
S8 z_T = y_T >> (sizeof (type_t) * CHAR_BIT - log2 (N));
|
1624 |
|
|
S7 z_t = (type) z_T;
|
1625 |
|
|
S6 w_t = b_t + z_t;
|
1626 |
|
|
S5 x_t = w_t & (N - 1);
|
1627 |
|
|
S4' a_t = x_t - z_t;
|
1628 |
|
|
|
1629 |
|
|
Output:
|
1630 |
|
|
|
1631 |
|
|
* TYPE_IN: The type of the input arguments to the pattern.
|
1632 |
|
|
|
1633 |
|
|
* TYPE_OUT: The type of the output of this pattern.
|
1634 |
|
|
|
1635 |
|
|
* Return value: A new stmt that will be used to replace the division
|
1636 |
|
|
S1 or modulo S4 stmt. */
|
1637 |
|
|
|
1638 |
|
|
static gimple
|
1639 |
|
|
vect_recog_sdivmod_pow2_pattern (VEC (gimple, heap) **stmts,
|
1640 |
|
|
tree *type_in, tree *type_out)
|
1641 |
|
|
{
|
1642 |
|
|
gimple last_stmt = VEC_pop (gimple, *stmts);
|
1643 |
|
|
tree oprnd0, oprnd1, vectype, itype, cond;
|
1644 |
|
|
gimple pattern_stmt, def_stmt;
|
1645 |
|
|
enum tree_code rhs_code;
|
1646 |
|
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
|
1647 |
|
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
|
1648 |
|
|
optab optab;
|
1649 |
|
|
|
1650 |
|
|
if (!is_gimple_assign (last_stmt))
|
1651 |
|
|
return NULL;
|
1652 |
|
|
|
1653 |
|
|
rhs_code = gimple_assign_rhs_code (last_stmt);
|
1654 |
|
|
switch (rhs_code)
|
1655 |
|
|
{
|
1656 |
|
|
case TRUNC_DIV_EXPR:
|
1657 |
|
|
case TRUNC_MOD_EXPR:
|
1658 |
|
|
break;
|
1659 |
|
|
default:
|
1660 |
|
|
return NULL;
|
1661 |
|
|
}
|
1662 |
|
|
|
1663 |
|
|
if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
|
1664 |
|
|
return NULL;
|
1665 |
|
|
|
1666 |
|
|
oprnd0 = gimple_assign_rhs1 (last_stmt);
|
1667 |
|
|
oprnd1 = gimple_assign_rhs2 (last_stmt);
|
1668 |
|
|
itype = TREE_TYPE (oprnd0);
|
1669 |
|
|
if (TREE_CODE (oprnd0) != SSA_NAME
|
1670 |
|
|
|| TREE_CODE (oprnd1) != INTEGER_CST
|
1671 |
|
|
|| TREE_CODE (itype) != INTEGER_TYPE
|
1672 |
|
|
|| TYPE_UNSIGNED (itype)
|
1673 |
|
|
|| TYPE_PRECISION (itype) != GET_MODE_PRECISION (TYPE_MODE (itype))
|
1674 |
|
|
|| !integer_pow2p (oprnd1)
|
1675 |
|
|
|| tree_int_cst_sgn (oprnd1) != 1)
|
1676 |
|
|
return NULL;
|
1677 |
|
|
|
1678 |
|
|
vectype = get_vectype_for_scalar_type (itype);
|
1679 |
|
|
if (vectype == NULL_TREE)
|
1680 |
|
|
return NULL;
|
1681 |
|
|
|
1682 |
|
|
/* If the target can handle vectorized division or modulo natively,
|
1683 |
|
|
don't attempt to optimize this. */
|
1684 |
|
|
optab = optab_for_tree_code (rhs_code, vectype, optab_default);
|
1685 |
|
|
if (optab != NULL)
|
1686 |
|
|
{
|
1687 |
|
|
enum machine_mode vec_mode = TYPE_MODE (vectype);
|
1688 |
|
|
int icode = (int) optab_handler (optab, vec_mode);
|
1689 |
|
|
if (icode != CODE_FOR_nothing
|
1690 |
|
|
|| GET_MODE_SIZE (vec_mode) == UNITS_PER_WORD)
|
1691 |
|
|
return NULL;
|
1692 |
|
|
}
|
1693 |
|
|
|
1694 |
|
|
/* Pattern detected. */
|
1695 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
1696 |
|
|
fprintf (vect_dump, "vect_recog_sdivmod_pow2_pattern: detected: ");
|
1697 |
|
|
|
1698 |
|
|
cond = build2 (LT_EXPR, boolean_type_node, oprnd0, build_int_cst (itype, 0));
|
1699 |
|
|
if (rhs_code == TRUNC_DIV_EXPR)
|
1700 |
|
|
{
|
1701 |
|
|
tree var = vect_recog_temp_ssa_var (itype, NULL);
|
1702 |
|
|
def_stmt
|
1703 |
|
|
= gimple_build_assign_with_ops3 (COND_EXPR, var, cond,
|
1704 |
|
|
fold_build2 (MINUS_EXPR, itype,
|
1705 |
|
|
oprnd1,
|
1706 |
|
|
build_int_cst (itype,
|
1707 |
|
|
1)),
|
1708 |
|
|
build_int_cst (itype, 0));
|
1709 |
|
|
new_pattern_def_seq (stmt_vinfo, def_stmt);
|
1710 |
|
|
var = vect_recog_temp_ssa_var (itype, NULL);
|
1711 |
|
|
def_stmt
|
1712 |
|
|
= gimple_build_assign_with_ops (PLUS_EXPR, var, oprnd0,
|
1713 |
|
|
gimple_assign_lhs (def_stmt));
|
1714 |
|
|
append_pattern_def_seq (stmt_vinfo, def_stmt);
|
1715 |
|
|
|
1716 |
|
|
pattern_stmt
|
1717 |
|
|
= gimple_build_assign_with_ops (RSHIFT_EXPR,
|
1718 |
|
|
vect_recog_temp_ssa_var (itype, NULL),
|
1719 |
|
|
var,
|
1720 |
|
|
build_int_cst (itype,
|
1721 |
|
|
tree_log2 (oprnd1)));
|
1722 |
|
|
}
|
1723 |
|
|
else
|
1724 |
|
|
{
|
1725 |
|
|
tree signmask;
|
1726 |
|
|
STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo) = NULL;
|
1727 |
|
|
if (compare_tree_int (oprnd1, 2) == 0)
|
1728 |
|
|
{
|
1729 |
|
|
signmask = vect_recog_temp_ssa_var (itype, NULL);
|
1730 |
|
|
def_stmt
|
1731 |
|
|
= gimple_build_assign_with_ops3 (COND_EXPR, signmask, cond,
|
1732 |
|
|
build_int_cst (itype, 1),
|
1733 |
|
|
build_int_cst (itype, 0));
|
1734 |
|
|
append_pattern_def_seq (stmt_vinfo, def_stmt);
|
1735 |
|
|
}
|
1736 |
|
|
else
|
1737 |
|
|
{
|
1738 |
|
|
tree utype
|
1739 |
|
|
= build_nonstandard_integer_type (TYPE_PRECISION (itype), 1);
|
1740 |
|
|
tree vecutype = get_vectype_for_scalar_type (utype);
|
1741 |
|
|
tree shift
|
1742 |
|
|
= build_int_cst (utype, GET_MODE_BITSIZE (TYPE_MODE (itype))
|
1743 |
|
|
- tree_log2 (oprnd1));
|
1744 |
|
|
tree var = vect_recog_temp_ssa_var (utype, NULL);
|
1745 |
|
|
stmt_vec_info def_stmt_vinfo;
|
1746 |
|
|
|
1747 |
|
|
def_stmt
|
1748 |
|
|
= gimple_build_assign_with_ops3 (COND_EXPR, var, cond,
|
1749 |
|
|
build_int_cst (utype, -1),
|
1750 |
|
|
build_int_cst (utype, 0));
|
1751 |
|
|
def_stmt_vinfo = new_stmt_vec_info (def_stmt, loop_vinfo, NULL);
|
1752 |
|
|
set_vinfo_for_stmt (def_stmt, def_stmt_vinfo);
|
1753 |
|
|
STMT_VINFO_VECTYPE (def_stmt_vinfo) = vecutype;
|
1754 |
|
|
append_pattern_def_seq (stmt_vinfo, def_stmt);
|
1755 |
|
|
var = vect_recog_temp_ssa_var (utype, NULL);
|
1756 |
|
|
def_stmt
|
1757 |
|
|
= gimple_build_assign_with_ops (RSHIFT_EXPR, var,
|
1758 |
|
|
gimple_assign_lhs (def_stmt),
|
1759 |
|
|
shift);
|
1760 |
|
|
def_stmt_vinfo = new_stmt_vec_info (def_stmt, loop_vinfo, NULL);
|
1761 |
|
|
set_vinfo_for_stmt (def_stmt, def_stmt_vinfo);
|
1762 |
|
|
STMT_VINFO_VECTYPE (def_stmt_vinfo) = vecutype;
|
1763 |
|
|
append_pattern_def_seq (stmt_vinfo, def_stmt);
|
1764 |
|
|
signmask = vect_recog_temp_ssa_var (itype, NULL);
|
1765 |
|
|
def_stmt
|
1766 |
|
|
= gimple_build_assign_with_ops (NOP_EXPR, signmask, var,
|
1767 |
|
|
NULL_TREE);
|
1768 |
|
|
append_pattern_def_seq (stmt_vinfo, def_stmt);
|
1769 |
|
|
}
|
1770 |
|
|
def_stmt
|
1771 |
|
|
= gimple_build_assign_with_ops (PLUS_EXPR,
|
1772 |
|
|
vect_recog_temp_ssa_var (itype, NULL),
|
1773 |
|
|
oprnd0, signmask);
|
1774 |
|
|
append_pattern_def_seq (stmt_vinfo, def_stmt);
|
1775 |
|
|
def_stmt
|
1776 |
|
|
= gimple_build_assign_with_ops (BIT_AND_EXPR,
|
1777 |
|
|
vect_recog_temp_ssa_var (itype, NULL),
|
1778 |
|
|
gimple_assign_lhs (def_stmt),
|
1779 |
|
|
fold_build2 (MINUS_EXPR, itype,
|
1780 |
|
|
oprnd1,
|
1781 |
|
|
build_int_cst (itype,
|
1782 |
|
|
1)));
|
1783 |
|
|
append_pattern_def_seq (stmt_vinfo, def_stmt);
|
1784 |
|
|
|
1785 |
|
|
pattern_stmt
|
1786 |
|
|
= gimple_build_assign_with_ops (MINUS_EXPR,
|
1787 |
|
|
vect_recog_temp_ssa_var (itype, NULL),
|
1788 |
|
|
gimple_assign_lhs (def_stmt),
|
1789 |
|
|
signmask);
|
1790 |
|
|
}
|
1791 |
|
|
|
1792 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
1793 |
|
|
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
|
1794 |
|
|
|
1795 |
|
|
VEC_safe_push (gimple, heap, *stmts, last_stmt);
|
1796 |
|
|
|
1797 |
|
|
*type_in = vectype;
|
1798 |
|
|
*type_out = vectype;
|
1799 |
|
|
return pattern_stmt;
|
1800 |
|
|
}
|
1801 |
|
|
|
1802 |
|
|
/* Function vect_recog_mixed_size_cond_pattern
|
1803 |
|
|
|
1804 |
|
|
Try to find the following pattern:
|
1805 |
|
|
|
1806 |
|
|
type x_t, y_t;
|
1807 |
|
|
TYPE a_T, b_T, c_T;
|
1808 |
|
|
loop:
|
1809 |
|
|
S1 a_T = x_t CMP y_t ? b_T : c_T;
|
1810 |
|
|
|
1811 |
|
|
where type 'TYPE' is an integral type which has different size
|
1812 |
|
|
from 'type'. b_T and c_T are constants and if 'TYPE' is wider
|
1813 |
|
|
than 'type', the constants need to fit into an integer type
|
1814 |
|
|
with the same width as 'type'.
|
1815 |
|
|
|
1816 |
|
|
Input:
|
1817 |
|
|
|
1818 |
|
|
* LAST_STMT: A stmt from which the pattern search begins.
|
1819 |
|
|
|
1820 |
|
|
Output:
|
1821 |
|
|
|
1822 |
|
|
* TYPE_IN: The type of the input arguments to the pattern.
|
1823 |
|
|
|
1824 |
|
|
* TYPE_OUT: The type of the output of this pattern.
|
1825 |
|
|
|
1826 |
|
|
* Return value: A new stmt that will be used to replace the pattern.
|
1827 |
|
|
Additionally a def_stmt is added.
|
1828 |
|
|
|
1829 |
|
|
a_it = x_t CMP y_t ? b_it : c_it;
|
1830 |
|
|
a_T = (TYPE) a_it; */
|
1831 |
|
|
|
1832 |
|
|
static gimple
|
1833 |
|
|
vect_recog_mixed_size_cond_pattern (VEC (gimple, heap) **stmts, tree *type_in,
|
1834 |
|
|
tree *type_out)
|
1835 |
|
|
{
|
1836 |
|
|
gimple last_stmt = VEC_index (gimple, *stmts, 0);
|
1837 |
|
|
tree cond_expr, then_clause, else_clause;
|
1838 |
|
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt), def_stmt_info;
|
1839 |
|
|
tree type, vectype, comp_vectype, itype, vecitype;
|
1840 |
|
|
enum machine_mode cmpmode;
|
1841 |
|
|
gimple pattern_stmt, def_stmt;
|
1842 |
|
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
|
1843 |
|
|
|
1844 |
|
|
if (!is_gimple_assign (last_stmt)
|
1845 |
|
|
|| gimple_assign_rhs_code (last_stmt) != COND_EXPR
|
1846 |
|
|
|| STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_internal_def)
|
1847 |
|
|
return NULL;
|
1848 |
|
|
|
1849 |
|
|
cond_expr = gimple_assign_rhs1 (last_stmt);
|
1850 |
|
|
then_clause = gimple_assign_rhs2 (last_stmt);
|
1851 |
|
|
else_clause = gimple_assign_rhs3 (last_stmt);
|
1852 |
|
|
|
1853 |
|
|
if (TREE_CODE (then_clause) != INTEGER_CST
|
1854 |
|
|
|| TREE_CODE (else_clause) != INTEGER_CST)
|
1855 |
|
|
return NULL;
|
1856 |
|
|
|
1857 |
|
|
if (!COMPARISON_CLASS_P (cond_expr))
|
1858 |
|
|
return NULL;
|
1859 |
|
|
|
1860 |
|
|
comp_vectype
|
1861 |
|
|
= get_vectype_for_scalar_type (TREE_TYPE (TREE_OPERAND (cond_expr, 0)));
|
1862 |
|
|
if (comp_vectype == NULL_TREE)
|
1863 |
|
|
return NULL;
|
1864 |
|
|
|
1865 |
|
|
type = gimple_expr_type (last_stmt);
|
1866 |
|
|
cmpmode = GET_MODE_INNER (TYPE_MODE (comp_vectype));
|
1867 |
|
|
|
1868 |
|
|
if (GET_MODE_BITSIZE (TYPE_MODE (type)) == GET_MODE_BITSIZE (cmpmode))
|
1869 |
|
|
return NULL;
|
1870 |
|
|
|
1871 |
|
|
vectype = get_vectype_for_scalar_type (type);
|
1872 |
|
|
if (vectype == NULL_TREE)
|
1873 |
|
|
return NULL;
|
1874 |
|
|
|
1875 |
|
|
if (expand_vec_cond_expr_p (vectype, comp_vectype))
|
1876 |
|
|
return NULL;
|
1877 |
|
|
|
1878 |
|
|
itype = build_nonstandard_integer_type (GET_MODE_BITSIZE (cmpmode),
|
1879 |
|
|
TYPE_UNSIGNED (type));
|
1880 |
|
|
if (itype == NULL_TREE
|
1881 |
|
|
|| GET_MODE_BITSIZE (TYPE_MODE (itype)) != GET_MODE_BITSIZE (cmpmode))
|
1882 |
|
|
return NULL;
|
1883 |
|
|
|
1884 |
|
|
vecitype = get_vectype_for_scalar_type (itype);
|
1885 |
|
|
if (vecitype == NULL_TREE)
|
1886 |
|
|
return NULL;
|
1887 |
|
|
|
1888 |
|
|
if (!expand_vec_cond_expr_p (vecitype, comp_vectype))
|
1889 |
|
|
return NULL;
|
1890 |
|
|
|
1891 |
|
|
if (GET_MODE_BITSIZE (TYPE_MODE (type)) > GET_MODE_BITSIZE (cmpmode))
|
1892 |
|
|
{
|
1893 |
|
|
if (!int_fits_type_p (then_clause, itype)
|
1894 |
|
|
|| !int_fits_type_p (else_clause, itype))
|
1895 |
|
|
return NULL;
|
1896 |
|
|
}
|
1897 |
|
|
|
1898 |
|
|
def_stmt
|
1899 |
|
|
= gimple_build_assign_with_ops3 (COND_EXPR,
|
1900 |
|
|
vect_recog_temp_ssa_var (itype, NULL),
|
1901 |
|
|
unshare_expr (cond_expr),
|
1902 |
|
|
fold_convert (itype, then_clause),
|
1903 |
|
|
fold_convert (itype, else_clause));
|
1904 |
|
|
pattern_stmt
|
1905 |
|
|
= gimple_build_assign_with_ops (NOP_EXPR,
|
1906 |
|
|
vect_recog_temp_ssa_var (type, NULL),
|
1907 |
|
|
gimple_assign_lhs (def_stmt), NULL_TREE);
|
1908 |
|
|
|
1909 |
|
|
new_pattern_def_seq (stmt_vinfo, def_stmt);
|
1910 |
|
|
def_stmt_info = new_stmt_vec_info (def_stmt, loop_vinfo, NULL);
|
1911 |
|
|
set_vinfo_for_stmt (def_stmt, def_stmt_info);
|
1912 |
|
|
STMT_VINFO_VECTYPE (def_stmt_info) = vecitype;
|
1913 |
|
|
*type_in = vecitype;
|
1914 |
|
|
*type_out = vectype;
|
1915 |
|
|
|
1916 |
|
|
return pattern_stmt;
|
1917 |
|
|
}
|
1918 |
|
|
|
1919 |
|
|
|
1920 |
|
|
/* Helper function of vect_recog_bool_pattern. Called recursively, return
|
1921 |
|
|
true if bool VAR can be optimized that way. */
|
1922 |
|
|
|
1923 |
|
|
static bool
|
1924 |
|
|
check_bool_pattern (tree var, loop_vec_info loop_vinfo)
|
1925 |
|
|
{
|
1926 |
|
|
gimple def_stmt;
|
1927 |
|
|
enum vect_def_type dt;
|
1928 |
|
|
tree def, rhs1;
|
1929 |
|
|
enum tree_code rhs_code;
|
1930 |
|
|
|
1931 |
|
|
if (!vect_is_simple_use (var, NULL, loop_vinfo, NULL, &def_stmt, &def, &dt))
|
1932 |
|
|
return false;
|
1933 |
|
|
|
1934 |
|
|
if (dt != vect_internal_def)
|
1935 |
|
|
return false;
|
1936 |
|
|
|
1937 |
|
|
if (!is_gimple_assign (def_stmt))
|
1938 |
|
|
return false;
|
1939 |
|
|
|
1940 |
|
|
if (!has_single_use (def))
|
1941 |
|
|
return false;
|
1942 |
|
|
|
1943 |
|
|
rhs1 = gimple_assign_rhs1 (def_stmt);
|
1944 |
|
|
rhs_code = gimple_assign_rhs_code (def_stmt);
|
1945 |
|
|
switch (rhs_code)
|
1946 |
|
|
{
|
1947 |
|
|
case SSA_NAME:
|
1948 |
|
|
return check_bool_pattern (rhs1, loop_vinfo);
|
1949 |
|
|
|
1950 |
|
|
CASE_CONVERT:
|
1951 |
|
|
if ((TYPE_PRECISION (TREE_TYPE (rhs1)) != 1
|
1952 |
|
|
|| !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
|
1953 |
|
|
&& TREE_CODE (TREE_TYPE (rhs1)) != BOOLEAN_TYPE)
|
1954 |
|
|
return false;
|
1955 |
|
|
return check_bool_pattern (rhs1, loop_vinfo);
|
1956 |
|
|
|
1957 |
|
|
case BIT_NOT_EXPR:
|
1958 |
|
|
return check_bool_pattern (rhs1, loop_vinfo);
|
1959 |
|
|
|
1960 |
|
|
case BIT_AND_EXPR:
|
1961 |
|
|
case BIT_IOR_EXPR:
|
1962 |
|
|
case BIT_XOR_EXPR:
|
1963 |
|
|
if (!check_bool_pattern (rhs1, loop_vinfo))
|
1964 |
|
|
return false;
|
1965 |
|
|
return check_bool_pattern (gimple_assign_rhs2 (def_stmt), loop_vinfo);
|
1966 |
|
|
|
1967 |
|
|
default:
|
1968 |
|
|
if (TREE_CODE_CLASS (rhs_code) == tcc_comparison)
|
1969 |
|
|
{
|
1970 |
|
|
tree vecitype, comp_vectype;
|
1971 |
|
|
|
1972 |
|
|
/* If the comparison can throw, then is_gimple_condexpr will be
|
1973 |
|
|
false and we can't make a COND_EXPR/VEC_COND_EXPR out of it. */
|
1974 |
|
|
if (stmt_could_throw_p (def_stmt))
|
1975 |
|
|
return false;
|
1976 |
|
|
|
1977 |
|
|
comp_vectype = get_vectype_for_scalar_type (TREE_TYPE (rhs1));
|
1978 |
|
|
if (comp_vectype == NULL_TREE)
|
1979 |
|
|
return false;
|
1980 |
|
|
|
1981 |
|
|
if (TREE_CODE (TREE_TYPE (rhs1)) != INTEGER_TYPE)
|
1982 |
|
|
{
|
1983 |
|
|
enum machine_mode mode = TYPE_MODE (TREE_TYPE (rhs1));
|
1984 |
|
|
tree itype
|
1985 |
|
|
= build_nonstandard_integer_type (GET_MODE_BITSIZE (mode), 1);
|
1986 |
|
|
vecitype = get_vectype_for_scalar_type (itype);
|
1987 |
|
|
if (vecitype == NULL_TREE)
|
1988 |
|
|
return false;
|
1989 |
|
|
}
|
1990 |
|
|
else
|
1991 |
|
|
vecitype = comp_vectype;
|
1992 |
|
|
return expand_vec_cond_expr_p (vecitype, comp_vectype);
|
1993 |
|
|
}
|
1994 |
|
|
return false;
|
1995 |
|
|
}
|
1996 |
|
|
}
|
1997 |
|
|
|
1998 |
|
|
|
1999 |
|
|
/* Helper function of adjust_bool_pattern. Add a cast to TYPE to a previous
|
2000 |
|
|
stmt (SSA_NAME_DEF_STMT of VAR) by moving the COND_EXPR from RELATED_STMT
|
2001 |
|
|
to PATTERN_DEF_SEQ and adding a cast as RELATED_STMT. */
|
2002 |
|
|
|
2003 |
|
|
static tree
|
2004 |
|
|
adjust_bool_pattern_cast (tree type, tree var)
|
2005 |
|
|
{
|
2006 |
|
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (SSA_NAME_DEF_STMT (var));
|
2007 |
|
|
gimple cast_stmt, pattern_stmt;
|
2008 |
|
|
|
2009 |
|
|
gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo));
|
2010 |
|
|
pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo);
|
2011 |
|
|
new_pattern_def_seq (stmt_vinfo, pattern_stmt);
|
2012 |
|
|
cast_stmt
|
2013 |
|
|
= gimple_build_assign_with_ops (NOP_EXPR,
|
2014 |
|
|
vect_recog_temp_ssa_var (type, NULL),
|
2015 |
|
|
gimple_assign_lhs (pattern_stmt),
|
2016 |
|
|
NULL_TREE);
|
2017 |
|
|
STMT_VINFO_RELATED_STMT (stmt_vinfo) = cast_stmt;
|
2018 |
|
|
return gimple_assign_lhs (cast_stmt);
|
2019 |
|
|
}
|
2020 |
|
|
|
2021 |
|
|
|
2022 |
|
|
/* Helper function of vect_recog_bool_pattern. Do the actual transformations,
|
2023 |
|
|
recursively. VAR is an SSA_NAME that should be transformed from bool
|
2024 |
|
|
to a wider integer type, OUT_TYPE is the desired final integer type of
|
2025 |
|
|
the whole pattern, TRUEVAL should be NULL unless optimizing
|
2026 |
|
|
BIT_AND_EXPR into a COND_EXPR with one integer from one of the operands
|
2027 |
|
|
in the then_clause, STMTS is where statements with added pattern stmts
|
2028 |
|
|
should be pushed to. */
|
2029 |
|
|
|
2030 |
|
|
static tree
|
2031 |
|
|
adjust_bool_pattern (tree var, tree out_type, tree trueval,
|
2032 |
|
|
VEC (gimple, heap) **stmts)
|
2033 |
|
|
{
|
2034 |
|
|
gimple stmt = SSA_NAME_DEF_STMT (var);
|
2035 |
|
|
enum tree_code rhs_code, def_rhs_code;
|
2036 |
|
|
tree itype, cond_expr, rhs1, rhs2, irhs1, irhs2;
|
2037 |
|
|
location_t loc;
|
2038 |
|
|
gimple pattern_stmt, def_stmt;
|
2039 |
|
|
|
2040 |
|
|
rhs1 = gimple_assign_rhs1 (stmt);
|
2041 |
|
|
rhs2 = gimple_assign_rhs2 (stmt);
|
2042 |
|
|
rhs_code = gimple_assign_rhs_code (stmt);
|
2043 |
|
|
loc = gimple_location (stmt);
|
2044 |
|
|
switch (rhs_code)
|
2045 |
|
|
{
|
2046 |
|
|
case SSA_NAME:
|
2047 |
|
|
CASE_CONVERT:
|
2048 |
|
|
irhs1 = adjust_bool_pattern (rhs1, out_type, NULL_TREE, stmts);
|
2049 |
|
|
itype = TREE_TYPE (irhs1);
|
2050 |
|
|
pattern_stmt
|
2051 |
|
|
= gimple_build_assign_with_ops (SSA_NAME,
|
2052 |
|
|
vect_recog_temp_ssa_var (itype, NULL),
|
2053 |
|
|
irhs1, NULL_TREE);
|
2054 |
|
|
break;
|
2055 |
|
|
|
2056 |
|
|
case BIT_NOT_EXPR:
|
2057 |
|
|
irhs1 = adjust_bool_pattern (rhs1, out_type, NULL_TREE, stmts);
|
2058 |
|
|
itype = TREE_TYPE (irhs1);
|
2059 |
|
|
pattern_stmt
|
2060 |
|
|
= gimple_build_assign_with_ops (BIT_XOR_EXPR,
|
2061 |
|
|
vect_recog_temp_ssa_var (itype, NULL),
|
2062 |
|
|
irhs1, build_int_cst (itype, 1));
|
2063 |
|
|
break;
|
2064 |
|
|
|
2065 |
|
|
case BIT_AND_EXPR:
|
2066 |
|
|
/* Try to optimize x = y & (a < b ? 1 : 0); into
|
2067 |
|
|
x = (a < b ? y : 0);
|
2068 |
|
|
|
2069 |
|
|
E.g. for:
|
2070 |
|
|
bool a_b, b_b, c_b;
|
2071 |
|
|
TYPE d_T;
|
2072 |
|
|
|
2073 |
|
|
S1 a_b = x1 CMP1 y1;
|
2074 |
|
|
S2 b_b = x2 CMP2 y2;
|
2075 |
|
|
S3 c_b = a_b & b_b;
|
2076 |
|
|
S4 d_T = (TYPE) c_b;
|
2077 |
|
|
|
2078 |
|
|
we would normally emit:
|
2079 |
|
|
|
2080 |
|
|
S1' a_T = x1 CMP1 y1 ? 1 : 0;
|
2081 |
|
|
S2' b_T = x2 CMP2 y2 ? 1 : 0;
|
2082 |
|
|
S3' c_T = a_T & b_T;
|
2083 |
|
|
S4' d_T = c_T;
|
2084 |
|
|
|
2085 |
|
|
but we can save one stmt by using the
|
2086 |
|
|
result of one of the COND_EXPRs in the other COND_EXPR and leave
|
2087 |
|
|
BIT_AND_EXPR stmt out:
|
2088 |
|
|
|
2089 |
|
|
S1' a_T = x1 CMP1 y1 ? 1 : 0;
|
2090 |
|
|
S3' c_T = x2 CMP2 y2 ? a_T : 0;
|
2091 |
|
|
S4' f_T = c_T;
|
2092 |
|
|
|
2093 |
|
|
At least when VEC_COND_EXPR is implemented using masks
|
2094 |
|
|
cond ? 1 : 0 is as expensive as cond ? var : 0, in both cases it
|
2095 |
|
|
computes the comparison masks and ands it, in one case with
|
2096 |
|
|
all ones vector, in the other case with a vector register.
|
2097 |
|
|
Don't do this for BIT_IOR_EXPR, because cond ? 1 : var; is
|
2098 |
|
|
often more expensive. */
|
2099 |
|
|
def_stmt = SSA_NAME_DEF_STMT (rhs2);
|
2100 |
|
|
def_rhs_code = gimple_assign_rhs_code (def_stmt);
|
2101 |
|
|
if (TREE_CODE_CLASS (def_rhs_code) == tcc_comparison)
|
2102 |
|
|
{
|
2103 |
|
|
tree def_rhs1 = gimple_assign_rhs1 (def_stmt);
|
2104 |
|
|
irhs1 = adjust_bool_pattern (rhs1, out_type, NULL_TREE, stmts);
|
2105 |
|
|
if (TYPE_PRECISION (TREE_TYPE (irhs1))
|
2106 |
|
|
== GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (def_rhs1))))
|
2107 |
|
|
{
|
2108 |
|
|
gimple tstmt;
|
2109 |
|
|
stmt_vec_info stmt_def_vinfo = vinfo_for_stmt (def_stmt);
|
2110 |
|
|
irhs2 = adjust_bool_pattern (rhs2, out_type, irhs1, stmts);
|
2111 |
|
|
tstmt = VEC_pop (gimple, *stmts);
|
2112 |
|
|
gcc_assert (tstmt == def_stmt);
|
2113 |
|
|
VEC_quick_push (gimple, *stmts, stmt);
|
2114 |
|
|
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt))
|
2115 |
|
|
= STMT_VINFO_RELATED_STMT (stmt_def_vinfo);
|
2116 |
|
|
gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_def_vinfo));
|
2117 |
|
|
STMT_VINFO_RELATED_STMT (stmt_def_vinfo) = NULL;
|
2118 |
|
|
return irhs2;
|
2119 |
|
|
}
|
2120 |
|
|
else
|
2121 |
|
|
irhs2 = adjust_bool_pattern (rhs2, out_type, NULL_TREE, stmts);
|
2122 |
|
|
goto and_ior_xor;
|
2123 |
|
|
}
|
2124 |
|
|
def_stmt = SSA_NAME_DEF_STMT (rhs1);
|
2125 |
|
|
def_rhs_code = gimple_assign_rhs_code (def_stmt);
|
2126 |
|
|
if (TREE_CODE_CLASS (def_rhs_code) == tcc_comparison)
|
2127 |
|
|
{
|
2128 |
|
|
tree def_rhs1 = gimple_assign_rhs1 (def_stmt);
|
2129 |
|
|
irhs2 = adjust_bool_pattern (rhs2, out_type, NULL_TREE, stmts);
|
2130 |
|
|
if (TYPE_PRECISION (TREE_TYPE (irhs2))
|
2131 |
|
|
== GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (def_rhs1))))
|
2132 |
|
|
{
|
2133 |
|
|
gimple tstmt;
|
2134 |
|
|
stmt_vec_info stmt_def_vinfo = vinfo_for_stmt (def_stmt);
|
2135 |
|
|
irhs1 = adjust_bool_pattern (rhs1, out_type, irhs2, stmts);
|
2136 |
|
|
tstmt = VEC_pop (gimple, *stmts);
|
2137 |
|
|
gcc_assert (tstmt == def_stmt);
|
2138 |
|
|
VEC_quick_push (gimple, *stmts, stmt);
|
2139 |
|
|
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt))
|
2140 |
|
|
= STMT_VINFO_RELATED_STMT (stmt_def_vinfo);
|
2141 |
|
|
gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_def_vinfo));
|
2142 |
|
|
STMT_VINFO_RELATED_STMT (stmt_def_vinfo) = NULL;
|
2143 |
|
|
return irhs1;
|
2144 |
|
|
}
|
2145 |
|
|
else
|
2146 |
|
|
irhs1 = adjust_bool_pattern (rhs1, out_type, NULL_TREE, stmts);
|
2147 |
|
|
goto and_ior_xor;
|
2148 |
|
|
}
|
2149 |
|
|
/* FALLTHRU */
|
2150 |
|
|
case BIT_IOR_EXPR:
|
2151 |
|
|
case BIT_XOR_EXPR:
|
2152 |
|
|
irhs1 = adjust_bool_pattern (rhs1, out_type, NULL_TREE, stmts);
|
2153 |
|
|
irhs2 = adjust_bool_pattern (rhs2, out_type, NULL_TREE, stmts);
|
2154 |
|
|
and_ior_xor:
|
2155 |
|
|
if (TYPE_PRECISION (TREE_TYPE (irhs1))
|
2156 |
|
|
!= TYPE_PRECISION (TREE_TYPE (irhs2)))
|
2157 |
|
|
{
|
2158 |
|
|
int prec1 = TYPE_PRECISION (TREE_TYPE (irhs1));
|
2159 |
|
|
int prec2 = TYPE_PRECISION (TREE_TYPE (irhs2));
|
2160 |
|
|
int out_prec = TYPE_PRECISION (out_type);
|
2161 |
|
|
if (absu_hwi (out_prec - prec1) < absu_hwi (out_prec - prec2))
|
2162 |
|
|
irhs2 = adjust_bool_pattern_cast (TREE_TYPE (irhs1), rhs2);
|
2163 |
|
|
else if (absu_hwi (out_prec - prec1) > absu_hwi (out_prec - prec2))
|
2164 |
|
|
irhs1 = adjust_bool_pattern_cast (TREE_TYPE (irhs2), rhs1);
|
2165 |
|
|
else
|
2166 |
|
|
{
|
2167 |
|
|
irhs1 = adjust_bool_pattern_cast (out_type, rhs1);
|
2168 |
|
|
irhs2 = adjust_bool_pattern_cast (out_type, rhs2);
|
2169 |
|
|
}
|
2170 |
|
|
}
|
2171 |
|
|
itype = TREE_TYPE (irhs1);
|
2172 |
|
|
pattern_stmt
|
2173 |
|
|
= gimple_build_assign_with_ops (rhs_code,
|
2174 |
|
|
vect_recog_temp_ssa_var (itype, NULL),
|
2175 |
|
|
irhs1, irhs2);
|
2176 |
|
|
break;
|
2177 |
|
|
|
2178 |
|
|
default:
|
2179 |
|
|
gcc_assert (TREE_CODE_CLASS (rhs_code) == tcc_comparison);
|
2180 |
|
|
if (TREE_CODE (TREE_TYPE (rhs1)) != INTEGER_TYPE
|
2181 |
|
|
|| !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
|
2182 |
|
|
{
|
2183 |
|
|
enum machine_mode mode = TYPE_MODE (TREE_TYPE (rhs1));
|
2184 |
|
|
itype
|
2185 |
|
|
= build_nonstandard_integer_type (GET_MODE_BITSIZE (mode), 1);
|
2186 |
|
|
}
|
2187 |
|
|
else
|
2188 |
|
|
itype = TREE_TYPE (rhs1);
|
2189 |
|
|
cond_expr = build2_loc (loc, rhs_code, itype, rhs1, rhs2);
|
2190 |
|
|
if (trueval == NULL_TREE)
|
2191 |
|
|
trueval = build_int_cst (itype, 1);
|
2192 |
|
|
else
|
2193 |
|
|
gcc_checking_assert (useless_type_conversion_p (itype,
|
2194 |
|
|
TREE_TYPE (trueval)));
|
2195 |
|
|
pattern_stmt
|
2196 |
|
|
= gimple_build_assign_with_ops3 (COND_EXPR,
|
2197 |
|
|
vect_recog_temp_ssa_var (itype, NULL),
|
2198 |
|
|
cond_expr, trueval,
|
2199 |
|
|
build_int_cst (itype, 0));
|
2200 |
|
|
break;
|
2201 |
|
|
}
|
2202 |
|
|
|
2203 |
|
|
VEC_safe_push (gimple, heap, *stmts, stmt);
|
2204 |
|
|
gimple_set_location (pattern_stmt, loc);
|
2205 |
|
|
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt)) = pattern_stmt;
|
2206 |
|
|
return gimple_assign_lhs (pattern_stmt);
|
2207 |
|
|
}
|
2208 |
|
|
|
2209 |
|
|
|
2210 |
|
|
/* Function vect_recog_bool_pattern
|
2211 |
|
|
|
2212 |
|
|
Try to find pattern like following:
|
2213 |
|
|
|
2214 |
|
|
bool a_b, b_b, c_b, d_b, e_b;
|
2215 |
|
|
TYPE f_T;
|
2216 |
|
|
loop:
|
2217 |
|
|
S1 a_b = x1 CMP1 y1;
|
2218 |
|
|
S2 b_b = x2 CMP2 y2;
|
2219 |
|
|
S3 c_b = a_b & b_b;
|
2220 |
|
|
S4 d_b = x3 CMP3 y3;
|
2221 |
|
|
S5 e_b = c_b | d_b;
|
2222 |
|
|
S6 f_T = (TYPE) e_b;
|
2223 |
|
|
|
2224 |
|
|
where type 'TYPE' is an integral type.
|
2225 |
|
|
|
2226 |
|
|
Input:
|
2227 |
|
|
|
2228 |
|
|
* LAST_STMT: A stmt at the end from which the pattern
|
2229 |
|
|
search begins, i.e. cast of a bool to
|
2230 |
|
|
an integer type.
|
2231 |
|
|
|
2232 |
|
|
Output:
|
2233 |
|
|
|
2234 |
|
|
* TYPE_IN: The type of the input arguments to the pattern.
|
2235 |
|
|
|
2236 |
|
|
* TYPE_OUT: The type of the output of this pattern.
|
2237 |
|
|
|
2238 |
|
|
* Return value: A new stmt that will be used to replace the pattern.
|
2239 |
|
|
|
2240 |
|
|
Assuming size of TYPE is the same as size of all comparisons
|
2241 |
|
|
(otherwise some casts would be added where needed), the above
|
2242 |
|
|
sequence we create related pattern stmts:
|
2243 |
|
|
S1' a_T = x1 CMP1 y1 ? 1 : 0;
|
2244 |
|
|
S3' c_T = x2 CMP2 y2 ? a_T : 0;
|
2245 |
|
|
S4' d_T = x3 CMP3 y3 ? 1 : 0;
|
2246 |
|
|
S5' e_T = c_T | d_T;
|
2247 |
|
|
S6' f_T = e_T;
|
2248 |
|
|
|
2249 |
|
|
Instead of the above S3' we could emit:
|
2250 |
|
|
S2' b_T = x2 CMP2 y2 ? 1 : 0;
|
2251 |
|
|
S3' c_T = a_T | b_T;
|
2252 |
|
|
but the above is more efficient. */
|
2253 |
|
|
|
2254 |
|
|
static gimple
|
2255 |
|
|
vect_recog_bool_pattern (VEC (gimple, heap) **stmts, tree *type_in,
|
2256 |
|
|
tree *type_out)
|
2257 |
|
|
{
|
2258 |
|
|
gimple last_stmt = VEC_pop (gimple, *stmts);
|
2259 |
|
|
enum tree_code rhs_code;
|
2260 |
|
|
tree var, lhs, rhs, vectype;
|
2261 |
|
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
|
2262 |
|
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
|
2263 |
|
|
gimple pattern_stmt;
|
2264 |
|
|
|
2265 |
|
|
if (!is_gimple_assign (last_stmt))
|
2266 |
|
|
return NULL;
|
2267 |
|
|
|
2268 |
|
|
var = gimple_assign_rhs1 (last_stmt);
|
2269 |
|
|
lhs = gimple_assign_lhs (last_stmt);
|
2270 |
|
|
|
2271 |
|
|
if ((TYPE_PRECISION (TREE_TYPE (var)) != 1
|
2272 |
|
|
|| !TYPE_UNSIGNED (TREE_TYPE (var)))
|
2273 |
|
|
&& TREE_CODE (TREE_TYPE (var)) != BOOLEAN_TYPE)
|
2274 |
|
|
return NULL;
|
2275 |
|
|
|
2276 |
|
|
rhs_code = gimple_assign_rhs_code (last_stmt);
|
2277 |
|
|
if (CONVERT_EXPR_CODE_P (rhs_code))
|
2278 |
|
|
{
|
2279 |
|
|
if (TREE_CODE (TREE_TYPE (lhs)) != INTEGER_TYPE
|
2280 |
|
|
|| TYPE_PRECISION (TREE_TYPE (lhs)) == 1)
|
2281 |
|
|
return NULL;
|
2282 |
|
|
vectype = get_vectype_for_scalar_type (TREE_TYPE (lhs));
|
2283 |
|
|
if (vectype == NULL_TREE)
|
2284 |
|
|
return NULL;
|
2285 |
|
|
|
2286 |
|
|
if (!check_bool_pattern (var, loop_vinfo))
|
2287 |
|
|
return NULL;
|
2288 |
|
|
|
2289 |
|
|
rhs = adjust_bool_pattern (var, TREE_TYPE (lhs), NULL_TREE, stmts);
|
2290 |
|
|
lhs = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
|
2291 |
|
|
if (useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs)))
|
2292 |
|
|
pattern_stmt
|
2293 |
|
|
= gimple_build_assign_with_ops (SSA_NAME, lhs, rhs, NULL_TREE);
|
2294 |
|
|
else
|
2295 |
|
|
pattern_stmt
|
2296 |
|
|
= gimple_build_assign_with_ops (NOP_EXPR, lhs, rhs, NULL_TREE);
|
2297 |
|
|
*type_out = vectype;
|
2298 |
|
|
*type_in = vectype;
|
2299 |
|
|
VEC_safe_push (gimple, heap, *stmts, last_stmt);
|
2300 |
|
|
return pattern_stmt;
|
2301 |
|
|
}
|
2302 |
|
|
else if (rhs_code == SSA_NAME
|
2303 |
|
|
&& STMT_VINFO_DATA_REF (stmt_vinfo))
|
2304 |
|
|
{
|
2305 |
|
|
stmt_vec_info pattern_stmt_info;
|
2306 |
|
|
vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
|
2307 |
|
|
gcc_assert (vectype != NULL_TREE);
|
2308 |
|
|
if (!VECTOR_MODE_P (TYPE_MODE (vectype)))
|
2309 |
|
|
return NULL;
|
2310 |
|
|
if (!check_bool_pattern (var, loop_vinfo))
|
2311 |
|
|
return NULL;
|
2312 |
|
|
|
2313 |
|
|
rhs = adjust_bool_pattern (var, TREE_TYPE (vectype), NULL_TREE, stmts);
|
2314 |
|
|
lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vectype), lhs);
|
2315 |
|
|
if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs)))
|
2316 |
|
|
{
|
2317 |
|
|
tree rhs2 = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
|
2318 |
|
|
gimple cast_stmt
|
2319 |
|
|
= gimple_build_assign_with_ops (NOP_EXPR, rhs2, rhs, NULL_TREE);
|
2320 |
|
|
new_pattern_def_seq (stmt_vinfo, cast_stmt);
|
2321 |
|
|
rhs = rhs2;
|
2322 |
|
|
}
|
2323 |
|
|
pattern_stmt
|
2324 |
|
|
= gimple_build_assign_with_ops (SSA_NAME, lhs, rhs, NULL_TREE);
|
2325 |
|
|
pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo, NULL);
|
2326 |
|
|
set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info);
|
2327 |
|
|
STMT_VINFO_DATA_REF (pattern_stmt_info)
|
2328 |
|
|
= STMT_VINFO_DATA_REF (stmt_vinfo);
|
2329 |
|
|
STMT_VINFO_DR_BASE_ADDRESS (pattern_stmt_info)
|
2330 |
|
|
= STMT_VINFO_DR_BASE_ADDRESS (stmt_vinfo);
|
2331 |
|
|
STMT_VINFO_DR_INIT (pattern_stmt_info) = STMT_VINFO_DR_INIT (stmt_vinfo);
|
2332 |
|
|
STMT_VINFO_DR_OFFSET (pattern_stmt_info)
|
2333 |
|
|
= STMT_VINFO_DR_OFFSET (stmt_vinfo);
|
2334 |
|
|
STMT_VINFO_DR_STEP (pattern_stmt_info) = STMT_VINFO_DR_STEP (stmt_vinfo);
|
2335 |
|
|
STMT_VINFO_DR_ALIGNED_TO (pattern_stmt_info)
|
2336 |
|
|
= STMT_VINFO_DR_ALIGNED_TO (stmt_vinfo);
|
2337 |
|
|
DR_STMT (STMT_VINFO_DATA_REF (stmt_vinfo)) = pattern_stmt;
|
2338 |
|
|
*type_out = vectype;
|
2339 |
|
|
*type_in = vectype;
|
2340 |
|
|
VEC_safe_push (gimple, heap, *stmts, last_stmt);
|
2341 |
|
|
return pattern_stmt;
|
2342 |
|
|
}
|
2343 |
|
|
else
|
2344 |
|
|
return NULL;
|
2345 |
|
|
}
|
2346 |
|
|
|
2347 |
|
|
|
2348 |
|
|
/* Mark statements that are involved in a pattern. */
|
2349 |
|
|
|
2350 |
|
|
static inline void
|
2351 |
|
|
vect_mark_pattern_stmts (gimple orig_stmt, gimple pattern_stmt,
|
2352 |
|
|
tree pattern_vectype)
|
2353 |
|
|
{
|
2354 |
|
|
stmt_vec_info pattern_stmt_info, def_stmt_info;
|
2355 |
|
|
stmt_vec_info orig_stmt_info = vinfo_for_stmt (orig_stmt);
|
2356 |
|
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (orig_stmt_info);
|
2357 |
|
|
gimple def_stmt;
|
2358 |
|
|
|
2359 |
|
|
pattern_stmt_info = vinfo_for_stmt (pattern_stmt);
|
2360 |
|
|
if (pattern_stmt_info == NULL)
|
2361 |
|
|
{
|
2362 |
|
|
pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo, NULL);
|
2363 |
|
|
set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info);
|
2364 |
|
|
}
|
2365 |
|
|
gimple_set_bb (pattern_stmt, gimple_bb (orig_stmt));
|
2366 |
|
|
|
2367 |
|
|
STMT_VINFO_RELATED_STMT (pattern_stmt_info) = orig_stmt;
|
2368 |
|
|
STMT_VINFO_DEF_TYPE (pattern_stmt_info)
|
2369 |
|
|
= STMT_VINFO_DEF_TYPE (orig_stmt_info);
|
2370 |
|
|
STMT_VINFO_VECTYPE (pattern_stmt_info) = pattern_vectype;
|
2371 |
|
|
STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true;
|
2372 |
|
|
STMT_VINFO_RELATED_STMT (orig_stmt_info) = pattern_stmt;
|
2373 |
|
|
STMT_VINFO_PATTERN_DEF_SEQ (pattern_stmt_info)
|
2374 |
|
|
= STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
|
2375 |
|
|
if (STMT_VINFO_PATTERN_DEF_SEQ (pattern_stmt_info))
|
2376 |
|
|
{
|
2377 |
|
|
gimple_stmt_iterator si;
|
2378 |
|
|
for (si = gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (pattern_stmt_info));
|
2379 |
|
|
!gsi_end_p (si); gsi_next (&si))
|
2380 |
|
|
{
|
2381 |
|
|
def_stmt = gsi_stmt (si);
|
2382 |
|
|
def_stmt_info = vinfo_for_stmt (def_stmt);
|
2383 |
|
|
if (def_stmt_info == NULL)
|
2384 |
|
|
{
|
2385 |
|
|
def_stmt_info = new_stmt_vec_info (def_stmt, loop_vinfo, NULL);
|
2386 |
|
|
set_vinfo_for_stmt (def_stmt, def_stmt_info);
|
2387 |
|
|
}
|
2388 |
|
|
gimple_set_bb (def_stmt, gimple_bb (orig_stmt));
|
2389 |
|
|
STMT_VINFO_RELATED_STMT (def_stmt_info) = orig_stmt;
|
2390 |
|
|
STMT_VINFO_DEF_TYPE (def_stmt_info)
|
2391 |
|
|
= STMT_VINFO_DEF_TYPE (orig_stmt_info);
|
2392 |
|
|
if (STMT_VINFO_VECTYPE (def_stmt_info) == NULL_TREE)
|
2393 |
|
|
STMT_VINFO_VECTYPE (def_stmt_info) = pattern_vectype;
|
2394 |
|
|
}
|
2395 |
|
|
}
|
2396 |
|
|
}
|
2397 |
|
|
|
2398 |
|
|
/* Function vect_pattern_recog_1
|
2399 |
|
|
|
2400 |
|
|
Input:
|
2401 |
|
|
PATTERN_RECOG_FUNC: A pointer to a function that detects a certain
|
2402 |
|
|
computation pattern.
|
2403 |
|
|
STMT: A stmt from which the pattern search should start.
|
2404 |
|
|
|
2405 |
|
|
If PATTERN_RECOG_FUNC successfully detected the pattern, it creates an
|
2406 |
|
|
expression that computes the same functionality and can be used to
|
2407 |
|
|
replace the sequence of stmts that are involved in the pattern.
|
2408 |
|
|
|
2409 |
|
|
Output:
|
2410 |
|
|
This function checks if the expression returned by PATTERN_RECOG_FUNC is
|
2411 |
|
|
supported in vector form by the target. We use 'TYPE_IN' to obtain the
|
2412 |
|
|
relevant vector type. If 'TYPE_IN' is already a vector type, then this
|
2413 |
|
|
indicates that target support had already been checked by PATTERN_RECOG_FUNC.
|
2414 |
|
|
If 'TYPE_OUT' is also returned by PATTERN_RECOG_FUNC, we check that it fits
|
2415 |
|
|
to the available target pattern.
|
2416 |
|
|
|
2417 |
|
|
This function also does some bookkeeping, as explained in the documentation
|
2418 |
|
|
for vect_recog_pattern. */
|
2419 |
|
|
|
2420 |
|
|
static void
|
2421 |
|
|
vect_pattern_recog_1 (vect_recog_func_ptr vect_recog_func,
|
2422 |
|
|
gimple_stmt_iterator si,
|
2423 |
|
|
VEC (gimple, heap) **stmts_to_replace)
|
2424 |
|
|
{
|
2425 |
|
|
gimple stmt = gsi_stmt (si), pattern_stmt;
|
2426 |
|
|
stmt_vec_info stmt_info;
|
2427 |
|
|
loop_vec_info loop_vinfo;
|
2428 |
|
|
tree pattern_vectype;
|
2429 |
|
|
tree type_in, type_out;
|
2430 |
|
|
enum tree_code code;
|
2431 |
|
|
int i;
|
2432 |
|
|
gimple next;
|
2433 |
|
|
|
2434 |
|
|
VEC_truncate (gimple, *stmts_to_replace, 0);
|
2435 |
|
|
VEC_quick_push (gimple, *stmts_to_replace, stmt);
|
2436 |
|
|
pattern_stmt = (* vect_recog_func) (stmts_to_replace, &type_in, &type_out);
|
2437 |
|
|
if (!pattern_stmt)
|
2438 |
|
|
return;
|
2439 |
|
|
|
2440 |
|
|
stmt = VEC_last (gimple, *stmts_to_replace);
|
2441 |
|
|
stmt_info = vinfo_for_stmt (stmt);
|
2442 |
|
|
loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
2443 |
|
|
|
2444 |
|
|
if (VECTOR_MODE_P (TYPE_MODE (type_in)))
|
2445 |
|
|
{
|
2446 |
|
|
/* No need to check target support (already checked by the pattern
|
2447 |
|
|
recognition function). */
|
2448 |
|
|
pattern_vectype = type_out ? type_out : type_in;
|
2449 |
|
|
}
|
2450 |
|
|
else
|
2451 |
|
|
{
|
2452 |
|
|
enum machine_mode vec_mode;
|
2453 |
|
|
enum insn_code icode;
|
2454 |
|
|
optab optab;
|
2455 |
|
|
|
2456 |
|
|
/* Check target support */
|
2457 |
|
|
type_in = get_vectype_for_scalar_type (type_in);
|
2458 |
|
|
if (!type_in)
|
2459 |
|
|
return;
|
2460 |
|
|
if (type_out)
|
2461 |
|
|
type_out = get_vectype_for_scalar_type (type_out);
|
2462 |
|
|
else
|
2463 |
|
|
type_out = type_in;
|
2464 |
|
|
if (!type_out)
|
2465 |
|
|
return;
|
2466 |
|
|
pattern_vectype = type_out;
|
2467 |
|
|
|
2468 |
|
|
if (is_gimple_assign (pattern_stmt))
|
2469 |
|
|
code = gimple_assign_rhs_code (pattern_stmt);
|
2470 |
|
|
else
|
2471 |
|
|
{
|
2472 |
|
|
gcc_assert (is_gimple_call (pattern_stmt));
|
2473 |
|
|
code = CALL_EXPR;
|
2474 |
|
|
}
|
2475 |
|
|
|
2476 |
|
|
optab = optab_for_tree_code (code, type_in, optab_default);
|
2477 |
|
|
vec_mode = TYPE_MODE (type_in);
|
2478 |
|
|
if (!optab
|
2479 |
|
|
|| (icode = optab_handler (optab, vec_mode)) == CODE_FOR_nothing
|
2480 |
|
|
|| (insn_data[icode].operand[0].mode != TYPE_MODE (type_out)))
|
2481 |
|
|
return;
|
2482 |
|
|
}
|
2483 |
|
|
|
2484 |
|
|
/* Found a vectorizable pattern. */
|
2485 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
2486 |
|
|
{
|
2487 |
|
|
fprintf (vect_dump, "pattern recognized: ");
|
2488 |
|
|
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
|
2489 |
|
|
}
|
2490 |
|
|
|
2491 |
|
|
/* Mark the stmts that are involved in the pattern. */
|
2492 |
|
|
vect_mark_pattern_stmts (stmt, pattern_stmt, pattern_vectype);
|
2493 |
|
|
|
2494 |
|
|
/* Patterns cannot be vectorized using SLP, because they change the order of
|
2495 |
|
|
computation. */
|
2496 |
|
|
FOR_EACH_VEC_ELT (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i, next)
|
2497 |
|
|
if (next == stmt)
|
2498 |
|
|
VEC_ordered_remove (gimple, LOOP_VINFO_REDUCTIONS (loop_vinfo), i);
|
2499 |
|
|
|
2500 |
|
|
/* It is possible that additional pattern stmts are created and inserted in
|
2501 |
|
|
STMTS_TO_REPLACE. We create a stmt_info for each of them, and mark the
|
2502 |
|
|
relevant statements. */
|
2503 |
|
|
for (i = 0; VEC_iterate (gimple, *stmts_to_replace, i, stmt)
|
2504 |
|
|
&& (unsigned) i < (VEC_length (gimple, *stmts_to_replace) - 1);
|
2505 |
|
|
i++)
|
2506 |
|
|
{
|
2507 |
|
|
stmt_info = vinfo_for_stmt (stmt);
|
2508 |
|
|
pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
|
2509 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
2510 |
|
|
{
|
2511 |
|
|
fprintf (vect_dump, "additional pattern stmt: ");
|
2512 |
|
|
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
|
2513 |
|
|
}
|
2514 |
|
|
|
2515 |
|
|
vect_mark_pattern_stmts (stmt, pattern_stmt, NULL_TREE);
|
2516 |
|
|
}
|
2517 |
|
|
}
|
2518 |
|
|
|
2519 |
|
|
|
2520 |
|
|
/* Function vect_pattern_recog
|
2521 |
|
|
|
2522 |
|
|
Input:
|
2523 |
|
|
LOOP_VINFO - a struct_loop_info of a loop in which we want to look for
|
2524 |
|
|
computation idioms.
|
2525 |
|
|
|
2526 |
|
|
Output - for each computation idiom that is detected we create a new stmt
|
2527 |
|
|
that provides the same functionality and that can be vectorized. We
|
2528 |
|
|
also record some information in the struct_stmt_info of the relevant
|
2529 |
|
|
stmts, as explained below:
|
2530 |
|
|
|
2531 |
|
|
At the entry to this function we have the following stmts, with the
|
2532 |
|
|
following initial value in the STMT_VINFO fields:
|
2533 |
|
|
|
2534 |
|
|
stmt in_pattern_p related_stmt vec_stmt
|
2535 |
|
|
S1: a_i = .... - - -
|
2536 |
|
|
S2: a_2 = ..use(a_i).. - - -
|
2537 |
|
|
S3: a_1 = ..use(a_2).. - - -
|
2538 |
|
|
S4: a_0 = ..use(a_1).. - - -
|
2539 |
|
|
S5: ... = ..use(a_0).. - - -
|
2540 |
|
|
|
2541 |
|
|
Say the sequence {S1,S2,S3,S4} was detected as a pattern that can be
|
2542 |
|
|
represented by a single stmt. We then:
|
2543 |
|
|
- create a new stmt S6 equivalent to the pattern (the stmt is not
|
2544 |
|
|
inserted into the code)
|
2545 |
|
|
- fill in the STMT_VINFO fields as follows:
|
2546 |
|
|
|
2547 |
|
|
in_pattern_p related_stmt vec_stmt
|
2548 |
|
|
S1: a_i = .... - - -
|
2549 |
|
|
S2: a_2 = ..use(a_i).. - - -
|
2550 |
|
|
S3: a_1 = ..use(a_2).. - - -
|
2551 |
|
|
S4: a_0 = ..use(a_1).. true S6 -
|
2552 |
|
|
'---> S6: a_new = .... - S4 -
|
2553 |
|
|
S5: ... = ..use(a_0).. - - -
|
2554 |
|
|
|
2555 |
|
|
(the last stmt in the pattern (S4) and the new pattern stmt (S6) point
|
2556 |
|
|
to each other through the RELATED_STMT field).
|
2557 |
|
|
|
2558 |
|
|
S6 will be marked as relevant in vect_mark_stmts_to_be_vectorized instead
|
2559 |
|
|
of S4 because it will replace all its uses. Stmts {S1,S2,S3} will
|
2560 |
|
|
remain irrelevant unless used by stmts other than S4.
|
2561 |
|
|
|
2562 |
|
|
If vectorization succeeds, vect_transform_stmt will skip over {S1,S2,S3}
|
2563 |
|
|
(because they are marked as irrelevant). It will vectorize S6, and record
|
2564 |
|
|
a pointer to the new vector stmt VS6 from S6 (as usual).
|
2565 |
|
|
S4 will be skipped, and S5 will be vectorized as usual:
|
2566 |
|
|
|
2567 |
|
|
in_pattern_p related_stmt vec_stmt
|
2568 |
|
|
S1: a_i = .... - - -
|
2569 |
|
|
S2: a_2 = ..use(a_i).. - - -
|
2570 |
|
|
S3: a_1 = ..use(a_2).. - - -
|
2571 |
|
|
> VS6: va_new = .... - - -
|
2572 |
|
|
S4: a_0 = ..use(a_1).. true S6 VS6
|
2573 |
|
|
'---> S6: a_new = .... - S4 VS6
|
2574 |
|
|
> VS5: ... = ..vuse(va_new).. - - -
|
2575 |
|
|
S5: ... = ..use(a_0).. - - -
|
2576 |
|
|
|
2577 |
|
|
DCE could then get rid of {S1,S2,S3,S4,S5} (if their defs are not used
|
2578 |
|
|
elsewhere), and we'll end up with:
|
2579 |
|
|
|
2580 |
|
|
VS6: va_new = ....
|
2581 |
|
|
VS5: ... = ..vuse(va_new)..
|
2582 |
|
|
|
2583 |
|
|
In case of more than one pattern statements, e.g., widen-mult with
|
2584 |
|
|
intermediate type:
|
2585 |
|
|
|
2586 |
|
|
S1 a_t = ;
|
2587 |
|
|
S2 a_T = (TYPE) a_t;
|
2588 |
|
|
'--> S3: a_it = (interm_type) a_t;
|
2589 |
|
|
S4 prod_T = a_T * CONST;
|
2590 |
|
|
'--> S5: prod_T' = a_it w* CONST;
|
2591 |
|
|
|
2592 |
|
|
there may be other users of a_T outside the pattern. In that case S2 will
|
2593 |
|
|
be marked as relevant (as well as S3), and both S2 and S3 will be analyzed
|
2594 |
|
|
and vectorized. The vector stmt VS2 will be recorded in S2, and VS3 will
|
2595 |
|
|
be recorded in S3. */
|
2596 |
|
|
|
2597 |
|
|
void
|
2598 |
|
|
vect_pattern_recog (loop_vec_info loop_vinfo)
|
2599 |
|
|
{
|
2600 |
|
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
2601 |
|
|
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
|
2602 |
|
|
unsigned int nbbs = loop->num_nodes;
|
2603 |
|
|
gimple_stmt_iterator si;
|
2604 |
|
|
unsigned int i, j;
|
2605 |
|
|
vect_recog_func_ptr vect_recog_func;
|
2606 |
|
|
VEC (gimple, heap) *stmts_to_replace = VEC_alloc (gimple, heap, 1);
|
2607 |
|
|
|
2608 |
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
2609 |
|
|
fprintf (vect_dump, "=== vect_pattern_recog ===");
|
2610 |
|
|
|
2611 |
|
|
/* Scan through the loop stmts, applying the pattern recognition
|
2612 |
|
|
functions starting at each stmt visited: */
|
2613 |
|
|
for (i = 0; i < nbbs; i++)
|
2614 |
|
|
{
|
2615 |
|
|
basic_block bb = bbs[i];
|
2616 |
|
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
2617 |
|
|
{
|
2618 |
|
|
/* Scan over all generic vect_recog_xxx_pattern functions. */
|
2619 |
|
|
for (j = 0; j < NUM_PATTERNS; j++)
|
2620 |
|
|
{
|
2621 |
|
|
vect_recog_func = vect_vect_recog_func_ptrs[j];
|
2622 |
|
|
vect_pattern_recog_1 (vect_recog_func, si,
|
2623 |
|
|
&stmts_to_replace);
|
2624 |
|
|
}
|
2625 |
|
|
}
|
2626 |
|
|
}
|
2627 |
|
|
|
2628 |
|
|
VEC_free (gimple, heap, stmts_to_replace);
|
2629 |
|
|
}
|