/* Loop Vectorization
|
/* Loop Vectorization
|
Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
|
Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
|
Free Software Foundation, Inc.
|
Free Software Foundation, Inc.
|
Contributed by Dorit Naishlos <dorit@il.ibm.com> and
|
Contributed by Dorit Naishlos <dorit@il.ibm.com> and
|
Ira Rosen <irar@il.ibm.com>
|
Ira Rosen <irar@il.ibm.com>
|
|
|
This file is part of GCC.
|
This file is part of GCC.
|
|
|
GCC is free software; you can redistribute it and/or modify it under
|
GCC is free software; you can redistribute it and/or modify it under
|
the terms of the GNU General Public License as published by the Free
|
the terms of the GNU General Public License as published by the Free
|
Software Foundation; either version 3, or (at your option) any later
|
Software Foundation; either version 3, or (at your option) any later
|
version.
|
version.
|
|
|
GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
for more details.
|
for more details.
|
|
|
You should have received a copy of the GNU General Public License
|
You should have received a copy of the GNU General Public License
|
along with GCC; see the file COPYING3. If not see
|
along with GCC; see the file COPYING3. If not see
|
<http://www.gnu.org/licenses/>. */
|
<http://www.gnu.org/licenses/>. */
|
|
|
#include "config.h"
|
#include "config.h"
|
#include "system.h"
|
#include "system.h"
|
#include "coretypes.h"
|
#include "coretypes.h"
|
#include "tm.h"
|
#include "tm.h"
|
#include "ggc.h"
|
#include "ggc.h"
|
#include "tree.h"
|
#include "tree.h"
|
#include "basic-block.h"
|
#include "basic-block.h"
|
#include "diagnostic.h"
|
#include "diagnostic.h"
|
#include "tree-flow.h"
|
#include "tree-flow.h"
|
#include "tree-dump.h"
|
#include "tree-dump.h"
|
#include "cfgloop.h"
|
#include "cfgloop.h"
|
#include "cfglayout.h"
|
#include "cfglayout.h"
|
#include "expr.h"
|
#include "expr.h"
|
#include "recog.h"
|
#include "recog.h"
|
#include "optabs.h"
|
#include "optabs.h"
|
#include "params.h"
|
#include "params.h"
|
#include "toplev.h"
|
#include "toplev.h"
|
#include "tree-chrec.h"
|
#include "tree-chrec.h"
|
#include "tree-scalar-evolution.h"
|
#include "tree-scalar-evolution.h"
|
#include "tree-vectorizer.h"
|
#include "tree-vectorizer.h"
|
|
|
/* Loop Vectorization Pass.
|
/* Loop Vectorization Pass.
|
|
|
This pass tries to vectorize loops.
|
This pass tries to vectorize loops.
|
|
|
For example, the vectorizer transforms the following simple loop:
|
For example, the vectorizer transforms the following simple loop:
|
|
|
short a[N]; short b[N]; short c[N]; int i;
|
short a[N]; short b[N]; short c[N]; int i;
|
|
|
for (i=0; i<N; i++){
|
for (i=0; i<N; i++){
|
a[i] = b[i] + c[i];
|
a[i] = b[i] + c[i];
|
}
|
}
|
|
|
as if it was manually vectorized by rewriting the source code into:
|
as if it was manually vectorized by rewriting the source code into:
|
|
|
typedef int __attribute__((mode(V8HI))) v8hi;
|
typedef int __attribute__((mode(V8HI))) v8hi;
|
short a[N]; short b[N]; short c[N]; int i;
|
short a[N]; short b[N]; short c[N]; int i;
|
v8hi *pa = (v8hi*)a, *pb = (v8hi*)b, *pc = (v8hi*)c;
|
v8hi *pa = (v8hi*)a, *pb = (v8hi*)b, *pc = (v8hi*)c;
|
v8hi va, vb, vc;
|
v8hi va, vb, vc;
|
|
|
for (i=0; i<N/8; i++){
|
for (i=0; i<N/8; i++){
|
vb = pb[i];
|
vb = pb[i];
|
vc = pc[i];
|
vc = pc[i];
|
va = vb + vc;
|
va = vb + vc;
|
pa[i] = va;
|
pa[i] = va;
|
}
|
}
|
|
|
The main entry to this pass is vectorize_loops(), in which
|
The main entry to this pass is vectorize_loops(), in which
|
the vectorizer applies a set of analyses on a given set of loops,
|
the vectorizer applies a set of analyses on a given set of loops,
|
followed by the actual vectorization transformation for the loops that
|
followed by the actual vectorization transformation for the loops that
|
had successfully passed the analysis phase.
|
had successfully passed the analysis phase.
|
Throughout this pass we make a distinction between two types of
|
Throughout this pass we make a distinction between two types of
|
data: scalars (which are represented by SSA_NAMES), and memory references
|
data: scalars (which are represented by SSA_NAMES), and memory references
|
("data-refs"). These two types of data require different handling both
|
("data-refs"). These two types of data require different handling both
|
during analysis and transformation. The types of data-refs that the
|
during analysis and transformation. The types of data-refs that the
|
vectorizer currently supports are ARRAY_REFS which base is an array DECL
|
vectorizer currently supports are ARRAY_REFS which base is an array DECL
|
(not a pointer), and INDIRECT_REFS through pointers; both array and pointer
|
(not a pointer), and INDIRECT_REFS through pointers; both array and pointer
|
accesses are required to have a simple (consecutive) access pattern.
|
accesses are required to have a simple (consecutive) access pattern.
|
|
|
Analysis phase:
|
Analysis phase:
|
===============
|
===============
|
The driver for the analysis phase is vect_analyze_loop().
|
The driver for the analysis phase is vect_analyze_loop().
|
It applies a set of analyses, some of which rely on the scalar evolution
|
It applies a set of analyses, some of which rely on the scalar evolution
|
analyzer (scev) developed by Sebastian Pop.
|
analyzer (scev) developed by Sebastian Pop.
|
|
|
During the analysis phase the vectorizer records some information
|
During the analysis phase the vectorizer records some information
|
per stmt in a "stmt_vec_info" struct which is attached to each stmt in the
|
per stmt in a "stmt_vec_info" struct which is attached to each stmt in the
|
loop, as well as general information about the loop as a whole, which is
|
loop, as well as general information about the loop as a whole, which is
|
recorded in a "loop_vec_info" struct attached to each loop.
|
recorded in a "loop_vec_info" struct attached to each loop.
|
|
|
Transformation phase:
|
Transformation phase:
|
=====================
|
=====================
|
The loop transformation phase scans all the stmts in the loop, and
|
The loop transformation phase scans all the stmts in the loop, and
|
creates a vector stmt (or a sequence of stmts) for each scalar stmt S in
|
creates a vector stmt (or a sequence of stmts) for each scalar stmt S in
|
the loop that needs to be vectorized. It inserts the vector code sequence
|
the loop that needs to be vectorized. It inserts the vector code sequence
|
just before the scalar stmt S, and records a pointer to the vector code
|
just before the scalar stmt S, and records a pointer to the vector code
|
in STMT_VINFO_VEC_STMT (stmt_info) (stmt_info is the stmt_vec_info struct
|
in STMT_VINFO_VEC_STMT (stmt_info) (stmt_info is the stmt_vec_info struct
|
attached to S). This pointer will be used for the vectorization of following
|
attached to S). This pointer will be used for the vectorization of following
|
stmts which use the def of stmt S. Stmt S is removed if it writes to memory;
|
stmts which use the def of stmt S. Stmt S is removed if it writes to memory;
|
otherwise, we rely on dead code elimination for removing it.
|
otherwise, we rely on dead code elimination for removing it.
|
|
|
For example, say stmt S1 was vectorized into stmt VS1:
|
For example, say stmt S1 was vectorized into stmt VS1:
|
|
|
VS1: vb = px[i];
|
VS1: vb = px[i];
|
S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
|
S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
|
S2: a = b;
|
S2: a = b;
|
|
|
To vectorize stmt S2, the vectorizer first finds the stmt that defines
|
To vectorize stmt S2, the vectorizer first finds the stmt that defines
|
the operand 'b' (S1), and gets the relevant vector def 'vb' from the
|
the operand 'b' (S1), and gets the relevant vector def 'vb' from the
|
vector stmt VS1 pointed to by STMT_VINFO_VEC_STMT (stmt_info (S1)). The
|
vector stmt VS1 pointed to by STMT_VINFO_VEC_STMT (stmt_info (S1)). The
|
resulting sequence would be:
|
resulting sequence would be:
|
|
|
VS1: vb = px[i];
|
VS1: vb = px[i];
|
S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
|
S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
|
VS2: va = vb;
|
VS2: va = vb;
|
S2: a = b; STMT_VINFO_VEC_STMT (stmt_info (S2)) = VS2
|
S2: a = b; STMT_VINFO_VEC_STMT (stmt_info (S2)) = VS2
|
|
|
Operands that are not SSA_NAMEs, are data-refs that appear in
|
Operands that are not SSA_NAMEs, are data-refs that appear in
|
load/store operations (like 'x[i]' in S1), and are handled differently.
|
load/store operations (like 'x[i]' in S1), and are handled differently.
|
|
|
Target modeling:
|
Target modeling:
|
=================
|
=================
|
Currently the only target specific information that is used is the
|
Currently the only target specific information that is used is the
|
size of the vector (in bytes) - "UNITS_PER_SIMD_WORD". Targets that can
|
size of the vector (in bytes) - "UNITS_PER_SIMD_WORD". Targets that can
|
support different sizes of vectors, for now will need to specify one value
|
support different sizes of vectors, for now will need to specify one value
|
for "UNITS_PER_SIMD_WORD". More flexibility will be added in the future.
|
for "UNITS_PER_SIMD_WORD". More flexibility will be added in the future.
|
|
|
Since we only vectorize operations which vector form can be
|
Since we only vectorize operations which vector form can be
|
expressed using existing tree codes, to verify that an operation is
|
expressed using existing tree codes, to verify that an operation is
|
supported, the vectorizer checks the relevant optab at the relevant
|
supported, the vectorizer checks the relevant optab at the relevant
|
machine_mode (e.g, optab_handler (add_optab, V8HImode)->insn_code). If
|
machine_mode (e.g, optab_handler (add_optab, V8HImode)->insn_code). If
|
the value found is CODE_FOR_nothing, then there's no target support, and
|
the value found is CODE_FOR_nothing, then there's no target support, and
|
we can't vectorize the stmt.
|
we can't vectorize the stmt.
|
|
|
For additional information on this project see:
|
For additional information on this project see:
|
http://gcc.gnu.org/projects/tree-ssa/vectorization.html
|
http://gcc.gnu.org/projects/tree-ssa/vectorization.html
|
*/
|
*/
|
|
|
/* Function vect_determine_vectorization_factor
|
/* Function vect_determine_vectorization_factor
|
|
|
Determine the vectorization factor (VF). VF is the number of data elements
|
Determine the vectorization factor (VF). VF is the number of data elements
|
that are operated upon in parallel in a single iteration of the vectorized
|
that are operated upon in parallel in a single iteration of the vectorized
|
loop. For example, when vectorizing a loop that operates on 4byte elements,
|
loop. For example, when vectorizing a loop that operates on 4byte elements,
|
on a target with vector size (VS) 16byte, the VF is set to 4, since 4
|
on a target with vector size (VS) 16byte, the VF is set to 4, since 4
|
elements can fit in a single vector register.
|
elements can fit in a single vector register.
|
|
|
We currently support vectorization of loops in which all types operated upon
|
We currently support vectorization of loops in which all types operated upon
|
are of the same size. Therefore this function currently sets VF according to
|
are of the same size. Therefore this function currently sets VF according to
|
the size of the types operated upon, and fails if there are multiple sizes
|
the size of the types operated upon, and fails if there are multiple sizes
|
in the loop.
|
in the loop.
|
|
|
VF is also the factor by which the loop iterations are strip-mined, e.g.:
|
VF is also the factor by which the loop iterations are strip-mined, e.g.:
|
original loop:
|
original loop:
|
for (i=0; i<N; i++){
|
for (i=0; i<N; i++){
|
a[i] = b[i] + c[i];
|
a[i] = b[i] + c[i];
|
}
|
}
|
|
|
vectorized loop:
|
vectorized loop:
|
for (i=0; i<N; i+=VF){
|
for (i=0; i<N; i+=VF){
|
a[i:VF] = b[i:VF] + c[i:VF];
|
a[i:VF] = b[i:VF] + c[i:VF];
|
}
|
}
|
*/
|
*/
|
|
|
static bool
|
static bool
|
vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
|
vect_determine_vectorization_factor (loop_vec_info loop_vinfo)
|
{
|
{
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
|
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
|
int nbbs = loop->num_nodes;
|
int nbbs = loop->num_nodes;
|
gimple_stmt_iterator si;
|
gimple_stmt_iterator si;
|
unsigned int vectorization_factor = 0;
|
unsigned int vectorization_factor = 0;
|
tree scalar_type;
|
tree scalar_type;
|
gimple phi;
|
gimple phi;
|
tree vectype;
|
tree vectype;
|
unsigned int nunits;
|
unsigned int nunits;
|
stmt_vec_info stmt_info;
|
stmt_vec_info stmt_info;
|
int i;
|
int i;
|
HOST_WIDE_INT dummy;
|
HOST_WIDE_INT dummy;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "=== vect_determine_vectorization_factor ===");
|
fprintf (vect_dump, "=== vect_determine_vectorization_factor ===");
|
|
|
for (i = 0; i < nbbs; i++)
|
for (i = 0; i < nbbs; i++)
|
{
|
{
|
basic_block bb = bbs[i];
|
basic_block bb = bbs[i];
|
|
|
for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
|
for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
|
{
|
{
|
phi = gsi_stmt (si);
|
phi = gsi_stmt (si);
|
stmt_info = vinfo_for_stmt (phi);
|
stmt_info = vinfo_for_stmt (phi);
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "==> examining phi: ");
|
fprintf (vect_dump, "==> examining phi: ");
|
print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
|
}
|
}
|
|
|
gcc_assert (stmt_info);
|
gcc_assert (stmt_info);
|
|
|
if (STMT_VINFO_RELEVANT_P (stmt_info))
|
if (STMT_VINFO_RELEVANT_P (stmt_info))
|
{
|
{
|
gcc_assert (!STMT_VINFO_VECTYPE (stmt_info));
|
gcc_assert (!STMT_VINFO_VECTYPE (stmt_info));
|
scalar_type = TREE_TYPE (PHI_RESULT (phi));
|
scalar_type = TREE_TYPE (PHI_RESULT (phi));
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "get vectype for scalar type: ");
|
fprintf (vect_dump, "get vectype for scalar type: ");
|
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
|
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
|
}
|
}
|
|
|
vectype = get_vectype_for_scalar_type (scalar_type);
|
vectype = get_vectype_for_scalar_type (scalar_type);
|
if (!vectype)
|
if (!vectype)
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
{
|
{
|
fprintf (vect_dump,
|
fprintf (vect_dump,
|
"not vectorized: unsupported data-type ");
|
"not vectorized: unsupported data-type ");
|
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
|
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
|
}
|
}
|
return false;
|
return false;
|
}
|
}
|
STMT_VINFO_VECTYPE (stmt_info) = vectype;
|
STMT_VINFO_VECTYPE (stmt_info) = vectype;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "vectype: ");
|
fprintf (vect_dump, "vectype: ");
|
print_generic_expr (vect_dump, vectype, TDF_SLIM);
|
print_generic_expr (vect_dump, vectype, TDF_SLIM);
|
}
|
}
|
|
|
nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "nunits = %d", nunits);
|
fprintf (vect_dump, "nunits = %d", nunits);
|
|
|
if (!vectorization_factor
|
if (!vectorization_factor
|
|| (nunits > vectorization_factor))
|
|| (nunits > vectorization_factor))
|
vectorization_factor = nunits;
|
vectorization_factor = nunits;
|
}
|
}
|
}
|
}
|
|
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
{
|
{
|
gimple stmt = gsi_stmt (si);
|
gimple stmt = gsi_stmt (si);
|
stmt_info = vinfo_for_stmt (stmt);
|
stmt_info = vinfo_for_stmt (stmt);
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "==> examining statement: ");
|
fprintf (vect_dump, "==> examining statement: ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
gcc_assert (stmt_info);
|
gcc_assert (stmt_info);
|
|
|
/* skip stmts which do not need to be vectorized. */
|
/* skip stmts which do not need to be vectorized. */
|
if (!STMT_VINFO_RELEVANT_P (stmt_info)
|
if (!STMT_VINFO_RELEVANT_P (stmt_info)
|
&& !STMT_VINFO_LIVE_P (stmt_info))
|
&& !STMT_VINFO_LIVE_P (stmt_info))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "skip.");
|
fprintf (vect_dump, "skip.");
|
continue;
|
continue;
|
}
|
}
|
|
|
if (gimple_get_lhs (stmt) == NULL_TREE)
|
if (gimple_get_lhs (stmt) == NULL_TREE)
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
{
|
{
|
fprintf (vect_dump, "not vectorized: irregular stmt.");
|
fprintf (vect_dump, "not vectorized: irregular stmt.");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
return false;
|
return false;
|
}
|
}
|
|
|
if (VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt))))
|
if (VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt))))
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
{
|
{
|
fprintf (vect_dump, "not vectorized: vector stmt in loop:");
|
fprintf (vect_dump, "not vectorized: vector stmt in loop:");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
return false;
|
return false;
|
}
|
}
|
|
|
if (STMT_VINFO_VECTYPE (stmt_info))
|
if (STMT_VINFO_VECTYPE (stmt_info))
|
{
|
{
|
/* The only case when a vectype had been already set is for stmts
|
/* The only case when a vectype had been already set is for stmts
|
that contain a dataref, or for "pattern-stmts" (stmts generated
|
that contain a dataref, or for "pattern-stmts" (stmts generated
|
by the vectorizer to represent/replace a certain idiom). */
|
by the vectorizer to represent/replace a certain idiom). */
|
gcc_assert (STMT_VINFO_DATA_REF (stmt_info)
|
gcc_assert (STMT_VINFO_DATA_REF (stmt_info)
|
|| is_pattern_stmt_p (stmt_info));
|
|| is_pattern_stmt_p (stmt_info));
|
vectype = STMT_VINFO_VECTYPE (stmt_info);
|
vectype = STMT_VINFO_VECTYPE (stmt_info);
|
}
|
}
|
else
|
else
|
{
|
{
|
gcc_assert (!STMT_VINFO_DATA_REF (stmt_info)
|
gcc_assert (!STMT_VINFO_DATA_REF (stmt_info)
|
&& !is_pattern_stmt_p (stmt_info));
|
&& !is_pattern_stmt_p (stmt_info));
|
|
|
scalar_type = vect_get_smallest_scalar_type (stmt, &dummy,
|
scalar_type = vect_get_smallest_scalar_type (stmt, &dummy,
|
&dummy);
|
&dummy);
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "get vectype for scalar type: ");
|
fprintf (vect_dump, "get vectype for scalar type: ");
|
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
|
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
|
}
|
}
|
|
|
vectype = get_vectype_for_scalar_type (scalar_type);
|
vectype = get_vectype_for_scalar_type (scalar_type);
|
if (!vectype)
|
if (!vectype)
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
{
|
{
|
fprintf (vect_dump,
|
fprintf (vect_dump,
|
"not vectorized: unsupported data-type ");
|
"not vectorized: unsupported data-type ");
|
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
|
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
|
}
|
}
|
return false;
|
return false;
|
}
|
}
|
STMT_VINFO_VECTYPE (stmt_info) = vectype;
|
STMT_VINFO_VECTYPE (stmt_info) = vectype;
|
}
|
}
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "vectype: ");
|
fprintf (vect_dump, "vectype: ");
|
print_generic_expr (vect_dump, vectype, TDF_SLIM);
|
print_generic_expr (vect_dump, vectype, TDF_SLIM);
|
}
|
}
|
|
|
nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "nunits = %d", nunits);
|
fprintf (vect_dump, "nunits = %d", nunits);
|
|
|
if (!vectorization_factor
|
if (!vectorization_factor
|
|| (nunits > vectorization_factor))
|
|| (nunits > vectorization_factor))
|
vectorization_factor = nunits;
|
vectorization_factor = nunits;
|
|
|
}
|
}
|
}
|
}
|
|
|
/* TODO: Analyze cost. Decide if worth while to vectorize. */
|
/* TODO: Analyze cost. Decide if worth while to vectorize. */
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "vectorization factor = %d", vectorization_factor);
|
fprintf (vect_dump, "vectorization factor = %d", vectorization_factor);
|
if (vectorization_factor <= 1)
|
if (vectorization_factor <= 1)
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: unsupported data-type");
|
fprintf (vect_dump, "not vectorized: unsupported data-type");
|
return false;
|
return false;
|
}
|
}
|
LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
|
LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
|
|
/* Function vect_is_simple_iv_evolution.
|
/* Function vect_is_simple_iv_evolution.
|
|
|
FORNOW: A simple evolution of an induction variables in the loop is
|
FORNOW: A simple evolution of an induction variables in the loop is
|
considered a polynomial evolution with constant step. */
|
considered a polynomial evolution with constant step. */
|
|
|
static bool
|
static bool
|
vect_is_simple_iv_evolution (unsigned loop_nb, tree access_fn, tree * init,
|
vect_is_simple_iv_evolution (unsigned loop_nb, tree access_fn, tree * init,
|
tree * step)
|
tree * step)
|
{
|
{
|
tree init_expr;
|
tree init_expr;
|
tree step_expr;
|
tree step_expr;
|
tree evolution_part = evolution_part_in_loop_num (access_fn, loop_nb);
|
tree evolution_part = evolution_part_in_loop_num (access_fn, loop_nb);
|
|
|
/* When there is no evolution in this loop, the evolution function
|
/* When there is no evolution in this loop, the evolution function
|
is not "simple". */
|
is not "simple". */
|
if (evolution_part == NULL_TREE)
|
if (evolution_part == NULL_TREE)
|
return false;
|
return false;
|
|
|
/* When the evolution is a polynomial of degree >= 2
|
/* When the evolution is a polynomial of degree >= 2
|
the evolution function is not "simple". */
|
the evolution function is not "simple". */
|
if (tree_is_chrec (evolution_part))
|
if (tree_is_chrec (evolution_part))
|
return false;
|
return false;
|
|
|
step_expr = evolution_part;
|
step_expr = evolution_part;
|
init_expr = unshare_expr (initial_condition_in_loop_num (access_fn, loop_nb));
|
init_expr = unshare_expr (initial_condition_in_loop_num (access_fn, loop_nb));
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "step: ");
|
fprintf (vect_dump, "step: ");
|
print_generic_expr (vect_dump, step_expr, TDF_SLIM);
|
print_generic_expr (vect_dump, step_expr, TDF_SLIM);
|
fprintf (vect_dump, ", init: ");
|
fprintf (vect_dump, ", init: ");
|
print_generic_expr (vect_dump, init_expr, TDF_SLIM);
|
print_generic_expr (vect_dump, init_expr, TDF_SLIM);
|
}
|
}
|
|
|
*init = init_expr;
|
*init = init_expr;
|
*step = step_expr;
|
*step = step_expr;
|
|
|
if (TREE_CODE (step_expr) != INTEGER_CST)
|
if (TREE_CODE (step_expr) != INTEGER_CST)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "step unknown.");
|
fprintf (vect_dump, "step unknown.");
|
return false;
|
return false;
|
}
|
}
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
/* Function vect_analyze_scalar_cycles_1.
|
/* Function vect_analyze_scalar_cycles_1.
|
|
|
Examine the cross iteration def-use cycles of scalar variables
|
Examine the cross iteration def-use cycles of scalar variables
|
in LOOP. LOOP_VINFO represents the loop that is now being
|
in LOOP. LOOP_VINFO represents the loop that is now being
|
considered for vectorization (can be LOOP, or an outer-loop
|
considered for vectorization (can be LOOP, or an outer-loop
|
enclosing LOOP). */
|
enclosing LOOP). */
|
|
|
static void
|
static void
|
vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop)
|
vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop)
|
{
|
{
|
basic_block bb = loop->header;
|
basic_block bb = loop->header;
|
tree dumy;
|
tree dumy;
|
VEC(gimple,heap) *worklist = VEC_alloc (gimple, heap, 64);
|
VEC(gimple,heap) *worklist = VEC_alloc (gimple, heap, 64);
|
gimple_stmt_iterator gsi;
|
gimple_stmt_iterator gsi;
|
bool double_reduc;
|
bool double_reduc;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "=== vect_analyze_scalar_cycles ===");
|
fprintf (vect_dump, "=== vect_analyze_scalar_cycles ===");
|
|
|
/* First - identify all inductions. Reduction detection assumes that all the
|
/* First - identify all inductions. Reduction detection assumes that all the
|
inductions have been identified, therefore, this order must not be
|
inductions have been identified, therefore, this order must not be
|
changed. */
|
changed. */
|
for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
{
|
{
|
gimple phi = gsi_stmt (gsi);
|
gimple phi = gsi_stmt (gsi);
|
tree access_fn = NULL;
|
tree access_fn = NULL;
|
tree def = PHI_RESULT (phi);
|
tree def = PHI_RESULT (phi);
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi);
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi);
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "Analyze phi: ");
|
fprintf (vect_dump, "Analyze phi: ");
|
print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
|
}
|
}
|
|
|
/* Skip virtual phi's. The data dependences that are associated with
|
/* Skip virtual phi's. The data dependences that are associated with
|
virtual defs/uses (i.e., memory accesses) are analyzed elsewhere. */
|
virtual defs/uses (i.e., memory accesses) are analyzed elsewhere. */
|
if (!is_gimple_reg (SSA_NAME_VAR (def)))
|
if (!is_gimple_reg (SSA_NAME_VAR (def)))
|
continue;
|
continue;
|
|
|
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_unknown_def_type;
|
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_unknown_def_type;
|
|
|
/* Analyze the evolution function. */
|
/* Analyze the evolution function. */
|
access_fn = analyze_scalar_evolution (loop, def);
|
access_fn = analyze_scalar_evolution (loop, def);
|
if (access_fn && vect_print_dump_info (REPORT_DETAILS))
|
if (access_fn && vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "Access function of PHI: ");
|
fprintf (vect_dump, "Access function of PHI: ");
|
print_generic_expr (vect_dump, access_fn, TDF_SLIM);
|
print_generic_expr (vect_dump, access_fn, TDF_SLIM);
|
}
|
}
|
|
|
if (!access_fn
|
if (!access_fn
|
|| !vect_is_simple_iv_evolution (loop->num, access_fn, &dumy, &dumy))
|
|| !vect_is_simple_iv_evolution (loop->num, access_fn, &dumy, &dumy))
|
{
|
{
|
VEC_safe_push (gimple, heap, worklist, phi);
|
VEC_safe_push (gimple, heap, worklist, phi);
|
continue;
|
continue;
|
}
|
}
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "Detected induction.");
|
fprintf (vect_dump, "Detected induction.");
|
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_induction_def;
|
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_induction_def;
|
}
|
}
|
|
|
|
|
/* Second - identify all reductions and nested cycles. */
|
/* Second - identify all reductions and nested cycles. */
|
while (VEC_length (gimple, worklist) > 0)
|
while (VEC_length (gimple, worklist) > 0)
|
{
|
{
|
gimple phi = VEC_pop (gimple, worklist);
|
gimple phi = VEC_pop (gimple, worklist);
|
tree def = PHI_RESULT (phi);
|
tree def = PHI_RESULT (phi);
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi);
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi);
|
gimple reduc_stmt;
|
gimple reduc_stmt;
|
bool nested_cycle;
|
bool nested_cycle;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "Analyze phi: ");
|
fprintf (vect_dump, "Analyze phi: ");
|
print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
|
}
|
}
|
|
|
gcc_assert (is_gimple_reg (SSA_NAME_VAR (def)));
|
gcc_assert (is_gimple_reg (SSA_NAME_VAR (def)));
|
gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type);
|
gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type);
|
|
|
nested_cycle = (loop != LOOP_VINFO_LOOP (loop_vinfo));
|
nested_cycle = (loop != LOOP_VINFO_LOOP (loop_vinfo));
|
reduc_stmt = vect_is_simple_reduction (loop_vinfo, phi, !nested_cycle,
|
reduc_stmt = vect_is_simple_reduction (loop_vinfo, phi, !nested_cycle,
|
&double_reduc);
|
&double_reduc);
|
if (reduc_stmt)
|
if (reduc_stmt)
|
{
|
{
|
if (double_reduc)
|
if (double_reduc)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "Detected double reduction.");
|
fprintf (vect_dump, "Detected double reduction.");
|
|
|
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_double_reduction_def;
|
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_double_reduction_def;
|
STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
|
STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
|
vect_double_reduction_def;
|
vect_double_reduction_def;
|
}
|
}
|
else
|
else
|
{
|
{
|
if (nested_cycle)
|
if (nested_cycle)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "Detected vectorizable nested cycle.");
|
fprintf (vect_dump, "Detected vectorizable nested cycle.");
|
|
|
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_nested_cycle;
|
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_nested_cycle;
|
STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
|
STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
|
vect_nested_cycle;
|
vect_nested_cycle;
|
}
|
}
|
else
|
else
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "Detected reduction.");
|
fprintf (vect_dump, "Detected reduction.");
|
|
|
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_reduction_def;
|
STMT_VINFO_DEF_TYPE (stmt_vinfo) = vect_reduction_def;
|
STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
|
STMT_VINFO_DEF_TYPE (vinfo_for_stmt (reduc_stmt)) =
|
vect_reduction_def;
|
vect_reduction_def;
|
}
|
}
|
}
|
}
|
}
|
}
|
else
|
else
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "Unknown def-use cycle pattern.");
|
fprintf (vect_dump, "Unknown def-use cycle pattern.");
|
}
|
}
|
|
|
VEC_free (gimple, heap, worklist);
|
VEC_free (gimple, heap, worklist);
|
}
|
}
|
|
|
|
|
/* Function vect_analyze_scalar_cycles.
|
/* Function vect_analyze_scalar_cycles.
|
|
|
Examine the cross iteration def-use cycles of scalar variables, by
|
Examine the cross iteration def-use cycles of scalar variables, by
|
analyzing the loop-header PHIs of scalar variables; Classify each
|
analyzing the loop-header PHIs of scalar variables; Classify each
|
cycle as one of the following: invariant, induction, reduction, unknown.
|
cycle as one of the following: invariant, induction, reduction, unknown.
|
We do that for the loop represented by LOOP_VINFO, and also to its
|
We do that for the loop represented by LOOP_VINFO, and also to its
|
inner-loop, if exists.
|
inner-loop, if exists.
|
Examples for scalar cycles:
|
Examples for scalar cycles:
|
|
|
Example1: reduction:
|
Example1: reduction:
|
|
|
loop1:
|
loop1:
|
for (i=0; i<N; i++)
|
for (i=0; i<N; i++)
|
sum += a[i];
|
sum += a[i];
|
|
|
Example2: induction:
|
Example2: induction:
|
|
|
loop2:
|
loop2:
|
for (i=0; i<N; i++)
|
for (i=0; i<N; i++)
|
a[i] = i; */
|
a[i] = i; */
|
|
|
static void
|
static void
|
vect_analyze_scalar_cycles (loop_vec_info loop_vinfo)
|
vect_analyze_scalar_cycles (loop_vec_info loop_vinfo)
|
{
|
{
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
|
vect_analyze_scalar_cycles_1 (loop_vinfo, loop);
|
vect_analyze_scalar_cycles_1 (loop_vinfo, loop);
|
|
|
/* When vectorizing an outer-loop, the inner-loop is executed sequentially.
|
/* When vectorizing an outer-loop, the inner-loop is executed sequentially.
|
Reductions in such inner-loop therefore have different properties than
|
Reductions in such inner-loop therefore have different properties than
|
the reductions in the nest that gets vectorized:
|
the reductions in the nest that gets vectorized:
|
1. When vectorized, they are executed in the same order as in the original
|
1. When vectorized, they are executed in the same order as in the original
|
scalar loop, so we can't change the order of computation when
|
scalar loop, so we can't change the order of computation when
|
vectorizing them.
|
vectorizing them.
|
2. FIXME: Inner-loop reductions can be used in the inner-loop, so the
|
2. FIXME: Inner-loop reductions can be used in the inner-loop, so the
|
current checks are too strict. */
|
current checks are too strict. */
|
|
|
if (loop->inner)
|
if (loop->inner)
|
vect_analyze_scalar_cycles_1 (loop_vinfo, loop->inner);
|
vect_analyze_scalar_cycles_1 (loop_vinfo, loop->inner);
|
}
|
}
|
|
|
/* Function vect_get_loop_niters.
|
/* Function vect_get_loop_niters.
|
|
|
Determine how many iterations the loop is executed.
|
Determine how many iterations the loop is executed.
|
If an expression that represents the number of iterations
|
If an expression that represents the number of iterations
|
can be constructed, place it in NUMBER_OF_ITERATIONS.
|
can be constructed, place it in NUMBER_OF_ITERATIONS.
|
Return the loop exit condition. */
|
Return the loop exit condition. */
|
|
|
static gimple
|
static gimple
|
vect_get_loop_niters (struct loop *loop, tree *number_of_iterations)
|
vect_get_loop_niters (struct loop *loop, tree *number_of_iterations)
|
{
|
{
|
tree niters;
|
tree niters;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "=== get_loop_niters ===");
|
fprintf (vect_dump, "=== get_loop_niters ===");
|
|
|
niters = number_of_exit_cond_executions (loop);
|
niters = number_of_exit_cond_executions (loop);
|
|
|
if (niters != NULL_TREE
|
if (niters != NULL_TREE
|
&& niters != chrec_dont_know)
|
&& niters != chrec_dont_know)
|
{
|
{
|
*number_of_iterations = niters;
|
*number_of_iterations = niters;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "==> get_loop_niters:" );
|
fprintf (vect_dump, "==> get_loop_niters:" );
|
print_generic_expr (vect_dump, *number_of_iterations, TDF_SLIM);
|
print_generic_expr (vect_dump, *number_of_iterations, TDF_SLIM);
|
}
|
}
|
}
|
}
|
|
|
return get_loop_exit_condition (loop);
|
return get_loop_exit_condition (loop);
|
}
|
}
|
|
|
|
|
/* Function bb_in_loop_p
|
/* Function bb_in_loop_p
|
|
|
Used as predicate for dfs order traversal of the loop bbs. */
|
Used as predicate for dfs order traversal of the loop bbs. */
|
|
|
static bool
|
static bool
|
bb_in_loop_p (const_basic_block bb, const void *data)
|
bb_in_loop_p (const_basic_block bb, const void *data)
|
{
|
{
|
const struct loop *const loop = (const struct loop *)data;
|
const struct loop *const loop = (const struct loop *)data;
|
if (flow_bb_inside_loop_p (loop, bb))
|
if (flow_bb_inside_loop_p (loop, bb))
|
return true;
|
return true;
|
return false;
|
return false;
|
}
|
}
|
|
|
|
|
/* Function new_loop_vec_info.
|
/* Function new_loop_vec_info.
|
|
|
Create and initialize a new loop_vec_info struct for LOOP, as well as
|
Create and initialize a new loop_vec_info struct for LOOP, as well as
|
stmt_vec_info structs for all the stmts in LOOP. */
|
stmt_vec_info structs for all the stmts in LOOP. */
|
|
|
static loop_vec_info
|
static loop_vec_info
|
new_loop_vec_info (struct loop *loop)
|
new_loop_vec_info (struct loop *loop)
|
{
|
{
|
loop_vec_info res;
|
loop_vec_info res;
|
basic_block *bbs;
|
basic_block *bbs;
|
gimple_stmt_iterator si;
|
gimple_stmt_iterator si;
|
unsigned int i, nbbs;
|
unsigned int i, nbbs;
|
|
|
res = (loop_vec_info) xcalloc (1, sizeof (struct _loop_vec_info));
|
res = (loop_vec_info) xcalloc (1, sizeof (struct _loop_vec_info));
|
LOOP_VINFO_LOOP (res) = loop;
|
LOOP_VINFO_LOOP (res) = loop;
|
|
|
bbs = get_loop_body (loop);
|
bbs = get_loop_body (loop);
|
|
|
/* Create/Update stmt_info for all stmts in the loop. */
|
/* Create/Update stmt_info for all stmts in the loop. */
|
for (i = 0; i < loop->num_nodes; i++)
|
for (i = 0; i < loop->num_nodes; i++)
|
{
|
{
|
basic_block bb = bbs[i];
|
basic_block bb = bbs[i];
|
|
|
/* BBs in a nested inner-loop will have been already processed (because
|
/* BBs in a nested inner-loop will have been already processed (because
|
we will have called vect_analyze_loop_form for any nested inner-loop).
|
we will have called vect_analyze_loop_form for any nested inner-loop).
|
Therefore, for stmts in an inner-loop we just want to update the
|
Therefore, for stmts in an inner-loop we just want to update the
|
STMT_VINFO_LOOP_VINFO field of their stmt_info to point to the new
|
STMT_VINFO_LOOP_VINFO field of their stmt_info to point to the new
|
loop_info of the outer-loop we are currently considering to vectorize
|
loop_info of the outer-loop we are currently considering to vectorize
|
(instead of the loop_info of the inner-loop).
|
(instead of the loop_info of the inner-loop).
|
For stmts in other BBs we need to create a stmt_info from scratch. */
|
For stmts in other BBs we need to create a stmt_info from scratch. */
|
if (bb->loop_father != loop)
|
if (bb->loop_father != loop)
|
{
|
{
|
/* Inner-loop bb. */
|
/* Inner-loop bb. */
|
gcc_assert (loop->inner && bb->loop_father == loop->inner);
|
gcc_assert (loop->inner && bb->loop_father == loop->inner);
|
for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
|
for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
|
{
|
{
|
gimple phi = gsi_stmt (si);
|
gimple phi = gsi_stmt (si);
|
stmt_vec_info stmt_info = vinfo_for_stmt (phi);
|
stmt_vec_info stmt_info = vinfo_for_stmt (phi);
|
loop_vec_info inner_loop_vinfo =
|
loop_vec_info inner_loop_vinfo =
|
STMT_VINFO_LOOP_VINFO (stmt_info);
|
STMT_VINFO_LOOP_VINFO (stmt_info);
|
gcc_assert (loop->inner == LOOP_VINFO_LOOP (inner_loop_vinfo));
|
gcc_assert (loop->inner == LOOP_VINFO_LOOP (inner_loop_vinfo));
|
STMT_VINFO_LOOP_VINFO (stmt_info) = res;
|
STMT_VINFO_LOOP_VINFO (stmt_info) = res;
|
}
|
}
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
{
|
{
|
gimple stmt = gsi_stmt (si);
|
gimple stmt = gsi_stmt (si);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
loop_vec_info inner_loop_vinfo =
|
loop_vec_info inner_loop_vinfo =
|
STMT_VINFO_LOOP_VINFO (stmt_info);
|
STMT_VINFO_LOOP_VINFO (stmt_info);
|
gcc_assert (loop->inner == LOOP_VINFO_LOOP (inner_loop_vinfo));
|
gcc_assert (loop->inner == LOOP_VINFO_LOOP (inner_loop_vinfo));
|
STMT_VINFO_LOOP_VINFO (stmt_info) = res;
|
STMT_VINFO_LOOP_VINFO (stmt_info) = res;
|
}
|
}
|
}
|
}
|
else
|
else
|
{
|
{
|
/* bb in current nest. */
|
/* bb in current nest. */
|
for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
|
for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
|
{
|
{
|
gimple phi = gsi_stmt (si);
|
gimple phi = gsi_stmt (si);
|
gimple_set_uid (phi, 0);
|
gimple_set_uid (phi, 0);
|
set_vinfo_for_stmt (phi, new_stmt_vec_info (phi, res, NULL));
|
set_vinfo_for_stmt (phi, new_stmt_vec_info (phi, res, NULL));
|
}
|
}
|
|
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
{
|
{
|
gimple stmt = gsi_stmt (si);
|
gimple stmt = gsi_stmt (si);
|
gimple_set_uid (stmt, 0);
|
gimple_set_uid (stmt, 0);
|
set_vinfo_for_stmt (stmt, new_stmt_vec_info (stmt, res, NULL));
|
set_vinfo_for_stmt (stmt, new_stmt_vec_info (stmt, res, NULL));
|
}
|
}
|
}
|
}
|
}
|
}
|
|
|
/* CHECKME: We want to visit all BBs before their successors (except for
|
/* CHECKME: We want to visit all BBs before their successors (except for
|
latch blocks, for which this assertion wouldn't hold). In the simple
|
latch blocks, for which this assertion wouldn't hold). In the simple
|
case of the loop forms we allow, a dfs order of the BBs would the same
|
case of the loop forms we allow, a dfs order of the BBs would the same
|
as reversed postorder traversal, so we are safe. */
|
as reversed postorder traversal, so we are safe. */
|
|
|
free (bbs);
|
free (bbs);
|
bbs = XCNEWVEC (basic_block, loop->num_nodes);
|
bbs = XCNEWVEC (basic_block, loop->num_nodes);
|
nbbs = dfs_enumerate_from (loop->header, 0, bb_in_loop_p,
|
nbbs = dfs_enumerate_from (loop->header, 0, bb_in_loop_p,
|
bbs, loop->num_nodes, loop);
|
bbs, loop->num_nodes, loop);
|
gcc_assert (nbbs == loop->num_nodes);
|
gcc_assert (nbbs == loop->num_nodes);
|
|
|
LOOP_VINFO_BBS (res) = bbs;
|
LOOP_VINFO_BBS (res) = bbs;
|
LOOP_VINFO_NITERS (res) = NULL;
|
LOOP_VINFO_NITERS (res) = NULL;
|
LOOP_VINFO_NITERS_UNCHANGED (res) = NULL;
|
LOOP_VINFO_NITERS_UNCHANGED (res) = NULL;
|
LOOP_VINFO_COST_MODEL_MIN_ITERS (res) = 0;
|
LOOP_VINFO_COST_MODEL_MIN_ITERS (res) = 0;
|
LOOP_VINFO_VECTORIZABLE_P (res) = 0;
|
LOOP_VINFO_VECTORIZABLE_P (res) = 0;
|
LOOP_PEELING_FOR_ALIGNMENT (res) = 0;
|
LOOP_PEELING_FOR_ALIGNMENT (res) = 0;
|
LOOP_VINFO_VECT_FACTOR (res) = 0;
|
LOOP_VINFO_VECT_FACTOR (res) = 0;
|
LOOP_VINFO_DATAREFS (res) = VEC_alloc (data_reference_p, heap, 10);
|
LOOP_VINFO_DATAREFS (res) = VEC_alloc (data_reference_p, heap, 10);
|
LOOP_VINFO_DDRS (res) = VEC_alloc (ddr_p, heap, 10 * 10);
|
LOOP_VINFO_DDRS (res) = VEC_alloc (ddr_p, heap, 10 * 10);
|
LOOP_VINFO_UNALIGNED_DR (res) = NULL;
|
LOOP_VINFO_UNALIGNED_DR (res) = NULL;
|
LOOP_VINFO_MAY_MISALIGN_STMTS (res) =
|
LOOP_VINFO_MAY_MISALIGN_STMTS (res) =
|
VEC_alloc (gimple, heap,
|
VEC_alloc (gimple, heap,
|
PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIGNMENT_CHECKS));
|
PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIGNMENT_CHECKS));
|
LOOP_VINFO_MAY_ALIAS_DDRS (res) =
|
LOOP_VINFO_MAY_ALIAS_DDRS (res) =
|
VEC_alloc (ddr_p, heap,
|
VEC_alloc (ddr_p, heap,
|
PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS));
|
PARAM_VALUE (PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS));
|
LOOP_VINFO_STRIDED_STORES (res) = VEC_alloc (gimple, heap, 10);
|
LOOP_VINFO_STRIDED_STORES (res) = VEC_alloc (gimple, heap, 10);
|
LOOP_VINFO_SLP_INSTANCES (res) = VEC_alloc (slp_instance, heap, 10);
|
LOOP_VINFO_SLP_INSTANCES (res) = VEC_alloc (slp_instance, heap, 10);
|
LOOP_VINFO_SLP_UNROLLING_FACTOR (res) = 1;
|
LOOP_VINFO_SLP_UNROLLING_FACTOR (res) = 1;
|
|
|
return res;
|
return res;
|
}
|
}
|
|
|
|
|
/* Function destroy_loop_vec_info.
|
/* Function destroy_loop_vec_info.
|
|
|
Free LOOP_VINFO struct, as well as all the stmt_vec_info structs of all the
|
Free LOOP_VINFO struct, as well as all the stmt_vec_info structs of all the
|
stmts in the loop. */
|
stmts in the loop. */
|
|
|
void
|
void
|
destroy_loop_vec_info (loop_vec_info loop_vinfo, bool clean_stmts)
|
destroy_loop_vec_info (loop_vec_info loop_vinfo, bool clean_stmts)
|
{
|
{
|
struct loop *loop;
|
struct loop *loop;
|
basic_block *bbs;
|
basic_block *bbs;
|
int nbbs;
|
int nbbs;
|
gimple_stmt_iterator si;
|
gimple_stmt_iterator si;
|
int j;
|
int j;
|
VEC (slp_instance, heap) *slp_instances;
|
VEC (slp_instance, heap) *slp_instances;
|
slp_instance instance;
|
slp_instance instance;
|
|
|
if (!loop_vinfo)
|
if (!loop_vinfo)
|
return;
|
return;
|
|
|
loop = LOOP_VINFO_LOOP (loop_vinfo);
|
loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
|
bbs = LOOP_VINFO_BBS (loop_vinfo);
|
bbs = LOOP_VINFO_BBS (loop_vinfo);
|
nbbs = loop->num_nodes;
|
nbbs = loop->num_nodes;
|
|
|
if (!clean_stmts)
|
if (!clean_stmts)
|
{
|
{
|
free (LOOP_VINFO_BBS (loop_vinfo));
|
free (LOOP_VINFO_BBS (loop_vinfo));
|
free_data_refs (LOOP_VINFO_DATAREFS (loop_vinfo));
|
free_data_refs (LOOP_VINFO_DATAREFS (loop_vinfo));
|
free_dependence_relations (LOOP_VINFO_DDRS (loop_vinfo));
|
free_dependence_relations (LOOP_VINFO_DDRS (loop_vinfo));
|
VEC_free (gimple, heap, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo));
|
VEC_free (gimple, heap, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo));
|
|
|
free (loop_vinfo);
|
free (loop_vinfo);
|
loop->aux = NULL;
|
loop->aux = NULL;
|
return;
|
return;
|
}
|
}
|
|
|
for (j = 0; j < nbbs; j++)
|
for (j = 0; j < nbbs; j++)
|
{
|
{
|
basic_block bb = bbs[j];
|
basic_block bb = bbs[j];
|
for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
|
for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
|
free_stmt_vec_info (gsi_stmt (si));
|
free_stmt_vec_info (gsi_stmt (si));
|
|
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); )
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); )
|
{
|
{
|
gimple stmt = gsi_stmt (si);
|
gimple stmt = gsi_stmt (si);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
|
|
if (stmt_info)
|
if (stmt_info)
|
{
|
{
|
/* Check if this is a "pattern stmt" (introduced by the
|
/* Check if this is a "pattern stmt" (introduced by the
|
vectorizer during the pattern recognition pass). */
|
vectorizer during the pattern recognition pass). */
|
bool remove_stmt_p = false;
|
bool remove_stmt_p = false;
|
gimple orig_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
|
gimple orig_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
|
if (orig_stmt)
|
if (orig_stmt)
|
{
|
{
|
stmt_vec_info orig_stmt_info = vinfo_for_stmt (orig_stmt);
|
stmt_vec_info orig_stmt_info = vinfo_for_stmt (orig_stmt);
|
if (orig_stmt_info
|
if (orig_stmt_info
|
&& STMT_VINFO_IN_PATTERN_P (orig_stmt_info))
|
&& STMT_VINFO_IN_PATTERN_P (orig_stmt_info))
|
remove_stmt_p = true;
|
remove_stmt_p = true;
|
}
|
}
|
|
|
/* Free stmt_vec_info. */
|
/* Free stmt_vec_info. */
|
free_stmt_vec_info (stmt);
|
free_stmt_vec_info (stmt);
|
|
|
/* Remove dead "pattern stmts". */
|
/* Remove dead "pattern stmts". */
|
if (remove_stmt_p)
|
if (remove_stmt_p)
|
gsi_remove (&si, true);
|
gsi_remove (&si, true);
|
}
|
}
|
gsi_next (&si);
|
gsi_next (&si);
|
}
|
}
|
}
|
}
|
|
|
free (LOOP_VINFO_BBS (loop_vinfo));
|
free (LOOP_VINFO_BBS (loop_vinfo));
|
free_data_refs (LOOP_VINFO_DATAREFS (loop_vinfo));
|
free_data_refs (LOOP_VINFO_DATAREFS (loop_vinfo));
|
free_dependence_relations (LOOP_VINFO_DDRS (loop_vinfo));
|
free_dependence_relations (LOOP_VINFO_DDRS (loop_vinfo));
|
VEC_free (gimple, heap, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo));
|
VEC_free (gimple, heap, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo));
|
VEC_free (ddr_p, heap, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo));
|
VEC_free (ddr_p, heap, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo));
|
slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
|
slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
|
for (j = 0; VEC_iterate (slp_instance, slp_instances, j, instance); j++)
|
for (j = 0; VEC_iterate (slp_instance, slp_instances, j, instance); j++)
|
vect_free_slp_instance (instance);
|
vect_free_slp_instance (instance);
|
|
|
VEC_free (slp_instance, heap, LOOP_VINFO_SLP_INSTANCES (loop_vinfo));
|
VEC_free (slp_instance, heap, LOOP_VINFO_SLP_INSTANCES (loop_vinfo));
|
VEC_free (gimple, heap, LOOP_VINFO_STRIDED_STORES (loop_vinfo));
|
VEC_free (gimple, heap, LOOP_VINFO_STRIDED_STORES (loop_vinfo));
|
|
|
free (loop_vinfo);
|
free (loop_vinfo);
|
loop->aux = NULL;
|
loop->aux = NULL;
|
}
|
}
|
|
|
|
|
/* Function vect_analyze_loop_1.
|
/* Function vect_analyze_loop_1.
|
|
|
Apply a set of analyses on LOOP, and create a loop_vec_info struct
|
Apply a set of analyses on LOOP, and create a loop_vec_info struct
|
for it. The different analyses will record information in the
|
for it. The different analyses will record information in the
|
loop_vec_info struct. This is a subset of the analyses applied in
|
loop_vec_info struct. This is a subset of the analyses applied in
|
vect_analyze_loop, to be applied on an inner-loop nested in the loop
|
vect_analyze_loop, to be applied on an inner-loop nested in the loop
|
that is now considered for (outer-loop) vectorization. */
|
that is now considered for (outer-loop) vectorization. */
|
|
|
static loop_vec_info
|
static loop_vec_info
|
vect_analyze_loop_1 (struct loop *loop)
|
vect_analyze_loop_1 (struct loop *loop)
|
{
|
{
|
loop_vec_info loop_vinfo;
|
loop_vec_info loop_vinfo;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "===== analyze_loop_nest_1 =====");
|
fprintf (vect_dump, "===== analyze_loop_nest_1 =====");
|
|
|
/* Check the CFG characteristics of the loop (nesting, entry/exit, etc. */
|
/* Check the CFG characteristics of the loop (nesting, entry/exit, etc. */
|
|
|
loop_vinfo = vect_analyze_loop_form (loop);
|
loop_vinfo = vect_analyze_loop_form (loop);
|
if (!loop_vinfo)
|
if (!loop_vinfo)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "bad inner-loop form.");
|
fprintf (vect_dump, "bad inner-loop form.");
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
return loop_vinfo;
|
return loop_vinfo;
|
}
|
}
|
|
|
|
|
/* Function vect_analyze_loop_form.
|
/* Function vect_analyze_loop_form.
|
|
|
Verify that certain CFG restrictions hold, including:
|
Verify that certain CFG restrictions hold, including:
|
- the loop has a pre-header
|
- the loop has a pre-header
|
- the loop has a single entry and exit
|
- the loop has a single entry and exit
|
- the loop exit condition is simple enough, and the number of iterations
|
- the loop exit condition is simple enough, and the number of iterations
|
can be analyzed (a countable loop). */
|
can be analyzed (a countable loop). */
|
|
|
loop_vec_info
|
loop_vec_info
|
vect_analyze_loop_form (struct loop *loop)
|
vect_analyze_loop_form (struct loop *loop)
|
{
|
{
|
loop_vec_info loop_vinfo;
|
loop_vec_info loop_vinfo;
|
gimple loop_cond;
|
gimple loop_cond;
|
tree number_of_iterations = NULL;
|
tree number_of_iterations = NULL;
|
loop_vec_info inner_loop_vinfo = NULL;
|
loop_vec_info inner_loop_vinfo = NULL;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "=== vect_analyze_loop_form ===");
|
fprintf (vect_dump, "=== vect_analyze_loop_form ===");
|
|
|
/* Different restrictions apply when we are considering an inner-most loop,
|
/* Different restrictions apply when we are considering an inner-most loop,
|
vs. an outer (nested) loop.
|
vs. an outer (nested) loop.
|
(FORNOW. May want to relax some of these restrictions in the future). */
|
(FORNOW. May want to relax some of these restrictions in the future). */
|
|
|
if (!loop->inner)
|
if (!loop->inner)
|
{
|
{
|
/* Inner-most loop. We currently require that the number of BBs is
|
/* Inner-most loop. We currently require that the number of BBs is
|
exactly 2 (the header and latch). Vectorizable inner-most loops
|
exactly 2 (the header and latch). Vectorizable inner-most loops
|
look like this:
|
look like this:
|
|
|
(pre-header)
|
(pre-header)
|
|
|
|
|
header <--------+
|
header <--------+
|
| | |
|
| | |
|
| +--> latch --+
|
| +--> latch --+
|
|
|
|
|
(exit-bb) */
|
(exit-bb) */
|
|
|
if (loop->num_nodes != 2)
|
if (loop->num_nodes != 2)
|
{
|
{
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
fprintf (vect_dump, "not vectorized: control flow in loop.");
|
fprintf (vect_dump, "not vectorized: control flow in loop.");
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (empty_block_p (loop->header))
|
if (empty_block_p (loop->header))
|
{
|
{
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
fprintf (vect_dump, "not vectorized: empty loop.");
|
fprintf (vect_dump, "not vectorized: empty loop.");
|
return NULL;
|
return NULL;
|
}
|
}
|
}
|
}
|
else
|
else
|
{
|
{
|
struct loop *innerloop = loop->inner;
|
struct loop *innerloop = loop->inner;
|
edge entryedge;
|
edge entryedge;
|
|
|
/* Nested loop. We currently require that the loop is doubly-nested,
|
/* Nested loop. We currently require that the loop is doubly-nested,
|
contains a single inner loop, and the number of BBs is exactly 5.
|
contains a single inner loop, and the number of BBs is exactly 5.
|
Vectorizable outer-loops look like this:
|
Vectorizable outer-loops look like this:
|
|
|
(pre-header)
|
(pre-header)
|
|
|
|
|
header <---+
|
header <---+
|
| |
|
| |
|
inner-loop |
|
inner-loop |
|
| |
|
| |
|
tail ------+
|
tail ------+
|
|
|
|
|
(exit-bb)
|
(exit-bb)
|
|
|
The inner-loop has the properties expected of inner-most loops
|
The inner-loop has the properties expected of inner-most loops
|
as described above. */
|
as described above. */
|
|
|
if ((loop->inner)->inner || (loop->inner)->next)
|
if ((loop->inner)->inner || (loop->inner)->next)
|
{
|
{
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
fprintf (vect_dump, "not vectorized: multiple nested loops.");
|
fprintf (vect_dump, "not vectorized: multiple nested loops.");
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Analyze the inner-loop. */
|
/* Analyze the inner-loop. */
|
inner_loop_vinfo = vect_analyze_loop_1 (loop->inner);
|
inner_loop_vinfo = vect_analyze_loop_1 (loop->inner);
|
if (!inner_loop_vinfo)
|
if (!inner_loop_vinfo)
|
{
|
{
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
fprintf (vect_dump, "not vectorized: Bad inner loop.");
|
fprintf (vect_dump, "not vectorized: Bad inner loop.");
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (!expr_invariant_in_loop_p (loop,
|
if (!expr_invariant_in_loop_p (loop,
|
LOOP_VINFO_NITERS (inner_loop_vinfo)))
|
LOOP_VINFO_NITERS (inner_loop_vinfo)))
|
{
|
{
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
fprintf (vect_dump,
|
fprintf (vect_dump,
|
"not vectorized: inner-loop count not invariant.");
|
"not vectorized: inner-loop count not invariant.");
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (loop->num_nodes != 5)
|
if (loop->num_nodes != 5)
|
{
|
{
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
fprintf (vect_dump, "not vectorized: control flow in loop.");
|
fprintf (vect_dump, "not vectorized: control flow in loop.");
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
gcc_assert (EDGE_COUNT (innerloop->header->preds) == 2);
|
gcc_assert (EDGE_COUNT (innerloop->header->preds) == 2);
|
entryedge = EDGE_PRED (innerloop->header, 0);
|
entryedge = EDGE_PRED (innerloop->header, 0);
|
if (EDGE_PRED (innerloop->header, 0)->src == innerloop->latch)
|
if (EDGE_PRED (innerloop->header, 0)->src == innerloop->latch)
|
entryedge = EDGE_PRED (innerloop->header, 1);
|
entryedge = EDGE_PRED (innerloop->header, 1);
|
|
|
if (entryedge->src != loop->header
|
if (entryedge->src != loop->header
|
|| !single_exit (innerloop)
|
|| !single_exit (innerloop)
|
|| single_exit (innerloop)->dest != EDGE_PRED (loop->latch, 0)->src)
|
|| single_exit (innerloop)->dest != EDGE_PRED (loop->latch, 0)->src)
|
{
|
{
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
fprintf (vect_dump, "not vectorized: unsupported outerloop form.");
|
fprintf (vect_dump, "not vectorized: unsupported outerloop form.");
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "Considering outer-loop vectorization.");
|
fprintf (vect_dump, "Considering outer-loop vectorization.");
|
}
|
}
|
|
|
if (!single_exit (loop)
|
if (!single_exit (loop)
|
|| EDGE_COUNT (loop->header->preds) != 2)
|
|| EDGE_COUNT (loop->header->preds) != 2)
|
{
|
{
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
{
|
{
|
if (!single_exit (loop))
|
if (!single_exit (loop))
|
fprintf (vect_dump, "not vectorized: multiple exits.");
|
fprintf (vect_dump, "not vectorized: multiple exits.");
|
else if (EDGE_COUNT (loop->header->preds) != 2)
|
else if (EDGE_COUNT (loop->header->preds) != 2)
|
fprintf (vect_dump, "not vectorized: too many incoming edges.");
|
fprintf (vect_dump, "not vectorized: too many incoming edges.");
|
}
|
}
|
if (inner_loop_vinfo)
|
if (inner_loop_vinfo)
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* We assume that the loop exit condition is at the end of the loop. i.e,
|
/* We assume that the loop exit condition is at the end of the loop. i.e,
|
that the loop is represented as a do-while (with a proper if-guard
|
that the loop is represented as a do-while (with a proper if-guard
|
before the loop if needed), where the loop header contains all the
|
before the loop if needed), where the loop header contains all the
|
executable statements, and the latch is empty. */
|
executable statements, and the latch is empty. */
|
if (!empty_block_p (loop->latch)
|
if (!empty_block_p (loop->latch)
|
|| !gimple_seq_empty_p (phi_nodes (loop->latch)))
|
|| !gimple_seq_empty_p (phi_nodes (loop->latch)))
|
{
|
{
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
fprintf (vect_dump, "not vectorized: unexpected loop form.");
|
fprintf (vect_dump, "not vectorized: unexpected loop form.");
|
if (inner_loop_vinfo)
|
if (inner_loop_vinfo)
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Make sure there exists a single-predecessor exit bb: */
|
/* Make sure there exists a single-predecessor exit bb: */
|
if (!single_pred_p (single_exit (loop)->dest))
|
if (!single_pred_p (single_exit (loop)->dest))
|
{
|
{
|
edge e = single_exit (loop);
|
edge e = single_exit (loop);
|
if (!(e->flags & EDGE_ABNORMAL))
|
if (!(e->flags & EDGE_ABNORMAL))
|
{
|
{
|
split_loop_exit_edge (e);
|
split_loop_exit_edge (e);
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "split exit edge.");
|
fprintf (vect_dump, "split exit edge.");
|
}
|
}
|
else
|
else
|
{
|
{
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
fprintf (vect_dump, "not vectorized: abnormal loop exit edge.");
|
fprintf (vect_dump, "not vectorized: abnormal loop exit edge.");
|
if (inner_loop_vinfo)
|
if (inner_loop_vinfo)
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
}
|
}
|
|
|
loop_cond = vect_get_loop_niters (loop, &number_of_iterations);
|
loop_cond = vect_get_loop_niters (loop, &number_of_iterations);
|
if (!loop_cond)
|
if (!loop_cond)
|
{
|
{
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
fprintf (vect_dump, "not vectorized: complicated exit condition.");
|
fprintf (vect_dump, "not vectorized: complicated exit condition.");
|
if (inner_loop_vinfo)
|
if (inner_loop_vinfo)
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (!number_of_iterations)
|
if (!number_of_iterations)
|
{
|
{
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
fprintf (vect_dump,
|
fprintf (vect_dump,
|
"not vectorized: number of iterations cannot be computed.");
|
"not vectorized: number of iterations cannot be computed.");
|
if (inner_loop_vinfo)
|
if (inner_loop_vinfo)
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (chrec_contains_undetermined (number_of_iterations))
|
if (chrec_contains_undetermined (number_of_iterations))
|
{
|
{
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
if (vect_print_dump_info (REPORT_BAD_FORM_LOOPS))
|
fprintf (vect_dump, "Infinite number of iterations.");
|
fprintf (vect_dump, "Infinite number of iterations.");
|
if (inner_loop_vinfo)
|
if (inner_loop_vinfo)
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
destroy_loop_vec_info (inner_loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (!NITERS_KNOWN_P (number_of_iterations))
|
if (!NITERS_KNOWN_P (number_of_iterations))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "Symbolic number of iterations is ");
|
fprintf (vect_dump, "Symbolic number of iterations is ");
|
print_generic_expr (vect_dump, number_of_iterations, TDF_DETAILS);
|
print_generic_expr (vect_dump, number_of_iterations, TDF_DETAILS);
|
}
|
}
|
}
|
}
|
else if (TREE_INT_CST_LOW (number_of_iterations) == 0)
|
else if (TREE_INT_CST_LOW (number_of_iterations) == 0)
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: number of iterations = 0.");
|
fprintf (vect_dump, "not vectorized: number of iterations = 0.");
|
if (inner_loop_vinfo)
|
if (inner_loop_vinfo)
|
destroy_loop_vec_info (inner_loop_vinfo, false);
|
destroy_loop_vec_info (inner_loop_vinfo, false);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
loop_vinfo = new_loop_vec_info (loop);
|
loop_vinfo = new_loop_vec_info (loop);
|
LOOP_VINFO_NITERS (loop_vinfo) = number_of_iterations;
|
LOOP_VINFO_NITERS (loop_vinfo) = number_of_iterations;
|
LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo) = number_of_iterations;
|
LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo) = number_of_iterations;
|
|
|
STMT_VINFO_TYPE (vinfo_for_stmt (loop_cond)) = loop_exit_ctrl_vec_info_type;
|
STMT_VINFO_TYPE (vinfo_for_stmt (loop_cond)) = loop_exit_ctrl_vec_info_type;
|
|
|
/* CHECKME: May want to keep it around it in the future. */
|
/* CHECKME: May want to keep it around it in the future. */
|
if (inner_loop_vinfo)
|
if (inner_loop_vinfo)
|
destroy_loop_vec_info (inner_loop_vinfo, false);
|
destroy_loop_vec_info (inner_loop_vinfo, false);
|
|
|
gcc_assert (!loop->aux);
|
gcc_assert (!loop->aux);
|
loop->aux = loop_vinfo;
|
loop->aux = loop_vinfo;
|
return loop_vinfo;
|
return loop_vinfo;
|
}
|
}
|
|
|
|
|
/* Function vect_analyze_loop_operations.
|
/* Function vect_analyze_loop_operations.
|
|
|
Scan the loop stmts and make sure they are all vectorizable. */
|
Scan the loop stmts and make sure they are all vectorizable. */
|
|
|
static bool
|
static bool
|
vect_analyze_loop_operations (loop_vec_info loop_vinfo)
|
vect_analyze_loop_operations (loop_vec_info loop_vinfo)
|
{
|
{
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
|
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
|
int nbbs = loop->num_nodes;
|
int nbbs = loop->num_nodes;
|
gimple_stmt_iterator si;
|
gimple_stmt_iterator si;
|
unsigned int vectorization_factor = 0;
|
unsigned int vectorization_factor = 0;
|
int i;
|
int i;
|
gimple phi;
|
gimple phi;
|
stmt_vec_info stmt_info;
|
stmt_vec_info stmt_info;
|
bool need_to_vectorize = false;
|
bool need_to_vectorize = false;
|
int min_profitable_iters;
|
int min_profitable_iters;
|
int min_scalar_loop_bound;
|
int min_scalar_loop_bound;
|
unsigned int th;
|
unsigned int th;
|
bool only_slp_in_loop = true, ok;
|
bool only_slp_in_loop = true, ok;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "=== vect_analyze_loop_operations ===");
|
fprintf (vect_dump, "=== vect_analyze_loop_operations ===");
|
|
|
gcc_assert (LOOP_VINFO_VECT_FACTOR (loop_vinfo));
|
gcc_assert (LOOP_VINFO_VECT_FACTOR (loop_vinfo));
|
vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
|
|
for (i = 0; i < nbbs; i++)
|
for (i = 0; i < nbbs; i++)
|
{
|
{
|
basic_block bb = bbs[i];
|
basic_block bb = bbs[i];
|
|
|
for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
|
for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
|
{
|
{
|
phi = gsi_stmt (si);
|
phi = gsi_stmt (si);
|
ok = true;
|
ok = true;
|
|
|
stmt_info = vinfo_for_stmt (phi);
|
stmt_info = vinfo_for_stmt (phi);
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "examining phi: ");
|
fprintf (vect_dump, "examining phi: ");
|
print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
|
}
|
}
|
|
|
if (! is_loop_header_bb_p (bb))
|
if (! is_loop_header_bb_p (bb))
|
{
|
{
|
/* inner-loop loop-closed exit phi in outer-loop vectorization
|
/* inner-loop loop-closed exit phi in outer-loop vectorization
|
(i.e. a phi in the tail of the outer-loop).
|
(i.e. a phi in the tail of the outer-loop).
|
FORNOW: we currently don't support the case that these phis
|
FORNOW: we currently don't support the case that these phis
|
are not used in the outerloop (unless it is double reduction,
|
are not used in the outerloop (unless it is double reduction,
|
i.e., this phi is vect_reduction_def), cause this case
|
i.e., this phi is vect_reduction_def), cause this case
|
requires to actually do something here. */
|
requires to actually do something here. */
|
if ((!STMT_VINFO_RELEVANT_P (stmt_info)
|
if ((!STMT_VINFO_RELEVANT_P (stmt_info)
|
|| STMT_VINFO_LIVE_P (stmt_info))
|
|| STMT_VINFO_LIVE_P (stmt_info))
|
&& STMT_VINFO_DEF_TYPE (stmt_info)
|
&& STMT_VINFO_DEF_TYPE (stmt_info)
|
!= vect_double_reduction_def)
|
!= vect_double_reduction_def)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump,
|
fprintf (vect_dump,
|
"Unsupported loop-closed phi in outer-loop.");
|
"Unsupported loop-closed phi in outer-loop.");
|
return false;
|
return false;
|
}
|
}
|
continue;
|
continue;
|
}
|
}
|
|
|
gcc_assert (stmt_info);
|
gcc_assert (stmt_info);
|
|
|
if (STMT_VINFO_LIVE_P (stmt_info))
|
if (STMT_VINFO_LIVE_P (stmt_info))
|
{
|
{
|
/* FORNOW: not yet supported. */
|
/* FORNOW: not yet supported. */
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: value used after loop.");
|
fprintf (vect_dump, "not vectorized: value used after loop.");
|
return false;
|
return false;
|
}
|
}
|
|
|
if (STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_scope
|
if (STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_scope
|
&& STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def)
|
&& STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def)
|
{
|
{
|
/* A scalar-dependence cycle that we don't support. */
|
/* A scalar-dependence cycle that we don't support. */
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: scalar dependence cycle.");
|
fprintf (vect_dump, "not vectorized: scalar dependence cycle.");
|
return false;
|
return false;
|
}
|
}
|
|
|
if (STMT_VINFO_RELEVANT_P (stmt_info))
|
if (STMT_VINFO_RELEVANT_P (stmt_info))
|
{
|
{
|
need_to_vectorize = true;
|
need_to_vectorize = true;
|
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def)
|
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def)
|
ok = vectorizable_induction (phi, NULL, NULL);
|
ok = vectorizable_induction (phi, NULL, NULL);
|
}
|
}
|
|
|
if (!ok)
|
if (!ok)
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
{
|
{
|
fprintf (vect_dump,
|
fprintf (vect_dump,
|
"not vectorized: relevant phi not supported: ");
|
"not vectorized: relevant phi not supported: ");
|
print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
|
}
|
}
|
return false;
|
return false;
|
}
|
}
|
}
|
}
|
|
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
{
|
{
|
gimple stmt = gsi_stmt (si);
|
gimple stmt = gsi_stmt (si);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
|
|
gcc_assert (stmt_info);
|
gcc_assert (stmt_info);
|
|
|
if (!vect_analyze_stmt (stmt, &need_to_vectorize, NULL))
|
if (!vect_analyze_stmt (stmt, &need_to_vectorize, NULL))
|
return false;
|
return false;
|
|
|
if ((STMT_VINFO_RELEVANT_P (stmt_info)
|
if ((STMT_VINFO_RELEVANT_P (stmt_info)
|
|| VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
|
|| VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info)))
|
&& !PURE_SLP_STMT (stmt_info))
|
&& !PURE_SLP_STMT (stmt_info))
|
|
|
/* STMT needs both SLP and loop-based vectorization. */
|
/* STMT needs both SLP and loop-based vectorization. */
|
only_slp_in_loop = false;
|
only_slp_in_loop = false;
|
}
|
}
|
} /* bbs */
|
} /* bbs */
|
|
|
/* All operations in the loop are either irrelevant (deal with loop
|
/* All operations in the loop are either irrelevant (deal with loop
|
control, or dead), or only used outside the loop and can be moved
|
control, or dead), or only used outside the loop and can be moved
|
out of the loop (e.g. invariants, inductions). The loop can be
|
out of the loop (e.g. invariants, inductions). The loop can be
|
optimized away by scalar optimizations. We're better off not
|
optimized away by scalar optimizations. We're better off not
|
touching this loop. */
|
touching this loop. */
|
if (!need_to_vectorize)
|
if (!need_to_vectorize)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump,
|
fprintf (vect_dump,
|
"All the computation can be taken out of the loop.");
|
"All the computation can be taken out of the loop.");
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump,
|
fprintf (vect_dump,
|
"not vectorized: redundant loop. no profit to vectorize.");
|
"not vectorized: redundant loop. no profit to vectorize.");
|
return false;
|
return false;
|
}
|
}
|
|
|
/* If all the stmts in the loop can be SLPed, we perform only SLP, and
|
/* If all the stmts in the loop can be SLPed, we perform only SLP, and
|
vectorization factor of the loop is the unrolling factor required by the
|
vectorization factor of the loop is the unrolling factor required by the
|
SLP instances. If that unrolling factor is 1, we say, that we perform
|
SLP instances. If that unrolling factor is 1, we say, that we perform
|
pure SLP on loop - cross iteration parallelism is not exploited. */
|
pure SLP on loop - cross iteration parallelism is not exploited. */
|
if (only_slp_in_loop)
|
if (only_slp_in_loop)
|
vectorization_factor = LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo);
|
vectorization_factor = LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo);
|
else
|
else
|
vectorization_factor = least_common_multiple (vectorization_factor,
|
vectorization_factor = least_common_multiple (vectorization_factor,
|
LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo));
|
LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo));
|
|
|
LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
|
LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor;
|
|
|
if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
&& vect_print_dump_info (REPORT_DETAILS))
|
&& vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump,
|
fprintf (vect_dump,
|
"vectorization_factor = %d, niters = " HOST_WIDE_INT_PRINT_DEC,
|
"vectorization_factor = %d, niters = " HOST_WIDE_INT_PRINT_DEC,
|
vectorization_factor, LOOP_VINFO_INT_NITERS (loop_vinfo));
|
vectorization_factor, LOOP_VINFO_INT_NITERS (loop_vinfo));
|
|
|
if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
&& (LOOP_VINFO_INT_NITERS (loop_vinfo) < vectorization_factor))
|
&& (LOOP_VINFO_INT_NITERS (loop_vinfo) < vectorization_factor))
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: iteration count too small.");
|
fprintf (vect_dump, "not vectorized: iteration count too small.");
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump,"not vectorized: iteration count smaller than "
|
fprintf (vect_dump,"not vectorized: iteration count smaller than "
|
"vectorization factor.");
|
"vectorization factor.");
|
return false;
|
return false;
|
}
|
}
|
|
|
/* Analyze cost. Decide if worth while to vectorize. */
|
/* Analyze cost. Decide if worth while to vectorize. */
|
|
|
/* Once VF is set, SLP costs should be updated since the number of created
|
/* Once VF is set, SLP costs should be updated since the number of created
|
vector stmts depends on VF. */
|
vector stmts depends on VF. */
|
vect_update_slp_costs_according_to_vf (loop_vinfo);
|
vect_update_slp_costs_according_to_vf (loop_vinfo);
|
|
|
min_profitable_iters = vect_estimate_min_profitable_iters (loop_vinfo);
|
min_profitable_iters = vect_estimate_min_profitable_iters (loop_vinfo);
|
LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo) = min_profitable_iters;
|
LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo) = min_profitable_iters;
|
|
|
if (min_profitable_iters < 0)
|
if (min_profitable_iters < 0)
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: vectorization not profitable.");
|
fprintf (vect_dump, "not vectorized: vectorization not profitable.");
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "not vectorized: vector version will never be "
|
fprintf (vect_dump, "not vectorized: vector version will never be "
|
"profitable.");
|
"profitable.");
|
return false;
|
return false;
|
}
|
}
|
|
|
min_scalar_loop_bound = ((PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND)
|
min_scalar_loop_bound = ((PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND)
|
* vectorization_factor) - 1);
|
* vectorization_factor) - 1);
|
|
|
/* Use the cost model only if it is more conservative than user specified
|
/* Use the cost model only if it is more conservative than user specified
|
threshold. */
|
threshold. */
|
|
|
th = (unsigned) min_scalar_loop_bound;
|
th = (unsigned) min_scalar_loop_bound;
|
if (min_profitable_iters
|
if (min_profitable_iters
|
&& (!min_scalar_loop_bound
|
&& (!min_scalar_loop_bound
|
|| min_profitable_iters > min_scalar_loop_bound))
|
|| min_profitable_iters > min_scalar_loop_bound))
|
th = (unsigned) min_profitable_iters;
|
th = (unsigned) min_profitable_iters;
|
|
|
if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
&& LOOP_VINFO_INT_NITERS (loop_vinfo) <= th)
|
&& LOOP_VINFO_INT_NITERS (loop_vinfo) <= th)
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: vectorization not "
|
fprintf (vect_dump, "not vectorized: vectorization not "
|
"profitable.");
|
"profitable.");
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "not vectorized: iteration count smaller than "
|
fprintf (vect_dump, "not vectorized: iteration count smaller than "
|
"user specified loop bound parameter or minimum "
|
"user specified loop bound parameter or minimum "
|
"profitable iterations (whichever is more conservative).");
|
"profitable iterations (whichever is more conservative).");
|
return false;
|
return false;
|
}
|
}
|
|
|
if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
|| LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0
|
|| LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0
|
|| LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
|
|| LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "epilog loop required.");
|
fprintf (vect_dump, "epilog loop required.");
|
if (!vect_can_advance_ivs_p (loop_vinfo))
|
if (!vect_can_advance_ivs_p (loop_vinfo))
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump,
|
fprintf (vect_dump,
|
"not vectorized: can't create epilog loop 1.");
|
"not vectorized: can't create epilog loop 1.");
|
return false;
|
return false;
|
}
|
}
|
if (!slpeel_can_duplicate_loop_p (loop, single_exit (loop)))
|
if (!slpeel_can_duplicate_loop_p (loop, single_exit (loop)))
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump,
|
fprintf (vect_dump,
|
"not vectorized: can't create epilog loop 2.");
|
"not vectorized: can't create epilog loop 2.");
|
return false;
|
return false;
|
}
|
}
|
}
|
}
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
|
|
/* Function vect_analyze_loop.
|
/* Function vect_analyze_loop.
|
|
|
Apply a set of analyses on LOOP, and create a loop_vec_info struct
|
Apply a set of analyses on LOOP, and create a loop_vec_info struct
|
for it. The different analyses will record information in the
|
for it. The different analyses will record information in the
|
loop_vec_info struct. */
|
loop_vec_info struct. */
|
loop_vec_info
|
loop_vec_info
|
vect_analyze_loop (struct loop *loop)
|
vect_analyze_loop (struct loop *loop)
|
{
|
{
|
bool ok;
|
bool ok;
|
loop_vec_info loop_vinfo;
|
loop_vec_info loop_vinfo;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "===== analyze_loop_nest =====");
|
fprintf (vect_dump, "===== analyze_loop_nest =====");
|
|
|
if (loop_outer (loop)
|
if (loop_outer (loop)
|
&& loop_vec_info_for_loop (loop_outer (loop))
|
&& loop_vec_info_for_loop (loop_outer (loop))
|
&& LOOP_VINFO_VECTORIZABLE_P (loop_vec_info_for_loop (loop_outer (loop))))
|
&& LOOP_VINFO_VECTORIZABLE_P (loop_vec_info_for_loop (loop_outer (loop))))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "outer-loop already vectorized.");
|
fprintf (vect_dump, "outer-loop already vectorized.");
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Check the CFG characteristics of the loop (nesting, entry/exit, etc. */
|
/* Check the CFG characteristics of the loop (nesting, entry/exit, etc. */
|
|
|
loop_vinfo = vect_analyze_loop_form (loop);
|
loop_vinfo = vect_analyze_loop_form (loop);
|
if (!loop_vinfo)
|
if (!loop_vinfo)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "bad loop form.");
|
fprintf (vect_dump, "bad loop form.");
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Find all data references in the loop (which correspond to vdefs/vuses)
|
/* Find all data references in the loop (which correspond to vdefs/vuses)
|
and analyze their evolution in the loop.
|
and analyze their evolution in the loop.
|
|
|
FORNOW: Handle only simple, array references, which
|
FORNOW: Handle only simple, array references, which
|
alignment can be forced, and aligned pointer-references. */
|
alignment can be forced, and aligned pointer-references. */
|
|
|
ok = vect_analyze_data_refs (loop_vinfo, NULL);
|
ok = vect_analyze_data_refs (loop_vinfo, NULL);
|
if (!ok)
|
if (!ok)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "bad data references.");
|
fprintf (vect_dump, "bad data references.");
|
destroy_loop_vec_info (loop_vinfo, true);
|
destroy_loop_vec_info (loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Classify all cross-iteration scalar data-flow cycles.
|
/* Classify all cross-iteration scalar data-flow cycles.
|
Cross-iteration cycles caused by virtual phis are analyzed separately. */
|
Cross-iteration cycles caused by virtual phis are analyzed separately. */
|
|
|
vect_analyze_scalar_cycles (loop_vinfo);
|
vect_analyze_scalar_cycles (loop_vinfo);
|
|
|
vect_pattern_recog (loop_vinfo);
|
vect_pattern_recog (loop_vinfo);
|
|
|
/* Data-flow analysis to detect stmts that do not need to be vectorized. */
|
/* Data-flow analysis to detect stmts that do not need to be vectorized. */
|
|
|
ok = vect_mark_stmts_to_be_vectorized (loop_vinfo);
|
ok = vect_mark_stmts_to_be_vectorized (loop_vinfo);
|
if (!ok)
|
if (!ok)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "unexpected pattern.");
|
fprintf (vect_dump, "unexpected pattern.");
|
destroy_loop_vec_info (loop_vinfo, true);
|
destroy_loop_vec_info (loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Analyze the alignment of the data-refs in the loop.
|
/* Analyze the alignment of the data-refs in the loop.
|
Fail if a data reference is found that cannot be vectorized. */
|
Fail if a data reference is found that cannot be vectorized. */
|
|
|
ok = vect_analyze_data_refs_alignment (loop_vinfo, NULL);
|
ok = vect_analyze_data_refs_alignment (loop_vinfo, NULL);
|
if (!ok)
|
if (!ok)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "bad data alignment.");
|
fprintf (vect_dump, "bad data alignment.");
|
destroy_loop_vec_info (loop_vinfo, true);
|
destroy_loop_vec_info (loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
ok = vect_determine_vectorization_factor (loop_vinfo);
|
ok = vect_determine_vectorization_factor (loop_vinfo);
|
if (!ok)
|
if (!ok)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "can't determine vectorization factor.");
|
fprintf (vect_dump, "can't determine vectorization factor.");
|
destroy_loop_vec_info (loop_vinfo, true);
|
destroy_loop_vec_info (loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Analyze data dependences between the data-refs in the loop.
|
/* Analyze data dependences between the data-refs in the loop.
|
FORNOW: fail at the first data dependence that we encounter. */
|
FORNOW: fail at the first data dependence that we encounter. */
|
|
|
ok = vect_analyze_data_ref_dependences (loop_vinfo, NULL);
|
ok = vect_analyze_data_ref_dependences (loop_vinfo, NULL);
|
if (!ok)
|
if (!ok)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "bad data dependence.");
|
fprintf (vect_dump, "bad data dependence.");
|
destroy_loop_vec_info (loop_vinfo, true);
|
destroy_loop_vec_info (loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Analyze the access patterns of the data-refs in the loop (consecutive,
|
/* Analyze the access patterns of the data-refs in the loop (consecutive,
|
complex, etc.). FORNOW: Only handle consecutive access pattern. */
|
complex, etc.). FORNOW: Only handle consecutive access pattern. */
|
|
|
ok = vect_analyze_data_ref_accesses (loop_vinfo, NULL);
|
ok = vect_analyze_data_ref_accesses (loop_vinfo, NULL);
|
if (!ok)
|
if (!ok)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "bad data access.");
|
fprintf (vect_dump, "bad data access.");
|
destroy_loop_vec_info (loop_vinfo, true);
|
destroy_loop_vec_info (loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Prune the list of ddrs to be tested at run-time by versioning for alias.
|
/* Prune the list of ddrs to be tested at run-time by versioning for alias.
|
It is important to call pruning after vect_analyze_data_ref_accesses,
|
It is important to call pruning after vect_analyze_data_ref_accesses,
|
since we use grouping information gathered by interleaving analysis. */
|
since we use grouping information gathered by interleaving analysis. */
|
ok = vect_prune_runtime_alias_test_list (loop_vinfo);
|
ok = vect_prune_runtime_alias_test_list (loop_vinfo);
|
if (!ok)
|
if (!ok)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "too long list of versioning for alias "
|
fprintf (vect_dump, "too long list of versioning for alias "
|
"run-time tests.");
|
"run-time tests.");
|
destroy_loop_vec_info (loop_vinfo, true);
|
destroy_loop_vec_info (loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Check the SLP opportunities in the loop, analyze and build SLP trees. */
|
/* Check the SLP opportunities in the loop, analyze and build SLP trees. */
|
ok = vect_analyze_slp (loop_vinfo, NULL);
|
ok = vect_analyze_slp (loop_vinfo, NULL);
|
if (ok)
|
if (ok)
|
{
|
{
|
/* Decide which possible SLP instances to SLP. */
|
/* Decide which possible SLP instances to SLP. */
|
vect_make_slp_decision (loop_vinfo);
|
vect_make_slp_decision (loop_vinfo);
|
|
|
/* Find stmts that need to be both vectorized and SLPed. */
|
/* Find stmts that need to be both vectorized and SLPed. */
|
vect_detect_hybrid_slp (loop_vinfo);
|
vect_detect_hybrid_slp (loop_vinfo);
|
}
|
}
|
|
|
/* This pass will decide on using loop versioning and/or loop peeling in
|
/* This pass will decide on using loop versioning and/or loop peeling in
|
order to enhance the alignment of data references in the loop. */
|
order to enhance the alignment of data references in the loop. */
|
|
|
ok = vect_enhance_data_refs_alignment (loop_vinfo);
|
ok = vect_enhance_data_refs_alignment (loop_vinfo);
|
if (!ok)
|
if (!ok)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "bad data alignment.");
|
fprintf (vect_dump, "bad data alignment.");
|
destroy_loop_vec_info (loop_vinfo, true);
|
destroy_loop_vec_info (loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Scan all the operations in the loop and make sure they are
|
/* Scan all the operations in the loop and make sure they are
|
vectorizable. */
|
vectorizable. */
|
|
|
ok = vect_analyze_loop_operations (loop_vinfo);
|
ok = vect_analyze_loop_operations (loop_vinfo);
|
if (!ok)
|
if (!ok)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "bad operation or unsupported loop bound.");
|
fprintf (vect_dump, "bad operation or unsupported loop bound.");
|
destroy_loop_vec_info (loop_vinfo, true);
|
destroy_loop_vec_info (loop_vinfo, true);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
LOOP_VINFO_VECTORIZABLE_P (loop_vinfo) = 1;
|
LOOP_VINFO_VECTORIZABLE_P (loop_vinfo) = 1;
|
|
|
return loop_vinfo;
|
return loop_vinfo;
|
}
|
}
|
|
|
|
|
/* Function reduction_code_for_scalar_code
|
/* Function reduction_code_for_scalar_code
|
|
|
Input:
|
Input:
|
CODE - tree_code of a reduction operations.
|
CODE - tree_code of a reduction operations.
|
|
|
Output:
|
Output:
|
REDUC_CODE - the corresponding tree-code to be used to reduce the
|
REDUC_CODE - the corresponding tree-code to be used to reduce the
|
vector of partial results into a single scalar result (which
|
vector of partial results into a single scalar result (which
|
will also reside in a vector) or ERROR_MARK if the operation is
|
will also reside in a vector) or ERROR_MARK if the operation is
|
a supported reduction operation, but does not have such tree-code.
|
a supported reduction operation, but does not have such tree-code.
|
|
|
Return FALSE if CODE currently cannot be vectorized as reduction. */
|
Return FALSE if CODE currently cannot be vectorized as reduction. */
|
|
|
static bool
|
static bool
|
reduction_code_for_scalar_code (enum tree_code code,
|
reduction_code_for_scalar_code (enum tree_code code,
|
enum tree_code *reduc_code)
|
enum tree_code *reduc_code)
|
{
|
{
|
switch (code)
|
switch (code)
|
{
|
{
|
case MAX_EXPR:
|
case MAX_EXPR:
|
*reduc_code = REDUC_MAX_EXPR;
|
*reduc_code = REDUC_MAX_EXPR;
|
return true;
|
return true;
|
|
|
case MIN_EXPR:
|
case MIN_EXPR:
|
*reduc_code = REDUC_MIN_EXPR;
|
*reduc_code = REDUC_MIN_EXPR;
|
return true;
|
return true;
|
|
|
case PLUS_EXPR:
|
case PLUS_EXPR:
|
*reduc_code = REDUC_PLUS_EXPR;
|
*reduc_code = REDUC_PLUS_EXPR;
|
return true;
|
return true;
|
|
|
case MULT_EXPR:
|
case MULT_EXPR:
|
case MINUS_EXPR:
|
case MINUS_EXPR:
|
case BIT_IOR_EXPR:
|
case BIT_IOR_EXPR:
|
case BIT_XOR_EXPR:
|
case BIT_XOR_EXPR:
|
case BIT_AND_EXPR:
|
case BIT_AND_EXPR:
|
*reduc_code = ERROR_MARK;
|
*reduc_code = ERROR_MARK;
|
return true;
|
return true;
|
|
|
default:
|
default:
|
return false;
|
return false;
|
}
|
}
|
}
|
}
|
|
|
|
|
/* Error reporting helper for vect_is_simple_reduction below. GIMPLE statement
|
/* Error reporting helper for vect_is_simple_reduction below. GIMPLE statement
|
STMT is printed with a message MSG. */
|
STMT is printed with a message MSG. */
|
|
|
static void
|
static void
|
report_vect_op (gimple stmt, const char *msg)
|
report_vect_op (gimple stmt, const char *msg)
|
{
|
{
|
fprintf (vect_dump, "%s", msg);
|
fprintf (vect_dump, "%s", msg);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
|
|
/* Function vect_is_simple_reduction
|
/* Function vect_is_simple_reduction
|
|
|
(1) Detect a cross-iteration def-use cycle that represents a simple
|
(1) Detect a cross-iteration def-use cycle that represents a simple
|
reduction computation. We look for the following pattern:
|
reduction computation. We look for the following pattern:
|
|
|
loop_header:
|
loop_header:
|
a1 = phi < a0, a2 >
|
a1 = phi < a0, a2 >
|
a3 = ...
|
a3 = ...
|
a2 = operation (a3, a1)
|
a2 = operation (a3, a1)
|
|
|
such that:
|
such that:
|
1. operation is commutative and associative and it is safe to
|
1. operation is commutative and associative and it is safe to
|
change the order of the computation (if CHECK_REDUCTION is true)
|
change the order of the computation (if CHECK_REDUCTION is true)
|
2. no uses for a2 in the loop (a2 is used out of the loop)
|
2. no uses for a2 in the loop (a2 is used out of the loop)
|
3. no uses of a1 in the loop besides the reduction operation.
|
3. no uses of a1 in the loop besides the reduction operation.
|
|
|
Condition 1 is tested here.
|
Condition 1 is tested here.
|
Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.
|
Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.
|
|
|
(2) Detect a cross-iteration def-use cycle in nested loops, i.e.,
|
(2) Detect a cross-iteration def-use cycle in nested loops, i.e.,
|
nested cycles, if CHECK_REDUCTION is false.
|
nested cycles, if CHECK_REDUCTION is false.
|
|
|
(3) Detect cycles of phi nodes in outer-loop vectorization, i.e., double
|
(3) Detect cycles of phi nodes in outer-loop vectorization, i.e., double
|
reductions:
|
reductions:
|
|
|
a1 = phi < a0, a2 >
|
a1 = phi < a0, a2 >
|
inner loop (def of a3)
|
inner loop (def of a3)
|
a2 = phi < a3 >
|
a2 = phi < a3 >
|
*/
|
*/
|
|
|
gimple
|
gimple
|
vect_is_simple_reduction (loop_vec_info loop_info, gimple phi,
|
vect_is_simple_reduction (loop_vec_info loop_info, gimple phi,
|
bool check_reduction, bool *double_reduc)
|
bool check_reduction, bool *double_reduc)
|
{
|
{
|
struct loop *loop = (gimple_bb (phi))->loop_father;
|
struct loop *loop = (gimple_bb (phi))->loop_father;
|
struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info);
|
struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info);
|
edge latch_e = loop_latch_edge (loop);
|
edge latch_e = loop_latch_edge (loop);
|
tree loop_arg = PHI_ARG_DEF_FROM_EDGE (phi, latch_e);
|
tree loop_arg = PHI_ARG_DEF_FROM_EDGE (phi, latch_e);
|
gimple def_stmt, def1 = NULL, def2 = NULL;
|
gimple def_stmt, def1 = NULL, def2 = NULL;
|
enum tree_code code;
|
enum tree_code code;
|
tree op1, op2, op3 = NULL_TREE, op4 = NULL_TREE;
|
tree op1, op2, op3 = NULL_TREE, op4 = NULL_TREE;
|
tree type;
|
tree type;
|
int nloop_uses;
|
int nloop_uses;
|
tree name;
|
tree name;
|
imm_use_iterator imm_iter;
|
imm_use_iterator imm_iter;
|
use_operand_p use_p;
|
use_operand_p use_p;
|
bool phi_def;
|
bool phi_def;
|
|
|
*double_reduc = false;
|
*double_reduc = false;
|
|
|
/* If CHECK_REDUCTION is true, we assume inner-most loop vectorization,
|
/* If CHECK_REDUCTION is true, we assume inner-most loop vectorization,
|
otherwise, we assume outer loop vectorization. */
|
otherwise, we assume outer loop vectorization. */
|
gcc_assert ((check_reduction && loop == vect_loop)
|
gcc_assert ((check_reduction && loop == vect_loop)
|
|| (!check_reduction && flow_loop_nested_p (vect_loop, loop)));
|
|| (!check_reduction && flow_loop_nested_p (vect_loop, loop)));
|
|
|
name = PHI_RESULT (phi);
|
name = PHI_RESULT (phi);
|
nloop_uses = 0;
|
nloop_uses = 0;
|
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, name)
|
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, name)
|
{
|
{
|
gimple use_stmt = USE_STMT (use_p);
|
gimple use_stmt = USE_STMT (use_p);
|
if (is_gimple_debug (use_stmt))
|
if (is_gimple_debug (use_stmt))
|
continue;
|
continue;
|
if (flow_bb_inside_loop_p (loop, gimple_bb (use_stmt))
|
if (flow_bb_inside_loop_p (loop, gimple_bb (use_stmt))
|
&& vinfo_for_stmt (use_stmt)
|
&& vinfo_for_stmt (use_stmt)
|
&& !is_pattern_stmt_p (vinfo_for_stmt (use_stmt)))
|
&& !is_pattern_stmt_p (vinfo_for_stmt (use_stmt)))
|
nloop_uses++;
|
nloop_uses++;
|
if (nloop_uses > 1)
|
if (nloop_uses > 1)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "reduction used in loop.");
|
fprintf (vect_dump, "reduction used in loop.");
|
return NULL;
|
return NULL;
|
}
|
}
|
}
|
}
|
|
|
if (TREE_CODE (loop_arg) != SSA_NAME)
|
if (TREE_CODE (loop_arg) != SSA_NAME)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "reduction: not ssa_name: ");
|
fprintf (vect_dump, "reduction: not ssa_name: ");
|
print_generic_expr (vect_dump, loop_arg, TDF_SLIM);
|
print_generic_expr (vect_dump, loop_arg, TDF_SLIM);
|
}
|
}
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
def_stmt = SSA_NAME_DEF_STMT (loop_arg);
|
def_stmt = SSA_NAME_DEF_STMT (loop_arg);
|
if (!def_stmt)
|
if (!def_stmt)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "reduction: no def_stmt.");
|
fprintf (vect_dump, "reduction: no def_stmt.");
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (!is_gimple_assign (def_stmt) && gimple_code (def_stmt) != GIMPLE_PHI)
|
if (!is_gimple_assign (def_stmt) && gimple_code (def_stmt) != GIMPLE_PHI)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
print_gimple_stmt (vect_dump, def_stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, def_stmt, 0, TDF_SLIM);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (is_gimple_assign (def_stmt))
|
if (is_gimple_assign (def_stmt))
|
{
|
{
|
name = gimple_assign_lhs (def_stmt);
|
name = gimple_assign_lhs (def_stmt);
|
phi_def = false;
|
phi_def = false;
|
}
|
}
|
else
|
else
|
{
|
{
|
name = PHI_RESULT (def_stmt);
|
name = PHI_RESULT (def_stmt);
|
phi_def = true;
|
phi_def = true;
|
}
|
}
|
|
|
nloop_uses = 0;
|
nloop_uses = 0;
|
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, name)
|
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, name)
|
{
|
{
|
gimple use_stmt = USE_STMT (use_p);
|
gimple use_stmt = USE_STMT (use_p);
|
if (is_gimple_debug (use_stmt))
|
if (is_gimple_debug (use_stmt))
|
continue;
|
continue;
|
if (flow_bb_inside_loop_p (loop, gimple_bb (use_stmt))
|
if (flow_bb_inside_loop_p (loop, gimple_bb (use_stmt))
|
&& vinfo_for_stmt (use_stmt)
|
&& vinfo_for_stmt (use_stmt)
|
&& !is_pattern_stmt_p (vinfo_for_stmt (use_stmt)))
|
&& !is_pattern_stmt_p (vinfo_for_stmt (use_stmt)))
|
nloop_uses++;
|
nloop_uses++;
|
if (nloop_uses > 1)
|
if (nloop_uses > 1)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "reduction used in loop.");
|
fprintf (vect_dump, "reduction used in loop.");
|
return NULL;
|
return NULL;
|
}
|
}
|
}
|
}
|
|
|
/* If DEF_STMT is a phi node itself, we expect it to have a single argument
|
/* If DEF_STMT is a phi node itself, we expect it to have a single argument
|
defined in the inner loop. */
|
defined in the inner loop. */
|
if (phi_def)
|
if (phi_def)
|
{
|
{
|
op1 = PHI_ARG_DEF (def_stmt, 0);
|
op1 = PHI_ARG_DEF (def_stmt, 0);
|
|
|
if (gimple_phi_num_args (def_stmt) != 1
|
if (gimple_phi_num_args (def_stmt) != 1
|
|| TREE_CODE (op1) != SSA_NAME)
|
|| TREE_CODE (op1) != SSA_NAME)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "unsupported phi node definition.");
|
fprintf (vect_dump, "unsupported phi node definition.");
|
|
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
def1 = SSA_NAME_DEF_STMT (op1);
|
def1 = SSA_NAME_DEF_STMT (op1);
|
if (flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
|
if (flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
|
&& loop->inner
|
&& loop->inner
|
&& flow_bb_inside_loop_p (loop->inner, gimple_bb (def1))
|
&& flow_bb_inside_loop_p (loop->inner, gimple_bb (def1))
|
&& is_gimple_assign (def1))
|
&& is_gimple_assign (def1))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
report_vect_op (def_stmt, "detected double reduction: ");
|
report_vect_op (def_stmt, "detected double reduction: ");
|
|
|
*double_reduc = true;
|
*double_reduc = true;
|
return def_stmt;
|
return def_stmt;
|
}
|
}
|
|
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
code = gimple_assign_rhs_code (def_stmt);
|
code = gimple_assign_rhs_code (def_stmt);
|
|
|
if (check_reduction
|
if (check_reduction
|
&& (!commutative_tree_code (code) || !associative_tree_code (code)))
|
&& (!commutative_tree_code (code) || !associative_tree_code (code)))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
report_vect_op (def_stmt, "reduction: not commutative/associative: ");
|
report_vect_op (def_stmt, "reduction: not commutative/associative: ");
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (get_gimple_rhs_class (code) != GIMPLE_BINARY_RHS)
|
if (get_gimple_rhs_class (code) != GIMPLE_BINARY_RHS)
|
{
|
{
|
if (code != COND_EXPR)
|
if (code != COND_EXPR)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
report_vect_op (def_stmt, "reduction: not binary operation: ");
|
report_vect_op (def_stmt, "reduction: not binary operation: ");
|
|
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
op3 = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0);
|
op3 = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0);
|
if (COMPARISON_CLASS_P (op3))
|
if (COMPARISON_CLASS_P (op3))
|
{
|
{
|
op4 = TREE_OPERAND (op3, 1);
|
op4 = TREE_OPERAND (op3, 1);
|
op3 = TREE_OPERAND (op3, 0);
|
op3 = TREE_OPERAND (op3, 0);
|
}
|
}
|
|
|
op1 = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 1);
|
op1 = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 1);
|
op2 = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 2);
|
op2 = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 2);
|
|
|
if (TREE_CODE (op1) != SSA_NAME && TREE_CODE (op2) != SSA_NAME)
|
if (TREE_CODE (op1) != SSA_NAME && TREE_CODE (op2) != SSA_NAME)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
report_vect_op (def_stmt, "reduction: uses not ssa_names: ");
|
report_vect_op (def_stmt, "reduction: uses not ssa_names: ");
|
|
|
return NULL;
|
return NULL;
|
}
|
}
|
}
|
}
|
else
|
else
|
{
|
{
|
op1 = gimple_assign_rhs1 (def_stmt);
|
op1 = gimple_assign_rhs1 (def_stmt);
|
op2 = gimple_assign_rhs2 (def_stmt);
|
op2 = gimple_assign_rhs2 (def_stmt);
|
|
|
if (TREE_CODE (op1) != SSA_NAME || TREE_CODE (op2) != SSA_NAME)
|
if (TREE_CODE (op1) != SSA_NAME || TREE_CODE (op2) != SSA_NAME)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
report_vect_op (def_stmt, "reduction: uses not ssa_names: ");
|
report_vect_op (def_stmt, "reduction: uses not ssa_names: ");
|
|
|
return NULL;
|
return NULL;
|
}
|
}
|
}
|
}
|
|
|
type = TREE_TYPE (gimple_assign_lhs (def_stmt));
|
type = TREE_TYPE (gimple_assign_lhs (def_stmt));
|
if ((TREE_CODE (op1) == SSA_NAME
|
if ((TREE_CODE (op1) == SSA_NAME
|
&& !types_compatible_p (type,TREE_TYPE (op1)))
|
&& !types_compatible_p (type,TREE_TYPE (op1)))
|
|| (TREE_CODE (op2) == SSA_NAME
|
|| (TREE_CODE (op2) == SSA_NAME
|
&& !types_compatible_p (type, TREE_TYPE (op2)))
|
&& !types_compatible_p (type, TREE_TYPE (op2)))
|
|| (op3 && TREE_CODE (op3) == SSA_NAME
|
|| (op3 && TREE_CODE (op3) == SSA_NAME
|
&& !types_compatible_p (type, TREE_TYPE (op3)))
|
&& !types_compatible_p (type, TREE_TYPE (op3)))
|
|| (op4 && TREE_CODE (op4) == SSA_NAME
|
|| (op4 && TREE_CODE (op4) == SSA_NAME
|
&& !types_compatible_p (type, TREE_TYPE (op4))))
|
&& !types_compatible_p (type, TREE_TYPE (op4))))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "reduction: multiple types: operation type: ");
|
fprintf (vect_dump, "reduction: multiple types: operation type: ");
|
print_generic_expr (vect_dump, type, TDF_SLIM);
|
print_generic_expr (vect_dump, type, TDF_SLIM);
|
fprintf (vect_dump, ", operands types: ");
|
fprintf (vect_dump, ", operands types: ");
|
print_generic_expr (vect_dump, TREE_TYPE (op1), TDF_SLIM);
|
print_generic_expr (vect_dump, TREE_TYPE (op1), TDF_SLIM);
|
fprintf (vect_dump, ",");
|
fprintf (vect_dump, ",");
|
print_generic_expr (vect_dump, TREE_TYPE (op2), TDF_SLIM);
|
print_generic_expr (vect_dump, TREE_TYPE (op2), TDF_SLIM);
|
if (op3)
|
if (op3)
|
{
|
{
|
fprintf (vect_dump, ",");
|
fprintf (vect_dump, ",");
|
print_generic_expr (vect_dump, TREE_TYPE (op3), TDF_SLIM);
|
print_generic_expr (vect_dump, TREE_TYPE (op3), TDF_SLIM);
|
}
|
}
|
|
|
if (op4)
|
if (op4)
|
{
|
{
|
fprintf (vect_dump, ",");
|
fprintf (vect_dump, ",");
|
print_generic_expr (vect_dump, TREE_TYPE (op4), TDF_SLIM);
|
print_generic_expr (vect_dump, TREE_TYPE (op4), TDF_SLIM);
|
}
|
}
|
}
|
}
|
|
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Check that it's ok to change the order of the computation.
|
/* Check that it's ok to change the order of the computation.
|
Generally, when vectorizing a reduction we change the order of the
|
Generally, when vectorizing a reduction we change the order of the
|
computation. This may change the behavior of the program in some
|
computation. This may change the behavior of the program in some
|
cases, so we need to check that this is ok. One exception is when
|
cases, so we need to check that this is ok. One exception is when
|
vectorizing an outer-loop: the inner-loop is executed sequentially,
|
vectorizing an outer-loop: the inner-loop is executed sequentially,
|
and therefore vectorizing reductions in the inner-loop during
|
and therefore vectorizing reductions in the inner-loop during
|
outer-loop vectorization is safe. */
|
outer-loop vectorization is safe. */
|
|
|
/* CHECKME: check for !flag_finite_math_only too? */
|
/* CHECKME: check for !flag_finite_math_only too? */
|
if (SCALAR_FLOAT_TYPE_P (type) && !flag_associative_math
|
if (SCALAR_FLOAT_TYPE_P (type) && !flag_associative_math
|
&& check_reduction)
|
&& check_reduction)
|
{
|
{
|
/* Changing the order of operations changes the semantics. */
|
/* Changing the order of operations changes the semantics. */
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
report_vect_op (def_stmt, "reduction: unsafe fp math optimization: ");
|
report_vect_op (def_stmt, "reduction: unsafe fp math optimization: ");
|
return NULL;
|
return NULL;
|
}
|
}
|
else if (INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type)
|
else if (INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type)
|
&& check_reduction)
|
&& check_reduction)
|
{
|
{
|
/* Changing the order of operations changes the semantics. */
|
/* Changing the order of operations changes the semantics. */
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
report_vect_op (def_stmt, "reduction: unsafe int math optimization: ");
|
report_vect_op (def_stmt, "reduction: unsafe int math optimization: ");
|
return NULL;
|
return NULL;
|
}
|
}
|
else if (SAT_FIXED_POINT_TYPE_P (type) && check_reduction)
|
else if (SAT_FIXED_POINT_TYPE_P (type) && check_reduction)
|
{
|
{
|
/* Changing the order of operations changes the semantics. */
|
/* Changing the order of operations changes the semantics. */
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
report_vect_op (def_stmt,
|
report_vect_op (def_stmt,
|
"reduction: unsafe fixed-point math optimization: ");
|
"reduction: unsafe fixed-point math optimization: ");
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Reduction is safe. We're dealing with one of the following:
|
/* Reduction is safe. We're dealing with one of the following:
|
1) integer arithmetic and no trapv
|
1) integer arithmetic and no trapv
|
2) floating point arithmetic, and special flags permit this optimization
|
2) floating point arithmetic, and special flags permit this optimization
|
3) nested cycle (i.e., outer loop vectorization). */
|
3) nested cycle (i.e., outer loop vectorization). */
|
if (TREE_CODE (op1) == SSA_NAME)
|
if (TREE_CODE (op1) == SSA_NAME)
|
def1 = SSA_NAME_DEF_STMT (op1);
|
def1 = SSA_NAME_DEF_STMT (op1);
|
|
|
if (TREE_CODE (op2) == SSA_NAME)
|
if (TREE_CODE (op2) == SSA_NAME)
|
def2 = SSA_NAME_DEF_STMT (op2);
|
def2 = SSA_NAME_DEF_STMT (op2);
|
|
|
if (code != COND_EXPR
|
if (code != COND_EXPR
|
&& (!def1 || !def2 || gimple_nop_p (def1) || gimple_nop_p (def2)))
|
&& (!def1 || !def2 || gimple_nop_p (def1) || gimple_nop_p (def2)))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
report_vect_op (def_stmt, "reduction: no defs for operands: ");
|
report_vect_op (def_stmt, "reduction: no defs for operands: ");
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Check that one def is the reduction def, defined by PHI,
|
/* Check that one def is the reduction def, defined by PHI,
|
the other def is either defined in the loop ("vect_internal_def"),
|
the other def is either defined in the loop ("vect_internal_def"),
|
or it's an induction (defined by a loop-header phi-node). */
|
or it's an induction (defined by a loop-header phi-node). */
|
|
|
if (def2 && def2 == phi
|
if (def2 && def2 == phi
|
&& (code == COND_EXPR
|
&& (code == COND_EXPR
|
|| (def1 && flow_bb_inside_loop_p (loop, gimple_bb (def1))
|
|| (def1 && flow_bb_inside_loop_p (loop, gimple_bb (def1))
|
&& (is_gimple_assign (def1)
|
&& (is_gimple_assign (def1)
|
|| STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1))
|
|| STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1))
|
== vect_induction_def
|
== vect_induction_def
|
|| (gimple_code (def1) == GIMPLE_PHI
|
|| (gimple_code (def1) == GIMPLE_PHI
|
&& STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1))
|
&& STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1))
|
== vect_internal_def
|
== vect_internal_def
|
&& !is_loop_header_bb_p (gimple_bb (def1)))))))
|
&& !is_loop_header_bb_p (gimple_bb (def1)))))))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
report_vect_op (def_stmt, "detected reduction: ");
|
report_vect_op (def_stmt, "detected reduction: ");
|
return def_stmt;
|
return def_stmt;
|
}
|
}
|
else if (def1 && def1 == phi
|
else if (def1 && def1 == phi
|
&& (code == COND_EXPR
|
&& (code == COND_EXPR
|
|| (def2 && flow_bb_inside_loop_p (loop, gimple_bb (def2))
|
|| (def2 && flow_bb_inside_loop_p (loop, gimple_bb (def2))
|
&& (is_gimple_assign (def2)
|
&& (is_gimple_assign (def2)
|
|| STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2))
|
|| STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2))
|
== vect_induction_def
|
== vect_induction_def
|
|| (gimple_code (def2) == GIMPLE_PHI
|
|| (gimple_code (def2) == GIMPLE_PHI
|
&& STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2))
|
&& STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2))
|
== vect_internal_def
|
== vect_internal_def
|
&& !is_loop_header_bb_p (gimple_bb (def2)))))))
|
&& !is_loop_header_bb_p (gimple_bb (def2)))))))
|
{
|
{
|
if (check_reduction)
|
if (check_reduction)
|
{
|
{
|
/* Swap operands (just for simplicity - so that the rest of the code
|
/* Swap operands (just for simplicity - so that the rest of the code
|
can assume that the reduction variable is always the last (second)
|
can assume that the reduction variable is always the last (second)
|
argument). */
|
argument). */
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
report_vect_op (def_stmt,
|
report_vect_op (def_stmt,
|
"detected reduction: need to swap operands: ");
|
"detected reduction: need to swap operands: ");
|
|
|
swap_tree_operands (def_stmt, gimple_assign_rhs1_ptr (def_stmt),
|
swap_tree_operands (def_stmt, gimple_assign_rhs1_ptr (def_stmt),
|
gimple_assign_rhs2_ptr (def_stmt));
|
gimple_assign_rhs2_ptr (def_stmt));
|
}
|
}
|
else
|
else
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
report_vect_op (def_stmt, "detected reduction: ");
|
report_vect_op (def_stmt, "detected reduction: ");
|
}
|
}
|
|
|
return def_stmt;
|
return def_stmt;
|
}
|
}
|
else
|
else
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
report_vect_op (def_stmt, "reduction: unknown pattern: ");
|
report_vect_op (def_stmt, "reduction: unknown pattern: ");
|
|
|
return NULL;
|
return NULL;
|
}
|
}
|
}
|
}
|
|
|
|
|
/* Function vect_estimate_min_profitable_iters
|
/* Function vect_estimate_min_profitable_iters
|
|
|
Return the number of iterations required for the vector version of the
|
Return the number of iterations required for the vector version of the
|
loop to be profitable relative to the cost of the scalar version of the
|
loop to be profitable relative to the cost of the scalar version of the
|
loop.
|
loop.
|
|
|
TODO: Take profile info into account before making vectorization
|
TODO: Take profile info into account before making vectorization
|
decisions, if available. */
|
decisions, if available. */
|
|
|
int
|
int
|
vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo)
|
vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo)
|
{
|
{
|
int i;
|
int i;
|
int min_profitable_iters;
|
int min_profitable_iters;
|
int peel_iters_prologue;
|
int peel_iters_prologue;
|
int peel_iters_epilogue;
|
int peel_iters_epilogue;
|
int vec_inside_cost = 0;
|
int vec_inside_cost = 0;
|
int vec_outside_cost = 0;
|
int vec_outside_cost = 0;
|
int scalar_single_iter_cost = 0;
|
int scalar_single_iter_cost = 0;
|
int scalar_outside_cost = 0;
|
int scalar_outside_cost = 0;
|
int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
|
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
|
int nbbs = loop->num_nodes;
|
int nbbs = loop->num_nodes;
|
int byte_misalign = LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo);
|
int byte_misalign = LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo);
|
int peel_guard_costs = 0;
|
int peel_guard_costs = 0;
|
int innerloop_iters = 0, factor;
|
int innerloop_iters = 0, factor;
|
VEC (slp_instance, heap) *slp_instances;
|
VEC (slp_instance, heap) *slp_instances;
|
slp_instance instance;
|
slp_instance instance;
|
|
|
/* Cost model disabled. */
|
/* Cost model disabled. */
|
if (!flag_vect_cost_model)
|
if (!flag_vect_cost_model)
|
{
|
{
|
if (vect_print_dump_info (REPORT_COST))
|
if (vect_print_dump_info (REPORT_COST))
|
fprintf (vect_dump, "cost model disabled.");
|
fprintf (vect_dump, "cost model disabled.");
|
return 0;
|
return 0;
|
}
|
}
|
|
|
/* Requires loop versioning tests to handle misalignment. */
|
/* Requires loop versioning tests to handle misalignment. */
|
if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo))
|
if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo))
|
{
|
{
|
/* FIXME: Make cost depend on complexity of individual check. */
|
/* FIXME: Make cost depend on complexity of individual check. */
|
vec_outside_cost +=
|
vec_outside_cost +=
|
VEC_length (gimple, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo));
|
VEC_length (gimple, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo));
|
if (vect_print_dump_info (REPORT_COST))
|
if (vect_print_dump_info (REPORT_COST))
|
fprintf (vect_dump, "cost model: Adding cost of checks for loop "
|
fprintf (vect_dump, "cost model: Adding cost of checks for loop "
|
"versioning to treat misalignment.\n");
|
"versioning to treat misalignment.\n");
|
}
|
}
|
|
|
/* Requires loop versioning with alias checks. */
|
/* Requires loop versioning with alias checks. */
|
if (LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
|
if (LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
|
{
|
{
|
/* FIXME: Make cost depend on complexity of individual check. */
|
/* FIXME: Make cost depend on complexity of individual check. */
|
vec_outside_cost +=
|
vec_outside_cost +=
|
VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo));
|
VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo));
|
if (vect_print_dump_info (REPORT_COST))
|
if (vect_print_dump_info (REPORT_COST))
|
fprintf (vect_dump, "cost model: Adding cost of checks for loop "
|
fprintf (vect_dump, "cost model: Adding cost of checks for loop "
|
"versioning aliasing.\n");
|
"versioning aliasing.\n");
|
}
|
}
|
|
|
if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo)
|
if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo)
|
|| LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
|
|| LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
|
vec_outside_cost += TARG_COND_TAKEN_BRANCH_COST;
|
vec_outside_cost += TARG_COND_TAKEN_BRANCH_COST;
|
|
|
/* Count statements in scalar loop. Using this as scalar cost for a single
|
/* Count statements in scalar loop. Using this as scalar cost for a single
|
iteration for now.
|
iteration for now.
|
|
|
TODO: Add outer loop support.
|
TODO: Add outer loop support.
|
|
|
TODO: Consider assigning different costs to different scalar
|
TODO: Consider assigning different costs to different scalar
|
statements. */
|
statements. */
|
|
|
/* FORNOW. */
|
/* FORNOW. */
|
if (loop->inner)
|
if (loop->inner)
|
innerloop_iters = 50; /* FIXME */
|
innerloop_iters = 50; /* FIXME */
|
|
|
for (i = 0; i < nbbs; i++)
|
for (i = 0; i < nbbs; i++)
|
{
|
{
|
gimple_stmt_iterator si;
|
gimple_stmt_iterator si;
|
basic_block bb = bbs[i];
|
basic_block bb = bbs[i];
|
|
|
if (bb->loop_father == loop->inner)
|
if (bb->loop_father == loop->inner)
|
factor = innerloop_iters;
|
factor = innerloop_iters;
|
else
|
else
|
factor = 1;
|
factor = 1;
|
|
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
{
|
{
|
gimple stmt = gsi_stmt (si);
|
gimple stmt = gsi_stmt (si);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
/* Skip stmts that are not vectorized inside the loop. */
|
/* Skip stmts that are not vectorized inside the loop. */
|
if (!STMT_VINFO_RELEVANT_P (stmt_info)
|
if (!STMT_VINFO_RELEVANT_P (stmt_info)
|
&& (!STMT_VINFO_LIVE_P (stmt_info)
|
&& (!STMT_VINFO_LIVE_P (stmt_info)
|
|| STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def))
|
|| STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def))
|
continue;
|
continue;
|
scalar_single_iter_cost += cost_for_stmt (stmt) * factor;
|
scalar_single_iter_cost += cost_for_stmt (stmt) * factor;
|
vec_inside_cost += STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) * factor;
|
vec_inside_cost += STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) * factor;
|
/* FIXME: for stmts in the inner-loop in outer-loop vectorization,
|
/* FIXME: for stmts in the inner-loop in outer-loop vectorization,
|
some of the "outside" costs are generated inside the outer-loop. */
|
some of the "outside" costs are generated inside the outer-loop. */
|
vec_outside_cost += STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info);
|
vec_outside_cost += STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info);
|
}
|
}
|
}
|
}
|
|
|
/* Add additional cost for the peeled instructions in prologue and epilogue
|
/* Add additional cost for the peeled instructions in prologue and epilogue
|
loop.
|
loop.
|
|
|
FORNOW: If we don't know the value of peel_iters for prologue or epilogue
|
FORNOW: If we don't know the value of peel_iters for prologue or epilogue
|
at compile-time - we assume it's vf/2 (the worst would be vf-1).
|
at compile-time - we assume it's vf/2 (the worst would be vf-1).
|
|
|
TODO: Build an expression that represents peel_iters for prologue and
|
TODO: Build an expression that represents peel_iters for prologue and
|
epilogue to be used in a run-time test. */
|
epilogue to be used in a run-time test. */
|
|
|
if (byte_misalign < 0)
|
if (byte_misalign < 0)
|
{
|
{
|
peel_iters_prologue = vf/2;
|
peel_iters_prologue = vf/2;
|
if (vect_print_dump_info (REPORT_COST))
|
if (vect_print_dump_info (REPORT_COST))
|
fprintf (vect_dump, "cost model: "
|
fprintf (vect_dump, "cost model: "
|
"prologue peel iters set to vf/2.");
|
"prologue peel iters set to vf/2.");
|
|
|
/* If peeling for alignment is unknown, loop bound of main loop becomes
|
/* If peeling for alignment is unknown, loop bound of main loop becomes
|
unknown. */
|
unknown. */
|
peel_iters_epilogue = vf/2;
|
peel_iters_epilogue = vf/2;
|
if (vect_print_dump_info (REPORT_COST))
|
if (vect_print_dump_info (REPORT_COST))
|
fprintf (vect_dump, "cost model: "
|
fprintf (vect_dump, "cost model: "
|
"epilogue peel iters set to vf/2 because "
|
"epilogue peel iters set to vf/2 because "
|
"peeling for alignment is unknown .");
|
"peeling for alignment is unknown .");
|
|
|
/* If peeled iterations are unknown, count a taken branch and a not taken
|
/* If peeled iterations are unknown, count a taken branch and a not taken
|
branch per peeled loop. Even if scalar loop iterations are known,
|
branch per peeled loop. Even if scalar loop iterations are known,
|
vector iterations are not known since peeled prologue iterations are
|
vector iterations are not known since peeled prologue iterations are
|
not known. Hence guards remain the same. */
|
not known. Hence guards remain the same. */
|
peel_guard_costs += 2 * (TARG_COND_TAKEN_BRANCH_COST
|
peel_guard_costs += 2 * (TARG_COND_TAKEN_BRANCH_COST
|
+ TARG_COND_NOT_TAKEN_BRANCH_COST);
|
+ TARG_COND_NOT_TAKEN_BRANCH_COST);
|
}
|
}
|
else
|
else
|
{
|
{
|
if (byte_misalign)
|
if (byte_misalign)
|
{
|
{
|
struct data_reference *dr = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
|
struct data_reference *dr = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
|
int element_size = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (dr))));
|
int element_size = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (dr))));
|
tree vectype = STMT_VINFO_VECTYPE (vinfo_for_stmt (DR_STMT (dr)));
|
tree vectype = STMT_VINFO_VECTYPE (vinfo_for_stmt (DR_STMT (dr)));
|
int nelements = TYPE_VECTOR_SUBPARTS (vectype);
|
int nelements = TYPE_VECTOR_SUBPARTS (vectype);
|
|
|
peel_iters_prologue = nelements - (byte_misalign / element_size);
|
peel_iters_prologue = nelements - (byte_misalign / element_size);
|
}
|
}
|
else
|
else
|
peel_iters_prologue = 0;
|
peel_iters_prologue = 0;
|
|
|
if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
|
if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
|
{
|
{
|
peel_iters_epilogue = vf/2;
|
peel_iters_epilogue = vf/2;
|
if (vect_print_dump_info (REPORT_COST))
|
if (vect_print_dump_info (REPORT_COST))
|
fprintf (vect_dump, "cost model: "
|
fprintf (vect_dump, "cost model: "
|
"epilogue peel iters set to vf/2 because "
|
"epilogue peel iters set to vf/2 because "
|
"loop iterations are unknown .");
|
"loop iterations are unknown .");
|
|
|
/* If peeled iterations are known but number of scalar loop
|
/* If peeled iterations are known but number of scalar loop
|
iterations are unknown, count a taken branch per peeled loop. */
|
iterations are unknown, count a taken branch per peeled loop. */
|
peel_guard_costs += 2 * TARG_COND_TAKEN_BRANCH_COST;
|
peel_guard_costs += 2 * TARG_COND_TAKEN_BRANCH_COST;
|
|
|
}
|
}
|
else
|
else
|
{
|
{
|
int niters = LOOP_VINFO_INT_NITERS (loop_vinfo);
|
int niters = LOOP_VINFO_INT_NITERS (loop_vinfo);
|
peel_iters_prologue = niters < peel_iters_prologue ?
|
peel_iters_prologue = niters < peel_iters_prologue ?
|
niters : peel_iters_prologue;
|
niters : peel_iters_prologue;
|
peel_iters_epilogue = (niters - peel_iters_prologue) % vf;
|
peel_iters_epilogue = (niters - peel_iters_prologue) % vf;
|
}
|
}
|
}
|
}
|
|
|
vec_outside_cost += (peel_iters_prologue * scalar_single_iter_cost)
|
vec_outside_cost += (peel_iters_prologue * scalar_single_iter_cost)
|
+ (peel_iters_epilogue * scalar_single_iter_cost)
|
+ (peel_iters_epilogue * scalar_single_iter_cost)
|
+ peel_guard_costs;
|
+ peel_guard_costs;
|
|
|
/* FORNOW: The scalar outside cost is incremented in one of the
|
/* FORNOW: The scalar outside cost is incremented in one of the
|
following ways:
|
following ways:
|
|
|
1. The vectorizer checks for alignment and aliasing and generates
|
1. The vectorizer checks for alignment and aliasing and generates
|
a condition that allows dynamic vectorization. A cost model
|
a condition that allows dynamic vectorization. A cost model
|
check is ANDED with the versioning condition. Hence scalar code
|
check is ANDED with the versioning condition. Hence scalar code
|
path now has the added cost of the versioning check.
|
path now has the added cost of the versioning check.
|
|
|
if (cost > th & versioning_check)
|
if (cost > th & versioning_check)
|
jmp to vector code
|
jmp to vector code
|
|
|
Hence run-time scalar is incremented by not-taken branch cost.
|
Hence run-time scalar is incremented by not-taken branch cost.
|
|
|
2. The vectorizer then checks if a prologue is required. If the
|
2. The vectorizer then checks if a prologue is required. If the
|
cost model check was not done before during versioning, it has to
|
cost model check was not done before during versioning, it has to
|
be done before the prologue check.
|
be done before the prologue check.
|
|
|
if (cost <= th)
|
if (cost <= th)
|
prologue = scalar_iters
|
prologue = scalar_iters
|
if (prologue == 0)
|
if (prologue == 0)
|
jmp to vector code
|
jmp to vector code
|
else
|
else
|
execute prologue
|
execute prologue
|
if (prologue == num_iters)
|
if (prologue == num_iters)
|
go to exit
|
go to exit
|
|
|
Hence the run-time scalar cost is incremented by a taken branch,
|
Hence the run-time scalar cost is incremented by a taken branch,
|
plus a not-taken branch, plus a taken branch cost.
|
plus a not-taken branch, plus a taken branch cost.
|
|
|
3. The vectorizer then checks if an epilogue is required. If the
|
3. The vectorizer then checks if an epilogue is required. If the
|
cost model check was not done before during prologue check, it
|
cost model check was not done before during prologue check, it
|
has to be done with the epilogue check.
|
has to be done with the epilogue check.
|
|
|
if (prologue == 0)
|
if (prologue == 0)
|
jmp to vector code
|
jmp to vector code
|
else
|
else
|
execute prologue
|
execute prologue
|
if (prologue == num_iters)
|
if (prologue == num_iters)
|
go to exit
|
go to exit
|
vector code:
|
vector code:
|
if ((cost <= th) | (scalar_iters-prologue-epilogue == 0))
|
if ((cost <= th) | (scalar_iters-prologue-epilogue == 0))
|
jmp to epilogue
|
jmp to epilogue
|
|
|
Hence the run-time scalar cost should be incremented by 2 taken
|
Hence the run-time scalar cost should be incremented by 2 taken
|
branches.
|
branches.
|
|
|
TODO: The back end may reorder the BBS's differently and reverse
|
TODO: The back end may reorder the BBS's differently and reverse
|
conditions/branch directions. Change the estimates below to
|
conditions/branch directions. Change the estimates below to
|
something more reasonable. */
|
something more reasonable. */
|
|
|
/* If the number of iterations is known and we do not do versioning, we can
|
/* If the number of iterations is known and we do not do versioning, we can
|
decide whether to vectorize at compile time. Hence the scalar version
|
decide whether to vectorize at compile time. Hence the scalar version
|
do not carry cost model guard costs. */
|
do not carry cost model guard costs. */
|
if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
|| LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo)
|
|| LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo)
|
|| LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
|
|| LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
|
{
|
{
|
/* Cost model check occurs at versioning. */
|
/* Cost model check occurs at versioning. */
|
if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo)
|
if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo)
|
|| LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
|
|| LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
|
scalar_outside_cost += TARG_COND_NOT_TAKEN_BRANCH_COST;
|
scalar_outside_cost += TARG_COND_NOT_TAKEN_BRANCH_COST;
|
else
|
else
|
{
|
{
|
/* Cost model check occurs at prologue generation. */
|
/* Cost model check occurs at prologue generation. */
|
if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0)
|
if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0)
|
scalar_outside_cost += 2 * TARG_COND_TAKEN_BRANCH_COST
|
scalar_outside_cost += 2 * TARG_COND_TAKEN_BRANCH_COST
|
+ TARG_COND_NOT_TAKEN_BRANCH_COST;
|
+ TARG_COND_NOT_TAKEN_BRANCH_COST;
|
/* Cost model check occurs at epilogue generation. */
|
/* Cost model check occurs at epilogue generation. */
|
else
|
else
|
scalar_outside_cost += 2 * TARG_COND_TAKEN_BRANCH_COST;
|
scalar_outside_cost += 2 * TARG_COND_TAKEN_BRANCH_COST;
|
}
|
}
|
}
|
}
|
|
|
/* Add SLP costs. */
|
/* Add SLP costs. */
|
slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
|
slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
|
for (i = 0; VEC_iterate (slp_instance, slp_instances, i, instance); i++)
|
for (i = 0; VEC_iterate (slp_instance, slp_instances, i, instance); i++)
|
{
|
{
|
vec_outside_cost += SLP_INSTANCE_OUTSIDE_OF_LOOP_COST (instance);
|
vec_outside_cost += SLP_INSTANCE_OUTSIDE_OF_LOOP_COST (instance);
|
vec_inside_cost += SLP_INSTANCE_INSIDE_OF_LOOP_COST (instance);
|
vec_inside_cost += SLP_INSTANCE_INSIDE_OF_LOOP_COST (instance);
|
}
|
}
|
|
|
/* Calculate number of iterations required to make the vector version
|
/* Calculate number of iterations required to make the vector version
|
profitable, relative to the loop bodies only. The following condition
|
profitable, relative to the loop bodies only. The following condition
|
must hold true:
|
must hold true:
|
SIC * niters + SOC > VIC * ((niters-PL_ITERS-EP_ITERS)/VF) + VOC
|
SIC * niters + SOC > VIC * ((niters-PL_ITERS-EP_ITERS)/VF) + VOC
|
where
|
where
|
SIC = scalar iteration cost, VIC = vector iteration cost,
|
SIC = scalar iteration cost, VIC = vector iteration cost,
|
VOC = vector outside cost, VF = vectorization factor,
|
VOC = vector outside cost, VF = vectorization factor,
|
PL_ITERS = prologue iterations, EP_ITERS= epilogue iterations
|
PL_ITERS = prologue iterations, EP_ITERS= epilogue iterations
|
SOC = scalar outside cost for run time cost model check. */
|
SOC = scalar outside cost for run time cost model check. */
|
|
|
if ((scalar_single_iter_cost * vf) > vec_inside_cost)
|
if ((scalar_single_iter_cost * vf) > vec_inside_cost)
|
{
|
{
|
if (vec_outside_cost <= 0)
|
if (vec_outside_cost <= 0)
|
min_profitable_iters = 1;
|
min_profitable_iters = 1;
|
else
|
else
|
{
|
{
|
min_profitable_iters = ((vec_outside_cost - scalar_outside_cost) * vf
|
min_profitable_iters = ((vec_outside_cost - scalar_outside_cost) * vf
|
- vec_inside_cost * peel_iters_prologue
|
- vec_inside_cost * peel_iters_prologue
|
- vec_inside_cost * peel_iters_epilogue)
|
- vec_inside_cost * peel_iters_epilogue)
|
/ ((scalar_single_iter_cost * vf)
|
/ ((scalar_single_iter_cost * vf)
|
- vec_inside_cost);
|
- vec_inside_cost);
|
|
|
if ((scalar_single_iter_cost * vf * min_profitable_iters)
|
if ((scalar_single_iter_cost * vf * min_profitable_iters)
|
<= ((vec_inside_cost * min_profitable_iters)
|
<= ((vec_inside_cost * min_profitable_iters)
|
+ ((vec_outside_cost - scalar_outside_cost) * vf)))
|
+ ((vec_outside_cost - scalar_outside_cost) * vf)))
|
min_profitable_iters++;
|
min_profitable_iters++;
|
}
|
}
|
}
|
}
|
/* vector version will never be profitable. */
|
/* vector version will never be profitable. */
|
else
|
else
|
{
|
{
|
if (vect_print_dump_info (REPORT_COST))
|
if (vect_print_dump_info (REPORT_COST))
|
fprintf (vect_dump, "cost model: the vector iteration cost = %d "
|
fprintf (vect_dump, "cost model: the vector iteration cost = %d "
|
"divided by the scalar iteration cost = %d "
|
"divided by the scalar iteration cost = %d "
|
"is greater or equal to the vectorization factor = %d.",
|
"is greater or equal to the vectorization factor = %d.",
|
vec_inside_cost, scalar_single_iter_cost, vf);
|
vec_inside_cost, scalar_single_iter_cost, vf);
|
return -1;
|
return -1;
|
}
|
}
|
|
|
if (vect_print_dump_info (REPORT_COST))
|
if (vect_print_dump_info (REPORT_COST))
|
{
|
{
|
fprintf (vect_dump, "Cost model analysis: \n");
|
fprintf (vect_dump, "Cost model analysis: \n");
|
fprintf (vect_dump, " Vector inside of loop cost: %d\n",
|
fprintf (vect_dump, " Vector inside of loop cost: %d\n",
|
vec_inside_cost);
|
vec_inside_cost);
|
fprintf (vect_dump, " Vector outside of loop cost: %d\n",
|
fprintf (vect_dump, " Vector outside of loop cost: %d\n",
|
vec_outside_cost);
|
vec_outside_cost);
|
fprintf (vect_dump, " Scalar iteration cost: %d\n",
|
fprintf (vect_dump, " Scalar iteration cost: %d\n",
|
scalar_single_iter_cost);
|
scalar_single_iter_cost);
|
fprintf (vect_dump, " Scalar outside cost: %d\n", scalar_outside_cost);
|
fprintf (vect_dump, " Scalar outside cost: %d\n", scalar_outside_cost);
|
fprintf (vect_dump, " prologue iterations: %d\n",
|
fprintf (vect_dump, " prologue iterations: %d\n",
|
peel_iters_prologue);
|
peel_iters_prologue);
|
fprintf (vect_dump, " epilogue iterations: %d\n",
|
fprintf (vect_dump, " epilogue iterations: %d\n",
|
peel_iters_epilogue);
|
peel_iters_epilogue);
|
fprintf (vect_dump, " Calculated minimum iters for profitability: %d\n",
|
fprintf (vect_dump, " Calculated minimum iters for profitability: %d\n",
|
min_profitable_iters);
|
min_profitable_iters);
|
}
|
}
|
|
|
min_profitable_iters =
|
min_profitable_iters =
|
min_profitable_iters < vf ? vf : min_profitable_iters;
|
min_profitable_iters < vf ? vf : min_profitable_iters;
|
|
|
/* Because the condition we create is:
|
/* Because the condition we create is:
|
if (niters <= min_profitable_iters)
|
if (niters <= min_profitable_iters)
|
then skip the vectorized loop. */
|
then skip the vectorized loop. */
|
min_profitable_iters--;
|
min_profitable_iters--;
|
|
|
if (vect_print_dump_info (REPORT_COST))
|
if (vect_print_dump_info (REPORT_COST))
|
fprintf (vect_dump, " Profitability threshold = %d\n",
|
fprintf (vect_dump, " Profitability threshold = %d\n",
|
min_profitable_iters);
|
min_profitable_iters);
|
|
|
return min_profitable_iters;
|
return min_profitable_iters;
|
}
|
}
|
|
|
|
|
/* TODO: Close dependency between vect_model_*_cost and vectorizable_*
|
/* TODO: Close dependency between vect_model_*_cost and vectorizable_*
|
functions. Design better to avoid maintenance issues. */
|
functions. Design better to avoid maintenance issues. */
|
|
|
/* Function vect_model_reduction_cost.
|
/* Function vect_model_reduction_cost.
|
|
|
Models cost for a reduction operation, including the vector ops
|
Models cost for a reduction operation, including the vector ops
|
generated within the strip-mine loop, the initial definition before
|
generated within the strip-mine loop, the initial definition before
|
the loop, and the epilogue code that must be generated. */
|
the loop, and the epilogue code that must be generated. */
|
|
|
static bool
|
static bool
|
vect_model_reduction_cost (stmt_vec_info stmt_info, enum tree_code reduc_code,
|
vect_model_reduction_cost (stmt_vec_info stmt_info, enum tree_code reduc_code,
|
int ncopies)
|
int ncopies)
|
{
|
{
|
int outer_cost = 0;
|
int outer_cost = 0;
|
enum tree_code code;
|
enum tree_code code;
|
optab optab;
|
optab optab;
|
tree vectype;
|
tree vectype;
|
gimple stmt, orig_stmt;
|
gimple stmt, orig_stmt;
|
tree reduction_op;
|
tree reduction_op;
|
enum machine_mode mode;
|
enum machine_mode mode;
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
|
|
|
/* Cost of reduction op inside loop. */
|
/* Cost of reduction op inside loop. */
|
STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) += ncopies * TARG_VEC_STMT_COST;
|
STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) += ncopies * TARG_VEC_STMT_COST;
|
|
|
stmt = STMT_VINFO_STMT (stmt_info);
|
stmt = STMT_VINFO_STMT (stmt_info);
|
|
|
switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt)))
|
switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt)))
|
{
|
{
|
case GIMPLE_SINGLE_RHS:
|
case GIMPLE_SINGLE_RHS:
|
gcc_assert (TREE_OPERAND_LENGTH (gimple_assign_rhs1 (stmt)) == ternary_op);
|
gcc_assert (TREE_OPERAND_LENGTH (gimple_assign_rhs1 (stmt)) == ternary_op);
|
reduction_op = TREE_OPERAND (gimple_assign_rhs1 (stmt), 2);
|
reduction_op = TREE_OPERAND (gimple_assign_rhs1 (stmt), 2);
|
break;
|
break;
|
case GIMPLE_UNARY_RHS:
|
case GIMPLE_UNARY_RHS:
|
reduction_op = gimple_assign_rhs1 (stmt);
|
reduction_op = gimple_assign_rhs1 (stmt);
|
break;
|
break;
|
case GIMPLE_BINARY_RHS:
|
case GIMPLE_BINARY_RHS:
|
reduction_op = gimple_assign_rhs2 (stmt);
|
reduction_op = gimple_assign_rhs2 (stmt);
|
break;
|
break;
|
default:
|
default:
|
gcc_unreachable ();
|
gcc_unreachable ();
|
}
|
}
|
|
|
vectype = get_vectype_for_scalar_type (TREE_TYPE (reduction_op));
|
vectype = get_vectype_for_scalar_type (TREE_TYPE (reduction_op));
|
if (!vectype)
|
if (!vectype)
|
{
|
{
|
if (vect_print_dump_info (REPORT_COST))
|
if (vect_print_dump_info (REPORT_COST))
|
{
|
{
|
fprintf (vect_dump, "unsupported data-type ");
|
fprintf (vect_dump, "unsupported data-type ");
|
print_generic_expr (vect_dump, TREE_TYPE (reduction_op), TDF_SLIM);
|
print_generic_expr (vect_dump, TREE_TYPE (reduction_op), TDF_SLIM);
|
}
|
}
|
return false;
|
return false;
|
}
|
}
|
|
|
mode = TYPE_MODE (vectype);
|
mode = TYPE_MODE (vectype);
|
orig_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
|
orig_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
|
|
|
if (!orig_stmt)
|
if (!orig_stmt)
|
orig_stmt = STMT_VINFO_STMT (stmt_info);
|
orig_stmt = STMT_VINFO_STMT (stmt_info);
|
|
|
code = gimple_assign_rhs_code (orig_stmt);
|
code = gimple_assign_rhs_code (orig_stmt);
|
|
|
/* Add in cost for initial definition. */
|
/* Add in cost for initial definition. */
|
outer_cost += TARG_SCALAR_TO_VEC_COST;
|
outer_cost += TARG_SCALAR_TO_VEC_COST;
|
|
|
/* Determine cost of epilogue code.
|
/* Determine cost of epilogue code.
|
|
|
We have a reduction operator that will reduce the vector in one statement.
|
We have a reduction operator that will reduce the vector in one statement.
|
Also requires scalar extract. */
|
Also requires scalar extract. */
|
|
|
if (!nested_in_vect_loop_p (loop, orig_stmt))
|
if (!nested_in_vect_loop_p (loop, orig_stmt))
|
{
|
{
|
if (reduc_code != ERROR_MARK)
|
if (reduc_code != ERROR_MARK)
|
outer_cost += TARG_VEC_STMT_COST + TARG_VEC_TO_SCALAR_COST;
|
outer_cost += TARG_VEC_STMT_COST + TARG_VEC_TO_SCALAR_COST;
|
else
|
else
|
{
|
{
|
int vec_size_in_bits = tree_low_cst (TYPE_SIZE (vectype), 1);
|
int vec_size_in_bits = tree_low_cst (TYPE_SIZE (vectype), 1);
|
tree bitsize =
|
tree bitsize =
|
TYPE_SIZE (TREE_TYPE (gimple_assign_lhs (orig_stmt)));
|
TYPE_SIZE (TREE_TYPE (gimple_assign_lhs (orig_stmt)));
|
int element_bitsize = tree_low_cst (bitsize, 1);
|
int element_bitsize = tree_low_cst (bitsize, 1);
|
int nelements = vec_size_in_bits / element_bitsize;
|
int nelements = vec_size_in_bits / element_bitsize;
|
|
|
optab = optab_for_tree_code (code, vectype, optab_default);
|
optab = optab_for_tree_code (code, vectype, optab_default);
|
|
|
/* We have a whole vector shift available. */
|
/* We have a whole vector shift available. */
|
if (VECTOR_MODE_P (mode)
|
if (VECTOR_MODE_P (mode)
|
&& optab_handler (optab, mode)->insn_code != CODE_FOR_nothing
|
&& optab_handler (optab, mode)->insn_code != CODE_FOR_nothing
|
&& optab_handler (vec_shr_optab, mode)->insn_code != CODE_FOR_nothing)
|
&& optab_handler (vec_shr_optab, mode)->insn_code != CODE_FOR_nothing)
|
/* Final reduction via vector shifts and the reduction operator. Also
|
/* Final reduction via vector shifts and the reduction operator. Also
|
requires scalar extract. */
|
requires scalar extract. */
|
outer_cost += ((exact_log2(nelements) * 2) * TARG_VEC_STMT_COST
|
outer_cost += ((exact_log2(nelements) * 2) * TARG_VEC_STMT_COST
|
+ TARG_VEC_TO_SCALAR_COST);
|
+ TARG_VEC_TO_SCALAR_COST);
|
else
|
else
|
/* Use extracts and reduction op for final reduction. For N elements,
|
/* Use extracts and reduction op for final reduction. For N elements,
|
we have N extracts and N-1 reduction ops. */
|
we have N extracts and N-1 reduction ops. */
|
outer_cost += ((nelements + nelements - 1) * TARG_VEC_STMT_COST);
|
outer_cost += ((nelements + nelements - 1) * TARG_VEC_STMT_COST);
|
}
|
}
|
}
|
}
|
|
|
STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info) = outer_cost;
|
STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info) = outer_cost;
|
|
|
if (vect_print_dump_info (REPORT_COST))
|
if (vect_print_dump_info (REPORT_COST))
|
fprintf (vect_dump, "vect_model_reduction_cost: inside_cost = %d, "
|
fprintf (vect_dump, "vect_model_reduction_cost: inside_cost = %d, "
|
"outside_cost = %d .", STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info),
|
"outside_cost = %d .", STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info),
|
STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info));
|
STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info));
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
|
|
/* Function vect_model_induction_cost.
|
/* Function vect_model_induction_cost.
|
|
|
Models cost for induction operations. */
|
Models cost for induction operations. */
|
|
|
static void
|
static void
|
vect_model_induction_cost (stmt_vec_info stmt_info, int ncopies)
|
vect_model_induction_cost (stmt_vec_info stmt_info, int ncopies)
|
{
|
{
|
/* loop cost for vec_loop. */
|
/* loop cost for vec_loop. */
|
STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) = ncopies * TARG_VEC_STMT_COST;
|
STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) = ncopies * TARG_VEC_STMT_COST;
|
/* prologue cost for vec_init and vec_step. */
|
/* prologue cost for vec_init and vec_step. */
|
STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info) = 2 * TARG_SCALAR_TO_VEC_COST;
|
STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info) = 2 * TARG_SCALAR_TO_VEC_COST;
|
|
|
if (vect_print_dump_info (REPORT_COST))
|
if (vect_print_dump_info (REPORT_COST))
|
fprintf (vect_dump, "vect_model_induction_cost: inside_cost = %d, "
|
fprintf (vect_dump, "vect_model_induction_cost: inside_cost = %d, "
|
"outside_cost = %d .", STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info),
|
"outside_cost = %d .", STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info),
|
STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info));
|
STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info));
|
}
|
}
|
|
|
|
|
/* Function get_initial_def_for_induction
|
/* Function get_initial_def_for_induction
|
|
|
Input:
|
Input:
|
STMT - a stmt that performs an induction operation in the loop.
|
STMT - a stmt that performs an induction operation in the loop.
|
IV_PHI - the initial value of the induction variable
|
IV_PHI - the initial value of the induction variable
|
|
|
Output:
|
Output:
|
Return a vector variable, initialized with the first VF values of
|
Return a vector variable, initialized with the first VF values of
|
the induction variable. E.g., for an iv with IV_PHI='X' and
|
the induction variable. E.g., for an iv with IV_PHI='X' and
|
evolution S, for a vector of 4 units, we want to return:
|
evolution S, for a vector of 4 units, we want to return:
|
[X, X + S, X + 2*S, X + 3*S]. */
|
[X, X + S, X + 2*S, X + 3*S]. */
|
|
|
static tree
|
static tree
|
get_initial_def_for_induction (gimple iv_phi)
|
get_initial_def_for_induction (gimple iv_phi)
|
{
|
{
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (iv_phi);
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (iv_phi);
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
tree scalar_type = TREE_TYPE (gimple_phi_result (iv_phi));
|
tree scalar_type = TREE_TYPE (gimple_phi_result (iv_phi));
|
tree vectype;
|
tree vectype;
|
int nunits;
|
int nunits;
|
edge pe = loop_preheader_edge (loop);
|
edge pe = loop_preheader_edge (loop);
|
struct loop *iv_loop;
|
struct loop *iv_loop;
|
basic_block new_bb;
|
basic_block new_bb;
|
tree vec, vec_init, vec_step, t;
|
tree vec, vec_init, vec_step, t;
|
tree access_fn;
|
tree access_fn;
|
tree new_var;
|
tree new_var;
|
tree new_name;
|
tree new_name;
|
gimple init_stmt, induction_phi, new_stmt;
|
gimple init_stmt, induction_phi, new_stmt;
|
tree induc_def, vec_def, vec_dest;
|
tree induc_def, vec_def, vec_dest;
|
tree init_expr, step_expr;
|
tree init_expr, step_expr;
|
int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
int i;
|
int i;
|
bool ok;
|
bool ok;
|
int ncopies;
|
int ncopies;
|
tree expr;
|
tree expr;
|
stmt_vec_info phi_info = vinfo_for_stmt (iv_phi);
|
stmt_vec_info phi_info = vinfo_for_stmt (iv_phi);
|
bool nested_in_vect_loop = false;
|
bool nested_in_vect_loop = false;
|
gimple_seq stmts = NULL;
|
gimple_seq stmts = NULL;
|
imm_use_iterator imm_iter;
|
imm_use_iterator imm_iter;
|
use_operand_p use_p;
|
use_operand_p use_p;
|
gimple exit_phi;
|
gimple exit_phi;
|
edge latch_e;
|
edge latch_e;
|
tree loop_arg;
|
tree loop_arg;
|
gimple_stmt_iterator si;
|
gimple_stmt_iterator si;
|
basic_block bb = gimple_bb (iv_phi);
|
basic_block bb = gimple_bb (iv_phi);
|
tree stepvectype;
|
tree stepvectype;
|
|
|
vectype = get_vectype_for_scalar_type (scalar_type);
|
vectype = get_vectype_for_scalar_type (scalar_type);
|
gcc_assert (vectype);
|
gcc_assert (vectype);
|
nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
ncopies = vf / nunits;
|
ncopies = vf / nunits;
|
|
|
gcc_assert (phi_info);
|
gcc_assert (phi_info);
|
gcc_assert (ncopies >= 1);
|
gcc_assert (ncopies >= 1);
|
|
|
/* Find the first insertion point in the BB. */
|
/* Find the first insertion point in the BB. */
|
si = gsi_after_labels (bb);
|
si = gsi_after_labels (bb);
|
|
|
if (INTEGRAL_TYPE_P (scalar_type))
|
if (INTEGRAL_TYPE_P (scalar_type))
|
step_expr = build_int_cst (scalar_type, 0);
|
step_expr = build_int_cst (scalar_type, 0);
|
else if (POINTER_TYPE_P (scalar_type))
|
else if (POINTER_TYPE_P (scalar_type))
|
step_expr = build_int_cst (sizetype, 0);
|
step_expr = build_int_cst (sizetype, 0);
|
else
|
else
|
step_expr = build_real (scalar_type, dconst0);
|
step_expr = build_real (scalar_type, dconst0);
|
|
|
/* Is phi in an inner-loop, while vectorizing an enclosing outer-loop? */
|
/* Is phi in an inner-loop, while vectorizing an enclosing outer-loop? */
|
if (nested_in_vect_loop_p (loop, iv_phi))
|
if (nested_in_vect_loop_p (loop, iv_phi))
|
{
|
{
|
nested_in_vect_loop = true;
|
nested_in_vect_loop = true;
|
iv_loop = loop->inner;
|
iv_loop = loop->inner;
|
}
|
}
|
else
|
else
|
iv_loop = loop;
|
iv_loop = loop;
|
gcc_assert (iv_loop == (gimple_bb (iv_phi))->loop_father);
|
gcc_assert (iv_loop == (gimple_bb (iv_phi))->loop_father);
|
|
|
latch_e = loop_latch_edge (iv_loop);
|
latch_e = loop_latch_edge (iv_loop);
|
loop_arg = PHI_ARG_DEF_FROM_EDGE (iv_phi, latch_e);
|
loop_arg = PHI_ARG_DEF_FROM_EDGE (iv_phi, latch_e);
|
|
|
access_fn = analyze_scalar_evolution (iv_loop, PHI_RESULT (iv_phi));
|
access_fn = analyze_scalar_evolution (iv_loop, PHI_RESULT (iv_phi));
|
gcc_assert (access_fn);
|
gcc_assert (access_fn);
|
ok = vect_is_simple_iv_evolution (iv_loop->num, access_fn,
|
ok = vect_is_simple_iv_evolution (iv_loop->num, access_fn,
|
&init_expr, &step_expr);
|
&init_expr, &step_expr);
|
gcc_assert (ok);
|
gcc_assert (ok);
|
pe = loop_preheader_edge (iv_loop);
|
pe = loop_preheader_edge (iv_loop);
|
|
|
/* Create the vector that holds the initial_value of the induction. */
|
/* Create the vector that holds the initial_value of the induction. */
|
if (nested_in_vect_loop)
|
if (nested_in_vect_loop)
|
{
|
{
|
/* iv_loop is nested in the loop to be vectorized. init_expr had already
|
/* iv_loop is nested in the loop to be vectorized. init_expr had already
|
been created during vectorization of previous stmts; We obtain it from
|
been created during vectorization of previous stmts; We obtain it from
|
the STMT_VINFO_VEC_STMT of the defining stmt. */
|
the STMT_VINFO_VEC_STMT of the defining stmt. */
|
tree iv_def = PHI_ARG_DEF_FROM_EDGE (iv_phi,
|
tree iv_def = PHI_ARG_DEF_FROM_EDGE (iv_phi,
|
loop_preheader_edge (iv_loop));
|
loop_preheader_edge (iv_loop));
|
vec_init = vect_get_vec_def_for_operand (iv_def, iv_phi, NULL);
|
vec_init = vect_get_vec_def_for_operand (iv_def, iv_phi, NULL);
|
}
|
}
|
else
|
else
|
{
|
{
|
/* iv_loop is the loop to be vectorized. Create:
|
/* iv_loop is the loop to be vectorized. Create:
|
vec_init = [X, X+S, X+2*S, X+3*S] (S = step_expr, X = init_expr) */
|
vec_init = [X, X+S, X+2*S, X+3*S] (S = step_expr, X = init_expr) */
|
new_var = vect_get_new_vect_var (scalar_type, vect_scalar_var, "var_");
|
new_var = vect_get_new_vect_var (scalar_type, vect_scalar_var, "var_");
|
add_referenced_var (new_var);
|
add_referenced_var (new_var);
|
|
|
new_name = force_gimple_operand (init_expr, &stmts, false, new_var);
|
new_name = force_gimple_operand (init_expr, &stmts, false, new_var);
|
if (stmts)
|
if (stmts)
|
{
|
{
|
new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
|
new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
|
gcc_assert (!new_bb);
|
gcc_assert (!new_bb);
|
}
|
}
|
|
|
t = NULL_TREE;
|
t = NULL_TREE;
|
t = tree_cons (NULL_TREE, init_expr, t);
|
t = tree_cons (NULL_TREE, init_expr, t);
|
for (i = 1; i < nunits; i++)
|
for (i = 1; i < nunits; i++)
|
{
|
{
|
/* Create: new_name_i = new_name + step_expr */
|
/* Create: new_name_i = new_name + step_expr */
|
enum tree_code code = POINTER_TYPE_P (scalar_type)
|
enum tree_code code = POINTER_TYPE_P (scalar_type)
|
? POINTER_PLUS_EXPR : PLUS_EXPR;
|
? POINTER_PLUS_EXPR : PLUS_EXPR;
|
init_stmt = gimple_build_assign_with_ops (code, new_var,
|
init_stmt = gimple_build_assign_with_ops (code, new_var,
|
new_name, step_expr);
|
new_name, step_expr);
|
new_name = make_ssa_name (new_var, init_stmt);
|
new_name = make_ssa_name (new_var, init_stmt);
|
gimple_assign_set_lhs (init_stmt, new_name);
|
gimple_assign_set_lhs (init_stmt, new_name);
|
|
|
new_bb = gsi_insert_on_edge_immediate (pe, init_stmt);
|
new_bb = gsi_insert_on_edge_immediate (pe, init_stmt);
|
gcc_assert (!new_bb);
|
gcc_assert (!new_bb);
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "created new init_stmt: ");
|
fprintf (vect_dump, "created new init_stmt: ");
|
print_gimple_stmt (vect_dump, init_stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, init_stmt, 0, TDF_SLIM);
|
}
|
}
|
t = tree_cons (NULL_TREE, new_name, t);
|
t = tree_cons (NULL_TREE, new_name, t);
|
}
|
}
|
/* Create a vector from [new_name_0, new_name_1, ..., new_name_nunits-1] */
|
/* Create a vector from [new_name_0, new_name_1, ..., new_name_nunits-1] */
|
vec = build_constructor_from_list (vectype, nreverse (t));
|
vec = build_constructor_from_list (vectype, nreverse (t));
|
vec_init = vect_init_vector (iv_phi, vec, vectype, NULL);
|
vec_init = vect_init_vector (iv_phi, vec, vectype, NULL);
|
}
|
}
|
|
|
|
|
/* Create the vector that holds the step of the induction. */
|
/* Create the vector that holds the step of the induction. */
|
if (nested_in_vect_loop)
|
if (nested_in_vect_loop)
|
/* iv_loop is nested in the loop to be vectorized. Generate:
|
/* iv_loop is nested in the loop to be vectorized. Generate:
|
vec_step = [S, S, S, S] */
|
vec_step = [S, S, S, S] */
|
new_name = step_expr;
|
new_name = step_expr;
|
else
|
else
|
{
|
{
|
/* iv_loop is the loop to be vectorized. Generate:
|
/* iv_loop is the loop to be vectorized. Generate:
|
vec_step = [VF*S, VF*S, VF*S, VF*S] */
|
vec_step = [VF*S, VF*S, VF*S, VF*S] */
|
expr = build_int_cst (TREE_TYPE (step_expr), vf);
|
expr = build_int_cst (TREE_TYPE (step_expr), vf);
|
new_name = fold_build2 (MULT_EXPR, TREE_TYPE (step_expr),
|
new_name = fold_build2 (MULT_EXPR, TREE_TYPE (step_expr),
|
expr, step_expr);
|
expr, step_expr);
|
}
|
}
|
|
|
t = NULL_TREE;
|
t = NULL_TREE;
|
for (i = 0; i < nunits; i++)
|
for (i = 0; i < nunits; i++)
|
t = tree_cons (NULL_TREE, unshare_expr (new_name), t);
|
t = tree_cons (NULL_TREE, unshare_expr (new_name), t);
|
gcc_assert (CONSTANT_CLASS_P (new_name));
|
gcc_assert (CONSTANT_CLASS_P (new_name));
|
stepvectype = get_vectype_for_scalar_type (TREE_TYPE (new_name));
|
stepvectype = get_vectype_for_scalar_type (TREE_TYPE (new_name));
|
gcc_assert (stepvectype);
|
gcc_assert (stepvectype);
|
vec = build_vector (stepvectype, t);
|
vec = build_vector (stepvectype, t);
|
vec_step = vect_init_vector (iv_phi, vec, stepvectype, NULL);
|
vec_step = vect_init_vector (iv_phi, vec, stepvectype, NULL);
|
|
|
|
|
/* Create the following def-use cycle:
|
/* Create the following def-use cycle:
|
loop prolog:
|
loop prolog:
|
vec_init = ...
|
vec_init = ...
|
vec_step = ...
|
vec_step = ...
|
loop:
|
loop:
|
vec_iv = PHI <vec_init, vec_loop>
|
vec_iv = PHI <vec_init, vec_loop>
|
...
|
...
|
STMT
|
STMT
|
...
|
...
|
vec_loop = vec_iv + vec_step; */
|
vec_loop = vec_iv + vec_step; */
|
|
|
/* Create the induction-phi that defines the induction-operand. */
|
/* Create the induction-phi that defines the induction-operand. */
|
vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_");
|
vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_");
|
add_referenced_var (vec_dest);
|
add_referenced_var (vec_dest);
|
induction_phi = create_phi_node (vec_dest, iv_loop->header);
|
induction_phi = create_phi_node (vec_dest, iv_loop->header);
|
set_vinfo_for_stmt (induction_phi,
|
set_vinfo_for_stmt (induction_phi,
|
new_stmt_vec_info (induction_phi, loop_vinfo, NULL));
|
new_stmt_vec_info (induction_phi, loop_vinfo, NULL));
|
induc_def = PHI_RESULT (induction_phi);
|
induc_def = PHI_RESULT (induction_phi);
|
|
|
/* Create the iv update inside the loop */
|
/* Create the iv update inside the loop */
|
new_stmt = gimple_build_assign_with_ops (PLUS_EXPR, vec_dest,
|
new_stmt = gimple_build_assign_with_ops (PLUS_EXPR, vec_dest,
|
induc_def, vec_step);
|
induc_def, vec_step);
|
vec_def = make_ssa_name (vec_dest, new_stmt);
|
vec_def = make_ssa_name (vec_dest, new_stmt);
|
gimple_assign_set_lhs (new_stmt, vec_def);
|
gimple_assign_set_lhs (new_stmt, vec_def);
|
gsi_insert_before (&si, new_stmt, GSI_SAME_STMT);
|
gsi_insert_before (&si, new_stmt, GSI_SAME_STMT);
|
set_vinfo_for_stmt (new_stmt, new_stmt_vec_info (new_stmt, loop_vinfo,
|
set_vinfo_for_stmt (new_stmt, new_stmt_vec_info (new_stmt, loop_vinfo,
|
NULL));
|
NULL));
|
|
|
/* Set the arguments of the phi node: */
|
/* Set the arguments of the phi node: */
|
add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION);
|
add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION);
|
add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
|
add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop),
|
UNKNOWN_LOCATION);
|
UNKNOWN_LOCATION);
|
|
|
|
|
/* In case that vectorization factor (VF) is bigger than the number
|
/* In case that vectorization factor (VF) is bigger than the number
|
of elements that we can fit in a vectype (nunits), we have to generate
|
of elements that we can fit in a vectype (nunits), we have to generate
|
more than one vector stmt - i.e - we need to "unroll" the
|
more than one vector stmt - i.e - we need to "unroll" the
|
vector stmt by a factor VF/nunits. For more details see documentation
|
vector stmt by a factor VF/nunits. For more details see documentation
|
in vectorizable_operation. */
|
in vectorizable_operation. */
|
|
|
if (ncopies > 1)
|
if (ncopies > 1)
|
{
|
{
|
stmt_vec_info prev_stmt_vinfo;
|
stmt_vec_info prev_stmt_vinfo;
|
/* FORNOW. This restriction should be relaxed. */
|
/* FORNOW. This restriction should be relaxed. */
|
gcc_assert (!nested_in_vect_loop);
|
gcc_assert (!nested_in_vect_loop);
|
|
|
/* Create the vector that holds the step of the induction. */
|
/* Create the vector that holds the step of the induction. */
|
expr = build_int_cst (TREE_TYPE (step_expr), nunits);
|
expr = build_int_cst (TREE_TYPE (step_expr), nunits);
|
new_name = fold_build2 (MULT_EXPR, TREE_TYPE (step_expr),
|
new_name = fold_build2 (MULT_EXPR, TREE_TYPE (step_expr),
|
expr, step_expr);
|
expr, step_expr);
|
t = NULL_TREE;
|
t = NULL_TREE;
|
for (i = 0; i < nunits; i++)
|
for (i = 0; i < nunits; i++)
|
t = tree_cons (NULL_TREE, unshare_expr (new_name), t);
|
t = tree_cons (NULL_TREE, unshare_expr (new_name), t);
|
gcc_assert (CONSTANT_CLASS_P (new_name));
|
gcc_assert (CONSTANT_CLASS_P (new_name));
|
vec = build_vector (stepvectype, t);
|
vec = build_vector (stepvectype, t);
|
vec_step = vect_init_vector (iv_phi, vec, stepvectype, NULL);
|
vec_step = vect_init_vector (iv_phi, vec, stepvectype, NULL);
|
|
|
vec_def = induc_def;
|
vec_def = induc_def;
|
prev_stmt_vinfo = vinfo_for_stmt (induction_phi);
|
prev_stmt_vinfo = vinfo_for_stmt (induction_phi);
|
for (i = 1; i < ncopies; i++)
|
for (i = 1; i < ncopies; i++)
|
{
|
{
|
/* vec_i = vec_prev + vec_step */
|
/* vec_i = vec_prev + vec_step */
|
new_stmt = gimple_build_assign_with_ops (PLUS_EXPR, vec_dest,
|
new_stmt = gimple_build_assign_with_ops (PLUS_EXPR, vec_dest,
|
vec_def, vec_step);
|
vec_def, vec_step);
|
vec_def = make_ssa_name (vec_dest, new_stmt);
|
vec_def = make_ssa_name (vec_dest, new_stmt);
|
gimple_assign_set_lhs (new_stmt, vec_def);
|
gimple_assign_set_lhs (new_stmt, vec_def);
|
|
|
gsi_insert_before (&si, new_stmt, GSI_SAME_STMT);
|
gsi_insert_before (&si, new_stmt, GSI_SAME_STMT);
|
set_vinfo_for_stmt (new_stmt,
|
set_vinfo_for_stmt (new_stmt,
|
new_stmt_vec_info (new_stmt, loop_vinfo, NULL));
|
new_stmt_vec_info (new_stmt, loop_vinfo, NULL));
|
STMT_VINFO_RELATED_STMT (prev_stmt_vinfo) = new_stmt;
|
STMT_VINFO_RELATED_STMT (prev_stmt_vinfo) = new_stmt;
|
prev_stmt_vinfo = vinfo_for_stmt (new_stmt);
|
prev_stmt_vinfo = vinfo_for_stmt (new_stmt);
|
}
|
}
|
}
|
}
|
|
|
if (nested_in_vect_loop)
|
if (nested_in_vect_loop)
|
{
|
{
|
/* Find the loop-closed exit-phi of the induction, and record
|
/* Find the loop-closed exit-phi of the induction, and record
|
the final vector of induction results: */
|
the final vector of induction results: */
|
exit_phi = NULL;
|
exit_phi = NULL;
|
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, loop_arg)
|
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, loop_arg)
|
{
|
{
|
if (!flow_bb_inside_loop_p (iv_loop, gimple_bb (USE_STMT (use_p))))
|
if (!flow_bb_inside_loop_p (iv_loop, gimple_bb (USE_STMT (use_p))))
|
{
|
{
|
exit_phi = USE_STMT (use_p);
|
exit_phi = USE_STMT (use_p);
|
break;
|
break;
|
}
|
}
|
}
|
}
|
if (exit_phi)
|
if (exit_phi)
|
{
|
{
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (exit_phi);
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (exit_phi);
|
/* FORNOW. Currently not supporting the case that an inner-loop induction
|
/* FORNOW. Currently not supporting the case that an inner-loop induction
|
is not used in the outer-loop (i.e. only outside the outer-loop). */
|
is not used in the outer-loop (i.e. only outside the outer-loop). */
|
gcc_assert (STMT_VINFO_RELEVANT_P (stmt_vinfo)
|
gcc_assert (STMT_VINFO_RELEVANT_P (stmt_vinfo)
|
&& !STMT_VINFO_LIVE_P (stmt_vinfo));
|
&& !STMT_VINFO_LIVE_P (stmt_vinfo));
|
|
|
STMT_VINFO_VEC_STMT (stmt_vinfo) = new_stmt;
|
STMT_VINFO_VEC_STMT (stmt_vinfo) = new_stmt;
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "vector of inductions after inner-loop:");
|
fprintf (vect_dump, "vector of inductions after inner-loop:");
|
print_gimple_stmt (vect_dump, new_stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, new_stmt, 0, TDF_SLIM);
|
}
|
}
|
}
|
}
|
}
|
}
|
|
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "transform induction: created def-use cycle: ");
|
fprintf (vect_dump, "transform induction: created def-use cycle: ");
|
print_gimple_stmt (vect_dump, induction_phi, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, induction_phi, 0, TDF_SLIM);
|
fprintf (vect_dump, "\n");
|
fprintf (vect_dump, "\n");
|
print_gimple_stmt (vect_dump, SSA_NAME_DEF_STMT (vec_def), 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, SSA_NAME_DEF_STMT (vec_def), 0, TDF_SLIM);
|
}
|
}
|
|
|
STMT_VINFO_VEC_STMT (phi_info) = induction_phi;
|
STMT_VINFO_VEC_STMT (phi_info) = induction_phi;
|
return induc_def;
|
return induc_def;
|
}
|
}
|
|
|
|
|
/* Function get_initial_def_for_reduction
|
/* Function get_initial_def_for_reduction
|
|
|
Input:
|
Input:
|
STMT - a stmt that performs a reduction operation in the loop.
|
STMT - a stmt that performs a reduction operation in the loop.
|
INIT_VAL - the initial value of the reduction variable
|
INIT_VAL - the initial value of the reduction variable
|
|
|
Output:
|
Output:
|
ADJUSTMENT_DEF - a tree that holds a value to be added to the final result
|
ADJUSTMENT_DEF - a tree that holds a value to be added to the final result
|
of the reduction (used for adjusting the epilog - see below).
|
of the reduction (used for adjusting the epilog - see below).
|
Return a vector variable, initialized according to the operation that STMT
|
Return a vector variable, initialized according to the operation that STMT
|
performs. This vector will be used as the initial value of the
|
performs. This vector will be used as the initial value of the
|
vector of partial results.
|
vector of partial results.
|
|
|
Option1 (adjust in epilog): Initialize the vector as follows:
|
Option1 (adjust in epilog): Initialize the vector as follows:
|
add/bit or/xor: [0,0,...,0,0]
|
add/bit or/xor: [0,0,...,0,0]
|
mult/bit and: [1,1,...,1,1]
|
mult/bit and: [1,1,...,1,1]
|
min/max/cond_expr: [init_val,init_val,..,init_val,init_val]
|
min/max/cond_expr: [init_val,init_val,..,init_val,init_val]
|
and when necessary (e.g. add/mult case) let the caller know
|
and when necessary (e.g. add/mult case) let the caller know
|
that it needs to adjust the result by init_val.
|
that it needs to adjust the result by init_val.
|
|
|
Option2: Initialize the vector as follows:
|
Option2: Initialize the vector as follows:
|
add/bit or/xor: [init_val,0,0,...,0]
|
add/bit or/xor: [init_val,0,0,...,0]
|
mult/bit and: [init_val,1,1,...,1]
|
mult/bit and: [init_val,1,1,...,1]
|
min/max/cond_expr: [init_val,init_val,...,init_val]
|
min/max/cond_expr: [init_val,init_val,...,init_val]
|
and no adjustments are needed.
|
and no adjustments are needed.
|
|
|
For example, for the following code:
|
For example, for the following code:
|
|
|
s = init_val;
|
s = init_val;
|
for (i=0;i<n;i++)
|
for (i=0;i<n;i++)
|
s = s + a[i];
|
s = s + a[i];
|
|
|
STMT is 's = s + a[i]', and the reduction variable is 's'.
|
STMT is 's = s + a[i]', and the reduction variable is 's'.
|
For a vector of 4 units, we want to return either [0,0,0,init_val],
|
For a vector of 4 units, we want to return either [0,0,0,init_val],
|
or [0,0,0,0] and let the caller know that it needs to adjust
|
or [0,0,0,0] and let the caller know that it needs to adjust
|
the result at the end by 'init_val'.
|
the result at the end by 'init_val'.
|
|
|
FORNOW, we are using the 'adjust in epilog' scheme, because this way the
|
FORNOW, we are using the 'adjust in epilog' scheme, because this way the
|
initialization vector is simpler (same element in all entries), if
|
initialization vector is simpler (same element in all entries), if
|
ADJUSTMENT_DEF is not NULL, and Option2 otherwise.
|
ADJUSTMENT_DEF is not NULL, and Option2 otherwise.
|
|
|
A cost model should help decide between these two schemes. */
|
A cost model should help decide between these two schemes. */
|
|
|
tree
|
tree
|
get_initial_def_for_reduction (gimple stmt, tree init_val,
|
get_initial_def_for_reduction (gimple stmt, tree init_val,
|
tree *adjustment_def)
|
tree *adjustment_def)
|
{
|
{
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
tree scalar_type = TREE_TYPE (init_val);
|
tree scalar_type = TREE_TYPE (init_val);
|
tree vectype = get_vectype_for_scalar_type (scalar_type);
|
tree vectype = get_vectype_for_scalar_type (scalar_type);
|
int nunits;
|
int nunits;
|
enum tree_code code = gimple_assign_rhs_code (stmt);
|
enum tree_code code = gimple_assign_rhs_code (stmt);
|
tree def_for_init;
|
tree def_for_init;
|
tree init_def;
|
tree init_def;
|
tree t = NULL_TREE;
|
tree t = NULL_TREE;
|
int i;
|
int i;
|
bool nested_in_vect_loop = false;
|
bool nested_in_vect_loop = false;
|
tree init_value;
|
tree init_value;
|
REAL_VALUE_TYPE real_init_val = dconst0;
|
REAL_VALUE_TYPE real_init_val = dconst0;
|
int int_init_val = 0;
|
int int_init_val = 0;
|
gimple def_stmt = NULL;
|
gimple def_stmt = NULL;
|
|
|
gcc_assert (vectype);
|
gcc_assert (vectype);
|
nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
|
|
gcc_assert (POINTER_TYPE_P (scalar_type) || INTEGRAL_TYPE_P (scalar_type)
|
gcc_assert (POINTER_TYPE_P (scalar_type) || INTEGRAL_TYPE_P (scalar_type)
|
|| SCALAR_FLOAT_TYPE_P (scalar_type));
|
|| SCALAR_FLOAT_TYPE_P (scalar_type));
|
|
|
if (nested_in_vect_loop_p (loop, stmt))
|
if (nested_in_vect_loop_p (loop, stmt))
|
nested_in_vect_loop = true;
|
nested_in_vect_loop = true;
|
else
|
else
|
gcc_assert (loop == (gimple_bb (stmt))->loop_father);
|
gcc_assert (loop == (gimple_bb (stmt))->loop_father);
|
|
|
/* In case of double reduction we only create a vector variable to be put
|
/* In case of double reduction we only create a vector variable to be put
|
in the reduction phi node. The actual statement creation is done in
|
in the reduction phi node. The actual statement creation is done in
|
vect_create_epilog_for_reduction. */
|
vect_create_epilog_for_reduction. */
|
if (adjustment_def && nested_in_vect_loop
|
if (adjustment_def && nested_in_vect_loop
|
&& TREE_CODE (init_val) == SSA_NAME
|
&& TREE_CODE (init_val) == SSA_NAME
|
&& (def_stmt = SSA_NAME_DEF_STMT (init_val))
|
&& (def_stmt = SSA_NAME_DEF_STMT (init_val))
|
&& gimple_code (def_stmt) == GIMPLE_PHI
|
&& gimple_code (def_stmt) == GIMPLE_PHI
|
&& flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
|
&& flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
|
&& vinfo_for_stmt (def_stmt)
|
&& vinfo_for_stmt (def_stmt)
|
&& STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt))
|
&& STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt))
|
== vect_double_reduction_def)
|
== vect_double_reduction_def)
|
{
|
{
|
*adjustment_def = NULL;
|
*adjustment_def = NULL;
|
return vect_create_destination_var (init_val, vectype);
|
return vect_create_destination_var (init_val, vectype);
|
}
|
}
|
|
|
if (TREE_CONSTANT (init_val))
|
if (TREE_CONSTANT (init_val))
|
{
|
{
|
if (SCALAR_FLOAT_TYPE_P (scalar_type))
|
if (SCALAR_FLOAT_TYPE_P (scalar_type))
|
init_value = build_real (scalar_type, TREE_REAL_CST (init_val));
|
init_value = build_real (scalar_type, TREE_REAL_CST (init_val));
|
else
|
else
|
init_value = build_int_cst (scalar_type, TREE_INT_CST_LOW (init_val));
|
init_value = build_int_cst (scalar_type, TREE_INT_CST_LOW (init_val));
|
}
|
}
|
else
|
else
|
init_value = init_val;
|
init_value = init_val;
|
|
|
switch (code)
|
switch (code)
|
{
|
{
|
case WIDEN_SUM_EXPR:
|
case WIDEN_SUM_EXPR:
|
case DOT_PROD_EXPR:
|
case DOT_PROD_EXPR:
|
case PLUS_EXPR:
|
case PLUS_EXPR:
|
case MINUS_EXPR:
|
case MINUS_EXPR:
|
case BIT_IOR_EXPR:
|
case BIT_IOR_EXPR:
|
case BIT_XOR_EXPR:
|
case BIT_XOR_EXPR:
|
case MULT_EXPR:
|
case MULT_EXPR:
|
case BIT_AND_EXPR:
|
case BIT_AND_EXPR:
|
/* ADJUSMENT_DEF is NULL when called from
|
/* ADJUSMENT_DEF is NULL when called from
|
vect_create_epilog_for_reduction to vectorize double reduction. */
|
vect_create_epilog_for_reduction to vectorize double reduction. */
|
if (adjustment_def)
|
if (adjustment_def)
|
{
|
{
|
if (nested_in_vect_loop)
|
if (nested_in_vect_loop)
|
*adjustment_def = vect_get_vec_def_for_operand (init_val, stmt,
|
*adjustment_def = vect_get_vec_def_for_operand (init_val, stmt,
|
NULL);
|
NULL);
|
else
|
else
|
*adjustment_def = init_val;
|
*adjustment_def = init_val;
|
}
|
}
|
|
|
if (code == MULT_EXPR)
|
if (code == MULT_EXPR)
|
{
|
{
|
real_init_val = dconst1;
|
real_init_val = dconst1;
|
int_init_val = 1;
|
int_init_val = 1;
|
}
|
}
|
|
|
if (code == BIT_AND_EXPR)
|
if (code == BIT_AND_EXPR)
|
int_init_val = -1;
|
int_init_val = -1;
|
|
|
if (SCALAR_FLOAT_TYPE_P (scalar_type))
|
if (SCALAR_FLOAT_TYPE_P (scalar_type))
|
def_for_init = build_real (scalar_type, real_init_val);
|
def_for_init = build_real (scalar_type, real_init_val);
|
else
|
else
|
def_for_init = build_int_cst (scalar_type, int_init_val);
|
def_for_init = build_int_cst (scalar_type, int_init_val);
|
|
|
/* Create a vector of '0' or '1' except the first element. */
|
/* Create a vector of '0' or '1' except the first element. */
|
for (i = nunits - 2; i >= 0; --i)
|
for (i = nunits - 2; i >= 0; --i)
|
t = tree_cons (NULL_TREE, def_for_init, t);
|
t = tree_cons (NULL_TREE, def_for_init, t);
|
|
|
/* Option1: the first element is '0' or '1' as well. */
|
/* Option1: the first element is '0' or '1' as well. */
|
if (adjustment_def)
|
if (adjustment_def)
|
{
|
{
|
t = tree_cons (NULL_TREE, def_for_init, t);
|
t = tree_cons (NULL_TREE, def_for_init, t);
|
init_def = build_vector (vectype, t);
|
init_def = build_vector (vectype, t);
|
break;
|
break;
|
}
|
}
|
|
|
/* Option2: the first element is INIT_VAL. */
|
/* Option2: the first element is INIT_VAL. */
|
t = tree_cons (NULL_TREE, init_value, t);
|
t = tree_cons (NULL_TREE, init_value, t);
|
if (TREE_CONSTANT (init_val))
|
if (TREE_CONSTANT (init_val))
|
init_def = build_vector (vectype, t);
|
init_def = build_vector (vectype, t);
|
else
|
else
|
init_def = build_constructor_from_list (vectype, t);
|
init_def = build_constructor_from_list (vectype, t);
|
|
|
break;
|
break;
|
|
|
case MIN_EXPR:
|
case MIN_EXPR:
|
case MAX_EXPR:
|
case MAX_EXPR:
|
case COND_EXPR:
|
case COND_EXPR:
|
if (adjustment_def)
|
if (adjustment_def)
|
{
|
{
|
*adjustment_def = NULL_TREE;
|
*adjustment_def = NULL_TREE;
|
init_def = vect_get_vec_def_for_operand (init_val, stmt, NULL);
|
init_def = vect_get_vec_def_for_operand (init_val, stmt, NULL);
|
break;
|
break;
|
}
|
}
|
|
|
for (i = nunits - 1; i >= 0; --i)
|
for (i = nunits - 1; i >= 0; --i)
|
t = tree_cons (NULL_TREE, init_value, t);
|
t = tree_cons (NULL_TREE, init_value, t);
|
|
|
if (TREE_CONSTANT (init_val))
|
if (TREE_CONSTANT (init_val))
|
init_def = build_vector (vectype, t);
|
init_def = build_vector (vectype, t);
|
else
|
else
|
init_def = build_constructor_from_list (vectype, t);
|
init_def = build_constructor_from_list (vectype, t);
|
|
|
break;
|
break;
|
|
|
default:
|
default:
|
gcc_unreachable ();
|
gcc_unreachable ();
|
}
|
}
|
|
|
return init_def;
|
return init_def;
|
}
|
}
|
|
|
|
|
/* Function vect_create_epilog_for_reduction
|
/* Function vect_create_epilog_for_reduction
|
|
|
Create code at the loop-epilog to finalize the result of a reduction
|
Create code at the loop-epilog to finalize the result of a reduction
|
computation.
|
computation.
|
|
|
VECT_DEF is a vector of partial results.
|
VECT_DEF is a vector of partial results.
|
REDUC_CODE is the tree-code for the epilog reduction.
|
REDUC_CODE is the tree-code for the epilog reduction.
|
NCOPIES is > 1 in case the vectorization factor (VF) is bigger than the
|
NCOPIES is > 1 in case the vectorization factor (VF) is bigger than the
|
number of elements that we can fit in a vectype (nunits). In this case
|
number of elements that we can fit in a vectype (nunits). In this case
|
we have to generate more than one vector stmt - i.e - we need to "unroll"
|
we have to generate more than one vector stmt - i.e - we need to "unroll"
|
the vector stmt by a factor VF/nunits. For more details see documentation
|
the vector stmt by a factor VF/nunits. For more details see documentation
|
in vectorizable_operation.
|
in vectorizable_operation.
|
STMT is the scalar reduction stmt that is being vectorized.
|
STMT is the scalar reduction stmt that is being vectorized.
|
REDUCTION_PHI is the phi-node that carries the reduction computation.
|
REDUCTION_PHI is the phi-node that carries the reduction computation.
|
REDUC_INDEX is the index of the operand in the right hand side of the
|
REDUC_INDEX is the index of the operand in the right hand side of the
|
statement that is defined by REDUCTION_PHI.
|
statement that is defined by REDUCTION_PHI.
|
DOUBLE_REDUC is TRUE if double reduction phi nodes should be handled.
|
DOUBLE_REDUC is TRUE if double reduction phi nodes should be handled.
|
|
|
This function:
|
This function:
|
1. Creates the reduction def-use cycle: sets the arguments for
|
1. Creates the reduction def-use cycle: sets the arguments for
|
REDUCTION_PHI:
|
REDUCTION_PHI:
|
The loop-entry argument is the vectorized initial-value of the reduction.
|
The loop-entry argument is the vectorized initial-value of the reduction.
|
The loop-latch argument is VECT_DEF - the vector of partial sums.
|
The loop-latch argument is VECT_DEF - the vector of partial sums.
|
2. "Reduces" the vector of partial results VECT_DEF into a single result,
|
2. "Reduces" the vector of partial results VECT_DEF into a single result,
|
by applying the operation specified by REDUC_CODE if available, or by
|
by applying the operation specified by REDUC_CODE if available, or by
|
other means (whole-vector shifts or a scalar loop).
|
other means (whole-vector shifts or a scalar loop).
|
The function also creates a new phi node at the loop exit to preserve
|
The function also creates a new phi node at the loop exit to preserve
|
loop-closed form, as illustrated below.
|
loop-closed form, as illustrated below.
|
|
|
The flow at the entry to this function:
|
The flow at the entry to this function:
|
|
|
loop:
|
loop:
|
vec_def = phi <null, null> # REDUCTION_PHI
|
vec_def = phi <null, null> # REDUCTION_PHI
|
VECT_DEF = vector_stmt # vectorized form of STMT
|
VECT_DEF = vector_stmt # vectorized form of STMT
|
s_loop = scalar_stmt # (scalar) STMT
|
s_loop = scalar_stmt # (scalar) STMT
|
loop_exit:
|
loop_exit:
|
s_out0 = phi <s_loop> # (scalar) EXIT_PHI
|
s_out0 = phi <s_loop> # (scalar) EXIT_PHI
|
use <s_out0>
|
use <s_out0>
|
use <s_out0>
|
use <s_out0>
|
|
|
The above is transformed by this function into:
|
The above is transformed by this function into:
|
|
|
loop:
|
loop:
|
vec_def = phi <vec_init, VECT_DEF> # REDUCTION_PHI
|
vec_def = phi <vec_init, VECT_DEF> # REDUCTION_PHI
|
VECT_DEF = vector_stmt # vectorized form of STMT
|
VECT_DEF = vector_stmt # vectorized form of STMT
|
s_loop = scalar_stmt # (scalar) STMT
|
s_loop = scalar_stmt # (scalar) STMT
|
loop_exit:
|
loop_exit:
|
s_out0 = phi <s_loop> # (scalar) EXIT_PHI
|
s_out0 = phi <s_loop> # (scalar) EXIT_PHI
|
v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
|
v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
|
v_out2 = reduce <v_out1>
|
v_out2 = reduce <v_out1>
|
s_out3 = extract_field <v_out2, 0>
|
s_out3 = extract_field <v_out2, 0>
|
s_out4 = adjust_result <s_out3>
|
s_out4 = adjust_result <s_out3>
|
use <s_out4>
|
use <s_out4>
|
use <s_out4>
|
use <s_out4>
|
*/
|
*/
|
|
|
static void
|
static void
|
vect_create_epilog_for_reduction (tree vect_def, gimple stmt,
|
vect_create_epilog_for_reduction (tree vect_def, gimple stmt,
|
int ncopies,
|
int ncopies,
|
enum tree_code reduc_code,
|
enum tree_code reduc_code,
|
gimple reduction_phi,
|
gimple reduction_phi,
|
int reduc_index,
|
int reduc_index,
|
bool double_reduc)
|
bool double_reduc)
|
{
|
{
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
stmt_vec_info prev_phi_info;
|
stmt_vec_info prev_phi_info;
|
tree vectype;
|
tree vectype;
|
enum machine_mode mode;
|
enum machine_mode mode;
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo), *outer_loop = NULL;
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo), *outer_loop = NULL;
|
basic_block exit_bb;
|
basic_block exit_bb;
|
tree scalar_dest;
|
tree scalar_dest;
|
tree scalar_type;
|
tree scalar_type;
|
gimple new_phi = NULL, phi;
|
gimple new_phi = NULL, phi;
|
gimple_stmt_iterator exit_gsi;
|
gimple_stmt_iterator exit_gsi;
|
tree vec_dest;
|
tree vec_dest;
|
tree new_temp = NULL_TREE;
|
tree new_temp = NULL_TREE;
|
tree new_name;
|
tree new_name;
|
gimple epilog_stmt = NULL;
|
gimple epilog_stmt = NULL;
|
tree new_scalar_dest, new_dest;
|
tree new_scalar_dest, new_dest;
|
gimple exit_phi;
|
gimple exit_phi;
|
tree bitsize, bitpos;
|
tree bitsize, bitpos;
|
enum tree_code code = gimple_assign_rhs_code (stmt);
|
enum tree_code code = gimple_assign_rhs_code (stmt);
|
tree adjustment_def;
|
tree adjustment_def;
|
tree vec_initial_def, def;
|
tree vec_initial_def, def;
|
tree orig_name;
|
tree orig_name;
|
imm_use_iterator imm_iter;
|
imm_use_iterator imm_iter;
|
use_operand_p use_p;
|
use_operand_p use_p;
|
bool extract_scalar_result = false;
|
bool extract_scalar_result = false;
|
tree reduction_op, expr;
|
tree reduction_op, expr;
|
gimple orig_stmt;
|
gimple orig_stmt;
|
gimple use_stmt;
|
gimple use_stmt;
|
bool nested_in_vect_loop = false;
|
bool nested_in_vect_loop = false;
|
VEC(gimple,heap) *phis = NULL;
|
VEC(gimple,heap) *phis = NULL;
|
enum vect_def_type dt = vect_unknown_def_type;
|
enum vect_def_type dt = vect_unknown_def_type;
|
int j, i;
|
int j, i;
|
|
|
if (nested_in_vect_loop_p (loop, stmt))
|
if (nested_in_vect_loop_p (loop, stmt))
|
{
|
{
|
outer_loop = loop;
|
outer_loop = loop;
|
loop = loop->inner;
|
loop = loop->inner;
|
nested_in_vect_loop = true;
|
nested_in_vect_loop = true;
|
}
|
}
|
|
|
switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt)))
|
switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt)))
|
{
|
{
|
case GIMPLE_SINGLE_RHS:
|
case GIMPLE_SINGLE_RHS:
|
gcc_assert (TREE_OPERAND_LENGTH (gimple_assign_rhs1 (stmt))
|
gcc_assert (TREE_OPERAND_LENGTH (gimple_assign_rhs1 (stmt))
|
== ternary_op);
|
== ternary_op);
|
reduction_op = TREE_OPERAND (gimple_assign_rhs1 (stmt), reduc_index);
|
reduction_op = TREE_OPERAND (gimple_assign_rhs1 (stmt), reduc_index);
|
break;
|
break;
|
case GIMPLE_UNARY_RHS:
|
case GIMPLE_UNARY_RHS:
|
reduction_op = gimple_assign_rhs1 (stmt);
|
reduction_op = gimple_assign_rhs1 (stmt);
|
break;
|
break;
|
case GIMPLE_BINARY_RHS:
|
case GIMPLE_BINARY_RHS:
|
reduction_op = reduc_index ?
|
reduction_op = reduc_index ?
|
gimple_assign_rhs2 (stmt) : gimple_assign_rhs1 (stmt);
|
gimple_assign_rhs2 (stmt) : gimple_assign_rhs1 (stmt);
|
break;
|
break;
|
default:
|
default:
|
gcc_unreachable ();
|
gcc_unreachable ();
|
}
|
}
|
|
|
vectype = get_vectype_for_scalar_type (TREE_TYPE (reduction_op));
|
vectype = get_vectype_for_scalar_type (TREE_TYPE (reduction_op));
|
gcc_assert (vectype);
|
gcc_assert (vectype);
|
mode = TYPE_MODE (vectype);
|
mode = TYPE_MODE (vectype);
|
|
|
/*** 1. Create the reduction def-use cycle ***/
|
/*** 1. Create the reduction def-use cycle ***/
|
|
|
/* For the case of reduction, vect_get_vec_def_for_operand returns
|
/* For the case of reduction, vect_get_vec_def_for_operand returns
|
the scalar def before the loop, that defines the initial value
|
the scalar def before the loop, that defines the initial value
|
of the reduction variable. */
|
of the reduction variable. */
|
vec_initial_def = vect_get_vec_def_for_operand (reduction_op, stmt,
|
vec_initial_def = vect_get_vec_def_for_operand (reduction_op, stmt,
|
&adjustment_def);
|
&adjustment_def);
|
|
|
phi = reduction_phi;
|
phi = reduction_phi;
|
def = vect_def;
|
def = vect_def;
|
for (j = 0; j < ncopies; j++)
|
for (j = 0; j < ncopies; j++)
|
{
|
{
|
/* 1.1 set the loop-entry arg of the reduction-phi: */
|
/* 1.1 set the loop-entry arg of the reduction-phi: */
|
add_phi_arg (phi, vec_initial_def, loop_preheader_edge (loop),
|
add_phi_arg (phi, vec_initial_def, loop_preheader_edge (loop),
|
UNKNOWN_LOCATION);
|
UNKNOWN_LOCATION);
|
|
|
/* 1.2 set the loop-latch arg for the reduction-phi: */
|
/* 1.2 set the loop-latch arg for the reduction-phi: */
|
if (j > 0)
|
if (j > 0)
|
def = vect_get_vec_def_for_stmt_copy (dt, def);
|
def = vect_get_vec_def_for_stmt_copy (dt, def);
|
add_phi_arg (phi, def, loop_latch_edge (loop), UNKNOWN_LOCATION);
|
add_phi_arg (phi, def, loop_latch_edge (loop), UNKNOWN_LOCATION);
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "transform reduction: created def-use cycle: ");
|
fprintf (vect_dump, "transform reduction: created def-use cycle: ");
|
print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
|
fprintf (vect_dump, "\n");
|
fprintf (vect_dump, "\n");
|
print_gimple_stmt (vect_dump, SSA_NAME_DEF_STMT (def), 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, SSA_NAME_DEF_STMT (def), 0, TDF_SLIM);
|
}
|
}
|
|
|
phi = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (phi));
|
phi = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (phi));
|
}
|
}
|
|
|
/*** 2. Create epilog code
|
/*** 2. Create epilog code
|
The reduction epilog code operates across the elements of the vector
|
The reduction epilog code operates across the elements of the vector
|
of partial results computed by the vectorized loop.
|
of partial results computed by the vectorized loop.
|
The reduction epilog code consists of:
|
The reduction epilog code consists of:
|
step 1: compute the scalar result in a vector (v_out2)
|
step 1: compute the scalar result in a vector (v_out2)
|
step 2: extract the scalar result (s_out3) from the vector (v_out2)
|
step 2: extract the scalar result (s_out3) from the vector (v_out2)
|
step 3: adjust the scalar result (s_out3) if needed.
|
step 3: adjust the scalar result (s_out3) if needed.
|
|
|
Step 1 can be accomplished using one the following three schemes:
|
Step 1 can be accomplished using one the following three schemes:
|
(scheme 1) using reduc_code, if available.
|
(scheme 1) using reduc_code, if available.
|
(scheme 2) using whole-vector shifts, if available.
|
(scheme 2) using whole-vector shifts, if available.
|
(scheme 3) using a scalar loop. In this case steps 1+2 above are
|
(scheme 3) using a scalar loop. In this case steps 1+2 above are
|
combined.
|
combined.
|
|
|
The overall epilog code looks like this:
|
The overall epilog code looks like this:
|
|
|
s_out0 = phi <s_loop> # original EXIT_PHI
|
s_out0 = phi <s_loop> # original EXIT_PHI
|
v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
|
v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
|
v_out2 = reduce <v_out1> # step 1
|
v_out2 = reduce <v_out1> # step 1
|
s_out3 = extract_field <v_out2, 0> # step 2
|
s_out3 = extract_field <v_out2, 0> # step 2
|
s_out4 = adjust_result <s_out3> # step 3
|
s_out4 = adjust_result <s_out3> # step 3
|
|
|
(step 3 is optional, and steps 1 and 2 may be combined).
|
(step 3 is optional, and steps 1 and 2 may be combined).
|
Lastly, the uses of s_out0 are replaced by s_out4.
|
Lastly, the uses of s_out0 are replaced by s_out4.
|
|
|
***/
|
***/
|
|
|
/* 2.1 Create new loop-exit-phi to preserve loop-closed form:
|
/* 2.1 Create new loop-exit-phi to preserve loop-closed form:
|
v_out1 = phi <v_loop> */
|
v_out1 = phi <v_loop> */
|
|
|
exit_bb = single_exit (loop)->dest;
|
exit_bb = single_exit (loop)->dest;
|
def = vect_def;
|
def = vect_def;
|
prev_phi_info = NULL;
|
prev_phi_info = NULL;
|
for (j = 0; j < ncopies; j++)
|
for (j = 0; j < ncopies; j++)
|
{
|
{
|
phi = create_phi_node (SSA_NAME_VAR (vect_def), exit_bb);
|
phi = create_phi_node (SSA_NAME_VAR (vect_def), exit_bb);
|
set_vinfo_for_stmt (phi, new_stmt_vec_info (phi, loop_vinfo, NULL));
|
set_vinfo_for_stmt (phi, new_stmt_vec_info (phi, loop_vinfo, NULL));
|
if (j == 0)
|
if (j == 0)
|
new_phi = phi;
|
new_phi = phi;
|
else
|
else
|
{
|
{
|
def = vect_get_vec_def_for_stmt_copy (dt, def);
|
def = vect_get_vec_def_for_stmt_copy (dt, def);
|
STMT_VINFO_RELATED_STMT (prev_phi_info) = phi;
|
STMT_VINFO_RELATED_STMT (prev_phi_info) = phi;
|
}
|
}
|
SET_PHI_ARG_DEF (phi, single_exit (loop)->dest_idx, def);
|
SET_PHI_ARG_DEF (phi, single_exit (loop)->dest_idx, def);
|
prev_phi_info = vinfo_for_stmt (phi);
|
prev_phi_info = vinfo_for_stmt (phi);
|
}
|
}
|
|
|
exit_gsi = gsi_after_labels (exit_bb);
|
exit_gsi = gsi_after_labels (exit_bb);
|
|
|
/* 2.2 Get the relevant tree-code to use in the epilog for schemes 2,3
|
/* 2.2 Get the relevant tree-code to use in the epilog for schemes 2,3
|
(i.e. when reduc_code is not available) and in the final adjustment
|
(i.e. when reduc_code is not available) and in the final adjustment
|
code (if needed). Also get the original scalar reduction variable as
|
code (if needed). Also get the original scalar reduction variable as
|
defined in the loop. In case STMT is a "pattern-stmt" (i.e. - it
|
defined in the loop. In case STMT is a "pattern-stmt" (i.e. - it
|
represents a reduction pattern), the tree-code and scalar-def are
|
represents a reduction pattern), the tree-code and scalar-def are
|
taken from the original stmt that the pattern-stmt (STMT) replaces.
|
taken from the original stmt that the pattern-stmt (STMT) replaces.
|
Otherwise (it is a regular reduction) - the tree-code and scalar-def
|
Otherwise (it is a regular reduction) - the tree-code and scalar-def
|
are taken from STMT. */
|
are taken from STMT. */
|
|
|
orig_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
|
orig_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
|
if (!orig_stmt)
|
if (!orig_stmt)
|
{
|
{
|
/* Regular reduction */
|
/* Regular reduction */
|
orig_stmt = stmt;
|
orig_stmt = stmt;
|
}
|
}
|
else
|
else
|
{
|
{
|
/* Reduction pattern */
|
/* Reduction pattern */
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (orig_stmt);
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (orig_stmt);
|
gcc_assert (STMT_VINFO_IN_PATTERN_P (stmt_vinfo));
|
gcc_assert (STMT_VINFO_IN_PATTERN_P (stmt_vinfo));
|
gcc_assert (STMT_VINFO_RELATED_STMT (stmt_vinfo) == stmt);
|
gcc_assert (STMT_VINFO_RELATED_STMT (stmt_vinfo) == stmt);
|
}
|
}
|
|
|
code = gimple_assign_rhs_code (orig_stmt);
|
code = gimple_assign_rhs_code (orig_stmt);
|
scalar_dest = gimple_assign_lhs (orig_stmt);
|
scalar_dest = gimple_assign_lhs (orig_stmt);
|
scalar_type = TREE_TYPE (scalar_dest);
|
scalar_type = TREE_TYPE (scalar_dest);
|
new_scalar_dest = vect_create_destination_var (scalar_dest, NULL);
|
new_scalar_dest = vect_create_destination_var (scalar_dest, NULL);
|
bitsize = TYPE_SIZE (scalar_type);
|
bitsize = TYPE_SIZE (scalar_type);
|
|
|
/* For MINUS_EXPR the initial vector is [init_val,0,...,0], therefore,
|
/* For MINUS_EXPR the initial vector is [init_val,0,...,0], therefore,
|
partial results are added and not subtracted. */
|
partial results are added and not subtracted. */
|
if (code == MINUS_EXPR)
|
if (code == MINUS_EXPR)
|
code = PLUS_EXPR;
|
code = PLUS_EXPR;
|
|
|
/* In case this is a reduction in an inner-loop while vectorizing an outer
|
/* In case this is a reduction in an inner-loop while vectorizing an outer
|
loop - we don't need to extract a single scalar result at the end of the
|
loop - we don't need to extract a single scalar result at the end of the
|
inner-loop (unless it is double reduction, i.e., the use of reduction is
|
inner-loop (unless it is double reduction, i.e., the use of reduction is
|
outside the outer-loop). The final vector of partial results will be used
|
outside the outer-loop). The final vector of partial results will be used
|
in the vectorized outer-loop, or reduced to a scalar result at the end of
|
in the vectorized outer-loop, or reduced to a scalar result at the end of
|
the outer-loop. */
|
the outer-loop. */
|
if (nested_in_vect_loop && !double_reduc)
|
if (nested_in_vect_loop && !double_reduc)
|
goto vect_finalize_reduction;
|
goto vect_finalize_reduction;
|
|
|
/* The epilogue is created for the outer-loop, i.e., for the loop being
|
/* The epilogue is created for the outer-loop, i.e., for the loop being
|
vectorized. */
|
vectorized. */
|
if (double_reduc)
|
if (double_reduc)
|
loop = outer_loop;
|
loop = outer_loop;
|
|
|
/* FORNOW */
|
/* FORNOW */
|
gcc_assert (ncopies == 1);
|
gcc_assert (ncopies == 1);
|
|
|
/* 2.3 Create the reduction code, using one of the three schemes described
|
/* 2.3 Create the reduction code, using one of the three schemes described
|
above. */
|
above. */
|
|
|
if (reduc_code != ERROR_MARK)
|
if (reduc_code != ERROR_MARK)
|
{
|
{
|
tree tmp;
|
tree tmp;
|
|
|
/*** Case 1: Create:
|
/*** Case 1: Create:
|
v_out2 = reduc_expr <v_out1> */
|
v_out2 = reduc_expr <v_out1> */
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "Reduce using direct vector reduction.");
|
fprintf (vect_dump, "Reduce using direct vector reduction.");
|
|
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
tmp = build1 (reduc_code, vectype, PHI_RESULT (new_phi));
|
tmp = build1 (reduc_code, vectype, PHI_RESULT (new_phi));
|
epilog_stmt = gimple_build_assign (vec_dest, tmp);
|
epilog_stmt = gimple_build_assign (vec_dest, tmp);
|
new_temp = make_ssa_name (vec_dest, epilog_stmt);
|
new_temp = make_ssa_name (vec_dest, epilog_stmt);
|
gimple_assign_set_lhs (epilog_stmt, new_temp);
|
gimple_assign_set_lhs (epilog_stmt, new_temp);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
|
|
extract_scalar_result = true;
|
extract_scalar_result = true;
|
}
|
}
|
else
|
else
|
{
|
{
|
enum tree_code shift_code = ERROR_MARK;
|
enum tree_code shift_code = ERROR_MARK;
|
bool have_whole_vector_shift = true;
|
bool have_whole_vector_shift = true;
|
int bit_offset;
|
int bit_offset;
|
int element_bitsize = tree_low_cst (bitsize, 1);
|
int element_bitsize = tree_low_cst (bitsize, 1);
|
int vec_size_in_bits = tree_low_cst (TYPE_SIZE (vectype), 1);
|
int vec_size_in_bits = tree_low_cst (TYPE_SIZE (vectype), 1);
|
tree vec_temp;
|
tree vec_temp;
|
|
|
if (optab_handler (vec_shr_optab, mode)->insn_code != CODE_FOR_nothing)
|
if (optab_handler (vec_shr_optab, mode)->insn_code != CODE_FOR_nothing)
|
shift_code = VEC_RSHIFT_EXPR;
|
shift_code = VEC_RSHIFT_EXPR;
|
else
|
else
|
have_whole_vector_shift = false;
|
have_whole_vector_shift = false;
|
|
|
/* Regardless of whether we have a whole vector shift, if we're
|
/* Regardless of whether we have a whole vector shift, if we're
|
emulating the operation via tree-vect-generic, we don't want
|
emulating the operation via tree-vect-generic, we don't want
|
to use it. Only the first round of the reduction is likely
|
to use it. Only the first round of the reduction is likely
|
to still be profitable via emulation. */
|
to still be profitable via emulation. */
|
/* ??? It might be better to emit a reduction tree code here, so that
|
/* ??? It might be better to emit a reduction tree code here, so that
|
tree-vect-generic can expand the first round via bit tricks. */
|
tree-vect-generic can expand the first round via bit tricks. */
|
if (!VECTOR_MODE_P (mode))
|
if (!VECTOR_MODE_P (mode))
|
have_whole_vector_shift = false;
|
have_whole_vector_shift = false;
|
else
|
else
|
{
|
{
|
optab optab = optab_for_tree_code (code, vectype, optab_default);
|
optab optab = optab_for_tree_code (code, vectype, optab_default);
|
if (optab_handler (optab, mode)->insn_code == CODE_FOR_nothing)
|
if (optab_handler (optab, mode)->insn_code == CODE_FOR_nothing)
|
have_whole_vector_shift = false;
|
have_whole_vector_shift = false;
|
}
|
}
|
|
|
if (have_whole_vector_shift)
|
if (have_whole_vector_shift)
|
{
|
{
|
/*** Case 2: Create:
|
/*** Case 2: Create:
|
for (offset = VS/2; offset >= element_size; offset/=2)
|
for (offset = VS/2; offset >= element_size; offset/=2)
|
{
|
{
|
Create: va' = vec_shift <va, offset>
|
Create: va' = vec_shift <va, offset>
|
Create: va = vop <va, va'>
|
Create: va = vop <va, va'>
|
} */
|
} */
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "Reduce using vector shifts");
|
fprintf (vect_dump, "Reduce using vector shifts");
|
|
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
new_temp = PHI_RESULT (new_phi);
|
new_temp = PHI_RESULT (new_phi);
|
|
|
for (bit_offset = vec_size_in_bits/2;
|
for (bit_offset = vec_size_in_bits/2;
|
bit_offset >= element_bitsize;
|
bit_offset >= element_bitsize;
|
bit_offset /= 2)
|
bit_offset /= 2)
|
{
|
{
|
tree bitpos = size_int (bit_offset);
|
tree bitpos = size_int (bit_offset);
|
|
|
epilog_stmt = gimple_build_assign_with_ops (shift_code, vec_dest,
|
epilog_stmt = gimple_build_assign_with_ops (shift_code, vec_dest,
|
new_temp, bitpos);
|
new_temp, bitpos);
|
new_name = make_ssa_name (vec_dest, epilog_stmt);
|
new_name = make_ssa_name (vec_dest, epilog_stmt);
|
gimple_assign_set_lhs (epilog_stmt, new_name);
|
gimple_assign_set_lhs (epilog_stmt, new_name);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
|
|
epilog_stmt = gimple_build_assign_with_ops (code, vec_dest,
|
epilog_stmt = gimple_build_assign_with_ops (code, vec_dest,
|
new_name, new_temp);
|
new_name, new_temp);
|
new_temp = make_ssa_name (vec_dest, epilog_stmt);
|
new_temp = make_ssa_name (vec_dest, epilog_stmt);
|
gimple_assign_set_lhs (epilog_stmt, new_temp);
|
gimple_assign_set_lhs (epilog_stmt, new_temp);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
}
|
}
|
|
|
extract_scalar_result = true;
|
extract_scalar_result = true;
|
}
|
}
|
else
|
else
|
{
|
{
|
tree rhs;
|
tree rhs;
|
|
|
/*** Case 3: Create:
|
/*** Case 3: Create:
|
s = extract_field <v_out2, 0>
|
s = extract_field <v_out2, 0>
|
for (offset = element_size;
|
for (offset = element_size;
|
offset < vector_size;
|
offset < vector_size;
|
offset += element_size;)
|
offset += element_size;)
|
{
|
{
|
Create: s' = extract_field <v_out2, offset>
|
Create: s' = extract_field <v_out2, offset>
|
Create: s = op <s, s'>
|
Create: s = op <s, s'>
|
} */
|
} */
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "Reduce using scalar code. ");
|
fprintf (vect_dump, "Reduce using scalar code. ");
|
|
|
vec_temp = PHI_RESULT (new_phi);
|
vec_temp = PHI_RESULT (new_phi);
|
vec_size_in_bits = tree_low_cst (TYPE_SIZE (vectype), 1);
|
vec_size_in_bits = tree_low_cst (TYPE_SIZE (vectype), 1);
|
rhs = build3 (BIT_FIELD_REF, scalar_type, vec_temp, bitsize,
|
rhs = build3 (BIT_FIELD_REF, scalar_type, vec_temp, bitsize,
|
bitsize_zero_node);
|
bitsize_zero_node);
|
epilog_stmt = gimple_build_assign (new_scalar_dest, rhs);
|
epilog_stmt = gimple_build_assign (new_scalar_dest, rhs);
|
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
|
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
|
gimple_assign_set_lhs (epilog_stmt, new_temp);
|
gimple_assign_set_lhs (epilog_stmt, new_temp);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
|
|
for (bit_offset = element_bitsize;
|
for (bit_offset = element_bitsize;
|
bit_offset < vec_size_in_bits;
|
bit_offset < vec_size_in_bits;
|
bit_offset += element_bitsize)
|
bit_offset += element_bitsize)
|
{
|
{
|
tree bitpos = bitsize_int (bit_offset);
|
tree bitpos = bitsize_int (bit_offset);
|
tree rhs = build3 (BIT_FIELD_REF, scalar_type, vec_temp, bitsize,
|
tree rhs = build3 (BIT_FIELD_REF, scalar_type, vec_temp, bitsize,
|
bitpos);
|
bitpos);
|
|
|
epilog_stmt = gimple_build_assign (new_scalar_dest, rhs);
|
epilog_stmt = gimple_build_assign (new_scalar_dest, rhs);
|
new_name = make_ssa_name (new_scalar_dest, epilog_stmt);
|
new_name = make_ssa_name (new_scalar_dest, epilog_stmt);
|
gimple_assign_set_lhs (epilog_stmt, new_name);
|
gimple_assign_set_lhs (epilog_stmt, new_name);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
|
|
epilog_stmt = gimple_build_assign_with_ops (code,
|
epilog_stmt = gimple_build_assign_with_ops (code,
|
new_scalar_dest,
|
new_scalar_dest,
|
new_name, new_temp);
|
new_name, new_temp);
|
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
|
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
|
gimple_assign_set_lhs (epilog_stmt, new_temp);
|
gimple_assign_set_lhs (epilog_stmt, new_temp);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
}
|
}
|
|
|
extract_scalar_result = false;
|
extract_scalar_result = false;
|
}
|
}
|
}
|
}
|
|
|
/* 2.4 Extract the final scalar result. Create:
|
/* 2.4 Extract the final scalar result. Create:
|
s_out3 = extract_field <v_out2, bitpos> */
|
s_out3 = extract_field <v_out2, bitpos> */
|
|
|
if (extract_scalar_result)
|
if (extract_scalar_result)
|
{
|
{
|
tree rhs;
|
tree rhs;
|
|
|
gcc_assert (!nested_in_vect_loop || double_reduc);
|
gcc_assert (!nested_in_vect_loop || double_reduc);
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "extract scalar result");
|
fprintf (vect_dump, "extract scalar result");
|
|
|
if (BYTES_BIG_ENDIAN)
|
if (BYTES_BIG_ENDIAN)
|
bitpos = size_binop (MULT_EXPR,
|
bitpos = size_binop (MULT_EXPR,
|
bitsize_int (TYPE_VECTOR_SUBPARTS (vectype) - 1),
|
bitsize_int (TYPE_VECTOR_SUBPARTS (vectype) - 1),
|
TYPE_SIZE (scalar_type));
|
TYPE_SIZE (scalar_type));
|
else
|
else
|
bitpos = bitsize_zero_node;
|
bitpos = bitsize_zero_node;
|
|
|
rhs = build3 (BIT_FIELD_REF, scalar_type, new_temp, bitsize, bitpos);
|
rhs = build3 (BIT_FIELD_REF, scalar_type, new_temp, bitsize, bitpos);
|
epilog_stmt = gimple_build_assign (new_scalar_dest, rhs);
|
epilog_stmt = gimple_build_assign (new_scalar_dest, rhs);
|
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
|
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
|
gimple_assign_set_lhs (epilog_stmt, new_temp);
|
gimple_assign_set_lhs (epilog_stmt, new_temp);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
}
|
}
|
|
|
vect_finalize_reduction:
|
vect_finalize_reduction:
|
|
|
if (double_reduc)
|
if (double_reduc)
|
loop = loop->inner;
|
loop = loop->inner;
|
|
|
/* 2.5 Adjust the final result by the initial value of the reduction
|
/* 2.5 Adjust the final result by the initial value of the reduction
|
variable. (When such adjustment is not needed, then
|
variable. (When such adjustment is not needed, then
|
'adjustment_def' is zero). For example, if code is PLUS we create:
|
'adjustment_def' is zero). For example, if code is PLUS we create:
|
new_temp = loop_exit_def + adjustment_def */
|
new_temp = loop_exit_def + adjustment_def */
|
|
|
if (adjustment_def)
|
if (adjustment_def)
|
{
|
{
|
if (nested_in_vect_loop)
|
if (nested_in_vect_loop)
|
{
|
{
|
gcc_assert (TREE_CODE (TREE_TYPE (adjustment_def)) == VECTOR_TYPE);
|
gcc_assert (TREE_CODE (TREE_TYPE (adjustment_def)) == VECTOR_TYPE);
|
expr = build2 (code, vectype, PHI_RESULT (new_phi), adjustment_def);
|
expr = build2 (code, vectype, PHI_RESULT (new_phi), adjustment_def);
|
new_dest = vect_create_destination_var (scalar_dest, vectype);
|
new_dest = vect_create_destination_var (scalar_dest, vectype);
|
}
|
}
|
else
|
else
|
{
|
{
|
gcc_assert (TREE_CODE (TREE_TYPE (adjustment_def)) != VECTOR_TYPE);
|
gcc_assert (TREE_CODE (TREE_TYPE (adjustment_def)) != VECTOR_TYPE);
|
expr = build2 (code, scalar_type, new_temp, adjustment_def);
|
expr = build2 (code, scalar_type, new_temp, adjustment_def);
|
new_dest = vect_create_destination_var (scalar_dest, scalar_type);
|
new_dest = vect_create_destination_var (scalar_dest, scalar_type);
|
}
|
}
|
|
|
epilog_stmt = gimple_build_assign (new_dest, expr);
|
epilog_stmt = gimple_build_assign (new_dest, expr);
|
new_temp = make_ssa_name (new_dest, epilog_stmt);
|
new_temp = make_ssa_name (new_dest, epilog_stmt);
|
gimple_assign_set_lhs (epilog_stmt, new_temp);
|
gimple_assign_set_lhs (epilog_stmt, new_temp);
|
SSA_NAME_DEF_STMT (new_temp) = epilog_stmt;
|
SSA_NAME_DEF_STMT (new_temp) = epilog_stmt;
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
gsi_insert_before (&exit_gsi, epilog_stmt, GSI_SAME_STMT);
|
}
|
}
|
|
|
|
|
/* 2.6 Handle the loop-exit phi */
|
/* 2.6 Handle the loop-exit phi */
|
|
|
/* Replace uses of s_out0 with uses of s_out3:
|
/* Replace uses of s_out0 with uses of s_out3:
|
Find the loop-closed-use at the loop exit of the original scalar result.
|
Find the loop-closed-use at the loop exit of the original scalar result.
|
(The reduction result is expected to have two immediate uses - one at the
|
(The reduction result is expected to have two immediate uses - one at the
|
latch block, and one at the loop exit). */
|
latch block, and one at the loop exit). */
|
phis = VEC_alloc (gimple, heap, 10);
|
phis = VEC_alloc (gimple, heap, 10);
|
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest)
|
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest)
|
{
|
{
|
if (!flow_bb_inside_loop_p (loop, gimple_bb (USE_STMT (use_p))))
|
if (!flow_bb_inside_loop_p (loop, gimple_bb (USE_STMT (use_p))))
|
{
|
{
|
exit_phi = USE_STMT (use_p);
|
exit_phi = USE_STMT (use_p);
|
VEC_quick_push (gimple, phis, exit_phi);
|
VEC_quick_push (gimple, phis, exit_phi);
|
}
|
}
|
}
|
}
|
|
|
/* We expect to have found an exit_phi because of loop-closed-ssa form. */
|
/* We expect to have found an exit_phi because of loop-closed-ssa form. */
|
gcc_assert (!VEC_empty (gimple, phis));
|
gcc_assert (!VEC_empty (gimple, phis));
|
|
|
for (i = 0; VEC_iterate (gimple, phis, i, exit_phi); i++)
|
for (i = 0; VEC_iterate (gimple, phis, i, exit_phi); i++)
|
{
|
{
|
if (nested_in_vect_loop)
|
if (nested_in_vect_loop)
|
{
|
{
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (exit_phi);
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (exit_phi);
|
gimple vect_phi;
|
gimple vect_phi;
|
|
|
/* FORNOW. Currently not supporting the case that an inner-loop
|
/* FORNOW. Currently not supporting the case that an inner-loop
|
reduction is not used in the outer-loop (but only outside the
|
reduction is not used in the outer-loop (but only outside the
|
outer-loop), unless it is double reduction. */
|
outer-loop), unless it is double reduction. */
|
gcc_assert ((STMT_VINFO_RELEVANT_P (stmt_vinfo)
|
gcc_assert ((STMT_VINFO_RELEVANT_P (stmt_vinfo)
|
&& !STMT_VINFO_LIVE_P (stmt_vinfo)) || double_reduc);
|
&& !STMT_VINFO_LIVE_P (stmt_vinfo)) || double_reduc);
|
|
|
epilog_stmt = adjustment_def ? epilog_stmt : new_phi;
|
epilog_stmt = adjustment_def ? epilog_stmt : new_phi;
|
STMT_VINFO_VEC_STMT (stmt_vinfo) = epilog_stmt;
|
STMT_VINFO_VEC_STMT (stmt_vinfo) = epilog_stmt;
|
set_vinfo_for_stmt (epilog_stmt,
|
set_vinfo_for_stmt (epilog_stmt,
|
new_stmt_vec_info (epilog_stmt, loop_vinfo,
|
new_stmt_vec_info (epilog_stmt, loop_vinfo,
|
NULL));
|
NULL));
|
if (adjustment_def)
|
if (adjustment_def)
|
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (epilog_stmt)) =
|
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (epilog_stmt)) =
|
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (new_phi));
|
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (new_phi));
|
|
|
if (!double_reduc
|
if (!double_reduc
|
|| STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_double_reduction_def)
|
|| STMT_VINFO_DEF_TYPE (stmt_vinfo) != vect_double_reduction_def)
|
continue;
|
continue;
|
|
|
/* Handle double reduction:
|
/* Handle double reduction:
|
|
|
stmt1: s1 = phi <s0, s2> - double reduction phi (outer loop)
|
stmt1: s1 = phi <s0, s2> - double reduction phi (outer loop)
|
stmt2: s3 = phi <s1, s4> - (regular) reduction phi (inner loop)
|
stmt2: s3 = phi <s1, s4> - (regular) reduction phi (inner loop)
|
stmt3: s4 = use (s3) - (regular) reduction stmt (inner loop)
|
stmt3: s4 = use (s3) - (regular) reduction stmt (inner loop)
|
stmt4: s2 = phi <s4> - double reduction stmt (outer loop)
|
stmt4: s2 = phi <s4> - double reduction stmt (outer loop)
|
|
|
At that point the regular reduction (stmt2 and stmt3) is already
|
At that point the regular reduction (stmt2 and stmt3) is already
|
vectorized, as well as the exit phi node, stmt4.
|
vectorized, as well as the exit phi node, stmt4.
|
Here we vectorize the phi node of double reduction, stmt1, and
|
Here we vectorize the phi node of double reduction, stmt1, and
|
update all relevant statements. */
|
update all relevant statements. */
|
|
|
/* Go through all the uses of s2 to find double reduction phi node,
|
/* Go through all the uses of s2 to find double reduction phi node,
|
i.e., stmt1 above. */
|
i.e., stmt1 above. */
|
orig_name = PHI_RESULT (exit_phi);
|
orig_name = PHI_RESULT (exit_phi);
|
FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, orig_name)
|
FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, orig_name)
|
{
|
{
|
stmt_vec_info use_stmt_vinfo = vinfo_for_stmt (use_stmt);
|
stmt_vec_info use_stmt_vinfo = vinfo_for_stmt (use_stmt);
|
stmt_vec_info new_phi_vinfo;
|
stmt_vec_info new_phi_vinfo;
|
tree vect_phi_init, preheader_arg, vect_phi_res, init_def;
|
tree vect_phi_init, preheader_arg, vect_phi_res, init_def;
|
basic_block bb = gimple_bb (use_stmt);
|
basic_block bb = gimple_bb (use_stmt);
|
gimple use;
|
gimple use;
|
|
|
/* Check that USE_STMT is really double reduction phi node. */
|
/* Check that USE_STMT is really double reduction phi node. */
|
if (gimple_code (use_stmt) != GIMPLE_PHI
|
if (gimple_code (use_stmt) != GIMPLE_PHI
|
|| gimple_phi_num_args (use_stmt) != 2
|
|| gimple_phi_num_args (use_stmt) != 2
|
|| !use_stmt_vinfo
|
|| !use_stmt_vinfo
|
|| STMT_VINFO_DEF_TYPE (use_stmt_vinfo)
|
|| STMT_VINFO_DEF_TYPE (use_stmt_vinfo)
|
!= vect_double_reduction_def
|
!= vect_double_reduction_def
|
|| bb->loop_father != outer_loop)
|
|| bb->loop_father != outer_loop)
|
continue;
|
continue;
|
|
|
/* Create vector phi node for double reduction:
|
/* Create vector phi node for double reduction:
|
vs1 = phi <vs0, vs2>
|
vs1 = phi <vs0, vs2>
|
vs1 was created previously in this function by a call to
|
vs1 was created previously in this function by a call to
|
vect_get_vec_def_for_operand and is stored in vec_initial_def;
|
vect_get_vec_def_for_operand and is stored in vec_initial_def;
|
vs2 is defined by EPILOG_STMT, the vectorized EXIT_PHI;
|
vs2 is defined by EPILOG_STMT, the vectorized EXIT_PHI;
|
vs0 is created here. */
|
vs0 is created here. */
|
|
|
/* Create vector phi node. */
|
/* Create vector phi node. */
|
vect_phi = create_phi_node (vec_initial_def, bb);
|
vect_phi = create_phi_node (vec_initial_def, bb);
|
new_phi_vinfo = new_stmt_vec_info (vect_phi,
|
new_phi_vinfo = new_stmt_vec_info (vect_phi,
|
loop_vec_info_for_loop (outer_loop), NULL);
|
loop_vec_info_for_loop (outer_loop), NULL);
|
set_vinfo_for_stmt (vect_phi, new_phi_vinfo);
|
set_vinfo_for_stmt (vect_phi, new_phi_vinfo);
|
|
|
/* Create vs0 - initial def of the double reduction phi. */
|
/* Create vs0 - initial def of the double reduction phi. */
|
preheader_arg = PHI_ARG_DEF_FROM_EDGE (use_stmt,
|
preheader_arg = PHI_ARG_DEF_FROM_EDGE (use_stmt,
|
loop_preheader_edge (outer_loop));
|
loop_preheader_edge (outer_loop));
|
init_def = get_initial_def_for_reduction (stmt, preheader_arg,
|
init_def = get_initial_def_for_reduction (stmt, preheader_arg,
|
NULL);
|
NULL);
|
vect_phi_init = vect_init_vector (use_stmt, init_def, vectype,
|
vect_phi_init = vect_init_vector (use_stmt, init_def, vectype,
|
NULL);
|
NULL);
|
|
|
/* Update phi node arguments with vs0 and vs2. */
|
/* Update phi node arguments with vs0 and vs2. */
|
add_phi_arg (vect_phi, vect_phi_init,
|
add_phi_arg (vect_phi, vect_phi_init,
|
loop_preheader_edge (outer_loop), UNKNOWN_LOCATION);
|
loop_preheader_edge (outer_loop), UNKNOWN_LOCATION);
|
add_phi_arg (vect_phi, PHI_RESULT (epilog_stmt),
|
add_phi_arg (vect_phi, PHI_RESULT (epilog_stmt),
|
loop_latch_edge (outer_loop), UNKNOWN_LOCATION);
|
loop_latch_edge (outer_loop), UNKNOWN_LOCATION);
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "created double reduction phi node: ");
|
fprintf (vect_dump, "created double reduction phi node: ");
|
print_gimple_stmt (vect_dump, vect_phi, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, vect_phi, 0, TDF_SLIM);
|
}
|
}
|
|
|
vect_phi_res = PHI_RESULT (vect_phi);
|
vect_phi_res = PHI_RESULT (vect_phi);
|
|
|
/* Replace the use, i.e., set the correct vs1 in the regular
|
/* Replace the use, i.e., set the correct vs1 in the regular
|
reduction phi node. FORNOW, NCOPIES is always 1, so the loop
|
reduction phi node. FORNOW, NCOPIES is always 1, so the loop
|
is redundant. */
|
is redundant. */
|
use = reduction_phi;
|
use = reduction_phi;
|
for (j = 0; j < ncopies; j++)
|
for (j = 0; j < ncopies; j++)
|
{
|
{
|
edge pr_edge = loop_preheader_edge (loop);
|
edge pr_edge = loop_preheader_edge (loop);
|
SET_PHI_ARG_DEF (use, pr_edge->dest_idx, vect_phi_res);
|
SET_PHI_ARG_DEF (use, pr_edge->dest_idx, vect_phi_res);
|
use = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (use));
|
use = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (use));
|
}
|
}
|
}
|
}
|
}
|
}
|
|
|
/* Replace the uses: */
|
/* Replace the uses: */
|
orig_name = PHI_RESULT (exit_phi);
|
orig_name = PHI_RESULT (exit_phi);
|
FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, orig_name)
|
FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, orig_name)
|
FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
|
FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
|
SET_USE (use_p, new_temp);
|
SET_USE (use_p, new_temp);
|
}
|
}
|
|
|
VEC_free (gimple, heap, phis);
|
VEC_free (gimple, heap, phis);
|
}
|
}
|
|
|
|
|
/* Function vectorizable_reduction.
|
/* Function vectorizable_reduction.
|
|
|
Check if STMT performs a reduction operation that can be vectorized.
|
Check if STMT performs a reduction operation that can be vectorized.
|
If VEC_STMT is also passed, vectorize the STMT: create a vectorized
|
If VEC_STMT is also passed, vectorize the STMT: create a vectorized
|
stmt to replace it, put it in VEC_STMT, and insert it at GSI.
|
stmt to replace it, put it in VEC_STMT, and insert it at GSI.
|
Return FALSE if not a vectorizable STMT, TRUE otherwise.
|
Return FALSE if not a vectorizable STMT, TRUE otherwise.
|
|
|
This function also handles reduction idioms (patterns) that have been
|
This function also handles reduction idioms (patterns) that have been
|
recognized in advance during vect_pattern_recog. In this case, STMT may be
|
recognized in advance during vect_pattern_recog. In this case, STMT may be
|
of this form:
|
of this form:
|
X = pattern_expr (arg0, arg1, ..., X)
|
X = pattern_expr (arg0, arg1, ..., X)
|
and it's STMT_VINFO_RELATED_STMT points to the last stmt in the original
|
and it's STMT_VINFO_RELATED_STMT points to the last stmt in the original
|
sequence that had been detected and replaced by the pattern-stmt (STMT).
|
sequence that had been detected and replaced by the pattern-stmt (STMT).
|
|
|
In some cases of reduction patterns, the type of the reduction variable X is
|
In some cases of reduction patterns, the type of the reduction variable X is
|
different than the type of the other arguments of STMT.
|
different than the type of the other arguments of STMT.
|
In such cases, the vectype that is used when transforming STMT into a vector
|
In such cases, the vectype that is used when transforming STMT into a vector
|
stmt is different than the vectype that is used to determine the
|
stmt is different than the vectype that is used to determine the
|
vectorization factor, because it consists of a different number of elements
|
vectorization factor, because it consists of a different number of elements
|
than the actual number of elements that are being operated upon in parallel.
|
than the actual number of elements that are being operated upon in parallel.
|
|
|
For example, consider an accumulation of shorts into an int accumulator.
|
For example, consider an accumulation of shorts into an int accumulator.
|
On some targets it's possible to vectorize this pattern operating on 8
|
On some targets it's possible to vectorize this pattern operating on 8
|
shorts at a time (hence, the vectype for purposes of determining the
|
shorts at a time (hence, the vectype for purposes of determining the
|
vectorization factor should be V8HI); on the other hand, the vectype that
|
vectorization factor should be V8HI); on the other hand, the vectype that
|
is used to create the vector form is actually V4SI (the type of the result).
|
is used to create the vector form is actually V4SI (the type of the result).
|
|
|
Upon entry to this function, STMT_VINFO_VECTYPE records the vectype that
|
Upon entry to this function, STMT_VINFO_VECTYPE records the vectype that
|
indicates what is the actual level of parallelism (V8HI in the example), so
|
indicates what is the actual level of parallelism (V8HI in the example), so
|
that the right vectorization factor would be derived. This vectype
|
that the right vectorization factor would be derived. This vectype
|
corresponds to the type of arguments to the reduction stmt, and should *NOT*
|
corresponds to the type of arguments to the reduction stmt, and should *NOT*
|
be used to create the vectorized stmt. The right vectype for the vectorized
|
be used to create the vectorized stmt. The right vectype for the vectorized
|
stmt is obtained from the type of the result X:
|
stmt is obtained from the type of the result X:
|
get_vectype_for_scalar_type (TREE_TYPE (X))
|
get_vectype_for_scalar_type (TREE_TYPE (X))
|
|
|
This means that, contrary to "regular" reductions (or "regular" stmts in
|
This means that, contrary to "regular" reductions (or "regular" stmts in
|
general), the following equation:
|
general), the following equation:
|
STMT_VINFO_VECTYPE == get_vectype_for_scalar_type (TREE_TYPE (X))
|
STMT_VINFO_VECTYPE == get_vectype_for_scalar_type (TREE_TYPE (X))
|
does *NOT* necessarily hold for reduction patterns. */
|
does *NOT* necessarily hold for reduction patterns. */
|
|
|
bool
|
bool
|
vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
|
vectorizable_reduction (gimple stmt, gimple_stmt_iterator *gsi,
|
gimple *vec_stmt)
|
gimple *vec_stmt)
|
{
|
{
|
tree vec_dest;
|
tree vec_dest;
|
tree scalar_dest;
|
tree scalar_dest;
|
tree loop_vec_def0 = NULL_TREE, loop_vec_def1 = NULL_TREE;
|
tree loop_vec_def0 = NULL_TREE, loop_vec_def1 = NULL_TREE;
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
enum tree_code code, orig_code, epilog_reduc_code;
|
enum tree_code code, orig_code, epilog_reduc_code;
|
enum machine_mode vec_mode;
|
enum machine_mode vec_mode;
|
int op_type;
|
int op_type;
|
optab optab, reduc_optab;
|
optab optab, reduc_optab;
|
tree new_temp = NULL_TREE;
|
tree new_temp = NULL_TREE;
|
tree def;
|
tree def;
|
gimple def_stmt;
|
gimple def_stmt;
|
enum vect_def_type dt;
|
enum vect_def_type dt;
|
gimple new_phi = NULL;
|
gimple new_phi = NULL;
|
tree scalar_type;
|
tree scalar_type;
|
bool is_simple_use;
|
bool is_simple_use;
|
gimple orig_stmt;
|
gimple orig_stmt;
|
stmt_vec_info orig_stmt_info;
|
stmt_vec_info orig_stmt_info;
|
tree expr = NULL_TREE;
|
tree expr = NULL_TREE;
|
int i;
|
int i;
|
int nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
int nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
int ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
|
int ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
|
int epilog_copies;
|
int epilog_copies;
|
stmt_vec_info prev_stmt_info, prev_phi_info;
|
stmt_vec_info prev_stmt_info, prev_phi_info;
|
gimple first_phi = NULL;
|
gimple first_phi = NULL;
|
bool single_defuse_cycle = false;
|
bool single_defuse_cycle = false;
|
tree reduc_def = NULL_TREE;
|
tree reduc_def = NULL_TREE;
|
gimple new_stmt = NULL;
|
gimple new_stmt = NULL;
|
int j;
|
int j;
|
tree ops[3];
|
tree ops[3];
|
bool nested_cycle = false, found_nested_cycle_def = false;
|
bool nested_cycle = false, found_nested_cycle_def = false;
|
gimple reduc_def_stmt = NULL;
|
gimple reduc_def_stmt = NULL;
|
/* The default is that the reduction variable is the last in statement. */
|
/* The default is that the reduction variable is the last in statement. */
|
int reduc_index = 2;
|
int reduc_index = 2;
|
bool double_reduc = false, dummy;
|
bool double_reduc = false, dummy;
|
basic_block def_bb;
|
basic_block def_bb;
|
struct loop * def_stmt_loop, *outer_loop = NULL;
|
struct loop * def_stmt_loop, *outer_loop = NULL;
|
tree def_arg;
|
tree def_arg;
|
gimple def_arg_stmt;
|
gimple def_arg_stmt;
|
|
|
if (nested_in_vect_loop_p (loop, stmt))
|
if (nested_in_vect_loop_p (loop, stmt))
|
{
|
{
|
outer_loop = loop;
|
outer_loop = loop;
|
loop = loop->inner;
|
loop = loop->inner;
|
nested_cycle = true;
|
nested_cycle = true;
|
}
|
}
|
|
|
gcc_assert (ncopies >= 1);
|
gcc_assert (ncopies >= 1);
|
|
|
/* FORNOW: SLP not supported. */
|
/* FORNOW: SLP not supported. */
|
if (STMT_SLP_TYPE (stmt_info))
|
if (STMT_SLP_TYPE (stmt_info))
|
return false;
|
return false;
|
|
|
/* 1. Is vectorizable reduction? */
|
/* 1. Is vectorizable reduction? */
|
/* Not supportable if the reduction variable is used in the loop. */
|
/* Not supportable if the reduction variable is used in the loop. */
|
if (STMT_VINFO_RELEVANT (stmt_info) > vect_used_in_outer)
|
if (STMT_VINFO_RELEVANT (stmt_info) > vect_used_in_outer)
|
return false;
|
return false;
|
|
|
/* Reductions that are not used even in an enclosing outer-loop,
|
/* Reductions that are not used even in an enclosing outer-loop,
|
are expected to be "live" (used out of the loop). */
|
are expected to be "live" (used out of the loop). */
|
if (STMT_VINFO_RELEVANT (stmt_info) == vect_unused_in_scope
|
if (STMT_VINFO_RELEVANT (stmt_info) == vect_unused_in_scope
|
&& !STMT_VINFO_LIVE_P (stmt_info))
|
&& !STMT_VINFO_LIVE_P (stmt_info))
|
return false;
|
return false;
|
|
|
/* Make sure it was already recognized as a reduction computation. */
|
/* Make sure it was already recognized as a reduction computation. */
|
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def
|
if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def
|
&& STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle)
|
&& STMT_VINFO_DEF_TYPE (stmt_info) != vect_nested_cycle)
|
return false;
|
return false;
|
|
|
/* 2. Has this been recognized as a reduction pattern?
|
/* 2. Has this been recognized as a reduction pattern?
|
|
|
Check if STMT represents a pattern that has been recognized
|
Check if STMT represents a pattern that has been recognized
|
in earlier analysis stages. For stmts that represent a pattern,
|
in earlier analysis stages. For stmts that represent a pattern,
|
the STMT_VINFO_RELATED_STMT field records the last stmt in
|
the STMT_VINFO_RELATED_STMT field records the last stmt in
|
the original sequence that constitutes the pattern. */
|
the original sequence that constitutes the pattern. */
|
|
|
orig_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
|
orig_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
|
if (orig_stmt)
|
if (orig_stmt)
|
{
|
{
|
orig_stmt_info = vinfo_for_stmt (orig_stmt);
|
orig_stmt_info = vinfo_for_stmt (orig_stmt);
|
gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info) == stmt);
|
gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info) == stmt);
|
gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info));
|
gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info));
|
gcc_assert (!STMT_VINFO_IN_PATTERN_P (stmt_info));
|
gcc_assert (!STMT_VINFO_IN_PATTERN_P (stmt_info));
|
}
|
}
|
|
|
/* 3. Check the operands of the operation. The first operands are defined
|
/* 3. Check the operands of the operation. The first operands are defined
|
inside the loop body. The last operand is the reduction variable,
|
inside the loop body. The last operand is the reduction variable,
|
which is defined by the loop-header-phi. */
|
which is defined by the loop-header-phi. */
|
|
|
gcc_assert (is_gimple_assign (stmt));
|
gcc_assert (is_gimple_assign (stmt));
|
|
|
/* Flatten RHS */
|
/* Flatten RHS */
|
switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt)))
|
switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt)))
|
{
|
{
|
case GIMPLE_SINGLE_RHS:
|
case GIMPLE_SINGLE_RHS:
|
op_type = TREE_OPERAND_LENGTH (gimple_assign_rhs1 (stmt));
|
op_type = TREE_OPERAND_LENGTH (gimple_assign_rhs1 (stmt));
|
if (op_type == ternary_op)
|
if (op_type == ternary_op)
|
{
|
{
|
tree rhs = gimple_assign_rhs1 (stmt);
|
tree rhs = gimple_assign_rhs1 (stmt);
|
ops[0] = TREE_OPERAND (rhs, 0);
|
ops[0] = TREE_OPERAND (rhs, 0);
|
ops[1] = TREE_OPERAND (rhs, 1);
|
ops[1] = TREE_OPERAND (rhs, 1);
|
ops[2] = TREE_OPERAND (rhs, 2);
|
ops[2] = TREE_OPERAND (rhs, 2);
|
code = TREE_CODE (rhs);
|
code = TREE_CODE (rhs);
|
}
|
}
|
else
|
else
|
return false;
|
return false;
|
break;
|
break;
|
|
|
case GIMPLE_BINARY_RHS:
|
case GIMPLE_BINARY_RHS:
|
code = gimple_assign_rhs_code (stmt);
|
code = gimple_assign_rhs_code (stmt);
|
op_type = TREE_CODE_LENGTH (code);
|
op_type = TREE_CODE_LENGTH (code);
|
gcc_assert (op_type == binary_op);
|
gcc_assert (op_type == binary_op);
|
ops[0] = gimple_assign_rhs1 (stmt);
|
ops[0] = gimple_assign_rhs1 (stmt);
|
ops[1] = gimple_assign_rhs2 (stmt);
|
ops[1] = gimple_assign_rhs2 (stmt);
|
break;
|
break;
|
|
|
case GIMPLE_UNARY_RHS:
|
case GIMPLE_UNARY_RHS:
|
return false;
|
return false;
|
|
|
default:
|
default:
|
gcc_unreachable ();
|
gcc_unreachable ();
|
}
|
}
|
|
|
scalar_dest = gimple_assign_lhs (stmt);
|
scalar_dest = gimple_assign_lhs (stmt);
|
scalar_type = TREE_TYPE (scalar_dest);
|
scalar_type = TREE_TYPE (scalar_dest);
|
if (!POINTER_TYPE_P (scalar_type) && !INTEGRAL_TYPE_P (scalar_type)
|
if (!POINTER_TYPE_P (scalar_type) && !INTEGRAL_TYPE_P (scalar_type)
|
&& !SCALAR_FLOAT_TYPE_P (scalar_type))
|
&& !SCALAR_FLOAT_TYPE_P (scalar_type))
|
return false;
|
return false;
|
|
|
/* All uses but the last are expected to be defined in the loop.
|
/* All uses but the last are expected to be defined in the loop.
|
The last use is the reduction variable. In case of nested cycle this
|
The last use is the reduction variable. In case of nested cycle this
|
assumption is not true: we use reduc_index to record the index of the
|
assumption is not true: we use reduc_index to record the index of the
|
reduction variable. */
|
reduction variable. */
|
for (i = 0; i < op_type-1; i++)
|
for (i = 0; i < op_type-1; i++)
|
{
|
{
|
/* The condition of COND_EXPR is checked in vectorizable_condition(). */
|
/* The condition of COND_EXPR is checked in vectorizable_condition(). */
|
if (i == 0 && code == COND_EXPR)
|
if (i == 0 && code == COND_EXPR)
|
continue;
|
continue;
|
|
|
is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, NULL, &def_stmt,
|
is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, NULL, &def_stmt,
|
&def, &dt);
|
&def, &dt);
|
gcc_assert (is_simple_use);
|
gcc_assert (is_simple_use);
|
if (dt != vect_internal_def
|
if (dt != vect_internal_def
|
&& dt != vect_external_def
|
&& dt != vect_external_def
|
&& dt != vect_constant_def
|
&& dt != vect_constant_def
|
&& dt != vect_induction_def
|
&& dt != vect_induction_def
|
&& !(dt == vect_nested_cycle && nested_cycle))
|
&& !(dt == vect_nested_cycle && nested_cycle))
|
return false;
|
return false;
|
|
|
if (dt == vect_nested_cycle)
|
if (dt == vect_nested_cycle)
|
{
|
{
|
found_nested_cycle_def = true;
|
found_nested_cycle_def = true;
|
reduc_def_stmt = def_stmt;
|
reduc_def_stmt = def_stmt;
|
reduc_index = i;
|
reduc_index = i;
|
}
|
}
|
}
|
}
|
|
|
is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, NULL, &def_stmt,
|
is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, NULL, &def_stmt,
|
&def, &dt);
|
&def, &dt);
|
gcc_assert (is_simple_use);
|
gcc_assert (is_simple_use);
|
gcc_assert (dt == vect_reduction_def
|
gcc_assert (dt == vect_reduction_def
|
|| dt == vect_nested_cycle
|
|| dt == vect_nested_cycle
|
|| ((dt == vect_internal_def || dt == vect_external_def
|
|| ((dt == vect_internal_def || dt == vect_external_def
|
|| dt == vect_constant_def || dt == vect_induction_def)
|
|| dt == vect_constant_def || dt == vect_induction_def)
|
&& nested_cycle && found_nested_cycle_def));
|
&& nested_cycle && found_nested_cycle_def));
|
if (!found_nested_cycle_def)
|
if (!found_nested_cycle_def)
|
reduc_def_stmt = def_stmt;
|
reduc_def_stmt = def_stmt;
|
|
|
gcc_assert (gimple_code (reduc_def_stmt) == GIMPLE_PHI);
|
gcc_assert (gimple_code (reduc_def_stmt) == GIMPLE_PHI);
|
if (orig_stmt)
|
if (orig_stmt)
|
gcc_assert (orig_stmt == vect_is_simple_reduction (loop_vinfo,
|
gcc_assert (orig_stmt == vect_is_simple_reduction (loop_vinfo,
|
reduc_def_stmt,
|
reduc_def_stmt,
|
!nested_cycle,
|
!nested_cycle,
|
&dummy));
|
&dummy));
|
else
|
else
|
gcc_assert (stmt == vect_is_simple_reduction (loop_vinfo, reduc_def_stmt,
|
gcc_assert (stmt == vect_is_simple_reduction (loop_vinfo, reduc_def_stmt,
|
!nested_cycle, &dummy));
|
!nested_cycle, &dummy));
|
|
|
if (STMT_VINFO_LIVE_P (vinfo_for_stmt (reduc_def_stmt)))
|
if (STMT_VINFO_LIVE_P (vinfo_for_stmt (reduc_def_stmt)))
|
return false;
|
return false;
|
|
|
vec_mode = TYPE_MODE (vectype);
|
vec_mode = TYPE_MODE (vectype);
|
|
|
if (code == COND_EXPR)
|
if (code == COND_EXPR)
|
{
|
{
|
if (!vectorizable_condition (stmt, gsi, NULL, ops[reduc_index], 0))
|
if (!vectorizable_condition (stmt, gsi, NULL, ops[reduc_index], 0))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "unsupported condition in reduction");
|
fprintf (vect_dump, "unsupported condition in reduction");
|
|
|
return false;
|
return false;
|
}
|
}
|
}
|
}
|
else
|
else
|
{
|
{
|
/* 4. Supportable by target? */
|
/* 4. Supportable by target? */
|
|
|
/* 4.1. check support for the operation in the loop */
|
/* 4.1. check support for the operation in the loop */
|
optab = optab_for_tree_code (code, vectype, optab_default);
|
optab = optab_for_tree_code (code, vectype, optab_default);
|
if (!optab)
|
if (!optab)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "no optab.");
|
fprintf (vect_dump, "no optab.");
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
if (optab_handler (optab, vec_mode)->insn_code == CODE_FOR_nothing)
|
if (optab_handler (optab, vec_mode)->insn_code == CODE_FOR_nothing)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "op not supported by target.");
|
fprintf (vect_dump, "op not supported by target.");
|
|
|
if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
|
if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
|
|| LOOP_VINFO_VECT_FACTOR (loop_vinfo)
|
|| LOOP_VINFO_VECT_FACTOR (loop_vinfo)
|
< vect_min_worthwhile_factor (code))
|
< vect_min_worthwhile_factor (code))
|
return false;
|
return false;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "proceeding using word mode.");
|
fprintf (vect_dump, "proceeding using word mode.");
|
}
|
}
|
|
|
/* Worthwhile without SIMD support? */
|
/* Worthwhile without SIMD support? */
|
if (!VECTOR_MODE_P (TYPE_MODE (vectype))
|
if (!VECTOR_MODE_P (TYPE_MODE (vectype))
|
&& LOOP_VINFO_VECT_FACTOR (loop_vinfo)
|
&& LOOP_VINFO_VECT_FACTOR (loop_vinfo)
|
< vect_min_worthwhile_factor (code))
|
< vect_min_worthwhile_factor (code))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "not worthwhile without SIMD support.");
|
fprintf (vect_dump, "not worthwhile without SIMD support.");
|
|
|
return false;
|
return false;
|
}
|
}
|
}
|
}
|
|
|
/* 4.2. Check support for the epilog operation.
|
/* 4.2. Check support for the epilog operation.
|
|
|
If STMT represents a reduction pattern, then the type of the
|
If STMT represents a reduction pattern, then the type of the
|
reduction variable may be different than the type of the rest
|
reduction variable may be different than the type of the rest
|
of the arguments. For example, consider the case of accumulation
|
of the arguments. For example, consider the case of accumulation
|
of shorts into an int accumulator; The original code:
|
of shorts into an int accumulator; The original code:
|
S1: int_a = (int) short_a;
|
S1: int_a = (int) short_a;
|
orig_stmt-> S2: int_acc = plus <int_a ,int_acc>;
|
orig_stmt-> S2: int_acc = plus <int_a ,int_acc>;
|
|
|
was replaced with:
|
was replaced with:
|
STMT: int_acc = widen_sum <short_a, int_acc>
|
STMT: int_acc = widen_sum <short_a, int_acc>
|
|
|
This means that:
|
This means that:
|
1. The tree-code that is used to create the vector operation in the
|
1. The tree-code that is used to create the vector operation in the
|
epilog code (that reduces the partial results) is not the
|
epilog code (that reduces the partial results) is not the
|
tree-code of STMT, but is rather the tree-code of the original
|
tree-code of STMT, but is rather the tree-code of the original
|
stmt from the pattern that STMT is replacing. I.e, in the example
|
stmt from the pattern that STMT is replacing. I.e, in the example
|
above we want to use 'widen_sum' in the loop, but 'plus' in the
|
above we want to use 'widen_sum' in the loop, but 'plus' in the
|
epilog.
|
epilog.
|
2. The type (mode) we use to check available target support
|
2. The type (mode) we use to check available target support
|
for the vector operation to be created in the *epilog*, is
|
for the vector operation to be created in the *epilog*, is
|
determined by the type of the reduction variable (in the example
|
determined by the type of the reduction variable (in the example
|
above we'd check this: plus_optab[vect_int_mode]).
|
above we'd check this: plus_optab[vect_int_mode]).
|
However the type (mode) we use to check available target support
|
However the type (mode) we use to check available target support
|
for the vector operation to be created *inside the loop*, is
|
for the vector operation to be created *inside the loop*, is
|
determined by the type of the other arguments to STMT (in the
|
determined by the type of the other arguments to STMT (in the
|
example we'd check this: widen_sum_optab[vect_short_mode]).
|
example we'd check this: widen_sum_optab[vect_short_mode]).
|
|
|
This is contrary to "regular" reductions, in which the types of all
|
This is contrary to "regular" reductions, in which the types of all
|
the arguments are the same as the type of the reduction variable.
|
the arguments are the same as the type of the reduction variable.
|
For "regular" reductions we can therefore use the same vector type
|
For "regular" reductions we can therefore use the same vector type
|
(and also the same tree-code) when generating the epilog code and
|
(and also the same tree-code) when generating the epilog code and
|
when generating the code inside the loop. */
|
when generating the code inside the loop. */
|
|
|
if (orig_stmt)
|
if (orig_stmt)
|
{
|
{
|
/* This is a reduction pattern: get the vectype from the type of the
|
/* This is a reduction pattern: get the vectype from the type of the
|
reduction variable, and get the tree-code from orig_stmt. */
|
reduction variable, and get the tree-code from orig_stmt. */
|
orig_code = gimple_assign_rhs_code (orig_stmt);
|
orig_code = gimple_assign_rhs_code (orig_stmt);
|
vectype = get_vectype_for_scalar_type (TREE_TYPE (def));
|
vectype = get_vectype_for_scalar_type (TREE_TYPE (def));
|
if (!vectype)
|
if (!vectype)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "unsupported data-type ");
|
fprintf (vect_dump, "unsupported data-type ");
|
print_generic_expr (vect_dump, TREE_TYPE (def), TDF_SLIM);
|
print_generic_expr (vect_dump, TREE_TYPE (def), TDF_SLIM);
|
}
|
}
|
return false;
|
return false;
|
}
|
}
|
|
|
vec_mode = TYPE_MODE (vectype);
|
vec_mode = TYPE_MODE (vectype);
|
}
|
}
|
else
|
else
|
{
|
{
|
/* Regular reduction: use the same vectype and tree-code as used for
|
/* Regular reduction: use the same vectype and tree-code as used for
|
the vector code inside the loop can be used for the epilog code. */
|
the vector code inside the loop can be used for the epilog code. */
|
orig_code = code;
|
orig_code = code;
|
}
|
}
|
|
|
if (nested_cycle)
|
if (nested_cycle)
|
{
|
{
|
def_bb = gimple_bb (reduc_def_stmt);
|
def_bb = gimple_bb (reduc_def_stmt);
|
def_stmt_loop = def_bb->loop_father;
|
def_stmt_loop = def_bb->loop_father;
|
def_arg = PHI_ARG_DEF_FROM_EDGE (reduc_def_stmt,
|
def_arg = PHI_ARG_DEF_FROM_EDGE (reduc_def_stmt,
|
loop_preheader_edge (def_stmt_loop));
|
loop_preheader_edge (def_stmt_loop));
|
if (TREE_CODE (def_arg) == SSA_NAME
|
if (TREE_CODE (def_arg) == SSA_NAME
|
&& (def_arg_stmt = SSA_NAME_DEF_STMT (def_arg))
|
&& (def_arg_stmt = SSA_NAME_DEF_STMT (def_arg))
|
&& gimple_code (def_arg_stmt) == GIMPLE_PHI
|
&& gimple_code (def_arg_stmt) == GIMPLE_PHI
|
&& flow_bb_inside_loop_p (outer_loop, gimple_bb (def_arg_stmt))
|
&& flow_bb_inside_loop_p (outer_loop, gimple_bb (def_arg_stmt))
|
&& vinfo_for_stmt (def_arg_stmt)
|
&& vinfo_for_stmt (def_arg_stmt)
|
&& STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_arg_stmt))
|
&& STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_arg_stmt))
|
== vect_double_reduction_def)
|
== vect_double_reduction_def)
|
double_reduc = true;
|
double_reduc = true;
|
}
|
}
|
|
|
epilog_reduc_code = ERROR_MARK;
|
epilog_reduc_code = ERROR_MARK;
|
if (reduction_code_for_scalar_code (orig_code, &epilog_reduc_code))
|
if (reduction_code_for_scalar_code (orig_code, &epilog_reduc_code))
|
{
|
{
|
reduc_optab = optab_for_tree_code (epilog_reduc_code, vectype,
|
reduc_optab = optab_for_tree_code (epilog_reduc_code, vectype,
|
optab_default);
|
optab_default);
|
if (!reduc_optab)
|
if (!reduc_optab)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "no optab for reduction.");
|
fprintf (vect_dump, "no optab for reduction.");
|
|
|
epilog_reduc_code = ERROR_MARK;
|
epilog_reduc_code = ERROR_MARK;
|
}
|
}
|
|
|
if (reduc_optab
|
if (reduc_optab
|
&& optab_handler (reduc_optab, vec_mode)->insn_code
|
&& optab_handler (reduc_optab, vec_mode)->insn_code
|
== CODE_FOR_nothing)
|
== CODE_FOR_nothing)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "reduc op not supported by target.");
|
fprintf (vect_dump, "reduc op not supported by target.");
|
|
|
epilog_reduc_code = ERROR_MARK;
|
epilog_reduc_code = ERROR_MARK;
|
}
|
}
|
}
|
}
|
else
|
else
|
{
|
{
|
if (!nested_cycle || double_reduc)
|
if (!nested_cycle || double_reduc)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "no reduc code for scalar code.");
|
fprintf (vect_dump, "no reduc code for scalar code.");
|
|
|
return false;
|
return false;
|
}
|
}
|
}
|
}
|
|
|
if (double_reduc && ncopies > 1)
|
if (double_reduc && ncopies > 1)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "multiple types in double reduction");
|
fprintf (vect_dump, "multiple types in double reduction");
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
if (!vec_stmt) /* transformation not required. */
|
if (!vec_stmt) /* transformation not required. */
|
{
|
{
|
STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
|
STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type;
|
if (!vect_model_reduction_cost (stmt_info, epilog_reduc_code, ncopies))
|
if (!vect_model_reduction_cost (stmt_info, epilog_reduc_code, ncopies))
|
return false;
|
return false;
|
return true;
|
return true;
|
}
|
}
|
|
|
/** Transform. **/
|
/** Transform. **/
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "transform reduction.");
|
fprintf (vect_dump, "transform reduction.");
|
|
|
/* FORNOW: Multiple types are not supported for condition. */
|
/* FORNOW: Multiple types are not supported for condition. */
|
if (code == COND_EXPR)
|
if (code == COND_EXPR)
|
gcc_assert (ncopies == 1);
|
gcc_assert (ncopies == 1);
|
|
|
/* Create the destination vector */
|
/* Create the destination vector */
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
vec_dest = vect_create_destination_var (scalar_dest, vectype);
|
|
|
/* In case the vectorization factor (VF) is bigger than the number
|
/* In case the vectorization factor (VF) is bigger than the number
|
of elements that we can fit in a vectype (nunits), we have to generate
|
of elements that we can fit in a vectype (nunits), we have to generate
|
more than one vector stmt - i.e - we need to "unroll" the
|
more than one vector stmt - i.e - we need to "unroll" the
|
vector stmt by a factor VF/nunits. For more details see documentation
|
vector stmt by a factor VF/nunits. For more details see documentation
|
in vectorizable_operation. */
|
in vectorizable_operation. */
|
|
|
/* If the reduction is used in an outer loop we need to generate
|
/* If the reduction is used in an outer loop we need to generate
|
VF intermediate results, like so (e.g. for ncopies=2):
|
VF intermediate results, like so (e.g. for ncopies=2):
|
r0 = phi (init, r0)
|
r0 = phi (init, r0)
|
r1 = phi (init, r1)
|
r1 = phi (init, r1)
|
r0 = x0 + r0;
|
r0 = x0 + r0;
|
r1 = x1 + r1;
|
r1 = x1 + r1;
|
(i.e. we generate VF results in 2 registers).
|
(i.e. we generate VF results in 2 registers).
|
In this case we have a separate def-use cycle for each copy, and therefore
|
In this case we have a separate def-use cycle for each copy, and therefore
|
for each copy we get the vector def for the reduction variable from the
|
for each copy we get the vector def for the reduction variable from the
|
respective phi node created for this copy.
|
respective phi node created for this copy.
|
|
|
Otherwise (the reduction is unused in the loop nest), we can combine
|
Otherwise (the reduction is unused in the loop nest), we can combine
|
together intermediate results, like so (e.g. for ncopies=2):
|
together intermediate results, like so (e.g. for ncopies=2):
|
r = phi (init, r)
|
r = phi (init, r)
|
r = x0 + r;
|
r = x0 + r;
|
r = x1 + r;
|
r = x1 + r;
|
(i.e. we generate VF/2 results in a single register).
|
(i.e. we generate VF/2 results in a single register).
|
In this case for each copy we get the vector def for the reduction variable
|
In this case for each copy we get the vector def for the reduction variable
|
from the vectorized reduction operation generated in the previous iteration.
|
from the vectorized reduction operation generated in the previous iteration.
|
*/
|
*/
|
|
|
if (STMT_VINFO_RELEVANT (stmt_info) == vect_unused_in_scope)
|
if (STMT_VINFO_RELEVANT (stmt_info) == vect_unused_in_scope)
|
{
|
{
|
single_defuse_cycle = true;
|
single_defuse_cycle = true;
|
epilog_copies = 1;
|
epilog_copies = 1;
|
}
|
}
|
else
|
else
|
epilog_copies = ncopies;
|
epilog_copies = ncopies;
|
|
|
prev_stmt_info = NULL;
|
prev_stmt_info = NULL;
|
prev_phi_info = NULL;
|
prev_phi_info = NULL;
|
for (j = 0; j < ncopies; j++)
|
for (j = 0; j < ncopies; j++)
|
{
|
{
|
if (j == 0 || !single_defuse_cycle)
|
if (j == 0 || !single_defuse_cycle)
|
{
|
{
|
/* Create the reduction-phi that defines the reduction-operand. */
|
/* Create the reduction-phi that defines the reduction-operand. */
|
new_phi = create_phi_node (vec_dest, loop->header);
|
new_phi = create_phi_node (vec_dest, loop->header);
|
set_vinfo_for_stmt (new_phi, new_stmt_vec_info (new_phi, loop_vinfo,
|
set_vinfo_for_stmt (new_phi, new_stmt_vec_info (new_phi, loop_vinfo,
|
NULL));
|
NULL));
|
/* Get the vector def for the reduction variable from the phi
|
/* Get the vector def for the reduction variable from the phi
|
node. */
|
node. */
|
reduc_def = PHI_RESULT (new_phi);
|
reduc_def = PHI_RESULT (new_phi);
|
}
|
}
|
|
|
if (code == COND_EXPR)
|
if (code == COND_EXPR)
|
{
|
{
|
first_phi = new_phi;
|
first_phi = new_phi;
|
vectorizable_condition (stmt, gsi, vec_stmt, reduc_def, reduc_index);
|
vectorizable_condition (stmt, gsi, vec_stmt, reduc_def, reduc_index);
|
/* Multiple types are not supported for condition. */
|
/* Multiple types are not supported for condition. */
|
break;
|
break;
|
}
|
}
|
|
|
/* Handle uses. */
|
/* Handle uses. */
|
if (j == 0)
|
if (j == 0)
|
{
|
{
|
loop_vec_def0 = vect_get_vec_def_for_operand (ops[!reduc_index],
|
loop_vec_def0 = vect_get_vec_def_for_operand (ops[!reduc_index],
|
stmt, NULL);
|
stmt, NULL);
|
if (op_type == ternary_op)
|
if (op_type == ternary_op)
|
{
|
{
|
if (reduc_index == 0)
|
if (reduc_index == 0)
|
loop_vec_def1 = vect_get_vec_def_for_operand (ops[2], stmt,
|
loop_vec_def1 = vect_get_vec_def_for_operand (ops[2], stmt,
|
NULL);
|
NULL);
|
else
|
else
|
loop_vec_def1 = vect_get_vec_def_for_operand (ops[1], stmt,
|
loop_vec_def1 = vect_get_vec_def_for_operand (ops[1], stmt,
|
NULL);
|
NULL);
|
}
|
}
|
|
|
/* Get the vector def for the reduction variable from the phi
|
/* Get the vector def for the reduction variable from the phi
|
node. */
|
node. */
|
first_phi = new_phi;
|
first_phi = new_phi;
|
}
|
}
|
else
|
else
|
{
|
{
|
enum vect_def_type dt = vect_unknown_def_type; /* Dummy */
|
enum vect_def_type dt = vect_unknown_def_type; /* Dummy */
|
loop_vec_def0 = vect_get_vec_def_for_stmt_copy (dt, loop_vec_def0);
|
loop_vec_def0 = vect_get_vec_def_for_stmt_copy (dt, loop_vec_def0);
|
if (op_type == ternary_op)
|
if (op_type == ternary_op)
|
loop_vec_def1 = vect_get_vec_def_for_stmt_copy (dt, loop_vec_def1);
|
loop_vec_def1 = vect_get_vec_def_for_stmt_copy (dt, loop_vec_def1);
|
|
|
if (single_defuse_cycle)
|
if (single_defuse_cycle)
|
reduc_def = gimple_assign_lhs (new_stmt);
|
reduc_def = gimple_assign_lhs (new_stmt);
|
else
|
else
|
reduc_def = PHI_RESULT (new_phi);
|
reduc_def = PHI_RESULT (new_phi);
|
|
|
STMT_VINFO_RELATED_STMT (prev_phi_info) = new_phi;
|
STMT_VINFO_RELATED_STMT (prev_phi_info) = new_phi;
|
}
|
}
|
|
|
/* Arguments are ready. Create the new vector stmt. */
|
/* Arguments are ready. Create the new vector stmt. */
|
if (op_type == binary_op)
|
if (op_type == binary_op)
|
{
|
{
|
if (reduc_index == 0)
|
if (reduc_index == 0)
|
expr = build2 (code, vectype, reduc_def, loop_vec_def0);
|
expr = build2 (code, vectype, reduc_def, loop_vec_def0);
|
else
|
else
|
expr = build2 (code, vectype, loop_vec_def0, reduc_def);
|
expr = build2 (code, vectype, loop_vec_def0, reduc_def);
|
}
|
}
|
else
|
else
|
{
|
{
|
if (reduc_index == 0)
|
if (reduc_index == 0)
|
expr = build3 (code, vectype, reduc_def, loop_vec_def0,
|
expr = build3 (code, vectype, reduc_def, loop_vec_def0,
|
loop_vec_def1);
|
loop_vec_def1);
|
else
|
else
|
{
|
{
|
if (reduc_index == 1)
|
if (reduc_index == 1)
|
expr = build3 (code, vectype, loop_vec_def0, reduc_def,
|
expr = build3 (code, vectype, loop_vec_def0, reduc_def,
|
loop_vec_def1);
|
loop_vec_def1);
|
else
|
else
|
expr = build3 (code, vectype, loop_vec_def0, loop_vec_def1,
|
expr = build3 (code, vectype, loop_vec_def0, loop_vec_def1,
|
reduc_def);
|
reduc_def);
|
}
|
}
|
}
|
}
|
|
|
new_stmt = gimple_build_assign (vec_dest, expr);
|
new_stmt = gimple_build_assign (vec_dest, expr);
|
new_temp = make_ssa_name (vec_dest, new_stmt);
|
new_temp = make_ssa_name (vec_dest, new_stmt);
|
gimple_assign_set_lhs (new_stmt, new_temp);
|
gimple_assign_set_lhs (new_stmt, new_temp);
|
vect_finish_stmt_generation (stmt, new_stmt, gsi);
|
vect_finish_stmt_generation (stmt, new_stmt, gsi);
|
|
|
if (j == 0)
|
if (j == 0)
|
STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
|
STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
|
else
|
else
|
STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
|
STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
|
|
|
prev_stmt_info = vinfo_for_stmt (new_stmt);
|
prev_stmt_info = vinfo_for_stmt (new_stmt);
|
prev_phi_info = vinfo_for_stmt (new_phi);
|
prev_phi_info = vinfo_for_stmt (new_phi);
|
}
|
}
|
|
|
/* Finalize the reduction-phi (set its arguments) and create the
|
/* Finalize the reduction-phi (set its arguments) and create the
|
epilog reduction code. */
|
epilog reduction code. */
|
if (!single_defuse_cycle || code == COND_EXPR)
|
if (!single_defuse_cycle || code == COND_EXPR)
|
new_temp = gimple_assign_lhs (*vec_stmt);
|
new_temp = gimple_assign_lhs (*vec_stmt);
|
|
|
vect_create_epilog_for_reduction (new_temp, stmt, epilog_copies,
|
vect_create_epilog_for_reduction (new_temp, stmt, epilog_copies,
|
epilog_reduc_code, first_phi, reduc_index,
|
epilog_reduc_code, first_phi, reduc_index,
|
double_reduc);
|
double_reduc);
|
return true;
|
return true;
|
}
|
}
|
|
|
/* Function vect_min_worthwhile_factor.
|
/* Function vect_min_worthwhile_factor.
|
|
|
For a loop where we could vectorize the operation indicated by CODE,
|
For a loop where we could vectorize the operation indicated by CODE,
|
return the minimum vectorization factor that makes it worthwhile
|
return the minimum vectorization factor that makes it worthwhile
|
to use generic vectors. */
|
to use generic vectors. */
|
int
|
int
|
vect_min_worthwhile_factor (enum tree_code code)
|
vect_min_worthwhile_factor (enum tree_code code)
|
{
|
{
|
switch (code)
|
switch (code)
|
{
|
{
|
case PLUS_EXPR:
|
case PLUS_EXPR:
|
case MINUS_EXPR:
|
case MINUS_EXPR:
|
case NEGATE_EXPR:
|
case NEGATE_EXPR:
|
return 4;
|
return 4;
|
|
|
case BIT_AND_EXPR:
|
case BIT_AND_EXPR:
|
case BIT_IOR_EXPR:
|
case BIT_IOR_EXPR:
|
case BIT_XOR_EXPR:
|
case BIT_XOR_EXPR:
|
case BIT_NOT_EXPR:
|
case BIT_NOT_EXPR:
|
return 2;
|
return 2;
|
|
|
default:
|
default:
|
return INT_MAX;
|
return INT_MAX;
|
}
|
}
|
}
|
}
|
|
|
|
|
/* Function vectorizable_induction
|
/* Function vectorizable_induction
|
|
|
Check if PHI performs an induction computation that can be vectorized.
|
Check if PHI performs an induction computation that can be vectorized.
|
If VEC_STMT is also passed, vectorize the induction PHI: create a vectorized
|
If VEC_STMT is also passed, vectorize the induction PHI: create a vectorized
|
phi to replace it, put it in VEC_STMT, and add it to the same basic block.
|
phi to replace it, put it in VEC_STMT, and add it to the same basic block.
|
Return FALSE if not a vectorizable STMT, TRUE otherwise. */
|
Return FALSE if not a vectorizable STMT, TRUE otherwise. */
|
|
|
bool
|
bool
|
vectorizable_induction (gimple phi, gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED,
|
vectorizable_induction (gimple phi, gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED,
|
gimple *vec_stmt)
|
gimple *vec_stmt)
|
{
|
{
|
stmt_vec_info stmt_info = vinfo_for_stmt (phi);
|
stmt_vec_info stmt_info = vinfo_for_stmt (phi);
|
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
int nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
int nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
int ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
|
int ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits;
|
tree vec_def;
|
tree vec_def;
|
|
|
gcc_assert (ncopies >= 1);
|
gcc_assert (ncopies >= 1);
|
/* FORNOW. This restriction should be relaxed. */
|
/* FORNOW. This restriction should be relaxed. */
|
if (nested_in_vect_loop_p (loop, phi) && ncopies > 1)
|
if (nested_in_vect_loop_p (loop, phi) && ncopies > 1)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "multiple types in nested loop.");
|
fprintf (vect_dump, "multiple types in nested loop.");
|
return false;
|
return false;
|
}
|
}
|
|
|
if (!STMT_VINFO_RELEVANT_P (stmt_info))
|
if (!STMT_VINFO_RELEVANT_P (stmt_info))
|
return false;
|
return false;
|
|
|
/* FORNOW: SLP not supported. */
|
/* FORNOW: SLP not supported. */
|
if (STMT_SLP_TYPE (stmt_info))
|
if (STMT_SLP_TYPE (stmt_info))
|
return false;
|
return false;
|
|
|
gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def);
|
gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def);
|
|
|
if (gimple_code (phi) != GIMPLE_PHI)
|
if (gimple_code (phi) != GIMPLE_PHI)
|
return false;
|
return false;
|
|
|
if (!vec_stmt) /* transformation not required. */
|
if (!vec_stmt) /* transformation not required. */
|
{
|
{
|
STMT_VINFO_TYPE (stmt_info) = induc_vec_info_type;
|
STMT_VINFO_TYPE (stmt_info) = induc_vec_info_type;
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "=== vectorizable_induction ===");
|
fprintf (vect_dump, "=== vectorizable_induction ===");
|
vect_model_induction_cost (stmt_info, ncopies);
|
vect_model_induction_cost (stmt_info, ncopies);
|
return true;
|
return true;
|
}
|
}
|
|
|
/** Transform. **/
|
/** Transform. **/
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "transform induction phi.");
|
fprintf (vect_dump, "transform induction phi.");
|
|
|
vec_def = get_initial_def_for_induction (phi);
|
vec_def = get_initial_def_for_induction (phi);
|
*vec_stmt = SSA_NAME_DEF_STMT (vec_def);
|
*vec_stmt = SSA_NAME_DEF_STMT (vec_def);
|
return true;
|
return true;
|
}
|
}
|
|
|
/* Function vectorizable_live_operation.
|
/* Function vectorizable_live_operation.
|
|
|
STMT computes a value that is used outside the loop. Check if
|
STMT computes a value that is used outside the loop. Check if
|
it can be supported. */
|
it can be supported. */
|
|
|
bool
|
bool
|
vectorizable_live_operation (gimple stmt,
|
vectorizable_live_operation (gimple stmt,
|
gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED,
|
gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED,
|
gimple *vec_stmt ATTRIBUTE_UNUSED)
|
gimple *vec_stmt ATTRIBUTE_UNUSED)
|
{
|
{
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
int i;
|
int i;
|
int op_type;
|
int op_type;
|
tree op;
|
tree op;
|
tree def;
|
tree def;
|
gimple def_stmt;
|
gimple def_stmt;
|
enum vect_def_type dt;
|
enum vect_def_type dt;
|
enum tree_code code;
|
enum tree_code code;
|
enum gimple_rhs_class rhs_class;
|
enum gimple_rhs_class rhs_class;
|
|
|
gcc_assert (STMT_VINFO_LIVE_P (stmt_info));
|
gcc_assert (STMT_VINFO_LIVE_P (stmt_info));
|
|
|
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
|
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
|
return false;
|
return false;
|
|
|
if (!is_gimple_assign (stmt))
|
if (!is_gimple_assign (stmt))
|
return false;
|
return false;
|
|
|
if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
|
if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
|
return false;
|
return false;
|
|
|
/* FORNOW. CHECKME. */
|
/* FORNOW. CHECKME. */
|
if (nested_in_vect_loop_p (loop, stmt))
|
if (nested_in_vect_loop_p (loop, stmt))
|
return false;
|
return false;
|
|
|
code = gimple_assign_rhs_code (stmt);
|
code = gimple_assign_rhs_code (stmt);
|
op_type = TREE_CODE_LENGTH (code);
|
op_type = TREE_CODE_LENGTH (code);
|
rhs_class = get_gimple_rhs_class (code);
|
rhs_class = get_gimple_rhs_class (code);
|
gcc_assert (rhs_class != GIMPLE_UNARY_RHS || op_type == unary_op);
|
gcc_assert (rhs_class != GIMPLE_UNARY_RHS || op_type == unary_op);
|
gcc_assert (rhs_class != GIMPLE_BINARY_RHS || op_type == binary_op);
|
gcc_assert (rhs_class != GIMPLE_BINARY_RHS || op_type == binary_op);
|
|
|
/* FORNOW: support only if all uses are invariant. This means
|
/* FORNOW: support only if all uses are invariant. This means
|
that the scalar operations can remain in place, unvectorized.
|
that the scalar operations can remain in place, unvectorized.
|
The original last scalar value that they compute will be used. */
|
The original last scalar value that they compute will be used. */
|
|
|
for (i = 0; i < op_type; i++)
|
for (i = 0; i < op_type; i++)
|
{
|
{
|
if (rhs_class == GIMPLE_SINGLE_RHS)
|
if (rhs_class == GIMPLE_SINGLE_RHS)
|
op = TREE_OPERAND (gimple_op (stmt, 1), i);
|
op = TREE_OPERAND (gimple_op (stmt, 1), i);
|
else
|
else
|
op = gimple_op (stmt, i + 1);
|
op = gimple_op (stmt, i + 1);
|
if (op
|
if (op
|
&& !vect_is_simple_use (op, loop_vinfo, NULL, &def_stmt, &def, &dt))
|
&& !vect_is_simple_use (op, loop_vinfo, NULL, &def_stmt, &def, &dt))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "use not simple.");
|
fprintf (vect_dump, "use not simple.");
|
return false;
|
return false;
|
}
|
}
|
|
|
if (dt != vect_external_def && dt != vect_constant_def)
|
if (dt != vect_external_def && dt != vect_constant_def)
|
return false;
|
return false;
|
}
|
}
|
|
|
/* No transformation is required for the cases we currently support. */
|
/* No transformation is required for the cases we currently support. */
|
return true;
|
return true;
|
}
|
}
|
|
|
/* Kill any debug uses outside LOOP of SSA names defined in STMT. */
|
/* Kill any debug uses outside LOOP of SSA names defined in STMT. */
|
|
|
static void
|
static void
|
vect_loop_kill_debug_uses (struct loop *loop, gimple stmt)
|
vect_loop_kill_debug_uses (struct loop *loop, gimple stmt)
|
{
|
{
|
ssa_op_iter op_iter;
|
ssa_op_iter op_iter;
|
imm_use_iterator imm_iter;
|
imm_use_iterator imm_iter;
|
def_operand_p def_p;
|
def_operand_p def_p;
|
gimple ustmt;
|
gimple ustmt;
|
|
|
FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt, op_iter, SSA_OP_DEF)
|
FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt, op_iter, SSA_OP_DEF)
|
{
|
{
|
FOR_EACH_IMM_USE_STMT (ustmt, imm_iter, DEF_FROM_PTR (def_p))
|
FOR_EACH_IMM_USE_STMT (ustmt, imm_iter, DEF_FROM_PTR (def_p))
|
{
|
{
|
basic_block bb;
|
basic_block bb;
|
|
|
if (!is_gimple_debug (ustmt))
|
if (!is_gimple_debug (ustmt))
|
continue;
|
continue;
|
|
|
bb = gimple_bb (ustmt);
|
bb = gimple_bb (ustmt);
|
|
|
if (!flow_bb_inside_loop_p (loop, bb))
|
if (!flow_bb_inside_loop_p (loop, bb))
|
{
|
{
|
if (gimple_debug_bind_p (ustmt))
|
if (gimple_debug_bind_p (ustmt))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "killing debug use");
|
fprintf (vect_dump, "killing debug use");
|
|
|
gimple_debug_bind_reset_value (ustmt);
|
gimple_debug_bind_reset_value (ustmt);
|
update_stmt (ustmt);
|
update_stmt (ustmt);
|
}
|
}
|
else
|
else
|
gcc_unreachable ();
|
gcc_unreachable ();
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
|
|
/* Function vect_transform_loop.
|
/* Function vect_transform_loop.
|
|
|
The analysis phase has determined that the loop is vectorizable.
|
The analysis phase has determined that the loop is vectorizable.
|
Vectorize the loop - created vectorized stmts to replace the scalar
|
Vectorize the loop - created vectorized stmts to replace the scalar
|
stmts in the loop, and update the loop exit condition. */
|
stmts in the loop, and update the loop exit condition. */
|
|
|
void
|
void
|
vect_transform_loop (loop_vec_info loop_vinfo)
|
vect_transform_loop (loop_vec_info loop_vinfo)
|
{
|
{
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
|
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
|
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
|
int nbbs = loop->num_nodes;
|
int nbbs = loop->num_nodes;
|
gimple_stmt_iterator si;
|
gimple_stmt_iterator si;
|
int i;
|
int i;
|
tree ratio = NULL;
|
tree ratio = NULL;
|
int vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
int vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
bool strided_store;
|
bool strided_store;
|
bool slp_scheduled = false;
|
bool slp_scheduled = false;
|
unsigned int nunits;
|
unsigned int nunits;
|
tree cond_expr = NULL_TREE;
|
tree cond_expr = NULL_TREE;
|
gimple_seq cond_expr_stmt_list = NULL;
|
gimple_seq cond_expr_stmt_list = NULL;
|
bool do_peeling_for_loop_bound;
|
bool do_peeling_for_loop_bound;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "=== vec_transform_loop ===");
|
fprintf (vect_dump, "=== vec_transform_loop ===");
|
|
|
/* Peel the loop if there are data refs with unknown alignment.
|
/* Peel the loop if there are data refs with unknown alignment.
|
Only one data ref with unknown store is allowed. */
|
Only one data ref with unknown store is allowed. */
|
|
|
if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
|
if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
|
vect_do_peeling_for_alignment (loop_vinfo);
|
vect_do_peeling_for_alignment (loop_vinfo);
|
|
|
do_peeling_for_loop_bound
|
do_peeling_for_loop_bound
|
= (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
= (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
|| (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
|| (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|
&& LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0));
|
&& LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0));
|
|
|
if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo)
|
if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo)
|
|| LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
|
|| LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo))
|
vect_loop_versioning (loop_vinfo,
|
vect_loop_versioning (loop_vinfo,
|
!do_peeling_for_loop_bound,
|
!do_peeling_for_loop_bound,
|
&cond_expr, &cond_expr_stmt_list);
|
&cond_expr, &cond_expr_stmt_list);
|
|
|
/* If the loop has a symbolic number of iterations 'n' (i.e. it's not a
|
/* If the loop has a symbolic number of iterations 'n' (i.e. it's not a
|
compile time constant), or it is a constant that doesn't divide by the
|
compile time constant), or it is a constant that doesn't divide by the
|
vectorization factor, then an epilog loop needs to be created.
|
vectorization factor, then an epilog loop needs to be created.
|
We therefore duplicate the loop: the original loop will be vectorized,
|
We therefore duplicate the loop: the original loop will be vectorized,
|
and will compute the first (n/VF) iterations. The second copy of the loop
|
and will compute the first (n/VF) iterations. The second copy of the loop
|
will remain scalar and will compute the remaining (n%VF) iterations.
|
will remain scalar and will compute the remaining (n%VF) iterations.
|
(VF is the vectorization factor). */
|
(VF is the vectorization factor). */
|
|
|
if (do_peeling_for_loop_bound)
|
if (do_peeling_for_loop_bound)
|
vect_do_peeling_for_loop_bound (loop_vinfo, &ratio,
|
vect_do_peeling_for_loop_bound (loop_vinfo, &ratio,
|
cond_expr, cond_expr_stmt_list);
|
cond_expr, cond_expr_stmt_list);
|
else
|
else
|
ratio = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)),
|
ratio = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)),
|
LOOP_VINFO_INT_NITERS (loop_vinfo) / vectorization_factor);
|
LOOP_VINFO_INT_NITERS (loop_vinfo) / vectorization_factor);
|
|
|
/* 1) Make sure the loop header has exactly two entries
|
/* 1) Make sure the loop header has exactly two entries
|
2) Make sure we have a preheader basic block. */
|
2) Make sure we have a preheader basic block. */
|
|
|
gcc_assert (EDGE_COUNT (loop->header->preds) == 2);
|
gcc_assert (EDGE_COUNT (loop->header->preds) == 2);
|
|
|
split_edge (loop_preheader_edge (loop));
|
split_edge (loop_preheader_edge (loop));
|
|
|
/* FORNOW: the vectorizer supports only loops which body consist
|
/* FORNOW: the vectorizer supports only loops which body consist
|
of one basic block (header + empty latch). When the vectorizer will
|
of one basic block (header + empty latch). When the vectorizer will
|
support more involved loop forms, the order by which the BBs are
|
support more involved loop forms, the order by which the BBs are
|
traversed need to be reconsidered. */
|
traversed need to be reconsidered. */
|
|
|
for (i = 0; i < nbbs; i++)
|
for (i = 0; i < nbbs; i++)
|
{
|
{
|
basic_block bb = bbs[i];
|
basic_block bb = bbs[i];
|
stmt_vec_info stmt_info;
|
stmt_vec_info stmt_info;
|
gimple phi;
|
gimple phi;
|
|
|
for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
|
for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
|
{
|
{
|
phi = gsi_stmt (si);
|
phi = gsi_stmt (si);
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "------>vectorizing phi: ");
|
fprintf (vect_dump, "------>vectorizing phi: ");
|
print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, phi, 0, TDF_SLIM);
|
}
|
}
|
stmt_info = vinfo_for_stmt (phi);
|
stmt_info = vinfo_for_stmt (phi);
|
if (!stmt_info)
|
if (!stmt_info)
|
continue;
|
continue;
|
|
|
if (MAY_HAVE_DEBUG_STMTS && !STMT_VINFO_LIVE_P (stmt_info))
|
if (MAY_HAVE_DEBUG_STMTS && !STMT_VINFO_LIVE_P (stmt_info))
|
vect_loop_kill_debug_uses (loop, phi);
|
vect_loop_kill_debug_uses (loop, phi);
|
|
|
if (!STMT_VINFO_RELEVANT_P (stmt_info)
|
if (!STMT_VINFO_RELEVANT_P (stmt_info)
|
&& !STMT_VINFO_LIVE_P (stmt_info))
|
&& !STMT_VINFO_LIVE_P (stmt_info))
|
continue;
|
continue;
|
|
|
if ((TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info))
|
if ((TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info))
|
!= (unsigned HOST_WIDE_INT) vectorization_factor)
|
!= (unsigned HOST_WIDE_INT) vectorization_factor)
|
&& vect_print_dump_info (REPORT_DETAILS))
|
&& vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "multiple-types.");
|
fprintf (vect_dump, "multiple-types.");
|
|
|
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def)
|
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "transform phi.");
|
fprintf (vect_dump, "transform phi.");
|
vect_transform_stmt (phi, NULL, NULL, NULL, NULL);
|
vect_transform_stmt (phi, NULL, NULL, NULL, NULL);
|
}
|
}
|
}
|
}
|
|
|
for (si = gsi_start_bb (bb); !gsi_end_p (si);)
|
for (si = gsi_start_bb (bb); !gsi_end_p (si);)
|
{
|
{
|
gimple stmt = gsi_stmt (si);
|
gimple stmt = gsi_stmt (si);
|
bool is_store;
|
bool is_store;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "------>vectorizing statement: ");
|
fprintf (vect_dump, "------>vectorizing statement: ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
stmt_info = vinfo_for_stmt (stmt);
|
stmt_info = vinfo_for_stmt (stmt);
|
|
|
/* vector stmts created in the outer-loop during vectorization of
|
/* vector stmts created in the outer-loop during vectorization of
|
stmts in an inner-loop may not have a stmt_info, and do not
|
stmts in an inner-loop may not have a stmt_info, and do not
|
need to be vectorized. */
|
need to be vectorized. */
|
if (!stmt_info)
|
if (!stmt_info)
|
{
|
{
|
gsi_next (&si);
|
gsi_next (&si);
|
continue;
|
continue;
|
}
|
}
|
|
|
if (MAY_HAVE_DEBUG_STMTS && !STMT_VINFO_LIVE_P (stmt_info))
|
if (MAY_HAVE_DEBUG_STMTS && !STMT_VINFO_LIVE_P (stmt_info))
|
vect_loop_kill_debug_uses (loop, stmt);
|
vect_loop_kill_debug_uses (loop, stmt);
|
|
|
if (!STMT_VINFO_RELEVANT_P (stmt_info)
|
if (!STMT_VINFO_RELEVANT_P (stmt_info)
|
&& !STMT_VINFO_LIVE_P (stmt_info))
|
&& !STMT_VINFO_LIVE_P (stmt_info))
|
{
|
{
|
gsi_next (&si);
|
gsi_next (&si);
|
continue;
|
continue;
|
}
|
}
|
|
|
gcc_assert (STMT_VINFO_VECTYPE (stmt_info));
|
gcc_assert (STMT_VINFO_VECTYPE (stmt_info));
|
nunits =
|
nunits =
|
(unsigned int) TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info));
|
(unsigned int) TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info));
|
if (!STMT_SLP_TYPE (stmt_info)
|
if (!STMT_SLP_TYPE (stmt_info)
|
&& nunits != (unsigned int) vectorization_factor
|
&& nunits != (unsigned int) vectorization_factor
|
&& vect_print_dump_info (REPORT_DETAILS))
|
&& vect_print_dump_info (REPORT_DETAILS))
|
/* For SLP VF is set according to unrolling factor, and not to
|
/* For SLP VF is set according to unrolling factor, and not to
|
vector size, hence for SLP this print is not valid. */
|
vector size, hence for SLP this print is not valid. */
|
fprintf (vect_dump, "multiple-types.");
|
fprintf (vect_dump, "multiple-types.");
|
|
|
/* SLP. Schedule all the SLP instances when the first SLP stmt is
|
/* SLP. Schedule all the SLP instances when the first SLP stmt is
|
reached. */
|
reached. */
|
if (STMT_SLP_TYPE (stmt_info))
|
if (STMT_SLP_TYPE (stmt_info))
|
{
|
{
|
if (!slp_scheduled)
|
if (!slp_scheduled)
|
{
|
{
|
slp_scheduled = true;
|
slp_scheduled = true;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "=== scheduling SLP instances ===");
|
fprintf (vect_dump, "=== scheduling SLP instances ===");
|
|
|
vect_schedule_slp (loop_vinfo, NULL);
|
vect_schedule_slp (loop_vinfo, NULL);
|
}
|
}
|
|
|
/* Hybrid SLP stmts must be vectorized in addition to SLP. */
|
/* Hybrid SLP stmts must be vectorized in addition to SLP. */
|
if (!vinfo_for_stmt (stmt) || PURE_SLP_STMT (stmt_info))
|
if (!vinfo_for_stmt (stmt) || PURE_SLP_STMT (stmt_info))
|
{
|
{
|
gsi_next (&si);
|
gsi_next (&si);
|
continue;
|
continue;
|
}
|
}
|
}
|
}
|
|
|
/* -------- vectorize statement ------------ */
|
/* -------- vectorize statement ------------ */
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "transform statement.");
|
fprintf (vect_dump, "transform statement.");
|
|
|
strided_store = false;
|
strided_store = false;
|
is_store = vect_transform_stmt (stmt, &si, &strided_store, NULL, NULL);
|
is_store = vect_transform_stmt (stmt, &si, &strided_store, NULL, NULL);
|
if (is_store)
|
if (is_store)
|
{
|
{
|
if (STMT_VINFO_STRIDED_ACCESS (stmt_info))
|
if (STMT_VINFO_STRIDED_ACCESS (stmt_info))
|
{
|
{
|
/* Interleaving. If IS_STORE is TRUE, the vectorization of the
|
/* Interleaving. If IS_STORE is TRUE, the vectorization of the
|
interleaving chain was completed - free all the stores in
|
interleaving chain was completed - free all the stores in
|
the chain. */
|
the chain. */
|
vect_remove_stores (DR_GROUP_FIRST_DR (stmt_info));
|
vect_remove_stores (DR_GROUP_FIRST_DR (stmt_info));
|
gsi_remove (&si, true);
|
gsi_remove (&si, true);
|
continue;
|
continue;
|
}
|
}
|
else
|
else
|
{
|
{
|
/* Free the attached stmt_vec_info and remove the stmt. */
|
/* Free the attached stmt_vec_info and remove the stmt. */
|
free_stmt_vec_info (stmt);
|
free_stmt_vec_info (stmt);
|
gsi_remove (&si, true);
|
gsi_remove (&si, true);
|
continue;
|
continue;
|
}
|
}
|
}
|
}
|
gsi_next (&si);
|
gsi_next (&si);
|
} /* stmts in BB */
|
} /* stmts in BB */
|
} /* BBs in loop */
|
} /* BBs in loop */
|
|
|
slpeel_make_loop_iterate_ntimes (loop, ratio);
|
slpeel_make_loop_iterate_ntimes (loop, ratio);
|
|
|
/* The memory tags and pointers in vectorized statements need to
|
/* The memory tags and pointers in vectorized statements need to
|
have their SSA forms updated. FIXME, why can't this be delayed
|
have their SSA forms updated. FIXME, why can't this be delayed
|
until all the loops have been transformed? */
|
until all the loops have been transformed? */
|
update_ssa (TODO_update_ssa);
|
update_ssa (TODO_update_ssa);
|
|
|
if (vect_print_dump_info (REPORT_VECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_VECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "LOOP VECTORIZED.");
|
fprintf (vect_dump, "LOOP VECTORIZED.");
|
if (loop->inner && vect_print_dump_info (REPORT_VECTORIZED_LOCATIONS))
|
if (loop->inner && vect_print_dump_info (REPORT_VECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "OUTER LOOP VECTORIZED.");
|
fprintf (vect_dump, "OUTER LOOP VECTORIZED.");
|
}
|
}
|
|
|
