/* SLP - Basic Block Vectorization
|
/* SLP - Basic Block Vectorization
|
Copyright (C) 2007, 2008, 2009, 2010
|
Copyright (C) 2007, 2008, 2009, 2010
|
Free Software Foundation, Inc.
|
Free Software Foundation, Inc.
|
Contributed by Dorit Naishlos <dorit@il.ibm.com>
|
Contributed by Dorit Naishlos <dorit@il.ibm.com>
|
and Ira Rosen <irar@il.ibm.com>
|
and Ira Rosen <irar@il.ibm.com>
|
|
|
This file is part of GCC.
|
This file is part of GCC.
|
|
|
GCC is free software; you can redistribute it and/or modify it under
|
GCC is free software; you can redistribute it and/or modify it under
|
the terms of the GNU General Public License as published by the Free
|
the terms of the GNU General Public License as published by the Free
|
Software Foundation; either version 3, or (at your option) any later
|
Software Foundation; either version 3, or (at your option) any later
|
version.
|
version.
|
|
|
GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
GCC is distributed in the hope that it will be useful, but WITHOUT ANY
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
for more details.
|
for more details.
|
|
|
You should have received a copy of the GNU General Public License
|
You should have received a copy of the GNU General Public License
|
along with GCC; see the file COPYING3. If not see
|
along with GCC; see the file COPYING3. If not see
|
<http://www.gnu.org/licenses/>. */
|
<http://www.gnu.org/licenses/>. */
|
|
|
#include "config.h"
|
#include "config.h"
|
#include "system.h"
|
#include "system.h"
|
#include "coretypes.h"
|
#include "coretypes.h"
|
#include "tm.h"
|
#include "tm.h"
|
#include "ggc.h"
|
#include "ggc.h"
|
#include "tree.h"
|
#include "tree.h"
|
#include "target.h"
|
#include "target.h"
|
#include "basic-block.h"
|
#include "basic-block.h"
|
#include "diagnostic.h"
|
#include "diagnostic.h"
|
#include "tree-flow.h"
|
#include "tree-flow.h"
|
#include "tree-dump.h"
|
#include "tree-dump.h"
|
#include "cfgloop.h"
|
#include "cfgloop.h"
|
#include "cfglayout.h"
|
#include "cfglayout.h"
|
#include "expr.h"
|
#include "expr.h"
|
#include "recog.h"
|
#include "recog.h"
|
#include "optabs.h"
|
#include "optabs.h"
|
#include "tree-vectorizer.h"
|
#include "tree-vectorizer.h"
|
|
|
/* Extract the location of the basic block in the source code.
|
/* Extract the location of the basic block in the source code.
|
Return the basic block location if succeed and NULL if not. */
|
Return the basic block location if succeed and NULL if not. */
|
|
|
LOC
|
LOC
|
find_bb_location (basic_block bb)
|
find_bb_location (basic_block bb)
|
{
|
{
|
gimple stmt = NULL;
|
gimple stmt = NULL;
|
gimple_stmt_iterator si;
|
gimple_stmt_iterator si;
|
|
|
if (!bb)
|
if (!bb)
|
return UNKNOWN_LOC;
|
return UNKNOWN_LOC;
|
|
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
{
|
{
|
stmt = gsi_stmt (si);
|
stmt = gsi_stmt (si);
|
if (gimple_location (stmt) != UNKNOWN_LOC)
|
if (gimple_location (stmt) != UNKNOWN_LOC)
|
return gimple_location (stmt);
|
return gimple_location (stmt);
|
}
|
}
|
|
|
return UNKNOWN_LOC;
|
return UNKNOWN_LOC;
|
}
|
}
|
|
|
|
|
/* Recursively free the memory allocated for the SLP tree rooted at NODE. */
|
/* Recursively free the memory allocated for the SLP tree rooted at NODE. */
|
|
|
static void
|
static void
|
vect_free_slp_tree (slp_tree node)
|
vect_free_slp_tree (slp_tree node)
|
{
|
{
|
if (!node)
|
if (!node)
|
return;
|
return;
|
|
|
if (SLP_TREE_LEFT (node))
|
if (SLP_TREE_LEFT (node))
|
vect_free_slp_tree (SLP_TREE_LEFT (node));
|
vect_free_slp_tree (SLP_TREE_LEFT (node));
|
|
|
if (SLP_TREE_RIGHT (node))
|
if (SLP_TREE_RIGHT (node))
|
vect_free_slp_tree (SLP_TREE_RIGHT (node));
|
vect_free_slp_tree (SLP_TREE_RIGHT (node));
|
|
|
VEC_free (gimple, heap, SLP_TREE_SCALAR_STMTS (node));
|
VEC_free (gimple, heap, SLP_TREE_SCALAR_STMTS (node));
|
|
|
if (SLP_TREE_VEC_STMTS (node))
|
if (SLP_TREE_VEC_STMTS (node))
|
VEC_free (gimple, heap, SLP_TREE_VEC_STMTS (node));
|
VEC_free (gimple, heap, SLP_TREE_VEC_STMTS (node));
|
|
|
free (node);
|
free (node);
|
}
|
}
|
|
|
|
|
/* Free the memory allocated for the SLP instance. */
|
/* Free the memory allocated for the SLP instance. */
|
|
|
void
|
void
|
vect_free_slp_instance (slp_instance instance)
|
vect_free_slp_instance (slp_instance instance)
|
{
|
{
|
vect_free_slp_tree (SLP_INSTANCE_TREE (instance));
|
vect_free_slp_tree (SLP_INSTANCE_TREE (instance));
|
VEC_free (int, heap, SLP_INSTANCE_LOAD_PERMUTATION (instance));
|
VEC_free (int, heap, SLP_INSTANCE_LOAD_PERMUTATION (instance));
|
VEC_free (slp_tree, heap, SLP_INSTANCE_LOADS (instance));
|
VEC_free (slp_tree, heap, SLP_INSTANCE_LOADS (instance));
|
}
|
}
|
|
|
|
|
/* Get the defs for the rhs of STMT (collect them in DEF_STMTS0/1), check that
|
/* Get the defs for the rhs of STMT (collect them in DEF_STMTS0/1), check that
|
they are of a legal type and that they match the defs of the first stmt of
|
they are of a legal type and that they match the defs of the first stmt of
|
the SLP group (stored in FIRST_STMT_...). */
|
the SLP group (stored in FIRST_STMT_...). */
|
|
|
static bool
|
static bool
|
vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
|
vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
|
slp_tree slp_node, gimple stmt,
|
slp_tree slp_node, gimple stmt,
|
VEC (gimple, heap) **def_stmts0,
|
VEC (gimple, heap) **def_stmts0,
|
VEC (gimple, heap) **def_stmts1,
|
VEC (gimple, heap) **def_stmts1,
|
enum vect_def_type *first_stmt_dt0,
|
enum vect_def_type *first_stmt_dt0,
|
enum vect_def_type *first_stmt_dt1,
|
enum vect_def_type *first_stmt_dt1,
|
tree *first_stmt_def0_type,
|
tree *first_stmt_def0_type,
|
tree *first_stmt_def1_type,
|
tree *first_stmt_def1_type,
|
tree *first_stmt_const_oprnd,
|
tree *first_stmt_const_oprnd,
|
int ncopies_for_cost,
|
int ncopies_for_cost,
|
bool *pattern0, bool *pattern1)
|
bool *pattern0, bool *pattern1)
|
{
|
{
|
tree oprnd;
|
tree oprnd;
|
unsigned int i, number_of_oprnds;
|
unsigned int i, number_of_oprnds;
|
tree def;
|
tree def;
|
gimple def_stmt;
|
gimple def_stmt;
|
enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
|
enum vect_def_type dt[2] = {vect_unknown_def_type, vect_unknown_def_type};
|
stmt_vec_info stmt_info =
|
stmt_vec_info stmt_info =
|
vinfo_for_stmt (VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0));
|
vinfo_for_stmt (VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0));
|
enum gimple_rhs_class rhs_class;
|
enum gimple_rhs_class rhs_class;
|
struct loop *loop = NULL;
|
struct loop *loop = NULL;
|
|
|
if (loop_vinfo)
|
if (loop_vinfo)
|
loop = LOOP_VINFO_LOOP (loop_vinfo);
|
loop = LOOP_VINFO_LOOP (loop_vinfo);
|
|
|
rhs_class = get_gimple_rhs_class (gimple_assign_rhs_code (stmt));
|
rhs_class = get_gimple_rhs_class (gimple_assign_rhs_code (stmt));
|
number_of_oprnds = gimple_num_ops (stmt) - 1; /* RHS only */
|
number_of_oprnds = gimple_num_ops (stmt) - 1; /* RHS only */
|
|
|
for (i = 0; i < number_of_oprnds; i++)
|
for (i = 0; i < number_of_oprnds; i++)
|
{
|
{
|
oprnd = gimple_op (stmt, i + 1);
|
oprnd = gimple_op (stmt, i + 1);
|
|
|
if (!vect_is_simple_use (oprnd, loop_vinfo, bb_vinfo, &def_stmt, &def,
|
if (!vect_is_simple_use (oprnd, loop_vinfo, bb_vinfo, &def_stmt, &def,
|
&dt[i])
|
&dt[i])
|
|| (!def_stmt && dt[i] != vect_constant_def))
|
|| (!def_stmt && dt[i] != vect_constant_def))
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump, "Build SLP failed: can't find def for ");
|
fprintf (vect_dump, "Build SLP failed: can't find def for ");
|
print_generic_expr (vect_dump, oprnd, TDF_SLIM);
|
print_generic_expr (vect_dump, oprnd, TDF_SLIM);
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
/* Check if DEF_STMT is a part of a pattern in LOOP and get the def stmt
|
/* Check if DEF_STMT is a part of a pattern in LOOP and get the def stmt
|
from the pattern. Check that all the stmts of the node are in the
|
from the pattern. Check that all the stmts of the node are in the
|
pattern. */
|
pattern. */
|
if (loop && def_stmt && gimple_bb (def_stmt)
|
if (loop && def_stmt && gimple_bb (def_stmt)
|
&& flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
|
&& flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
|
&& vinfo_for_stmt (def_stmt)
|
&& vinfo_for_stmt (def_stmt)
|
&& STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (def_stmt)))
|
&& STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (def_stmt)))
|
{
|
{
|
if (!*first_stmt_dt0)
|
if (!*first_stmt_dt0)
|
*pattern0 = true;
|
*pattern0 = true;
|
else
|
else
|
{
|
{
|
if (i == 1 && !*first_stmt_dt1)
|
if (i == 1 && !*first_stmt_dt1)
|
*pattern1 = true;
|
*pattern1 = true;
|
else if ((i == 0 && !*pattern0) || (i == 1 && !*pattern1))
|
else if ((i == 0 && !*pattern0) || (i == 1 && !*pattern1))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "Build SLP failed: some of the stmts"
|
fprintf (vect_dump, "Build SLP failed: some of the stmts"
|
" are in a pattern, and others are not ");
|
" are in a pattern, and others are not ");
|
print_generic_expr (vect_dump, oprnd, TDF_SLIM);
|
print_generic_expr (vect_dump, oprnd, TDF_SLIM);
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
}
|
}
|
|
|
def_stmt = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt));
|
def_stmt = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt));
|
dt[i] = STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt));
|
dt[i] = STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt));
|
|
|
if (*dt == vect_unknown_def_type)
|
if (*dt == vect_unknown_def_type)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "Unsupported pattern.");
|
fprintf (vect_dump, "Unsupported pattern.");
|
return false;
|
return false;
|
}
|
}
|
|
|
switch (gimple_code (def_stmt))
|
switch (gimple_code (def_stmt))
|
{
|
{
|
case GIMPLE_PHI:
|
case GIMPLE_PHI:
|
def = gimple_phi_result (def_stmt);
|
def = gimple_phi_result (def_stmt);
|
break;
|
break;
|
|
|
case GIMPLE_ASSIGN:
|
case GIMPLE_ASSIGN:
|
def = gimple_assign_lhs (def_stmt);
|
def = gimple_assign_lhs (def_stmt);
|
break;
|
break;
|
|
|
default:
|
default:
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "unsupported defining stmt: ");
|
fprintf (vect_dump, "unsupported defining stmt: ");
|
return false;
|
return false;
|
}
|
}
|
}
|
}
|
|
|
if (!*first_stmt_dt0)
|
if (!*first_stmt_dt0)
|
{
|
{
|
/* op0 of the first stmt of the group - store its info. */
|
/* op0 of the first stmt of the group - store its info. */
|
*first_stmt_dt0 = dt[i];
|
*first_stmt_dt0 = dt[i];
|
if (def)
|
if (def)
|
*first_stmt_def0_type = TREE_TYPE (def);
|
*first_stmt_def0_type = TREE_TYPE (def);
|
else
|
else
|
*first_stmt_const_oprnd = oprnd;
|
*first_stmt_const_oprnd = oprnd;
|
|
|
/* Analyze costs (for the first stmt of the group only). */
|
/* Analyze costs (for the first stmt of the group only). */
|
if (rhs_class != GIMPLE_SINGLE_RHS)
|
if (rhs_class != GIMPLE_SINGLE_RHS)
|
/* Not memory operation (we don't call this functions for loads). */
|
/* Not memory operation (we don't call this functions for loads). */
|
vect_model_simple_cost (stmt_info, ncopies_for_cost, dt, slp_node);
|
vect_model_simple_cost (stmt_info, ncopies_for_cost, dt, slp_node);
|
else
|
else
|
/* Store. */
|
/* Store. */
|
vect_model_store_cost (stmt_info, ncopies_for_cost, dt[0], slp_node);
|
vect_model_store_cost (stmt_info, ncopies_for_cost, dt[0], slp_node);
|
}
|
}
|
|
|
else
|
else
|
{
|
{
|
if (!*first_stmt_dt1 && i == 1)
|
if (!*first_stmt_dt1 && i == 1)
|
{
|
{
|
/* op1 of the first stmt of the group - store its info. */
|
/* op1 of the first stmt of the group - store its info. */
|
*first_stmt_dt1 = dt[i];
|
*first_stmt_dt1 = dt[i];
|
if (def)
|
if (def)
|
*first_stmt_def1_type = TREE_TYPE (def);
|
*first_stmt_def1_type = TREE_TYPE (def);
|
else
|
else
|
{
|
{
|
/* We assume that the stmt contains only one constant
|
/* We assume that the stmt contains only one constant
|
operand. We fail otherwise, to be on the safe side. */
|
operand. We fail otherwise, to be on the safe side. */
|
if (*first_stmt_const_oprnd)
|
if (*first_stmt_const_oprnd)
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
fprintf (vect_dump, "Build SLP failed: two constant "
|
fprintf (vect_dump, "Build SLP failed: two constant "
|
"oprnds in stmt");
|
"oprnds in stmt");
|
return false;
|
return false;
|
}
|
}
|
*first_stmt_const_oprnd = oprnd;
|
*first_stmt_const_oprnd = oprnd;
|
}
|
}
|
}
|
}
|
else
|
else
|
{
|
{
|
/* Not first stmt of the group, check that the def-stmt/s match
|
/* Not first stmt of the group, check that the def-stmt/s match
|
the def-stmt/s of the first stmt. */
|
the def-stmt/s of the first stmt. */
|
if ((i == 0
|
if ((i == 0
|
&& (*first_stmt_dt0 != dt[i]
|
&& (*first_stmt_dt0 != dt[i]
|
|| (*first_stmt_def0_type && def
|
|| (*first_stmt_def0_type && def
|
&& !types_compatible_p (*first_stmt_def0_type,
|
&& !types_compatible_p (*first_stmt_def0_type,
|
TREE_TYPE (def)))))
|
TREE_TYPE (def)))))
|
|| (i == 1
|
|| (i == 1
|
&& (*first_stmt_dt1 != dt[i]
|
&& (*first_stmt_dt1 != dt[i]
|
|| (*first_stmt_def1_type && def
|
|| (*first_stmt_def1_type && def
|
&& !types_compatible_p (*first_stmt_def1_type,
|
&& !types_compatible_p (*first_stmt_def1_type,
|
TREE_TYPE (def)))))
|
TREE_TYPE (def)))))
|
|| (!def
|
|| (!def
|
&& !types_compatible_p (TREE_TYPE (*first_stmt_const_oprnd),
|
&& !types_compatible_p (TREE_TYPE (*first_stmt_const_oprnd),
|
TREE_TYPE (oprnd))))
|
TREE_TYPE (oprnd))))
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
fprintf (vect_dump, "Build SLP failed: different types ");
|
fprintf (vect_dump, "Build SLP failed: different types ");
|
|
|
return false;
|
return false;
|
}
|
}
|
}
|
}
|
}
|
}
|
|
|
/* Check the types of the definitions. */
|
/* Check the types of the definitions. */
|
switch (dt[i])
|
switch (dt[i])
|
{
|
{
|
case vect_constant_def:
|
case vect_constant_def:
|
case vect_external_def:
|
case vect_external_def:
|
break;
|
break;
|
|
|
case vect_internal_def:
|
case vect_internal_def:
|
if (i == 0)
|
if (i == 0)
|
VEC_safe_push (gimple, heap, *def_stmts0, def_stmt);
|
VEC_safe_push (gimple, heap, *def_stmts0, def_stmt);
|
else
|
else
|
VEC_safe_push (gimple, heap, *def_stmts1, def_stmt);
|
VEC_safe_push (gimple, heap, *def_stmts1, def_stmt);
|
break;
|
break;
|
|
|
default:
|
default:
|
/* FORNOW: Not supported. */
|
/* FORNOW: Not supported. */
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump, "Build SLP failed: illegal type of def ");
|
fprintf (vect_dump, "Build SLP failed: illegal type of def ");
|
print_generic_expr (vect_dump, def, TDF_SLIM);
|
print_generic_expr (vect_dump, def, TDF_SLIM);
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
}
|
}
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
|
|
/* Recursively build an SLP tree starting from NODE.
|
/* Recursively build an SLP tree starting from NODE.
|
Fail (and return FALSE) if def-stmts are not isomorphic, require data
|
Fail (and return FALSE) if def-stmts are not isomorphic, require data
|
permutation or are of unsupported types of operation. Otherwise, return
|
permutation or are of unsupported types of operation. Otherwise, return
|
TRUE. */
|
TRUE. */
|
|
|
static bool
|
static bool
|
vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
|
vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
|
slp_tree *node, unsigned int group_size,
|
slp_tree *node, unsigned int group_size,
|
int *inside_cost, int *outside_cost,
|
int *inside_cost, int *outside_cost,
|
int ncopies_for_cost, unsigned int *max_nunits,
|
int ncopies_for_cost, unsigned int *max_nunits,
|
VEC (int, heap) **load_permutation,
|
VEC (int, heap) **load_permutation,
|
VEC (slp_tree, heap) **loads,
|
VEC (slp_tree, heap) **loads,
|
unsigned int vectorization_factor)
|
unsigned int vectorization_factor)
|
{
|
{
|
VEC (gimple, heap) *def_stmts0 = VEC_alloc (gimple, heap, group_size);
|
VEC (gimple, heap) *def_stmts0 = VEC_alloc (gimple, heap, group_size);
|
VEC (gimple, heap) *def_stmts1 = VEC_alloc (gimple, heap, group_size);
|
VEC (gimple, heap) *def_stmts1 = VEC_alloc (gimple, heap, group_size);
|
unsigned int i;
|
unsigned int i;
|
VEC (gimple, heap) *stmts = SLP_TREE_SCALAR_STMTS (*node);
|
VEC (gimple, heap) *stmts = SLP_TREE_SCALAR_STMTS (*node);
|
gimple stmt = VEC_index (gimple, stmts, 0);
|
gimple stmt = VEC_index (gimple, stmts, 0);
|
enum vect_def_type first_stmt_dt0 = vect_uninitialized_def;
|
enum vect_def_type first_stmt_dt0 = vect_uninitialized_def;
|
enum vect_def_type first_stmt_dt1 = vect_uninitialized_def;
|
enum vect_def_type first_stmt_dt1 = vect_uninitialized_def;
|
enum tree_code first_stmt_code = ERROR_MARK, rhs_code;
|
enum tree_code first_stmt_code = ERROR_MARK, rhs_code;
|
tree first_stmt_def1_type = NULL_TREE, first_stmt_def0_type = NULL_TREE;
|
tree first_stmt_def1_type = NULL_TREE, first_stmt_def0_type = NULL_TREE;
|
tree lhs;
|
tree lhs;
|
bool stop_recursion = false, need_same_oprnds = false;
|
bool stop_recursion = false, need_same_oprnds = false;
|
tree vectype, scalar_type, first_op1 = NULL_TREE;
|
tree vectype, scalar_type, first_op1 = NULL_TREE;
|
unsigned int ncopies;
|
unsigned int ncopies;
|
optab optab;
|
optab optab;
|
int icode;
|
int icode;
|
enum machine_mode optab_op2_mode;
|
enum machine_mode optab_op2_mode;
|
enum machine_mode vec_mode;
|
enum machine_mode vec_mode;
|
tree first_stmt_const_oprnd = NULL_TREE;
|
tree first_stmt_const_oprnd = NULL_TREE;
|
struct data_reference *first_dr;
|
struct data_reference *first_dr;
|
bool pattern0 = false, pattern1 = false;
|
bool pattern0 = false, pattern1 = false;
|
HOST_WIDE_INT dummy;
|
HOST_WIDE_INT dummy;
|
bool permutation = false;
|
bool permutation = false;
|
unsigned int load_place;
|
unsigned int load_place;
|
gimple first_load;
|
gimple first_load;
|
|
|
/* For every stmt in NODE find its def stmt/s. */
|
/* For every stmt in NODE find its def stmt/s. */
|
for (i = 0; VEC_iterate (gimple, stmts, i, stmt); i++)
|
for (i = 0; VEC_iterate (gimple, stmts, i, stmt); i++)
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump, "Build SLP for ");
|
fprintf (vect_dump, "Build SLP for ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
lhs = gimple_get_lhs (stmt);
|
lhs = gimple_get_lhs (stmt);
|
if (lhs == NULL_TREE)
|
if (lhs == NULL_TREE)
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump,
|
fprintf (vect_dump,
|
"Build SLP failed: not GIMPLE_ASSIGN nor GIMPLE_CALL");
|
"Build SLP failed: not GIMPLE_ASSIGN nor GIMPLE_CALL");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
scalar_type = vect_get_smallest_scalar_type (stmt, &dummy, &dummy);
|
scalar_type = vect_get_smallest_scalar_type (stmt, &dummy, &dummy);
|
vectype = get_vectype_for_scalar_type (scalar_type);
|
vectype = get_vectype_for_scalar_type (scalar_type);
|
if (!vectype)
|
if (!vectype)
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump, "Build SLP failed: unsupported data-type ");
|
fprintf (vect_dump, "Build SLP failed: unsupported data-type ");
|
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
|
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
|
}
|
}
|
return false;
|
return false;
|
}
|
}
|
|
|
ncopies = vectorization_factor / TYPE_VECTOR_SUBPARTS (vectype);
|
ncopies = vectorization_factor / TYPE_VECTOR_SUBPARTS (vectype);
|
if (ncopies != 1)
|
if (ncopies != 1)
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
fprintf (vect_dump, "SLP with multiple types ");
|
fprintf (vect_dump, "SLP with multiple types ");
|
|
|
/* FORNOW: multiple types are unsupported in BB SLP. */
|
/* FORNOW: multiple types are unsupported in BB SLP. */
|
if (bb_vinfo)
|
if (bb_vinfo)
|
return false;
|
return false;
|
}
|
}
|
|
|
/* In case of multiple types we need to detect the smallest type. */
|
/* In case of multiple types we need to detect the smallest type. */
|
if (*max_nunits < TYPE_VECTOR_SUBPARTS (vectype))
|
if (*max_nunits < TYPE_VECTOR_SUBPARTS (vectype))
|
*max_nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
*max_nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
|
|
if (is_gimple_call (stmt))
|
if (is_gimple_call (stmt))
|
rhs_code = CALL_EXPR;
|
rhs_code = CALL_EXPR;
|
else
|
else
|
rhs_code = gimple_assign_rhs_code (stmt);
|
rhs_code = gimple_assign_rhs_code (stmt);
|
|
|
/* Check the operation. */
|
/* Check the operation. */
|
if (i == 0)
|
if (i == 0)
|
{
|
{
|
first_stmt_code = rhs_code;
|
first_stmt_code = rhs_code;
|
|
|
/* Shift arguments should be equal in all the packed stmts for a
|
/* Shift arguments should be equal in all the packed stmts for a
|
vector shift with scalar shift operand. */
|
vector shift with scalar shift operand. */
|
if (rhs_code == LSHIFT_EXPR || rhs_code == RSHIFT_EXPR
|
if (rhs_code == LSHIFT_EXPR || rhs_code == RSHIFT_EXPR
|
|| rhs_code == LROTATE_EXPR
|
|| rhs_code == LROTATE_EXPR
|
|| rhs_code == RROTATE_EXPR)
|
|| rhs_code == RROTATE_EXPR)
|
{
|
{
|
vec_mode = TYPE_MODE (vectype);
|
vec_mode = TYPE_MODE (vectype);
|
|
|
/* First see if we have a vector/vector shift. */
|
/* First see if we have a vector/vector shift. */
|
optab = optab_for_tree_code (rhs_code, vectype,
|
optab = optab_for_tree_code (rhs_code, vectype,
|
optab_vector);
|
optab_vector);
|
|
|
if (!optab
|
if (!optab
|
|| (optab->handlers[(int) vec_mode].insn_code
|
|| (optab->handlers[(int) vec_mode].insn_code
|
== CODE_FOR_nothing))
|
== CODE_FOR_nothing))
|
{
|
{
|
/* No vector/vector shift, try for a vector/scalar shift. */
|
/* No vector/vector shift, try for a vector/scalar shift. */
|
optab = optab_for_tree_code (rhs_code, vectype,
|
optab = optab_for_tree_code (rhs_code, vectype,
|
optab_scalar);
|
optab_scalar);
|
|
|
if (!optab)
|
if (!optab)
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
fprintf (vect_dump, "Build SLP failed: no optab.");
|
fprintf (vect_dump, "Build SLP failed: no optab.");
|
return false;
|
return false;
|
}
|
}
|
icode = (int) optab->handlers[(int) vec_mode].insn_code;
|
icode = (int) optab->handlers[(int) vec_mode].insn_code;
|
if (icode == CODE_FOR_nothing)
|
if (icode == CODE_FOR_nothing)
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
fprintf (vect_dump, "Build SLP failed: "
|
fprintf (vect_dump, "Build SLP failed: "
|
"op not supported by target.");
|
"op not supported by target.");
|
return false;
|
return false;
|
}
|
}
|
optab_op2_mode = insn_data[icode].operand[2].mode;
|
optab_op2_mode = insn_data[icode].operand[2].mode;
|
if (!VECTOR_MODE_P (optab_op2_mode))
|
if (!VECTOR_MODE_P (optab_op2_mode))
|
{
|
{
|
need_same_oprnds = true;
|
need_same_oprnds = true;
|
first_op1 = gimple_assign_rhs2 (stmt);
|
first_op1 = gimple_assign_rhs2 (stmt);
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
else
|
else
|
{
|
{
|
if (first_stmt_code != rhs_code
|
if (first_stmt_code != rhs_code
|
&& (first_stmt_code != IMAGPART_EXPR
|
&& (first_stmt_code != IMAGPART_EXPR
|
|| rhs_code != REALPART_EXPR)
|
|| rhs_code != REALPART_EXPR)
|
&& (first_stmt_code != REALPART_EXPR
|
&& (first_stmt_code != REALPART_EXPR
|
|| rhs_code != IMAGPART_EXPR))
|
|| rhs_code != IMAGPART_EXPR))
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump,
|
fprintf (vect_dump,
|
"Build SLP failed: different operation in stmt ");
|
"Build SLP failed: different operation in stmt ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
if (need_same_oprnds
|
if (need_same_oprnds
|
&& !operand_equal_p (first_op1, gimple_assign_rhs2 (stmt), 0))
|
&& !operand_equal_p (first_op1, gimple_assign_rhs2 (stmt), 0))
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump,
|
fprintf (vect_dump,
|
"Build SLP failed: different shift arguments in ");
|
"Build SLP failed: different shift arguments in ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
}
|
}
|
|
|
/* Strided store or load. */
|
/* Strided store or load. */
|
if (STMT_VINFO_STRIDED_ACCESS (vinfo_for_stmt (stmt)))
|
if (STMT_VINFO_STRIDED_ACCESS (vinfo_for_stmt (stmt)))
|
{
|
{
|
if (REFERENCE_CLASS_P (lhs))
|
if (REFERENCE_CLASS_P (lhs))
|
{
|
{
|
/* Store. */
|
/* Store. */
|
if (!vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo, *node,
|
if (!vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo, *node,
|
stmt, &def_stmts0, &def_stmts1,
|
stmt, &def_stmts0, &def_stmts1,
|
&first_stmt_dt0,
|
&first_stmt_dt0,
|
&first_stmt_dt1,
|
&first_stmt_dt1,
|
&first_stmt_def0_type,
|
&first_stmt_def0_type,
|
&first_stmt_def1_type,
|
&first_stmt_def1_type,
|
&first_stmt_const_oprnd,
|
&first_stmt_const_oprnd,
|
ncopies_for_cost,
|
ncopies_for_cost,
|
&pattern0, &pattern1))
|
&pattern0, &pattern1))
|
return false;
|
return false;
|
}
|
}
|
else
|
else
|
{
|
{
|
/* Load. */
|
/* Load. */
|
/* FORNOW: Check that there is no gap between the loads. */
|
/* FORNOW: Check that there is no gap between the loads. */
|
if ((DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)) == stmt
|
if ((DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)) == stmt
|
&& DR_GROUP_GAP (vinfo_for_stmt (stmt)) != 0)
|
&& DR_GROUP_GAP (vinfo_for_stmt (stmt)) != 0)
|
|| (DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)) != stmt
|
|| (DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)) != stmt
|
&& DR_GROUP_GAP (vinfo_for_stmt (stmt)) != 1))
|
&& DR_GROUP_GAP (vinfo_for_stmt (stmt)) != 1))
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump, "Build SLP failed: strided "
|
fprintf (vect_dump, "Build SLP failed: strided "
|
"loads have gaps ");
|
"loads have gaps ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
/* Check that the size of interleaved loads group is not
|
/* Check that the size of interleaved loads group is not
|
greater than the SLP group size. */
|
greater than the SLP group size. */
|
if (DR_GROUP_SIZE (vinfo_for_stmt (stmt))
|
if (DR_GROUP_SIZE (vinfo_for_stmt (stmt))
|
> ncopies * group_size)
|
> ncopies * group_size)
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump, "Build SLP failed: the number of "
|
fprintf (vect_dump, "Build SLP failed: the number of "
|
"interleaved loads is greater than"
|
"interleaved loads is greater than"
|
" the SLP group size ");
|
" the SLP group size ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
first_load = DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt));
|
first_load = DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt));
|
|
|
if (first_load == stmt)
|
if (first_load == stmt)
|
{
|
{
|
first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt));
|
first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt));
|
if (vect_supportable_dr_alignment (first_dr)
|
if (vect_supportable_dr_alignment (first_dr)
|
== dr_unaligned_unsupported)
|
== dr_unaligned_unsupported)
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump, "Build SLP failed: unsupported "
|
fprintf (vect_dump, "Build SLP failed: unsupported "
|
"unaligned load ");
|
"unaligned load ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
/* Analyze costs (for the first stmt in the group). */
|
/* Analyze costs (for the first stmt in the group). */
|
vect_model_load_cost (vinfo_for_stmt (stmt),
|
vect_model_load_cost (vinfo_for_stmt (stmt),
|
ncopies_for_cost, *node);
|
ncopies_for_cost, *node);
|
}
|
}
|
|
|
/* Store the place of this load in the interleaving chain. In
|
/* Store the place of this load in the interleaving chain. In
|
case that permutation is needed we later decide if a specific
|
case that permutation is needed we later decide if a specific
|
permutation is supported. */
|
permutation is supported. */
|
load_place = vect_get_place_in_interleaving_chain (stmt,
|
load_place = vect_get_place_in_interleaving_chain (stmt,
|
first_load);
|
first_load);
|
if (load_place != i)
|
if (load_place != i)
|
permutation = true;
|
permutation = true;
|
|
|
VEC_safe_push (int, heap, *load_permutation, load_place);
|
VEC_safe_push (int, heap, *load_permutation, load_place);
|
|
|
/* We stop the tree when we reach a group of loads. */
|
/* We stop the tree when we reach a group of loads. */
|
stop_recursion = true;
|
stop_recursion = true;
|
continue;
|
continue;
|
}
|
}
|
} /* Strided access. */
|
} /* Strided access. */
|
else
|
else
|
{
|
{
|
if (TREE_CODE_CLASS (rhs_code) == tcc_reference)
|
if (TREE_CODE_CLASS (rhs_code) == tcc_reference)
|
{
|
{
|
/* Not strided load. */
|
/* Not strided load. */
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump, "Build SLP failed: not strided load ");
|
fprintf (vect_dump, "Build SLP failed: not strided load ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
/* FORNOW: Not strided loads are not supported. */
|
/* FORNOW: Not strided loads are not supported. */
|
return false;
|
return false;
|
}
|
}
|
|
|
/* Not memory operation. */
|
/* Not memory operation. */
|
if (TREE_CODE_CLASS (rhs_code) != tcc_binary
|
if (TREE_CODE_CLASS (rhs_code) != tcc_binary
|
&& TREE_CODE_CLASS (rhs_code) != tcc_unary)
|
&& TREE_CODE_CLASS (rhs_code) != tcc_unary)
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump, "Build SLP failed: operation");
|
fprintf (vect_dump, "Build SLP failed: operation");
|
fprintf (vect_dump, " unsupported ");
|
fprintf (vect_dump, " unsupported ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
/* Find the def-stmts. */
|
/* Find the def-stmts. */
|
if (!vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo, *node, stmt,
|
if (!vect_get_and_check_slp_defs (loop_vinfo, bb_vinfo, *node, stmt,
|
&def_stmts0, &def_stmts1,
|
&def_stmts0, &def_stmts1,
|
&first_stmt_dt0, &first_stmt_dt1,
|
&first_stmt_dt0, &first_stmt_dt1,
|
&first_stmt_def0_type,
|
&first_stmt_def0_type,
|
&first_stmt_def1_type,
|
&first_stmt_def1_type,
|
&first_stmt_const_oprnd,
|
&first_stmt_const_oprnd,
|
ncopies_for_cost,
|
ncopies_for_cost,
|
&pattern0, &pattern1))
|
&pattern0, &pattern1))
|
return false;
|
return false;
|
}
|
}
|
}
|
}
|
|
|
/* Add the costs of the node to the overall instance costs. */
|
/* Add the costs of the node to the overall instance costs. */
|
*inside_cost += SLP_TREE_INSIDE_OF_LOOP_COST (*node);
|
*inside_cost += SLP_TREE_INSIDE_OF_LOOP_COST (*node);
|
*outside_cost += SLP_TREE_OUTSIDE_OF_LOOP_COST (*node);
|
*outside_cost += SLP_TREE_OUTSIDE_OF_LOOP_COST (*node);
|
|
|
/* Strided loads were reached - stop the recursion. */
|
/* Strided loads were reached - stop the recursion. */
|
if (stop_recursion)
|
if (stop_recursion)
|
{
|
{
|
if (permutation)
|
if (permutation)
|
{
|
{
|
VEC_safe_push (slp_tree, heap, *loads, *node);
|
VEC_safe_push (slp_tree, heap, *loads, *node);
|
*inside_cost += TARG_VEC_PERMUTE_COST * group_size;
|
*inside_cost += TARG_VEC_PERMUTE_COST * group_size;
|
}
|
}
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
/* Create SLP_TREE nodes for the definition node/s. */
|
/* Create SLP_TREE nodes for the definition node/s. */
|
if (first_stmt_dt0 == vect_internal_def)
|
if (first_stmt_dt0 == vect_internal_def)
|
{
|
{
|
slp_tree left_node = XNEW (struct _slp_tree);
|
slp_tree left_node = XNEW (struct _slp_tree);
|
SLP_TREE_SCALAR_STMTS (left_node) = def_stmts0;
|
SLP_TREE_SCALAR_STMTS (left_node) = def_stmts0;
|
SLP_TREE_VEC_STMTS (left_node) = NULL;
|
SLP_TREE_VEC_STMTS (left_node) = NULL;
|
SLP_TREE_LEFT (left_node) = NULL;
|
SLP_TREE_LEFT (left_node) = NULL;
|
SLP_TREE_RIGHT (left_node) = NULL;
|
SLP_TREE_RIGHT (left_node) = NULL;
|
SLP_TREE_OUTSIDE_OF_LOOP_COST (left_node) = 0;
|
SLP_TREE_OUTSIDE_OF_LOOP_COST (left_node) = 0;
|
SLP_TREE_INSIDE_OF_LOOP_COST (left_node) = 0;
|
SLP_TREE_INSIDE_OF_LOOP_COST (left_node) = 0;
|
if (!vect_build_slp_tree (loop_vinfo, bb_vinfo, &left_node, group_size,
|
if (!vect_build_slp_tree (loop_vinfo, bb_vinfo, &left_node, group_size,
|
inside_cost, outside_cost, ncopies_for_cost,
|
inside_cost, outside_cost, ncopies_for_cost,
|
max_nunits, load_permutation, loads,
|
max_nunits, load_permutation, loads,
|
vectorization_factor))
|
vectorization_factor))
|
return false;
|
return false;
|
|
|
SLP_TREE_LEFT (*node) = left_node;
|
SLP_TREE_LEFT (*node) = left_node;
|
}
|
}
|
|
|
if (first_stmt_dt1 == vect_internal_def)
|
if (first_stmt_dt1 == vect_internal_def)
|
{
|
{
|
slp_tree right_node = XNEW (struct _slp_tree);
|
slp_tree right_node = XNEW (struct _slp_tree);
|
SLP_TREE_SCALAR_STMTS (right_node) = def_stmts1;
|
SLP_TREE_SCALAR_STMTS (right_node) = def_stmts1;
|
SLP_TREE_VEC_STMTS (right_node) = NULL;
|
SLP_TREE_VEC_STMTS (right_node) = NULL;
|
SLP_TREE_LEFT (right_node) = NULL;
|
SLP_TREE_LEFT (right_node) = NULL;
|
SLP_TREE_RIGHT (right_node) = NULL;
|
SLP_TREE_RIGHT (right_node) = NULL;
|
SLP_TREE_OUTSIDE_OF_LOOP_COST (right_node) = 0;
|
SLP_TREE_OUTSIDE_OF_LOOP_COST (right_node) = 0;
|
SLP_TREE_INSIDE_OF_LOOP_COST (right_node) = 0;
|
SLP_TREE_INSIDE_OF_LOOP_COST (right_node) = 0;
|
if (!vect_build_slp_tree (loop_vinfo, bb_vinfo, &right_node, group_size,
|
if (!vect_build_slp_tree (loop_vinfo, bb_vinfo, &right_node, group_size,
|
inside_cost, outside_cost, ncopies_for_cost,
|
inside_cost, outside_cost, ncopies_for_cost,
|
max_nunits, load_permutation, loads,
|
max_nunits, load_permutation, loads,
|
vectorization_factor))
|
vectorization_factor))
|
return false;
|
return false;
|
|
|
SLP_TREE_RIGHT (*node) = right_node;
|
SLP_TREE_RIGHT (*node) = right_node;
|
}
|
}
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
|
|
static void
|
static void
|
vect_print_slp_tree (slp_tree node)
|
vect_print_slp_tree (slp_tree node)
|
{
|
{
|
int i;
|
int i;
|
gimple stmt;
|
gimple stmt;
|
|
|
if (!node)
|
if (!node)
|
return;
|
return;
|
|
|
fprintf (vect_dump, "node ");
|
fprintf (vect_dump, "node ");
|
for (i = 0; VEC_iterate (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt); i++)
|
for (i = 0; VEC_iterate (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt); i++)
|
{
|
{
|
fprintf (vect_dump, "\n\tstmt %d ", i);
|
fprintf (vect_dump, "\n\tstmt %d ", i);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
fprintf (vect_dump, "\n");
|
fprintf (vect_dump, "\n");
|
|
|
vect_print_slp_tree (SLP_TREE_LEFT (node));
|
vect_print_slp_tree (SLP_TREE_LEFT (node));
|
vect_print_slp_tree (SLP_TREE_RIGHT (node));
|
vect_print_slp_tree (SLP_TREE_RIGHT (node));
|
}
|
}
|
|
|
|
|
/* Mark the tree rooted at NODE with MARK (PURE_SLP or HYBRID).
|
/* Mark the tree rooted at NODE with MARK (PURE_SLP or HYBRID).
|
If MARK is HYBRID, it refers to a specific stmt in NODE (the stmt at index
|
If MARK is HYBRID, it refers to a specific stmt in NODE (the stmt at index
|
J). Otherwise, MARK is PURE_SLP and J is -1, which indicates that all the
|
J). Otherwise, MARK is PURE_SLP and J is -1, which indicates that all the
|
stmts in NODE are to be marked. */
|
stmts in NODE are to be marked. */
|
|
|
static void
|
static void
|
vect_mark_slp_stmts (slp_tree node, enum slp_vect_type mark, int j)
|
vect_mark_slp_stmts (slp_tree node, enum slp_vect_type mark, int j)
|
{
|
{
|
int i;
|
int i;
|
gimple stmt;
|
gimple stmt;
|
|
|
if (!node)
|
if (!node)
|
return;
|
return;
|
|
|
for (i = 0; VEC_iterate (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt); i++)
|
for (i = 0; VEC_iterate (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt); i++)
|
if (j < 0 || i == j)
|
if (j < 0 || i == j)
|
STMT_SLP_TYPE (vinfo_for_stmt (stmt)) = mark;
|
STMT_SLP_TYPE (vinfo_for_stmt (stmt)) = mark;
|
|
|
vect_mark_slp_stmts (SLP_TREE_LEFT (node), mark, j);
|
vect_mark_slp_stmts (SLP_TREE_LEFT (node), mark, j);
|
vect_mark_slp_stmts (SLP_TREE_RIGHT (node), mark, j);
|
vect_mark_slp_stmts (SLP_TREE_RIGHT (node), mark, j);
|
}
|
}
|
|
|
|
|
/* Mark the statements of the tree rooted at NODE as relevant (vect_used). */
|
/* Mark the statements of the tree rooted at NODE as relevant (vect_used). */
|
|
|
static void
|
static void
|
vect_mark_slp_stmts_relevant (slp_tree node)
|
vect_mark_slp_stmts_relevant (slp_tree node)
|
{
|
{
|
int i;
|
int i;
|
gimple stmt;
|
gimple stmt;
|
stmt_vec_info stmt_info;
|
stmt_vec_info stmt_info;
|
|
|
if (!node)
|
if (!node)
|
return;
|
return;
|
|
|
for (i = 0; VEC_iterate (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt); i++)
|
for (i = 0; VEC_iterate (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt); i++)
|
{
|
{
|
stmt_info = vinfo_for_stmt (stmt);
|
stmt_info = vinfo_for_stmt (stmt);
|
gcc_assert (!STMT_VINFO_RELEVANT (stmt_info)
|
gcc_assert (!STMT_VINFO_RELEVANT (stmt_info)
|
|| STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_scope);
|
|| STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_scope);
|
STMT_VINFO_RELEVANT (stmt_info) = vect_used_in_scope;
|
STMT_VINFO_RELEVANT (stmt_info) = vect_used_in_scope;
|
}
|
}
|
|
|
vect_mark_slp_stmts_relevant (SLP_TREE_LEFT (node));
|
vect_mark_slp_stmts_relevant (SLP_TREE_LEFT (node));
|
vect_mark_slp_stmts_relevant (SLP_TREE_RIGHT (node));
|
vect_mark_slp_stmts_relevant (SLP_TREE_RIGHT (node));
|
}
|
}
|
|
|
|
|
/* Check if the permutation required by the SLP INSTANCE is supported.
|
/* Check if the permutation required by the SLP INSTANCE is supported.
|
Reorganize the SLP nodes stored in SLP_INSTANCE_LOADS if needed. */
|
Reorganize the SLP nodes stored in SLP_INSTANCE_LOADS if needed. */
|
|
|
static bool
|
static bool
|
vect_supported_slp_permutation_p (slp_instance instance)
|
vect_supported_slp_permutation_p (slp_instance instance)
|
{
|
{
|
slp_tree node = VEC_index (slp_tree, SLP_INSTANCE_LOADS (instance), 0);
|
slp_tree node = VEC_index (slp_tree, SLP_INSTANCE_LOADS (instance), 0);
|
gimple stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (node), 0);
|
gimple stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (node), 0);
|
gimple first_load = DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt));
|
gimple first_load = DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt));
|
VEC (slp_tree, heap) *sorted_loads = NULL;
|
VEC (slp_tree, heap) *sorted_loads = NULL;
|
int index;
|
int index;
|
slp_tree *tmp_loads = NULL;
|
slp_tree *tmp_loads = NULL;
|
int group_size = SLP_INSTANCE_GROUP_SIZE (instance), i, j;
|
int group_size = SLP_INSTANCE_GROUP_SIZE (instance), i, j;
|
slp_tree load;
|
slp_tree load;
|
|
|
/* FORNOW: The only supported loads permutation is loads from the same
|
/* FORNOW: The only supported loads permutation is loads from the same
|
location in all the loads in the node, when the data-refs in
|
location in all the loads in the node, when the data-refs in
|
nodes of LOADS constitute an interleaving chain.
|
nodes of LOADS constitute an interleaving chain.
|
Sort the nodes according to the order of accesses in the chain. */
|
Sort the nodes according to the order of accesses in the chain. */
|
tmp_loads = (slp_tree *) xmalloc (sizeof (slp_tree) * group_size);
|
tmp_loads = (slp_tree *) xmalloc (sizeof (slp_tree) * group_size);
|
for (i = 0, j = 0;
|
for (i = 0, j = 0;
|
VEC_iterate (int, SLP_INSTANCE_LOAD_PERMUTATION (instance), i, index)
|
VEC_iterate (int, SLP_INSTANCE_LOAD_PERMUTATION (instance), i, index)
|
&& VEC_iterate (slp_tree, SLP_INSTANCE_LOADS (instance), j, load);
|
&& VEC_iterate (slp_tree, SLP_INSTANCE_LOADS (instance), j, load);
|
i += group_size, j++)
|
i += group_size, j++)
|
{
|
{
|
gimple scalar_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (load), 0);
|
gimple scalar_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (load), 0);
|
/* Check that the loads are all in the same interleaving chain. */
|
/* Check that the loads are all in the same interleaving chain. */
|
if (DR_GROUP_FIRST_DR (vinfo_for_stmt (scalar_stmt)) != first_load)
|
if (DR_GROUP_FIRST_DR (vinfo_for_stmt (scalar_stmt)) != first_load)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "Build SLP failed: unsupported data "
|
fprintf (vect_dump, "Build SLP failed: unsupported data "
|
"permutation ");
|
"permutation ");
|
print_gimple_stmt (vect_dump, scalar_stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, scalar_stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
free (tmp_loads);
|
free (tmp_loads);
|
return false;
|
return false;
|
}
|
}
|
|
|
tmp_loads[index] = load;
|
tmp_loads[index] = load;
|
}
|
}
|
|
|
sorted_loads = VEC_alloc (slp_tree, heap, group_size);
|
sorted_loads = VEC_alloc (slp_tree, heap, group_size);
|
for (i = 0; i < group_size; i++)
|
for (i = 0; i < group_size; i++)
|
VEC_safe_push (slp_tree, heap, sorted_loads, tmp_loads[i]);
|
VEC_safe_push (slp_tree, heap, sorted_loads, tmp_loads[i]);
|
|
|
VEC_free (slp_tree, heap, SLP_INSTANCE_LOADS (instance));
|
VEC_free (slp_tree, heap, SLP_INSTANCE_LOADS (instance));
|
SLP_INSTANCE_LOADS (instance) = sorted_loads;
|
SLP_INSTANCE_LOADS (instance) = sorted_loads;
|
free (tmp_loads);
|
free (tmp_loads);
|
|
|
if (!vect_transform_slp_perm_load (stmt, NULL, NULL,
|
if (!vect_transform_slp_perm_load (stmt, NULL, NULL,
|
SLP_INSTANCE_UNROLLING_FACTOR (instance),
|
SLP_INSTANCE_UNROLLING_FACTOR (instance),
|
instance, true))
|
instance, true))
|
return false;
|
return false;
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
|
|
/* Check if the required load permutation is supported.
|
/* Check if the required load permutation is supported.
|
LOAD_PERMUTATION contains a list of indices of the loads.
|
LOAD_PERMUTATION contains a list of indices of the loads.
|
In SLP this permutation is relative to the order of strided stores that are
|
In SLP this permutation is relative to the order of strided stores that are
|
the base of the SLP instance. */
|
the base of the SLP instance. */
|
|
|
static bool
|
static bool
|
vect_supported_load_permutation_p (slp_instance slp_instn, int group_size,
|
vect_supported_load_permutation_p (slp_instance slp_instn, int group_size,
|
VEC (int, heap) *load_permutation)
|
VEC (int, heap) *load_permutation)
|
{
|
{
|
int i = 0, j, prev = -1, next, k;
|
int i = 0, j, prev = -1, next, k;
|
bool supported;
|
bool supported;
|
sbitmap load_index;
|
sbitmap load_index;
|
|
|
/* FORNOW: permutations are only supported in SLP. */
|
/* FORNOW: permutations are only supported in SLP. */
|
if (!slp_instn)
|
if (!slp_instn)
|
return false;
|
return false;
|
|
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump, "Load permutation ");
|
fprintf (vect_dump, "Load permutation ");
|
for (i = 0; VEC_iterate (int, load_permutation, i, next); i++)
|
for (i = 0; VEC_iterate (int, load_permutation, i, next); i++)
|
fprintf (vect_dump, "%d ", next);
|
fprintf (vect_dump, "%d ", next);
|
}
|
}
|
|
|
/* FORNOW: the only supported permutation is 0..01..1.. of length equal to
|
/* FORNOW: the only supported permutation is 0..01..1.. of length equal to
|
GROUP_SIZE and where each sequence of same drs is of GROUP_SIZE length as
|
GROUP_SIZE and where each sequence of same drs is of GROUP_SIZE length as
|
well. */
|
well. */
|
if (VEC_length (int, load_permutation)
|
if (VEC_length (int, load_permutation)
|
!= (unsigned int) (group_size * group_size))
|
!= (unsigned int) (group_size * group_size))
|
return false;
|
return false;
|
|
|
supported = true;
|
supported = true;
|
load_index = sbitmap_alloc (group_size);
|
load_index = sbitmap_alloc (group_size);
|
sbitmap_zero (load_index);
|
sbitmap_zero (load_index);
|
for (j = 0; j < group_size; j++)
|
for (j = 0; j < group_size; j++)
|
{
|
{
|
for (i = j * group_size, k = 0;
|
for (i = j * group_size, k = 0;
|
VEC_iterate (int, load_permutation, i, next) && k < group_size;
|
VEC_iterate (int, load_permutation, i, next) && k < group_size;
|
i++, k++)
|
i++, k++)
|
{
|
{
|
if (i != j * group_size && next != prev)
|
if (i != j * group_size && next != prev)
|
{
|
{
|
supported = false;
|
supported = false;
|
break;
|
break;
|
}
|
}
|
|
|
prev = next;
|
prev = next;
|
}
|
}
|
|
|
if (TEST_BIT (load_index, prev))
|
if (TEST_BIT (load_index, prev))
|
{
|
{
|
supported = false;
|
supported = false;
|
break;
|
break;
|
}
|
}
|
|
|
SET_BIT (load_index, prev);
|
SET_BIT (load_index, prev);
|
}
|
}
|
|
|
for (j = 0; j < group_size; j++)
|
for (j = 0; j < group_size; j++)
|
if (!TEST_BIT (load_index, j))
|
if (!TEST_BIT (load_index, j))
|
return false;
|
return false;
|
|
|
sbitmap_free (load_index);
|
sbitmap_free (load_index);
|
|
|
if (supported && i == group_size * group_size
|
if (supported && i == group_size * group_size
|
&& vect_supported_slp_permutation_p (slp_instn))
|
&& vect_supported_slp_permutation_p (slp_instn))
|
return true;
|
return true;
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
|
|
/* Find the first load in the loop that belongs to INSTANCE.
|
/* Find the first load in the loop that belongs to INSTANCE.
|
When loads are in several SLP nodes, there can be a case in which the first
|
When loads are in several SLP nodes, there can be a case in which the first
|
load does not appear in the first SLP node to be transformed, causing
|
load does not appear in the first SLP node to be transformed, causing
|
incorrect order of statements. Since we generate all the loads together,
|
incorrect order of statements. Since we generate all the loads together,
|
they must be inserted before the first load of the SLP instance and not
|
they must be inserted before the first load of the SLP instance and not
|
before the first load of the first node of the instance. */
|
before the first load of the first node of the instance. */
|
static gimple
|
static gimple
|
vect_find_first_load_in_slp_instance (slp_instance instance)
|
vect_find_first_load_in_slp_instance (slp_instance instance)
|
{
|
{
|
int i, j;
|
int i, j;
|
slp_tree load_node;
|
slp_tree load_node;
|
gimple first_load = NULL, load;
|
gimple first_load = NULL, load;
|
|
|
for (i = 0;
|
for (i = 0;
|
VEC_iterate (slp_tree, SLP_INSTANCE_LOADS (instance), i, load_node);
|
VEC_iterate (slp_tree, SLP_INSTANCE_LOADS (instance), i, load_node);
|
i++)
|
i++)
|
for (j = 0;
|
for (j = 0;
|
VEC_iterate (gimple, SLP_TREE_SCALAR_STMTS (load_node), j, load);
|
VEC_iterate (gimple, SLP_TREE_SCALAR_STMTS (load_node), j, load);
|
j++)
|
j++)
|
first_load = get_earlier_stmt (load, first_load);
|
first_load = get_earlier_stmt (load, first_load);
|
|
|
return first_load;
|
return first_load;
|
}
|
}
|
|
|
|
|
/* Analyze an SLP instance starting from a group of strided stores. Call
|
/* Analyze an SLP instance starting from a group of strided stores. Call
|
vect_build_slp_tree to build a tree of packed stmts if possible.
|
vect_build_slp_tree to build a tree of packed stmts if possible.
|
Return FALSE if it's impossible to SLP any stmt in the loop. */
|
Return FALSE if it's impossible to SLP any stmt in the loop. */
|
|
|
static bool
|
static bool
|
vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
|
vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo,
|
gimple stmt)
|
gimple stmt)
|
{
|
{
|
slp_instance new_instance;
|
slp_instance new_instance;
|
slp_tree node = XNEW (struct _slp_tree);
|
slp_tree node = XNEW (struct _slp_tree);
|
unsigned int group_size = DR_GROUP_SIZE (vinfo_for_stmt (stmt));
|
unsigned int group_size = DR_GROUP_SIZE (vinfo_for_stmt (stmt));
|
unsigned int unrolling_factor = 1, nunits;
|
unsigned int unrolling_factor = 1, nunits;
|
tree vectype, scalar_type;
|
tree vectype, scalar_type;
|
gimple next;
|
gimple next;
|
unsigned int vectorization_factor = 0;
|
unsigned int vectorization_factor = 0;
|
int inside_cost = 0, outside_cost = 0, ncopies_for_cost;
|
int inside_cost = 0, outside_cost = 0, ncopies_for_cost;
|
unsigned int max_nunits = 0;
|
unsigned int max_nunits = 0;
|
VEC (int, heap) *load_permutation;
|
VEC (int, heap) *load_permutation;
|
VEC (slp_tree, heap) *loads;
|
VEC (slp_tree, heap) *loads;
|
|
|
scalar_type = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (
|
scalar_type = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (
|
vinfo_for_stmt (stmt))));
|
vinfo_for_stmt (stmt))));
|
vectype = get_vectype_for_scalar_type (scalar_type);
|
vectype = get_vectype_for_scalar_type (scalar_type);
|
if (!vectype)
|
if (!vectype)
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump, "Build SLP failed: unsupported data-type ");
|
fprintf (vect_dump, "Build SLP failed: unsupported data-type ");
|
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
|
print_generic_expr (vect_dump, scalar_type, TDF_SLIM);
|
}
|
}
|
return false;
|
return false;
|
}
|
}
|
|
|
nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
if (loop_vinfo)
|
if (loop_vinfo)
|
vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
else
|
else
|
/* No multitypes in BB SLP. */
|
/* No multitypes in BB SLP. */
|
vectorization_factor = nunits;
|
vectorization_factor = nunits;
|
|
|
/* Calculate the unrolling factor. */
|
/* Calculate the unrolling factor. */
|
unrolling_factor = least_common_multiple (nunits, group_size) / group_size;
|
unrolling_factor = least_common_multiple (nunits, group_size) / group_size;
|
if (unrolling_factor != 1 && !loop_vinfo)
|
if (unrolling_factor != 1 && !loop_vinfo)
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
fprintf (vect_dump, "Build SLP failed: unrolling required in basic"
|
fprintf (vect_dump, "Build SLP failed: unrolling required in basic"
|
" block SLP");
|
" block SLP");
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
/* Create a node (a root of the SLP tree) for the packed strided stores. */
|
/* Create a node (a root of the SLP tree) for the packed strided stores. */
|
SLP_TREE_SCALAR_STMTS (node) = VEC_alloc (gimple, heap, group_size);
|
SLP_TREE_SCALAR_STMTS (node) = VEC_alloc (gimple, heap, group_size);
|
next = stmt;
|
next = stmt;
|
/* Collect the stores and store them in SLP_TREE_SCALAR_STMTS. */
|
/* Collect the stores and store them in SLP_TREE_SCALAR_STMTS. */
|
while (next)
|
while (next)
|
{
|
{
|
VEC_safe_push (gimple, heap, SLP_TREE_SCALAR_STMTS (node), next);
|
VEC_safe_push (gimple, heap, SLP_TREE_SCALAR_STMTS (node), next);
|
next = DR_GROUP_NEXT_DR (vinfo_for_stmt (next));
|
next = DR_GROUP_NEXT_DR (vinfo_for_stmt (next));
|
}
|
}
|
|
|
SLP_TREE_VEC_STMTS (node) = NULL;
|
SLP_TREE_VEC_STMTS (node) = NULL;
|
SLP_TREE_NUMBER_OF_VEC_STMTS (node) = 0;
|
SLP_TREE_NUMBER_OF_VEC_STMTS (node) = 0;
|
SLP_TREE_LEFT (node) = NULL;
|
SLP_TREE_LEFT (node) = NULL;
|
SLP_TREE_RIGHT (node) = NULL;
|
SLP_TREE_RIGHT (node) = NULL;
|
SLP_TREE_OUTSIDE_OF_LOOP_COST (node) = 0;
|
SLP_TREE_OUTSIDE_OF_LOOP_COST (node) = 0;
|
SLP_TREE_INSIDE_OF_LOOP_COST (node) = 0;
|
SLP_TREE_INSIDE_OF_LOOP_COST (node) = 0;
|
|
|
/* Calculate the number of vector stmts to create based on the unrolling
|
/* Calculate the number of vector stmts to create based on the unrolling
|
factor (number of vectors is 1 if NUNITS >= GROUP_SIZE, and is
|
factor (number of vectors is 1 if NUNITS >= GROUP_SIZE, and is
|
GROUP_SIZE / NUNITS otherwise. */
|
GROUP_SIZE / NUNITS otherwise. */
|
ncopies_for_cost = unrolling_factor * group_size / nunits;
|
ncopies_for_cost = unrolling_factor * group_size / nunits;
|
|
|
load_permutation = VEC_alloc (int, heap, group_size * group_size);
|
load_permutation = VEC_alloc (int, heap, group_size * group_size);
|
loads = VEC_alloc (slp_tree, heap, group_size);
|
loads = VEC_alloc (slp_tree, heap, group_size);
|
|
|
/* Build the tree for the SLP instance. */
|
/* Build the tree for the SLP instance. */
|
if (vect_build_slp_tree (loop_vinfo, bb_vinfo, &node, group_size,
|
if (vect_build_slp_tree (loop_vinfo, bb_vinfo, &node, group_size,
|
&inside_cost, &outside_cost, ncopies_for_cost,
|
&inside_cost, &outside_cost, ncopies_for_cost,
|
&max_nunits, &load_permutation, &loads,
|
&max_nunits, &load_permutation, &loads,
|
vectorization_factor))
|
vectorization_factor))
|
{
|
{
|
/* Create a new SLP instance. */
|
/* Create a new SLP instance. */
|
new_instance = XNEW (struct _slp_instance);
|
new_instance = XNEW (struct _slp_instance);
|
SLP_INSTANCE_TREE (new_instance) = node;
|
SLP_INSTANCE_TREE (new_instance) = node;
|
SLP_INSTANCE_GROUP_SIZE (new_instance) = group_size;
|
SLP_INSTANCE_GROUP_SIZE (new_instance) = group_size;
|
/* Calculate the unrolling factor based on the smallest type in the
|
/* Calculate the unrolling factor based on the smallest type in the
|
loop. */
|
loop. */
|
if (max_nunits > nunits)
|
if (max_nunits > nunits)
|
unrolling_factor = least_common_multiple (max_nunits, group_size)
|
unrolling_factor = least_common_multiple (max_nunits, group_size)
|
/ group_size;
|
/ group_size;
|
|
|
SLP_INSTANCE_UNROLLING_FACTOR (new_instance) = unrolling_factor;
|
SLP_INSTANCE_UNROLLING_FACTOR (new_instance) = unrolling_factor;
|
SLP_INSTANCE_OUTSIDE_OF_LOOP_COST (new_instance) = outside_cost;
|
SLP_INSTANCE_OUTSIDE_OF_LOOP_COST (new_instance) = outside_cost;
|
SLP_INSTANCE_INSIDE_OF_LOOP_COST (new_instance) = inside_cost;
|
SLP_INSTANCE_INSIDE_OF_LOOP_COST (new_instance) = inside_cost;
|
SLP_INSTANCE_LOADS (new_instance) = loads;
|
SLP_INSTANCE_LOADS (new_instance) = loads;
|
SLP_INSTANCE_FIRST_LOAD_STMT (new_instance) = NULL;
|
SLP_INSTANCE_FIRST_LOAD_STMT (new_instance) = NULL;
|
SLP_INSTANCE_LOAD_PERMUTATION (new_instance) = load_permutation;
|
SLP_INSTANCE_LOAD_PERMUTATION (new_instance) = load_permutation;
|
if (VEC_length (slp_tree, loads))
|
if (VEC_length (slp_tree, loads))
|
{
|
{
|
if (!vect_supported_load_permutation_p (new_instance, group_size,
|
if (!vect_supported_load_permutation_p (new_instance, group_size,
|
load_permutation))
|
load_permutation))
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
{
|
{
|
fprintf (vect_dump, "Build SLP failed: unsupported load "
|
fprintf (vect_dump, "Build SLP failed: unsupported load "
|
"permutation ");
|
"permutation ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
vect_free_slp_instance (new_instance);
|
vect_free_slp_instance (new_instance);
|
return false;
|
return false;
|
}
|
}
|
|
|
SLP_INSTANCE_FIRST_LOAD_STMT (new_instance)
|
SLP_INSTANCE_FIRST_LOAD_STMT (new_instance)
|
= vect_find_first_load_in_slp_instance (new_instance);
|
= vect_find_first_load_in_slp_instance (new_instance);
|
}
|
}
|
else
|
else
|
VEC_free (int, heap, SLP_INSTANCE_LOAD_PERMUTATION (new_instance));
|
VEC_free (int, heap, SLP_INSTANCE_LOAD_PERMUTATION (new_instance));
|
|
|
if (loop_vinfo)
|
if (loop_vinfo)
|
VEC_safe_push (slp_instance, heap,
|
VEC_safe_push (slp_instance, heap,
|
LOOP_VINFO_SLP_INSTANCES (loop_vinfo),
|
LOOP_VINFO_SLP_INSTANCES (loop_vinfo),
|
new_instance);
|
new_instance);
|
else
|
else
|
VEC_safe_push (slp_instance, heap, BB_VINFO_SLP_INSTANCES (bb_vinfo),
|
VEC_safe_push (slp_instance, heap, BB_VINFO_SLP_INSTANCES (bb_vinfo),
|
new_instance);
|
new_instance);
|
|
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
vect_print_slp_tree (node);
|
vect_print_slp_tree (node);
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
/* Failed to SLP. */
|
/* Failed to SLP. */
|
/* Free the allocated memory. */
|
/* Free the allocated memory. */
|
vect_free_slp_tree (node);
|
vect_free_slp_tree (node);
|
VEC_free (int, heap, load_permutation);
|
VEC_free (int, heap, load_permutation);
|
VEC_free (slp_tree, heap, loads);
|
VEC_free (slp_tree, heap, loads);
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
|
|
/* Check if there are stmts in the loop can be vectorized using SLP. Build SLP
|
/* Check if there are stmts in the loop can be vectorized using SLP. Build SLP
|
trees of packed scalar stmts if SLP is possible. */
|
trees of packed scalar stmts if SLP is possible. */
|
|
|
bool
|
bool
|
vect_analyze_slp (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
|
vect_analyze_slp (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
|
{
|
{
|
unsigned int i;
|
unsigned int i;
|
VEC (gimple, heap) *strided_stores;
|
VEC (gimple, heap) *strided_stores;
|
gimple store;
|
gimple store;
|
bool ok = false;
|
bool ok = false;
|
|
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
fprintf (vect_dump, "=== vect_analyze_slp ===");
|
fprintf (vect_dump, "=== vect_analyze_slp ===");
|
|
|
if (loop_vinfo)
|
if (loop_vinfo)
|
strided_stores = LOOP_VINFO_STRIDED_STORES (loop_vinfo);
|
strided_stores = LOOP_VINFO_STRIDED_STORES (loop_vinfo);
|
else
|
else
|
strided_stores = BB_VINFO_STRIDED_STORES (bb_vinfo);
|
strided_stores = BB_VINFO_STRIDED_STORES (bb_vinfo);
|
|
|
for (i = 0; VEC_iterate (gimple, strided_stores, i, store); i++)
|
for (i = 0; VEC_iterate (gimple, strided_stores, i, store); i++)
|
if (vect_analyze_slp_instance (loop_vinfo, bb_vinfo, store))
|
if (vect_analyze_slp_instance (loop_vinfo, bb_vinfo, store))
|
ok = true;
|
ok = true;
|
|
|
if (bb_vinfo && !ok)
|
if (bb_vinfo && !ok)
|
{
|
{
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
fprintf (vect_dump, "Failed to SLP the basic block.");
|
fprintf (vect_dump, "Failed to SLP the basic block.");
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
|
|
/* For each possible SLP instance decide whether to SLP it and calculate overall
|
/* For each possible SLP instance decide whether to SLP it and calculate overall
|
unrolling factor needed to SLP the loop. */
|
unrolling factor needed to SLP the loop. */
|
|
|
void
|
void
|
vect_make_slp_decision (loop_vec_info loop_vinfo)
|
vect_make_slp_decision (loop_vec_info loop_vinfo)
|
{
|
{
|
unsigned int i, unrolling_factor = 1;
|
unsigned int i, unrolling_factor = 1;
|
VEC (slp_instance, heap) *slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
|
VEC (slp_instance, heap) *slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
|
slp_instance instance;
|
slp_instance instance;
|
int decided_to_slp = 0;
|
int decided_to_slp = 0;
|
|
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
fprintf (vect_dump, "=== vect_make_slp_decision ===");
|
fprintf (vect_dump, "=== vect_make_slp_decision ===");
|
|
|
for (i = 0; VEC_iterate (slp_instance, slp_instances, i, instance); i++)
|
for (i = 0; VEC_iterate (slp_instance, slp_instances, i, instance); i++)
|
{
|
{
|
/* FORNOW: SLP if you can. */
|
/* FORNOW: SLP if you can. */
|
if (unrolling_factor < SLP_INSTANCE_UNROLLING_FACTOR (instance))
|
if (unrolling_factor < SLP_INSTANCE_UNROLLING_FACTOR (instance))
|
unrolling_factor = SLP_INSTANCE_UNROLLING_FACTOR (instance);
|
unrolling_factor = SLP_INSTANCE_UNROLLING_FACTOR (instance);
|
|
|
/* Mark all the stmts that belong to INSTANCE as PURE_SLP stmts. Later we
|
/* Mark all the stmts that belong to INSTANCE as PURE_SLP stmts. Later we
|
call vect_detect_hybrid_slp () to find stmts that need hybrid SLP and
|
call vect_detect_hybrid_slp () to find stmts that need hybrid SLP and
|
loop-based vectorization. Such stmts will be marked as HYBRID. */
|
loop-based vectorization. Such stmts will be marked as HYBRID. */
|
vect_mark_slp_stmts (SLP_INSTANCE_TREE (instance), pure_slp, -1);
|
vect_mark_slp_stmts (SLP_INSTANCE_TREE (instance), pure_slp, -1);
|
decided_to_slp++;
|
decided_to_slp++;
|
}
|
}
|
|
|
LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo) = unrolling_factor;
|
LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo) = unrolling_factor;
|
|
|
if (decided_to_slp && vect_print_dump_info (REPORT_SLP))
|
if (decided_to_slp && vect_print_dump_info (REPORT_SLP))
|
fprintf (vect_dump, "Decided to SLP %d instances. Unrolling factor %d",
|
fprintf (vect_dump, "Decided to SLP %d instances. Unrolling factor %d",
|
decided_to_slp, unrolling_factor);
|
decided_to_slp, unrolling_factor);
|
}
|
}
|
|
|
|
|
/* Find stmts that must be both vectorized and SLPed (since they feed stmts that
|
/* Find stmts that must be both vectorized and SLPed (since they feed stmts that
|
can't be SLPed) in the tree rooted at NODE. Mark such stmts as HYBRID. */
|
can't be SLPed) in the tree rooted at NODE. Mark such stmts as HYBRID. */
|
|
|
static void
|
static void
|
vect_detect_hybrid_slp_stmts (slp_tree node)
|
vect_detect_hybrid_slp_stmts (slp_tree node)
|
{
|
{
|
int i;
|
int i;
|
gimple stmt;
|
gimple stmt;
|
imm_use_iterator imm_iter;
|
imm_use_iterator imm_iter;
|
gimple use_stmt;
|
gimple use_stmt;
|
stmt_vec_info stmt_vinfo;
|
stmt_vec_info stmt_vinfo;
|
|
|
if (!node)
|
if (!node)
|
return;
|
return;
|
|
|
for (i = 0; VEC_iterate (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt); i++)
|
for (i = 0; VEC_iterate (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt); i++)
|
if (PURE_SLP_STMT (vinfo_for_stmt (stmt))
|
if (PURE_SLP_STMT (vinfo_for_stmt (stmt))
|
&& TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME)
|
&& TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME)
|
FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, gimple_op (stmt, 0))
|
FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, gimple_op (stmt, 0))
|
if ((stmt_vinfo = vinfo_for_stmt (use_stmt))
|
if ((stmt_vinfo = vinfo_for_stmt (use_stmt))
|
&& !STMT_SLP_TYPE (stmt_vinfo)
|
&& !STMT_SLP_TYPE (stmt_vinfo)
|
&& (STMT_VINFO_RELEVANT (stmt_vinfo)
|
&& (STMT_VINFO_RELEVANT (stmt_vinfo)
|
|| VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_vinfo))))
|
|| VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_vinfo))))
|
vect_mark_slp_stmts (node, hybrid, i);
|
vect_mark_slp_stmts (node, hybrid, i);
|
|
|
vect_detect_hybrid_slp_stmts (SLP_TREE_LEFT (node));
|
vect_detect_hybrid_slp_stmts (SLP_TREE_LEFT (node));
|
vect_detect_hybrid_slp_stmts (SLP_TREE_RIGHT (node));
|
vect_detect_hybrid_slp_stmts (SLP_TREE_RIGHT (node));
|
}
|
}
|
|
|
|
|
/* Find stmts that must be both vectorized and SLPed. */
|
/* Find stmts that must be both vectorized and SLPed. */
|
|
|
void
|
void
|
vect_detect_hybrid_slp (loop_vec_info loop_vinfo)
|
vect_detect_hybrid_slp (loop_vec_info loop_vinfo)
|
{
|
{
|
unsigned int i;
|
unsigned int i;
|
VEC (slp_instance, heap) *slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
|
VEC (slp_instance, heap) *slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
|
slp_instance instance;
|
slp_instance instance;
|
|
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
fprintf (vect_dump, "=== vect_detect_hybrid_slp ===");
|
fprintf (vect_dump, "=== vect_detect_hybrid_slp ===");
|
|
|
for (i = 0; VEC_iterate (slp_instance, slp_instances, i, instance); i++)
|
for (i = 0; VEC_iterate (slp_instance, slp_instances, i, instance); i++)
|
vect_detect_hybrid_slp_stmts (SLP_INSTANCE_TREE (instance));
|
vect_detect_hybrid_slp_stmts (SLP_INSTANCE_TREE (instance));
|
}
|
}
|
|
|
|
|
/* Create and initialize a new bb_vec_info struct for BB, as well as
|
/* Create and initialize a new bb_vec_info struct for BB, as well as
|
stmt_vec_info structs for all the stmts in it. */
|
stmt_vec_info structs for all the stmts in it. */
|
|
|
static bb_vec_info
|
static bb_vec_info
|
new_bb_vec_info (basic_block bb)
|
new_bb_vec_info (basic_block bb)
|
{
|
{
|
bb_vec_info res = NULL;
|
bb_vec_info res = NULL;
|
gimple_stmt_iterator gsi;
|
gimple_stmt_iterator gsi;
|
|
|
res = (bb_vec_info) xcalloc (1, sizeof (struct _bb_vec_info));
|
res = (bb_vec_info) xcalloc (1, sizeof (struct _bb_vec_info));
|
BB_VINFO_BB (res) = bb;
|
BB_VINFO_BB (res) = bb;
|
|
|
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
{
|
{
|
gimple stmt = gsi_stmt (gsi);
|
gimple stmt = gsi_stmt (gsi);
|
gimple_set_uid (stmt, 0);
|
gimple_set_uid (stmt, 0);
|
set_vinfo_for_stmt (stmt, new_stmt_vec_info (stmt, NULL, res));
|
set_vinfo_for_stmt (stmt, new_stmt_vec_info (stmt, NULL, res));
|
}
|
}
|
|
|
BB_VINFO_STRIDED_STORES (res) = VEC_alloc (gimple, heap, 10);
|
BB_VINFO_STRIDED_STORES (res) = VEC_alloc (gimple, heap, 10);
|
BB_VINFO_SLP_INSTANCES (res) = VEC_alloc (slp_instance, heap, 2);
|
BB_VINFO_SLP_INSTANCES (res) = VEC_alloc (slp_instance, heap, 2);
|
|
|
bb->aux = res;
|
bb->aux = res;
|
return res;
|
return res;
|
}
|
}
|
|
|
|
|
/* Free BB_VINFO struct, as well as all the stmt_vec_info structs of all the
|
/* Free BB_VINFO struct, as well as all the stmt_vec_info structs of all the
|
stmts in the basic block. */
|
stmts in the basic block. */
|
|
|
static void
|
static void
|
destroy_bb_vec_info (bb_vec_info bb_vinfo)
|
destroy_bb_vec_info (bb_vec_info bb_vinfo)
|
{
|
{
|
basic_block bb;
|
basic_block bb;
|
gimple_stmt_iterator si;
|
gimple_stmt_iterator si;
|
|
|
if (!bb_vinfo)
|
if (!bb_vinfo)
|
return;
|
return;
|
|
|
bb = BB_VINFO_BB (bb_vinfo);
|
bb = BB_VINFO_BB (bb_vinfo);
|
|
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
{
|
{
|
gimple stmt = gsi_stmt (si);
|
gimple stmt = gsi_stmt (si);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
|
|
if (stmt_info)
|
if (stmt_info)
|
/* Free stmt_vec_info. */
|
/* Free stmt_vec_info. */
|
free_stmt_vec_info (stmt);
|
free_stmt_vec_info (stmt);
|
}
|
}
|
|
|
VEC_free (gimple, heap, BB_VINFO_STRIDED_STORES (bb_vinfo));
|
VEC_free (gimple, heap, BB_VINFO_STRIDED_STORES (bb_vinfo));
|
VEC_free (slp_instance, heap, BB_VINFO_SLP_INSTANCES (bb_vinfo));
|
VEC_free (slp_instance, heap, BB_VINFO_SLP_INSTANCES (bb_vinfo));
|
free (bb_vinfo);
|
free (bb_vinfo);
|
bb->aux = NULL;
|
bb->aux = NULL;
|
}
|
}
|
|
|
|
|
/* Analyze statements contained in SLP tree node after recursively analyzing
|
/* Analyze statements contained in SLP tree node after recursively analyzing
|
the subtree. Return TRUE if the operations are supported. */
|
the subtree. Return TRUE if the operations are supported. */
|
|
|
static bool
|
static bool
|
vect_slp_analyze_node_operations (bb_vec_info bb_vinfo, slp_tree node)
|
vect_slp_analyze_node_operations (bb_vec_info bb_vinfo, slp_tree node)
|
{
|
{
|
bool dummy;
|
bool dummy;
|
int i;
|
int i;
|
gimple stmt;
|
gimple stmt;
|
|
|
if (!node)
|
if (!node)
|
return true;
|
return true;
|
|
|
if (!vect_slp_analyze_node_operations (bb_vinfo, SLP_TREE_LEFT (node))
|
if (!vect_slp_analyze_node_operations (bb_vinfo, SLP_TREE_LEFT (node))
|
|| !vect_slp_analyze_node_operations (bb_vinfo, SLP_TREE_RIGHT (node)))
|
|| !vect_slp_analyze_node_operations (bb_vinfo, SLP_TREE_RIGHT (node)))
|
return false;
|
return false;
|
|
|
for (i = 0; VEC_iterate (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt); i++)
|
for (i = 0; VEC_iterate (gimple, SLP_TREE_SCALAR_STMTS (node), i, stmt); i++)
|
{
|
{
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
gcc_assert (stmt_info);
|
gcc_assert (stmt_info);
|
gcc_assert (PURE_SLP_STMT (stmt_info));
|
gcc_assert (PURE_SLP_STMT (stmt_info));
|
|
|
if (!vect_analyze_stmt (stmt, &dummy, node))
|
if (!vect_analyze_stmt (stmt, &dummy, node))
|
return false;
|
return false;
|
}
|
}
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
|
|
/* Analyze statements in SLP instances of the basic block. Return TRUE if the
|
/* Analyze statements in SLP instances of the basic block. Return TRUE if the
|
operations are supported. */
|
operations are supported. */
|
|
|
static bool
|
static bool
|
vect_slp_analyze_operations (bb_vec_info bb_vinfo)
|
vect_slp_analyze_operations (bb_vec_info bb_vinfo)
|
{
|
{
|
VEC (slp_instance, heap) *slp_instances = BB_VINFO_SLP_INSTANCES (bb_vinfo);
|
VEC (slp_instance, heap) *slp_instances = BB_VINFO_SLP_INSTANCES (bb_vinfo);
|
slp_instance instance;
|
slp_instance instance;
|
int i;
|
int i;
|
|
|
for (i = 0; VEC_iterate (slp_instance, slp_instances, i, instance); )
|
for (i = 0; VEC_iterate (slp_instance, slp_instances, i, instance); )
|
{
|
{
|
if (!vect_slp_analyze_node_operations (bb_vinfo,
|
if (!vect_slp_analyze_node_operations (bb_vinfo,
|
SLP_INSTANCE_TREE (instance)))
|
SLP_INSTANCE_TREE (instance)))
|
{
|
{
|
vect_free_slp_instance (instance);
|
vect_free_slp_instance (instance);
|
VEC_ordered_remove (slp_instance, slp_instances, i);
|
VEC_ordered_remove (slp_instance, slp_instances, i);
|
}
|
}
|
else
|
else
|
i++;
|
i++;
|
}
|
}
|
|
|
if (!VEC_length (slp_instance, slp_instances))
|
if (!VEC_length (slp_instance, slp_instances))
|
return false;
|
return false;
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
|
|
/* Cheick if the basic block can be vectorized. */
|
/* Cheick if the basic block can be vectorized. */
|
|
|
bb_vec_info
|
bb_vec_info
|
vect_slp_analyze_bb (basic_block bb)
|
vect_slp_analyze_bb (basic_block bb)
|
{
|
{
|
bb_vec_info bb_vinfo;
|
bb_vec_info bb_vinfo;
|
VEC (ddr_p, heap) *ddrs;
|
VEC (ddr_p, heap) *ddrs;
|
VEC (slp_instance, heap) *slp_instances;
|
VEC (slp_instance, heap) *slp_instances;
|
slp_instance instance;
|
slp_instance instance;
|
int i, insns = 0;
|
int i, insns = 0;
|
gimple_stmt_iterator gsi;
|
gimple_stmt_iterator gsi;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "===vect_slp_analyze_bb===\n");
|
fprintf (vect_dump, "===vect_slp_analyze_bb===\n");
|
|
|
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
{
|
{
|
gimple stmt = gsi_stmt (gsi);
|
gimple stmt = gsi_stmt (gsi);
|
if (!is_gimple_debug (stmt)
|
if (!is_gimple_debug (stmt)
|
&& !gimple_nop_p (stmt)
|
&& !gimple_nop_p (stmt)
|
&& gimple_code (stmt) != GIMPLE_LABEL)
|
&& gimple_code (stmt) != GIMPLE_LABEL)
|
insns++;
|
insns++;
|
}
|
}
|
|
|
if (insns > PARAM_VALUE (PARAM_SLP_MAX_INSNS_IN_BB))
|
if (insns > PARAM_VALUE (PARAM_SLP_MAX_INSNS_IN_BB))
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: too many instructions in basic "
|
fprintf (vect_dump, "not vectorized: too many instructions in basic "
|
"block.\n");
|
"block.\n");
|
|
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
bb_vinfo = new_bb_vec_info (bb);
|
bb_vinfo = new_bb_vec_info (bb);
|
if (!bb_vinfo)
|
if (!bb_vinfo)
|
return NULL;
|
return NULL;
|
|
|
if (!vect_analyze_data_refs (NULL, bb_vinfo))
|
if (!vect_analyze_data_refs (NULL, bb_vinfo))
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: unhandled data-ref in basic "
|
fprintf (vect_dump, "not vectorized: unhandled data-ref in basic "
|
"block.\n");
|
"block.\n");
|
|
|
destroy_bb_vec_info (bb_vinfo);
|
destroy_bb_vec_info (bb_vinfo);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
ddrs = BB_VINFO_DDRS (bb_vinfo);
|
ddrs = BB_VINFO_DDRS (bb_vinfo);
|
if (!VEC_length (ddr_p, ddrs))
|
if (!VEC_length (ddr_p, ddrs))
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: not enough data-refs in basic "
|
fprintf (vect_dump, "not vectorized: not enough data-refs in basic "
|
"block.\n");
|
"block.\n");
|
|
|
destroy_bb_vec_info (bb_vinfo);
|
destroy_bb_vec_info (bb_vinfo);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (!vect_analyze_data_refs_alignment (NULL, bb_vinfo))
|
if (!vect_analyze_data_refs_alignment (NULL, bb_vinfo))
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: bad data alignment in basic "
|
fprintf (vect_dump, "not vectorized: bad data alignment in basic "
|
"block.\n");
|
"block.\n");
|
|
|
destroy_bb_vec_info (bb_vinfo);
|
destroy_bb_vec_info (bb_vinfo);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (!vect_analyze_data_ref_dependences (NULL, bb_vinfo))
|
if (!vect_analyze_data_ref_dependences (NULL, bb_vinfo))
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: unhandled data dependence in basic"
|
fprintf (vect_dump, "not vectorized: unhandled data dependence in basic"
|
" block.\n");
|
" block.\n");
|
|
|
destroy_bb_vec_info (bb_vinfo);
|
destroy_bb_vec_info (bb_vinfo);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (!vect_analyze_data_ref_accesses (NULL, bb_vinfo))
|
if (!vect_analyze_data_ref_accesses (NULL, bb_vinfo))
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: unhandled data access in basic "
|
fprintf (vect_dump, "not vectorized: unhandled data access in basic "
|
"block.\n");
|
"block.\n");
|
|
|
destroy_bb_vec_info (bb_vinfo);
|
destroy_bb_vec_info (bb_vinfo);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (!vect_verify_datarefs_alignment (NULL, bb_vinfo))
|
if (!vect_verify_datarefs_alignment (NULL, bb_vinfo))
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: unsupported alignment in basic "
|
fprintf (vect_dump, "not vectorized: unsupported alignment in basic "
|
"block.\n");
|
"block.\n");
|
|
|
destroy_bb_vec_info (bb_vinfo);
|
destroy_bb_vec_info (bb_vinfo);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
/* Check the SLP opportunities in the basic block, analyze and build SLP
|
/* Check the SLP opportunities in the basic block, analyze and build SLP
|
trees. */
|
trees. */
|
if (!vect_analyze_slp (NULL, bb_vinfo))
|
if (!vect_analyze_slp (NULL, bb_vinfo))
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: failed to find SLP opportunities "
|
fprintf (vect_dump, "not vectorized: failed to find SLP opportunities "
|
"in basic block.\n");
|
"in basic block.\n");
|
|
|
destroy_bb_vec_info (bb_vinfo);
|
destroy_bb_vec_info (bb_vinfo);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
slp_instances = BB_VINFO_SLP_INSTANCES (bb_vinfo);
|
slp_instances = BB_VINFO_SLP_INSTANCES (bb_vinfo);
|
|
|
/* Mark all the statements that we want to vectorize as pure SLP and
|
/* Mark all the statements that we want to vectorize as pure SLP and
|
relevant. */
|
relevant. */
|
for (i = 0; VEC_iterate (slp_instance, slp_instances, i, instance); i++)
|
for (i = 0; VEC_iterate (slp_instance, slp_instances, i, instance); i++)
|
{
|
{
|
vect_mark_slp_stmts (SLP_INSTANCE_TREE (instance), pure_slp, -1);
|
vect_mark_slp_stmts (SLP_INSTANCE_TREE (instance), pure_slp, -1);
|
vect_mark_slp_stmts_relevant (SLP_INSTANCE_TREE (instance));
|
vect_mark_slp_stmts_relevant (SLP_INSTANCE_TREE (instance));
|
}
|
}
|
|
|
if (!vect_slp_analyze_operations (bb_vinfo))
|
if (!vect_slp_analyze_operations (bb_vinfo))
|
{
|
{
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "not vectorized: bad operation in basic block.\n");
|
fprintf (vect_dump, "not vectorized: bad operation in basic block.\n");
|
|
|
destroy_bb_vec_info (bb_vinfo);
|
destroy_bb_vec_info (bb_vinfo);
|
return NULL;
|
return NULL;
|
}
|
}
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "Basic block will be vectorized using SLP\n");
|
fprintf (vect_dump, "Basic block will be vectorized using SLP\n");
|
|
|
return bb_vinfo;
|
return bb_vinfo;
|
}
|
}
|
|
|
|
|
/* SLP costs are calculated according to SLP instance unrolling factor (i.e.,
|
/* SLP costs are calculated according to SLP instance unrolling factor (i.e.,
|
the number of created vector stmts depends on the unrolling factor). However,
|
the number of created vector stmts depends on the unrolling factor). However,
|
the actual number of vector stmts for every SLP node depends on VF which is
|
the actual number of vector stmts for every SLP node depends on VF which is
|
set later in vect_analyze_operations(). Hence, SLP costs should be updated.
|
set later in vect_analyze_operations(). Hence, SLP costs should be updated.
|
In this function we assume that the inside costs calculated in
|
In this function we assume that the inside costs calculated in
|
vect_model_xxx_cost are linear in ncopies. */
|
vect_model_xxx_cost are linear in ncopies. */
|
|
|
void
|
void
|
vect_update_slp_costs_according_to_vf (loop_vec_info loop_vinfo)
|
vect_update_slp_costs_according_to_vf (loop_vec_info loop_vinfo)
|
{
|
{
|
unsigned int i, vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
unsigned int i, vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
VEC (slp_instance, heap) *slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
|
VEC (slp_instance, heap) *slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
|
slp_instance instance;
|
slp_instance instance;
|
|
|
if (vect_print_dump_info (REPORT_SLP))
|
if (vect_print_dump_info (REPORT_SLP))
|
fprintf (vect_dump, "=== vect_update_slp_costs_according_to_vf ===");
|
fprintf (vect_dump, "=== vect_update_slp_costs_according_to_vf ===");
|
|
|
for (i = 0; VEC_iterate (slp_instance, slp_instances, i, instance); i++)
|
for (i = 0; VEC_iterate (slp_instance, slp_instances, i, instance); i++)
|
/* We assume that costs are linear in ncopies. */
|
/* We assume that costs are linear in ncopies. */
|
SLP_INSTANCE_INSIDE_OF_LOOP_COST (instance) *= vf
|
SLP_INSTANCE_INSIDE_OF_LOOP_COST (instance) *= vf
|
/ SLP_INSTANCE_UNROLLING_FACTOR (instance);
|
/ SLP_INSTANCE_UNROLLING_FACTOR (instance);
|
}
|
}
|
|
|
|
|
/* For constant and loop invariant defs of SLP_NODE this function returns
|
/* For constant and loop invariant defs of SLP_NODE this function returns
|
(vector) defs (VEC_OPRNDS) that will be used in the vectorized stmts.
|
(vector) defs (VEC_OPRNDS) that will be used in the vectorized stmts.
|
OP_NUM determines if we gather defs for operand 0 or operand 1 of the scalar
|
OP_NUM determines if we gather defs for operand 0 or operand 1 of the scalar
|
stmts. NUMBER_OF_VECTORS is the number of vector defs to create. */
|
stmts. NUMBER_OF_VECTORS is the number of vector defs to create. */
|
|
|
static void
|
static void
|
vect_get_constant_vectors (slp_tree slp_node, VEC(tree,heap) **vec_oprnds,
|
vect_get_constant_vectors (slp_tree slp_node, VEC(tree,heap) **vec_oprnds,
|
unsigned int op_num, unsigned int number_of_vectors)
|
unsigned int op_num, unsigned int number_of_vectors)
|
{
|
{
|
VEC (gimple, heap) *stmts = SLP_TREE_SCALAR_STMTS (slp_node);
|
VEC (gimple, heap) *stmts = SLP_TREE_SCALAR_STMTS (slp_node);
|
gimple stmt = VEC_index (gimple, stmts, 0);
|
gimple stmt = VEC_index (gimple, stmts, 0);
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
|
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
|
int nunits;
|
int nunits;
|
tree vec_cst;
|
tree vec_cst;
|
tree t = NULL_TREE;
|
tree t = NULL_TREE;
|
int j, number_of_places_left_in_vector;
|
int j, number_of_places_left_in_vector;
|
tree vector_type;
|
tree vector_type;
|
tree op, vop;
|
tree op, vop;
|
int group_size = VEC_length (gimple, stmts);
|
int group_size = VEC_length (gimple, stmts);
|
unsigned int vec_num, i;
|
unsigned int vec_num, i;
|
int number_of_copies = 1;
|
int number_of_copies = 1;
|
VEC (tree, heap) *voprnds = VEC_alloc (tree, heap, number_of_vectors);
|
VEC (tree, heap) *voprnds = VEC_alloc (tree, heap, number_of_vectors);
|
bool constant_p, is_store;
|
bool constant_p, is_store;
|
|
|
if (STMT_VINFO_DATA_REF (stmt_vinfo))
|
if (STMT_VINFO_DATA_REF (stmt_vinfo))
|
{
|
{
|
is_store = true;
|
is_store = true;
|
op = gimple_assign_rhs1 (stmt);
|
op = gimple_assign_rhs1 (stmt);
|
}
|
}
|
else
|
else
|
{
|
{
|
is_store = false;
|
is_store = false;
|
op = gimple_op (stmt, op_num + 1);
|
op = gimple_op (stmt, op_num + 1);
|
}
|
}
|
|
|
if (CONSTANT_CLASS_P (op))
|
if (CONSTANT_CLASS_P (op))
|
constant_p = true;
|
constant_p = true;
|
else
|
else
|
constant_p = false;
|
constant_p = false;
|
|
|
vector_type = get_vectype_for_scalar_type (TREE_TYPE (op));
|
vector_type = get_vectype_for_scalar_type (TREE_TYPE (op));
|
gcc_assert (vector_type);
|
gcc_assert (vector_type);
|
|
|
nunits = TYPE_VECTOR_SUBPARTS (vector_type);
|
nunits = TYPE_VECTOR_SUBPARTS (vector_type);
|
|
|
/* NUMBER_OF_COPIES is the number of times we need to use the same values in
|
/* NUMBER_OF_COPIES is the number of times we need to use the same values in
|
created vectors. It is greater than 1 if unrolling is performed.
|
created vectors. It is greater than 1 if unrolling is performed.
|
|
|
For example, we have two scalar operands, s1 and s2 (e.g., group of
|
For example, we have two scalar operands, s1 and s2 (e.g., group of
|
strided accesses of size two), while NUNITS is four (i.e., four scalars
|
strided accesses of size two), while NUNITS is four (i.e., four scalars
|
of this type can be packed in a vector). The output vector will contain
|
of this type can be packed in a vector). The output vector will contain
|
two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
|
two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
|
will be 2).
|
will be 2).
|
|
|
If GROUP_SIZE > NUNITS, the scalars will be split into several vectors
|
If GROUP_SIZE > NUNITS, the scalars will be split into several vectors
|
containing the operands.
|
containing the operands.
|
|
|
For example, NUNITS is four as before, and the group size is 8
|
For example, NUNITS is four as before, and the group size is 8
|
(s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
|
(s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
|
{s5, s6, s7, s8}. */
|
{s5, s6, s7, s8}. */
|
|
|
number_of_copies = least_common_multiple (nunits, group_size) / group_size;
|
number_of_copies = least_common_multiple (nunits, group_size) / group_size;
|
|
|
number_of_places_left_in_vector = nunits;
|
number_of_places_left_in_vector = nunits;
|
for (j = 0; j < number_of_copies; j++)
|
for (j = 0; j < number_of_copies; j++)
|
{
|
{
|
for (i = group_size - 1; VEC_iterate (gimple, stmts, i, stmt); i--)
|
for (i = group_size - 1; VEC_iterate (gimple, stmts, i, stmt); i--)
|
{
|
{
|
if (is_store)
|
if (is_store)
|
op = gimple_assign_rhs1 (stmt);
|
op = gimple_assign_rhs1 (stmt);
|
else
|
else
|
op = gimple_op (stmt, op_num + 1);
|
op = gimple_op (stmt, op_num + 1);
|
|
|
/* Create 'vect_ = {op0,op1,...,opn}'. */
|
/* Create 'vect_ = {op0,op1,...,opn}'. */
|
t = tree_cons (NULL_TREE, op, t);
|
t = tree_cons (NULL_TREE, op, t);
|
|
|
number_of_places_left_in_vector--;
|
number_of_places_left_in_vector--;
|
|
|
if (number_of_places_left_in_vector == 0)
|
if (number_of_places_left_in_vector == 0)
|
{
|
{
|
number_of_places_left_in_vector = nunits;
|
number_of_places_left_in_vector = nunits;
|
|
|
if (constant_p)
|
if (constant_p)
|
vec_cst = build_vector (vector_type, t);
|
vec_cst = build_vector (vector_type, t);
|
else
|
else
|
vec_cst = build_constructor_from_list (vector_type, t);
|
vec_cst = build_constructor_from_list (vector_type, t);
|
VEC_quick_push (tree, voprnds,
|
VEC_quick_push (tree, voprnds,
|
vect_init_vector (stmt, vec_cst, vector_type, NULL));
|
vect_init_vector (stmt, vec_cst, vector_type, NULL));
|
t = NULL_TREE;
|
t = NULL_TREE;
|
}
|
}
|
}
|
}
|
}
|
}
|
|
|
/* Since the vectors are created in the reverse order, we should invert
|
/* Since the vectors are created in the reverse order, we should invert
|
them. */
|
them. */
|
vec_num = VEC_length (tree, voprnds);
|
vec_num = VEC_length (tree, voprnds);
|
for (j = vec_num - 1; j >= 0; j--)
|
for (j = vec_num - 1; j >= 0; j--)
|
{
|
{
|
vop = VEC_index (tree, voprnds, j);
|
vop = VEC_index (tree, voprnds, j);
|
VEC_quick_push (tree, *vec_oprnds, vop);
|
VEC_quick_push (tree, *vec_oprnds, vop);
|
}
|
}
|
|
|
VEC_free (tree, heap, voprnds);
|
VEC_free (tree, heap, voprnds);
|
|
|
/* In case that VF is greater than the unrolling factor needed for the SLP
|
/* In case that VF is greater than the unrolling factor needed for the SLP
|
group of stmts, NUMBER_OF_VECTORS to be created is greater than
|
group of stmts, NUMBER_OF_VECTORS to be created is greater than
|
NUMBER_OF_SCALARS/NUNITS or NUNITS/NUMBER_OF_SCALARS, and hence we have
|
NUMBER_OF_SCALARS/NUNITS or NUNITS/NUMBER_OF_SCALARS, and hence we have
|
to replicate the vectors. */
|
to replicate the vectors. */
|
while (number_of_vectors > VEC_length (tree, *vec_oprnds))
|
while (number_of_vectors > VEC_length (tree, *vec_oprnds))
|
{
|
{
|
for (i = 0; VEC_iterate (tree, *vec_oprnds, i, vop) && i < vec_num; i++)
|
for (i = 0; VEC_iterate (tree, *vec_oprnds, i, vop) && i < vec_num; i++)
|
VEC_quick_push (tree, *vec_oprnds, vop);
|
VEC_quick_push (tree, *vec_oprnds, vop);
|
}
|
}
|
}
|
}
|
|
|
|
|
/* Get vectorized definitions from SLP_NODE that contains corresponding
|
/* Get vectorized definitions from SLP_NODE that contains corresponding
|
vectorized def-stmts. */
|
vectorized def-stmts. */
|
|
|
static void
|
static void
|
vect_get_slp_vect_defs (slp_tree slp_node, VEC (tree,heap) **vec_oprnds)
|
vect_get_slp_vect_defs (slp_tree slp_node, VEC (tree,heap) **vec_oprnds)
|
{
|
{
|
tree vec_oprnd;
|
tree vec_oprnd;
|
gimple vec_def_stmt;
|
gimple vec_def_stmt;
|
unsigned int i;
|
unsigned int i;
|
|
|
gcc_assert (SLP_TREE_VEC_STMTS (slp_node));
|
gcc_assert (SLP_TREE_VEC_STMTS (slp_node));
|
|
|
for (i = 0;
|
for (i = 0;
|
VEC_iterate (gimple, SLP_TREE_VEC_STMTS (slp_node), i, vec_def_stmt);
|
VEC_iterate (gimple, SLP_TREE_VEC_STMTS (slp_node), i, vec_def_stmt);
|
i++)
|
i++)
|
{
|
{
|
gcc_assert (vec_def_stmt);
|
gcc_assert (vec_def_stmt);
|
vec_oprnd = gimple_get_lhs (vec_def_stmt);
|
vec_oprnd = gimple_get_lhs (vec_def_stmt);
|
VEC_quick_push (tree, *vec_oprnds, vec_oprnd);
|
VEC_quick_push (tree, *vec_oprnds, vec_oprnd);
|
}
|
}
|
}
|
}
|
|
|
|
|
/* Get vectorized definitions for SLP_NODE.
|
/* Get vectorized definitions for SLP_NODE.
|
If the scalar definitions are loop invariants or constants, collect them and
|
If the scalar definitions are loop invariants or constants, collect them and
|
call vect_get_constant_vectors() to create vector stmts.
|
call vect_get_constant_vectors() to create vector stmts.
|
Otherwise, the def-stmts must be already vectorized and the vectorized stmts
|
Otherwise, the def-stmts must be already vectorized and the vectorized stmts
|
must be stored in the LEFT/RIGHT node of SLP_NODE, and we call
|
must be stored in the LEFT/RIGHT node of SLP_NODE, and we call
|
vect_get_slp_vect_defs() to retrieve them.
|
vect_get_slp_vect_defs() to retrieve them.
|
If VEC_OPRNDS1 is NULL, don't get vector defs for the second operand (from
|
If VEC_OPRNDS1 is NULL, don't get vector defs for the second operand (from
|
the right node. This is used when the second operand must remain scalar. */
|
the right node. This is used when the second operand must remain scalar. */
|
|
|
void
|
void
|
vect_get_slp_defs (slp_tree slp_node, VEC (tree,heap) **vec_oprnds0,
|
vect_get_slp_defs (slp_tree slp_node, VEC (tree,heap) **vec_oprnds0,
|
VEC (tree,heap) **vec_oprnds1)
|
VEC (tree,heap) **vec_oprnds1)
|
{
|
{
|
gimple first_stmt;
|
gimple first_stmt;
|
enum tree_code code;
|
enum tree_code code;
|
int number_of_vects;
|
int number_of_vects;
|
HOST_WIDE_INT lhs_size_unit, rhs_size_unit;
|
HOST_WIDE_INT lhs_size_unit, rhs_size_unit;
|
|
|
first_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0);
|
first_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0);
|
/* The number of vector defs is determined by the number of vector statements
|
/* The number of vector defs is determined by the number of vector statements
|
in the node from which we get those statements. */
|
in the node from which we get those statements. */
|
if (SLP_TREE_LEFT (slp_node))
|
if (SLP_TREE_LEFT (slp_node))
|
number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (SLP_TREE_LEFT (slp_node));
|
number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (SLP_TREE_LEFT (slp_node));
|
else
|
else
|
{
|
{
|
number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
|
number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
|
/* Number of vector stmts was calculated according to LHS in
|
/* Number of vector stmts was calculated according to LHS in
|
vect_schedule_slp_instance(), fix it by replacing LHS with RHS, if
|
vect_schedule_slp_instance(), fix it by replacing LHS with RHS, if
|
necessary. See vect_get_smallest_scalar_type() for details. */
|
necessary. See vect_get_smallest_scalar_type() for details. */
|
vect_get_smallest_scalar_type (first_stmt, &lhs_size_unit,
|
vect_get_smallest_scalar_type (first_stmt, &lhs_size_unit,
|
&rhs_size_unit);
|
&rhs_size_unit);
|
if (rhs_size_unit != lhs_size_unit)
|
if (rhs_size_unit != lhs_size_unit)
|
{
|
{
|
number_of_vects *= rhs_size_unit;
|
number_of_vects *= rhs_size_unit;
|
number_of_vects /= lhs_size_unit;
|
number_of_vects /= lhs_size_unit;
|
}
|
}
|
}
|
}
|
|
|
/* Allocate memory for vectorized defs. */
|
/* Allocate memory for vectorized defs. */
|
*vec_oprnds0 = VEC_alloc (tree, heap, number_of_vects);
|
*vec_oprnds0 = VEC_alloc (tree, heap, number_of_vects);
|
|
|
/* SLP_NODE corresponds either to a group of stores or to a group of
|
/* SLP_NODE corresponds either to a group of stores or to a group of
|
unary/binary operations. We don't call this function for loads. */
|
unary/binary operations. We don't call this function for loads. */
|
if (SLP_TREE_LEFT (slp_node))
|
if (SLP_TREE_LEFT (slp_node))
|
/* The defs are already vectorized. */
|
/* The defs are already vectorized. */
|
vect_get_slp_vect_defs (SLP_TREE_LEFT (slp_node), vec_oprnds0);
|
vect_get_slp_vect_defs (SLP_TREE_LEFT (slp_node), vec_oprnds0);
|
else
|
else
|
/* Build vectors from scalar defs. */
|
/* Build vectors from scalar defs. */
|
vect_get_constant_vectors (slp_node, vec_oprnds0, 0, number_of_vects);
|
vect_get_constant_vectors (slp_node, vec_oprnds0, 0, number_of_vects);
|
|
|
if (STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)))
|
if (STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)))
|
/* Since we don't call this function with loads, this is a group of
|
/* Since we don't call this function with loads, this is a group of
|
stores. */
|
stores. */
|
return;
|
return;
|
|
|
code = gimple_assign_rhs_code (first_stmt);
|
code = gimple_assign_rhs_code (first_stmt);
|
if (get_gimple_rhs_class (code) != GIMPLE_BINARY_RHS || !vec_oprnds1)
|
if (get_gimple_rhs_class (code) != GIMPLE_BINARY_RHS || !vec_oprnds1)
|
return;
|
return;
|
|
|
/* The number of vector defs is determined by the number of vector statements
|
/* The number of vector defs is determined by the number of vector statements
|
in the node from which we get those statements. */
|
in the node from which we get those statements. */
|
if (SLP_TREE_RIGHT (slp_node))
|
if (SLP_TREE_RIGHT (slp_node))
|
number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (SLP_TREE_RIGHT (slp_node));
|
number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (SLP_TREE_RIGHT (slp_node));
|
else
|
else
|
number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
|
number_of_vects = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
|
|
|
*vec_oprnds1 = VEC_alloc (tree, heap, number_of_vects);
|
*vec_oprnds1 = VEC_alloc (tree, heap, number_of_vects);
|
|
|
if (SLP_TREE_RIGHT (slp_node))
|
if (SLP_TREE_RIGHT (slp_node))
|
/* The defs are already vectorized. */
|
/* The defs are already vectorized. */
|
vect_get_slp_vect_defs (SLP_TREE_RIGHT (slp_node), vec_oprnds1);
|
vect_get_slp_vect_defs (SLP_TREE_RIGHT (slp_node), vec_oprnds1);
|
else
|
else
|
/* Build vectors from scalar defs. */
|
/* Build vectors from scalar defs. */
|
vect_get_constant_vectors (slp_node, vec_oprnds1, 1, number_of_vects);
|
vect_get_constant_vectors (slp_node, vec_oprnds1, 1, number_of_vects);
|
}
|
}
|
|
|
|
|
/* Create NCOPIES permutation statements using the mask MASK_BYTES (by
|
/* Create NCOPIES permutation statements using the mask MASK_BYTES (by
|
building a vector of type MASK_TYPE from it) and two input vectors placed in
|
building a vector of type MASK_TYPE from it) and two input vectors placed in
|
DR_CHAIN at FIRST_VEC_INDX and SECOND_VEC_INDX for the first copy and
|
DR_CHAIN at FIRST_VEC_INDX and SECOND_VEC_INDX for the first copy and
|
shifting by STRIDE elements of DR_CHAIN for every copy.
|
shifting by STRIDE elements of DR_CHAIN for every copy.
|
(STRIDE is the number of vectorized stmts for NODE divided by the number of
|
(STRIDE is the number of vectorized stmts for NODE divided by the number of
|
copies).
|
copies).
|
VECT_STMTS_COUNTER specifies the index in the vectorized stmts of NODE, where
|
VECT_STMTS_COUNTER specifies the index in the vectorized stmts of NODE, where
|
the created stmts must be inserted. */
|
the created stmts must be inserted. */
|
|
|
static inline void
|
static inline void
|
vect_create_mask_and_perm (gimple stmt, gimple next_scalar_stmt,
|
vect_create_mask_and_perm (gimple stmt, gimple next_scalar_stmt,
|
tree mask, int first_vec_indx, int second_vec_indx,
|
tree mask, int first_vec_indx, int second_vec_indx,
|
gimple_stmt_iterator *gsi, slp_tree node,
|
gimple_stmt_iterator *gsi, slp_tree node,
|
tree builtin_decl, tree vectype,
|
tree builtin_decl, tree vectype,
|
VEC(tree,heap) *dr_chain,
|
VEC(tree,heap) *dr_chain,
|
int ncopies, int vect_stmts_counter)
|
int ncopies, int vect_stmts_counter)
|
{
|
{
|
tree perm_dest;
|
tree perm_dest;
|
gimple perm_stmt = NULL;
|
gimple perm_stmt = NULL;
|
stmt_vec_info next_stmt_info;
|
stmt_vec_info next_stmt_info;
|
int i, stride;
|
int i, stride;
|
tree first_vec, second_vec, data_ref;
|
tree first_vec, second_vec, data_ref;
|
VEC (tree, heap) *params = NULL;
|
VEC (tree, heap) *params = NULL;
|
|
|
stride = SLP_TREE_NUMBER_OF_VEC_STMTS (node) / ncopies;
|
stride = SLP_TREE_NUMBER_OF_VEC_STMTS (node) / ncopies;
|
|
|
/* Initialize the vect stmts of NODE to properly insert the generated
|
/* Initialize the vect stmts of NODE to properly insert the generated
|
stmts later. */
|
stmts later. */
|
for (i = VEC_length (gimple, SLP_TREE_VEC_STMTS (node));
|
for (i = VEC_length (gimple, SLP_TREE_VEC_STMTS (node));
|
i < (int) SLP_TREE_NUMBER_OF_VEC_STMTS (node); i++)
|
i < (int) SLP_TREE_NUMBER_OF_VEC_STMTS (node); i++)
|
VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (node), NULL);
|
VEC_quick_push (gimple, SLP_TREE_VEC_STMTS (node), NULL);
|
|
|
perm_dest = vect_create_destination_var (gimple_assign_lhs (stmt), vectype);
|
perm_dest = vect_create_destination_var (gimple_assign_lhs (stmt), vectype);
|
for (i = 0; i < ncopies; i++)
|
for (i = 0; i < ncopies; i++)
|
{
|
{
|
first_vec = VEC_index (tree, dr_chain, first_vec_indx);
|
first_vec = VEC_index (tree, dr_chain, first_vec_indx);
|
second_vec = VEC_index (tree, dr_chain, second_vec_indx);
|
second_vec = VEC_index (tree, dr_chain, second_vec_indx);
|
|
|
/* Build argument list for the vectorized call. */
|
/* Build argument list for the vectorized call. */
|
VEC_free (tree, heap, params);
|
VEC_free (tree, heap, params);
|
params = VEC_alloc (tree, heap, 3);
|
params = VEC_alloc (tree, heap, 3);
|
VEC_quick_push (tree, params, first_vec);
|
VEC_quick_push (tree, params, first_vec);
|
VEC_quick_push (tree, params, second_vec);
|
VEC_quick_push (tree, params, second_vec);
|
VEC_quick_push (tree, params, mask);
|
VEC_quick_push (tree, params, mask);
|
|
|
/* Generate the permute statement. */
|
/* Generate the permute statement. */
|
perm_stmt = gimple_build_call_vec (builtin_decl, params);
|
perm_stmt = gimple_build_call_vec (builtin_decl, params);
|
data_ref = make_ssa_name (perm_dest, perm_stmt);
|
data_ref = make_ssa_name (perm_dest, perm_stmt);
|
gimple_call_set_lhs (perm_stmt, data_ref);
|
gimple_call_set_lhs (perm_stmt, data_ref);
|
vect_finish_stmt_generation (stmt, perm_stmt, gsi);
|
vect_finish_stmt_generation (stmt, perm_stmt, gsi);
|
|
|
/* Store the vector statement in NODE. */
|
/* Store the vector statement in NODE. */
|
VEC_replace (gimple, SLP_TREE_VEC_STMTS (node),
|
VEC_replace (gimple, SLP_TREE_VEC_STMTS (node),
|
stride * i + vect_stmts_counter, perm_stmt);
|
stride * i + vect_stmts_counter, perm_stmt);
|
|
|
first_vec_indx += stride;
|
first_vec_indx += stride;
|
second_vec_indx += stride;
|
second_vec_indx += stride;
|
}
|
}
|
|
|
/* Mark the scalar stmt as vectorized. */
|
/* Mark the scalar stmt as vectorized. */
|
next_stmt_info = vinfo_for_stmt (next_scalar_stmt);
|
next_stmt_info = vinfo_for_stmt (next_scalar_stmt);
|
STMT_VINFO_VEC_STMT (next_stmt_info) = perm_stmt;
|
STMT_VINFO_VEC_STMT (next_stmt_info) = perm_stmt;
|
}
|
}
|
|
|
|
|
/* Given FIRST_MASK_ELEMENT - the mask element in element representation,
|
/* Given FIRST_MASK_ELEMENT - the mask element in element representation,
|
return in CURRENT_MASK_ELEMENT its equivalent in target specific
|
return in CURRENT_MASK_ELEMENT its equivalent in target specific
|
representation. Check that the mask is valid and return FALSE if not.
|
representation. Check that the mask is valid and return FALSE if not.
|
Return TRUE in NEED_NEXT_VECTOR if the permutation requires to move to
|
Return TRUE in NEED_NEXT_VECTOR if the permutation requires to move to
|
the next vector, i.e., the current first vector is not needed. */
|
the next vector, i.e., the current first vector is not needed. */
|
|
|
static bool
|
static bool
|
vect_get_mask_element (gimple stmt, int first_mask_element, int m,
|
vect_get_mask_element (gimple stmt, int first_mask_element, int m,
|
int mask_nunits, bool only_one_vec, int index,
|
int mask_nunits, bool only_one_vec, int index,
|
int *mask, int *current_mask_element,
|
int *mask, int *current_mask_element,
|
bool *need_next_vector)
|
bool *need_next_vector)
|
{
|
{
|
int i;
|
int i;
|
static int number_of_mask_fixes = 1;
|
static int number_of_mask_fixes = 1;
|
static bool mask_fixed = false;
|
static bool mask_fixed = false;
|
static bool needs_first_vector = false;
|
static bool needs_first_vector = false;
|
|
|
/* Convert to target specific representation. */
|
/* Convert to target specific representation. */
|
*current_mask_element = first_mask_element + m;
|
*current_mask_element = first_mask_element + m;
|
/* Adjust the value in case it's a mask for second and third vectors. */
|
/* Adjust the value in case it's a mask for second and third vectors. */
|
*current_mask_element -= mask_nunits * (number_of_mask_fixes - 1);
|
*current_mask_element -= mask_nunits * (number_of_mask_fixes - 1);
|
|
|
if (*current_mask_element < mask_nunits)
|
if (*current_mask_element < mask_nunits)
|
needs_first_vector = true;
|
needs_first_vector = true;
|
|
|
/* We have only one input vector to permute but the mask accesses values in
|
/* We have only one input vector to permute but the mask accesses values in
|
the next vector as well. */
|
the next vector as well. */
|
if (only_one_vec && *current_mask_element >= mask_nunits)
|
if (only_one_vec && *current_mask_element >= mask_nunits)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "permutation requires at least two vectors ");
|
fprintf (vect_dump, "permutation requires at least two vectors ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
/* The mask requires the next vector. */
|
/* The mask requires the next vector. */
|
if (*current_mask_element >= mask_nunits * 2)
|
if (*current_mask_element >= mask_nunits * 2)
|
{
|
{
|
if (needs_first_vector || mask_fixed)
|
if (needs_first_vector || mask_fixed)
|
{
|
{
|
/* We either need the first vector too or have already moved to the
|
/* We either need the first vector too or have already moved to the
|
next vector. In both cases, this permutation needs three
|
next vector. In both cases, this permutation needs three
|
vectors. */
|
vectors. */
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "permutation requires at "
|
fprintf (vect_dump, "permutation requires at "
|
"least three vectors ");
|
"least three vectors ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
/* We move to the next vector, dropping the first one and working with
|
/* We move to the next vector, dropping the first one and working with
|
the second and the third - we need to adjust the values of the mask
|
the second and the third - we need to adjust the values of the mask
|
accordingly. */
|
accordingly. */
|
*current_mask_element -= mask_nunits * number_of_mask_fixes;
|
*current_mask_element -= mask_nunits * number_of_mask_fixes;
|
|
|
for (i = 0; i < index; i++)
|
for (i = 0; i < index; i++)
|
mask[i] -= mask_nunits * number_of_mask_fixes;
|
mask[i] -= mask_nunits * number_of_mask_fixes;
|
|
|
(number_of_mask_fixes)++;
|
(number_of_mask_fixes)++;
|
mask_fixed = true;
|
mask_fixed = true;
|
}
|
}
|
|
|
*need_next_vector = mask_fixed;
|
*need_next_vector = mask_fixed;
|
|
|
/* This was the last element of this mask. Start a new one. */
|
/* This was the last element of this mask. Start a new one. */
|
if (index == mask_nunits - 1)
|
if (index == mask_nunits - 1)
|
{
|
{
|
number_of_mask_fixes = 1;
|
number_of_mask_fixes = 1;
|
mask_fixed = false;
|
mask_fixed = false;
|
needs_first_vector = false;
|
needs_first_vector = false;
|
}
|
}
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
|
|
/* Generate vector permute statements from a list of loads in DR_CHAIN.
|
/* Generate vector permute statements from a list of loads in DR_CHAIN.
|
If ANALYZE_ONLY is TRUE, only check that it is possible to create valid
|
If ANALYZE_ONLY is TRUE, only check that it is possible to create valid
|
permute statements for SLP_NODE_INSTANCE. */
|
permute statements for SLP_NODE_INSTANCE. */
|
bool
|
bool
|
vect_transform_slp_perm_load (gimple stmt, VEC (tree, heap) *dr_chain,
|
vect_transform_slp_perm_load (gimple stmt, VEC (tree, heap) *dr_chain,
|
gimple_stmt_iterator *gsi, int vf,
|
gimple_stmt_iterator *gsi, int vf,
|
slp_instance slp_node_instance, bool analyze_only)
|
slp_instance slp_node_instance, bool analyze_only)
|
{
|
{
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
|
tree mask_element_type = NULL_TREE, mask_type;
|
tree mask_element_type = NULL_TREE, mask_type;
|
int i, j, k, m, scale, mask_nunits, nunits, vec_index = 0, scalar_index;
|
int i, j, k, m, scale, mask_nunits, nunits, vec_index = 0, scalar_index;
|
slp_tree node;
|
slp_tree node;
|
tree vectype = STMT_VINFO_VECTYPE (stmt_info), builtin_decl;
|
tree vectype = STMT_VINFO_VECTYPE (stmt_info), builtin_decl;
|
gimple next_scalar_stmt;
|
gimple next_scalar_stmt;
|
int group_size = SLP_INSTANCE_GROUP_SIZE (slp_node_instance);
|
int group_size = SLP_INSTANCE_GROUP_SIZE (slp_node_instance);
|
int first_mask_element;
|
int first_mask_element;
|
int index, unroll_factor, *mask, current_mask_element, ncopies;
|
int index, unroll_factor, *mask, current_mask_element, ncopies;
|
bool only_one_vec = false, need_next_vector = false;
|
bool only_one_vec = false, need_next_vector = false;
|
int first_vec_index, second_vec_index, orig_vec_stmts_num, vect_stmts_counter;
|
int first_vec_index, second_vec_index, orig_vec_stmts_num, vect_stmts_counter;
|
|
|
if (!targetm.vectorize.builtin_vec_perm)
|
if (!targetm.vectorize.builtin_vec_perm)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "no builtin for vect permute for ");
|
fprintf (vect_dump, "no builtin for vect permute for ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
builtin_decl = targetm.vectorize.builtin_vec_perm (vectype,
|
builtin_decl = targetm.vectorize.builtin_vec_perm (vectype,
|
&mask_element_type);
|
&mask_element_type);
|
if (!builtin_decl || !mask_element_type)
|
if (!builtin_decl || !mask_element_type)
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "no builtin for vect permute for ");
|
fprintf (vect_dump, "no builtin for vect permute for ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
mask_type = get_vectype_for_scalar_type (mask_element_type);
|
mask_type = get_vectype_for_scalar_type (mask_element_type);
|
mask_nunits = TYPE_VECTOR_SUBPARTS (mask_type);
|
mask_nunits = TYPE_VECTOR_SUBPARTS (mask_type);
|
mask = (int *) xmalloc (sizeof (int) * mask_nunits);
|
mask = (int *) xmalloc (sizeof (int) * mask_nunits);
|
nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
nunits = TYPE_VECTOR_SUBPARTS (vectype);
|
scale = mask_nunits / nunits;
|
scale = mask_nunits / nunits;
|
unroll_factor = SLP_INSTANCE_UNROLLING_FACTOR (slp_node_instance);
|
unroll_factor = SLP_INSTANCE_UNROLLING_FACTOR (slp_node_instance);
|
|
|
/* The number of vector stmts to generate based only on SLP_NODE_INSTANCE
|
/* The number of vector stmts to generate based only on SLP_NODE_INSTANCE
|
unrolling factor. */
|
unrolling factor. */
|
orig_vec_stmts_num = group_size *
|
orig_vec_stmts_num = group_size *
|
SLP_INSTANCE_UNROLLING_FACTOR (slp_node_instance) / nunits;
|
SLP_INSTANCE_UNROLLING_FACTOR (slp_node_instance) / nunits;
|
if (orig_vec_stmts_num == 1)
|
if (orig_vec_stmts_num == 1)
|
only_one_vec = true;
|
only_one_vec = true;
|
|
|
/* Number of copies is determined by the final vectorization factor
|
/* Number of copies is determined by the final vectorization factor
|
relatively to SLP_NODE_INSTANCE unrolling factor. */
|
relatively to SLP_NODE_INSTANCE unrolling factor. */
|
ncopies = vf / SLP_INSTANCE_UNROLLING_FACTOR (slp_node_instance);
|
ncopies = vf / SLP_INSTANCE_UNROLLING_FACTOR (slp_node_instance);
|
|
|
/* Generate permutation masks for every NODE. Number of masks for each NODE
|
/* Generate permutation masks for every NODE. Number of masks for each NODE
|
is equal to GROUP_SIZE.
|
is equal to GROUP_SIZE.
|
E.g., we have a group of three nodes with three loads from the same
|
E.g., we have a group of three nodes with three loads from the same
|
location in each node, and the vector size is 4. I.e., we have a
|
location in each node, and the vector size is 4. I.e., we have a
|
a0b0c0a1b1c1... sequence and we need to create the following vectors:
|
a0b0c0a1b1c1... sequence and we need to create the following vectors:
|
for a's: a0a0a0a1 a1a1a2a2 a2a3a3a3
|
for a's: a0a0a0a1 a1a1a2a2 a2a3a3a3
|
for b's: b0b0b0b1 b1b1b2b2 b2b3b3b3
|
for b's: b0b0b0b1 b1b1b2b2 b2b3b3b3
|
...
|
...
|
|
|
The masks for a's should be: {0,0,0,3} {3,3,6,6} {6,9,9,9} (in target
|
The masks for a's should be: {0,0,0,3} {3,3,6,6} {6,9,9,9} (in target
|
scpecific type, e.g., in bytes for Altivec.
|
scpecific type, e.g., in bytes for Altivec.
|
The last mask is illegal since we assume two operands for permute
|
The last mask is illegal since we assume two operands for permute
|
operation, and the mask element values can't be outside that range. Hence,
|
operation, and the mask element values can't be outside that range. Hence,
|
the last mask must be converted into {2,5,5,5}.
|
the last mask must be converted into {2,5,5,5}.
|
For the first two permutations we need the first and the second input
|
For the first two permutations we need the first and the second input
|
vectors: {a0,b0,c0,a1} and {b1,c1,a2,b2}, and for the last permutation
|
vectors: {a0,b0,c0,a1} and {b1,c1,a2,b2}, and for the last permutation
|
we need the second and the third vectors: {b1,c1,a2,b2} and
|
we need the second and the third vectors: {b1,c1,a2,b2} and
|
{c2,a3,b3,c3}. */
|
{c2,a3,b3,c3}. */
|
|
|
for (i = 0;
|
for (i = 0;
|
VEC_iterate (slp_tree, SLP_INSTANCE_LOADS (slp_node_instance),
|
VEC_iterate (slp_tree, SLP_INSTANCE_LOADS (slp_node_instance),
|
i, node);
|
i, node);
|
i++)
|
i++)
|
{
|
{
|
scalar_index = 0;
|
scalar_index = 0;
|
index = 0;
|
index = 0;
|
vect_stmts_counter = 0;
|
vect_stmts_counter = 0;
|
vec_index = 0;
|
vec_index = 0;
|
first_vec_index = vec_index++;
|
first_vec_index = vec_index++;
|
if (only_one_vec)
|
if (only_one_vec)
|
second_vec_index = first_vec_index;
|
second_vec_index = first_vec_index;
|
else
|
else
|
second_vec_index = vec_index++;
|
second_vec_index = vec_index++;
|
|
|
for (j = 0; j < unroll_factor; j++)
|
for (j = 0; j < unroll_factor; j++)
|
{
|
{
|
for (k = 0; k < group_size; k++)
|
for (k = 0; k < group_size; k++)
|
{
|
{
|
first_mask_element = (i + j * group_size) * scale;
|
first_mask_element = (i + j * group_size) * scale;
|
for (m = 0; m < scale; m++)
|
for (m = 0; m < scale; m++)
|
{
|
{
|
if (!vect_get_mask_element (stmt, first_mask_element, m,
|
if (!vect_get_mask_element (stmt, first_mask_element, m,
|
mask_nunits, only_one_vec, index, mask,
|
mask_nunits, only_one_vec, index, mask,
|
¤t_mask_element, &need_next_vector))
|
¤t_mask_element, &need_next_vector))
|
return false;
|
return false;
|
|
|
mask[index++] = current_mask_element;
|
mask[index++] = current_mask_element;
|
}
|
}
|
|
|
if (index == mask_nunits)
|
if (index == mask_nunits)
|
{
|
{
|
tree mask_vec = NULL;
|
tree mask_vec = NULL;
|
|
|
while (--index >= 0)
|
while (--index >= 0)
|
{
|
{
|
tree t = build_int_cst (mask_element_type, mask[index]);
|
tree t = build_int_cst (mask_element_type, mask[index]);
|
mask_vec = tree_cons (NULL, t, mask_vec);
|
mask_vec = tree_cons (NULL, t, mask_vec);
|
}
|
}
|
mask_vec = build_vector (mask_type, mask_vec);
|
mask_vec = build_vector (mask_type, mask_vec);
|
index = 0;
|
index = 0;
|
|
|
if (!targetm.vectorize.builtin_vec_perm_ok (vectype,
|
if (!targetm.vectorize.builtin_vec_perm_ok (vectype,
|
mask_vec))
|
mask_vec))
|
{
|
{
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "unsupported vect permute ");
|
fprintf (vect_dump, "unsupported vect permute ");
|
print_generic_expr (vect_dump, mask_vec, 0);
|
print_generic_expr (vect_dump, mask_vec, 0);
|
}
|
}
|
free (mask);
|
free (mask);
|
return false;
|
return false;
|
}
|
}
|
|
|
if (!analyze_only)
|
if (!analyze_only)
|
{
|
{
|
if (need_next_vector)
|
if (need_next_vector)
|
{
|
{
|
first_vec_index = second_vec_index;
|
first_vec_index = second_vec_index;
|
second_vec_index = vec_index;
|
second_vec_index = vec_index;
|
}
|
}
|
|
|
next_scalar_stmt = VEC_index (gimple,
|
next_scalar_stmt = VEC_index (gimple,
|
SLP_TREE_SCALAR_STMTS (node), scalar_index++);
|
SLP_TREE_SCALAR_STMTS (node), scalar_index++);
|
|
|
vect_create_mask_and_perm (stmt, next_scalar_stmt,
|
vect_create_mask_and_perm (stmt, next_scalar_stmt,
|
mask_vec, first_vec_index, second_vec_index,
|
mask_vec, first_vec_index, second_vec_index,
|
gsi, node, builtin_decl, vectype, dr_chain,
|
gsi, node, builtin_decl, vectype, dr_chain,
|
ncopies, vect_stmts_counter++);
|
ncopies, vect_stmts_counter++);
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
|
|
free (mask);
|
free (mask);
|
return true;
|
return true;
|
}
|
}
|
|
|
|
|
|
|
/* Vectorize SLP instance tree in postorder. */
|
/* Vectorize SLP instance tree in postorder. */
|
|
|
static bool
|
static bool
|
vect_schedule_slp_instance (slp_tree node, slp_instance instance,
|
vect_schedule_slp_instance (slp_tree node, slp_instance instance,
|
unsigned int vectorization_factor)
|
unsigned int vectorization_factor)
|
{
|
{
|
gimple stmt;
|
gimple stmt;
|
bool strided_store, is_store;
|
bool strided_store, is_store;
|
gimple_stmt_iterator si;
|
gimple_stmt_iterator si;
|
stmt_vec_info stmt_info;
|
stmt_vec_info stmt_info;
|
unsigned int vec_stmts_size, nunits, group_size;
|
unsigned int vec_stmts_size, nunits, group_size;
|
tree vectype;
|
tree vectype;
|
int i;
|
int i;
|
slp_tree loads_node;
|
slp_tree loads_node;
|
|
|
if (!node)
|
if (!node)
|
return false;
|
return false;
|
|
|
vect_schedule_slp_instance (SLP_TREE_LEFT (node), instance,
|
vect_schedule_slp_instance (SLP_TREE_LEFT (node), instance,
|
vectorization_factor);
|
vectorization_factor);
|
vect_schedule_slp_instance (SLP_TREE_RIGHT (node), instance,
|
vect_schedule_slp_instance (SLP_TREE_RIGHT (node), instance,
|
vectorization_factor);
|
vectorization_factor);
|
|
|
stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (node), 0);
|
stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (node), 0);
|
stmt_info = vinfo_for_stmt (stmt);
|
stmt_info = vinfo_for_stmt (stmt);
|
|
|
/* VECTYPE is the type of the destination. */
|
/* VECTYPE is the type of the destination. */
|
vectype = get_vectype_for_scalar_type (TREE_TYPE (gimple_assign_lhs (stmt)));
|
vectype = get_vectype_for_scalar_type (TREE_TYPE (gimple_assign_lhs (stmt)));
|
nunits = (unsigned int) TYPE_VECTOR_SUBPARTS (vectype);
|
nunits = (unsigned int) TYPE_VECTOR_SUBPARTS (vectype);
|
group_size = SLP_INSTANCE_GROUP_SIZE (instance);
|
group_size = SLP_INSTANCE_GROUP_SIZE (instance);
|
|
|
/* For each SLP instance calculate number of vector stmts to be created
|
/* For each SLP instance calculate number of vector stmts to be created
|
for the scalar stmts in each node of the SLP tree. Number of vector
|
for the scalar stmts in each node of the SLP tree. Number of vector
|
elements in one vector iteration is the number of scalar elements in
|
elements in one vector iteration is the number of scalar elements in
|
one scalar iteration (GROUP_SIZE) multiplied by VF divided by vector
|
one scalar iteration (GROUP_SIZE) multiplied by VF divided by vector
|
size. */
|
size. */
|
vec_stmts_size = (vectorization_factor * group_size) / nunits;
|
vec_stmts_size = (vectorization_factor * group_size) / nunits;
|
|
|
/* In case of load permutation we have to allocate vectorized statements for
|
/* In case of load permutation we have to allocate vectorized statements for
|
all the nodes that participate in that permutation. */
|
all the nodes that participate in that permutation. */
|
if (SLP_INSTANCE_LOAD_PERMUTATION (instance))
|
if (SLP_INSTANCE_LOAD_PERMUTATION (instance))
|
{
|
{
|
for (i = 0;
|
for (i = 0;
|
VEC_iterate (slp_tree, SLP_INSTANCE_LOADS (instance), i, loads_node);
|
VEC_iterate (slp_tree, SLP_INSTANCE_LOADS (instance), i, loads_node);
|
i++)
|
i++)
|
{
|
{
|
if (!SLP_TREE_VEC_STMTS (loads_node))
|
if (!SLP_TREE_VEC_STMTS (loads_node))
|
{
|
{
|
SLP_TREE_VEC_STMTS (loads_node) = VEC_alloc (gimple, heap,
|
SLP_TREE_VEC_STMTS (loads_node) = VEC_alloc (gimple, heap,
|
vec_stmts_size);
|
vec_stmts_size);
|
SLP_TREE_NUMBER_OF_VEC_STMTS (loads_node) = vec_stmts_size;
|
SLP_TREE_NUMBER_OF_VEC_STMTS (loads_node) = vec_stmts_size;
|
}
|
}
|
}
|
}
|
}
|
}
|
|
|
if (!SLP_TREE_VEC_STMTS (node))
|
if (!SLP_TREE_VEC_STMTS (node))
|
{
|
{
|
SLP_TREE_VEC_STMTS (node) = VEC_alloc (gimple, heap, vec_stmts_size);
|
SLP_TREE_VEC_STMTS (node) = VEC_alloc (gimple, heap, vec_stmts_size);
|
SLP_TREE_NUMBER_OF_VEC_STMTS (node) = vec_stmts_size;
|
SLP_TREE_NUMBER_OF_VEC_STMTS (node) = vec_stmts_size;
|
}
|
}
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "------>vectorizing SLP node starting from: ");
|
fprintf (vect_dump, "------>vectorizing SLP node starting from: ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
/* Loads should be inserted before the first load. */
|
/* Loads should be inserted before the first load. */
|
if (SLP_INSTANCE_FIRST_LOAD_STMT (instance)
|
if (SLP_INSTANCE_FIRST_LOAD_STMT (instance)
|
&& STMT_VINFO_STRIDED_ACCESS (stmt_info)
|
&& STMT_VINFO_STRIDED_ACCESS (stmt_info)
|
&& !REFERENCE_CLASS_P (gimple_get_lhs (stmt)))
|
&& !REFERENCE_CLASS_P (gimple_get_lhs (stmt)))
|
si = gsi_for_stmt (SLP_INSTANCE_FIRST_LOAD_STMT (instance));
|
si = gsi_for_stmt (SLP_INSTANCE_FIRST_LOAD_STMT (instance));
|
else
|
else
|
si = gsi_for_stmt (stmt);
|
si = gsi_for_stmt (stmt);
|
|
|
is_store = vect_transform_stmt (stmt, &si, &strided_store, node, instance);
|
is_store = vect_transform_stmt (stmt, &si, &strided_store, node, instance);
|
if (is_store)
|
if (is_store)
|
{
|
{
|
if (DR_GROUP_FIRST_DR (stmt_info))
|
if (DR_GROUP_FIRST_DR (stmt_info))
|
/* If IS_STORE is TRUE, the vectorization of the
|
/* If IS_STORE is TRUE, the vectorization of the
|
interleaving chain was completed - free all the stores in
|
interleaving chain was completed - free all the stores in
|
the chain. */
|
the chain. */
|
vect_remove_stores (DR_GROUP_FIRST_DR (stmt_info));
|
vect_remove_stores (DR_GROUP_FIRST_DR (stmt_info));
|
else
|
else
|
/* FORNOW: SLP originates only from strided stores. */
|
/* FORNOW: SLP originates only from strided stores. */
|
gcc_unreachable ();
|
gcc_unreachable ();
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
/* FORNOW: SLP originates only from strided stores. */
|
/* FORNOW: SLP originates only from strided stores. */
|
return false;
|
return false;
|
}
|
}
|
|
|
|
|
bool
|
bool
|
vect_schedule_slp (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
|
vect_schedule_slp (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
|
{
|
{
|
VEC (slp_instance, heap) *slp_instances;
|
VEC (slp_instance, heap) *slp_instances;
|
slp_instance instance;
|
slp_instance instance;
|
unsigned int i, vf;
|
unsigned int i, vf;
|
bool is_store = false;
|
bool is_store = false;
|
|
|
if (loop_vinfo)
|
if (loop_vinfo)
|
{
|
{
|
slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
|
slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
|
vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
|
}
|
}
|
else
|
else
|
{
|
{
|
slp_instances = BB_VINFO_SLP_INSTANCES (bb_vinfo);
|
slp_instances = BB_VINFO_SLP_INSTANCES (bb_vinfo);
|
vf = 1;
|
vf = 1;
|
}
|
}
|
|
|
for (i = 0; VEC_iterate (slp_instance, slp_instances, i, instance); i++)
|
for (i = 0; VEC_iterate (slp_instance, slp_instances, i, instance); i++)
|
{
|
{
|
/* Schedule the tree of INSTANCE. */
|
/* Schedule the tree of INSTANCE. */
|
is_store = vect_schedule_slp_instance (SLP_INSTANCE_TREE (instance),
|
is_store = vect_schedule_slp_instance (SLP_INSTANCE_TREE (instance),
|
instance, vf);
|
instance, vf);
|
if (vect_print_dump_info (REPORT_VECTORIZED_LOCATIONS)
|
if (vect_print_dump_info (REPORT_VECTORIZED_LOCATIONS)
|
|| vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
|| vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
|
fprintf (vect_dump, "vectorizing stmts using SLP.");
|
fprintf (vect_dump, "vectorizing stmts using SLP.");
|
}
|
}
|
|
|
return is_store;
|
return is_store;
|
}
|
}
|
|
|
|
|
/* Vectorize the basic block. */
|
/* Vectorize the basic block. */
|
|
|
void
|
void
|
vect_slp_transform_bb (basic_block bb)
|
vect_slp_transform_bb (basic_block bb)
|
{
|
{
|
bb_vec_info bb_vinfo = vec_info_for_bb (bb);
|
bb_vec_info bb_vinfo = vec_info_for_bb (bb);
|
gimple_stmt_iterator si;
|
gimple_stmt_iterator si;
|
|
|
gcc_assert (bb_vinfo);
|
gcc_assert (bb_vinfo);
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "SLPing BB\n");
|
fprintf (vect_dump, "SLPing BB\n");
|
|
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
|
{
|
{
|
gimple stmt = gsi_stmt (si);
|
gimple stmt = gsi_stmt (si);
|
stmt_vec_info stmt_info;
|
stmt_vec_info stmt_info;
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
{
|
{
|
fprintf (vect_dump, "------>SLPing statement: ");
|
fprintf (vect_dump, "------>SLPing statement: ");
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
|
}
|
}
|
|
|
stmt_info = vinfo_for_stmt (stmt);
|
stmt_info = vinfo_for_stmt (stmt);
|
gcc_assert (stmt_info);
|
gcc_assert (stmt_info);
|
|
|
/* Schedule all the SLP instances when the first SLP stmt is reached. */
|
/* Schedule all the SLP instances when the first SLP stmt is reached. */
|
if (STMT_SLP_TYPE (stmt_info))
|
if (STMT_SLP_TYPE (stmt_info))
|
{
|
{
|
vect_schedule_slp (NULL, bb_vinfo);
|
vect_schedule_slp (NULL, bb_vinfo);
|
break;
|
break;
|
}
|
}
|
}
|
}
|
|
|
mark_sym_for_renaming (gimple_vop (cfun));
|
mark_sym_for_renaming (gimple_vop (cfun));
|
/* The memory tags and pointers in vectorized statements need to
|
/* The memory tags and pointers in vectorized statements need to
|
have their SSA forms updated. FIXME, why can't this be delayed
|
have their SSA forms updated. FIXME, why can't this be delayed
|
until all the loops have been transformed? */
|
until all the loops have been transformed? */
|
update_ssa (TODO_update_ssa);
|
update_ssa (TODO_update_ssa);
|
|
|
if (vect_print_dump_info (REPORT_DETAILS))
|
if (vect_print_dump_info (REPORT_DETAILS))
|
fprintf (vect_dump, "BASIC BLOCK VECTORIZED\n");
|
fprintf (vect_dump, "BASIC BLOCK VECTORIZED\n");
|
|
|
destroy_bb_vec_info (bb_vinfo);
|
destroy_bb_vec_info (bb_vinfo);
|
}
|
}
|
|
|
|
|
