/* Global, SSA-based optimizations using mathematical identities.
|
/* Global, SSA-based optimizations using mathematical identities.
|
Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011
|
Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011
|
Free Software Foundation, Inc.
|
Free Software Foundation, Inc.
|
|
|
This file is part of GCC.
|
This file is part of GCC.
|
|
|
GCC is free software; you can redistribute it and/or modify it
|
GCC is free software; you can redistribute it and/or modify it
|
under the terms of the GNU General Public License as published by the
|
under the terms of the GNU General Public License as published by the
|
Free Software Foundation; either version 3, or (at your option) any
|
Free Software Foundation; either version 3, or (at your option) any
|
later version.
|
later version.
|
|
|
GCC is distributed in the hope that it will be useful, but WITHOUT
|
GCC is distributed in the hope that it will be useful, but WITHOUT
|
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
for more details.
|
for more details.
|
|
|
You should have received a copy of the GNU General Public License
|
You should have received a copy of the GNU General Public License
|
along with GCC; see the file COPYING3. If not see
|
along with GCC; see the file COPYING3. If not see
|
<http://www.gnu.org/licenses/>. */
|
<http://www.gnu.org/licenses/>. */
|
|
|
/* Currently, the only mini-pass in this file tries to CSE reciprocal
|
/* Currently, the only mini-pass in this file tries to CSE reciprocal
|
operations. These are common in sequences such as this one:
|
operations. These are common in sequences such as this one:
|
|
|
modulus = sqrt(x*x + y*y + z*z);
|
modulus = sqrt(x*x + y*y + z*z);
|
x = x / modulus;
|
x = x / modulus;
|
y = y / modulus;
|
y = y / modulus;
|
z = z / modulus;
|
z = z / modulus;
|
|
|
that can be optimized to
|
that can be optimized to
|
|
|
modulus = sqrt(x*x + y*y + z*z);
|
modulus = sqrt(x*x + y*y + z*z);
|
rmodulus = 1.0 / modulus;
|
rmodulus = 1.0 / modulus;
|
x = x * rmodulus;
|
x = x * rmodulus;
|
y = y * rmodulus;
|
y = y * rmodulus;
|
z = z * rmodulus;
|
z = z * rmodulus;
|
|
|
We do this for loop invariant divisors, and with this pass whenever
|
We do this for loop invariant divisors, and with this pass whenever
|
we notice that a division has the same divisor multiple times.
|
we notice that a division has the same divisor multiple times.
|
|
|
Of course, like in PRE, we don't insert a division if a dominator
|
Of course, like in PRE, we don't insert a division if a dominator
|
already has one. However, this cannot be done as an extension of
|
already has one. However, this cannot be done as an extension of
|
PRE for several reasons.
|
PRE for several reasons.
|
|
|
First of all, with some experiments it was found out that the
|
First of all, with some experiments it was found out that the
|
transformation is not always useful if there are only two divisions
|
transformation is not always useful if there are only two divisions
|
hy the same divisor. This is probably because modern processors
|
hy the same divisor. This is probably because modern processors
|
can pipeline the divisions; on older, in-order processors it should
|
can pipeline the divisions; on older, in-order processors it should
|
still be effective to optimize two divisions by the same number.
|
still be effective to optimize two divisions by the same number.
|
We make this a param, and it shall be called N in the remainder of
|
We make this a param, and it shall be called N in the remainder of
|
this comment.
|
this comment.
|
|
|
Second, if trapping math is active, we have less freedom on where
|
Second, if trapping math is active, we have less freedom on where
|
to insert divisions: we can only do so in basic blocks that already
|
to insert divisions: we can only do so in basic blocks that already
|
contain one. (If divisions don't trap, instead, we can insert
|
contain one. (If divisions don't trap, instead, we can insert
|
divisions elsewhere, which will be in blocks that are common dominators
|
divisions elsewhere, which will be in blocks that are common dominators
|
of those that have the division).
|
of those that have the division).
|
|
|
We really don't want to compute the reciprocal unless a division will
|
We really don't want to compute the reciprocal unless a division will
|
be found. To do this, we won't insert the division in a basic block
|
be found. To do this, we won't insert the division in a basic block
|
that has less than N divisions *post-dominating* it.
|
that has less than N divisions *post-dominating* it.
|
|
|
The algorithm constructs a subset of the dominator tree, holding the
|
The algorithm constructs a subset of the dominator tree, holding the
|
blocks containing the divisions and the common dominators to them,
|
blocks containing the divisions and the common dominators to them,
|
and walk it twice. The first walk is in post-order, and it annotates
|
and walk it twice. The first walk is in post-order, and it annotates
|
each block with the number of divisions that post-dominate it: this
|
each block with the number of divisions that post-dominate it: this
|
gives information on where divisions can be inserted profitably.
|
gives information on where divisions can be inserted profitably.
|
The second walk is in pre-order, and it inserts divisions as explained
|
The second walk is in pre-order, and it inserts divisions as explained
|
above, and replaces divisions by multiplications.
|
above, and replaces divisions by multiplications.
|
|
|
In the best case, the cost of the pass is O(n_statements). In the
|
In the best case, the cost of the pass is O(n_statements). In the
|
worst-case, the cost is due to creating the dominator tree subset,
|
worst-case, the cost is due to creating the dominator tree subset,
|
with a cost of O(n_basic_blocks ^ 2); however this can only happen
|
with a cost of O(n_basic_blocks ^ 2); however this can only happen
|
for n_statements / n_basic_blocks statements. So, the amortized cost
|
for n_statements / n_basic_blocks statements. So, the amortized cost
|
of creating the dominator tree subset is O(n_basic_blocks) and the
|
of creating the dominator tree subset is O(n_basic_blocks) and the
|
worst-case cost of the pass is O(n_statements * n_basic_blocks).
|
worst-case cost of the pass is O(n_statements * n_basic_blocks).
|
|
|
More practically, the cost will be small because there are few
|
More practically, the cost will be small because there are few
|
divisions, and they tend to be in the same basic block, so insert_bb
|
divisions, and they tend to be in the same basic block, so insert_bb
|
is called very few times.
|
is called very few times.
|
|
|
If we did this using domwalk.c, an efficient implementation would have
|
If we did this using domwalk.c, an efficient implementation would have
|
to work on all the variables in a single pass, because we could not
|
to work on all the variables in a single pass, because we could not
|
work on just a subset of the dominator tree, as we do now, and the
|
work on just a subset of the dominator tree, as we do now, and the
|
cost would also be something like O(n_statements * n_basic_blocks).
|
cost would also be something like O(n_statements * n_basic_blocks).
|
The data structures would be more complex in order to work on all the
|
The data structures would be more complex in order to work on all the
|
variables in a single pass. */
|
variables in a single pass. */
|
|
|
#include "config.h"
|
#include "config.h"
|
#include "system.h"
|
#include "system.h"
|
#include "coretypes.h"
|
#include "coretypes.h"
|
#include "tm.h"
|
#include "tm.h"
|
#include "flags.h"
|
#include "flags.h"
|
#include "tree.h"
|
#include "tree.h"
|
#include "tree-flow.h"
|
#include "tree-flow.h"
|
#include "timevar.h"
|
#include "timevar.h"
|
#include "tree-pass.h"
|
#include "tree-pass.h"
|
#include "alloc-pool.h"
|
#include "alloc-pool.h"
|
#include "basic-block.h"
|
#include "basic-block.h"
|
#include "target.h"
|
#include "target.h"
|
#include "gimple-pretty-print.h"
|
#include "gimple-pretty-print.h"
|
|
|
/* FIXME: RTL headers have to be included here for optabs. */
|
/* FIXME: RTL headers have to be included here for optabs. */
|
#include "rtl.h" /* Because optabs.h wants enum rtx_code. */
|
#include "rtl.h" /* Because optabs.h wants enum rtx_code. */
|
#include "expr.h" /* Because optabs.h wants sepops. */
|
#include "expr.h" /* Because optabs.h wants sepops. */
|
#include "optabs.h"
|
#include "optabs.h"
|
|
|
/* This structure represents one basic block that either computes a
|
/* This structure represents one basic block that either computes a
|
division, or is a common dominator for basic block that compute a
|
division, or is a common dominator for basic block that compute a
|
division. */
|
division. */
|
struct occurrence {
|
struct occurrence {
|
/* The basic block represented by this structure. */
|
/* The basic block represented by this structure. */
|
basic_block bb;
|
basic_block bb;
|
|
|
/* If non-NULL, the SSA_NAME holding the definition for a reciprocal
|
/* If non-NULL, the SSA_NAME holding the definition for a reciprocal
|
inserted in BB. */
|
inserted in BB. */
|
tree recip_def;
|
tree recip_def;
|
|
|
/* If non-NULL, the GIMPLE_ASSIGN for a reciprocal computation that
|
/* If non-NULL, the GIMPLE_ASSIGN for a reciprocal computation that
|
was inserted in BB. */
|
was inserted in BB. */
|
gimple recip_def_stmt;
|
gimple recip_def_stmt;
|
|
|
/* Pointer to a list of "struct occurrence"s for blocks dominated
|
/* Pointer to a list of "struct occurrence"s for blocks dominated
|
by BB. */
|
by BB. */
|
struct occurrence *children;
|
struct occurrence *children;
|
|
|
/* Pointer to the next "struct occurrence"s in the list of blocks
|
/* Pointer to the next "struct occurrence"s in the list of blocks
|
sharing a common dominator. */
|
sharing a common dominator. */
|
struct occurrence *next;
|
struct occurrence *next;
|
|
|
/* The number of divisions that are in BB before compute_merit. The
|
/* The number of divisions that are in BB before compute_merit. The
|
number of divisions that are in BB or post-dominate it after
|
number of divisions that are in BB or post-dominate it after
|
compute_merit. */
|
compute_merit. */
|
int num_divisions;
|
int num_divisions;
|
|
|
/* True if the basic block has a division, false if it is a common
|
/* True if the basic block has a division, false if it is a common
|
dominator for basic blocks that do. If it is false and trapping
|
dominator for basic blocks that do. If it is false and trapping
|
math is active, BB is not a candidate for inserting a reciprocal. */
|
math is active, BB is not a candidate for inserting a reciprocal. */
|
bool bb_has_division;
|
bool bb_has_division;
|
};
|
};
|
|
|
static struct
|
static struct
|
{
|
{
|
/* Number of 1.0/X ops inserted. */
|
/* Number of 1.0/X ops inserted. */
|
int rdivs_inserted;
|
int rdivs_inserted;
|
|
|
/* Number of 1.0/FUNC ops inserted. */
|
/* Number of 1.0/FUNC ops inserted. */
|
int rfuncs_inserted;
|
int rfuncs_inserted;
|
} reciprocal_stats;
|
} reciprocal_stats;
|
|
|
static struct
|
static struct
|
{
|
{
|
/* Number of cexpi calls inserted. */
|
/* Number of cexpi calls inserted. */
|
int inserted;
|
int inserted;
|
} sincos_stats;
|
} sincos_stats;
|
|
|
static struct
|
static struct
|
{
|
{
|
/* Number of hand-written 32-bit bswaps found. */
|
/* Number of hand-written 32-bit bswaps found. */
|
int found_32bit;
|
int found_32bit;
|
|
|
/* Number of hand-written 64-bit bswaps found. */
|
/* Number of hand-written 64-bit bswaps found. */
|
int found_64bit;
|
int found_64bit;
|
} bswap_stats;
|
} bswap_stats;
|
|
|
static struct
|
static struct
|
{
|
{
|
/* Number of widening multiplication ops inserted. */
|
/* Number of widening multiplication ops inserted. */
|
int widen_mults_inserted;
|
int widen_mults_inserted;
|
|
|
/* Number of integer multiply-and-accumulate ops inserted. */
|
/* Number of integer multiply-and-accumulate ops inserted. */
|
int maccs_inserted;
|
int maccs_inserted;
|
|
|
/* Number of fp fused multiply-add ops inserted. */
|
/* Number of fp fused multiply-add ops inserted. */
|
int fmas_inserted;
|
int fmas_inserted;
|
} widen_mul_stats;
|
} widen_mul_stats;
|
|
|
/* The instance of "struct occurrence" representing the highest
|
/* The instance of "struct occurrence" representing the highest
|
interesting block in the dominator tree. */
|
interesting block in the dominator tree. */
|
static struct occurrence *occ_head;
|
static struct occurrence *occ_head;
|
|
|
/* Allocation pool for getting instances of "struct occurrence". */
|
/* Allocation pool for getting instances of "struct occurrence". */
|
static alloc_pool occ_pool;
|
static alloc_pool occ_pool;
|
|
|
|
|
|
|
/* Allocate and return a new struct occurrence for basic block BB, and
|
/* Allocate and return a new struct occurrence for basic block BB, and
|
whose children list is headed by CHILDREN. */
|
whose children list is headed by CHILDREN. */
|
static struct occurrence *
|
static struct occurrence *
|
occ_new (basic_block bb, struct occurrence *children)
|
occ_new (basic_block bb, struct occurrence *children)
|
{
|
{
|
struct occurrence *occ;
|
struct occurrence *occ;
|
|
|
bb->aux = occ = (struct occurrence *) pool_alloc (occ_pool);
|
bb->aux = occ = (struct occurrence *) pool_alloc (occ_pool);
|
memset (occ, 0, sizeof (struct occurrence));
|
memset (occ, 0, sizeof (struct occurrence));
|
|
|
occ->bb = bb;
|
occ->bb = bb;
|
occ->children = children;
|
occ->children = children;
|
return occ;
|
return occ;
|
}
|
}
|
|
|
|
|
/* Insert NEW_OCC into our subset of the dominator tree. P_HEAD points to a
|
/* Insert NEW_OCC into our subset of the dominator tree. P_HEAD points to a
|
list of "struct occurrence"s, one per basic block, having IDOM as
|
list of "struct occurrence"s, one per basic block, having IDOM as
|
their common dominator.
|
their common dominator.
|
|
|
We try to insert NEW_OCC as deep as possible in the tree, and we also
|
We try to insert NEW_OCC as deep as possible in the tree, and we also
|
insert any other block that is a common dominator for BB and one
|
insert any other block that is a common dominator for BB and one
|
block already in the tree. */
|
block already in the tree. */
|
|
|
static void
|
static void
|
insert_bb (struct occurrence *new_occ, basic_block idom,
|
insert_bb (struct occurrence *new_occ, basic_block idom,
|
struct occurrence **p_head)
|
struct occurrence **p_head)
|
{
|
{
|
struct occurrence *occ, **p_occ;
|
struct occurrence *occ, **p_occ;
|
|
|
for (p_occ = p_head; (occ = *p_occ) != NULL; )
|
for (p_occ = p_head; (occ = *p_occ) != NULL; )
|
{
|
{
|
basic_block bb = new_occ->bb, occ_bb = occ->bb;
|
basic_block bb = new_occ->bb, occ_bb = occ->bb;
|
basic_block dom = nearest_common_dominator (CDI_DOMINATORS, occ_bb, bb);
|
basic_block dom = nearest_common_dominator (CDI_DOMINATORS, occ_bb, bb);
|
if (dom == bb)
|
if (dom == bb)
|
{
|
{
|
/* BB dominates OCC_BB. OCC becomes NEW_OCC's child: remove OCC
|
/* BB dominates OCC_BB. OCC becomes NEW_OCC's child: remove OCC
|
from its list. */
|
from its list. */
|
*p_occ = occ->next;
|
*p_occ = occ->next;
|
occ->next = new_occ->children;
|
occ->next = new_occ->children;
|
new_occ->children = occ;
|
new_occ->children = occ;
|
|
|
/* Try the next block (it may as well be dominated by BB). */
|
/* Try the next block (it may as well be dominated by BB). */
|
}
|
}
|
|
|
else if (dom == occ_bb)
|
else if (dom == occ_bb)
|
{
|
{
|
/* OCC_BB dominates BB. Tail recurse to look deeper. */
|
/* OCC_BB dominates BB. Tail recurse to look deeper. */
|
insert_bb (new_occ, dom, &occ->children);
|
insert_bb (new_occ, dom, &occ->children);
|
return;
|
return;
|
}
|
}
|
|
|
else if (dom != idom)
|
else if (dom != idom)
|
{
|
{
|
gcc_assert (!dom->aux);
|
gcc_assert (!dom->aux);
|
|
|
/* There is a dominator between IDOM and BB, add it and make
|
/* There is a dominator between IDOM and BB, add it and make
|
two children out of NEW_OCC and OCC. First, remove OCC from
|
two children out of NEW_OCC and OCC. First, remove OCC from
|
its list. */
|
its list. */
|
*p_occ = occ->next;
|
*p_occ = occ->next;
|
new_occ->next = occ;
|
new_occ->next = occ;
|
occ->next = NULL;
|
occ->next = NULL;
|
|
|
/* None of the previous blocks has DOM as a dominator: if we tail
|
/* None of the previous blocks has DOM as a dominator: if we tail
|
recursed, we would reexamine them uselessly. Just switch BB with
|
recursed, we would reexamine them uselessly. Just switch BB with
|
DOM, and go on looking for blocks dominated by DOM. */
|
DOM, and go on looking for blocks dominated by DOM. */
|
new_occ = occ_new (dom, new_occ);
|
new_occ = occ_new (dom, new_occ);
|
}
|
}
|
|
|
else
|
else
|
{
|
{
|
/* Nothing special, go on with the next element. */
|
/* Nothing special, go on with the next element. */
|
p_occ = &occ->next;
|
p_occ = &occ->next;
|
}
|
}
|
}
|
}
|
|
|
/* No place was found as a child of IDOM. Make BB a sibling of IDOM. */
|
/* No place was found as a child of IDOM. Make BB a sibling of IDOM. */
|
new_occ->next = *p_head;
|
new_occ->next = *p_head;
|
*p_head = new_occ;
|
*p_head = new_occ;
|
}
|
}
|
|
|
/* Register that we found a division in BB. */
|
/* Register that we found a division in BB. */
|
|
|
static inline void
|
static inline void
|
register_division_in (basic_block bb)
|
register_division_in (basic_block bb)
|
{
|
{
|
struct occurrence *occ;
|
struct occurrence *occ;
|
|
|
occ = (struct occurrence *) bb->aux;
|
occ = (struct occurrence *) bb->aux;
|
if (!occ)
|
if (!occ)
|
{
|
{
|
occ = occ_new (bb, NULL);
|
occ = occ_new (bb, NULL);
|
insert_bb (occ, ENTRY_BLOCK_PTR, &occ_head);
|
insert_bb (occ, ENTRY_BLOCK_PTR, &occ_head);
|
}
|
}
|
|
|
occ->bb_has_division = true;
|
occ->bb_has_division = true;
|
occ->num_divisions++;
|
occ->num_divisions++;
|
}
|
}
|
|
|
|
|
/* Compute the number of divisions that postdominate each block in OCC and
|
/* Compute the number of divisions that postdominate each block in OCC and
|
its children. */
|
its children. */
|
|
|
static void
|
static void
|
compute_merit (struct occurrence *occ)
|
compute_merit (struct occurrence *occ)
|
{
|
{
|
struct occurrence *occ_child;
|
struct occurrence *occ_child;
|
basic_block dom = occ->bb;
|
basic_block dom = occ->bb;
|
|
|
for (occ_child = occ->children; occ_child; occ_child = occ_child->next)
|
for (occ_child = occ->children; occ_child; occ_child = occ_child->next)
|
{
|
{
|
basic_block bb;
|
basic_block bb;
|
if (occ_child->children)
|
if (occ_child->children)
|
compute_merit (occ_child);
|
compute_merit (occ_child);
|
|
|
if (flag_exceptions)
|
if (flag_exceptions)
|
bb = single_noncomplex_succ (dom);
|
bb = single_noncomplex_succ (dom);
|
else
|
else
|
bb = dom;
|
bb = dom;
|
|
|
if (dominated_by_p (CDI_POST_DOMINATORS, bb, occ_child->bb))
|
if (dominated_by_p (CDI_POST_DOMINATORS, bb, occ_child->bb))
|
occ->num_divisions += occ_child->num_divisions;
|
occ->num_divisions += occ_child->num_divisions;
|
}
|
}
|
}
|
}
|
|
|
|
|
/* Return whether USE_STMT is a floating-point division by DEF. */
|
/* Return whether USE_STMT is a floating-point division by DEF. */
|
static inline bool
|
static inline bool
|
is_division_by (gimple use_stmt, tree def)
|
is_division_by (gimple use_stmt, tree def)
|
{
|
{
|
return is_gimple_assign (use_stmt)
|
return is_gimple_assign (use_stmt)
|
&& gimple_assign_rhs_code (use_stmt) == RDIV_EXPR
|
&& gimple_assign_rhs_code (use_stmt) == RDIV_EXPR
|
&& gimple_assign_rhs2 (use_stmt) == def
|
&& gimple_assign_rhs2 (use_stmt) == def
|
/* Do not recognize x / x as valid division, as we are getting
|
/* Do not recognize x / x as valid division, as we are getting
|
confused later by replacing all immediate uses x in such
|
confused later by replacing all immediate uses x in such
|
a stmt. */
|
a stmt. */
|
&& gimple_assign_rhs1 (use_stmt) != def;
|
&& gimple_assign_rhs1 (use_stmt) != def;
|
}
|
}
|
|
|
/* Walk the subset of the dominator tree rooted at OCC, setting the
|
/* Walk the subset of the dominator tree rooted at OCC, setting the
|
RECIP_DEF field to a definition of 1.0 / DEF that can be used in
|
RECIP_DEF field to a definition of 1.0 / DEF that can be used in
|
the given basic block. The field may be left NULL, of course,
|
the given basic block. The field may be left NULL, of course,
|
if it is not possible or profitable to do the optimization.
|
if it is not possible or profitable to do the optimization.
|
|
|
DEF_BSI is an iterator pointing at the statement defining DEF.
|
DEF_BSI is an iterator pointing at the statement defining DEF.
|
If RECIP_DEF is set, a dominator already has a computation that can
|
If RECIP_DEF is set, a dominator already has a computation that can
|
be used. */
|
be used. */
|
|
|
static void
|
static void
|
insert_reciprocals (gimple_stmt_iterator *def_gsi, struct occurrence *occ,
|
insert_reciprocals (gimple_stmt_iterator *def_gsi, struct occurrence *occ,
|
tree def, tree recip_def, int threshold)
|
tree def, tree recip_def, int threshold)
|
{
|
{
|
tree type;
|
tree type;
|
gimple new_stmt;
|
gimple new_stmt;
|
gimple_stmt_iterator gsi;
|
gimple_stmt_iterator gsi;
|
struct occurrence *occ_child;
|
struct occurrence *occ_child;
|
|
|
if (!recip_def
|
if (!recip_def
|
&& (occ->bb_has_division || !flag_trapping_math)
|
&& (occ->bb_has_division || !flag_trapping_math)
|
&& occ->num_divisions >= threshold)
|
&& occ->num_divisions >= threshold)
|
{
|
{
|
/* Make a variable with the replacement and substitute it. */
|
/* Make a variable with the replacement and substitute it. */
|
type = TREE_TYPE (def);
|
type = TREE_TYPE (def);
|
recip_def = make_rename_temp (type, "reciptmp");
|
recip_def = make_rename_temp (type, "reciptmp");
|
new_stmt = gimple_build_assign_with_ops (RDIV_EXPR, recip_def,
|
new_stmt = gimple_build_assign_with_ops (RDIV_EXPR, recip_def,
|
build_one_cst (type), def);
|
build_one_cst (type), def);
|
|
|
if (occ->bb_has_division)
|
if (occ->bb_has_division)
|
{
|
{
|
/* Case 1: insert before an existing division. */
|
/* Case 1: insert before an existing division. */
|
gsi = gsi_after_labels (occ->bb);
|
gsi = gsi_after_labels (occ->bb);
|
while (!gsi_end_p (gsi) && !is_division_by (gsi_stmt (gsi), def))
|
while (!gsi_end_p (gsi) && !is_division_by (gsi_stmt (gsi), def))
|
gsi_next (&gsi);
|
gsi_next (&gsi);
|
|
|
gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
|
gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
|
}
|
}
|
else if (def_gsi && occ->bb == def_gsi->bb)
|
else if (def_gsi && occ->bb == def_gsi->bb)
|
{
|
{
|
/* Case 2: insert right after the definition. Note that this will
|
/* Case 2: insert right after the definition. Note that this will
|
never happen if the definition statement can throw, because in
|
never happen if the definition statement can throw, because in
|
that case the sole successor of the statement's basic block will
|
that case the sole successor of the statement's basic block will
|
dominate all the uses as well. */
|
dominate all the uses as well. */
|
gsi_insert_after (def_gsi, new_stmt, GSI_NEW_STMT);
|
gsi_insert_after (def_gsi, new_stmt, GSI_NEW_STMT);
|
}
|
}
|
else
|
else
|
{
|
{
|
/* Case 3: insert in a basic block not containing defs/uses. */
|
/* Case 3: insert in a basic block not containing defs/uses. */
|
gsi = gsi_after_labels (occ->bb);
|
gsi = gsi_after_labels (occ->bb);
|
gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
|
gsi_insert_before (&gsi, new_stmt, GSI_SAME_STMT);
|
}
|
}
|
|
|
reciprocal_stats.rdivs_inserted++;
|
reciprocal_stats.rdivs_inserted++;
|
|
|
occ->recip_def_stmt = new_stmt;
|
occ->recip_def_stmt = new_stmt;
|
}
|
}
|
|
|
occ->recip_def = recip_def;
|
occ->recip_def = recip_def;
|
for (occ_child = occ->children; occ_child; occ_child = occ_child->next)
|
for (occ_child = occ->children; occ_child; occ_child = occ_child->next)
|
insert_reciprocals (def_gsi, occ_child, def, recip_def, threshold);
|
insert_reciprocals (def_gsi, occ_child, def, recip_def, threshold);
|
}
|
}
|
|
|
|
|
/* Replace the division at USE_P with a multiplication by the reciprocal, if
|
/* Replace the division at USE_P with a multiplication by the reciprocal, if
|
possible. */
|
possible. */
|
|
|
static inline void
|
static inline void
|
replace_reciprocal (use_operand_p use_p)
|
replace_reciprocal (use_operand_p use_p)
|
{
|
{
|
gimple use_stmt = USE_STMT (use_p);
|
gimple use_stmt = USE_STMT (use_p);
|
basic_block bb = gimple_bb (use_stmt);
|
basic_block bb = gimple_bb (use_stmt);
|
struct occurrence *occ = (struct occurrence *) bb->aux;
|
struct occurrence *occ = (struct occurrence *) bb->aux;
|
|
|
if (optimize_bb_for_speed_p (bb)
|
if (optimize_bb_for_speed_p (bb)
|
&& occ->recip_def && use_stmt != occ->recip_def_stmt)
|
&& occ->recip_def && use_stmt != occ->recip_def_stmt)
|
{
|
{
|
gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
|
gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
|
gimple_assign_set_rhs_code (use_stmt, MULT_EXPR);
|
gimple_assign_set_rhs_code (use_stmt, MULT_EXPR);
|
SET_USE (use_p, occ->recip_def);
|
SET_USE (use_p, occ->recip_def);
|
fold_stmt_inplace (&gsi);
|
fold_stmt_inplace (&gsi);
|
update_stmt (use_stmt);
|
update_stmt (use_stmt);
|
}
|
}
|
}
|
}
|
|
|
|
|
/* Free OCC and return one more "struct occurrence" to be freed. */
|
/* Free OCC and return one more "struct occurrence" to be freed. */
|
|
|
static struct occurrence *
|
static struct occurrence *
|
free_bb (struct occurrence *occ)
|
free_bb (struct occurrence *occ)
|
{
|
{
|
struct occurrence *child, *next;
|
struct occurrence *child, *next;
|
|
|
/* First get the two pointers hanging off OCC. */
|
/* First get the two pointers hanging off OCC. */
|
next = occ->next;
|
next = occ->next;
|
child = occ->children;
|
child = occ->children;
|
occ->bb->aux = NULL;
|
occ->bb->aux = NULL;
|
pool_free (occ_pool, occ);
|
pool_free (occ_pool, occ);
|
|
|
/* Now ensure that we don't recurse unless it is necessary. */
|
/* Now ensure that we don't recurse unless it is necessary. */
|
if (!child)
|
if (!child)
|
return next;
|
return next;
|
else
|
else
|
{
|
{
|
while (next)
|
while (next)
|
next = free_bb (next);
|
next = free_bb (next);
|
|
|
return child;
|
return child;
|
}
|
}
|
}
|
}
|
|
|
|
|
/* Look for floating-point divisions among DEF's uses, and try to
|
/* Look for floating-point divisions among DEF's uses, and try to
|
replace them by multiplications with the reciprocal. Add
|
replace them by multiplications with the reciprocal. Add
|
as many statements computing the reciprocal as needed.
|
as many statements computing the reciprocal as needed.
|
|
|
DEF must be a GIMPLE register of a floating-point type. */
|
DEF must be a GIMPLE register of a floating-point type. */
|
|
|
static void
|
static void
|
execute_cse_reciprocals_1 (gimple_stmt_iterator *def_gsi, tree def)
|
execute_cse_reciprocals_1 (gimple_stmt_iterator *def_gsi, tree def)
|
{
|
{
|
use_operand_p use_p;
|
use_operand_p use_p;
|
imm_use_iterator use_iter;
|
imm_use_iterator use_iter;
|
struct occurrence *occ;
|
struct occurrence *occ;
|
int count = 0, threshold;
|
int count = 0, threshold;
|
|
|
gcc_assert (FLOAT_TYPE_P (TREE_TYPE (def)) && is_gimple_reg (def));
|
gcc_assert (FLOAT_TYPE_P (TREE_TYPE (def)) && is_gimple_reg (def));
|
|
|
FOR_EACH_IMM_USE_FAST (use_p, use_iter, def)
|
FOR_EACH_IMM_USE_FAST (use_p, use_iter, def)
|
{
|
{
|
gimple use_stmt = USE_STMT (use_p);
|
gimple use_stmt = USE_STMT (use_p);
|
if (is_division_by (use_stmt, def))
|
if (is_division_by (use_stmt, def))
|
{
|
{
|
register_division_in (gimple_bb (use_stmt));
|
register_division_in (gimple_bb (use_stmt));
|
count++;
|
count++;
|
}
|
}
|
}
|
}
|
|
|
/* Do the expensive part only if we can hope to optimize something. */
|
/* Do the expensive part only if we can hope to optimize something. */
|
threshold = targetm.min_divisions_for_recip_mul (TYPE_MODE (TREE_TYPE (def)));
|
threshold = targetm.min_divisions_for_recip_mul (TYPE_MODE (TREE_TYPE (def)));
|
if (count >= threshold)
|
if (count >= threshold)
|
{
|
{
|
gimple use_stmt;
|
gimple use_stmt;
|
for (occ = occ_head; occ; occ = occ->next)
|
for (occ = occ_head; occ; occ = occ->next)
|
{
|
{
|
compute_merit (occ);
|
compute_merit (occ);
|
insert_reciprocals (def_gsi, occ, def, NULL, threshold);
|
insert_reciprocals (def_gsi, occ, def, NULL, threshold);
|
}
|
}
|
|
|
FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, def)
|
FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, def)
|
{
|
{
|
if (is_division_by (use_stmt, def))
|
if (is_division_by (use_stmt, def))
|
{
|
{
|
FOR_EACH_IMM_USE_ON_STMT (use_p, use_iter)
|
FOR_EACH_IMM_USE_ON_STMT (use_p, use_iter)
|
replace_reciprocal (use_p);
|
replace_reciprocal (use_p);
|
}
|
}
|
}
|
}
|
}
|
}
|
|
|
for (occ = occ_head; occ; )
|
for (occ = occ_head; occ; )
|
occ = free_bb (occ);
|
occ = free_bb (occ);
|
|
|
occ_head = NULL;
|
occ_head = NULL;
|
}
|
}
|
|
|
static bool
|
static bool
|
gate_cse_reciprocals (void)
|
gate_cse_reciprocals (void)
|
{
|
{
|
return optimize && flag_reciprocal_math;
|
return optimize && flag_reciprocal_math;
|
}
|
}
|
|
|
/* Go through all the floating-point SSA_NAMEs, and call
|
/* Go through all the floating-point SSA_NAMEs, and call
|
execute_cse_reciprocals_1 on each of them. */
|
execute_cse_reciprocals_1 on each of them. */
|
static unsigned int
|
static unsigned int
|
execute_cse_reciprocals (void)
|
execute_cse_reciprocals (void)
|
{
|
{
|
basic_block bb;
|
basic_block bb;
|
tree arg;
|
tree arg;
|
|
|
occ_pool = create_alloc_pool ("dominators for recip",
|
occ_pool = create_alloc_pool ("dominators for recip",
|
sizeof (struct occurrence),
|
sizeof (struct occurrence),
|
n_basic_blocks / 3 + 1);
|
n_basic_blocks / 3 + 1);
|
|
|
memset (&reciprocal_stats, 0, sizeof (reciprocal_stats));
|
memset (&reciprocal_stats, 0, sizeof (reciprocal_stats));
|
calculate_dominance_info (CDI_DOMINATORS);
|
calculate_dominance_info (CDI_DOMINATORS);
|
calculate_dominance_info (CDI_POST_DOMINATORS);
|
calculate_dominance_info (CDI_POST_DOMINATORS);
|
|
|
#ifdef ENABLE_CHECKING
|
#ifdef ENABLE_CHECKING
|
FOR_EACH_BB (bb)
|
FOR_EACH_BB (bb)
|
gcc_assert (!bb->aux);
|
gcc_assert (!bb->aux);
|
#endif
|
#endif
|
|
|
for (arg = DECL_ARGUMENTS (cfun->decl); arg; arg = DECL_CHAIN (arg))
|
for (arg = DECL_ARGUMENTS (cfun->decl); arg; arg = DECL_CHAIN (arg))
|
if (gimple_default_def (cfun, arg)
|
if (gimple_default_def (cfun, arg)
|
&& FLOAT_TYPE_P (TREE_TYPE (arg))
|
&& FLOAT_TYPE_P (TREE_TYPE (arg))
|
&& is_gimple_reg (arg))
|
&& is_gimple_reg (arg))
|
execute_cse_reciprocals_1 (NULL, gimple_default_def (cfun, arg));
|
execute_cse_reciprocals_1 (NULL, gimple_default_def (cfun, arg));
|
|
|
FOR_EACH_BB (bb)
|
FOR_EACH_BB (bb)
|
{
|
{
|
gimple_stmt_iterator gsi;
|
gimple_stmt_iterator gsi;
|
gimple phi;
|
gimple phi;
|
tree def;
|
tree def;
|
|
|
for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
{
|
{
|
phi = gsi_stmt (gsi);
|
phi = gsi_stmt (gsi);
|
def = PHI_RESULT (phi);
|
def = PHI_RESULT (phi);
|
if (FLOAT_TYPE_P (TREE_TYPE (def))
|
if (FLOAT_TYPE_P (TREE_TYPE (def))
|
&& is_gimple_reg (def))
|
&& is_gimple_reg (def))
|
execute_cse_reciprocals_1 (NULL, def);
|
execute_cse_reciprocals_1 (NULL, def);
|
}
|
}
|
|
|
for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
{
|
{
|
gimple stmt = gsi_stmt (gsi);
|
gimple stmt = gsi_stmt (gsi);
|
|
|
if (gimple_has_lhs (stmt)
|
if (gimple_has_lhs (stmt)
|
&& (def = SINGLE_SSA_TREE_OPERAND (stmt, SSA_OP_DEF)) != NULL
|
&& (def = SINGLE_SSA_TREE_OPERAND (stmt, SSA_OP_DEF)) != NULL
|
&& FLOAT_TYPE_P (TREE_TYPE (def))
|
&& FLOAT_TYPE_P (TREE_TYPE (def))
|
&& TREE_CODE (def) == SSA_NAME)
|
&& TREE_CODE (def) == SSA_NAME)
|
execute_cse_reciprocals_1 (&gsi, def);
|
execute_cse_reciprocals_1 (&gsi, def);
|
}
|
}
|
|
|
if (optimize_bb_for_size_p (bb))
|
if (optimize_bb_for_size_p (bb))
|
continue;
|
continue;
|
|
|
/* Scan for a/func(b) and convert it to reciprocal a*rfunc(b). */
|
/* Scan for a/func(b) and convert it to reciprocal a*rfunc(b). */
|
for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
{
|
{
|
gimple stmt = gsi_stmt (gsi);
|
gimple stmt = gsi_stmt (gsi);
|
tree fndecl;
|
tree fndecl;
|
|
|
if (is_gimple_assign (stmt)
|
if (is_gimple_assign (stmt)
|
&& gimple_assign_rhs_code (stmt) == RDIV_EXPR)
|
&& gimple_assign_rhs_code (stmt) == RDIV_EXPR)
|
{
|
{
|
tree arg1 = gimple_assign_rhs2 (stmt);
|
tree arg1 = gimple_assign_rhs2 (stmt);
|
gimple stmt1;
|
gimple stmt1;
|
|
|
if (TREE_CODE (arg1) != SSA_NAME)
|
if (TREE_CODE (arg1) != SSA_NAME)
|
continue;
|
continue;
|
|
|
stmt1 = SSA_NAME_DEF_STMT (arg1);
|
stmt1 = SSA_NAME_DEF_STMT (arg1);
|
|
|
if (is_gimple_call (stmt1)
|
if (is_gimple_call (stmt1)
|
&& gimple_call_lhs (stmt1)
|
&& gimple_call_lhs (stmt1)
|
&& (fndecl = gimple_call_fndecl (stmt1))
|
&& (fndecl = gimple_call_fndecl (stmt1))
|
&& (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
|
&& (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
|
|| DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD))
|
|| DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD))
|
{
|
{
|
enum built_in_function code;
|
enum built_in_function code;
|
bool md_code, fail;
|
bool md_code, fail;
|
imm_use_iterator ui;
|
imm_use_iterator ui;
|
use_operand_p use_p;
|
use_operand_p use_p;
|
|
|
code = DECL_FUNCTION_CODE (fndecl);
|
code = DECL_FUNCTION_CODE (fndecl);
|
md_code = DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD;
|
md_code = DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD;
|
|
|
fndecl = targetm.builtin_reciprocal (code, md_code, false);
|
fndecl = targetm.builtin_reciprocal (code, md_code, false);
|
if (!fndecl)
|
if (!fndecl)
|
continue;
|
continue;
|
|
|
/* Check that all uses of the SSA name are divisions,
|
/* Check that all uses of the SSA name are divisions,
|
otherwise replacing the defining statement will do
|
otherwise replacing the defining statement will do
|
the wrong thing. */
|
the wrong thing. */
|
fail = false;
|
fail = false;
|
FOR_EACH_IMM_USE_FAST (use_p, ui, arg1)
|
FOR_EACH_IMM_USE_FAST (use_p, ui, arg1)
|
{
|
{
|
gimple stmt2 = USE_STMT (use_p);
|
gimple stmt2 = USE_STMT (use_p);
|
if (is_gimple_debug (stmt2))
|
if (is_gimple_debug (stmt2))
|
continue;
|
continue;
|
if (!is_gimple_assign (stmt2)
|
if (!is_gimple_assign (stmt2)
|
|| gimple_assign_rhs_code (stmt2) != RDIV_EXPR
|
|| gimple_assign_rhs_code (stmt2) != RDIV_EXPR
|
|| gimple_assign_rhs1 (stmt2) == arg1
|
|| gimple_assign_rhs1 (stmt2) == arg1
|
|| gimple_assign_rhs2 (stmt2) != arg1)
|
|| gimple_assign_rhs2 (stmt2) != arg1)
|
{
|
{
|
fail = true;
|
fail = true;
|
break;
|
break;
|
}
|
}
|
}
|
}
|
if (fail)
|
if (fail)
|
continue;
|
continue;
|
|
|
gimple_replace_lhs (stmt1, arg1);
|
gimple_replace_lhs (stmt1, arg1);
|
gimple_call_set_fndecl (stmt1, fndecl);
|
gimple_call_set_fndecl (stmt1, fndecl);
|
update_stmt (stmt1);
|
update_stmt (stmt1);
|
reciprocal_stats.rfuncs_inserted++;
|
reciprocal_stats.rfuncs_inserted++;
|
|
|
FOR_EACH_IMM_USE_STMT (stmt, ui, arg1)
|
FOR_EACH_IMM_USE_STMT (stmt, ui, arg1)
|
{
|
{
|
gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
|
gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
|
gimple_assign_set_rhs_code (stmt, MULT_EXPR);
|
gimple_assign_set_rhs_code (stmt, MULT_EXPR);
|
fold_stmt_inplace (&gsi);
|
fold_stmt_inplace (&gsi);
|
update_stmt (stmt);
|
update_stmt (stmt);
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
|
|
statistics_counter_event (cfun, "reciprocal divs inserted",
|
statistics_counter_event (cfun, "reciprocal divs inserted",
|
reciprocal_stats.rdivs_inserted);
|
reciprocal_stats.rdivs_inserted);
|
statistics_counter_event (cfun, "reciprocal functions inserted",
|
statistics_counter_event (cfun, "reciprocal functions inserted",
|
reciprocal_stats.rfuncs_inserted);
|
reciprocal_stats.rfuncs_inserted);
|
|
|
free_dominance_info (CDI_DOMINATORS);
|
free_dominance_info (CDI_DOMINATORS);
|
free_dominance_info (CDI_POST_DOMINATORS);
|
free_dominance_info (CDI_POST_DOMINATORS);
|
free_alloc_pool (occ_pool);
|
free_alloc_pool (occ_pool);
|
return 0;
|
return 0;
|
}
|
}
|
|
|
struct gimple_opt_pass pass_cse_reciprocals =
|
struct gimple_opt_pass pass_cse_reciprocals =
|
{
|
{
|
{
|
{
|
GIMPLE_PASS,
|
GIMPLE_PASS,
|
"recip", /* name */
|
"recip", /* name */
|
gate_cse_reciprocals, /* gate */
|
gate_cse_reciprocals, /* gate */
|
execute_cse_reciprocals, /* execute */
|
execute_cse_reciprocals, /* execute */
|
NULL, /* sub */
|
NULL, /* sub */
|
NULL, /* next */
|
NULL, /* next */
|
0, /* static_pass_number */
|
0, /* static_pass_number */
|
TV_NONE, /* tv_id */
|
TV_NONE, /* tv_id */
|
PROP_ssa, /* properties_required */
|
PROP_ssa, /* properties_required */
|
0, /* properties_provided */
|
0, /* properties_provided */
|
0, /* properties_destroyed */
|
0, /* properties_destroyed */
|
0, /* todo_flags_start */
|
0, /* todo_flags_start */
|
TODO_update_ssa | TODO_verify_ssa
|
TODO_update_ssa | TODO_verify_ssa
|
| TODO_verify_stmts /* todo_flags_finish */
|
| TODO_verify_stmts /* todo_flags_finish */
|
}
|
}
|
};
|
};
|
|
|
/* Records an occurrence at statement USE_STMT in the vector of trees
|
/* Records an occurrence at statement USE_STMT in the vector of trees
|
STMTS if it is dominated by *TOP_BB or dominates it or this basic block
|
STMTS if it is dominated by *TOP_BB or dominates it or this basic block
|
is not yet initialized. Returns true if the occurrence was pushed on
|
is not yet initialized. Returns true if the occurrence was pushed on
|
the vector. Adjusts *TOP_BB to be the basic block dominating all
|
the vector. Adjusts *TOP_BB to be the basic block dominating all
|
statements in the vector. */
|
statements in the vector. */
|
|
|
static bool
|
static bool
|
maybe_record_sincos (VEC(gimple, heap) **stmts,
|
maybe_record_sincos (VEC(gimple, heap) **stmts,
|
basic_block *top_bb, gimple use_stmt)
|
basic_block *top_bb, gimple use_stmt)
|
{
|
{
|
basic_block use_bb = gimple_bb (use_stmt);
|
basic_block use_bb = gimple_bb (use_stmt);
|
if (*top_bb
|
if (*top_bb
|
&& (*top_bb == use_bb
|
&& (*top_bb == use_bb
|
|| dominated_by_p (CDI_DOMINATORS, use_bb, *top_bb)))
|
|| dominated_by_p (CDI_DOMINATORS, use_bb, *top_bb)))
|
VEC_safe_push (gimple, heap, *stmts, use_stmt);
|
VEC_safe_push (gimple, heap, *stmts, use_stmt);
|
else if (!*top_bb
|
else if (!*top_bb
|
|| dominated_by_p (CDI_DOMINATORS, *top_bb, use_bb))
|
|| dominated_by_p (CDI_DOMINATORS, *top_bb, use_bb))
|
{
|
{
|
VEC_safe_push (gimple, heap, *stmts, use_stmt);
|
VEC_safe_push (gimple, heap, *stmts, use_stmt);
|
*top_bb = use_bb;
|
*top_bb = use_bb;
|
}
|
}
|
else
|
else
|
return false;
|
return false;
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
/* Look for sin, cos and cexpi calls with the same argument NAME and
|
/* Look for sin, cos and cexpi calls with the same argument NAME and
|
create a single call to cexpi CSEing the result in this case.
|
create a single call to cexpi CSEing the result in this case.
|
We first walk over all immediate uses of the argument collecting
|
We first walk over all immediate uses of the argument collecting
|
statements that we can CSE in a vector and in a second pass replace
|
statements that we can CSE in a vector and in a second pass replace
|
the statement rhs with a REALPART or IMAGPART expression on the
|
the statement rhs with a REALPART or IMAGPART expression on the
|
result of the cexpi call we insert before the use statement that
|
result of the cexpi call we insert before the use statement that
|
dominates all other candidates. */
|
dominates all other candidates. */
|
|
|
static bool
|
static bool
|
execute_cse_sincos_1 (tree name)
|
execute_cse_sincos_1 (tree name)
|
{
|
{
|
gimple_stmt_iterator gsi;
|
gimple_stmt_iterator gsi;
|
imm_use_iterator use_iter;
|
imm_use_iterator use_iter;
|
tree fndecl, res, type;
|
tree fndecl, res, type;
|
gimple def_stmt, use_stmt, stmt;
|
gimple def_stmt, use_stmt, stmt;
|
int seen_cos = 0, seen_sin = 0, seen_cexpi = 0;
|
int seen_cos = 0, seen_sin = 0, seen_cexpi = 0;
|
VEC(gimple, heap) *stmts = NULL;
|
VEC(gimple, heap) *stmts = NULL;
|
basic_block top_bb = NULL;
|
basic_block top_bb = NULL;
|
int i;
|
int i;
|
bool cfg_changed = false;
|
bool cfg_changed = false;
|
|
|
type = TREE_TYPE (name);
|
type = TREE_TYPE (name);
|
FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, name)
|
FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, name)
|
{
|
{
|
if (gimple_code (use_stmt) != GIMPLE_CALL
|
if (gimple_code (use_stmt) != GIMPLE_CALL
|
|| !gimple_call_lhs (use_stmt)
|
|| !gimple_call_lhs (use_stmt)
|
|| !(fndecl = gimple_call_fndecl (use_stmt))
|
|| !(fndecl = gimple_call_fndecl (use_stmt))
|
|| DECL_BUILT_IN_CLASS (fndecl) != BUILT_IN_NORMAL)
|
|| DECL_BUILT_IN_CLASS (fndecl) != BUILT_IN_NORMAL)
|
continue;
|
continue;
|
|
|
switch (DECL_FUNCTION_CODE (fndecl))
|
switch (DECL_FUNCTION_CODE (fndecl))
|
{
|
{
|
CASE_FLT_FN (BUILT_IN_COS):
|
CASE_FLT_FN (BUILT_IN_COS):
|
seen_cos |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
|
seen_cos |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
|
break;
|
break;
|
|
|
CASE_FLT_FN (BUILT_IN_SIN):
|
CASE_FLT_FN (BUILT_IN_SIN):
|
seen_sin |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
|
seen_sin |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
|
break;
|
break;
|
|
|
CASE_FLT_FN (BUILT_IN_CEXPI):
|
CASE_FLT_FN (BUILT_IN_CEXPI):
|
seen_cexpi |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
|
seen_cexpi |= maybe_record_sincos (&stmts, &top_bb, use_stmt) ? 1 : 0;
|
break;
|
break;
|
|
|
default:;
|
default:;
|
}
|
}
|
}
|
}
|
|
|
if (seen_cos + seen_sin + seen_cexpi <= 1)
|
if (seen_cos + seen_sin + seen_cexpi <= 1)
|
{
|
{
|
VEC_free(gimple, heap, stmts);
|
VEC_free(gimple, heap, stmts);
|
return false;
|
return false;
|
}
|
}
|
|
|
/* Simply insert cexpi at the beginning of top_bb but not earlier than
|
/* Simply insert cexpi at the beginning of top_bb but not earlier than
|
the name def statement. */
|
the name def statement. */
|
fndecl = mathfn_built_in (type, BUILT_IN_CEXPI);
|
fndecl = mathfn_built_in (type, BUILT_IN_CEXPI);
|
if (!fndecl)
|
if (!fndecl)
|
return false;
|
return false;
|
res = create_tmp_reg (TREE_TYPE (TREE_TYPE (fndecl)), "sincostmp");
|
res = create_tmp_reg (TREE_TYPE (TREE_TYPE (fndecl)), "sincostmp");
|
stmt = gimple_build_call (fndecl, 1, name);
|
stmt = gimple_build_call (fndecl, 1, name);
|
res = make_ssa_name (res, stmt);
|
res = make_ssa_name (res, stmt);
|
gimple_call_set_lhs (stmt, res);
|
gimple_call_set_lhs (stmt, res);
|
|
|
def_stmt = SSA_NAME_DEF_STMT (name);
|
def_stmt = SSA_NAME_DEF_STMT (name);
|
if (!SSA_NAME_IS_DEFAULT_DEF (name)
|
if (!SSA_NAME_IS_DEFAULT_DEF (name)
|
&& gimple_code (def_stmt) != GIMPLE_PHI
|
&& gimple_code (def_stmt) != GIMPLE_PHI
|
&& gimple_bb (def_stmt) == top_bb)
|
&& gimple_bb (def_stmt) == top_bb)
|
{
|
{
|
gsi = gsi_for_stmt (def_stmt);
|
gsi = gsi_for_stmt (def_stmt);
|
gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
|
gsi_insert_after (&gsi, stmt, GSI_SAME_STMT);
|
}
|
}
|
else
|
else
|
{
|
{
|
gsi = gsi_after_labels (top_bb);
|
gsi = gsi_after_labels (top_bb);
|
gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
|
gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
|
}
|
}
|
update_stmt (stmt);
|
update_stmt (stmt);
|
sincos_stats.inserted++;
|
sincos_stats.inserted++;
|
|
|
/* And adjust the recorded old call sites. */
|
/* And adjust the recorded old call sites. */
|
for (i = 0; VEC_iterate(gimple, stmts, i, use_stmt); ++i)
|
for (i = 0; VEC_iterate(gimple, stmts, i, use_stmt); ++i)
|
{
|
{
|
tree rhs = NULL;
|
tree rhs = NULL;
|
fndecl = gimple_call_fndecl (use_stmt);
|
fndecl = gimple_call_fndecl (use_stmt);
|
|
|
switch (DECL_FUNCTION_CODE (fndecl))
|
switch (DECL_FUNCTION_CODE (fndecl))
|
{
|
{
|
CASE_FLT_FN (BUILT_IN_COS):
|
CASE_FLT_FN (BUILT_IN_COS):
|
rhs = fold_build1 (REALPART_EXPR, type, res);
|
rhs = fold_build1 (REALPART_EXPR, type, res);
|
break;
|
break;
|
|
|
CASE_FLT_FN (BUILT_IN_SIN):
|
CASE_FLT_FN (BUILT_IN_SIN):
|
rhs = fold_build1 (IMAGPART_EXPR, type, res);
|
rhs = fold_build1 (IMAGPART_EXPR, type, res);
|
break;
|
break;
|
|
|
CASE_FLT_FN (BUILT_IN_CEXPI):
|
CASE_FLT_FN (BUILT_IN_CEXPI):
|
rhs = res;
|
rhs = res;
|
break;
|
break;
|
|
|
default:;
|
default:;
|
gcc_unreachable ();
|
gcc_unreachable ();
|
}
|
}
|
|
|
/* Replace call with a copy. */
|
/* Replace call with a copy. */
|
stmt = gimple_build_assign (gimple_call_lhs (use_stmt), rhs);
|
stmt = gimple_build_assign (gimple_call_lhs (use_stmt), rhs);
|
|
|
gsi = gsi_for_stmt (use_stmt);
|
gsi = gsi_for_stmt (use_stmt);
|
gsi_replace (&gsi, stmt, true);
|
gsi_replace (&gsi, stmt, true);
|
if (gimple_purge_dead_eh_edges (gimple_bb (stmt)))
|
if (gimple_purge_dead_eh_edges (gimple_bb (stmt)))
|
cfg_changed = true;
|
cfg_changed = true;
|
}
|
}
|
|
|
VEC_free(gimple, heap, stmts);
|
VEC_free(gimple, heap, stmts);
|
|
|
return cfg_changed;
|
return cfg_changed;
|
}
|
}
|
|
|
/* To evaluate powi(x,n), the floating point value x raised to the
|
/* To evaluate powi(x,n), the floating point value x raised to the
|
constant integer exponent n, we use a hybrid algorithm that
|
constant integer exponent n, we use a hybrid algorithm that
|
combines the "window method" with look-up tables. For an
|
combines the "window method" with look-up tables. For an
|
introduction to exponentiation algorithms and "addition chains",
|
introduction to exponentiation algorithms and "addition chains",
|
see section 4.6.3, "Evaluation of Powers" of Donald E. Knuth,
|
see section 4.6.3, "Evaluation of Powers" of Donald E. Knuth,
|
"Seminumerical Algorithms", Vol. 2, "The Art of Computer Programming",
|
"Seminumerical Algorithms", Vol. 2, "The Art of Computer Programming",
|
3rd Edition, 1998, and Daniel M. Gordon, "A Survey of Fast Exponentiation
|
3rd Edition, 1998, and Daniel M. Gordon, "A Survey of Fast Exponentiation
|
Methods", Journal of Algorithms, Vol. 27, pp. 129-146, 1998. */
|
Methods", Journal of Algorithms, Vol. 27, pp. 129-146, 1998. */
|
|
|
/* Provide a default value for POWI_MAX_MULTS, the maximum number of
|
/* Provide a default value for POWI_MAX_MULTS, the maximum number of
|
multiplications to inline before calling the system library's pow
|
multiplications to inline before calling the system library's pow
|
function. powi(x,n) requires at worst 2*bits(n)-2 multiplications,
|
function. powi(x,n) requires at worst 2*bits(n)-2 multiplications,
|
so this default never requires calling pow, powf or powl. */
|
so this default never requires calling pow, powf or powl. */
|
|
|
#ifndef POWI_MAX_MULTS
|
#ifndef POWI_MAX_MULTS
|
#define POWI_MAX_MULTS (2*HOST_BITS_PER_WIDE_INT-2)
|
#define POWI_MAX_MULTS (2*HOST_BITS_PER_WIDE_INT-2)
|
#endif
|
#endif
|
|
|
/* The size of the "optimal power tree" lookup table. All
|
/* The size of the "optimal power tree" lookup table. All
|
exponents less than this value are simply looked up in the
|
exponents less than this value are simply looked up in the
|
powi_table below. This threshold is also used to size the
|
powi_table below. This threshold is also used to size the
|
cache of pseudo registers that hold intermediate results. */
|
cache of pseudo registers that hold intermediate results. */
|
#define POWI_TABLE_SIZE 256
|
#define POWI_TABLE_SIZE 256
|
|
|
/* The size, in bits of the window, used in the "window method"
|
/* The size, in bits of the window, used in the "window method"
|
exponentiation algorithm. This is equivalent to a radix of
|
exponentiation algorithm. This is equivalent to a radix of
|
(1<<POWI_WINDOW_SIZE) in the corresponding "m-ary method". */
|
(1<<POWI_WINDOW_SIZE) in the corresponding "m-ary method". */
|
#define POWI_WINDOW_SIZE 3
|
#define POWI_WINDOW_SIZE 3
|
|
|
/* The following table is an efficient representation of an
|
/* The following table is an efficient representation of an
|
"optimal power tree". For each value, i, the corresponding
|
"optimal power tree". For each value, i, the corresponding
|
value, j, in the table states than an optimal evaluation
|
value, j, in the table states than an optimal evaluation
|
sequence for calculating pow(x,i) can be found by evaluating
|
sequence for calculating pow(x,i) can be found by evaluating
|
pow(x,j)*pow(x,i-j). An optimal power tree for the first
|
pow(x,j)*pow(x,i-j). An optimal power tree for the first
|
100 integers is given in Knuth's "Seminumerical algorithms". */
|
100 integers is given in Knuth's "Seminumerical algorithms". */
|
|
|
static const unsigned char powi_table[POWI_TABLE_SIZE] =
|
static const unsigned char powi_table[POWI_TABLE_SIZE] =
|
{
|
{
|
0, 1, 1, 2, 2, 3, 3, 4, /* 0 - 7 */
|
0, 1, 1, 2, 2, 3, 3, 4, /* 0 - 7 */
|
4, 6, 5, 6, 6, 10, 7, 9, /* 8 - 15 */
|
4, 6, 5, 6, 6, 10, 7, 9, /* 8 - 15 */
|
8, 16, 9, 16, 10, 12, 11, 13, /* 16 - 23 */
|
8, 16, 9, 16, 10, 12, 11, 13, /* 16 - 23 */
|
12, 17, 13, 18, 14, 24, 15, 26, /* 24 - 31 */
|
12, 17, 13, 18, 14, 24, 15, 26, /* 24 - 31 */
|
16, 17, 17, 19, 18, 33, 19, 26, /* 32 - 39 */
|
16, 17, 17, 19, 18, 33, 19, 26, /* 32 - 39 */
|
20, 25, 21, 40, 22, 27, 23, 44, /* 40 - 47 */
|
20, 25, 21, 40, 22, 27, 23, 44, /* 40 - 47 */
|
24, 32, 25, 34, 26, 29, 27, 44, /* 48 - 55 */
|
24, 32, 25, 34, 26, 29, 27, 44, /* 48 - 55 */
|
28, 31, 29, 34, 30, 60, 31, 36, /* 56 - 63 */
|
28, 31, 29, 34, 30, 60, 31, 36, /* 56 - 63 */
|
32, 64, 33, 34, 34, 46, 35, 37, /* 64 - 71 */
|
32, 64, 33, 34, 34, 46, 35, 37, /* 64 - 71 */
|
36, 65, 37, 50, 38, 48, 39, 69, /* 72 - 79 */
|
36, 65, 37, 50, 38, 48, 39, 69, /* 72 - 79 */
|
40, 49, 41, 43, 42, 51, 43, 58, /* 80 - 87 */
|
40, 49, 41, 43, 42, 51, 43, 58, /* 80 - 87 */
|
44, 64, 45, 47, 46, 59, 47, 76, /* 88 - 95 */
|
44, 64, 45, 47, 46, 59, 47, 76, /* 88 - 95 */
|
48, 65, 49, 66, 50, 67, 51, 66, /* 96 - 103 */
|
48, 65, 49, 66, 50, 67, 51, 66, /* 96 - 103 */
|
52, 70, 53, 74, 54, 104, 55, 74, /* 104 - 111 */
|
52, 70, 53, 74, 54, 104, 55, 74, /* 104 - 111 */
|
56, 64, 57, 69, 58, 78, 59, 68, /* 112 - 119 */
|
56, 64, 57, 69, 58, 78, 59, 68, /* 112 - 119 */
|
60, 61, 61, 80, 62, 75, 63, 68, /* 120 - 127 */
|
60, 61, 61, 80, 62, 75, 63, 68, /* 120 - 127 */
|
64, 65, 65, 128, 66, 129, 67, 90, /* 128 - 135 */
|
64, 65, 65, 128, 66, 129, 67, 90, /* 128 - 135 */
|
68, 73, 69, 131, 70, 94, 71, 88, /* 136 - 143 */
|
68, 73, 69, 131, 70, 94, 71, 88, /* 136 - 143 */
|
72, 128, 73, 98, 74, 132, 75, 121, /* 144 - 151 */
|
72, 128, 73, 98, 74, 132, 75, 121, /* 144 - 151 */
|
76, 102, 77, 124, 78, 132, 79, 106, /* 152 - 159 */
|
76, 102, 77, 124, 78, 132, 79, 106, /* 152 - 159 */
|
80, 97, 81, 160, 82, 99, 83, 134, /* 160 - 167 */
|
80, 97, 81, 160, 82, 99, 83, 134, /* 160 - 167 */
|
84, 86, 85, 95, 86, 160, 87, 100, /* 168 - 175 */
|
84, 86, 85, 95, 86, 160, 87, 100, /* 168 - 175 */
|
88, 113, 89, 98, 90, 107, 91, 122, /* 176 - 183 */
|
88, 113, 89, 98, 90, 107, 91, 122, /* 176 - 183 */
|
92, 111, 93, 102, 94, 126, 95, 150, /* 184 - 191 */
|
92, 111, 93, 102, 94, 126, 95, 150, /* 184 - 191 */
|
96, 128, 97, 130, 98, 133, 99, 195, /* 192 - 199 */
|
96, 128, 97, 130, 98, 133, 99, 195, /* 192 - 199 */
|
100, 128, 101, 123, 102, 164, 103, 138, /* 200 - 207 */
|
100, 128, 101, 123, 102, 164, 103, 138, /* 200 - 207 */
|
104, 145, 105, 146, 106, 109, 107, 149, /* 208 - 215 */
|
104, 145, 105, 146, 106, 109, 107, 149, /* 208 - 215 */
|
108, 200, 109, 146, 110, 170, 111, 157, /* 216 - 223 */
|
108, 200, 109, 146, 110, 170, 111, 157, /* 216 - 223 */
|
112, 128, 113, 130, 114, 182, 115, 132, /* 224 - 231 */
|
112, 128, 113, 130, 114, 182, 115, 132, /* 224 - 231 */
|
116, 200, 117, 132, 118, 158, 119, 206, /* 232 - 239 */
|
116, 200, 117, 132, 118, 158, 119, 206, /* 232 - 239 */
|
120, 240, 121, 162, 122, 147, 123, 152, /* 240 - 247 */
|
120, 240, 121, 162, 122, 147, 123, 152, /* 240 - 247 */
|
124, 166, 125, 214, 126, 138, 127, 153, /* 248 - 255 */
|
124, 166, 125, 214, 126, 138, 127, 153, /* 248 - 255 */
|
};
|
};
|
|
|
|
|
/* Return the number of multiplications required to calculate
|
/* Return the number of multiplications required to calculate
|
powi(x,n) where n is less than POWI_TABLE_SIZE. This is a
|
powi(x,n) where n is less than POWI_TABLE_SIZE. This is a
|
subroutine of powi_cost. CACHE is an array indicating
|
subroutine of powi_cost. CACHE is an array indicating
|
which exponents have already been calculated. */
|
which exponents have already been calculated. */
|
|
|
static int
|
static int
|
powi_lookup_cost (unsigned HOST_WIDE_INT n, bool *cache)
|
powi_lookup_cost (unsigned HOST_WIDE_INT n, bool *cache)
|
{
|
{
|
/* If we've already calculated this exponent, then this evaluation
|
/* If we've already calculated this exponent, then this evaluation
|
doesn't require any additional multiplications. */
|
doesn't require any additional multiplications. */
|
if (cache[n])
|
if (cache[n])
|
return 0;
|
return 0;
|
|
|
cache[n] = true;
|
cache[n] = true;
|
return powi_lookup_cost (n - powi_table[n], cache)
|
return powi_lookup_cost (n - powi_table[n], cache)
|
+ powi_lookup_cost (powi_table[n], cache) + 1;
|
+ powi_lookup_cost (powi_table[n], cache) + 1;
|
}
|
}
|
|
|
/* Return the number of multiplications required to calculate
|
/* Return the number of multiplications required to calculate
|
powi(x,n) for an arbitrary x, given the exponent N. This
|
powi(x,n) for an arbitrary x, given the exponent N. This
|
function needs to be kept in sync with powi_as_mults below. */
|
function needs to be kept in sync with powi_as_mults below. */
|
|
|
static int
|
static int
|
powi_cost (HOST_WIDE_INT n)
|
powi_cost (HOST_WIDE_INT n)
|
{
|
{
|
bool cache[POWI_TABLE_SIZE];
|
bool cache[POWI_TABLE_SIZE];
|
unsigned HOST_WIDE_INT digit;
|
unsigned HOST_WIDE_INT digit;
|
unsigned HOST_WIDE_INT val;
|
unsigned HOST_WIDE_INT val;
|
int result;
|
int result;
|
|
|
if (n == 0)
|
if (n == 0)
|
return 0;
|
return 0;
|
|
|
/* Ignore the reciprocal when calculating the cost. */
|
/* Ignore the reciprocal when calculating the cost. */
|
val = (n < 0) ? -n : n;
|
val = (n < 0) ? -n : n;
|
|
|
/* Initialize the exponent cache. */
|
/* Initialize the exponent cache. */
|
memset (cache, 0, POWI_TABLE_SIZE * sizeof (bool));
|
memset (cache, 0, POWI_TABLE_SIZE * sizeof (bool));
|
cache[1] = true;
|
cache[1] = true;
|
|
|
result = 0;
|
result = 0;
|
|
|
while (val >= POWI_TABLE_SIZE)
|
while (val >= POWI_TABLE_SIZE)
|
{
|
{
|
if (val & 1)
|
if (val & 1)
|
{
|
{
|
digit = val & ((1 << POWI_WINDOW_SIZE) - 1);
|
digit = val & ((1 << POWI_WINDOW_SIZE) - 1);
|
result += powi_lookup_cost (digit, cache)
|
result += powi_lookup_cost (digit, cache)
|
+ POWI_WINDOW_SIZE + 1;
|
+ POWI_WINDOW_SIZE + 1;
|
val >>= POWI_WINDOW_SIZE;
|
val >>= POWI_WINDOW_SIZE;
|
}
|
}
|
else
|
else
|
{
|
{
|
val >>= 1;
|
val >>= 1;
|
result++;
|
result++;
|
}
|
}
|
}
|
}
|
|
|
return result + powi_lookup_cost (val, cache);
|
return result + powi_lookup_cost (val, cache);
|
}
|
}
|
|
|
/* Recursive subroutine of powi_as_mults. This function takes the
|
/* Recursive subroutine of powi_as_mults. This function takes the
|
array, CACHE, of already calculated exponents and an exponent N and
|
array, CACHE, of already calculated exponents and an exponent N and
|
returns a tree that corresponds to CACHE[1]**N, with type TYPE. */
|
returns a tree that corresponds to CACHE[1]**N, with type TYPE. */
|
|
|
static tree
|
static tree
|
powi_as_mults_1 (gimple_stmt_iterator *gsi, location_t loc, tree type,
|
powi_as_mults_1 (gimple_stmt_iterator *gsi, location_t loc, tree type,
|
HOST_WIDE_INT n, tree *cache, tree target)
|
HOST_WIDE_INT n, tree *cache, tree target)
|
{
|
{
|
tree op0, op1, ssa_target;
|
tree op0, op1, ssa_target;
|
unsigned HOST_WIDE_INT digit;
|
unsigned HOST_WIDE_INT digit;
|
gimple mult_stmt;
|
gimple mult_stmt;
|
|
|
if (n < POWI_TABLE_SIZE && cache[n])
|
if (n < POWI_TABLE_SIZE && cache[n])
|
return cache[n];
|
return cache[n];
|
|
|
ssa_target = make_ssa_name (target, NULL);
|
ssa_target = make_ssa_name (target, NULL);
|
|
|
if (n < POWI_TABLE_SIZE)
|
if (n < POWI_TABLE_SIZE)
|
{
|
{
|
cache[n] = ssa_target;
|
cache[n] = ssa_target;
|
op0 = powi_as_mults_1 (gsi, loc, type, n - powi_table[n], cache, target);
|
op0 = powi_as_mults_1 (gsi, loc, type, n - powi_table[n], cache, target);
|
op1 = powi_as_mults_1 (gsi, loc, type, powi_table[n], cache, target);
|
op1 = powi_as_mults_1 (gsi, loc, type, powi_table[n], cache, target);
|
}
|
}
|
else if (n & 1)
|
else if (n & 1)
|
{
|
{
|
digit = n & ((1 << POWI_WINDOW_SIZE) - 1);
|
digit = n & ((1 << POWI_WINDOW_SIZE) - 1);
|
op0 = powi_as_mults_1 (gsi, loc, type, n - digit, cache, target);
|
op0 = powi_as_mults_1 (gsi, loc, type, n - digit, cache, target);
|
op1 = powi_as_mults_1 (gsi, loc, type, digit, cache, target);
|
op1 = powi_as_mults_1 (gsi, loc, type, digit, cache, target);
|
}
|
}
|
else
|
else
|
{
|
{
|
op0 = powi_as_mults_1 (gsi, loc, type, n >> 1, cache, target);
|
op0 = powi_as_mults_1 (gsi, loc, type, n >> 1, cache, target);
|
op1 = op0;
|
op1 = op0;
|
}
|
}
|
|
|
mult_stmt = gimple_build_assign_with_ops (MULT_EXPR, ssa_target, op0, op1);
|
mult_stmt = gimple_build_assign_with_ops (MULT_EXPR, ssa_target, op0, op1);
|
gimple_set_location (mult_stmt, loc);
|
gimple_set_location (mult_stmt, loc);
|
gsi_insert_before (gsi, mult_stmt, GSI_SAME_STMT);
|
gsi_insert_before (gsi, mult_stmt, GSI_SAME_STMT);
|
|
|
return ssa_target;
|
return ssa_target;
|
}
|
}
|
|
|
/* Convert ARG0**N to a tree of multiplications of ARG0 with itself.
|
/* Convert ARG0**N to a tree of multiplications of ARG0 with itself.
|
This function needs to be kept in sync with powi_cost above. */
|
This function needs to be kept in sync with powi_cost above. */
|
|
|
static tree
|
static tree
|
powi_as_mults (gimple_stmt_iterator *gsi, location_t loc,
|
powi_as_mults (gimple_stmt_iterator *gsi, location_t loc,
|
tree arg0, HOST_WIDE_INT n)
|
tree arg0, HOST_WIDE_INT n)
|
{
|
{
|
tree cache[POWI_TABLE_SIZE], result, type = TREE_TYPE (arg0), target;
|
tree cache[POWI_TABLE_SIZE], result, type = TREE_TYPE (arg0), target;
|
gimple div_stmt;
|
gimple div_stmt;
|
|
|
if (n == 0)
|
if (n == 0)
|
return build_real (type, dconst1);
|
return build_real (type, dconst1);
|
|
|
memset (cache, 0, sizeof (cache));
|
memset (cache, 0, sizeof (cache));
|
cache[1] = arg0;
|
cache[1] = arg0;
|
|
|
target = create_tmp_reg (type, "powmult");
|
target = create_tmp_reg (type, "powmult");
|
add_referenced_var (target);
|
add_referenced_var (target);
|
|
|
result = powi_as_mults_1 (gsi, loc, type, (n < 0) ? -n : n, cache, target);
|
result = powi_as_mults_1 (gsi, loc, type, (n < 0) ? -n : n, cache, target);
|
|
|
if (n >= 0)
|
if (n >= 0)
|
return result;
|
return result;
|
|
|
/* If the original exponent was negative, reciprocate the result. */
|
/* If the original exponent was negative, reciprocate the result. */
|
target = make_ssa_name (target, NULL);
|
target = make_ssa_name (target, NULL);
|
div_stmt = gimple_build_assign_with_ops (RDIV_EXPR, target,
|
div_stmt = gimple_build_assign_with_ops (RDIV_EXPR, target,
|
build_real (type, dconst1),
|
build_real (type, dconst1),
|
result);
|
result);
|
gimple_set_location (div_stmt, loc);
|
gimple_set_location (div_stmt, loc);
|
gsi_insert_before (gsi, div_stmt, GSI_SAME_STMT);
|
gsi_insert_before (gsi, div_stmt, GSI_SAME_STMT);
|
|
|
return target;
|
return target;
|
}
|
}
|
|
|
/* ARG0 and N are the two arguments to a powi builtin in GSI with
|
/* ARG0 and N are the two arguments to a powi builtin in GSI with
|
location info LOC. If the arguments are appropriate, create an
|
location info LOC. If the arguments are appropriate, create an
|
equivalent sequence of statements prior to GSI using an optimal
|
equivalent sequence of statements prior to GSI using an optimal
|
number of multiplications, and return an expession holding the
|
number of multiplications, and return an expession holding the
|
result. */
|
result. */
|
|
|
static tree
|
static tree
|
gimple_expand_builtin_powi (gimple_stmt_iterator *gsi, location_t loc,
|
gimple_expand_builtin_powi (gimple_stmt_iterator *gsi, location_t loc,
|
tree arg0, HOST_WIDE_INT n)
|
tree arg0, HOST_WIDE_INT n)
|
{
|
{
|
/* Avoid largest negative number. */
|
/* Avoid largest negative number. */
|
if (n != -n
|
if (n != -n
|
&& ((n >= -1 && n <= 2)
|
&& ((n >= -1 && n <= 2)
|
|| (optimize_function_for_speed_p (cfun)
|
|| (optimize_function_for_speed_p (cfun)
|
&& powi_cost (n) <= POWI_MAX_MULTS)))
|
&& powi_cost (n) <= POWI_MAX_MULTS)))
|
return powi_as_mults (gsi, loc, arg0, n);
|
return powi_as_mults (gsi, loc, arg0, n);
|
|
|
return NULL_TREE;
|
return NULL_TREE;
|
}
|
}
|
|
|
/* Build a gimple call statement that calls FN with argument ARG.
|
/* Build a gimple call statement that calls FN with argument ARG.
|
Set the lhs of the call statement to a fresh SSA name for
|
Set the lhs of the call statement to a fresh SSA name for
|
variable VAR. If VAR is NULL, first allocate it. Insert the
|
variable VAR. If VAR is NULL, first allocate it. Insert the
|
statement prior to GSI's current position, and return the fresh
|
statement prior to GSI's current position, and return the fresh
|
SSA name. */
|
SSA name. */
|
|
|
static tree
|
static tree
|
build_and_insert_call (gimple_stmt_iterator *gsi, location_t loc,
|
build_and_insert_call (gimple_stmt_iterator *gsi, location_t loc,
|
tree *var, tree fn, tree arg)
|
tree *var, tree fn, tree arg)
|
{
|
{
|
gimple call_stmt;
|
gimple call_stmt;
|
tree ssa_target;
|
tree ssa_target;
|
|
|
if (!*var)
|
if (!*var)
|
{
|
{
|
*var = create_tmp_reg (TREE_TYPE (arg), "powroot");
|
*var = create_tmp_reg (TREE_TYPE (arg), "powroot");
|
add_referenced_var (*var);
|
add_referenced_var (*var);
|
}
|
}
|
|
|
call_stmt = gimple_build_call (fn, 1, arg);
|
call_stmt = gimple_build_call (fn, 1, arg);
|
ssa_target = make_ssa_name (*var, NULL);
|
ssa_target = make_ssa_name (*var, NULL);
|
gimple_set_lhs (call_stmt, ssa_target);
|
gimple_set_lhs (call_stmt, ssa_target);
|
gimple_set_location (call_stmt, loc);
|
gimple_set_location (call_stmt, loc);
|
gsi_insert_before (gsi, call_stmt, GSI_SAME_STMT);
|
gsi_insert_before (gsi, call_stmt, GSI_SAME_STMT);
|
|
|
return ssa_target;
|
return ssa_target;
|
}
|
}
|
|
|
/* Build a gimple binary operation with the given CODE and arguments
|
/* Build a gimple binary operation with the given CODE and arguments
|
ARG0, ARG1, assigning the result to a new SSA name for variable
|
ARG0, ARG1, assigning the result to a new SSA name for variable
|
TARGET. Insert the statement prior to GSI's current position, and
|
TARGET. Insert the statement prior to GSI's current position, and
|
return the fresh SSA name.*/
|
return the fresh SSA name.*/
|
|
|
static tree
|
static tree
|
build_and_insert_binop (gimple_stmt_iterator *gsi, location_t loc,
|
build_and_insert_binop (gimple_stmt_iterator *gsi, location_t loc,
|
tree target, enum tree_code code, tree arg0, tree arg1)
|
tree target, enum tree_code code, tree arg0, tree arg1)
|
{
|
{
|
tree result = make_ssa_name (target, NULL);
|
tree result = make_ssa_name (target, NULL);
|
gimple stmt = gimple_build_assign_with_ops (code, result, arg0, arg1);
|
gimple stmt = gimple_build_assign_with_ops (code, result, arg0, arg1);
|
gimple_set_location (stmt, loc);
|
gimple_set_location (stmt, loc);
|
gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
|
gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
|
return result;
|
return result;
|
}
|
}
|
|
|
/* Build a gimple reference operation with the given CODE and argument
|
/* Build a gimple reference operation with the given CODE and argument
|
ARG, assigning the result to a new SSA name for variable TARGET.
|
ARG, assigning the result to a new SSA name for variable TARGET.
|
Insert the statement prior to GSI's current position, and return
|
Insert the statement prior to GSI's current position, and return
|
the fresh SSA name. */
|
the fresh SSA name. */
|
|
|
static inline tree
|
static inline tree
|
build_and_insert_ref (gimple_stmt_iterator *gsi, location_t loc, tree type,
|
build_and_insert_ref (gimple_stmt_iterator *gsi, location_t loc, tree type,
|
tree target, enum tree_code code, tree arg0)
|
tree target, enum tree_code code, tree arg0)
|
{
|
{
|
tree result = make_ssa_name (target, NULL);
|
tree result = make_ssa_name (target, NULL);
|
gimple stmt = gimple_build_assign (result, build1 (code, type, arg0));
|
gimple stmt = gimple_build_assign (result, build1 (code, type, arg0));
|
gimple_set_location (stmt, loc);
|
gimple_set_location (stmt, loc);
|
gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
|
gsi_insert_before (gsi, stmt, GSI_SAME_STMT);
|
return result;
|
return result;
|
}
|
}
|
|
|
/* Build a gimple assignment to cast VAL to TARGET. Insert the statement
|
/* Build a gimple assignment to cast VAL to TARGET. Insert the statement
|
prior to GSI's current position, and return the fresh SSA name. */
|
prior to GSI's current position, and return the fresh SSA name. */
|
|
|
static tree
|
static tree
|
build_and_insert_cast (gimple_stmt_iterator *gsi, location_t loc,
|
build_and_insert_cast (gimple_stmt_iterator *gsi, location_t loc,
|
tree target, tree val)
|
tree target, tree val)
|
{
|
{
|
return build_and_insert_binop (gsi, loc, target, CONVERT_EXPR, val, NULL);
|
return build_and_insert_binop (gsi, loc, target, CONVERT_EXPR, val, NULL);
|
}
|
}
|
|
|
/* ARG0 and ARG1 are the two arguments to a pow builtin call in GSI
|
/* ARG0 and ARG1 are the two arguments to a pow builtin call in GSI
|
with location info LOC. If possible, create an equivalent and
|
with location info LOC. If possible, create an equivalent and
|
less expensive sequence of statements prior to GSI, and return an
|
less expensive sequence of statements prior to GSI, and return an
|
expession holding the result. */
|
expession holding the result. */
|
|
|
static tree
|
static tree
|
gimple_expand_builtin_pow (gimple_stmt_iterator *gsi, location_t loc,
|
gimple_expand_builtin_pow (gimple_stmt_iterator *gsi, location_t loc,
|
tree arg0, tree arg1)
|
tree arg0, tree arg1)
|
{
|
{
|
REAL_VALUE_TYPE c, cint, dconst1_4, dconst3_4, dconst1_3, dconst1_6;
|
REAL_VALUE_TYPE c, cint, dconst1_4, dconst3_4, dconst1_3, dconst1_6;
|
REAL_VALUE_TYPE c2, dconst3;
|
REAL_VALUE_TYPE c2, dconst3;
|
HOST_WIDE_INT n;
|
HOST_WIDE_INT n;
|
tree type, sqrtfn, cbrtfn, sqrt_arg0, sqrt_sqrt, result, cbrt_x, powi_cbrt_x;
|
tree type, sqrtfn, cbrtfn, sqrt_arg0, sqrt_sqrt, result, cbrt_x, powi_cbrt_x;
|
tree target = NULL_TREE;
|
tree target = NULL_TREE;
|
enum machine_mode mode;
|
enum machine_mode mode;
|
bool hw_sqrt_exists;
|
bool hw_sqrt_exists;
|
|
|
/* If the exponent isn't a constant, there's nothing of interest
|
/* If the exponent isn't a constant, there's nothing of interest
|
to be done. */
|
to be done. */
|
if (TREE_CODE (arg1) != REAL_CST)
|
if (TREE_CODE (arg1) != REAL_CST)
|
return NULL_TREE;
|
return NULL_TREE;
|
|
|
/* If the exponent is equivalent to an integer, expand to an optimal
|
/* If the exponent is equivalent to an integer, expand to an optimal
|
multiplication sequence when profitable. */
|
multiplication sequence when profitable. */
|
c = TREE_REAL_CST (arg1);
|
c = TREE_REAL_CST (arg1);
|
n = real_to_integer (&c);
|
n = real_to_integer (&c);
|
real_from_integer (&cint, VOIDmode, n, n < 0 ? -1 : 0, 0);
|
real_from_integer (&cint, VOIDmode, n, n < 0 ? -1 : 0, 0);
|
|
|
if (real_identical (&c, &cint)
|
if (real_identical (&c, &cint)
|
&& ((n >= -1 && n <= 2)
|
&& ((n >= -1 && n <= 2)
|
|| (flag_unsafe_math_optimizations
|
|| (flag_unsafe_math_optimizations
|
&& optimize_insn_for_speed_p ()
|
&& optimize_insn_for_speed_p ()
|
&& powi_cost (n) <= POWI_MAX_MULTS)))
|
&& powi_cost (n) <= POWI_MAX_MULTS)))
|
return gimple_expand_builtin_powi (gsi, loc, arg0, n);
|
return gimple_expand_builtin_powi (gsi, loc, arg0, n);
|
|
|
/* Attempt various optimizations using sqrt and cbrt. */
|
/* Attempt various optimizations using sqrt and cbrt. */
|
type = TREE_TYPE (arg0);
|
type = TREE_TYPE (arg0);
|
mode = TYPE_MODE (type);
|
mode = TYPE_MODE (type);
|
sqrtfn = mathfn_built_in (type, BUILT_IN_SQRT);
|
sqrtfn = mathfn_built_in (type, BUILT_IN_SQRT);
|
|
|
/* Optimize pow(x,0.5) = sqrt(x). This replacement is always safe
|
/* Optimize pow(x,0.5) = sqrt(x). This replacement is always safe
|
unless signed zeros must be maintained. pow(-0,0.5) = +0, while
|
unless signed zeros must be maintained. pow(-0,0.5) = +0, while
|
sqrt(-0) = -0. */
|
sqrt(-0) = -0. */
|
if (sqrtfn
|
if (sqrtfn
|
&& REAL_VALUES_EQUAL (c, dconsthalf)
|
&& REAL_VALUES_EQUAL (c, dconsthalf)
|
&& !HONOR_SIGNED_ZEROS (mode))
|
&& !HONOR_SIGNED_ZEROS (mode))
|
return build_and_insert_call (gsi, loc, &target, sqrtfn, arg0);
|
return build_and_insert_call (gsi, loc, &target, sqrtfn, arg0);
|
|
|
/* Optimize pow(x,0.25) = sqrt(sqrt(x)). Assume on most machines that
|
/* Optimize pow(x,0.25) = sqrt(sqrt(x)). Assume on most machines that
|
a builtin sqrt instruction is smaller than a call to pow with 0.25,
|
a builtin sqrt instruction is smaller than a call to pow with 0.25,
|
so do this optimization even if -Os. Don't do this optimization
|
so do this optimization even if -Os. Don't do this optimization
|
if we don't have a hardware sqrt insn. */
|
if we don't have a hardware sqrt insn. */
|
dconst1_4 = dconst1;
|
dconst1_4 = dconst1;
|
SET_REAL_EXP (&dconst1_4, REAL_EXP (&dconst1_4) - 2);
|
SET_REAL_EXP (&dconst1_4, REAL_EXP (&dconst1_4) - 2);
|
hw_sqrt_exists = optab_handler (sqrt_optab, mode) != CODE_FOR_nothing;
|
hw_sqrt_exists = optab_handler (sqrt_optab, mode) != CODE_FOR_nothing;
|
|
|
if (flag_unsafe_math_optimizations
|
if (flag_unsafe_math_optimizations
|
&& sqrtfn
|
&& sqrtfn
|
&& REAL_VALUES_EQUAL (c, dconst1_4)
|
&& REAL_VALUES_EQUAL (c, dconst1_4)
|
&& hw_sqrt_exists)
|
&& hw_sqrt_exists)
|
{
|
{
|
/* sqrt(x) */
|
/* sqrt(x) */
|
sqrt_arg0 = build_and_insert_call (gsi, loc, &target, sqrtfn, arg0);
|
sqrt_arg0 = build_and_insert_call (gsi, loc, &target, sqrtfn, arg0);
|
|
|
/* sqrt(sqrt(x)) */
|
/* sqrt(sqrt(x)) */
|
return build_and_insert_call (gsi, loc, &target, sqrtfn, sqrt_arg0);
|
return build_and_insert_call (gsi, loc, &target, sqrtfn, sqrt_arg0);
|
}
|
}
|
|
|
/* Optimize pow(x,0.75) = sqrt(x) * sqrt(sqrt(x)) unless we are
|
/* Optimize pow(x,0.75) = sqrt(x) * sqrt(sqrt(x)) unless we are
|
optimizing for space. Don't do this optimization if we don't have
|
optimizing for space. Don't do this optimization if we don't have
|
a hardware sqrt insn. */
|
a hardware sqrt insn. */
|
real_from_integer (&dconst3_4, VOIDmode, 3, 0, 0);
|
real_from_integer (&dconst3_4, VOIDmode, 3, 0, 0);
|
SET_REAL_EXP (&dconst3_4, REAL_EXP (&dconst3_4) - 2);
|
SET_REAL_EXP (&dconst3_4, REAL_EXP (&dconst3_4) - 2);
|
|
|
if (flag_unsafe_math_optimizations
|
if (flag_unsafe_math_optimizations
|
&& sqrtfn
|
&& sqrtfn
|
&& optimize_function_for_speed_p (cfun)
|
&& optimize_function_for_speed_p (cfun)
|
&& REAL_VALUES_EQUAL (c, dconst3_4)
|
&& REAL_VALUES_EQUAL (c, dconst3_4)
|
&& hw_sqrt_exists)
|
&& hw_sqrt_exists)
|
{
|
{
|
/* sqrt(x) */
|
/* sqrt(x) */
|
sqrt_arg0 = build_and_insert_call (gsi, loc, &target, sqrtfn, arg0);
|
sqrt_arg0 = build_and_insert_call (gsi, loc, &target, sqrtfn, arg0);
|
|
|
/* sqrt(sqrt(x)) */
|
/* sqrt(sqrt(x)) */
|
sqrt_sqrt = build_and_insert_call (gsi, loc, &target, sqrtfn, sqrt_arg0);
|
sqrt_sqrt = build_and_insert_call (gsi, loc, &target, sqrtfn, sqrt_arg0);
|
|
|
/* sqrt(x) * sqrt(sqrt(x)) */
|
/* sqrt(x) * sqrt(sqrt(x)) */
|
return build_and_insert_binop (gsi, loc, target, MULT_EXPR,
|
return build_and_insert_binop (gsi, loc, target, MULT_EXPR,
|
sqrt_arg0, sqrt_sqrt);
|
sqrt_arg0, sqrt_sqrt);
|
}
|
}
|
|
|
/* Optimize pow(x,1./3.) = cbrt(x). This requires unsafe math
|
/* Optimize pow(x,1./3.) = cbrt(x). This requires unsafe math
|
optimizations since 1./3. is not exactly representable. If x
|
optimizations since 1./3. is not exactly representable. If x
|
is negative and finite, the correct value of pow(x,1./3.) is
|
is negative and finite, the correct value of pow(x,1./3.) is
|
a NaN with the "invalid" exception raised, because the value
|
a NaN with the "invalid" exception raised, because the value
|
of 1./3. actually has an even denominator. The correct value
|
of 1./3. actually has an even denominator. The correct value
|
of cbrt(x) is a negative real value. */
|
of cbrt(x) is a negative real value. */
|
cbrtfn = mathfn_built_in (type, BUILT_IN_CBRT);
|
cbrtfn = mathfn_built_in (type, BUILT_IN_CBRT);
|
dconst1_3 = real_value_truncate (mode, dconst_third ());
|
dconst1_3 = real_value_truncate (mode, dconst_third ());
|
|
|
if (flag_unsafe_math_optimizations
|
if (flag_unsafe_math_optimizations
|
&& cbrtfn
|
&& cbrtfn
|
&& (gimple_val_nonnegative_real_p (arg0) || !HONOR_NANS (mode))
|
&& (gimple_val_nonnegative_real_p (arg0) || !HONOR_NANS (mode))
|
&& REAL_VALUES_EQUAL (c, dconst1_3))
|
&& REAL_VALUES_EQUAL (c, dconst1_3))
|
return build_and_insert_call (gsi, loc, &target, cbrtfn, arg0);
|
return build_and_insert_call (gsi, loc, &target, cbrtfn, arg0);
|
|
|
/* Optimize pow(x,1./6.) = cbrt(sqrt(x)). Don't do this optimization
|
/* Optimize pow(x,1./6.) = cbrt(sqrt(x)). Don't do this optimization
|
if we don't have a hardware sqrt insn. */
|
if we don't have a hardware sqrt insn. */
|
dconst1_6 = dconst1_3;
|
dconst1_6 = dconst1_3;
|
SET_REAL_EXP (&dconst1_6, REAL_EXP (&dconst1_6) - 1);
|
SET_REAL_EXP (&dconst1_6, REAL_EXP (&dconst1_6) - 1);
|
|
|
if (flag_unsafe_math_optimizations
|
if (flag_unsafe_math_optimizations
|
&& sqrtfn
|
&& sqrtfn
|
&& cbrtfn
|
&& cbrtfn
|
&& (gimple_val_nonnegative_real_p (arg0) || !HONOR_NANS (mode))
|
&& (gimple_val_nonnegative_real_p (arg0) || !HONOR_NANS (mode))
|
&& optimize_function_for_speed_p (cfun)
|
&& optimize_function_for_speed_p (cfun)
|
&& hw_sqrt_exists
|
&& hw_sqrt_exists
|
&& REAL_VALUES_EQUAL (c, dconst1_6))
|
&& REAL_VALUES_EQUAL (c, dconst1_6))
|
{
|
{
|
/* sqrt(x) */
|
/* sqrt(x) */
|
sqrt_arg0 = build_and_insert_call (gsi, loc, &target, sqrtfn, arg0);
|
sqrt_arg0 = build_and_insert_call (gsi, loc, &target, sqrtfn, arg0);
|
|
|
/* cbrt(sqrt(x)) */
|
/* cbrt(sqrt(x)) */
|
return build_and_insert_call (gsi, loc, &target, cbrtfn, sqrt_arg0);
|
return build_and_insert_call (gsi, loc, &target, cbrtfn, sqrt_arg0);
|
}
|
}
|
|
|
/* Optimize pow(x,c), where n = 2c for some nonzero integer n, into
|
/* Optimize pow(x,c), where n = 2c for some nonzero integer n, into
|
|
|
sqrt(x) * powi(x, n/2), n > 0;
|
sqrt(x) * powi(x, n/2), n > 0;
|
1.0 / (sqrt(x) * powi(x, abs(n/2))), n < 0.
|
1.0 / (sqrt(x) * powi(x, abs(n/2))), n < 0.
|
|
|
Do not calculate the powi factor when n/2 = 0. */
|
Do not calculate the powi factor when n/2 = 0. */
|
real_arithmetic (&c2, MULT_EXPR, &c, &dconst2);
|
real_arithmetic (&c2, MULT_EXPR, &c, &dconst2);
|
n = real_to_integer (&c2);
|
n = real_to_integer (&c2);
|
real_from_integer (&cint, VOIDmode, n, n < 0 ? -1 : 0, 0);
|
real_from_integer (&cint, VOIDmode, n, n < 0 ? -1 : 0, 0);
|
|
|
if (flag_unsafe_math_optimizations
|
if (flag_unsafe_math_optimizations
|
&& sqrtfn
|
&& sqrtfn
|
&& real_identical (&c2, &cint))
|
&& real_identical (&c2, &cint))
|
{
|
{
|
tree powi_x_ndiv2 = NULL_TREE;
|
tree powi_x_ndiv2 = NULL_TREE;
|
|
|
/* Attempt to fold powi(arg0, abs(n/2)) into multiplies. If not
|
/* Attempt to fold powi(arg0, abs(n/2)) into multiplies. If not
|
possible or profitable, give up. Skip the degenerate case when
|
possible or profitable, give up. Skip the degenerate case when
|
n is 1 or -1, where the result is always 1. */
|
n is 1 or -1, where the result is always 1. */
|
if (absu_hwi (n) != 1)
|
if (absu_hwi (n) != 1)
|
{
|
{
|
powi_x_ndiv2 = gimple_expand_builtin_powi (gsi, loc, arg0,
|
powi_x_ndiv2 = gimple_expand_builtin_powi (gsi, loc, arg0,
|
abs_hwi (n / 2));
|
abs_hwi (n / 2));
|
if (!powi_x_ndiv2)
|
if (!powi_x_ndiv2)
|
return NULL_TREE;
|
return NULL_TREE;
|
}
|
}
|
|
|
/* Calculate sqrt(x). When n is not 1 or -1, multiply it by the
|
/* Calculate sqrt(x). When n is not 1 or -1, multiply it by the
|
result of the optimal multiply sequence just calculated. */
|
result of the optimal multiply sequence just calculated. */
|
sqrt_arg0 = build_and_insert_call (gsi, loc, &target, sqrtfn, arg0);
|
sqrt_arg0 = build_and_insert_call (gsi, loc, &target, sqrtfn, arg0);
|
|
|
if (absu_hwi (n) == 1)
|
if (absu_hwi (n) == 1)
|
result = sqrt_arg0;
|
result = sqrt_arg0;
|
else
|
else
|
result = build_and_insert_binop (gsi, loc, target, MULT_EXPR,
|
result = build_and_insert_binop (gsi, loc, target, MULT_EXPR,
|
sqrt_arg0, powi_x_ndiv2);
|
sqrt_arg0, powi_x_ndiv2);
|
|
|
/* If n is negative, reciprocate the result. */
|
/* If n is negative, reciprocate the result. */
|
if (n < 0)
|
if (n < 0)
|
result = build_and_insert_binop (gsi, loc, target, RDIV_EXPR,
|
result = build_and_insert_binop (gsi, loc, target, RDIV_EXPR,
|
build_real (type, dconst1), result);
|
build_real (type, dconst1), result);
|
return result;
|
return result;
|
}
|
}
|
|
|
/* Optimize pow(x,c), where 3c = n for some nonzero integer n, into
|
/* Optimize pow(x,c), where 3c = n for some nonzero integer n, into
|
|
|
powi(x, n/3) * powi(cbrt(x), n%3), n > 0;
|
powi(x, n/3) * powi(cbrt(x), n%3), n > 0;
|
1.0 / (powi(x, abs(n)/3) * powi(cbrt(x), abs(n)%3)), n < 0.
|
1.0 / (powi(x, abs(n)/3) * powi(cbrt(x), abs(n)%3)), n < 0.
|
|
|
Do not calculate the first factor when n/3 = 0. As cbrt(x) is
|
Do not calculate the first factor when n/3 = 0. As cbrt(x) is
|
different from pow(x, 1./3.) due to rounding and behavior with
|
different from pow(x, 1./3.) due to rounding and behavior with
|
negative x, we need to constrain this transformation to unsafe
|
negative x, we need to constrain this transformation to unsafe
|
math and positive x or finite math. */
|
math and positive x or finite math. */
|
real_from_integer (&dconst3, VOIDmode, 3, 0, 0);
|
real_from_integer (&dconst3, VOIDmode, 3, 0, 0);
|
real_arithmetic (&c2, MULT_EXPR, &c, &dconst3);
|
real_arithmetic (&c2, MULT_EXPR, &c, &dconst3);
|
real_round (&c2, mode, &c2);
|
real_round (&c2, mode, &c2);
|
n = real_to_integer (&c2);
|
n = real_to_integer (&c2);
|
real_from_integer (&cint, VOIDmode, n, n < 0 ? -1 : 0, 0);
|
real_from_integer (&cint, VOIDmode, n, n < 0 ? -1 : 0, 0);
|
real_arithmetic (&c2, RDIV_EXPR, &cint, &dconst3);
|
real_arithmetic (&c2, RDIV_EXPR, &cint, &dconst3);
|
real_convert (&c2, mode, &c2);
|
real_convert (&c2, mode, &c2);
|
|
|
if (flag_unsafe_math_optimizations
|
if (flag_unsafe_math_optimizations
|
&& cbrtfn
|
&& cbrtfn
|
&& (gimple_val_nonnegative_real_p (arg0) || !HONOR_NANS (mode))
|
&& (gimple_val_nonnegative_real_p (arg0) || !HONOR_NANS (mode))
|
&& real_identical (&c2, &c)
|
&& real_identical (&c2, &c)
|
&& optimize_function_for_speed_p (cfun)
|
&& optimize_function_for_speed_p (cfun)
|
&& powi_cost (n / 3) <= POWI_MAX_MULTS)
|
&& powi_cost (n / 3) <= POWI_MAX_MULTS)
|
{
|
{
|
tree powi_x_ndiv3 = NULL_TREE;
|
tree powi_x_ndiv3 = NULL_TREE;
|
|
|
/* Attempt to fold powi(arg0, abs(n/3)) into multiplies. If not
|
/* Attempt to fold powi(arg0, abs(n/3)) into multiplies. If not
|
possible or profitable, give up. Skip the degenerate case when
|
possible or profitable, give up. Skip the degenerate case when
|
abs(n) < 3, where the result is always 1. */
|
abs(n) < 3, where the result is always 1. */
|
if (absu_hwi (n) >= 3)
|
if (absu_hwi (n) >= 3)
|
{
|
{
|
powi_x_ndiv3 = gimple_expand_builtin_powi (gsi, loc, arg0,
|
powi_x_ndiv3 = gimple_expand_builtin_powi (gsi, loc, arg0,
|
abs_hwi (n / 3));
|
abs_hwi (n / 3));
|
if (!powi_x_ndiv3)
|
if (!powi_x_ndiv3)
|
return NULL_TREE;
|
return NULL_TREE;
|
}
|
}
|
|
|
/* Calculate powi(cbrt(x), n%3). Don't use gimple_expand_builtin_powi
|
/* Calculate powi(cbrt(x), n%3). Don't use gimple_expand_builtin_powi
|
as that creates an unnecessary variable. Instead, just produce
|
as that creates an unnecessary variable. Instead, just produce
|
either cbrt(x) or cbrt(x) * cbrt(x). */
|
either cbrt(x) or cbrt(x) * cbrt(x). */
|
cbrt_x = build_and_insert_call (gsi, loc, &target, cbrtfn, arg0);
|
cbrt_x = build_and_insert_call (gsi, loc, &target, cbrtfn, arg0);
|
|
|
if (absu_hwi (n) % 3 == 1)
|
if (absu_hwi (n) % 3 == 1)
|
powi_cbrt_x = cbrt_x;
|
powi_cbrt_x = cbrt_x;
|
else
|
else
|
powi_cbrt_x = build_and_insert_binop (gsi, loc, target, MULT_EXPR,
|
powi_cbrt_x = build_and_insert_binop (gsi, loc, target, MULT_EXPR,
|
cbrt_x, cbrt_x);
|
cbrt_x, cbrt_x);
|
|
|
/* Multiply the two subexpressions, unless powi(x,abs(n)/3) = 1. */
|
/* Multiply the two subexpressions, unless powi(x,abs(n)/3) = 1. */
|
if (absu_hwi (n) < 3)
|
if (absu_hwi (n) < 3)
|
result = powi_cbrt_x;
|
result = powi_cbrt_x;
|
else
|
else
|
result = build_and_insert_binop (gsi, loc, target, MULT_EXPR,
|
result = build_and_insert_binop (gsi, loc, target, MULT_EXPR,
|
powi_x_ndiv3, powi_cbrt_x);
|
powi_x_ndiv3, powi_cbrt_x);
|
|
|
/* If n is negative, reciprocate the result. */
|
/* If n is negative, reciprocate the result. */
|
if (n < 0)
|
if (n < 0)
|
result = build_and_insert_binop (gsi, loc, target, RDIV_EXPR,
|
result = build_and_insert_binop (gsi, loc, target, RDIV_EXPR,
|
build_real (type, dconst1), result);
|
build_real (type, dconst1), result);
|
|
|
return result;
|
return result;
|
}
|
}
|
|
|
/* No optimizations succeeded. */
|
/* No optimizations succeeded. */
|
return NULL_TREE;
|
return NULL_TREE;
|
}
|
}
|
|
|
/* ARG is the argument to a cabs builtin call in GSI with location info
|
/* ARG is the argument to a cabs builtin call in GSI with location info
|
LOC. Create a sequence of statements prior to GSI that calculates
|
LOC. Create a sequence of statements prior to GSI that calculates
|
sqrt(R*R + I*I), where R and I are the real and imaginary components
|
sqrt(R*R + I*I), where R and I are the real and imaginary components
|
of ARG, respectively. Return an expression holding the result. */
|
of ARG, respectively. Return an expression holding the result. */
|
|
|
static tree
|
static tree
|
gimple_expand_builtin_cabs (gimple_stmt_iterator *gsi, location_t loc, tree arg)
|
gimple_expand_builtin_cabs (gimple_stmt_iterator *gsi, location_t loc, tree arg)
|
{
|
{
|
tree target, real_part, imag_part, addend1, addend2, sum, result;
|
tree target, real_part, imag_part, addend1, addend2, sum, result;
|
tree type = TREE_TYPE (TREE_TYPE (arg));
|
tree type = TREE_TYPE (TREE_TYPE (arg));
|
tree sqrtfn = mathfn_built_in (type, BUILT_IN_SQRT);
|
tree sqrtfn = mathfn_built_in (type, BUILT_IN_SQRT);
|
enum machine_mode mode = TYPE_MODE (type);
|
enum machine_mode mode = TYPE_MODE (type);
|
|
|
if (!flag_unsafe_math_optimizations
|
if (!flag_unsafe_math_optimizations
|
|| !optimize_bb_for_speed_p (gimple_bb (gsi_stmt (*gsi)))
|
|| !optimize_bb_for_speed_p (gimple_bb (gsi_stmt (*gsi)))
|
|| !sqrtfn
|
|| !sqrtfn
|
|| optab_handler (sqrt_optab, mode) == CODE_FOR_nothing)
|
|| optab_handler (sqrt_optab, mode) == CODE_FOR_nothing)
|
return NULL_TREE;
|
return NULL_TREE;
|
|
|
target = create_tmp_reg (type, "cabs");
|
target = create_tmp_reg (type, "cabs");
|
add_referenced_var (target);
|
add_referenced_var (target);
|
|
|
real_part = build_and_insert_ref (gsi, loc, type, target,
|
real_part = build_and_insert_ref (gsi, loc, type, target,
|
REALPART_EXPR, arg);
|
REALPART_EXPR, arg);
|
addend1 = build_and_insert_binop (gsi, loc, target, MULT_EXPR,
|
addend1 = build_and_insert_binop (gsi, loc, target, MULT_EXPR,
|
real_part, real_part);
|
real_part, real_part);
|
imag_part = build_and_insert_ref (gsi, loc, type, target,
|
imag_part = build_and_insert_ref (gsi, loc, type, target,
|
IMAGPART_EXPR, arg);
|
IMAGPART_EXPR, arg);
|
addend2 = build_and_insert_binop (gsi, loc, target, MULT_EXPR,
|
addend2 = build_and_insert_binop (gsi, loc, target, MULT_EXPR,
|
imag_part, imag_part);
|
imag_part, imag_part);
|
sum = build_and_insert_binop (gsi, loc, target, PLUS_EXPR, addend1, addend2);
|
sum = build_and_insert_binop (gsi, loc, target, PLUS_EXPR, addend1, addend2);
|
result = build_and_insert_call (gsi, loc, &target, sqrtfn, sum);
|
result = build_and_insert_call (gsi, loc, &target, sqrtfn, sum);
|
|
|
return result;
|
return result;
|
}
|
}
|
|
|
/* Go through all calls to sin, cos and cexpi and call execute_cse_sincos_1
|
/* Go through all calls to sin, cos and cexpi and call execute_cse_sincos_1
|
on the SSA_NAME argument of each of them. Also expand powi(x,n) into
|
on the SSA_NAME argument of each of them. Also expand powi(x,n) into
|
an optimal number of multiplies, when n is a constant. */
|
an optimal number of multiplies, when n is a constant. */
|
|
|
static unsigned int
|
static unsigned int
|
execute_cse_sincos (void)
|
execute_cse_sincos (void)
|
{
|
{
|
basic_block bb;
|
basic_block bb;
|
bool cfg_changed = false;
|
bool cfg_changed = false;
|
|
|
calculate_dominance_info (CDI_DOMINATORS);
|
calculate_dominance_info (CDI_DOMINATORS);
|
memset (&sincos_stats, 0, sizeof (sincos_stats));
|
memset (&sincos_stats, 0, sizeof (sincos_stats));
|
|
|
FOR_EACH_BB (bb)
|
FOR_EACH_BB (bb)
|
{
|
{
|
gimple_stmt_iterator gsi;
|
gimple_stmt_iterator gsi;
|
|
|
for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
|
{
|
{
|
gimple stmt = gsi_stmt (gsi);
|
gimple stmt = gsi_stmt (gsi);
|
tree fndecl;
|
tree fndecl;
|
|
|
if (is_gimple_call (stmt)
|
if (is_gimple_call (stmt)
|
&& gimple_call_lhs (stmt)
|
&& gimple_call_lhs (stmt)
|
&& (fndecl = gimple_call_fndecl (stmt))
|
&& (fndecl = gimple_call_fndecl (stmt))
|
&& DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
|
&& DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
|
{
|
{
|
tree arg, arg0, arg1, result;
|
tree arg, arg0, arg1, result;
|
HOST_WIDE_INT n;
|
HOST_WIDE_INT n;
|
location_t loc;
|
location_t loc;
|
|
|
switch (DECL_FUNCTION_CODE (fndecl))
|
switch (DECL_FUNCTION_CODE (fndecl))
|
{
|
{
|
CASE_FLT_FN (BUILT_IN_COS):
|
CASE_FLT_FN (BUILT_IN_COS):
|
CASE_FLT_FN (BUILT_IN_SIN):
|
CASE_FLT_FN (BUILT_IN_SIN):
|
CASE_FLT_FN (BUILT_IN_CEXPI):
|
CASE_FLT_FN (BUILT_IN_CEXPI):
|
/* Make sure we have either sincos or cexp. */
|
/* Make sure we have either sincos or cexp. */
|
if (!TARGET_HAS_SINCOS && !TARGET_C99_FUNCTIONS)
|
if (!TARGET_HAS_SINCOS && !TARGET_C99_FUNCTIONS)
|
break;
|
break;
|
|
|
arg = gimple_call_arg (stmt, 0);
|
arg = gimple_call_arg (stmt, 0);
|
if (TREE_CODE (arg) == SSA_NAME)
|
if (TREE_CODE (arg) == SSA_NAME)
|
cfg_changed |= execute_cse_sincos_1 (arg);
|
cfg_changed |= execute_cse_sincos_1 (arg);
|
break;
|
break;
|
|
|
CASE_FLT_FN (BUILT_IN_POW):
|
CASE_FLT_FN (BUILT_IN_POW):
|
arg0 = gimple_call_arg (stmt, 0);
|
arg0 = gimple_call_arg (stmt, 0);
|
arg1 = gimple_call_arg (stmt, 1);
|
arg1 = gimple_call_arg (stmt, 1);
|
|
|
loc = gimple_location (stmt);
|
loc = gimple_location (stmt);
|
result = gimple_expand_builtin_pow (&gsi, loc, arg0, arg1);
|
result = gimple_expand_builtin_pow (&gsi, loc, arg0, arg1);
|
|
|
if (result)
|
if (result)
|
{
|
{
|
tree lhs = gimple_get_lhs (stmt);
|
tree lhs = gimple_get_lhs (stmt);
|
gimple new_stmt = gimple_build_assign (lhs, result);
|
gimple new_stmt = gimple_build_assign (lhs, result);
|
gimple_set_location (new_stmt, loc);
|
gimple_set_location (new_stmt, loc);
|
unlink_stmt_vdef (stmt);
|
unlink_stmt_vdef (stmt);
|
gsi_replace (&gsi, new_stmt, true);
|
gsi_replace (&gsi, new_stmt, true);
|
}
|
}
|
break;
|
break;
|
|
|
CASE_FLT_FN (BUILT_IN_POWI):
|
CASE_FLT_FN (BUILT_IN_POWI):
|
arg0 = gimple_call_arg (stmt, 0);
|
arg0 = gimple_call_arg (stmt, 0);
|
arg1 = gimple_call_arg (stmt, 1);
|
arg1 = gimple_call_arg (stmt, 1);
|
if (!host_integerp (arg1, 0))
|
if (!host_integerp (arg1, 0))
|
break;
|
break;
|
|
|
n = TREE_INT_CST_LOW (arg1);
|
n = TREE_INT_CST_LOW (arg1);
|
loc = gimple_location (stmt);
|
loc = gimple_location (stmt);
|
result = gimple_expand_builtin_powi (&gsi, loc, arg0, n);
|
result = gimple_expand_builtin_powi (&gsi, loc, arg0, n);
|
|
|
if (result)
|
if (result)
|
{
|
{
|
tree lhs = gimple_get_lhs (stmt);
|
tree lhs = gimple_get_lhs (stmt);
|
gimple new_stmt = gimple_build_assign (lhs, result);
|
gimple new_stmt = gimple_build_assign (lhs, result);
|
gimple_set_location (new_stmt, loc);
|
gimple_set_location (new_stmt, loc);
|
unlink_stmt_vdef (stmt);
|
unlink_stmt_vdef (stmt);
|
gsi_replace (&gsi, new_stmt, true);
|
gsi_replace (&gsi, new_stmt, true);
|
}
|
}
|
break;
|
break;
|
|
|
CASE_FLT_FN (BUILT_IN_CABS):
|
CASE_FLT_FN (BUILT_IN_CABS):
|
arg0 = gimple_call_arg (stmt, 0);
|
arg0 = gimple_call_arg (stmt, 0);
|
loc = gimple_location (stmt);
|
loc = gimple_location (stmt);
|
result = gimple_expand_builtin_cabs (&gsi, loc, arg0);
|
result = gimple_expand_builtin_cabs (&gsi, loc, arg0);
|
|
|
if (result)
|
if (result)
|
{
|
{
|
tree lhs = gimple_get_lhs (stmt);
|
tree lhs = gimple_get_lhs (stmt);
|
gimple new_stmt = gimple_build_assign (lhs, result);
|
gimple new_stmt = gimple_build_assign (lhs, result);
|
gimple_set_location (new_stmt, loc);
|
gimple_set_location (new_stmt, loc);
|
unlink_stmt_vdef (stmt);
|
unlink_stmt_vdef (stmt);
|
gsi_replace (&gsi, new_stmt, true);
|
gsi_replace (&gsi, new_stmt, true);
|
}
|
}
|
break;
|
break;
|
|
|
default:;
|
default:;
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
}
|
|
|
statistics_counter_event (cfun, "sincos statements inserted",
|
statistics_counter_event (cfun, "sincos statements inserted",
|
sincos_stats.inserted);
|
sincos_stats.inserted);
|
|
|
free_dominance_info (CDI_DOMINATORS);
|
free_dominance_info (CDI_DOMINATORS);
|
return cfg_changed ? TODO_cleanup_cfg : 0;
|
return cfg_changed ? TODO_cleanup_cfg : 0;
|
}
|
}
|
|
|
static bool
|
static bool
|
gate_cse_sincos (void)
|
gate_cse_sincos (void)
|
{
|
{
|
/* We no longer require either sincos or cexp, since powi expansion
|
/* We no longer require either sincos or cexp, since powi expansion
|
piggybacks on this pass. */
|
piggybacks on this pass. */
|
return optimize;
|
return optimize;
|
}
|
}
|
|
|
struct gimple_opt_pass pass_cse_sincos =
|
struct gimple_opt_pass pass_cse_sincos =
|
{
|
{
|
{
|
{
|
GIMPLE_PASS,
|
GIMPLE_PASS,
|
"sincos", /* name */
|
"sincos", /* name */
|
gate_cse_sincos, /* gate */
|
gate_cse_sincos, /* gate */
|
execute_cse_sincos, /* execute */
|
execute_cse_sincos, /* execute */
|
NULL, /* sub */
|
NULL, /* sub */
|
NULL, /* next */
|
NULL, /* next */
|
0, /* static_pass_number */
|
0, /* static_pass_number */
|
TV_NONE, /* tv_id */
|
TV_NONE, /* tv_id */
|
PROP_ssa, /* properties_required */
|
PROP_ssa, /* properties_required */
|
0, /* properties_provided */
|
0, /* properties_provided */
|
0, /* properties_destroyed */
|
0, /* properties_destroyed */
|
0, /* todo_flags_start */
|
0, /* todo_flags_start */
|
TODO_update_ssa | TODO_verify_ssa
|
TODO_update_ssa | TODO_verify_ssa
|
| TODO_verify_stmts /* todo_flags_finish */
|
| TODO_verify_stmts /* todo_flags_finish */
|
}
|
}
|
};
|
};
|
|
|
/* A symbolic number is used to detect byte permutation and selection
|
/* A symbolic number is used to detect byte permutation and selection
|
patterns. Therefore the field N contains an artificial number
|
patterns. Therefore the field N contains an artificial number
|
consisting of byte size markers:
|
consisting of byte size markers:
|
|
|
0 - byte has the value 0
|
0 - byte has the value 0
|
1..size - byte contains the content of the byte
|
1..size - byte contains the content of the byte
|
number indexed with that value minus one */
|
number indexed with that value minus one */
|
|
|
struct symbolic_number {
|
struct symbolic_number {
|
unsigned HOST_WIDEST_INT n;
|
unsigned HOST_WIDEST_INT n;
|
int size;
|
int size;
|
};
|
};
|
|
|
/* Perform a SHIFT or ROTATE operation by COUNT bits on symbolic
|
/* Perform a SHIFT or ROTATE operation by COUNT bits on symbolic
|
number N. Return false if the requested operation is not permitted
|
number N. Return false if the requested operation is not permitted
|
on a symbolic number. */
|
on a symbolic number. */
|
|
|
static inline bool
|
static inline bool
|
do_shift_rotate (enum tree_code code,
|
do_shift_rotate (enum tree_code code,
|
struct symbolic_number *n,
|
struct symbolic_number *n,
|
int count)
|
int count)
|
{
|
{
|
if (count % 8 != 0)
|
if (count % 8 != 0)
|
return false;
|
return false;
|
|
|
/* Zero out the extra bits of N in order to avoid them being shifted
|
/* Zero out the extra bits of N in order to avoid them being shifted
|
into the significant bits. */
|
into the significant bits. */
|
if (n->size < (int)sizeof (HOST_WIDEST_INT))
|
if (n->size < (int)sizeof (HOST_WIDEST_INT))
|
n->n &= ((unsigned HOST_WIDEST_INT)1 << (n->size * BITS_PER_UNIT)) - 1;
|
n->n &= ((unsigned HOST_WIDEST_INT)1 << (n->size * BITS_PER_UNIT)) - 1;
|
|
|
switch (code)
|
switch (code)
|
{
|
{
|
case LSHIFT_EXPR:
|
case LSHIFT_EXPR:
|
n->n <<= count;
|
n->n <<= count;
|
break;
|
break;
|
case RSHIFT_EXPR:
|
case RSHIFT_EXPR:
|
n->n >>= count;
|
n->n >>= count;
|
break;
|
break;
|
case LROTATE_EXPR:
|
case LROTATE_EXPR:
|
n->n = (n->n << count) | (n->n >> ((n->size * BITS_PER_UNIT) - count));
|
n->n = (n->n << count) | (n->n >> ((n->size * BITS_PER_UNIT) - count));
|
break;
|
break;
|
case RROTATE_EXPR:
|
case RROTATE_EXPR:
|
n->n = (n->n >> count) | (n->n << ((n->size * BITS_PER_UNIT) - count));
|
n->n = (n->n >> count) | (n->n << ((n->size * BITS_PER_UNIT) - count));
|
break;
|
break;
|
default:
|
default:
|
return false;
|
return false;
|
}
|
}
|
/* Zero unused bits for size. */
|
/* Zero unused bits for size. */
|
if (n->size < (int)sizeof (HOST_WIDEST_INT))
|
if (n->size < (int)sizeof (HOST_WIDEST_INT))
|
n->n &= ((unsigned HOST_WIDEST_INT)1 << (n->size * BITS_PER_UNIT)) - 1;
|
n->n &= ((unsigned HOST_WIDEST_INT)1 << (n->size * BITS_PER_UNIT)) - 1;
|
return true;
|
return true;
|
}
|
}
|
|
|
/* Perform sanity checking for the symbolic number N and the gimple
|
/* Perform sanity checking for the symbolic number N and the gimple
|
statement STMT. */
|
statement STMT. */
|
|
|
static inline bool
|
static inline bool
|
verify_symbolic_number_p (struct symbolic_number *n, gimple stmt)
|
verify_symbolic_number_p (struct symbolic_number *n, gimple stmt)
|
{
|
{
|
tree lhs_type;
|
tree lhs_type;
|
|
|
lhs_type = gimple_expr_type (stmt);
|
lhs_type = gimple_expr_type (stmt);
|
|
|
if (TREE_CODE (lhs_type) != INTEGER_TYPE)
|
if (TREE_CODE (lhs_type) != INTEGER_TYPE)
|
return false;
|
return false;
|
|
|
if (TYPE_PRECISION (lhs_type) != n->size * BITS_PER_UNIT)
|
if (TYPE_PRECISION (lhs_type) != n->size * BITS_PER_UNIT)
|
return false;
|
return false;
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
/* find_bswap_1 invokes itself recursively with N and tries to perform
|
/* find_bswap_1 invokes itself recursively with N and tries to perform
|
the operation given by the rhs of STMT on the result. If the
|
the operation given by the rhs of STMT on the result. If the
|
operation could successfully be executed the function returns the
|
operation could successfully be executed the function returns the
|
tree expression of the source operand and NULL otherwise. */
|
tree expression of the source operand and NULL otherwise. */
|
|
|
static tree
|
static tree
|
find_bswap_1 (gimple stmt, struct symbolic_number *n, int limit)
|
find_bswap_1 (gimple stmt, struct symbolic_number *n, int limit)
|
{
|
{
|
enum tree_code code;
|
enum tree_code code;
|
tree rhs1, rhs2 = NULL;
|
tree rhs1, rhs2 = NULL;
|
gimple rhs1_stmt, rhs2_stmt;
|
gimple rhs1_stmt, rhs2_stmt;
|
tree source_expr1;
|
tree source_expr1;
|
enum gimple_rhs_class rhs_class;
|
enum gimple_rhs_class rhs_class;
|
|
|
if (!limit || !is_gimple_assign (stmt))
|
if (!limit || !is_gimple_assign (stmt))
|
return NULL_TREE;
|
return NULL_TREE;
|
|
|
rhs1 = gimple_assign_rhs1 (stmt);
|
rhs1 = gimple_assign_rhs1 (stmt);
|
|
|
if (TREE_CODE (rhs1) != SSA_NAME)
|
if (TREE_CODE (rhs1) != SSA_NAME)
|
return NULL_TREE;
|
return NULL_TREE;
|
|
|
code = gimple_assign_rhs_code (stmt);
|
code = gimple_assign_rhs_code (stmt);
|
rhs_class = gimple_assign_rhs_class (stmt);
|
rhs_class = gimple_assign_rhs_class (stmt);
|
rhs1_stmt = SSA_NAME_DEF_STMT (rhs1);
|
rhs1_stmt = SSA_NAME_DEF_STMT (rhs1);
|
|
|
if (rhs_class == GIMPLE_BINARY_RHS)
|
if (rhs_class == GIMPLE_BINARY_RHS)
|
rhs2 = gimple_assign_rhs2 (stmt);
|
rhs2 = gimple_assign_rhs2 (stmt);
|
|
|
/* Handle unary rhs and binary rhs with integer constants as second
|
/* Handle unary rhs and binary rhs with integer constants as second
|
operand. */
|
operand. */
|
|
|
if (rhs_class == GIMPLE_UNARY_RHS
|
if (rhs_class == GIMPLE_UNARY_RHS
|
|| (rhs_class == GIMPLE_BINARY_RHS
|
|| (rhs_class == GIMPLE_BINARY_RHS
|
&& TREE_CODE (rhs2) == INTEGER_CST))
|
&& TREE_CODE (rhs2) == INTEGER_CST))
|
{
|
{
|
if (code != BIT_AND_EXPR
|
if (code != BIT_AND_EXPR
|
&& code != LSHIFT_EXPR
|
&& code != LSHIFT_EXPR
|
&& code != RSHIFT_EXPR
|
&& code != RSHIFT_EXPR
|
&& code != LROTATE_EXPR
|
&& code != LROTATE_EXPR
|
&& code != RROTATE_EXPR
|
&& code != RROTATE_EXPR
|
&& code != NOP_EXPR
|
&& code != NOP_EXPR
|
&& code != CONVERT_EXPR)
|
&& code != CONVERT_EXPR)
|
return NULL_TREE;
|
return NULL_TREE;
|
|
|
source_expr1 = find_bswap_1 (rhs1_stmt, n, limit - 1);
|
source_expr1 = find_bswap_1 (rhs1_stmt, n, limit - 1);
|
|
|
/* If find_bswap_1 returned NULL STMT is a leaf node and we have
|
/* If find_bswap_1 returned NULL STMT is a leaf node and we have
|
to initialize the symbolic number. */
|
to initialize the symbolic number. */
|
if (!source_expr1)
|
if (!source_expr1)
|
{
|
{
|
/* Set up the symbolic number N by setting each byte to a
|
/* Set up the symbolic number N by setting each byte to a
|
value between 1 and the byte size of rhs1. The highest
|
value between 1 and the byte size of rhs1. The highest
|
order byte is set to n->size and the lowest order
|
order byte is set to n->size and the lowest order
|
byte to 1. */
|
byte to 1. */
|
n->size = TYPE_PRECISION (TREE_TYPE (rhs1));
|
n->size = TYPE_PRECISION (TREE_TYPE (rhs1));
|
if (n->size % BITS_PER_UNIT != 0)
|
if (n->size % BITS_PER_UNIT != 0)
|
return NULL_TREE;
|
return NULL_TREE;
|
n->size /= BITS_PER_UNIT;
|
n->size /= BITS_PER_UNIT;
|
n->n = (sizeof (HOST_WIDEST_INT) < 8 ? 0 :
|
n->n = (sizeof (HOST_WIDEST_INT) < 8 ? 0 :
|
(unsigned HOST_WIDEST_INT)0x08070605 << 32 | 0x04030201);
|
(unsigned HOST_WIDEST_INT)0x08070605 << 32 | 0x04030201);
|
|
|
if (n->size < (int)sizeof (HOST_WIDEST_INT))
|
if (n->size < (int)sizeof (HOST_WIDEST_INT))
|
n->n &= ((unsigned HOST_WIDEST_INT)1 <<
|
n->n &= ((unsigned HOST_WIDEST_INT)1 <<
|
(n->size * BITS_PER_UNIT)) - 1;
|
(n->size * BITS_PER_UNIT)) - 1;
|
|
|
source_expr1 = rhs1;
|
source_expr1 = rhs1;
|
}
|
}
|
|
|
switch (code)
|
switch (code)
|
{
|
{
|
case BIT_AND_EXPR:
|
case BIT_AND_EXPR:
|
{
|
{
|
int i;
|
int i;
|
unsigned HOST_WIDEST_INT val = widest_int_cst_value (rhs2);
|
unsigned HOST_WIDEST_INT val = widest_int_cst_value (rhs2);
|
unsigned HOST_WIDEST_INT tmp = val;
|
unsigned HOST_WIDEST_INT tmp = val;
|
|
|
/* Only constants masking full bytes are allowed. */
|
/* Only constants masking full bytes are allowed. */
|
for (i = 0; i < n->size; i++, tmp >>= BITS_PER_UNIT)
|
for (i = 0; i < n->size; i++, tmp >>= BITS_PER_UNIT)
|
if ((tmp & 0xff) != 0 && (tmp & 0xff) != 0xff)
|
if ((tmp & 0xff) != 0 && (tmp & 0xff) != 0xff)
|
return NULL_TREE;
|
return NULL_TREE;
|
|
|
n->n &= val;
|
n->n &= val;
|
}
|
}
|
break;
|
break;
|
case LSHIFT_EXPR:
|
case LSHIFT_EXPR:
|
case RSHIFT_EXPR:
|
case RSHIFT_EXPR:
|
case LROTATE_EXPR:
|
case LROTATE_EXPR:
|
case RROTATE_EXPR:
|
case RROTATE_EXPR:
|
if (!do_shift_rotate (code, n, (int)TREE_INT_CST_LOW (rhs2)))
|
if (!do_shift_rotate (code, n, (int)TREE_INT_CST_LOW (rhs2)))
|
return NULL_TREE;
|
return NULL_TREE;
|
break;
|
break;
|
CASE_CONVERT:
|
CASE_CONVERT:
|
{
|
{
|
int type_size;
|
int type_size;
|
|
|
type_size = TYPE_PRECISION (gimple_expr_type (stmt));
|
type_size = TYPE_PRECISION (gimple_expr_type (stmt));
|
if (type_size % BITS_PER_UNIT != 0)
|
if (type_size % BITS_PER_UNIT != 0)
|
return NULL_TREE;
|
return NULL_TREE;
|
|
|
if (type_size / BITS_PER_UNIT < (int)(sizeof (HOST_WIDEST_INT)))
|
if (type_size / BITS_PER_UNIT < (int)(sizeof (HOST_WIDEST_INT)))
|
{
|
{
|
/* If STMT casts to a smaller type mask out the bits not
|
/* If STMT casts to a smaller type mask out the bits not
|
belonging to the target type. */
|
belonging to the target type. */
|
n->n &= ((unsigned HOST_WIDEST_INT)1 << type_size) - 1;
|
n->n &= ((unsigned HOST_WIDEST_INT)1 << type_size) - 1;
|
}
|
}
|
n->size = type_size / BITS_PER_UNIT;
|
n->size = type_size / BITS_PER_UNIT;
|
}
|
}
|
break;
|
break;
|
default:
|
default:
|
return NULL_TREE;
|
return NULL_TREE;
|
};
|
};
|
return verify_symbolic_number_p (n, stmt) ? source_expr1 : NULL;
|
return verify_symbolic_number_p (n, stmt) ? source_expr1 : NULL;
|
}
|
}
|
|
|
/* Handle binary rhs. */
|
/* Handle binary rhs. */
|
|
|
if (rhs_class == GIMPLE_BINARY_RHS)
|
if (rhs_class == GIMPLE_BINARY_RHS)
|
{
|
{
|
struct symbolic_number n1, n2;
|
struct symbolic_number n1, n2;
|
tree source_expr2;
|
tree source_expr2;
|
|
|
if (code != BIT_IOR_EXPR)
|
if (code != BIT_IOR_EXPR)
|
return NULL_TREE;
|
return NULL_TREE;
|
|
|
if (TREE_CODE (rhs2) != SSA_NAME)
|
if (TREE_CODE (rhs2) != SSA_NAME)
|
return NULL_TREE;
|
return NULL_TREE;
|
|
|
rhs2_stmt = SSA_NAME_DEF_STMT (rhs2);
|
rhs2_stmt = SSA_NAME_DEF_STMT (rhs2);
|
|
|
switch (code)
|
switch (code)
|
{
|
{
|
case BIT_IOR_EXPR:
|
case BIT_IOR_EXPR:
|
source_expr1 = find_bswap_1 (rhs1_stmt, &n1, limit - 1);
|
source_expr1 = find_bswap_1 (rhs1_stmt, &n1, limit - 1);
|
|
|
if (!source_expr1)
|
if (!source_expr1)
|
return NULL_TREE;
|
return NULL_TREE;
|
|
|
source_expr2 = find_bswap_1 (rhs2_stmt, &n2, limit - 1);
|
source_expr2 = find_bswap_1 (rhs2_stmt, &n2, limit - 1);
|
|
|
if (source_expr1 != source_expr2
|
if (source_expr1 != source_expr2
|
|| n1.size != n2.size)
|
|| n1.size != n2.size)
|
return NULL_TREE;
|
return NULL_TREE;
|
|
|
n->size = n1.size;
|
n->size = n1.size;
|
n->n = n1.n | n2.n;
|
n->n = n1.n | n2.n;
|
|
|
if (!verify_symbolic_number_p (n, stmt))
|
if (!verify_symbolic_number_p (n, stmt))
|
return NULL_TREE;
|
return NULL_TREE;
|
|
|
break;
|
break;
|
default:
|
default:
|
return NULL_TREE;
|
return NULL_TREE;
|
}
|
}
|
return source_expr1;
|
return source_expr1;
|
}
|
}
|
return NULL_TREE;
|
return NULL_TREE;
|
}
|
}
|
|
|
/* Check if STMT completes a bswap implementation consisting of ORs,
|
/* Check if STMT completes a bswap implementation consisting of ORs,
|
SHIFTs and ANDs. Return the source tree expression on which the
|
SHIFTs and ANDs. Return the source tree expression on which the
|
byte swap is performed and NULL if no bswap was found. */
|
byte swap is performed and NULL if no bswap was found. */
|
|
|
static tree
|
static tree
|
find_bswap (gimple stmt)
|
find_bswap (gimple stmt)
|
{
|
{
|
/* The number which the find_bswap result should match in order to
|
/* The number which the find_bswap result should match in order to
|
have a full byte swap. The number is shifted to the left according
|
have a full byte swap. The number is shifted to the left according
|
to the size of the symbolic number before using it. */
|
to the size of the symbolic number before using it. */
|
unsigned HOST_WIDEST_INT cmp =
|
unsigned HOST_WIDEST_INT cmp =
|
sizeof (HOST_WIDEST_INT) < 8 ? 0 :
|
sizeof (HOST_WIDEST_INT) < 8 ? 0 :
|
(unsigned HOST_WIDEST_INT)0x01020304 << 32 | 0x05060708;
|
(unsigned HOST_WIDEST_INT)0x01020304 << 32 | 0x05060708;
|
|
|
struct symbolic_number n;
|
struct symbolic_number n;
|
tree source_expr;
|
tree source_expr;
|
int limit;
|
int limit;
|
|
|
/* The last parameter determines the depth search limit. It usually
|
/* The last parameter determines the depth search limit. It usually
|
correlates directly to the number of bytes to be touched. We
|
correlates directly to the number of bytes to be touched. We
|
increase that number by three here in order to also
|
increase that number by three here in order to also
|
cover signed -> unsigned converions of the src operand as can be seen
|
cover signed -> unsigned converions of the src operand as can be seen
|
in libgcc, and for initial shift/and operation of the src operand. */
|
in libgcc, and for initial shift/and operation of the src operand. */
|
limit = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (gimple_expr_type (stmt)));
|
limit = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (gimple_expr_type (stmt)));
|
limit += 1 + (int) ceil_log2 ((unsigned HOST_WIDE_INT) limit);
|
limit += 1 + (int) ceil_log2 ((unsigned HOST_WIDE_INT) limit);
|
source_expr = find_bswap_1 (stmt, &n, limit);
|
source_expr = find_bswap_1 (stmt, &n, limit);
|
|
|
if (!source_expr)
|
if (!source_expr)
|
return NULL_TREE;
|
return NULL_TREE;
|
|
|
/* Zero out the extra bits of N and CMP. */
|
/* Zero out the extra bits of N and CMP. */
|
if (n.size < (int)sizeof (HOST_WIDEST_INT))
|
if (n.size < (int)sizeof (HOST_WIDEST_INT))
|
{
|
{
|
unsigned HOST_WIDEST_INT mask =
|
unsigned HOST_WIDEST_INT mask =
|
((unsigned HOST_WIDEST_INT)1 << (n.size * BITS_PER_UNIT)) - 1;
|
((unsigned HOST_WIDEST_INT)1 << (n.size * BITS_PER_UNIT)) - 1;
|
|
|
n.n &= mask;
|
n.n &= mask;
|
cmp >>= (sizeof (HOST_WIDEST_INT) - n.size) * BITS_PER_UNIT;
|
cmp >>= (sizeof (HOST_WIDEST_INT) - n.size) * BITS_PER_UNIT;
|
}
|
}
|
|
|
/* A complete byte swap should make the symbolic number to start
|
/* A complete byte swap should make the symbolic number to start
|
with the largest digit in the highest order byte. */
|
with the largest digit in the highest order byte. */
|
if (cmp != n.n)
|
if (cmp != n.n)
|
return NULL_TREE;
|
return NULL_TREE;
|
|
|
return source_expr;
|
return source_expr;
|
}
|
}
|
|
|
/* Find manual byte swap implementations and turn them into a bswap
|
/* Find manual byte swap implementations and turn them into a bswap
|
builtin invokation. */
|
builtin invokation. */
|
|
|
static unsigned int
|
static unsigned int
|
execute_optimize_bswap (void)
|
execute_optimize_bswap (void)
|
{
|
{
|
basic_block bb;
|
basic_block bb;
|
bool bswap32_p, bswap64_p;
|
bool bswap32_p, bswap64_p;
|
bool changed = false;
|
bool changed = false;
|
tree bswap32_type = NULL_TREE, bswap64_type = NULL_TREE;
|
tree bswap32_type = NULL_TREE, bswap64_type = NULL_TREE;
|
|
|
if (BITS_PER_UNIT != 8)
|
if (BITS_PER_UNIT != 8)
|
return 0;
|
return 0;
|
|
|
if (sizeof (HOST_WIDEST_INT) < 8)
|
if (sizeof (HOST_WIDEST_INT) < 8)
|
return 0;
|
return 0;
|
|
|
bswap32_p = (builtin_decl_explicit_p (BUILT_IN_BSWAP32)
|
bswap32_p = (builtin_decl_explicit_p (BUILT_IN_BSWAP32)
|
&& optab_handler (bswap_optab, SImode) != CODE_FOR_nothing);
|
&& optab_handler (bswap_optab, SImode) != CODE_FOR_nothing);
|
bswap64_p = (builtin_decl_explicit_p (BUILT_IN_BSWAP64)
|
bswap64_p = (builtin_decl_explicit_p (BUILT_IN_BSWAP64)
|
&& (optab_handler (bswap_optab, DImode) != CODE_FOR_nothing
|
&& (optab_handler (bswap_optab, DImode) != CODE_FOR_nothing
|
|| (bswap32_p && word_mode == SImode)));
|
|| (bswap32_p && word_mode == SImode)));
|
|
|
if (!bswap32_p && !bswap64_p)
|
if (!bswap32_p && !bswap64_p)
|
return 0;
|
return 0;
|
|
|
/* Determine the argument type of the builtins. The code later on
|
/* Determine the argument type of the builtins. The code later on
|
assumes that the return and argument type are the same. */
|
assumes that the return and argument type are the same. */
|
if (bswap32_p)
|
if (bswap32_p)
|
{
|
{
|
tree fndecl = builtin_decl_explicit (BUILT_IN_BSWAP32);
|
tree fndecl = builtin_decl_explicit (BUILT_IN_BSWAP32);
|
bswap32_type = TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (fndecl)));
|
bswap32_type = TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (fndecl)));
|
}
|
}
|
|
|
if (bswap64_p)
|
if (bswap64_p)
|
{
|
{
|
tree fndecl = builtin_decl_explicit (BUILT_IN_BSWAP64);
|
tree fndecl = builtin_decl_explicit (BUILT_IN_BSWAP64);
|
bswap64_type = TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (fndecl)));
|
bswap64_type = TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (fndecl)));
|
}
|
}
|
|
|
memset (&bswap_stats, 0, sizeof (bswap_stats));
|
memset (&bswap_stats, 0, sizeof (bswap_stats));
|
|
|
FOR_EACH_BB (bb)
|
FOR_EACH_BB (bb)
|
{
|
{
|
gimple_stmt_iterator gsi;
|
gimple_stmt_iterator gsi;
|
|
|
/* We do a reverse scan for bswap patterns to make sure we get the
|
/* We do a reverse scan for bswap patterns to make sure we get the
|
widest match. As bswap pattern matching doesn't handle
|
widest match. As bswap pattern matching doesn't handle
|
previously inserted smaller bswap replacements as sub-
|
previously inserted smaller bswap replacements as sub-
|
patterns, the wider variant wouldn't be detected. */
|
patterns, the wider variant wouldn't be detected. */
|
for (gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
|
for (gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
|
{
|
{
|
gimple stmt = gsi_stmt (gsi);
|
gimple stmt = gsi_stmt (gsi);
|
tree bswap_src, bswap_type;
|
tree bswap_src, bswap_type;
|
tree bswap_tmp;
|
tree bswap_tmp;
|
tree fndecl = NULL_TREE;
|
tree fndecl = NULL_TREE;
|
int type_size;
|
int type_size;
|
gimple call;
|
gimple call;
|
|
|
if (!is_gimple_assign (stmt)
|
if (!is_gimple_assign (stmt)
|
|| gimple_assign_rhs_code (stmt) != BIT_IOR_EXPR)
|
|| gimple_assign_rhs_code (stmt) != BIT_IOR_EXPR)
|
continue;
|
continue;
|
|
|
type_size = TYPE_PRECISION (gimple_expr_type (stmt));
|
type_size = TYPE_PRECISION (gimple_expr_type (stmt));
|
|
|
switch (type_size)
|
switch (type_size)
|
{
|
{
|
case 32:
|
case 32:
|
if (bswap32_p)
|
if (bswap32_p)
|
{
|
{
|
fndecl = builtin_decl_explicit (BUILT_IN_BSWAP32);
|
fndecl = builtin_decl_explicit (BUILT_IN_BSWAP32);
|
bswap_type = bswap32_type;
|
bswap_type = bswap32_type;
|
}
|
}
|
break;
|
break;
|
case 64:
|
case 64:
|
if (bswap64_p)
|
if (bswap64_p)
|
{
|
{
|
fndecl = builtin_decl_explicit (BUILT_IN_BSWAP64);
|
fndecl = builtin_decl_explicit (BUILT_IN_BSWAP64);
|
bswap_type = bswap64_type;
|
bswap_type = bswap64_type;
|
}
|
}
|
break;
|
break;
|
default:
|
default:
|
continue;
|
continue;
|
}
|
}
|
|
|
if (!fndecl)
|
if (!fndecl)
|
continue;
|
continue;
|
|
|
bswap_src = find_bswap (stmt);
|
bswap_src = find_bswap (stmt);
|
|
|
if (!bswap_src)
|
if (!bswap_src)
|
continue;
|
continue;
|
|
|
changed = true;
|
changed = true;
|
if (type_size == 32)
|
if (type_size == 32)
|
bswap_stats.found_32bit++;
|
bswap_stats.found_32bit++;
|
else
|
else
|
bswap_stats.found_64bit++;
|
bswap_stats.found_64bit++;
|
|
|
bswap_tmp = bswap_src;
|
bswap_tmp = bswap_src;
|
|
|
/* Convert the src expression if necessary. */
|
/* Convert the src expression if necessary. */
|
if (!useless_type_conversion_p (TREE_TYPE (bswap_tmp), bswap_type))
|
if (!useless_type_conversion_p (TREE_TYPE (bswap_tmp), bswap_type))
|
{
|
{
|
gimple convert_stmt;
|
gimple convert_stmt;
|
|
|
bswap_tmp = create_tmp_var (bswap_type, "bswapsrc");
|
bswap_tmp = create_tmp_var (bswap_type, "bswapsrc");
|
add_referenced_var (bswap_tmp);
|
add_referenced_var (bswap_tmp);
|
bswap_tmp = make_ssa_name (bswap_tmp, NULL);
|
bswap_tmp = make_ssa_name (bswap_tmp, NULL);
|
|
|
convert_stmt = gimple_build_assign_with_ops (
|
convert_stmt = gimple_build_assign_with_ops (
|
CONVERT_EXPR, bswap_tmp, bswap_src, NULL);
|
CONVERT_EXPR, bswap_tmp, bswap_src, NULL);
|
gsi_insert_before (&gsi, convert_stmt, GSI_SAME_STMT);
|
gsi_insert_before (&gsi, convert_stmt, GSI_SAME_STMT);
|
}
|
}
|
|
|
call = gimple_build_call (fndecl, 1, bswap_tmp);
|
call = gimple_build_call (fndecl, 1, bswap_tmp);
|
|
|
bswap_tmp = gimple_assign_lhs (stmt);
|
bswap_tmp = gimple_assign_lhs (stmt);
|
|
|
/* Convert the result if necessary. */
|
/* Convert the result if necessary. */
|
if (!useless_type_conversion_p (TREE_TYPE (bswap_tmp), bswap_type))
|
if (!useless_type_conversion_p (TREE_TYPE (bswap_tmp), bswap_type))
|
{
|
{
|
gimple convert_stmt;
|
gimple convert_stmt;
|
|
|
bswap_tmp = create_tmp_var (bswap_type, "bswapdst");
|
bswap_tmp = create_tmp_var (bswap_type, "bswapdst");
|
add_referenced_var (bswap_tmp);
|
add_referenced_var (bswap_tmp);
|
bswap_tmp = make_ssa_name (bswap_tmp, NULL);
|
bswap_tmp = make_ssa_name (bswap_tmp, NULL);
|
convert_stmt = gimple_build_assign_with_ops (
|
convert_stmt = gimple_build_assign_with_ops (
|
CONVERT_EXPR, gimple_assign_lhs (stmt), bswap_tmp, NULL);
|
CONVERT_EXPR, gimple_assign_lhs (stmt), bswap_tmp, NULL);
|
gsi_insert_after (&gsi, convert_stmt, GSI_SAME_STMT);
|
gsi_insert_after (&gsi, convert_stmt, GSI_SAME_STMT);
|
}
|
}
|
|
|
gimple_call_set_lhs (call, bswap_tmp);
|
gimple_call_set_lhs (call, bswap_tmp);
|
|
|
if (dump_file)
|
if (dump_file)
|
{
|
{
|
fprintf (dump_file, "%d bit bswap implementation found at: ",
|
fprintf (dump_file, "%d bit bswap implementation found at: ",
|
(int)type_size);
|
(int)type_size);
|
print_gimple_stmt (dump_file, stmt, 0, 0);
|
print_gimple_stmt (dump_file, stmt, 0, 0);
|
}
|
}
|
|
|
gsi_insert_after (&gsi, call, GSI_SAME_STMT);
|
gsi_insert_after (&gsi, call, GSI_SAME_STMT);
|
gsi_remove (&gsi, true);
|
gsi_remove (&gsi, true);
|
}
|
}
|
}
|
}
|
|
|
statistics_counter_event (cfun, "32-bit bswap implementations found",
|
statistics_counter_event (cfun, "32-bit bswap implementations found",
|
bswap_stats.found_32bit);
|
bswap_stats.found_32bit);
|
statistics_counter_event (cfun, "64-bit bswap implementations found",
|
statistics_counter_event (cfun, "64-bit bswap implementations found",
|
bswap_stats.found_64bit);
|
bswap_stats.found_64bit);
|
|
|
return (changed ? TODO_update_ssa | TODO_verify_ssa
|
return (changed ? TODO_update_ssa | TODO_verify_ssa
|
| TODO_verify_stmts : 0);
|
| TODO_verify_stmts : 0);
|
}
|
}
|
|
|
static bool
|
static bool
|
gate_optimize_bswap (void)
|
gate_optimize_bswap (void)
|
{
|
{
|
return flag_expensive_optimizations && optimize;
|
return flag_expensive_optimizations && optimize;
|
}
|
}
|
|
|
struct gimple_opt_pass pass_optimize_bswap =
|
struct gimple_opt_pass pass_optimize_bswap =
|
{
|
{
|
{
|
{
|
GIMPLE_PASS,
|
GIMPLE_PASS,
|
"bswap", /* name */
|
"bswap", /* name */
|
gate_optimize_bswap, /* gate */
|
gate_optimize_bswap, /* gate */
|
execute_optimize_bswap, /* execute */
|
execute_optimize_bswap, /* execute */
|
NULL, /* sub */
|
NULL, /* sub */
|
NULL, /* next */
|
NULL, /* next */
|
0, /* static_pass_number */
|
0, /* static_pass_number */
|
TV_NONE, /* tv_id */
|
TV_NONE, /* tv_id */
|
PROP_ssa, /* properties_required */
|
PROP_ssa, /* properties_required */
|
0, /* properties_provided */
|
0, /* properties_provided */
|
0, /* properties_destroyed */
|
0, /* properties_destroyed */
|
0, /* todo_flags_start */
|
0, /* todo_flags_start */
|
0 /* todo_flags_finish */
|
0 /* todo_flags_finish */
|
}
|
}
|
};
|
};
|
|
|
/* Return true if RHS is a suitable operand for a widening multiplication,
|
/* Return true if RHS is a suitable operand for a widening multiplication,
|
assuming a target type of TYPE.
|
assuming a target type of TYPE.
|
There are two cases:
|
There are two cases:
|
|
|
- RHS makes some value at least twice as wide. Store that value
|
- RHS makes some value at least twice as wide. Store that value
|
in *NEW_RHS_OUT if so, and store its type in *TYPE_OUT.
|
in *NEW_RHS_OUT if so, and store its type in *TYPE_OUT.
|
|
|
- RHS is an integer constant. Store that value in *NEW_RHS_OUT if so,
|
- RHS is an integer constant. Store that value in *NEW_RHS_OUT if so,
|
but leave *TYPE_OUT untouched. */
|
but leave *TYPE_OUT untouched. */
|
|
|
static bool
|
static bool
|
is_widening_mult_rhs_p (tree type, tree rhs, tree *type_out,
|
is_widening_mult_rhs_p (tree type, tree rhs, tree *type_out,
|
tree *new_rhs_out)
|
tree *new_rhs_out)
|
{
|
{
|
gimple stmt;
|
gimple stmt;
|
tree type1, rhs1;
|
tree type1, rhs1;
|
enum tree_code rhs_code;
|
enum tree_code rhs_code;
|
|
|
if (TREE_CODE (rhs) == SSA_NAME)
|
if (TREE_CODE (rhs) == SSA_NAME)
|
{
|
{
|
stmt = SSA_NAME_DEF_STMT (rhs);
|
stmt = SSA_NAME_DEF_STMT (rhs);
|
if (is_gimple_assign (stmt))
|
if (is_gimple_assign (stmt))
|
{
|
{
|
rhs_code = gimple_assign_rhs_code (stmt);
|
rhs_code = gimple_assign_rhs_code (stmt);
|
if (TREE_CODE (type) == INTEGER_TYPE
|
if (TREE_CODE (type) == INTEGER_TYPE
|
? !CONVERT_EXPR_CODE_P (rhs_code)
|
? !CONVERT_EXPR_CODE_P (rhs_code)
|
: rhs_code != FIXED_CONVERT_EXPR)
|
: rhs_code != FIXED_CONVERT_EXPR)
|
rhs1 = rhs;
|
rhs1 = rhs;
|
else
|
else
|
{
|
{
|
rhs1 = gimple_assign_rhs1 (stmt);
|
rhs1 = gimple_assign_rhs1 (stmt);
|
|
|
if (TREE_CODE (rhs1) == INTEGER_CST)
|
if (TREE_CODE (rhs1) == INTEGER_CST)
|
{
|
{
|
*new_rhs_out = rhs1;
|
*new_rhs_out = rhs1;
|
*type_out = NULL;
|
*type_out = NULL;
|
return true;
|
return true;
|
}
|
}
|
}
|
}
|
}
|
}
|
else
|
else
|
rhs1 = rhs;
|
rhs1 = rhs;
|
|
|
type1 = TREE_TYPE (rhs1);
|
type1 = TREE_TYPE (rhs1);
|
|
|
if (TREE_CODE (type1) != TREE_CODE (type)
|
if (TREE_CODE (type1) != TREE_CODE (type)
|
|| TYPE_PRECISION (type1) * 2 > TYPE_PRECISION (type))
|
|| TYPE_PRECISION (type1) * 2 > TYPE_PRECISION (type))
|
return false;
|
return false;
|
|
|
*new_rhs_out = rhs1;
|
*new_rhs_out = rhs1;
|
*type_out = type1;
|
*type_out = type1;
|
return true;
|
return true;
|
}
|
}
|
|
|
if (TREE_CODE (rhs) == INTEGER_CST)
|
if (TREE_CODE (rhs) == INTEGER_CST)
|
{
|
{
|
*new_rhs_out = rhs;
|
*new_rhs_out = rhs;
|
*type_out = NULL;
|
*type_out = NULL;
|
return true;
|
return true;
|
}
|
}
|
|
|
return false;
|
return false;
|
}
|
}
|
|
|
/* Return true if STMT performs a widening multiplication, assuming the
|
/* Return true if STMT performs a widening multiplication, assuming the
|
output type is TYPE. If so, store the unwidened types of the operands
|
output type is TYPE. If so, store the unwidened types of the operands
|
in *TYPE1_OUT and *TYPE2_OUT respectively. Also fill *RHS1_OUT and
|
in *TYPE1_OUT and *TYPE2_OUT respectively. Also fill *RHS1_OUT and
|
*RHS2_OUT such that converting those operands to types *TYPE1_OUT
|
*RHS2_OUT such that converting those operands to types *TYPE1_OUT
|
and *TYPE2_OUT would give the operands of the multiplication. */
|
and *TYPE2_OUT would give the operands of the multiplication. */
|
|
|
static bool
|
static bool
|
is_widening_mult_p (gimple stmt,
|
is_widening_mult_p (gimple stmt,
|
tree *type1_out, tree *rhs1_out,
|
tree *type1_out, tree *rhs1_out,
|
tree *type2_out, tree *rhs2_out)
|
tree *type2_out, tree *rhs2_out)
|
{
|
{
|
tree type = TREE_TYPE (gimple_assign_lhs (stmt));
|
tree type = TREE_TYPE (gimple_assign_lhs (stmt));
|
|
|
if (TREE_CODE (type) != INTEGER_TYPE
|
if (TREE_CODE (type) != INTEGER_TYPE
|
&& TREE_CODE (type) != FIXED_POINT_TYPE)
|
&& TREE_CODE (type) != FIXED_POINT_TYPE)
|
return false;
|
return false;
|
|
|
if (!is_widening_mult_rhs_p (type, gimple_assign_rhs1 (stmt), type1_out,
|
if (!is_widening_mult_rhs_p (type, gimple_assign_rhs1 (stmt), type1_out,
|
rhs1_out))
|
rhs1_out))
|
return false;
|
return false;
|
|
|
if (!is_widening_mult_rhs_p (type, gimple_assign_rhs2 (stmt), type2_out,
|
if (!is_widening_mult_rhs_p (type, gimple_assign_rhs2 (stmt), type2_out,
|
rhs2_out))
|
rhs2_out))
|
return false;
|
return false;
|
|
|
if (*type1_out == NULL)
|
if (*type1_out == NULL)
|
{
|
{
|
if (*type2_out == NULL || !int_fits_type_p (*rhs1_out, *type2_out))
|
if (*type2_out == NULL || !int_fits_type_p (*rhs1_out, *type2_out))
|
return false;
|
return false;
|
*type1_out = *type2_out;
|
*type1_out = *type2_out;
|
}
|
}
|
|
|
if (*type2_out == NULL)
|
if (*type2_out == NULL)
|
{
|
{
|
if (!int_fits_type_p (*rhs2_out, *type1_out))
|
if (!int_fits_type_p (*rhs2_out, *type1_out))
|
return false;
|
return false;
|
*type2_out = *type1_out;
|
*type2_out = *type1_out;
|
}
|
}
|
|
|
/* Ensure that the larger of the two operands comes first. */
|
/* Ensure that the larger of the two operands comes first. */
|
if (TYPE_PRECISION (*type1_out) < TYPE_PRECISION (*type2_out))
|
if (TYPE_PRECISION (*type1_out) < TYPE_PRECISION (*type2_out))
|
{
|
{
|
tree tmp;
|
tree tmp;
|
tmp = *type1_out;
|
tmp = *type1_out;
|
*type1_out = *type2_out;
|
*type1_out = *type2_out;
|
*type2_out = tmp;
|
*type2_out = tmp;
|
tmp = *rhs1_out;
|
tmp = *rhs1_out;
|
*rhs1_out = *rhs2_out;
|
*rhs1_out = *rhs2_out;
|
*rhs2_out = tmp;
|
*rhs2_out = tmp;
|
}
|
}
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
/* Process a single gimple statement STMT, which has a MULT_EXPR as
|
/* Process a single gimple statement STMT, which has a MULT_EXPR as
|
its rhs, and try to convert it into a WIDEN_MULT_EXPR. The return
|
its rhs, and try to convert it into a WIDEN_MULT_EXPR. The return
|
value is true iff we converted the statement. */
|
value is true iff we converted the statement. */
|
|
|
static bool
|
static bool
|
convert_mult_to_widen (gimple stmt, gimple_stmt_iterator *gsi)
|
convert_mult_to_widen (gimple stmt, gimple_stmt_iterator *gsi)
|
{
|
{
|
tree lhs, rhs1, rhs2, type, type1, type2, tmp = NULL;
|
tree lhs, rhs1, rhs2, type, type1, type2, tmp = NULL;
|
enum insn_code handler;
|
enum insn_code handler;
|
enum machine_mode to_mode, from_mode, actual_mode;
|
enum machine_mode to_mode, from_mode, actual_mode;
|
optab op;
|
optab op;
|
int actual_precision;
|
int actual_precision;
|
location_t loc = gimple_location (stmt);
|
location_t loc = gimple_location (stmt);
|
bool from_unsigned1, from_unsigned2;
|
bool from_unsigned1, from_unsigned2;
|
|
|
lhs = gimple_assign_lhs (stmt);
|
lhs = gimple_assign_lhs (stmt);
|
type = TREE_TYPE (lhs);
|
type = TREE_TYPE (lhs);
|
if (TREE_CODE (type) != INTEGER_TYPE)
|
if (TREE_CODE (type) != INTEGER_TYPE)
|
return false;
|
return false;
|
|
|
if (!is_widening_mult_p (stmt, &type1, &rhs1, &type2, &rhs2))
|
if (!is_widening_mult_p (stmt, &type1, &rhs1, &type2, &rhs2))
|
return false;
|
return false;
|
|
|
to_mode = TYPE_MODE (type);
|
to_mode = TYPE_MODE (type);
|
from_mode = TYPE_MODE (type1);
|
from_mode = TYPE_MODE (type1);
|
from_unsigned1 = TYPE_UNSIGNED (type1);
|
from_unsigned1 = TYPE_UNSIGNED (type1);
|
from_unsigned2 = TYPE_UNSIGNED (type2);
|
from_unsigned2 = TYPE_UNSIGNED (type2);
|
|
|
if (from_unsigned1 && from_unsigned2)
|
if (from_unsigned1 && from_unsigned2)
|
op = umul_widen_optab;
|
op = umul_widen_optab;
|
else if (!from_unsigned1 && !from_unsigned2)
|
else if (!from_unsigned1 && !from_unsigned2)
|
op = smul_widen_optab;
|
op = smul_widen_optab;
|
else
|
else
|
op = usmul_widen_optab;
|
op = usmul_widen_optab;
|
|
|
handler = find_widening_optab_handler_and_mode (op, to_mode, from_mode,
|
handler = find_widening_optab_handler_and_mode (op, to_mode, from_mode,
|
0, &actual_mode);
|
0, &actual_mode);
|
|
|
if (handler == CODE_FOR_nothing)
|
if (handler == CODE_FOR_nothing)
|
{
|
{
|
if (op != smul_widen_optab)
|
if (op != smul_widen_optab)
|
{
|
{
|
/* We can use a signed multiply with unsigned types as long as
|
/* We can use a signed multiply with unsigned types as long as
|
there is a wider mode to use, or it is the smaller of the two
|
there is a wider mode to use, or it is the smaller of the two
|
types that is unsigned. Note that type1 >= type2, always. */
|
types that is unsigned. Note that type1 >= type2, always. */
|
if ((TYPE_UNSIGNED (type1)
|
if ((TYPE_UNSIGNED (type1)
|
&& TYPE_PRECISION (type1) == GET_MODE_PRECISION (from_mode))
|
&& TYPE_PRECISION (type1) == GET_MODE_PRECISION (from_mode))
|
|| (TYPE_UNSIGNED (type2)
|
|| (TYPE_UNSIGNED (type2)
|
&& TYPE_PRECISION (type2) == GET_MODE_PRECISION (from_mode)))
|
&& TYPE_PRECISION (type2) == GET_MODE_PRECISION (from_mode)))
|
{
|
{
|
from_mode = GET_MODE_WIDER_MODE (from_mode);
|
from_mode = GET_MODE_WIDER_MODE (from_mode);
|
if (GET_MODE_SIZE (to_mode) <= GET_MODE_SIZE (from_mode))
|
if (GET_MODE_SIZE (to_mode) <= GET_MODE_SIZE (from_mode))
|
return false;
|
return false;
|
}
|
}
|
|
|
op = smul_widen_optab;
|
op = smul_widen_optab;
|
handler = find_widening_optab_handler_and_mode (op, to_mode,
|
handler = find_widening_optab_handler_and_mode (op, to_mode,
|
from_mode, 0,
|
from_mode, 0,
|
&actual_mode);
|
&actual_mode);
|
|
|
if (handler == CODE_FOR_nothing)
|
if (handler == CODE_FOR_nothing)
|
return false;
|
return false;
|
|
|
from_unsigned1 = from_unsigned2 = false;
|
from_unsigned1 = from_unsigned2 = false;
|
}
|
}
|
else
|
else
|
return false;
|
return false;
|
}
|
}
|
|
|
/* Ensure that the inputs to the handler are in the correct precison
|
/* Ensure that the inputs to the handler are in the correct precison
|
for the opcode. This will be the full mode size. */
|
for the opcode. This will be the full mode size. */
|
actual_precision = GET_MODE_PRECISION (actual_mode);
|
actual_precision = GET_MODE_PRECISION (actual_mode);
|
if (actual_precision != TYPE_PRECISION (type1)
|
if (actual_precision != TYPE_PRECISION (type1)
|
|| from_unsigned1 != TYPE_UNSIGNED (type1))
|
|| from_unsigned1 != TYPE_UNSIGNED (type1))
|
{
|
{
|
tmp = create_tmp_var (build_nonstandard_integer_type
|
tmp = create_tmp_var (build_nonstandard_integer_type
|
(actual_precision, from_unsigned1),
|
(actual_precision, from_unsigned1),
|
NULL);
|
NULL);
|
rhs1 = build_and_insert_cast (gsi, loc, tmp, rhs1);
|
rhs1 = build_and_insert_cast (gsi, loc, tmp, rhs1);
|
}
|
}
|
if (actual_precision != TYPE_PRECISION (type2)
|
if (actual_precision != TYPE_PRECISION (type2)
|
|| from_unsigned2 != TYPE_UNSIGNED (type2))
|
|| from_unsigned2 != TYPE_UNSIGNED (type2))
|
{
|
{
|
/* Reuse the same type info, if possible. */
|
/* Reuse the same type info, if possible. */
|
if (!tmp || from_unsigned1 != from_unsigned2)
|
if (!tmp || from_unsigned1 != from_unsigned2)
|
tmp = create_tmp_var (build_nonstandard_integer_type
|
tmp = create_tmp_var (build_nonstandard_integer_type
|
(actual_precision, from_unsigned2),
|
(actual_precision, from_unsigned2),
|
NULL);
|
NULL);
|
rhs2 = build_and_insert_cast (gsi, loc, tmp, rhs2);
|
rhs2 = build_and_insert_cast (gsi, loc, tmp, rhs2);
|
}
|
}
|
|
|
/* Handle constants. */
|
/* Handle constants. */
|
if (TREE_CODE (rhs1) == INTEGER_CST)
|
if (TREE_CODE (rhs1) == INTEGER_CST)
|
rhs1 = fold_convert (type1, rhs1);
|
rhs1 = fold_convert (type1, rhs1);
|
if (TREE_CODE (rhs2) == INTEGER_CST)
|
if (TREE_CODE (rhs2) == INTEGER_CST)
|
rhs2 = fold_convert (type2, rhs2);
|
rhs2 = fold_convert (type2, rhs2);
|
|
|
gimple_assign_set_rhs1 (stmt, rhs1);
|
gimple_assign_set_rhs1 (stmt, rhs1);
|
gimple_assign_set_rhs2 (stmt, rhs2);
|
gimple_assign_set_rhs2 (stmt, rhs2);
|
gimple_assign_set_rhs_code (stmt, WIDEN_MULT_EXPR);
|
gimple_assign_set_rhs_code (stmt, WIDEN_MULT_EXPR);
|
update_stmt (stmt);
|
update_stmt (stmt);
|
widen_mul_stats.widen_mults_inserted++;
|
widen_mul_stats.widen_mults_inserted++;
|
return true;
|
return true;
|
}
|
}
|
|
|
/* Process a single gimple statement STMT, which is found at the
|
/* Process a single gimple statement STMT, which is found at the
|
iterator GSI and has a either a PLUS_EXPR or a MINUS_EXPR as its
|
iterator GSI and has a either a PLUS_EXPR or a MINUS_EXPR as its
|
rhs (given by CODE), and try to convert it into a
|
rhs (given by CODE), and try to convert it into a
|
WIDEN_MULT_PLUS_EXPR or a WIDEN_MULT_MINUS_EXPR. The return value
|
WIDEN_MULT_PLUS_EXPR or a WIDEN_MULT_MINUS_EXPR. The return value
|
is true iff we converted the statement. */
|
is true iff we converted the statement. */
|
|
|
static bool
|
static bool
|
convert_plusminus_to_widen (gimple_stmt_iterator *gsi, gimple stmt,
|
convert_plusminus_to_widen (gimple_stmt_iterator *gsi, gimple stmt,
|
enum tree_code code)
|
enum tree_code code)
|
{
|
{
|
gimple rhs1_stmt = NULL, rhs2_stmt = NULL;
|
gimple rhs1_stmt = NULL, rhs2_stmt = NULL;
|
gimple conv1_stmt = NULL, conv2_stmt = NULL, conv_stmt;
|
gimple conv1_stmt = NULL, conv2_stmt = NULL, conv_stmt;
|
tree type, type1, type2, optype, tmp = NULL;
|
tree type, type1, type2, optype, tmp = NULL;
|
tree lhs, rhs1, rhs2, mult_rhs1, mult_rhs2, add_rhs;
|
tree lhs, rhs1, rhs2, mult_rhs1, mult_rhs2, add_rhs;
|
enum tree_code rhs1_code = ERROR_MARK, rhs2_code = ERROR_MARK;
|
enum tree_code rhs1_code = ERROR_MARK, rhs2_code = ERROR_MARK;
|
optab this_optab;
|
optab this_optab;
|
enum tree_code wmult_code;
|
enum tree_code wmult_code;
|
enum insn_code handler;
|
enum insn_code handler;
|
enum machine_mode to_mode, from_mode, actual_mode;
|
enum machine_mode to_mode, from_mode, actual_mode;
|
location_t loc = gimple_location (stmt);
|
location_t loc = gimple_location (stmt);
|
int actual_precision;
|
int actual_precision;
|
bool from_unsigned1, from_unsigned2;
|
bool from_unsigned1, from_unsigned2;
|
|
|
lhs = gimple_assign_lhs (stmt);
|
lhs = gimple_assign_lhs (stmt);
|
type = TREE_TYPE (lhs);
|
type = TREE_TYPE (lhs);
|
if (TREE_CODE (type) != INTEGER_TYPE
|
if (TREE_CODE (type) != INTEGER_TYPE
|
&& TREE_CODE (type) != FIXED_POINT_TYPE)
|
&& TREE_CODE (type) != FIXED_POINT_TYPE)
|
return false;
|
return false;
|
|
|
if (code == MINUS_EXPR)
|
if (code == MINUS_EXPR)
|
wmult_code = WIDEN_MULT_MINUS_EXPR;
|
wmult_code = WIDEN_MULT_MINUS_EXPR;
|
else
|
else
|
wmult_code = WIDEN_MULT_PLUS_EXPR;
|
wmult_code = WIDEN_MULT_PLUS_EXPR;
|
|
|
rhs1 = gimple_assign_rhs1 (stmt);
|
rhs1 = gimple_assign_rhs1 (stmt);
|
rhs2 = gimple_assign_rhs2 (stmt);
|
rhs2 = gimple_assign_rhs2 (stmt);
|
|
|
if (TREE_CODE (rhs1) == SSA_NAME)
|
if (TREE_CODE (rhs1) == SSA_NAME)
|
{
|
{
|
rhs1_stmt = SSA_NAME_DEF_STMT (rhs1);
|
rhs1_stmt = SSA_NAME_DEF_STMT (rhs1);
|
if (is_gimple_assign (rhs1_stmt))
|
if (is_gimple_assign (rhs1_stmt))
|
rhs1_code = gimple_assign_rhs_code (rhs1_stmt);
|
rhs1_code = gimple_assign_rhs_code (rhs1_stmt);
|
}
|
}
|
|
|
if (TREE_CODE (rhs2) == SSA_NAME)
|
if (TREE_CODE (rhs2) == SSA_NAME)
|
{
|
{
|
rhs2_stmt = SSA_NAME_DEF_STMT (rhs2);
|
rhs2_stmt = SSA_NAME_DEF_STMT (rhs2);
|
if (is_gimple_assign (rhs2_stmt))
|
if (is_gimple_assign (rhs2_stmt))
|
rhs2_code = gimple_assign_rhs_code (rhs2_stmt);
|
rhs2_code = gimple_assign_rhs_code (rhs2_stmt);
|
}
|
}
|
|
|
/* Allow for one conversion statement between the multiply
|
/* Allow for one conversion statement between the multiply
|
and addition/subtraction statement. If there are more than
|
and addition/subtraction statement. If there are more than
|
one conversions then we assume they would invalidate this
|
one conversions then we assume they would invalidate this
|
transformation. If that's not the case then they should have
|
transformation. If that's not the case then they should have
|
been folded before now. */
|
been folded before now. */
|
if (CONVERT_EXPR_CODE_P (rhs1_code))
|
if (CONVERT_EXPR_CODE_P (rhs1_code))
|
{
|
{
|
conv1_stmt = rhs1_stmt;
|
conv1_stmt = rhs1_stmt;
|
rhs1 = gimple_assign_rhs1 (rhs1_stmt);
|
rhs1 = gimple_assign_rhs1 (rhs1_stmt);
|
if (TREE_CODE (rhs1) == SSA_NAME)
|
if (TREE_CODE (rhs1) == SSA_NAME)
|
{
|
{
|
rhs1_stmt = SSA_NAME_DEF_STMT (rhs1);
|
rhs1_stmt = SSA_NAME_DEF_STMT (rhs1);
|
if (is_gimple_assign (rhs1_stmt))
|
if (is_gimple_assign (rhs1_stmt))
|
rhs1_code = gimple_assign_rhs_code (rhs1_stmt);
|
rhs1_code = gimple_assign_rhs_code (rhs1_stmt);
|
}
|
}
|
else
|
else
|
return false;
|
return false;
|
}
|
}
|
if (CONVERT_EXPR_CODE_P (rhs2_code))
|
if (CONVERT_EXPR_CODE_P (rhs2_code))
|
{
|
{
|
conv2_stmt = rhs2_stmt;
|
conv2_stmt = rhs2_stmt;
|
rhs2 = gimple_assign_rhs1 (rhs2_stmt);
|
rhs2 = gimple_assign_rhs1 (rhs2_stmt);
|
if (TREE_CODE (rhs2) == SSA_NAME)
|
if (TREE_CODE (rhs2) == SSA_NAME)
|
{
|
{
|
rhs2_stmt = SSA_NAME_DEF_STMT (rhs2);
|
rhs2_stmt = SSA_NAME_DEF_STMT (rhs2);
|
if (is_gimple_assign (rhs2_stmt))
|
if (is_gimple_assign (rhs2_stmt))
|
rhs2_code = gimple_assign_rhs_code (rhs2_stmt);
|
rhs2_code = gimple_assign_rhs_code (rhs2_stmt);
|
}
|
}
|
else
|
else
|
return false;
|
return false;
|
}
|
}
|
|
|
/* If code is WIDEN_MULT_EXPR then it would seem unnecessary to call
|
/* If code is WIDEN_MULT_EXPR then it would seem unnecessary to call
|
is_widening_mult_p, but we still need the rhs returns.
|
is_widening_mult_p, but we still need the rhs returns.
|
|
|
It might also appear that it would be sufficient to use the existing
|
It might also appear that it would be sufficient to use the existing
|
operands of the widening multiply, but that would limit the choice of
|
operands of the widening multiply, but that would limit the choice of
|
multiply-and-accumulate instructions. */
|
multiply-and-accumulate instructions. */
|
if (code == PLUS_EXPR
|
if (code == PLUS_EXPR
|
&& (rhs1_code == MULT_EXPR || rhs1_code == WIDEN_MULT_EXPR))
|
&& (rhs1_code == MULT_EXPR || rhs1_code == WIDEN_MULT_EXPR))
|
{
|
{
|
if (!is_widening_mult_p (rhs1_stmt, &type1, &mult_rhs1,
|
if (!is_widening_mult_p (rhs1_stmt, &type1, &mult_rhs1,
|
&type2, &mult_rhs2))
|
&type2, &mult_rhs2))
|
return false;
|
return false;
|
add_rhs = rhs2;
|
add_rhs = rhs2;
|
conv_stmt = conv1_stmt;
|
conv_stmt = conv1_stmt;
|
}
|
}
|
else if (rhs2_code == MULT_EXPR || rhs2_code == WIDEN_MULT_EXPR)
|
else if (rhs2_code == MULT_EXPR || rhs2_code == WIDEN_MULT_EXPR)
|
{
|
{
|
if (!is_widening_mult_p (rhs2_stmt, &type1, &mult_rhs1,
|
if (!is_widening_mult_p (rhs2_stmt, &type1, &mult_rhs1,
|
&type2, &mult_rhs2))
|
&type2, &mult_rhs2))
|
return false;
|
return false;
|
add_rhs = rhs1;
|
add_rhs = rhs1;
|
conv_stmt = conv2_stmt;
|
conv_stmt = conv2_stmt;
|
}
|
}
|
else
|
else
|
return false;
|
return false;
|
|
|
to_mode = TYPE_MODE (type);
|
to_mode = TYPE_MODE (type);
|
from_mode = TYPE_MODE (type1);
|
from_mode = TYPE_MODE (type1);
|
from_unsigned1 = TYPE_UNSIGNED (type1);
|
from_unsigned1 = TYPE_UNSIGNED (type1);
|
from_unsigned2 = TYPE_UNSIGNED (type2);
|
from_unsigned2 = TYPE_UNSIGNED (type2);
|
optype = type1;
|
optype = type1;
|
|
|
/* There's no such thing as a mixed sign madd yet, so use a wider mode. */
|
/* There's no such thing as a mixed sign madd yet, so use a wider mode. */
|
if (from_unsigned1 != from_unsigned2)
|
if (from_unsigned1 != from_unsigned2)
|
{
|
{
|
if (!INTEGRAL_TYPE_P (type))
|
if (!INTEGRAL_TYPE_P (type))
|
return false;
|
return false;
|
/* We can use a signed multiply with unsigned types as long as
|
/* We can use a signed multiply with unsigned types as long as
|
there is a wider mode to use, or it is the smaller of the two
|
there is a wider mode to use, or it is the smaller of the two
|
types that is unsigned. Note that type1 >= type2, always. */
|
types that is unsigned. Note that type1 >= type2, always. */
|
if ((from_unsigned1
|
if ((from_unsigned1
|
&& TYPE_PRECISION (type1) == GET_MODE_PRECISION (from_mode))
|
&& TYPE_PRECISION (type1) == GET_MODE_PRECISION (from_mode))
|
|| (from_unsigned2
|
|| (from_unsigned2
|
&& TYPE_PRECISION (type2) == GET_MODE_PRECISION (from_mode)))
|
&& TYPE_PRECISION (type2) == GET_MODE_PRECISION (from_mode)))
|
{
|
{
|
from_mode = GET_MODE_WIDER_MODE (from_mode);
|
from_mode = GET_MODE_WIDER_MODE (from_mode);
|
if (GET_MODE_SIZE (from_mode) >= GET_MODE_SIZE (to_mode))
|
if (GET_MODE_SIZE (from_mode) >= GET_MODE_SIZE (to_mode))
|
return false;
|
return false;
|
}
|
}
|
|
|
from_unsigned1 = from_unsigned2 = false;
|
from_unsigned1 = from_unsigned2 = false;
|
optype = build_nonstandard_integer_type (GET_MODE_PRECISION (from_mode),
|
optype = build_nonstandard_integer_type (GET_MODE_PRECISION (from_mode),
|
false);
|
false);
|
}
|
}
|
|
|
/* If there was a conversion between the multiply and addition
|
/* If there was a conversion between the multiply and addition
|
then we need to make sure it fits a multiply-and-accumulate.
|
then we need to make sure it fits a multiply-and-accumulate.
|
The should be a single mode change which does not change the
|
The should be a single mode change which does not change the
|
value. */
|
value. */
|
if (conv_stmt)
|
if (conv_stmt)
|
{
|
{
|
/* We use the original, unmodified data types for this. */
|
/* We use the original, unmodified data types for this. */
|
tree from_type = TREE_TYPE (gimple_assign_rhs1 (conv_stmt));
|
tree from_type = TREE_TYPE (gimple_assign_rhs1 (conv_stmt));
|
tree to_type = TREE_TYPE (gimple_assign_lhs (conv_stmt));
|
tree to_type = TREE_TYPE (gimple_assign_lhs (conv_stmt));
|
int data_size = TYPE_PRECISION (type1) + TYPE_PRECISION (type2);
|
int data_size = TYPE_PRECISION (type1) + TYPE_PRECISION (type2);
|
bool is_unsigned = TYPE_UNSIGNED (type1) && TYPE_UNSIGNED (type2);
|
bool is_unsigned = TYPE_UNSIGNED (type1) && TYPE_UNSIGNED (type2);
|
|
|
if (TYPE_PRECISION (from_type) > TYPE_PRECISION (to_type))
|
if (TYPE_PRECISION (from_type) > TYPE_PRECISION (to_type))
|
{
|
{
|
/* Conversion is a truncate. */
|
/* Conversion is a truncate. */
|
if (TYPE_PRECISION (to_type) < data_size)
|
if (TYPE_PRECISION (to_type) < data_size)
|
return false;
|
return false;
|
}
|
}
|
else if (TYPE_PRECISION (from_type) < TYPE_PRECISION (to_type))
|
else if (TYPE_PRECISION (from_type) < TYPE_PRECISION (to_type))
|
{
|
{
|
/* Conversion is an extend. Check it's the right sort. */
|
/* Conversion is an extend. Check it's the right sort. */
|
if (TYPE_UNSIGNED (from_type) != is_unsigned
|
if (TYPE_UNSIGNED (from_type) != is_unsigned
|
&& !(is_unsigned && TYPE_PRECISION (from_type) > data_size))
|
&& !(is_unsigned && TYPE_PRECISION (from_type) > data_size))
|
return false;
|
return false;
|
}
|
}
|
/* else convert is a no-op for our purposes. */
|
/* else convert is a no-op for our purposes. */
|
}
|
}
|
|
|
/* Verify that the machine can perform a widening multiply
|
/* Verify that the machine can perform a widening multiply
|
accumulate in this mode/signedness combination, otherwise
|
accumulate in this mode/signedness combination, otherwise
|
this transformation is likely to pessimize code. */
|
this transformation is likely to pessimize code. */
|
this_optab = optab_for_tree_code (wmult_code, optype, optab_default);
|
this_optab = optab_for_tree_code (wmult_code, optype, optab_default);
|
handler = find_widening_optab_handler_and_mode (this_optab, to_mode,
|
handler = find_widening_optab_handler_and_mode (this_optab, to_mode,
|
from_mode, 0, &actual_mode);
|
from_mode, 0, &actual_mode);
|
|
|
if (handler == CODE_FOR_nothing)
|
if (handler == CODE_FOR_nothing)
|
return false;
|
return false;
|
|
|
/* Ensure that the inputs to the handler are in the correct precison
|
/* Ensure that the inputs to the handler are in the correct precison
|
for the opcode. This will be the full mode size. */
|
for the opcode. This will be the full mode size. */
|
actual_precision = GET_MODE_PRECISION (actual_mode);
|
actual_precision = GET_MODE_PRECISION (actual_mode);
|
if (actual_precision != TYPE_PRECISION (type1)
|
if (actual_precision != TYPE_PRECISION (type1)
|
|| from_unsigned1 != TYPE_UNSIGNED (type1))
|
|| from_unsigned1 != TYPE_UNSIGNED (type1))
|
{
|
{
|
tmp = create_tmp_var (build_nonstandard_integer_type
|
tmp = create_tmp_var (build_nonstandard_integer_type
|
(actual_precision, from_unsigned1),
|
(actual_precision, from_unsigned1),
|
NULL);
|
NULL);
|
mult_rhs1 = build_and_insert_cast (gsi, loc, tmp, mult_rhs1);
|
mult_rhs1 = build_and_insert_cast (gsi, loc, tmp, mult_rhs1);
|
}
|
}
|
if (actual_precision != TYPE_PRECISION (type2)
|
if (actual_precision != TYPE_PRECISION (type2)
|
|| from_unsigned2 != TYPE_UNSIGNED (type2))
|
|| from_unsigned2 != TYPE_UNSIGNED (type2))
|
{
|
{
|
if (!tmp || from_unsigned1 != from_unsigned2)
|
if (!tmp || from_unsigned1 != from_unsigned2)
|
tmp = create_tmp_var (build_nonstandard_integer_type
|
tmp = create_tmp_var (build_nonstandard_integer_type
|
(actual_precision, from_unsigned2),
|
(actual_precision, from_unsigned2),
|
NULL);
|
NULL);
|
mult_rhs2 = build_and_insert_cast (gsi, loc, tmp, mult_rhs2);
|
mult_rhs2 = build_and_insert_cast (gsi, loc, tmp, mult_rhs2);
|
}
|
}
|
|
|
if (!useless_type_conversion_p (type, TREE_TYPE (add_rhs)))
|
if (!useless_type_conversion_p (type, TREE_TYPE (add_rhs)))
|
add_rhs = build_and_insert_cast (gsi, loc, create_tmp_var (type, NULL),
|
add_rhs = build_and_insert_cast (gsi, loc, create_tmp_var (type, NULL),
|
add_rhs);
|
add_rhs);
|
|
|
/* Handle constants. */
|
/* Handle constants. */
|
if (TREE_CODE (mult_rhs1) == INTEGER_CST)
|
if (TREE_CODE (mult_rhs1) == INTEGER_CST)
|
mult_rhs1 = fold_convert (type1, mult_rhs1);
|
mult_rhs1 = fold_convert (type1, mult_rhs1);
|
if (TREE_CODE (mult_rhs2) == INTEGER_CST)
|
if (TREE_CODE (mult_rhs2) == INTEGER_CST)
|
mult_rhs2 = fold_convert (type2, mult_rhs2);
|
mult_rhs2 = fold_convert (type2, mult_rhs2);
|
|
|
gimple_assign_set_rhs_with_ops_1 (gsi, wmult_code, mult_rhs1, mult_rhs2,
|
gimple_assign_set_rhs_with_ops_1 (gsi, wmult_code, mult_rhs1, mult_rhs2,
|
add_rhs);
|
add_rhs);
|
update_stmt (gsi_stmt (*gsi));
|
update_stmt (gsi_stmt (*gsi));
|
widen_mul_stats.maccs_inserted++;
|
widen_mul_stats.maccs_inserted++;
|
return true;
|
return true;
|
}
|
}
|
|
|
/* Combine the multiplication at MUL_STMT with operands MULOP1 and MULOP2
|
/* Combine the multiplication at MUL_STMT with operands MULOP1 and MULOP2
|
with uses in additions and subtractions to form fused multiply-add
|
with uses in additions and subtractions to form fused multiply-add
|
operations. Returns true if successful and MUL_STMT should be removed. */
|
operations. Returns true if successful and MUL_STMT should be removed. */
|
|
|
static bool
|
static bool
|
convert_mult_to_fma (gimple mul_stmt, tree op1, tree op2)
|
convert_mult_to_fma (gimple mul_stmt, tree op1, tree op2)
|
{
|
{
|
tree mul_result = gimple_get_lhs (mul_stmt);
|
tree mul_result = gimple_get_lhs (mul_stmt);
|
tree type = TREE_TYPE (mul_result);
|
tree type = TREE_TYPE (mul_result);
|
gimple use_stmt, neguse_stmt, fma_stmt;
|
gimple use_stmt, neguse_stmt, fma_stmt;
|
use_operand_p use_p;
|
use_operand_p use_p;
|
imm_use_iterator imm_iter;
|
imm_use_iterator imm_iter;
|
|
|
if (FLOAT_TYPE_P (type)
|
if (FLOAT_TYPE_P (type)
|
&& flag_fp_contract_mode == FP_CONTRACT_OFF)
|
&& flag_fp_contract_mode == FP_CONTRACT_OFF)
|
return false;
|
return false;
|
|
|
/* We don't want to do bitfield reduction ops. */
|
/* We don't want to do bitfield reduction ops. */
|
if (INTEGRAL_TYPE_P (type)
|
if (INTEGRAL_TYPE_P (type)
|
&& (TYPE_PRECISION (type)
|
&& (TYPE_PRECISION (type)
|
!= GET_MODE_PRECISION (TYPE_MODE (type))))
|
!= GET_MODE_PRECISION (TYPE_MODE (type))))
|
return false;
|
return false;
|
|
|
/* If the target doesn't support it, don't generate it. We assume that
|
/* If the target doesn't support it, don't generate it. We assume that
|
if fma isn't available then fms, fnma or fnms are not either. */
|
if fma isn't available then fms, fnma or fnms are not either. */
|
if (optab_handler (fma_optab, TYPE_MODE (type)) == CODE_FOR_nothing)
|
if (optab_handler (fma_optab, TYPE_MODE (type)) == CODE_FOR_nothing)
|
return false;
|
return false;
|
|
|
/* If the multiplication has zero uses, it is kept around probably because
|
/* If the multiplication has zero uses, it is kept around probably because
|
of -fnon-call-exceptions. Don't optimize it away in that case,
|
of -fnon-call-exceptions. Don't optimize it away in that case,
|
it is DCE job. */
|
it is DCE job. */
|
if (has_zero_uses (mul_result))
|
if (has_zero_uses (mul_result))
|
return false;
|
return false;
|
|
|
/* Make sure that the multiplication statement becomes dead after
|
/* Make sure that the multiplication statement becomes dead after
|
the transformation, thus that all uses are transformed to FMAs.
|
the transformation, thus that all uses are transformed to FMAs.
|
This means we assume that an FMA operation has the same cost
|
This means we assume that an FMA operation has the same cost
|
as an addition. */
|
as an addition. */
|
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, mul_result)
|
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, mul_result)
|
{
|
{
|
enum tree_code use_code;
|
enum tree_code use_code;
|
tree result = mul_result;
|
tree result = mul_result;
|
bool negate_p = false;
|
bool negate_p = false;
|
|
|
use_stmt = USE_STMT (use_p);
|
use_stmt = USE_STMT (use_p);
|
|
|
if (is_gimple_debug (use_stmt))
|
if (is_gimple_debug (use_stmt))
|
continue;
|
continue;
|
|
|
/* For now restrict this operations to single basic blocks. In theory
|
/* For now restrict this operations to single basic blocks. In theory
|
we would want to support sinking the multiplication in
|
we would want to support sinking the multiplication in
|
m = a*b;
|
m = a*b;
|
if ()
|
if ()
|
ma = m + c;
|
ma = m + c;
|
else
|
else
|
d = m;
|
d = m;
|
to form a fma in the then block and sink the multiplication to the
|
to form a fma in the then block and sink the multiplication to the
|
else block. */
|
else block. */
|
if (gimple_bb (use_stmt) != gimple_bb (mul_stmt))
|
if (gimple_bb (use_stmt) != gimple_bb (mul_stmt))
|
return false;
|
return false;
|
|
|
if (!is_gimple_assign (use_stmt))
|
if (!is_gimple_assign (use_stmt))
|
return false;
|
return false;
|
|
|
use_code = gimple_assign_rhs_code (use_stmt);
|
use_code = gimple_assign_rhs_code (use_stmt);
|
|
|
/* A negate on the multiplication leads to FNMA. */
|
/* A negate on the multiplication leads to FNMA. */
|
if (use_code == NEGATE_EXPR)
|
if (use_code == NEGATE_EXPR)
|
{
|
{
|
ssa_op_iter iter;
|
ssa_op_iter iter;
|
use_operand_p usep;
|
use_operand_p usep;
|
|
|
result = gimple_assign_lhs (use_stmt);
|
result = gimple_assign_lhs (use_stmt);
|
|
|
/* Make sure the negate statement becomes dead with this
|
/* Make sure the negate statement becomes dead with this
|
single transformation. */
|
single transformation. */
|
if (!single_imm_use (gimple_assign_lhs (use_stmt),
|
if (!single_imm_use (gimple_assign_lhs (use_stmt),
|
&use_p, &neguse_stmt))
|
&use_p, &neguse_stmt))
|
return false;
|
return false;
|
|
|
/* Make sure the multiplication isn't also used on that stmt. */
|
/* Make sure the multiplication isn't also used on that stmt. */
|
FOR_EACH_PHI_OR_STMT_USE (usep, neguse_stmt, iter, SSA_OP_USE)
|
FOR_EACH_PHI_OR_STMT_USE (usep, neguse_stmt, iter, SSA_OP_USE)
|
if (USE_FROM_PTR (usep) == mul_result)
|
if (USE_FROM_PTR (usep) == mul_result)
|
return false;
|
return false;
|
|
|
/* Re-validate. */
|
/* Re-validate. */
|
use_stmt = neguse_stmt;
|
use_stmt = neguse_stmt;
|
if (gimple_bb (use_stmt) != gimple_bb (mul_stmt))
|
if (gimple_bb (use_stmt) != gimple_bb (mul_stmt))
|
return false;
|
return false;
|
if (!is_gimple_assign (use_stmt))
|
if (!is_gimple_assign (use_stmt))
|
return false;
|
return false;
|
|
|
use_code = gimple_assign_rhs_code (use_stmt);
|
use_code = gimple_assign_rhs_code (use_stmt);
|
negate_p = true;
|
negate_p = true;
|
}
|
}
|
|
|
switch (use_code)
|
switch (use_code)
|
{
|
{
|
case MINUS_EXPR:
|
case MINUS_EXPR:
|
if (gimple_assign_rhs2 (use_stmt) == result)
|
if (gimple_assign_rhs2 (use_stmt) == result)
|
negate_p = !negate_p;
|
negate_p = !negate_p;
|
break;
|
break;
|
case PLUS_EXPR:
|
case PLUS_EXPR:
|
break;
|
break;
|
default:
|
default:
|
/* FMA can only be formed from PLUS and MINUS. */
|
/* FMA can only be formed from PLUS and MINUS. */
|
return false;
|
return false;
|
}
|
}
|
|
|
/* We can't handle a * b + a * b. */
|
/* We can't handle a * b + a * b. */
|
if (gimple_assign_rhs1 (use_stmt) == gimple_assign_rhs2 (use_stmt))
|
if (gimple_assign_rhs1 (use_stmt) == gimple_assign_rhs2 (use_stmt))
|
return false;
|
return false;
|
|
|
/* While it is possible to validate whether or not the exact form
|
/* While it is possible to validate whether or not the exact form
|
that we've recognized is available in the backend, the assumption
|
that we've recognized is available in the backend, the assumption
|
is that the transformation is never a loss. For instance, suppose
|
is that the transformation is never a loss. For instance, suppose
|
the target only has the plain FMA pattern available. Consider
|
the target only has the plain FMA pattern available. Consider
|
a*b-c -> fma(a,b,-c): we've exchanged MUL+SUB for FMA+NEG, which
|
a*b-c -> fma(a,b,-c): we've exchanged MUL+SUB for FMA+NEG, which
|
is still two operations. Consider -(a*b)-c -> fma(-a,b,-c): we
|
is still two operations. Consider -(a*b)-c -> fma(-a,b,-c): we
|
still have 3 operations, but in the FMA form the two NEGs are
|
still have 3 operations, but in the FMA form the two NEGs are
|
independant and could be run in parallel. */
|
independant and could be run in parallel. */
|
}
|
}
|
|
|
FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, mul_result)
|
FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, mul_result)
|
{
|
{
|
gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
|
gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
|
enum tree_code use_code;
|
enum tree_code use_code;
|
tree addop, mulop1 = op1, result = mul_result;
|
tree addop, mulop1 = op1, result = mul_result;
|
bool negate_p = false;
|
bool negate_p = false;
|
|
|
if (is_gimple_debug (use_stmt))
|
if (is_gimple_debug (use_stmt))
|
continue;
|
continue;
|
|
|
use_code = gimple_assign_rhs_code (use_stmt);
|
use_code = gimple_assign_rhs_code (use_stmt);
|
if (use_code == NEGATE_EXPR)
|
if (use_code == NEGATE_EXPR)
|
{
|
{
|
result = gimple_assign_lhs (use_stmt);
|
result = gimple_assign_lhs (use_stmt);
|
single_imm_use (gimple_assign_lhs (use_stmt), &use_p, &neguse_stmt);
|
single_imm_use (gimple_assign_lhs (use_stmt), &use_p, &neguse_stmt);
|
gsi_remove (&gsi, true);
|
gsi_remove (&gsi, true);
|
release_defs (use_stmt);
|
release_defs (use_stmt);
|
|
|
use_stmt = neguse_stmt;
|
use_stmt = neguse_stmt;
|
gsi = gsi_for_stmt (use_stmt);
|
gsi = gsi_for_stmt (use_stmt);
|
use_code = gimple_assign_rhs_code (use_stmt);
|
use_code = gimple_assign_rhs_code (use_stmt);
|
negate_p = true;
|
negate_p = true;
|
}
|
}
|
|
|
if (gimple_assign_rhs1 (use_stmt) == result)
|
if (gimple_assign_rhs1 (use_stmt) == result)
|
{
|
{
|
addop = gimple_assign_rhs2 (use_stmt);
|
addop = gimple_assign_rhs2 (use_stmt);
|
/* a * b - c -> a * b + (-c) */
|
/* a * b - c -> a * b + (-c) */
|
if (gimple_assign_rhs_code (use_stmt) == MINUS_EXPR)
|
if (gimple_assign_rhs_code (use_stmt) == MINUS_EXPR)
|
addop = force_gimple_operand_gsi (&gsi,
|
addop = force_gimple_operand_gsi (&gsi,
|
build1 (NEGATE_EXPR,
|
build1 (NEGATE_EXPR,
|
type, addop),
|
type, addop),
|
true, NULL_TREE, true,
|
true, NULL_TREE, true,
|
GSI_SAME_STMT);
|
GSI_SAME_STMT);
|
}
|
}
|
else
|
else
|
{
|
{
|
addop = gimple_assign_rhs1 (use_stmt);
|
addop = gimple_assign_rhs1 (use_stmt);
|
/* a - b * c -> (-b) * c + a */
|
/* a - b * c -> (-b) * c + a */
|
if (gimple_assign_rhs_code (use_stmt) == MINUS_EXPR)
|
if (gimple_assign_rhs_code (use_stmt) == MINUS_EXPR)
|
negate_p = !negate_p;
|
negate_p = !negate_p;
|
}
|
}
|
|
|
if (negate_p)
|
if (negate_p)
|
mulop1 = force_gimple_operand_gsi (&gsi,
|
mulop1 = force_gimple_operand_gsi (&gsi,
|
build1 (NEGATE_EXPR,
|
build1 (NEGATE_EXPR,
|
type, mulop1),
|
type, mulop1),
|
true, NULL_TREE, true,
|
true, NULL_TREE, true,
|
GSI_SAME_STMT);
|
GSI_SAME_STMT);
|
|
|
fma_stmt = gimple_build_assign_with_ops3 (FMA_EXPR,
|
fma_stmt = gimple_build_assign_with_ops3 (FMA_EXPR,
|
gimple_assign_lhs (use_stmt),
|
gimple_assign_lhs (use_stmt),
|
mulop1, op2,
|
mulop1, op2,
|
addop);
|
addop);
|
gsi_replace (&gsi, fma_stmt, true);
|
gsi_replace (&gsi, fma_stmt, true);
|
widen_mul_stats.fmas_inserted++;
|
widen_mul_stats.fmas_inserted++;
|
}
|
}
|
|
|
return true;
|
return true;
|
}
|
}
|
|
|
/* Find integer multiplications where the operands are extended from
|
/* Find integer multiplications where the operands are extended from
|
smaller types, and replace the MULT_EXPR with a WIDEN_MULT_EXPR
|
smaller types, and replace the MULT_EXPR with a WIDEN_MULT_EXPR
|
where appropriate. */
|
where appropriate. */
|
|
|
static unsigned int
|
static unsigned int
|
execute_optimize_widening_mul (void)
|
execute_optimize_widening_mul (void)
|
{
|
{
|
basic_block bb;
|
basic_block bb;
|
bool cfg_changed = false;
|
bool cfg_changed = false;
|
|
|
memset (&widen_mul_stats, 0, sizeof (widen_mul_stats));
|
memset (&widen_mul_stats, 0, sizeof (widen_mul_stats));
|
|
|
FOR_EACH_BB (bb)
|
FOR_EACH_BB (bb)
|
{
|
{
|
gimple_stmt_iterator gsi;
|
gimple_stmt_iterator gsi;
|
|
|
for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi);)
|
for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi);)
|
{
|
{
|
gimple stmt = gsi_stmt (gsi);
|
gimple stmt = gsi_stmt (gsi);
|
enum tree_code code;
|
enum tree_code code;
|
|
|
if (is_gimple_assign (stmt))
|
if (is_gimple_assign (stmt))
|
{
|
{
|
code = gimple_assign_rhs_code (stmt);
|
code = gimple_assign_rhs_code (stmt);
|
switch (code)
|
switch (code)
|
{
|
{
|
case MULT_EXPR:
|
case MULT_EXPR:
|
if (!convert_mult_to_widen (stmt, &gsi)
|
if (!convert_mult_to_widen (stmt, &gsi)
|
&& convert_mult_to_fma (stmt,
|
&& convert_mult_to_fma (stmt,
|
gimple_assign_rhs1 (stmt),
|
gimple_assign_rhs1 (stmt),
|
gimple_assign_rhs2 (stmt)))
|
gimple_assign_rhs2 (stmt)))
|
{
|
{
|
gsi_remove (&gsi, true);
|
gsi_remove (&gsi, true);
|
release_defs (stmt);
|
release_defs (stmt);
|
continue;
|
continue;
|
}
|
}
|
break;
|
break;
|
|
|
case PLUS_EXPR:
|
case PLUS_EXPR:
|
case MINUS_EXPR:
|
case MINUS_EXPR:
|
convert_plusminus_to_widen (&gsi, stmt, code);
|
convert_plusminus_to_widen (&gsi, stmt, code);
|
break;
|
break;
|
|
|
default:;
|
default:;
|
}
|
}
|
}
|
}
|
else if (is_gimple_call (stmt)
|
else if (is_gimple_call (stmt)
|
&& gimple_call_lhs (stmt))
|
&& gimple_call_lhs (stmt))
|
{
|
{
|
tree fndecl = gimple_call_fndecl (stmt);
|
tree fndecl = gimple_call_fndecl (stmt);
|
if (fndecl
|
if (fndecl
|
&& DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
|
&& DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
|
{
|
{
|
switch (DECL_FUNCTION_CODE (fndecl))
|
switch (DECL_FUNCTION_CODE (fndecl))
|
{
|
{
|
case BUILT_IN_POWF:
|
case BUILT_IN_POWF:
|
case BUILT_IN_POW:
|
case BUILT_IN_POW:
|
case BUILT_IN_POWL:
|
case BUILT_IN_POWL:
|
if (TREE_CODE (gimple_call_arg (stmt, 1)) == REAL_CST
|
if (TREE_CODE (gimple_call_arg (stmt, 1)) == REAL_CST
|
&& REAL_VALUES_EQUAL
|
&& REAL_VALUES_EQUAL
|
(TREE_REAL_CST (gimple_call_arg (stmt, 1)),
|
(TREE_REAL_CST (gimple_call_arg (stmt, 1)),
|
dconst2)
|
dconst2)
|
&& convert_mult_to_fma (stmt,
|
&& convert_mult_to_fma (stmt,
|
gimple_call_arg (stmt, 0),
|
gimple_call_arg (stmt, 0),
|
gimple_call_arg (stmt, 0)))
|
gimple_call_arg (stmt, 0)))
|
{
|
{
|
unlink_stmt_vdef (stmt);
|
unlink_stmt_vdef (stmt);
|
gsi_remove (&gsi, true);
|
gsi_remove (&gsi, true);
|
release_defs (stmt);
|
release_defs (stmt);
|
if (gimple_purge_dead_eh_edges (bb))
|
if (gimple_purge_dead_eh_edges (bb))
|
cfg_changed = true;
|
cfg_changed = true;
|
continue;
|
continue;
|
}
|
}
|
break;
|
break;
|
|
|
default:;
|
default:;
|
}
|
}
|
}
|
}
|
}
|
}
|
gsi_next (&gsi);
|
gsi_next (&gsi);
|
}
|
}
|
}
|
}
|
|
|
statistics_counter_event (cfun, "widening multiplications inserted",
|
statistics_counter_event (cfun, "widening multiplications inserted",
|
widen_mul_stats.widen_mults_inserted);
|
widen_mul_stats.widen_mults_inserted);
|
statistics_counter_event (cfun, "widening maccs inserted",
|
statistics_counter_event (cfun, "widening maccs inserted",
|
widen_mul_stats.maccs_inserted);
|
widen_mul_stats.maccs_inserted);
|
statistics_counter_event (cfun, "fused multiply-adds inserted",
|
statistics_counter_event (cfun, "fused multiply-adds inserted",
|
widen_mul_stats.fmas_inserted);
|
widen_mul_stats.fmas_inserted);
|
|
|
return cfg_changed ? TODO_cleanup_cfg : 0;
|
return cfg_changed ? TODO_cleanup_cfg : 0;
|
}
|
}
|
|
|
static bool
|
static bool
|
gate_optimize_widening_mul (void)
|
gate_optimize_widening_mul (void)
|
{
|
{
|
return flag_expensive_optimizations && optimize;
|
return flag_expensive_optimizations && optimize;
|
}
|
}
|
|
|
struct gimple_opt_pass pass_optimize_widening_mul =
|
struct gimple_opt_pass pass_optimize_widening_mul =
|
{
|
{
|
{
|
{
|
GIMPLE_PASS,
|
GIMPLE_PASS,
|
"widening_mul", /* name */
|
"widening_mul", /* name */
|
gate_optimize_widening_mul, /* gate */
|
gate_optimize_widening_mul, /* gate */
|
execute_optimize_widening_mul, /* execute */
|
execute_optimize_widening_mul, /* execute */
|
NULL, /* sub */
|
NULL, /* sub */
|
NULL, /* next */
|
NULL, /* next */
|
0, /* static_pass_number */
|
0, /* static_pass_number */
|
TV_NONE, /* tv_id */
|
TV_NONE, /* tv_id */
|
PROP_ssa, /* properties_required */
|
PROP_ssa, /* properties_required */
|
0, /* properties_provided */
|
0, /* properties_provided */
|
0, /* properties_destroyed */
|
0, /* properties_destroyed */
|
0, /* todo_flags_start */
|
0, /* todo_flags_start */
|
TODO_verify_ssa
|
TODO_verify_ssa
|
| TODO_verify_stmts
|
| TODO_verify_stmts
|
| TODO_update_ssa /* todo_flags_finish */
|
| TODO_update_ssa /* todo_flags_finish */
|
}
|
}
|
};
|
};
|
|
|
