Subversion Repositories openrisc

/* Loop Vectorization

   Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc.

   Contributed by Dorit Naishlos <dorit@il.ibm.com>

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it under

the terms of the GNU General Public License as published by the Free

Software Foundation; either version 3, or (at your option) any later

version.

GCC is distributed in the hope that it will be useful, but WITHOUT ANY

WARRANTY; without even the implied warranty of MERCHANTABILITY or

FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

for more details.

You should have received a copy of the GNU General Public License

along with GCC; see the file COPYING3.  If not see

<http://www.gnu.org/licenses/>.  */

/* Loop Vectorization Pass.

   This pass tries to vectorize loops. This first implementation focuses on

   simple inner-most loops, with no conditional control flow, and a set of

   simple operations which vector form can be expressed using existing

   tree codes (PLUS, MULT etc).

   For example, the vectorizer transforms the following simple loop:

        short a[N]; short b[N]; short c[N]; int i;

        for (i=0; i<N; i++){

          a[i] = b[i] + c[i];

        }

   as if it was manually vectorized by rewriting the source code into:

        typedef int __attribute__((mode(V8HI))) v8hi;

        short a[N];  short b[N]; short c[N];   int i;

        v8hi *pa = (v8hi*)a, *pb = (v8hi*)b, *pc = (v8hi*)c;

        v8hi va, vb, vc;

        for (i=0; i<N/8; i++){

          vb = pb[i];

          vc = pc[i];

          va = vb + vc;

          pa[i] = va;

        }

        The main entry to this pass is vectorize_loops(), in which

   the vectorizer applies a set of analyses on a given set of loops,

   followed by the actual vectorization transformation for the loops that

   had successfully passed the analysis phase.

        Throughout this pass we make a distinction between two types of

   data: scalars (which are represented by SSA_NAMES), and memory references

   ("data-refs"). These two types of data require different handling both

   during analysis and transformation. The types of data-refs that the

   vectorizer currently supports are ARRAY_REFS which base is an array DECL

   (not a pointer), and INDIRECT_REFS through pointers; both array and pointer

   accesses are required to have a  simple (consecutive) access pattern.

   Analysis phase:

   ===============

        The driver for the analysis phase is vect_analyze_loop_nest().

   It applies a set of analyses, some of which rely on the scalar evolution

   analyzer (scev) developed by Sebastian Pop.

        During the analysis phase the vectorizer records some information

   per stmt in a "stmt_vec_info" struct which is attached to each stmt in the

   loop, as well as general information about the loop as a whole, which is

   recorded in a "loop_vec_info" struct attached to each loop.

   Transformation phase:

   =====================

        The loop transformation phase scans all the stmts in the loop, and

   creates a vector stmt (or a sequence of stmts) for each scalar stmt S in

   the loop that needs to be vectorized. It insert the vector code sequence

   just before the scalar stmt S, and records a pointer to the vector code

   in STMT_VINFO_VEC_STMT (stmt_info) (stmt_info is the stmt_vec_info struct

   attached to S). This pointer will be used for the vectorization of following

   stmts which use the def of stmt S. Stmt S is removed if it writes to memory;

   otherwise, we rely on dead code elimination for removing it.

        For example, say stmt S1 was vectorized into stmt VS1:

   VS1: vb = px[i];

   S1:  b = x[i];    STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1

   S2:  a = b;

   To vectorize stmt S2, the vectorizer first finds the stmt that defines

   the operand 'b' (S1), and gets the relevant vector def 'vb' from the

   vector stmt VS1 pointed to by STMT_VINFO_VEC_STMT (stmt_info (S1)). The

   resulting sequence would be:

   VS1: vb = px[i];

   S1:  b = x[i];       STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1

   VS2: va = vb;

   S2:  a = b;          STMT_VINFO_VEC_STMT (stmt_info (S2)) = VS2

        Operands that are not SSA_NAMEs, are data-refs that appear in

   load/store operations (like 'x[i]' in S1), and are handled differently.

   Target modeling:

   =================

        Currently the only target specific information that is used is the

   size of the vector (in bytes) - "UNITS_PER_SIMD_WORD". Targets that can

   support different sizes of vectors, for now will need to specify one value

   for "UNITS_PER_SIMD_WORD". More flexibility will be added in the future.

        Since we only vectorize operations which vector form can be

   expressed using existing tree codes, to verify that an operation is

   supported, the vectorizer checks the relevant optab at the relevant

   machine_mode (e.g, add_optab->handlers[(int) V8HImode].insn_code). If

   the value found is CODE_FOR_nothing, then there's no target support, and

   we can't vectorize the stmt.

   For additional information on this project see:

   http://gcc.gnu.org/projects/tree-ssa/vectorization.html

*/

#include "config.h"

#include "system.h"

#include "coretypes.h"

#include "tm.h"

#include "ggc.h"

#include "tree.h"

#include "target.h"

#include "rtl.h"

#include "basic-block.h"

#include "diagnostic.h"

#include "tree-flow.h"

#include "tree-dump.h"

#include "timevar.h"

#include "cfgloop.h"

#include "cfglayout.h"

#include "expr.h"

#include "optabs.h"

#include "params.h"

#include "toplev.h"

#include "tree-chrec.h"

#include "tree-data-ref.h"

#include "tree-scalar-evolution.h"

#include "input.h"

#include "tree-vectorizer.h"

#include "tree-pass.h"

/*************************************************************************

  Simple Loop Peeling Utilities

 *************************************************************************/

static struct loop *slpeel_tree_duplicate_loop_to_edge_cfg

  (struct loop *, struct loops *, edge);

static void slpeel_update_phis_for_duplicate_loop

  (struct loop *, struct loop *, bool after);

static void slpeel_update_phi_nodes_for_guard1

  (edge, struct loop *, bool, basic_block *, bitmap *);

static void slpeel_update_phi_nodes_for_guard2

  (edge, struct loop *, bool, basic_block *);

static edge slpeel_add_loop_guard (basic_block, tree, basic_block, basic_block);

static void rename_use_op (use_operand_p);

static void rename_variables_in_bb (basic_block);

static void rename_variables_in_loop (struct loop *);

/*************************************************************************

  General Vectorization Utilities

 *************************************************************************/

static void vect_set_dump_settings (void);

/* vect_dump will be set to stderr or dump_file if exist.  */

FILE *vect_dump;

/* vect_verbosity_level set to an invalid value

   to mark that it's uninitialized.  */

enum verbosity_levels vect_verbosity_level = MAX_VERBOSITY_LEVEL;

/* Number of loops, at the beginning of vectorization.  */

unsigned int vect_loops_num;

/* Loop location.  */

static LOC vect_loop_location;

/* Bitmap of virtual variables to be renamed.  */

bitmap vect_vnames_to_rename;

/*************************************************************************

  Simple Loop Peeling Utilities

  Utilities to support loop peeling for vectorization purposes.

 *************************************************************************/

/* Renames the use *OP_P.  */

static void

rename_use_op (use_operand_p op_p)

{

  tree new_name;

  if (TREE_CODE (USE_FROM_PTR (op_p)) != SSA_NAME)

    return;

  new_name = get_current_def (USE_FROM_PTR (op_p));

  /* Something defined outside of the loop.  */

  if (!new_name)

    return;

  /* An ordinary ssa name defined in the loop.  */

  SET_USE (op_p, new_name);

}

/* Renames the variables in basic block BB.  */

static void

rename_variables_in_bb (basic_block bb)

{

  tree phi;

  block_stmt_iterator bsi;

  tree stmt;

  use_operand_p use_p;

  ssa_op_iter iter;

  edge e;

  edge_iterator ei;

  struct loop *loop = bb->loop_father;

  for (bsi = bsi_start (bb); !bsi_end_p (bsi); bsi_next (&bsi))

    {

      stmt = bsi_stmt (bsi);

      FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter,

                                 (SSA_OP_ALL_USES | SSA_OP_ALL_KILLS))

        rename_use_op (use_p);

    }

  FOR_EACH_EDGE (e, ei, bb->succs)

    {

      if (!flow_bb_inside_loop_p (loop, e->dest))

        continue;

      for (phi = phi_nodes (e->dest); phi; phi = PHI_CHAIN (phi))

        rename_use_op (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e));

    }

}

/* Renames variables in new generated LOOP.  */

static void

rename_variables_in_loop (struct loop *loop)

{

  unsigned i;

  basic_block *bbs;

  bbs = get_loop_body (loop);

  for (i = 0; i < loop->num_nodes; i++)

    rename_variables_in_bb (bbs[i]);

  free (bbs);

}

/* Update the PHI nodes of NEW_LOOP.

   NEW_LOOP is a duplicate of ORIG_LOOP.

   AFTER indicates whether NEW_LOOP executes before or after ORIG_LOOP:

   AFTER is true if NEW_LOOP executes after ORIG_LOOP, and false if it

   executes before it.  */

static void

slpeel_update_phis_for_duplicate_loop (struct loop *orig_loop,

                                       struct loop *new_loop, bool after)

{

  tree new_ssa_name;

  tree phi_new, phi_orig;

  tree def;

  edge orig_loop_latch = loop_latch_edge (orig_loop);

  edge orig_entry_e = loop_preheader_edge (orig_loop);

  edge new_loop_exit_e = new_loop->single_exit;

  edge new_loop_entry_e = loop_preheader_edge (new_loop);

  edge entry_arg_e = (after ? orig_loop_latch : orig_entry_e);

  /*

     step 1. For each loop-header-phi:

             Add the first phi argument for the phi in NEW_LOOP

            (the one associated with the entry of NEW_LOOP)

     step 2. For each loop-header-phi:

             Add the second phi argument for the phi in NEW_LOOP

            (the one associated with the latch of NEW_LOOP)

     step 3. Update the phis in the successor block of NEW_LOOP.

        case 1: NEW_LOOP was placed before ORIG_LOOP:

                The successor block of NEW_LOOP is the header of ORIG_LOOP.

                Updating the phis in the successor block can therefore be done

                along with the scanning of the loop header phis, because the

                header blocks of ORIG_LOOP and NEW_LOOP have exactly the same

                phi nodes, organized in the same order.

        case 2: NEW_LOOP was placed after ORIG_LOOP:

                The successor block of NEW_LOOP is the original exit block of

                ORIG_LOOP - the phis to be updated are the loop-closed-ssa phis.

                We postpone updating these phis to a later stage (when

                loop guards are added).

   */

  /* Scan the phis in the headers of the old and new loops

     (they are organized in exactly the same order).  */

  for (phi_new = phi_nodes (new_loop->header),

       phi_orig = phi_nodes (orig_loop->header);

       phi_new && phi_orig;

       phi_new = PHI_CHAIN (phi_new), phi_orig = PHI_CHAIN (phi_orig))

    {

      /* step 1.  */

      def = PHI_ARG_DEF_FROM_EDGE (phi_orig, entry_arg_e);

      add_phi_arg (phi_new, def, new_loop_entry_e);

      /* step 2.  */

      def = PHI_ARG_DEF_FROM_EDGE (phi_orig, orig_loop_latch);

      if (TREE_CODE (def) != SSA_NAME)

        continue;

      new_ssa_name = get_current_def (def);

      if (!new_ssa_name)

        {

          /* This only happens if there are no definitions

             inside the loop. use the phi_result in this case.  */

          new_ssa_name = PHI_RESULT (phi_new);

        }

      /* An ordinary ssa name defined in the loop.  */

      add_phi_arg (phi_new, new_ssa_name, loop_latch_edge (new_loop));

      /* step 3 (case 1).  */

      if (!after)

        {

          gcc_assert (new_loop_exit_e == orig_entry_e);

          SET_PHI_ARG_DEF (phi_orig,

                           new_loop_exit_e->dest_idx,

                           new_ssa_name);

        }

    }

}

/* Update PHI nodes for a guard of the LOOP.

   Input:

   - LOOP, GUARD_EDGE: LOOP is a loop for which we added guard code that

        controls whether LOOP is to be executed.  GUARD_EDGE is the edge that

        originates from the guard-bb, skips LOOP and reaches the (unique) exit

        bb of LOOP.  This loop-exit-bb is an empty bb with one successor.

        We denote this bb NEW_MERGE_BB because before the guard code was added

        it had a single predecessor (the LOOP header), and now it became a merge

        point of two paths - the path that ends with the LOOP exit-edge, and

        the path that ends with GUARD_EDGE.

   - NEW_EXIT_BB: New basic block that is added by this function between LOOP

        and NEW_MERGE_BB. It is used to place loop-closed-ssa-form exit-phis.

   ===> The CFG before the guard-code was added:

        LOOP_header_bb:

          loop_body

          if (exit_loop) goto update_bb

          else           goto LOOP_header_bb

        update_bb:

   ==> The CFG after the guard-code was added:

        guard_bb:

          if (LOOP_guard_condition) goto new_merge_bb

          else                      goto LOOP_header_bb

        LOOP_header_bb:

          loop_body

          if (exit_loop_condition) goto new_merge_bb

          else                     goto LOOP_header_bb

        new_merge_bb:

          goto update_bb

        update_bb:

   ==> The CFG after this function:

        guard_bb:

          if (LOOP_guard_condition) goto new_merge_bb

          else                      goto LOOP_header_bb

        LOOP_header_bb:

          loop_body

          if (exit_loop_condition) goto new_exit_bb

          else                     goto LOOP_header_bb

        new_exit_bb:

        new_merge_bb:

          goto update_bb

        update_bb:

   This function:

   1. creates and updates the relevant phi nodes to account for the new

      incoming edge (GUARD_EDGE) into NEW_MERGE_BB. This involves:

      1.1. Create phi nodes at NEW_MERGE_BB.

      1.2. Update the phi nodes at the successor of NEW_MERGE_BB (denoted

           UPDATE_BB).  UPDATE_BB was the exit-bb of LOOP before NEW_MERGE_BB

   2. preserves loop-closed-ssa-form by creating the required phi nodes

      at the exit of LOOP (i.e, in NEW_EXIT_BB).

   There are two flavors to this function:

   slpeel_update_phi_nodes_for_guard1:

     Here the guard controls whether we enter or skip LOOP, where LOOP is a

     prolog_loop (loop1 below), and the new phis created in NEW_MERGE_BB are

     for variables that have phis in the loop header.

   slpeel_update_phi_nodes_for_guard2:

     Here the guard controls whether we enter or skip LOOP, where LOOP is an

     epilog_loop (loop2 below), and the new phis created in NEW_MERGE_BB are

     for variables that have phis in the loop exit.

   I.E., the overall structure is:

        loop1_preheader_bb:

                guard1 (goto loop1/merg1_bb)

        loop1

        loop1_exit_bb:

                guard2 (goto merge1_bb/merge2_bb)

        merge1_bb

        loop2

        loop2_exit_bb

        merge2_bb

        next_bb

   slpeel_update_phi_nodes_for_guard1 takes care of creating phis in

   loop1_exit_bb and merge1_bb. These are entry phis (phis for the vars

   that have phis in loop1->header).

   slpeel_update_phi_nodes_for_guard2 takes care of creating phis in

   loop2_exit_bb and merge2_bb. These are exit phis (phis for the vars

   that have phis in next_bb). It also adds some of these phis to

   loop1_exit_bb.

   slpeel_update_phi_nodes_for_guard1 is always called before

   slpeel_update_phi_nodes_for_guard2. They are both needed in order

   to create correct data-flow and loop-closed-ssa-form.

   Generally slpeel_update_phi_nodes_for_guard1 creates phis for variables

   that change between iterations of a loop (and therefore have a phi-node

   at the loop entry), whereas slpeel_update_phi_nodes_for_guard2 creates

   phis for variables that are used out of the loop (and therefore have

   loop-closed exit phis). Some variables may be both updated between

   iterations and used after the loop. This is why in loop1_exit_bb we

   may need both entry_phis (created by slpeel_update_phi_nodes_for_guard1)

   and exit phis (created by slpeel_update_phi_nodes_for_guard2).

   - IS_NEW_LOOP: if IS_NEW_LOOP is true, then LOOP is a newly created copy of

     an original loop. i.e., we have:

           orig_loop

           guard_bb (goto LOOP/new_merge)

           new_loop <-- LOOP

           new_exit

           new_merge

           next_bb

     If IS_NEW_LOOP is false, then LOOP is an original loop, in which case we

     have:

           new_loop

           guard_bb (goto LOOP/new_merge)

           orig_loop <-- LOOP

           new_exit

           new_merge

           next_bb

     The SSA names defined in the original loop have a current

     reaching definition that that records the corresponding new

     ssa-name used in the new duplicated loop copy.

  */

/* Function slpeel_update_phi_nodes_for_guard1

   Input:

   - GUARD_EDGE, LOOP, IS_NEW_LOOP, NEW_EXIT_BB - as explained above.

   - DEFS - a bitmap of ssa names to mark new names for which we recorded

            information.

   In the context of the overall structure, we have:

        loop1_preheader_bb:

                guard1 (goto loop1/merg1_bb)

LOOP->  loop1

        loop1_exit_bb:

                guard2 (goto merge1_bb/merge2_bb)

        merge1_bb

        loop2

        loop2_exit_bb

        merge2_bb

        next_bb

   For each name updated between loop iterations (i.e - for each name that has

   an entry (loop-header) phi in LOOP) we create a new phi in:

   1. merge1_bb (to account for the edge from guard1)

   2. loop1_exit_bb (an exit-phi to keep LOOP in loop-closed form)

*/

static void

slpeel_update_phi_nodes_for_guard1 (edge guard_edge, struct loop *loop,

                                    bool is_new_loop, basic_block *new_exit_bb,

                                    bitmap *defs)

{

  tree orig_phi, new_phi;

  tree update_phi, update_phi2;

  tree guard_arg, loop_arg;

  basic_block new_merge_bb = guard_edge->dest;

  edge e = EDGE_SUCC (new_merge_bb, 0);

  basic_block update_bb = e->dest;

  basic_block orig_bb = loop->header;

  edge new_exit_e;

  tree current_new_name;

  tree name;

  /* Create new bb between loop and new_merge_bb.  */

  *new_exit_bb = split_edge (loop->single_exit);

  add_bb_to_loop (*new_exit_bb, loop->outer);

  new_exit_e = EDGE_SUCC (*new_exit_bb, 0);

  for (orig_phi = phi_nodes (orig_bb), update_phi = phi_nodes (update_bb);

       orig_phi && update_phi;

       orig_phi = PHI_CHAIN (orig_phi), update_phi = PHI_CHAIN (update_phi))

    {

      /* Virtual phi; Mark it for renaming. We actually want to call

         mar_sym_for_renaming, but since all ssa renaming datastructures

         are going to be freed before we get to call ssa_upate, we just

         record this name for now in a bitmap, and will mark it for

         renaming later.  */

      name = PHI_RESULT (orig_phi);

      if (!is_gimple_reg (SSA_NAME_VAR (name)))

        bitmap_set_bit (vect_vnames_to_rename, SSA_NAME_VERSION (name));

      /** 1. Handle new-merge-point phis  **/

      /* 1.1. Generate new phi node in NEW_MERGE_BB:  */

      new_phi = create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi)),

                                 new_merge_bb);

      /* 1.2. NEW_MERGE_BB has two incoming edges: GUARD_EDGE and the exit-edge

            of LOOP. Set the two phi args in NEW_PHI for these edges:  */

      loop_arg = PHI_ARG_DEF_FROM_EDGE (orig_phi, EDGE_SUCC (loop->latch, 0));

      guard_arg = PHI_ARG_DEF_FROM_EDGE (orig_phi, loop_preheader_edge (loop));

      add_phi_arg (new_phi, loop_arg, new_exit_e);

      add_phi_arg (new_phi, guard_arg, guard_edge);

      /* 1.3. Update phi in successor block.  */

      gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi, e) == loop_arg

                  || PHI_ARG_DEF_FROM_EDGE (update_phi, e) == guard_arg);

      SET_PHI_ARG_DEF (update_phi, e->dest_idx, PHI_RESULT (new_phi));

      update_phi2 = new_phi;

      /** 2. Handle loop-closed-ssa-form phis  **/

      /* 2.1. Generate new phi node in NEW_EXIT_BB:  */

      new_phi = create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi)),

                                 *new_exit_bb);

      /* 2.2. NEW_EXIT_BB has one incoming edge: the exit-edge of the loop.  */

      add_phi_arg (new_phi, loop_arg, loop->single_exit);

      /* 2.3. Update phi in successor of NEW_EXIT_BB:  */

      gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi2, new_exit_e) == loop_arg);

      SET_PHI_ARG_DEF (update_phi2, new_exit_e->dest_idx, PHI_RESULT (new_phi));

      /* 2.4. Record the newly created name with set_current_def.

         We want to find a name such that

                name = get_current_def (orig_loop_name)

         and to set its current definition as follows:

                set_current_def (name, new_phi_name)

         If LOOP is a new loop then loop_arg is already the name we're

         looking for. If LOOP is the original loop, then loop_arg is

         the orig_loop_name and the relevant name is recorded in its

         current reaching definition.  */

      if (is_new_loop)

        current_new_name = loop_arg;

      else

        {

          current_new_name = get_current_def (loop_arg);

          /* current_def is not available only if the variable does not

             change inside the loop, in which case we also don't care

             about recording a current_def for it because we won't be

             trying to create loop-exit-phis for it.  */

          if (!current_new_name)

            continue;

        }

      gcc_assert (get_current_def (current_new_name) == NULL_TREE);

      set_current_def (current_new_name, PHI_RESULT (new_phi));

      bitmap_set_bit (*defs, SSA_NAME_VERSION (current_new_name));

    }

  set_phi_nodes (new_merge_bb, phi_reverse (phi_nodes (new_merge_bb)));

}

/* Function slpeel_update_phi_nodes_for_guard2

   Input:

   - GUARD_EDGE, LOOP, IS_NEW_LOOP, NEW_EXIT_BB - as explained above.

   In the context of the overall structure, we have:

        loop1_preheader_bb:

                guard1 (goto loop1/merg1_bb)

        loop1

        loop1_exit_bb:

                guard2 (goto merge1_bb/merge2_bb)

        merge1_bb

LOOP->  loop2

        loop2_exit_bb

        merge2_bb

        next_bb

   For each name used out side the loop (i.e - for each name that has an exit

   phi in next_bb) we create a new phi in:

   1. merge2_bb (to account for the edge from guard_bb)

   2. loop2_exit_bb (an exit-phi to keep LOOP in loop-closed form)

   3. guard2 bb (an exit phi to keep the preceding loop in loop-closed form),

      if needed (if it wasn't handled by slpeel_update_phis_nodes_for_phi1).

*/

static void

slpeel_update_phi_nodes_for_guard2 (edge guard_edge, struct loop *loop,

                                    bool is_new_loop, basic_block *new_exit_bb)

{

  tree orig_phi, new_phi;

  tree update_phi, update_phi2;

  tree guard_arg, loop_arg;

  basic_block new_merge_bb = guard_edge->dest;

  edge e = EDGE_SUCC (new_merge_bb, 0);

  basic_block update_bb = e->dest;

  edge new_exit_e;

  tree orig_def, orig_def_new_name;

  tree new_name, new_name2;

  tree arg;

  /* Create new bb between loop and new_merge_bb.  */

  *new_exit_bb = split_edge (loop->single_exit);

  add_bb_to_loop (*new_exit_bb, loop->outer);

  new_exit_e = EDGE_SUCC (*new_exit_bb, 0);

  for (update_phi = phi_nodes (update_bb); update_phi;

       update_phi = PHI_CHAIN (update_phi))

    {

      orig_phi = update_phi;

      orig_def = PHI_ARG_DEF_FROM_EDGE (orig_phi, e);

      /* This loop-closed-phi actually doesn't represent a use

         out of the loop - the phi arg is a constant.  */

      if (TREE_CODE (orig_def) != SSA_NAME)

        continue;

      orig_def_new_name = get_current_def (orig_def);

      arg = NULL_TREE;

      /** 1. Handle new-merge-point phis  **/

      /* 1.1. Generate new phi node in NEW_MERGE_BB:  */

      new_phi = create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi)),

                                 new_merge_bb);

      /* 1.2. NEW_MERGE_BB has two incoming edges: GUARD_EDGE and the exit-edge

            of LOOP. Set the two PHI args in NEW_PHI for these edges:  */

      new_name = orig_def;

      new_name2 = NULL_TREE;

      if (orig_def_new_name)

        {

          new_name = orig_def_new_name;

          /* Some variables have both loop-entry-phis and loop-exit-phis.

             Such variables were given yet newer names by phis placed in

             guard_bb by slpeel_update_phi_nodes_for_guard1. I.e:

             new_name2 = get_current_def (get_current_def (orig_name)).  */

          new_name2 = get_current_def (new_name);

        }

      if (is_new_loop)

        {

          guard_arg = orig_def;

          loop_arg = new_name;

        }

      else

        {

          guard_arg = new_name;

          loop_arg = orig_def;

        }

      if (new_name2)

        guard_arg = new_name2;

      add_phi_arg (new_phi, loop_arg, new_exit_e);

      add_phi_arg (new_phi, guard_arg, guard_edge);

      /* 1.3. Update phi in successor block.  */

      gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi, e) == orig_def);

      SET_PHI_ARG_DEF (update_phi, e->dest_idx, PHI_RESULT (new_phi));

      update_phi2 = new_phi;

      /** 2. Handle loop-closed-ssa-form phis  **/

      /* 2.1. Generate new phi node in NEW_EXIT_BB:  */

      new_phi = create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi)),

                                 *new_exit_bb);

      /* 2.2. NEW_EXIT_BB has one incoming edge: the exit-edge of the loop.  */

      add_phi_arg (new_phi, loop_arg, loop->single_exit);

      /* 2.3. Update phi in successor of NEW_EXIT_BB:  */

      gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi2, new_exit_e) == loop_arg);

      SET_PHI_ARG_DEF (update_phi2, new_exit_e->dest_idx, PHI_RESULT (new_phi));

      /** 3. Handle loop-closed-ssa-form phis for first loop  **/

      /* 3.1. Find the relevant names that need an exit-phi in

         GUARD_BB, i.e. names for which

         slpeel_update_phi_nodes_for_guard1 had not already created a

         phi node. This is the case for names that are used outside

         the loop (and therefore need an exit phi) but are not updated

         across loop iterations (and therefore don't have a

         loop-header-phi).

         slpeel_update_phi_nodes_for_guard1 is responsible for

         creating loop-exit phis in GUARD_BB for names that have a

         loop-header-phi.  When such a phi is created we also record

         the new name in its current definition.  If this new name

         exists, then guard_arg was set to this new name (see 1.2

         above).  Therefore, if guard_arg is not this new name, this

         is an indication that an exit-phi in GUARD_BB was not yet

         created, so we take care of it here.  */

      if (guard_arg == new_name2)

        continue;

      arg = guard_arg;

      /* 3.2. Generate new phi node in GUARD_BB:  */

      new_phi = create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi)),

                                 guard_edge->src);

      /* 3.3. GUARD_BB has one incoming edge:  */

      gcc_assert (EDGE_COUNT (guard_edge->src->preds) == 1);

      add_phi_arg (new_phi, arg, EDGE_PRED (guard_edge->src, 0));

      /* 3.4. Update phi in successor of GUARD_BB:  */

      gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi2, guard_edge)

                                                                == guard_arg);

      SET_PHI_ARG_DEF (update_phi2, guard_edge->dest_idx, PHI_RESULT (new_phi));

    }

  set_phi_nodes (new_merge_bb, phi_reverse (phi_nodes (new_merge_bb)));

}

/* Make the LOOP iterate NITERS times. This is done by adding a new IV

   that starts at zero, increases by one and its limit is NITERS.

   Assumption: the exit-condition of LOOP is the last stmt in the loop.  */

void

slpeel_make_loop_iterate_ntimes (struct loop *loop, tree niters)

{

  tree indx_before_incr, indx_after_incr, cond_stmt, cond;

  tree orig_cond;

  edge exit_edge = loop->single_exit;

  block_stmt_iterator loop_cond_bsi;

  block_stmt_iterator incr_bsi;

  bool insert_after;

  tree begin_label = tree_block_label (loop->latch);

  tree exit_label = tree_block_label (loop->single_exit->dest);

  tree init = build_int_cst (TREE_TYPE (niters), 0);

  tree step = build_int_cst (TREE_TYPE (niters), 1);

  tree then_label;

  tree else_label;

  LOC loop_loc;

  orig_cond = get_loop_exit_condition (loop);

  gcc_assert (orig_cond);

  loop_cond_bsi = bsi_for_stmt (orig_cond);

  standard_iv_increment_position (loop, &incr_bsi, &insert_after);

  create_iv (init, step, NULL_TREE, loop,

             &incr_bsi, insert_after, &indx_before_incr, &indx_after_incr);

  if (exit_edge->flags & EDGE_TRUE_VALUE) /* 'then' edge exits the loop.  */

    {

      cond = build2 (GE_EXPR, boolean_type_node, indx_after_incr, niters);

      then_label = build1 (GOTO_EXPR, void_type_node, exit_label);

      else_label = build1 (GOTO_EXPR, void_type_node, begin_label);

    }