Subversion Repositories openrisc_me

/* Transformation Utilities for Loop Vectorization.

   Copyright (C) 2003,2004,2005,2006, 2007 Free Software Foundation, Inc.

   Contributed by Dorit Naishlos <dorit@il.ibm.com>

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it under

the terms of the GNU General Public License as published by the Free

Software Foundation; either version 3, or (at your option) any later

version.

GCC is distributed in the hope that it will be useful, but WITHOUT ANY

WARRANTY; without even the implied warranty of MERCHANTABILITY or

FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

for more details.

You should have received a copy of the GNU General Public License

along with GCC; see the file COPYING3.  If not see

<http://www.gnu.org/licenses/>.  */

#include "config.h"

#include "system.h"

#include "coretypes.h"

#include "tm.h"

#include "ggc.h"

#include "tree.h"

#include "target.h"

#include "rtl.h"

#include "basic-block.h"

#include "diagnostic.h"

#include "tree-flow.h"

#include "tree-dump.h"

#include "timevar.h"

#include "cfgloop.h"

#include "expr.h"

#include "optabs.h"

#include "recog.h"

#include "tree-data-ref.h"

#include "tree-chrec.h"

#include "tree-scalar-evolution.h"

#include "tree-vectorizer.h"

#include "langhooks.h"

#include "tree-pass.h"

#include "toplev.h"

#include "real.h"

/* Utility functions for the code transformation.  */

static bool vect_transform_stmt (tree, block_stmt_iterator *);

static void vect_align_data_ref (tree);

static tree vect_create_destination_var (tree, tree);

static tree vect_create_data_ref_ptr

  (tree, block_stmt_iterator *, tree, tree *, bool);

static tree vect_create_addr_base_for_vector_ref (tree, tree *, tree);

static tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);

static tree vect_get_vec_def_for_operand (tree, tree, tree *);

static tree vect_init_vector (tree, tree);

static void vect_finish_stmt_generation

  (tree stmt, tree vec_stmt, block_stmt_iterator *bsi);

static bool vect_is_simple_cond (tree, loop_vec_info);

static void update_vuses_to_preheader (tree, struct loop*);

static void vect_create_epilog_for_reduction (tree, tree, enum tree_code, tree);

static tree get_initial_def_for_reduction (tree, tree, tree *);

/* Utility function dealing with loop peeling (not peeling itself).  */

static void vect_generate_tmps_on_preheader

  (loop_vec_info, tree *, tree *, tree *);

static tree vect_build_loop_niters (loop_vec_info);

static void vect_update_ivs_after_vectorizer (loop_vec_info, tree, edge);

static tree vect_gen_niters_for_prolog_loop (loop_vec_info, tree);

static void vect_update_init_of_dr (struct data_reference *, tree niters);

static void vect_update_inits_of_drs (loop_vec_info, tree);

static void vect_do_peeling_for_alignment (loop_vec_info, struct loops *);

static void vect_do_peeling_for_loop_bound

  (loop_vec_info, tree *, struct loops *);

static int vect_min_worthwhile_factor (enum tree_code);

/* Function vect_get_new_vect_var.

   Returns a name for a new variable. The current naming scheme appends the

   prefix "vect_" or "vect_p" (depending on the value of VAR_KIND) to

   the name of vectorizer generated variables, and appends that to NAME if

   provided.  */

static tree

vect_get_new_vect_var (tree type, enum vect_var_kind var_kind, const char *name)

{

  const char *prefix;

  tree new_vect_var;

  switch (var_kind)

  {

  case vect_simple_var:

    prefix = "vect_";

    break;

  case vect_scalar_var:

    prefix = "stmp_";

    break;

  case vect_pointer_var:

    prefix = "vect_p";

    break;

  default:

    gcc_unreachable ();

  }

  if (name)

    new_vect_var = create_tmp_var (type, concat (prefix, name, NULL));

  else

    new_vect_var = create_tmp_var (type, prefix);

  return new_vect_var;

}

/* Function vect_create_addr_base_for_vector_ref.

   Create an expression that computes the address of the first memory location

   that will be accessed for a data reference.

   Input:

   STMT: The statement containing the data reference.

   NEW_STMT_LIST: Must be initialized to NULL_TREE or a statement list.

   OFFSET: Optional. If supplied, it is be added to the initial address.

   Output:

   1. Return an SSA_NAME whose value is the address of the memory location of

      the first vector of the data reference.

   2. If new_stmt_list is not NULL_TREE after return then the caller must insert

      these statement(s) which define the returned SSA_NAME.

   FORNOW: We are only handling array accesses with step 1.  */

static tree

vect_create_addr_base_for_vector_ref (tree stmt,

                                      tree *new_stmt_list,

                                      tree offset)

{

  stmt_vec_info stmt_info = vinfo_for_stmt (stmt);

  struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);

  tree data_ref_base = unshare_expr (DR_BASE_ADDRESS (dr));

  tree base_name = build_fold_indirect_ref (data_ref_base);

  tree ref = DR_REF (dr);

  tree scalar_type = TREE_TYPE (ref);

  tree scalar_ptr_type = build_pointer_type (scalar_type);

  tree vec_stmt;

  tree new_temp;

  tree addr_base, addr_expr;

  tree dest, new_stmt;

  tree base_offset = unshare_expr (DR_OFFSET (dr));

  tree init = unshare_expr (DR_INIT (dr));

  /* Create base_offset */

  base_offset = size_binop (PLUS_EXPR, base_offset, init);

  dest = create_tmp_var (TREE_TYPE (base_offset), "base_off");

  add_referenced_var (dest);

  base_offset = force_gimple_operand (base_offset, &new_stmt, false, dest);

  append_to_statement_list_force (new_stmt, new_stmt_list);

  if (offset)

    {

      tree tmp = create_tmp_var (TREE_TYPE (base_offset), "offset");

      add_referenced_var (tmp);

      offset = fold_build2 (MULT_EXPR, TREE_TYPE (offset), offset,

                            DR_STEP (dr));

      base_offset = fold_build2 (PLUS_EXPR, TREE_TYPE (base_offset),

                                 base_offset, offset);

      base_offset = force_gimple_operand (base_offset, &new_stmt, false, tmp);

      append_to_statement_list_force (new_stmt, new_stmt_list);

    }

  /* base + base_offset */

  addr_base = fold_build2 (PLUS_EXPR, TREE_TYPE (data_ref_base), data_ref_base,

                           base_offset);

  /* addr_expr = addr_base */

  addr_expr = vect_get_new_vect_var (scalar_ptr_type, vect_pointer_var,

                                     get_name (base_name));

  add_referenced_var (addr_expr);

  vec_stmt = build2 (MODIFY_EXPR, void_type_node, addr_expr, addr_base);

  new_temp = make_ssa_name (addr_expr, vec_stmt);

  TREE_OPERAND (vec_stmt, 0) = new_temp;

  append_to_statement_list_force (vec_stmt, new_stmt_list);

  if (vect_print_dump_info (REPORT_DETAILS))

    {

      fprintf (vect_dump, "created ");

      print_generic_expr (vect_dump, vec_stmt, TDF_SLIM);

    }

  return new_temp;

}

/* Function vect_align_data_ref.

   Handle misalignment of a memory accesses.

   FORNOW: Can't handle misaligned accesses.

   Make sure that the dataref is aligned.  */

static void

vect_align_data_ref (tree stmt)

{

  stmt_vec_info stmt_info = vinfo_for_stmt (stmt);

  struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);

  /* FORNOW: can't handle misaligned accesses;

             all accesses expected to be aligned.  */

  gcc_assert (aligned_access_p (dr));

}

/* Function vect_create_data_ref_ptr.

   Create a memory reference expression for vector access, to be used in a

   vector load/store stmt. The reference is based on a new pointer to vector

   type (vp).

   Input:

   1. STMT: a stmt that references memory. Expected to be of the form

         MODIFY_EXPR <name, data-ref> or MODIFY_EXPR <data-ref, name>.

   2. BSI: block_stmt_iterator where new stmts can be added.

   3. OFFSET (optional): an offset to be added to the initial address accessed

        by the data-ref in STMT.

   4. ONLY_INIT: indicate if vp is to be updated in the loop, or remain

        pointing to the initial address.

   Output:

   1. Declare a new ptr to vector_type, and have it point to the base of the

      data reference (initial addressed accessed by the data reference).

      For example, for vector of type V8HI, the following code is generated:

      v8hi *vp;

      vp = (v8hi *)initial_address;

      if OFFSET is not supplied:

         initial_address = &a[init];

      if OFFSET is supplied:

         initial_address = &a[init + OFFSET];

      Return the initial_address in INITIAL_ADDRESS.

   2. If ONLY_INIT is true, return the initial pointer.  Otherwise, create

      a data-reference in the loop based on the new vector pointer vp.  This

      new data reference will by some means be updated each iteration of

      the loop.  Return the pointer vp'.

   FORNOW: handle only aligned and consecutive accesses.  */

static tree

vect_create_data_ref_ptr (tree stmt,

                          block_stmt_iterator *bsi ATTRIBUTE_UNUSED,

                          tree offset, tree *initial_address, bool only_init)

{

  tree base_name;

  stmt_vec_info stmt_info = vinfo_for_stmt (stmt);

  loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);

  struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);

  tree vectype = STMT_VINFO_VECTYPE (stmt_info);

  tree vect_ptr_type;

  tree vect_ptr;

  tree tag;

  tree new_temp;

  tree vec_stmt;

  tree new_stmt_list = NULL_TREE;

  edge pe = loop_preheader_edge (loop);

  basic_block new_bb;

  tree vect_ptr_init;

  struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);

  base_name =  build_fold_indirect_ref (unshare_expr (DR_BASE_ADDRESS (dr)));

  if (vect_print_dump_info (REPORT_DETAILS))

    {

      tree data_ref_base = base_name;

      fprintf (vect_dump, "create vector-pointer variable to type: ");

      print_generic_expr (vect_dump, vectype, TDF_SLIM);

      if (TREE_CODE (data_ref_base) == VAR_DECL)

        fprintf (vect_dump, "  vectorizing a one dimensional array ref: ");

      else if (TREE_CODE (data_ref_base) == ARRAY_REF)

        fprintf (vect_dump, "  vectorizing a multidimensional array ref: ");

      else if (TREE_CODE (data_ref_base) == COMPONENT_REF)

        fprintf (vect_dump, "  vectorizing a record based array ref: ");

      else if (TREE_CODE (data_ref_base) == SSA_NAME)

        fprintf (vect_dump, "  vectorizing a pointer ref: ");

      print_generic_expr (vect_dump, base_name, TDF_SLIM);

    }

  /** (1) Create the new vector-pointer variable:  **/

  vect_ptr_type = build_pointer_type (vectype);

  vect_ptr = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var,

                                    get_name (base_name));

  add_referenced_var (vect_ptr);

  /** (2) Add aliasing information to the new vector-pointer:

          (The points-to info (DR_PTR_INFO) may be defined later.)  **/

  tag = DR_MEMTAG (dr);

  gcc_assert (tag);

  /* If tag is a variable (and NOT_A_TAG) than a new symbol memory

     tag must be created with tag added to its may alias list.  */

  if (!MTAG_P (tag))

    new_type_alias (vect_ptr, tag, DR_REF (dr));

  else

    var_ann (vect_ptr)->symbol_mem_tag = tag;

  var_ann (vect_ptr)->subvars = DR_SUBVARS (dr);

  /** (3) Calculate the initial address the vector-pointer, and set

          the vector-pointer to point to it before the loop:  **/

  /* Create: (&(base[init_val+offset]) in the loop preheader.  */

  new_temp = vect_create_addr_base_for_vector_ref (stmt, &new_stmt_list,

                                                   offset);

  pe = loop_preheader_edge (loop);

  new_bb = bsi_insert_on_edge_immediate (pe, new_stmt_list);

  gcc_assert (!new_bb);

  *initial_address = new_temp;

  /* Create: p = (vectype *) initial_base  */

  vec_stmt = fold_convert (vect_ptr_type, new_temp);

  vec_stmt = build2 (MODIFY_EXPR, void_type_node, vect_ptr, vec_stmt);

  vect_ptr_init = make_ssa_name (vect_ptr, vec_stmt);

  TREE_OPERAND (vec_stmt, 0) = vect_ptr_init;

  new_bb = bsi_insert_on_edge_immediate (pe, vec_stmt);

  gcc_assert (!new_bb);

  /** (4) Handle the updating of the vector-pointer inside the loop: **/

  if (only_init) /* No update in loop is required.  */

    {

      /* Copy the points-to information if it exists. */

      if (DR_PTR_INFO (dr))

        duplicate_ssa_name_ptr_info (vect_ptr_init, DR_PTR_INFO (dr));

      return vect_ptr_init;

    }

  else

    {

      block_stmt_iterator incr_bsi;

      bool insert_after;

      tree indx_before_incr, indx_after_incr;

      tree incr;

      standard_iv_increment_position (loop, &incr_bsi, &insert_after);

      create_iv (vect_ptr_init,

                 fold_convert (vect_ptr_type, TYPE_SIZE_UNIT (vectype)),

                 NULL_TREE, loop, &incr_bsi, insert_after,

                 &indx_before_incr, &indx_after_incr);

      incr = bsi_stmt (incr_bsi);

      set_stmt_info (stmt_ann (incr),

                     new_stmt_vec_info (incr, loop_vinfo));

      /* Copy the points-to information if it exists. */

      if (DR_PTR_INFO (dr))

        {

          duplicate_ssa_name_ptr_info (indx_before_incr, DR_PTR_INFO (dr));

          duplicate_ssa_name_ptr_info (indx_after_incr, DR_PTR_INFO (dr));

        }

      merge_alias_info (vect_ptr_init, indx_before_incr);

      merge_alias_info (vect_ptr_init, indx_after_incr);

      return indx_before_incr;

    }

}

/* Function vect_create_destination_var.

   Create a new temporary of type VECTYPE.  */

static tree

vect_create_destination_var (tree scalar_dest, tree vectype)

{

  tree vec_dest;

  const char *new_name;

  tree type;

  enum vect_var_kind kind;

  kind = vectype ? vect_simple_var : vect_scalar_var;

  type = vectype ? vectype : TREE_TYPE (scalar_dest);

  gcc_assert (TREE_CODE (scalar_dest) == SSA_NAME);

  new_name = get_name (scalar_dest);

  if (!new_name)

    new_name = "var_";

  vec_dest = vect_get_new_vect_var (type, vect_simple_var, new_name);

  add_referenced_var (vec_dest);

  return vec_dest;

}

/* Function vect_init_vector.

   Insert a new stmt (INIT_STMT) that initializes a new vector variable with

   the vector elements of VECTOR_VAR. Return the DEF of INIT_STMT. It will be

   used in the vectorization of STMT.  */

static tree

vect_init_vector (tree stmt, tree vector_var)

{

  stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);

  loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);

  struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);

  tree new_var;

  tree init_stmt;

  tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);

  tree vec_oprnd;

  edge pe;

  tree new_temp;

  basic_block new_bb;

  new_var = vect_get_new_vect_var (vectype, vect_simple_var, "cst_");

  add_referenced_var (new_var);

  init_stmt = build2 (MODIFY_EXPR, vectype, new_var, vector_var);

  new_temp = make_ssa_name (new_var, init_stmt);

  TREE_OPERAND (init_stmt, 0) = new_temp;

  pe = loop_preheader_edge (loop);

  new_bb = bsi_insert_on_edge_immediate (pe, init_stmt);

  gcc_assert (!new_bb);

  if (vect_print_dump_info (REPORT_DETAILS))

    {

      fprintf (vect_dump, "created new init_stmt: ");

      print_generic_expr (vect_dump, init_stmt, TDF_SLIM);

    }

  vec_oprnd = TREE_OPERAND (init_stmt, 0);

  return vec_oprnd;

}

/* Function vect_get_vec_def_for_operand.

   OP is an operand in STMT. This function returns a (vector) def that will be

   used in the vectorized stmt for STMT.

   In the case that OP is an SSA_NAME which is defined in the loop, then

   STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.

   In case OP is an invariant or constant, a new stmt that creates a vector def

   needs to be introduced.  */

static tree

vect_get_vec_def_for_operand (tree op, tree stmt, tree *scalar_def)

{

  tree vec_oprnd;

  tree vec_stmt;

  tree def_stmt;

  stmt_vec_info def_stmt_info = NULL;

  stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);

  tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);

  int nunits = TYPE_VECTOR_SUBPARTS (vectype);

  loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);

  struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);

  tree vec_inv;

  tree vec_cst;

  tree t = NULL_TREE;

  tree def;

  int i;

  enum vect_def_type dt;

  bool is_simple_use;

  if (vect_print_dump_info (REPORT_DETAILS))

    {

      fprintf (vect_dump, "vect_get_vec_def_for_operand: ");

      print_generic_expr (vect_dump, op, TDF_SLIM);

    }

  is_simple_use = vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt);

  gcc_assert (is_simple_use);

  if (vect_print_dump_info (REPORT_DETAILS))

    {

      if (def)

        {

          fprintf (vect_dump, "def =  ");

          print_generic_expr (vect_dump, def, TDF_SLIM);

        }

      if (def_stmt)

        {

          fprintf (vect_dump, "  def_stmt =  ");

          print_generic_expr (vect_dump, def_stmt, TDF_SLIM);

        }

    }

  switch (dt)

    {

    /* Case 1: operand is a constant.  */

    case vect_constant_def:

      {

        if (scalar_def)

          *scalar_def = op;

        /* Create 'vect_cst_ = {cst,cst,...,cst}'  */

        if (vect_print_dump_info (REPORT_DETAILS))

          fprintf (vect_dump, "Create vector_cst. nunits = %d", nunits);

        for (i = nunits - 1; i >= 0; --i)

          {

            t = tree_cons (NULL_TREE, op, t);

          }

        vec_cst = build_vector (vectype, t);

        return vect_init_vector (stmt, vec_cst);

      }

    /* Case 2: operand is defined outside the loop - loop invariant.  */

    case vect_invariant_def:

      {

        if (scalar_def)

          *scalar_def = def;

        /* Create 'vec_inv = {inv,inv,..,inv}'  */

        if (vect_print_dump_info (REPORT_DETAILS))

          fprintf (vect_dump, "Create vector_inv.");

        for (i = nunits - 1; i >= 0; --i)

          {

            t = tree_cons (NULL_TREE, def, t);

          }

        /* FIXME: use build_constructor directly.  */

        vec_inv = build_constructor_from_list (vectype, t);

        return vect_init_vector (stmt, vec_inv);

      }

    /* Case 3: operand is defined inside the loop.  */

    case vect_loop_def:

      {

        if (scalar_def)

          *scalar_def = def_stmt;

        /* Get the def from the vectorized stmt.  */

        def_stmt_info = vinfo_for_stmt (def_stmt);

        vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);

        gcc_assert (vec_stmt);

        vec_oprnd = TREE_OPERAND (vec_stmt, 0);

        return vec_oprnd;

      }

    /* Case 4: operand is defined by a loop header phi - reduction  */

    case vect_reduction_def:

      {

        gcc_assert (TREE_CODE (def_stmt) == PHI_NODE);

        /* Get the def before the loop  */

        op = PHI_ARG_DEF_FROM_EDGE (def_stmt, loop_preheader_edge (loop));

        return get_initial_def_for_reduction (stmt, op, scalar_def);

     }

    /* Case 5: operand is defined by loop-header phi - induction.  */

    case vect_induction_def:

      {

        if (vect_print_dump_info (REPORT_DETAILS))

          fprintf (vect_dump, "induction - unsupported.");

        internal_error ("no support for induction"); /* FORNOW */

      }

    default:

      gcc_unreachable ();

    }

}

/* Function vect_finish_stmt_generation.

   Insert a new stmt.  */

static void

vect_finish_stmt_generation (tree stmt, tree vec_stmt, block_stmt_iterator *bsi)

{

  bsi_insert_before (bsi, vec_stmt, BSI_SAME_STMT);

  if (vect_print_dump_info (REPORT_DETAILS))

    {

      fprintf (vect_dump, "add new stmt: ");

      print_generic_expr (vect_dump, vec_stmt, TDF_SLIM);

    }

  /* Make sure bsi points to the stmt that is being vectorized.  */

  gcc_assert (stmt == bsi_stmt (*bsi));

#ifdef USE_MAPPED_LOCATION

  SET_EXPR_LOCATION (vec_stmt, EXPR_LOCATION (stmt));

#else

  SET_EXPR_LOCUS (vec_stmt, EXPR_LOCUS (stmt));

#endif

}

#define ADJUST_IN_EPILOG 1

/* Function get_initial_def_for_reduction

   Input:

   STMT - a stmt that performs a reduction operation in the loop.

   INIT_VAL - the initial value of the reduction variable

   Output:

   SCALAR_DEF - a tree that holds a value to be added to the final result

        of the reduction (used for "ADJUST_IN_EPILOG" - see below).

   Return a vector variable, initialized according to the operation that STMT

        performs. This vector will be used as the initial value of the

        vector of partial results.

   Option1 ("ADJUST_IN_EPILOG"): Initialize the vector as follows:

     add:         [0,0,...,0,0]

     mult:        [1,1,...,1,1]

     min/max:     [init_val,init_val,..,init_val,init_val]

     bit and/or:  [init_val,init_val,..,init_val,init_val]

   and when necessary (e.g. add/mult case) let the caller know

   that it needs to adjust the result by init_val.

   Option2: Initialize the vector as follows:

     add:         [0,0,...,0,init_val]

     mult:        [1,1,...,1,init_val]

     min/max:     [init_val,init_val,...,init_val]

     bit and/or:  [init_val,init_val,...,init_val]

   and no adjustments are needed.

   For example, for the following code:

   s = init_val;

   for (i=0;i<n;i++)

     s = s + a[i];

   STMT is 's = s + a[i]', and the reduction variable is 's'.

   For a vector of 4 units, we want to return either [0,0,0,init_val],

   or [0,0,0,0] and let the caller know that it needs to adjust

   the result at the end by 'init_val'.

   FORNOW: We use the "ADJUST_IN_EPILOG" scheme.

   TODO: Use some cost-model to estimate which scheme is more profitable.

*/

static tree

get_initial_def_for_reduction (tree stmt, tree init_val, tree *scalar_def)

{

  stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);

  tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);

  int nunits = GET_MODE_NUNITS (TYPE_MODE (vectype));

  int nelements;

  enum tree_code code = TREE_CODE (TREE_OPERAND (stmt, 1));

  tree type = TREE_TYPE (init_val);

  tree def;

  tree vec, t = NULL_TREE;

  bool need_epilog_adjust;

  int i;

  gcc_assert (INTEGRAL_TYPE_P (type) || SCALAR_FLOAT_TYPE_P (type));

  switch (code)

  {

  case WIDEN_SUM_EXPR:

  case DOT_PROD_EXPR:

  case PLUS_EXPR:

    if (INTEGRAL_TYPE_P (type))

      def = build_int_cst (type, 0);

    else

      def = build_real (type, dconst0);

#ifdef ADJUST_IN_EPILOG

    /* All the 'nunits' elements are set to 0. The final result will be

       adjusted by 'init_val' at the loop epilog.  */

    nelements = nunits;

    need_epilog_adjust = true;

#else

    /* 'nunits - 1' elements are set to 0; The last element is set to

        'init_val'.  No further adjustments at the epilog are needed.  */

    nelements = nunits - 1;

    need_epilog_adjust = false;

#endif

    break;

  case MIN_EXPR:

  case MAX_EXPR:

    def = init_val;

    nelements = nunits;

    need_epilog_adjust = false;

    break;

  default:

    gcc_unreachable ();

  }

  for (i = nelements - 1; i >= 0; --i)

    t = tree_cons (NULL_TREE, def, t);

  if (nelements == nunits - 1)

    {

      /* Set the last element of the vector.  */

      t = tree_cons (NULL_TREE, init_val, t);

      nelements += 1;

    }

  gcc_assert (nelements == nunits);

  if (TREE_CODE (init_val) == INTEGER_CST || TREE_CODE (init_val) == REAL_CST)

    vec = build_vector (vectype, t);

  else

    vec = build_constructor_from_list (vectype, t);

  if (!need_epilog_adjust)

    *scalar_def = NULL_TREE;

  else

    *scalar_def = init_val;

  return vect_init_vector (stmt, vec);

}

/* Function vect_create_epilog_for_reduction

   Create code at the loop-epilog to finalize the result of a reduction

   computation.

   VECT_DEF is a vector of partial results.

   REDUC_CODE is the tree-code for the epilog reduction.

   STMT is the scalar reduction stmt that is being vectorized.