| 1 | 684 | jeremybenn | /* Swing Modulo Scheduling implementation.
 | 
      
         | 2 |  |  |    Copyright (C) 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
 | 
      
         | 3 |  |  |    Free Software Foundation, Inc.
 | 
      
         | 4 |  |  |    Contributed by Ayal Zaks and Mustafa Hagog <zaks,mustafa@il.ibm.com>
 | 
      
         | 5 |  |  |  
 | 
      
         | 6 |  |  | This file is part of GCC.
 | 
      
         | 7 |  |  |  
 | 
      
         | 8 |  |  | GCC is free software; you can redistribute it and/or modify it under
 | 
      
         | 9 |  |  | the terms of the GNU General Public License as published by the Free
 | 
      
         | 10 |  |  | Software Foundation; either version 3, or (at your option) any later
 | 
      
         | 11 |  |  | version.
 | 
      
         | 12 |  |  |  
 | 
      
         | 13 |  |  | GCC is distributed in the hope that it will be useful, but WITHOUT ANY
 | 
      
         | 14 |  |  | WARRANTY; without even the implied warranty of MERCHANTABILITY or
 | 
      
         | 15 |  |  | FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 | 
      
         | 16 |  |  | for more details.
 | 
      
         | 17 |  |  |  
 | 
      
         | 18 |  |  | You should have received a copy of the GNU General Public License
 | 
      
         | 19 |  |  | along with GCC; see the file COPYING3.  If not see
 | 
      
         | 20 |  |  | <http://www.gnu.org/licenses/>.  */
 | 
      
         | 21 |  |  |  
 | 
      
         | 22 |  |  |  
 | 
      
         | 23 |  |  | #include "config.h"
 | 
      
         | 24 |  |  | #include "system.h"
 | 
      
         | 25 |  |  | #include "coretypes.h"
 | 
      
         | 26 |  |  | #include "tm.h"
 | 
      
         | 27 |  |  | #include "diagnostic-core.h"
 | 
      
         | 28 |  |  | #include "rtl.h"
 | 
      
         | 29 |  |  | #include "tm_p.h"
 | 
      
         | 30 |  |  | #include "hard-reg-set.h"
 | 
      
         | 31 |  |  | #include "regs.h"
 | 
      
         | 32 |  |  | #include "function.h"
 | 
      
         | 33 |  |  | #include "flags.h"
 | 
      
         | 34 |  |  | #include "insn-config.h"
 | 
      
         | 35 |  |  | #include "insn-attr.h"
 | 
      
         | 36 |  |  | #include "except.h"
 | 
      
         | 37 |  |  | #include "recog.h"
 | 
      
         | 38 |  |  | #include "sched-int.h"
 | 
      
         | 39 |  |  | #include "target.h"
 | 
      
         | 40 |  |  | #include "cfglayout.h"
 | 
      
         | 41 |  |  | #include "cfgloop.h"
 | 
      
         | 42 |  |  | #include "cfghooks.h"
 | 
      
         | 43 |  |  | #include "expr.h"
 | 
      
         | 44 |  |  | #include "params.h"
 | 
      
         | 45 |  |  | #include "gcov-io.h"
 | 
      
         | 46 |  |  | #include "ddg.h"
 | 
      
         | 47 |  |  | #include "timevar.h"
 | 
      
         | 48 |  |  | #include "tree-pass.h"
 | 
      
         | 49 |  |  | #include "dbgcnt.h"
 | 
      
         | 50 |  |  | #include "df.h"
 | 
      
         | 51 |  |  |  
 | 
      
         | 52 |  |  | #ifdef INSN_SCHEDULING
 | 
      
         | 53 |  |  |  
 | 
      
         | 54 |  |  | /* This file contains the implementation of the Swing Modulo Scheduler,
 | 
      
         | 55 |  |  |    described in the following references:
 | 
      
         | 56 |  |  |    [1] J. Llosa, A. Gonzalez, E. Ayguade, M. Valero., and J. Eckhardt.
 | 
      
         | 57 |  |  |        Lifetime--sensitive modulo scheduling in a production environment.
 | 
      
         | 58 |  |  |        IEEE Trans. on Comps., 50(3), March 2001
 | 
      
         | 59 |  |  |    [2] J. Llosa, A. Gonzalez, E. Ayguade, and M. Valero.
 | 
      
         | 60 |  |  |        Swing Modulo Scheduling: A Lifetime Sensitive Approach.
 | 
      
         | 61 |  |  |        PACT '96 , pages 80-87, October 1996 (Boston - Massachusetts - USA).
 | 
      
         | 62 |  |  |  
 | 
      
         | 63 |  |  |    The basic structure is:
 | 
      
         | 64 |  |  |    1. Build a data-dependence graph (DDG) for each loop.
 | 
      
         | 65 |  |  |    2. Use the DDG to order the insns of a loop (not in topological order
 | 
      
         | 66 |  |  |       necessarily, but rather) trying to place each insn after all its
 | 
      
         | 67 |  |  |       predecessors _or_ after all its successors.
 | 
      
         | 68 |  |  |    3. Compute MII: a lower bound on the number of cycles to schedule the loop.
 | 
      
         | 69 |  |  |    4. Use the ordering to perform list-scheduling of the loop:
 | 
      
         | 70 |  |  |       1. Set II = MII.  We will try to schedule the loop within II cycles.
 | 
      
         | 71 |  |  |       2. Try to schedule the insns one by one according to the ordering.
 | 
      
         | 72 |  |  |          For each insn compute an interval of cycles by considering already-
 | 
      
         | 73 |  |  |          scheduled preds and succs (and associated latencies); try to place
 | 
      
         | 74 |  |  |          the insn in the cycles of this window checking for potential
 | 
      
         | 75 |  |  |          resource conflicts (using the DFA interface).
 | 
      
         | 76 |  |  |          Note: this is different from the cycle-scheduling of schedule_insns;
 | 
      
         | 77 |  |  |          here the insns are not scheduled monotonically top-down (nor bottom-
 | 
      
         | 78 |  |  |          up).
 | 
      
         | 79 |  |  |       3. If failed in scheduling all insns - bump II++ and try again, unless
 | 
      
         | 80 |  |  |          II reaches an upper bound MaxII, in which case report failure.
 | 
      
         | 81 |  |  |    5. If we succeeded in scheduling the loop within II cycles, we now
 | 
      
         | 82 |  |  |       generate prolog and epilog, decrease the counter of the loop, and
 | 
      
         | 83 |  |  |       perform modulo variable expansion for live ranges that span more than
 | 
      
         | 84 |  |  |       II cycles (i.e. use register copies to prevent a def from overwriting
 | 
      
         | 85 |  |  |       itself before reaching the use).
 | 
      
         | 86 |  |  |  
 | 
      
         | 87 |  |  |     SMS works with countable loops (1) whose control part can be easily
 | 
      
         | 88 |  |  |     decoupled from the rest of the loop and (2) whose loop count can
 | 
      
         | 89 |  |  |     be easily adjusted.  This is because we peel a constant number of
 | 
      
         | 90 |  |  |     iterations into a prologue and epilogue for which we want to avoid
 | 
      
         | 91 |  |  |     emitting the control part, and a kernel which is to iterate that
 | 
      
         | 92 |  |  |     constant number of iterations less than the original loop.  So the
 | 
      
         | 93 |  |  |     control part should be a set of insns clearly identified and having
 | 
      
         | 94 |  |  |     its own iv, not otherwise used in the loop (at-least for now), which
 | 
      
         | 95 |  |  |     initializes a register before the loop to the number of iterations.
 | 
      
         | 96 |  |  |     Currently SMS relies on the do-loop pattern to recognize such loops,
 | 
      
         | 97 |  |  |     where (1) the control part comprises of all insns defining and/or
 | 
      
         | 98 |  |  |     using a certain 'count' register and (2) the loop count can be
 | 
      
         | 99 |  |  |     adjusted by modifying this register prior to the loop.
 | 
      
         | 100 |  |  |     TODO: Rely on cfgloop analysis instead.  */
 | 
      
         | 101 |  |  |  
 | 
      
         | 102 |  |  | /* This page defines partial-schedule structures and functions for
 | 
      
         | 103 |  |  |    modulo scheduling.  */
 | 
      
         | 104 |  |  |  
 | 
      
         | 105 |  |  | typedef struct partial_schedule *partial_schedule_ptr;
 | 
      
         | 106 |  |  | typedef struct ps_insn *ps_insn_ptr;
 | 
      
         | 107 |  |  |  
 | 
      
         | 108 |  |  | /* The minimum (absolute) cycle that a node of ps was scheduled in.  */
 | 
      
         | 109 |  |  | #define PS_MIN_CYCLE(ps) (((partial_schedule_ptr)(ps))->min_cycle)
 | 
      
         | 110 |  |  |  
 | 
      
         | 111 |  |  | /* The maximum (absolute) cycle that a node of ps was scheduled in.  */
 | 
      
         | 112 |  |  | #define PS_MAX_CYCLE(ps) (((partial_schedule_ptr)(ps))->max_cycle)
 | 
      
         | 113 |  |  |  
 | 
      
         | 114 |  |  | /* Perform signed modulo, always returning a non-negative value.  */
 | 
      
         | 115 |  |  | #define SMODULO(x,y) ((x) % (y) < 0 ? ((x) % (y) + (y)) : (x) % (y))
 | 
      
         | 116 |  |  |  
 | 
      
         | 117 |  |  | /* The number of different iterations the nodes in ps span, assuming
 | 
      
         | 118 |  |  |    the stage boundaries are placed efficiently.  */
 | 
      
         | 119 |  |  | #define CALC_STAGE_COUNT(max_cycle,min_cycle,ii) ((max_cycle - min_cycle \
 | 
      
         | 120 |  |  |                          + 1 + ii - 1) / ii)
 | 
      
         | 121 |  |  | /* The stage count of ps.  */
 | 
      
         | 122 |  |  | #define PS_STAGE_COUNT(ps) (((partial_schedule_ptr)(ps))->stage_count)
 | 
      
         | 123 |  |  |  
 | 
      
         | 124 |  |  | /* A single instruction in the partial schedule.  */
 | 
      
         | 125 |  |  | struct ps_insn
 | 
      
         | 126 |  |  | {
 | 
      
         | 127 |  |  |   /* Identifies the instruction to be scheduled.  Values smaller than
 | 
      
         | 128 |  |  |      the ddg's num_nodes refer directly to ddg nodes.  A value of
 | 
      
         | 129 |  |  |      X - num_nodes refers to register move X.  */
 | 
      
         | 130 |  |  |   int id;
 | 
      
         | 131 |  |  |  
 | 
      
         | 132 |  |  |   /* The (absolute) cycle in which the PS instruction is scheduled.
 | 
      
         | 133 |  |  |      Same as SCHED_TIME (node).  */
 | 
      
         | 134 |  |  |   int cycle;
 | 
      
         | 135 |  |  |  
 | 
      
         | 136 |  |  |   /* The next/prev PS_INSN in the same row.  */
 | 
      
         | 137 |  |  |   ps_insn_ptr next_in_row,
 | 
      
         | 138 |  |  |               prev_in_row;
 | 
      
         | 139 |  |  |  
 | 
      
         | 140 |  |  | };
 | 
      
         | 141 |  |  |  
 | 
      
         | 142 |  |  | /* Information about a register move that has been added to a partial
 | 
      
         | 143 |  |  |    schedule.  */
 | 
      
         | 144 |  |  | struct ps_reg_move_info
 | 
      
         | 145 |  |  | {
 | 
      
         | 146 |  |  |   /* The source of the move is defined by the ps_insn with id DEF.
 | 
      
         | 147 |  |  |      The destination is used by the ps_insns with the ids in USES.  */
 | 
      
         | 148 |  |  |   int def;
 | 
      
         | 149 |  |  |   sbitmap uses;
 | 
      
         | 150 |  |  |  
 | 
      
         | 151 |  |  |   /* The original form of USES' instructions used OLD_REG, but they
 | 
      
         | 152 |  |  |      should now use NEW_REG.  */
 | 
      
         | 153 |  |  |   rtx old_reg;
 | 
      
         | 154 |  |  |   rtx new_reg;
 | 
      
         | 155 |  |  |  
 | 
      
         | 156 |  |  |   /* The number of consecutive stages that the move occupies.  */
 | 
      
         | 157 |  |  |   int num_consecutive_stages;
 | 
      
         | 158 |  |  |  
 | 
      
         | 159 |  |  |   /* An instruction that sets NEW_REG to the correct value.  The first
 | 
      
         | 160 |  |  |      move associated with DEF will have an rhs of OLD_REG; later moves
 | 
      
         | 161 |  |  |      use the result of the previous move.  */
 | 
      
         | 162 |  |  |   rtx insn;
 | 
      
         | 163 |  |  | };
 | 
      
         | 164 |  |  |  
 | 
      
         | 165 |  |  | typedef struct ps_reg_move_info ps_reg_move_info;
 | 
      
         | 166 |  |  | DEF_VEC_O (ps_reg_move_info);
 | 
      
         | 167 |  |  | DEF_VEC_ALLOC_O (ps_reg_move_info, heap);
 | 
      
         | 168 |  |  |  
 | 
      
         | 169 |  |  | /* Holds the partial schedule as an array of II rows.  Each entry of the
 | 
      
         | 170 |  |  |    array points to a linked list of PS_INSNs, which represents the
 | 
      
         | 171 |  |  |    instructions that are scheduled for that row.  */
 | 
      
         | 172 |  |  | struct partial_schedule
 | 
      
         | 173 |  |  | {
 | 
      
         | 174 |  |  |   int ii;       /* Number of rows in the partial schedule.  */
 | 
      
         | 175 |  |  |   int history;  /* Threshold for conflict checking using DFA.  */
 | 
      
         | 176 |  |  |  
 | 
      
         | 177 |  |  |   /* rows[i] points to linked list of insns scheduled in row i (0<=i<ii).  */
 | 
      
         | 178 |  |  |   ps_insn_ptr *rows;
 | 
      
         | 179 |  |  |  
 | 
      
         | 180 |  |  |   /* All the moves added for this partial schedule.  Index X has
 | 
      
         | 181 |  |  |      a ps_insn id of X + g->num_nodes.  */
 | 
      
         | 182 |  |  |   VEC (ps_reg_move_info, heap) *reg_moves;
 | 
      
         | 183 |  |  |  
 | 
      
         | 184 |  |  |   /*  rows_length[i] holds the number of instructions in the row.
 | 
      
         | 185 |  |  |       It is used only (as an optimization) to back off quickly from
 | 
      
         | 186 |  |  |       trying to schedule a node in a full row; that is, to avoid running
 | 
      
         | 187 |  |  |       through futile DFA state transitions.  */
 | 
      
         | 188 |  |  |   int *rows_length;
 | 
      
         | 189 |  |  |  
 | 
      
         | 190 |  |  |   /* The earliest absolute cycle of an insn in the partial schedule.  */
 | 
      
         | 191 |  |  |   int min_cycle;
 | 
      
         | 192 |  |  |  
 | 
      
         | 193 |  |  |   /* The latest absolute cycle of an insn in the partial schedule.  */
 | 
      
         | 194 |  |  |   int max_cycle;
 | 
      
         | 195 |  |  |  
 | 
      
         | 196 |  |  |   ddg_ptr g;    /* The DDG of the insns in the partial schedule.  */
 | 
      
         | 197 |  |  |  
 | 
      
         | 198 |  |  |   int stage_count;  /* The stage count of the partial schedule.  */
 | 
      
         | 199 |  |  | };
 | 
      
         | 200 |  |  |  
 | 
      
         | 201 |  |  |  
 | 
      
         | 202 |  |  | static partial_schedule_ptr create_partial_schedule (int ii, ddg_ptr, int history);
 | 
      
         | 203 |  |  | static void free_partial_schedule (partial_schedule_ptr);
 | 
      
         | 204 |  |  | static void reset_partial_schedule (partial_schedule_ptr, int new_ii);
 | 
      
         | 205 |  |  | void print_partial_schedule (partial_schedule_ptr, FILE *);
 | 
      
         | 206 |  |  | static void verify_partial_schedule (partial_schedule_ptr, sbitmap);
 | 
      
         | 207 |  |  | static ps_insn_ptr ps_add_node_check_conflicts (partial_schedule_ptr,
 | 
      
         | 208 |  |  |                                                 int, int, sbitmap, sbitmap);
 | 
      
         | 209 |  |  | static void rotate_partial_schedule (partial_schedule_ptr, int);
 | 
      
         | 210 |  |  | void set_row_column_for_ps (partial_schedule_ptr);
 | 
      
         | 211 |  |  | static void ps_insert_empty_row (partial_schedule_ptr, int, sbitmap);
 | 
      
         | 212 |  |  | static int compute_split_row (sbitmap, int, int, int, ddg_node_ptr);
 | 
      
         | 213 |  |  |  
 | 
      
         | 214 |  |  |  
 | 
      
         | 215 |  |  | /* This page defines constants and structures for the modulo scheduling
 | 
      
         | 216 |  |  |    driver.  */
 | 
      
         | 217 |  |  |  
 | 
      
         | 218 |  |  | static int sms_order_nodes (ddg_ptr, int, int *, int *);
 | 
      
         | 219 |  |  | static void set_node_sched_params (ddg_ptr);
 | 
      
         | 220 |  |  | static partial_schedule_ptr sms_schedule_by_order (ddg_ptr, int, int, int *);
 | 
      
         | 221 |  |  | static void permute_partial_schedule (partial_schedule_ptr, rtx);
 | 
      
         | 222 |  |  | static void generate_prolog_epilog (partial_schedule_ptr, struct loop *,
 | 
      
         | 223 |  |  |                                     rtx, rtx);
 | 
      
         | 224 |  |  | static int calculate_stage_count (partial_schedule_ptr, int);
 | 
      
         | 225 |  |  | static void calculate_must_precede_follow (ddg_node_ptr, int, int,
 | 
      
         | 226 |  |  |                                            int, int, sbitmap, sbitmap, sbitmap);
 | 
      
         | 227 |  |  | static int get_sched_window (partial_schedule_ptr, ddg_node_ptr,
 | 
      
         | 228 |  |  |                              sbitmap, int, int *, int *, int *);
 | 
      
         | 229 |  |  | static bool try_scheduling_node_in_cycle (partial_schedule_ptr, int, int,
 | 
      
         | 230 |  |  |                                           sbitmap, int *, sbitmap, sbitmap);
 | 
      
         | 231 |  |  | static void remove_node_from_ps (partial_schedule_ptr, ps_insn_ptr);
 | 
      
         | 232 |  |  |  
 | 
      
         | 233 |  |  | #define NODE_ASAP(node) ((node)->aux.count)
 | 
      
         | 234 |  |  |  
 | 
      
         | 235 |  |  | #define SCHED_PARAMS(x) VEC_index (node_sched_params, node_sched_param_vec, x)
 | 
      
         | 236 |  |  | #define SCHED_TIME(x) (SCHED_PARAMS (x)->time)
 | 
      
         | 237 |  |  | #define SCHED_ROW(x) (SCHED_PARAMS (x)->row)
 | 
      
         | 238 |  |  | #define SCHED_STAGE(x) (SCHED_PARAMS (x)->stage)
 | 
      
         | 239 |  |  | #define SCHED_COLUMN(x) (SCHED_PARAMS (x)->column)
 | 
      
         | 240 |  |  |  
 | 
      
         | 241 |  |  | /* The scheduling parameters held for each node.  */
 | 
      
         | 242 |  |  | typedef struct node_sched_params
 | 
      
         | 243 |  |  | {
 | 
      
         | 244 |  |  |   int time;     /* The absolute scheduling cycle.  */
 | 
      
         | 245 |  |  |  
 | 
      
         | 246 |  |  |   int row;    /* Holds time % ii.  */
 | 
      
         | 247 |  |  |   int stage;  /* Holds time / ii.  */
 | 
      
         | 248 |  |  |  
 | 
      
         | 249 |  |  |   /* The column of a node inside the ps.  If nodes u, v are on the same row,
 | 
      
         | 250 |  |  |      u will precede v if column (u) < column (v).  */
 | 
      
         | 251 |  |  |   int column;
 | 
      
         | 252 |  |  | } *node_sched_params_ptr;
 | 
      
         | 253 |  |  |  
 | 
      
         | 254 |  |  | typedef struct node_sched_params node_sched_params;
 | 
      
         | 255 |  |  | DEF_VEC_O (node_sched_params);
 | 
      
         | 256 |  |  | DEF_VEC_ALLOC_O (node_sched_params, heap);
 | 
      
         | 257 |  |  |  
 | 
      
         | 258 |  |  | /* The following three functions are copied from the current scheduler
 | 
      
         | 259 |  |  |    code in order to use sched_analyze() for computing the dependencies.
 | 
      
         | 260 |  |  |    They are used when initializing the sched_info structure.  */
 | 
      
         | 261 |  |  | static const char *
 | 
      
         | 262 |  |  | sms_print_insn (const_rtx insn, int aligned ATTRIBUTE_UNUSED)
 | 
      
         | 263 |  |  | {
 | 
      
         | 264 |  |  |   static char tmp[80];
 | 
      
         | 265 |  |  |  
 | 
      
         | 266 |  |  |   sprintf (tmp, "i%4d", INSN_UID (insn));
 | 
      
         | 267 |  |  |   return tmp;
 | 
      
         | 268 |  |  | }
 | 
      
         | 269 |  |  |  
 | 
      
         | 270 |  |  | static void
 | 
      
         | 271 |  |  | compute_jump_reg_dependencies (rtx insn ATTRIBUTE_UNUSED,
 | 
      
         | 272 |  |  |                                regset used ATTRIBUTE_UNUSED)
 | 
      
         | 273 |  |  | {
 | 
      
         | 274 |  |  | }
 | 
      
         | 275 |  |  |  
 | 
      
         | 276 |  |  | static struct common_sched_info_def sms_common_sched_info;
 | 
      
         | 277 |  |  |  
 | 
      
         | 278 |  |  | static struct sched_deps_info_def sms_sched_deps_info =
 | 
      
         | 279 |  |  |   {
 | 
      
         | 280 |  |  |     compute_jump_reg_dependencies,
 | 
      
         | 281 |  |  |     NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
 | 
      
         | 282 |  |  |     NULL,
 | 
      
         | 283 |  |  |     0, 0, 0
 | 
      
         | 284 |  |  |   };
 | 
      
         | 285 |  |  |  
 | 
      
         | 286 |  |  | static struct haifa_sched_info sms_sched_info =
 | 
      
         | 287 |  |  | {
 | 
      
         | 288 |  |  |   NULL,
 | 
      
         | 289 |  |  |   NULL,
 | 
      
         | 290 |  |  |   NULL,
 | 
      
         | 291 |  |  |   NULL,
 | 
      
         | 292 |  |  |   NULL,
 | 
      
         | 293 |  |  |   sms_print_insn,
 | 
      
         | 294 |  |  |   NULL,
 | 
      
         | 295 |  |  |   NULL, /* insn_finishes_block_p */
 | 
      
         | 296 |  |  |   NULL, NULL,
 | 
      
         | 297 |  |  |   NULL, NULL,
 | 
      
         | 298 |  |  |   0, 0,
 | 
      
         | 299 |  |  |  
 | 
      
         | 300 |  |  |   NULL, NULL, NULL, NULL,
 | 
      
         | 301 |  |  |   NULL, NULL,
 | 
      
         | 302 |  |  |  
 | 
      
         | 303 |  |  | };
 | 
      
         | 304 |  |  |  
 | 
      
         | 305 |  |  | /* Partial schedule instruction ID in PS is a register move.  Return
 | 
      
         | 306 |  |  |    information about it.  */
 | 
      
         | 307 |  |  | static struct ps_reg_move_info *
 | 
      
         | 308 |  |  | ps_reg_move (partial_schedule_ptr ps, int id)
 | 
      
         | 309 |  |  | {
 | 
      
         | 310 |  |  |   gcc_checking_assert (id >= ps->g->num_nodes);
 | 
      
         | 311 |  |  |   return VEC_index (ps_reg_move_info, ps->reg_moves, id - ps->g->num_nodes);
 | 
      
         | 312 |  |  | }
 | 
      
         | 313 |  |  |  
 | 
      
         | 314 |  |  | /* Return the rtl instruction that is being scheduled by partial schedule
 | 
      
         | 315 |  |  |    instruction ID, which belongs to schedule PS.  */
 | 
      
         | 316 |  |  | static rtx
 | 
      
         | 317 |  |  | ps_rtl_insn (partial_schedule_ptr ps, int id)
 | 
      
         | 318 |  |  | {
 | 
      
         | 319 |  |  |   if (id < ps->g->num_nodes)
 | 
      
         | 320 |  |  |     return ps->g->nodes[id].insn;
 | 
      
         | 321 |  |  |   else
 | 
      
         | 322 |  |  |     return ps_reg_move (ps, id)->insn;
 | 
      
         | 323 |  |  | }
 | 
      
         | 324 |  |  |  
 | 
      
         | 325 |  |  | /* Partial schedule instruction ID, which belongs to PS, occured in
 | 
      
         | 326 |  |  |    the original (unscheduled) loop.  Return the first instruction
 | 
      
         | 327 |  |  |    in the loop that was associated with ps_rtl_insn (PS, ID).
 | 
      
         | 328 |  |  |    If the instruction had some notes before it, this is the first
 | 
      
         | 329 |  |  |    of those notes.  */
 | 
      
         | 330 |  |  | static rtx
 | 
      
         | 331 |  |  | ps_first_note (partial_schedule_ptr ps, int id)
 | 
      
         | 332 |  |  | {
 | 
      
         | 333 |  |  |   gcc_assert (id < ps->g->num_nodes);
 | 
      
         | 334 |  |  |   return ps->g->nodes[id].first_note;
 | 
      
         | 335 |  |  | }
 | 
      
         | 336 |  |  |  
 | 
      
         | 337 |  |  | /* Return the number of consecutive stages that are occupied by
 | 
      
         | 338 |  |  |    partial schedule instruction ID in PS.  */
 | 
      
         | 339 |  |  | static int
 | 
      
         | 340 |  |  | ps_num_consecutive_stages (partial_schedule_ptr ps, int id)
 | 
      
         | 341 |  |  | {
 | 
      
         | 342 |  |  |   if (id < ps->g->num_nodes)
 | 
      
         | 343 |  |  |     return 1;
 | 
      
         | 344 |  |  |   else
 | 
      
         | 345 |  |  |     return ps_reg_move (ps, id)->num_consecutive_stages;
 | 
      
         | 346 |  |  | }
 | 
      
         | 347 |  |  |  
 | 
      
         | 348 |  |  | /* Given HEAD and TAIL which are the first and last insns in a loop;
 | 
      
         | 349 |  |  |    return the register which controls the loop.  Return zero if it has
 | 
      
         | 350 |  |  |    more than one occurrence in the loop besides the control part or the
 | 
      
         | 351 |  |  |    do-loop pattern is not of the form we expect.  */
 | 
      
         | 352 |  |  | static rtx
 | 
      
         | 353 |  |  | doloop_register_get (rtx head ATTRIBUTE_UNUSED, rtx tail ATTRIBUTE_UNUSED)
 | 
      
         | 354 |  |  | {
 | 
      
         | 355 |  |  | #ifdef HAVE_doloop_end
 | 
      
         | 356 |  |  |   rtx reg, condition, insn, first_insn_not_to_check;
 | 
      
         | 357 |  |  |  
 | 
      
         | 358 |  |  |   if (!JUMP_P (tail))
 | 
      
         | 359 |  |  |     return NULL_RTX;
 | 
      
         | 360 |  |  |  
 | 
      
         | 361 |  |  |   /* TODO: Free SMS's dependence on doloop_condition_get.  */
 | 
      
         | 362 |  |  |   condition = doloop_condition_get (tail);
 | 
      
         | 363 |  |  |   if (! condition)
 | 
      
         | 364 |  |  |     return NULL_RTX;
 | 
      
         | 365 |  |  |  
 | 
      
         | 366 |  |  |   if (REG_P (XEXP (condition, 0)))
 | 
      
         | 367 |  |  |     reg = XEXP (condition, 0);
 | 
      
         | 368 |  |  |   else if (GET_CODE (XEXP (condition, 0)) == PLUS
 | 
      
         | 369 |  |  |            && REG_P (XEXP (XEXP (condition, 0), 0)))
 | 
      
         | 370 |  |  |     reg = XEXP (XEXP (condition, 0), 0);
 | 
      
         | 371 |  |  |   else
 | 
      
         | 372 |  |  |     gcc_unreachable ();
 | 
      
         | 373 |  |  |  
 | 
      
         | 374 |  |  |   /* Check that the COUNT_REG has no other occurrences in the loop
 | 
      
         | 375 |  |  |      until the decrement.  We assume the control part consists of
 | 
      
         | 376 |  |  |      either a single (parallel) branch-on-count or a (non-parallel)
 | 
      
         | 377 |  |  |      branch immediately preceded by a single (decrement) insn.  */
 | 
      
         | 378 |  |  |   first_insn_not_to_check = (GET_CODE (PATTERN (tail)) == PARALLEL ? tail
 | 
      
         | 379 |  |  |                              : prev_nondebug_insn (tail));
 | 
      
         | 380 |  |  |  
 | 
      
         | 381 |  |  |   for (insn = head; insn != first_insn_not_to_check; insn = NEXT_INSN (insn))
 | 
      
         | 382 |  |  |     if (!DEBUG_INSN_P (insn) && reg_mentioned_p (reg, insn))
 | 
      
         | 383 |  |  |       {
 | 
      
         | 384 |  |  |         if (dump_file)
 | 
      
         | 385 |  |  |         {
 | 
      
         | 386 |  |  |           fprintf (dump_file, "SMS count_reg found ");
 | 
      
         | 387 |  |  |           print_rtl_single (dump_file, reg);
 | 
      
         | 388 |  |  |           fprintf (dump_file, " outside control in insn:\n");
 | 
      
         | 389 |  |  |           print_rtl_single (dump_file, insn);
 | 
      
         | 390 |  |  |         }
 | 
      
         | 391 |  |  |  
 | 
      
         | 392 |  |  |         return NULL_RTX;
 | 
      
         | 393 |  |  |       }
 | 
      
         | 394 |  |  |  
 | 
      
         | 395 |  |  |   return reg;
 | 
      
         | 396 |  |  | #else
 | 
      
         | 397 |  |  |   return NULL_RTX;
 | 
      
         | 398 |  |  | #endif
 | 
      
         | 399 |  |  | }
 | 
      
         | 400 |  |  |  
 | 
      
         | 401 |  |  | /* Check if COUNT_REG is set to a constant in the PRE_HEADER block, so
 | 
      
         | 402 |  |  |    that the number of iterations is a compile-time constant.  If so,
 | 
      
         | 403 |  |  |    return the rtx that sets COUNT_REG to a constant, and set COUNT to
 | 
      
         | 404 |  |  |    this constant.  Otherwise return 0.  */
 | 
      
         | 405 |  |  | static rtx
 | 
      
         | 406 |  |  | const_iteration_count (rtx count_reg, basic_block pre_header,
 | 
      
         | 407 |  |  |                        HOST_WIDEST_INT * count)
 | 
      
         | 408 |  |  | {
 | 
      
         | 409 |  |  |   rtx insn;
 | 
      
         | 410 |  |  |   rtx head, tail;
 | 
      
         | 411 |  |  |  
 | 
      
         | 412 |  |  |   if (! pre_header)
 | 
      
         | 413 |  |  |     return NULL_RTX;
 | 
      
         | 414 |  |  |  
 | 
      
         | 415 |  |  |   get_ebb_head_tail (pre_header, pre_header, &head, &tail);
 | 
      
         | 416 |  |  |  
 | 
      
         | 417 |  |  |   for (insn = tail; insn != PREV_INSN (head); insn = PREV_INSN (insn))
 | 
      
         | 418 |  |  |     if (NONDEBUG_INSN_P (insn) && single_set (insn) &&
 | 
      
         | 419 |  |  |         rtx_equal_p (count_reg, SET_DEST (single_set (insn))))
 | 
      
         | 420 |  |  |       {
 | 
      
         | 421 |  |  |         rtx pat = single_set (insn);
 | 
      
         | 422 |  |  |  
 | 
      
         | 423 |  |  |         if (CONST_INT_P (SET_SRC (pat)))
 | 
      
         | 424 |  |  |           {
 | 
      
         | 425 |  |  |             *count = INTVAL (SET_SRC (pat));
 | 
      
         | 426 |  |  |             return insn;
 | 
      
         | 427 |  |  |           }
 | 
      
         | 428 |  |  |  
 | 
      
         | 429 |  |  |         return NULL_RTX;
 | 
      
         | 430 |  |  |       }
 | 
      
         | 431 |  |  |  
 | 
      
         | 432 |  |  |   return NULL_RTX;
 | 
      
         | 433 |  |  | }
 | 
      
         | 434 |  |  |  
 | 
      
         | 435 |  |  | /* A very simple resource-based lower bound on the initiation interval.
 | 
      
         | 436 |  |  |    ??? Improve the accuracy of this bound by considering the
 | 
      
         | 437 |  |  |    utilization of various units.  */
 | 
      
         | 438 |  |  | static int
 | 
      
         | 439 |  |  | res_MII (ddg_ptr g)
 | 
      
         | 440 |  |  | {
 | 
      
         | 441 |  |  |   if (targetm.sched.sms_res_mii)
 | 
      
         | 442 |  |  |     return targetm.sched.sms_res_mii (g);
 | 
      
         | 443 |  |  |  
 | 
      
         | 444 |  |  |   return ((g->num_nodes - g->num_debug) / issue_rate);
 | 
      
         | 445 |  |  | }
 | 
      
         | 446 |  |  |  
 | 
      
         | 447 |  |  |  
 | 
      
         | 448 |  |  | /* A vector that contains the sched data for each ps_insn.  */
 | 
      
         | 449 |  |  | static VEC (node_sched_params, heap) *node_sched_param_vec;
 | 
      
         | 450 |  |  |  
 | 
      
         | 451 |  |  | /* Allocate sched_params for each node and initialize it.  */
 | 
      
         | 452 |  |  | static void
 | 
      
         | 453 |  |  | set_node_sched_params (ddg_ptr g)
 | 
      
         | 454 |  |  | {
 | 
      
         | 455 |  |  |   VEC_truncate (node_sched_params, node_sched_param_vec, 0);
 | 
      
         | 456 |  |  |   VEC_safe_grow_cleared (node_sched_params, heap,
 | 
      
         | 457 |  |  |                          node_sched_param_vec, g->num_nodes);
 | 
      
         | 458 |  |  | }
 | 
      
         | 459 |  |  |  
 | 
      
         | 460 |  |  | /* Make sure that node_sched_param_vec has an entry for every move in PS.  */
 | 
      
         | 461 |  |  | static void
 | 
      
         | 462 |  |  | extend_node_sched_params (partial_schedule_ptr ps)
 | 
      
         | 463 |  |  | {
 | 
      
         | 464 |  |  |   VEC_safe_grow_cleared (node_sched_params, heap, node_sched_param_vec,
 | 
      
         | 465 |  |  |                          ps->g->num_nodes + VEC_length (ps_reg_move_info,
 | 
      
         | 466 |  |  |                                                         ps->reg_moves));
 | 
      
         | 467 |  |  | }
 | 
      
         | 468 |  |  |  
 | 
      
         | 469 |  |  | /* Update the sched_params (time, row and stage) for node U using the II,
 | 
      
         | 470 |  |  |    the CYCLE of U and MIN_CYCLE.
 | 
      
         | 471 |  |  |    We're not simply taking the following
 | 
      
         | 472 |  |  |    SCHED_STAGE (u) = CALC_STAGE_COUNT (SCHED_TIME (u), min_cycle, ii);
 | 
      
         | 473 |  |  |    because the stages may not be aligned on cycle 0.  */
 | 
      
         | 474 |  |  | static void
 | 
      
         | 475 |  |  | update_node_sched_params (int u, int ii, int cycle, int min_cycle)
 | 
      
         | 476 |  |  | {
 | 
      
         | 477 |  |  |   int sc_until_cycle_zero;
 | 
      
         | 478 |  |  |   int stage;
 | 
      
         | 479 |  |  |  
 | 
      
         | 480 |  |  |   SCHED_TIME (u) = cycle;
 | 
      
         | 481 |  |  |   SCHED_ROW (u) = SMODULO (cycle, ii);
 | 
      
         | 482 |  |  |  
 | 
      
         | 483 |  |  |   /* The calculation of stage count is done adding the number
 | 
      
         | 484 |  |  |      of stages before cycle zero and after cycle zero.  */
 | 
      
         | 485 |  |  |   sc_until_cycle_zero = CALC_STAGE_COUNT (-1, min_cycle, ii);
 | 
      
         | 486 |  |  |  
 | 
      
         | 487 |  |  |   if (SCHED_TIME (u) < 0)
 | 
      
         | 488 |  |  |     {
 | 
      
         | 489 |  |  |       stage = CALC_STAGE_COUNT (-1, SCHED_TIME (u), ii);
 | 
      
         | 490 |  |  |       SCHED_STAGE (u) = sc_until_cycle_zero - stage;
 | 
      
         | 491 |  |  |     }
 | 
      
         | 492 |  |  |   else
 | 
      
         | 493 |  |  |     {
 | 
      
         | 494 |  |  |       stage = CALC_STAGE_COUNT (SCHED_TIME (u), 0, ii);
 | 
      
         | 495 |  |  |       SCHED_STAGE (u) = sc_until_cycle_zero + stage - 1;
 | 
      
         | 496 |  |  |     }
 | 
      
         | 497 |  |  | }
 | 
      
         | 498 |  |  |  
 | 
      
         | 499 |  |  | static void
 | 
      
         | 500 |  |  | print_node_sched_params (FILE *file, int num_nodes, partial_schedule_ptr ps)
 | 
      
         | 501 |  |  | {
 | 
      
         | 502 |  |  |   int i;
 | 
      
         | 503 |  |  |  
 | 
      
         | 504 |  |  |   if (! file)
 | 
      
         | 505 |  |  |     return;
 | 
      
         | 506 |  |  |   for (i = 0; i < num_nodes; i++)
 | 
      
         | 507 |  |  |     {
 | 
      
         | 508 |  |  |       node_sched_params_ptr nsp = SCHED_PARAMS (i);
 | 
      
         | 509 |  |  |  
 | 
      
         | 510 |  |  |       fprintf (file, "Node = %d; INSN = %d\n", i,
 | 
      
         | 511 |  |  |                INSN_UID (ps_rtl_insn (ps, i)));
 | 
      
         | 512 |  |  |       fprintf (file, " asap = %d:\n", NODE_ASAP (&ps->g->nodes[i]));
 | 
      
         | 513 |  |  |       fprintf (file, " time = %d:\n", nsp->time);
 | 
      
         | 514 |  |  |       fprintf (file, " stage = %d:\n", nsp->stage);
 | 
      
         | 515 |  |  |     }
 | 
      
         | 516 |  |  | }
 | 
      
         | 517 |  |  |  
 | 
      
         | 518 |  |  | /* Set SCHED_COLUMN for each instruction in row ROW of PS.  */
 | 
      
         | 519 |  |  | static void
 | 
      
         | 520 |  |  | set_columns_for_row (partial_schedule_ptr ps, int row)
 | 
      
         | 521 |  |  | {
 | 
      
         | 522 |  |  |   ps_insn_ptr cur_insn;
 | 
      
         | 523 |  |  |   int column;
 | 
      
         | 524 |  |  |  
 | 
      
         | 525 |  |  |   column = 0;
 | 
      
         | 526 |  |  |   for (cur_insn = ps->rows[row]; cur_insn; cur_insn = cur_insn->next_in_row)
 | 
      
         | 527 |  |  |     SCHED_COLUMN (cur_insn->id) = column++;
 | 
      
         | 528 |  |  | }
 | 
      
         | 529 |  |  |  
 | 
      
         | 530 |  |  | /* Set SCHED_COLUMN for each instruction in PS.  */
 | 
      
         | 531 |  |  | static void
 | 
      
         | 532 |  |  | set_columns_for_ps (partial_schedule_ptr ps)
 | 
      
         | 533 |  |  | {
 | 
      
         | 534 |  |  |   int row;
 | 
      
         | 535 |  |  |  
 | 
      
         | 536 |  |  |   for (row = 0; row < ps->ii; row++)
 | 
      
         | 537 |  |  |     set_columns_for_row (ps, row);
 | 
      
         | 538 |  |  | }
 | 
      
         | 539 |  |  |  
 | 
      
         | 540 |  |  | /* Try to schedule the move with ps_insn identifier I_REG_MOVE in PS.
 | 
      
         | 541 |  |  |    Its single predecessor has already been scheduled, as has its
 | 
      
         | 542 |  |  |    ddg node successors.  (The move may have also another move as its
 | 
      
         | 543 |  |  |    successor, in which case that successor will be scheduled later.)
 | 
      
         | 544 |  |  |  
 | 
      
         | 545 |  |  |    The move is part of a chain that satisfies register dependencies
 | 
      
         | 546 |  |  |    between a producing ddg node and various consuming ddg nodes.
 | 
      
         | 547 |  |  |    If some of these dependencies have a distance of 1 (meaning that
 | 
      
         | 548 |  |  |    the use is upward-exposed) then DISTANCE1_USES is nonnull and
 | 
      
         | 549 |  |  |    contains the set of uses with distance-1 dependencies.
 | 
      
         | 550 |  |  |    DISTANCE1_USES is null otherwise.
 | 
      
         | 551 |  |  |  
 | 
      
         | 552 |  |  |    MUST_FOLLOW is a scratch bitmap that is big enough to hold
 | 
      
         | 553 |  |  |    all current ps_insn ids.
 | 
      
         | 554 |  |  |  
 | 
      
         | 555 |  |  |    Return true on success.  */
 | 
      
         | 556 |  |  | static bool
 | 
      
         | 557 |  |  | schedule_reg_move (partial_schedule_ptr ps, int i_reg_move,
 | 
      
         | 558 |  |  |                    sbitmap distance1_uses, sbitmap must_follow)
 | 
      
         | 559 |  |  | {
 | 
      
         | 560 |  |  |   unsigned int u;
 | 
      
         | 561 |  |  |   int this_time, this_distance, this_start, this_end, this_latency;
 | 
      
         | 562 |  |  |   int start, end, c, ii;
 | 
      
         | 563 |  |  |   sbitmap_iterator sbi;
 | 
      
         | 564 |  |  |   ps_reg_move_info *move;
 | 
      
         | 565 |  |  |   rtx this_insn;
 | 
      
         | 566 |  |  |   ps_insn_ptr psi;
 | 
      
         | 567 |  |  |  
 | 
      
         | 568 |  |  |   move = ps_reg_move (ps, i_reg_move);
 | 
      
         | 569 |  |  |   ii = ps->ii;
 | 
      
         | 570 |  |  |   if (dump_file)
 | 
      
         | 571 |  |  |     {
 | 
      
         | 572 |  |  |       fprintf (dump_file, "Scheduling register move INSN %d; ii = %d"
 | 
      
         | 573 |  |  |                ", min cycle = %d\n\n", INSN_UID (move->insn), ii,
 | 
      
         | 574 |  |  |                PS_MIN_CYCLE (ps));
 | 
      
         | 575 |  |  |       print_rtl_single (dump_file, move->insn);
 | 
      
         | 576 |  |  |       fprintf (dump_file, "\n%11s %11s %5s\n", "start", "end", "time");
 | 
      
         | 577 |  |  |       fprintf (dump_file, "=========== =========== =====\n");
 | 
      
         | 578 |  |  |     }
 | 
      
         | 579 |  |  |  
 | 
      
         | 580 |  |  |   start = INT_MIN;
 | 
      
         | 581 |  |  |   end = INT_MAX;
 | 
      
         | 582 |  |  |  
 | 
      
         | 583 |  |  |   /* For dependencies of distance 1 between a producer ddg node A
 | 
      
         | 584 |  |  |      and consumer ddg node B, we have a chain of dependencies:
 | 
      
         | 585 |  |  |  
 | 
      
         | 586 |  |  |         A --(T,L1,1)--> M1 --(T,L2,0)--> M2 ... --(T,Ln,0)--> B
 | 
      
         | 587 |  |  |  
 | 
      
         | 588 |  |  |      where Mi is the ith move.  For dependencies of distance 0 between
 | 
      
         | 589 |  |  |      a producer ddg node A and consumer ddg node C, we have a chain of
 | 
      
         | 590 |  |  |      dependencies:
 | 
      
         | 591 |  |  |  
 | 
      
         | 592 |  |  |         A --(T,L1',0)--> M1' --(T,L2',0)--> M2' ... --(T,Ln',0)--> C
 | 
      
         | 593 |  |  |  
 | 
      
         | 594 |  |  |      where Mi' occupies the same position as Mi but occurs a stage later.
 | 
      
         | 595 |  |  |      We can only schedule each move once, so if we have both types of
 | 
      
         | 596 |  |  |      chain, we model the second as:
 | 
      
         | 597 |  |  |  
 | 
      
         | 598 |  |  |         A --(T,L1',1)--> M1 --(T,L2',0)--> M2 ... --(T,Ln',-1)--> C
 | 
      
         | 599 |  |  |  
 | 
      
         | 600 |  |  |      First handle the dependencies between the previously-scheduled
 | 
      
         | 601 |  |  |      predecessor and the move.  */
 | 
      
         | 602 |  |  |   this_insn = ps_rtl_insn (ps, move->def);
 | 
      
         | 603 |  |  |   this_latency = insn_latency (this_insn, move->insn);
 | 
      
         | 604 |  |  |   this_distance = distance1_uses && move->def < ps->g->num_nodes ? 1 : 0;
 | 
      
         | 605 |  |  |   this_time = SCHED_TIME (move->def) - this_distance * ii;
 | 
      
         | 606 |  |  |   this_start = this_time + this_latency;
 | 
      
         | 607 |  |  |   this_end = this_time + ii;
 | 
      
         | 608 |  |  |   if (dump_file)
 | 
      
         | 609 |  |  |     fprintf (dump_file, "%11d %11d %5d %d --(T,%d,%d)--> %d\n",
 | 
      
         | 610 |  |  |              this_start, this_end, SCHED_TIME (move->def),
 | 
      
         | 611 |  |  |              INSN_UID (this_insn), this_latency, this_distance,
 | 
      
         | 612 |  |  |              INSN_UID (move->insn));
 | 
      
         | 613 |  |  |  
 | 
      
         | 614 |  |  |   if (start < this_start)
 | 
      
         | 615 |  |  |     start = this_start;
 | 
      
         | 616 |  |  |   if (end > this_end)
 | 
      
         | 617 |  |  |     end = this_end;
 | 
      
         | 618 |  |  |  
 | 
      
         | 619 |  |  |   /* Handle the dependencies between the move and previously-scheduled
 | 
      
         | 620 |  |  |      successors.  */
 | 
      
         | 621 |  |  |   EXECUTE_IF_SET_IN_SBITMAP (move->uses, 0, u, sbi)
 | 
      
         | 622 |  |  |     {
 | 
      
         | 623 |  |  |       this_insn = ps_rtl_insn (ps, u);
 | 
      
         | 624 |  |  |       this_latency = insn_latency (move->insn, this_insn);
 | 
      
         | 625 |  |  |       if (distance1_uses && !TEST_BIT (distance1_uses, u))
 | 
      
         | 626 |  |  |         this_distance = -1;
 | 
      
         | 627 |  |  |       else
 | 
      
         | 628 |  |  |         this_distance = 0;
 | 
      
         | 629 |  |  |       this_time = SCHED_TIME (u) + this_distance * ii;
 | 
      
         | 630 |  |  |       this_start = this_time - ii;
 | 
      
         | 631 |  |  |       this_end = this_time - this_latency;
 | 
      
         | 632 |  |  |       if (dump_file)
 | 
      
         | 633 |  |  |         fprintf (dump_file, "%11d %11d %5d %d --(T,%d,%d)--> %d\n",
 | 
      
         | 634 |  |  |                  this_start, this_end, SCHED_TIME (u), INSN_UID (move->insn),
 | 
      
         | 635 |  |  |                  this_latency, this_distance, INSN_UID (this_insn));
 | 
      
         | 636 |  |  |  
 | 
      
         | 637 |  |  |       if (start < this_start)
 | 
      
         | 638 |  |  |         start = this_start;
 | 
      
         | 639 |  |  |       if (end > this_end)
 | 
      
         | 640 |  |  |         end = this_end;
 | 
      
         | 641 |  |  |     }
 | 
      
         | 642 |  |  |  
 | 
      
         | 643 |  |  |   if (dump_file)
 | 
      
         | 644 |  |  |     {
 | 
      
         | 645 |  |  |       fprintf (dump_file, "----------- ----------- -----\n");
 | 
      
         | 646 |  |  |       fprintf (dump_file, "%11d %11d %5s %s\n", start, end, "", "(max, min)");
 | 
      
         | 647 |  |  |     }
 | 
      
         | 648 |  |  |  
 | 
      
         | 649 |  |  |   sbitmap_zero (must_follow);
 | 
      
         | 650 |  |  |   SET_BIT (must_follow, move->def);
 | 
      
         | 651 |  |  |  
 | 
      
         | 652 |  |  |   start = MAX (start, end - (ii - 1));
 | 
      
         | 653 |  |  |   for (c = end; c >= start; c--)
 | 
      
         | 654 |  |  |     {
 | 
      
         | 655 |  |  |       psi = ps_add_node_check_conflicts (ps, i_reg_move, c,
 | 
      
         | 656 |  |  |                                          move->uses, must_follow);
 | 
      
         | 657 |  |  |       if (psi)
 | 
      
         | 658 |  |  |         {
 | 
      
         | 659 |  |  |           update_node_sched_params (i_reg_move, ii, c, PS_MIN_CYCLE (ps));
 | 
      
         | 660 |  |  |           if (dump_file)
 | 
      
         | 661 |  |  |             fprintf (dump_file, "\nScheduled register move INSN %d at"
 | 
      
         | 662 |  |  |                      " time %d, row %d\n\n", INSN_UID (move->insn), c,
 | 
      
         | 663 |  |  |                      SCHED_ROW (i_reg_move));
 | 
      
         | 664 |  |  |           return true;
 | 
      
         | 665 |  |  |         }
 | 
      
         | 666 |  |  |     }
 | 
      
         | 667 |  |  |  
 | 
      
         | 668 |  |  |   if (dump_file)
 | 
      
         | 669 |  |  |     fprintf (dump_file, "\nNo available slot\n\n");
 | 
      
         | 670 |  |  |  
 | 
      
         | 671 |  |  |   return false;
 | 
      
         | 672 |  |  | }
 | 
      
         | 673 |  |  |  
 | 
      
         | 674 |  |  | /*
 | 
      
         | 675 |  |  |    Breaking intra-loop register anti-dependences:
 | 
      
         | 676 |  |  |    Each intra-loop register anti-dependence implies a cross-iteration true
 | 
      
         | 677 |  |  |    dependence of distance 1. Therefore, we can remove such false dependencies
 | 
      
         | 678 |  |  |    and figure out if the partial schedule broke them by checking if (for a
 | 
      
         | 679 |  |  |    true-dependence of distance 1): SCHED_TIME (def) < SCHED_TIME (use) and
 | 
      
         | 680 |  |  |    if so generate a register move.   The number of such moves is equal to:
 | 
      
         | 681 |  |  |               SCHED_TIME (use) - SCHED_TIME (def)       { 0 broken
 | 
      
         | 682 |  |  |    nreg_moves = ----------------------------------- + 1 - {   dependence.
 | 
      
         | 683 |  |  |                             ii                          { 1 if not.
 | 
      
         | 684 |  |  | */
 | 
      
         | 685 |  |  | static bool
 | 
      
         | 686 |  |  | schedule_reg_moves (partial_schedule_ptr ps)
 | 
      
         | 687 |  |  | {
 | 
      
         | 688 |  |  |   ddg_ptr g = ps->g;
 | 
      
         | 689 |  |  |   int ii = ps->ii;
 | 
      
         | 690 |  |  |   int i;
 | 
      
         | 691 |  |  |  
 | 
      
         | 692 |  |  |   for (i = 0; i < g->num_nodes; i++)
 | 
      
         | 693 |  |  |     {
 | 
      
         | 694 |  |  |       ddg_node_ptr u = &g->nodes[i];
 | 
      
         | 695 |  |  |       ddg_edge_ptr e;
 | 
      
         | 696 |  |  |       int nreg_moves = 0, i_reg_move;
 | 
      
         | 697 |  |  |       rtx prev_reg, old_reg;
 | 
      
         | 698 |  |  |       int first_move;
 | 
      
         | 699 |  |  |       int distances[2];
 | 
      
         | 700 |  |  |       sbitmap must_follow;
 | 
      
         | 701 |  |  |       sbitmap distance1_uses;
 | 
      
         | 702 |  |  |       rtx set = single_set (u->insn);
 | 
      
         | 703 |  |  |  
 | 
      
         | 704 |  |  |       /* Skip instructions that do not set a register.  */
 | 
      
         | 705 |  |  |       if ((set && !REG_P (SET_DEST (set))))
 | 
      
         | 706 |  |  |         continue;
 | 
      
         | 707 |  |  |  
 | 
      
         | 708 |  |  |       /* Compute the number of reg_moves needed for u, by looking at life
 | 
      
         | 709 |  |  |          ranges started at u (excluding self-loops).  */
 | 
      
         | 710 |  |  |       distances[0] = distances[1] = false;
 | 
      
         | 711 |  |  |       for (e = u->out; e; e = e->next_out)
 | 
      
         | 712 |  |  |         if (e->type == TRUE_DEP && e->dest != e->src)
 | 
      
         | 713 |  |  |           {
 | 
      
         | 714 |  |  |             int nreg_moves4e = (SCHED_TIME (e->dest->cuid)
 | 
      
         | 715 |  |  |                                 - SCHED_TIME (e->src->cuid)) / ii;
 | 
      
         | 716 |  |  |  
 | 
      
         | 717 |  |  |             if (e->distance == 1)
 | 
      
         | 718 |  |  |               nreg_moves4e = (SCHED_TIME (e->dest->cuid)
 | 
      
         | 719 |  |  |                               - SCHED_TIME (e->src->cuid) + ii) / ii;
 | 
      
         | 720 |  |  |  
 | 
      
         | 721 |  |  |             /* If dest precedes src in the schedule of the kernel, then dest
 | 
      
         | 722 |  |  |                will read before src writes and we can save one reg_copy.  */
 | 
      
         | 723 |  |  |             if (SCHED_ROW (e->dest->cuid) == SCHED_ROW (e->src->cuid)
 | 
      
         | 724 |  |  |                 && SCHED_COLUMN (e->dest->cuid) < SCHED_COLUMN (e->src->cuid))
 | 
      
         | 725 |  |  |               nreg_moves4e--;
 | 
      
         | 726 |  |  |  
 | 
      
         | 727 |  |  |             if (nreg_moves4e >= 1)
 | 
      
         | 728 |  |  |               {
 | 
      
         | 729 |  |  |                 /* !single_set instructions are not supported yet and
 | 
      
         | 730 |  |  |                    thus we do not except to encounter them in the loop
 | 
      
         | 731 |  |  |                    except from the doloop part.  For the latter case
 | 
      
         | 732 |  |  |                    we assume no regmoves are generated as the doloop
 | 
      
         | 733 |  |  |                    instructions are tied to the branch with an edge.  */
 | 
      
         | 734 |  |  |                 gcc_assert (set);
 | 
      
         | 735 |  |  |                 /* If the instruction contains auto-inc register then
 | 
      
         | 736 |  |  |                    validate that the regmov is being generated for the
 | 
      
         | 737 |  |  |                    target regsiter rather then the inc'ed register.     */
 | 
      
         | 738 |  |  |                 gcc_assert (!autoinc_var_is_used_p (u->insn, e->dest->insn));
 | 
      
         | 739 |  |  |               }
 | 
      
         | 740 |  |  |  
 | 
      
         | 741 |  |  |             if (nreg_moves4e)
 | 
      
         | 742 |  |  |               {
 | 
      
         | 743 |  |  |                 gcc_assert (e->distance < 2);
 | 
      
         | 744 |  |  |                 distances[e->distance] = true;
 | 
      
         | 745 |  |  |               }
 | 
      
         | 746 |  |  |             nreg_moves = MAX (nreg_moves, nreg_moves4e);
 | 
      
         | 747 |  |  |           }
 | 
      
         | 748 |  |  |  
 | 
      
         | 749 |  |  |       if (nreg_moves == 0)
 | 
      
         | 750 |  |  |         continue;
 | 
      
         | 751 |  |  |  
 | 
      
         | 752 |  |  |       /* Create NREG_MOVES register moves.  */
 | 
      
         | 753 |  |  |       first_move = VEC_length (ps_reg_move_info, ps->reg_moves);
 | 
      
         | 754 |  |  |       VEC_safe_grow_cleared (ps_reg_move_info, heap, ps->reg_moves,
 | 
      
         | 755 |  |  |                              first_move + nreg_moves);
 | 
      
         | 756 |  |  |       extend_node_sched_params (ps);
 | 
      
         | 757 |  |  |  
 | 
      
         | 758 |  |  |       /* Record the moves associated with this node.  */
 | 
      
         | 759 |  |  |       first_move += ps->g->num_nodes;
 | 
      
         | 760 |  |  |  
 | 
      
         | 761 |  |  |       /* Generate each move.  */
 | 
      
         | 762 |  |  |       old_reg = prev_reg = SET_DEST (single_set (u->insn));
 | 
      
         | 763 |  |  |       for (i_reg_move = 0; i_reg_move < nreg_moves; i_reg_move++)
 | 
      
         | 764 |  |  |         {
 | 
      
         | 765 |  |  |           ps_reg_move_info *move = ps_reg_move (ps, first_move + i_reg_move);
 | 
      
         | 766 |  |  |  
 | 
      
         | 767 |  |  |           move->def = i_reg_move > 0 ? first_move + i_reg_move - 1 : i;
 | 
      
         | 768 |  |  |           move->uses = sbitmap_alloc (first_move + nreg_moves);
 | 
      
         | 769 |  |  |           move->old_reg = old_reg;
 | 
      
         | 770 |  |  |           move->new_reg = gen_reg_rtx (GET_MODE (prev_reg));
 | 
      
         | 771 |  |  |           move->num_consecutive_stages = distances[0] && distances[1] ? 2 : 1;
 | 
      
         | 772 |  |  |           move->insn = gen_move_insn (move->new_reg, copy_rtx (prev_reg));
 | 
      
         | 773 |  |  |           sbitmap_zero (move->uses);
 | 
      
         | 774 |  |  |  
 | 
      
         | 775 |  |  |           prev_reg = move->new_reg;
 | 
      
         | 776 |  |  |         }
 | 
      
         | 777 |  |  |  
 | 
      
         | 778 |  |  |       distance1_uses = distances[1] ? sbitmap_alloc (g->num_nodes) : NULL;
 | 
      
         | 779 |  |  |  
 | 
      
         | 780 |  |  |       /* Every use of the register defined by node may require a different
 | 
      
         | 781 |  |  |          copy of this register, depending on the time the use is scheduled.
 | 
      
         | 782 |  |  |          Record which uses require which move results.  */
 | 
      
         | 783 |  |  |       for (e = u->out; e; e = e->next_out)
 | 
      
         | 784 |  |  |         if (e->type == TRUE_DEP && e->dest != e->src)
 | 
      
         | 785 |  |  |           {
 | 
      
         | 786 |  |  |             int dest_copy = (SCHED_TIME (e->dest->cuid)
 | 
      
         | 787 |  |  |                              - SCHED_TIME (e->src->cuid)) / ii;
 | 
      
         | 788 |  |  |  
 | 
      
         | 789 |  |  |             if (e->distance == 1)
 | 
      
         | 790 |  |  |               dest_copy = (SCHED_TIME (e->dest->cuid)
 | 
      
         | 791 |  |  |                            - SCHED_TIME (e->src->cuid) + ii) / ii;
 | 
      
         | 792 |  |  |  
 | 
      
         | 793 |  |  |             if (SCHED_ROW (e->dest->cuid) == SCHED_ROW (e->src->cuid)
 | 
      
         | 794 |  |  |                 && SCHED_COLUMN (e->dest->cuid) < SCHED_COLUMN (e->src->cuid))
 | 
      
         | 795 |  |  |               dest_copy--;
 | 
      
         | 796 |  |  |  
 | 
      
         | 797 |  |  |             if (dest_copy)
 | 
      
         | 798 |  |  |               {
 | 
      
         | 799 |  |  |                 ps_reg_move_info *move;
 | 
      
         | 800 |  |  |  
 | 
      
         | 801 |  |  |                 move = ps_reg_move (ps, first_move + dest_copy - 1);
 | 
      
         | 802 |  |  |                 SET_BIT (move->uses, e->dest->cuid);
 | 
      
         | 803 |  |  |                 if (e->distance == 1)
 | 
      
         | 804 |  |  |                   SET_BIT (distance1_uses, e->dest->cuid);
 | 
      
         | 805 |  |  |               }
 | 
      
         | 806 |  |  |           }
 | 
      
         | 807 |  |  |  
 | 
      
         | 808 |  |  |       must_follow = sbitmap_alloc (first_move + nreg_moves);
 | 
      
         | 809 |  |  |       for (i_reg_move = 0; i_reg_move < nreg_moves; i_reg_move++)
 | 
      
         | 810 |  |  |         if (!schedule_reg_move (ps, first_move + i_reg_move,
 | 
      
         | 811 |  |  |                                 distance1_uses, must_follow))
 | 
      
         | 812 |  |  |           break;
 | 
      
         | 813 |  |  |       sbitmap_free (must_follow);
 | 
      
         | 814 |  |  |       if (distance1_uses)
 | 
      
         | 815 |  |  |         sbitmap_free (distance1_uses);
 | 
      
         | 816 |  |  |       if (i_reg_move < nreg_moves)
 | 
      
         | 817 |  |  |         return false;
 | 
      
         | 818 |  |  |     }
 | 
      
         | 819 |  |  |   return true;
 | 
      
         | 820 |  |  | }
 | 
      
         | 821 |  |  |  
 | 
      
         | 822 |  |  | /* Emit the moves associatied with PS.  Apply the substitutions
 | 
      
         | 823 |  |  |    associated with them.  */
 | 
      
         | 824 |  |  | static void
 | 
      
         | 825 |  |  | apply_reg_moves (partial_schedule_ptr ps)
 | 
      
         | 826 |  |  | {
 | 
      
         | 827 |  |  |   ps_reg_move_info *move;
 | 
      
         | 828 |  |  |   int i;
 | 
      
         | 829 |  |  |  
 | 
      
         | 830 |  |  |   FOR_EACH_VEC_ELT (ps_reg_move_info, ps->reg_moves, i, move)
 | 
      
         | 831 |  |  |     {
 | 
      
         | 832 |  |  |       unsigned int i_use;
 | 
      
         | 833 |  |  |       sbitmap_iterator sbi;
 | 
      
         | 834 |  |  |  
 | 
      
         | 835 |  |  |       EXECUTE_IF_SET_IN_SBITMAP (move->uses, 0, i_use, sbi)
 | 
      
         | 836 |  |  |         {
 | 
      
         | 837 |  |  |           replace_rtx (ps->g->nodes[i_use].insn, move->old_reg, move->new_reg);
 | 
      
         | 838 |  |  |           df_insn_rescan (ps->g->nodes[i_use].insn);
 | 
      
         | 839 |  |  |         }
 | 
      
         | 840 |  |  |     }
 | 
      
         | 841 |  |  | }
 | 
      
         | 842 |  |  |  
 | 
      
         | 843 |  |  | /* Bump the SCHED_TIMEs of all nodes by AMOUNT.  Set the values of
 | 
      
         | 844 |  |  |    SCHED_ROW and SCHED_STAGE.  Instruction scheduled on cycle AMOUNT
 | 
      
         | 845 |  |  |    will move to cycle zero.  */
 | 
      
         | 846 |  |  | static void
 | 
      
         | 847 |  |  | reset_sched_times (partial_schedule_ptr ps, int amount)
 | 
      
         | 848 |  |  | {
 | 
      
         | 849 |  |  |   int row;
 | 
      
         | 850 |  |  |   int ii = ps->ii;
 | 
      
         | 851 |  |  |   ps_insn_ptr crr_insn;
 | 
      
         | 852 |  |  |  
 | 
      
         | 853 |  |  |   for (row = 0; row < ii; row++)
 | 
      
         | 854 |  |  |     for (crr_insn = ps->rows[row]; crr_insn; crr_insn = crr_insn->next_in_row)
 | 
      
         | 855 |  |  |       {
 | 
      
         | 856 |  |  |         int u = crr_insn->id;
 | 
      
         | 857 |  |  |         int normalized_time = SCHED_TIME (u) - amount;
 | 
      
         | 858 |  |  |         int new_min_cycle = PS_MIN_CYCLE (ps) - amount;
 | 
      
         | 859 |  |  |  
 | 
      
         | 860 |  |  |         if (dump_file)
 | 
      
         | 861 |  |  |           {
 | 
      
         | 862 |  |  |             /* Print the scheduling times after the rotation.  */
 | 
      
         | 863 |  |  |             rtx insn = ps_rtl_insn (ps, u);
 | 
      
         | 864 |  |  |  
 | 
      
         | 865 |  |  |             fprintf (dump_file, "crr_insn->node=%d (insn id %d), "
 | 
      
         | 866 |  |  |                      "crr_insn->cycle=%d, min_cycle=%d", u,
 | 
      
         | 867 |  |  |                      INSN_UID (insn), normalized_time, new_min_cycle);
 | 
      
         | 868 |  |  |             if (JUMP_P (insn))
 | 
      
         | 869 |  |  |               fprintf (dump_file, " (branch)");
 | 
      
         | 870 |  |  |             fprintf (dump_file, "\n");
 | 
      
         | 871 |  |  |           }
 | 
      
         | 872 |  |  |  
 | 
      
         | 873 |  |  |         gcc_assert (SCHED_TIME (u) >= ps->min_cycle);
 | 
      
         | 874 |  |  |         gcc_assert (SCHED_TIME (u) <= ps->max_cycle);
 | 
      
         | 875 |  |  |  
 | 
      
         | 876 |  |  |         crr_insn->cycle = normalized_time;
 | 
      
         | 877 |  |  |         update_node_sched_params (u, ii, normalized_time, new_min_cycle);
 | 
      
         | 878 |  |  |       }
 | 
      
         | 879 |  |  | }
 | 
      
         | 880 |  |  |  
 | 
      
         | 881 |  |  | /* Permute the insns according to their order in PS, from row 0 to
 | 
      
         | 882 |  |  |    row ii-1, and position them right before LAST.  This schedules
 | 
      
         | 883 |  |  |    the insns of the loop kernel.  */
 | 
      
         | 884 |  |  | static void
 | 
      
         | 885 |  |  | permute_partial_schedule (partial_schedule_ptr ps, rtx last)
 | 
      
         | 886 |  |  | {
 | 
      
         | 887 |  |  |   int ii = ps->ii;
 | 
      
         | 888 |  |  |   int row;
 | 
      
         | 889 |  |  |   ps_insn_ptr ps_ij;
 | 
      
         | 890 |  |  |  
 | 
      
         | 891 |  |  |   for (row = 0; row < ii ; row++)
 | 
      
         | 892 |  |  |     for (ps_ij = ps->rows[row]; ps_ij; ps_ij = ps_ij->next_in_row)
 | 
      
         | 893 |  |  |       {
 | 
      
         | 894 |  |  |         rtx insn = ps_rtl_insn (ps, ps_ij->id);
 | 
      
         | 895 |  |  |  
 | 
      
         | 896 |  |  |         if (PREV_INSN (last) != insn)
 | 
      
         | 897 |  |  |           {
 | 
      
         | 898 |  |  |             if (ps_ij->id < ps->g->num_nodes)
 | 
      
         | 899 |  |  |               reorder_insns_nobb (ps_first_note (ps, ps_ij->id), insn,
 | 
      
         | 900 |  |  |                                   PREV_INSN (last));
 | 
      
         | 901 |  |  |             else
 | 
      
         | 902 |  |  |               add_insn_before (insn, last, NULL);
 | 
      
         | 903 |  |  |           }
 | 
      
         | 904 |  |  |       }
 | 
      
         | 905 |  |  | }
 | 
      
         | 906 |  |  |  
 | 
      
         | 907 |  |  | /* Set bitmaps TMP_FOLLOW and TMP_PRECEDE to MUST_FOLLOW and MUST_PRECEDE
 | 
      
         | 908 |  |  |    respectively only if cycle C falls on the border of the scheduling
 | 
      
         | 909 |  |  |    window boundaries marked by START and END cycles.  STEP is the
 | 
      
         | 910 |  |  |    direction of the window.  */
 | 
      
         | 911 |  |  | static inline void
 | 
      
         | 912 |  |  | set_must_precede_follow (sbitmap *tmp_follow, sbitmap must_follow,
 | 
      
         | 913 |  |  |                          sbitmap *tmp_precede, sbitmap must_precede, int c,
 | 
      
         | 914 |  |  |                          int start, int end, int step)
 | 
      
         | 915 |  |  | {
 | 
      
         | 916 |  |  |   *tmp_precede = NULL;
 | 
      
         | 917 |  |  |   *tmp_follow = NULL;
 | 
      
         | 918 |  |  |  
 | 
      
         | 919 |  |  |   if (c == start)
 | 
      
         | 920 |  |  |     {
 | 
      
         | 921 |  |  |       if (step == 1)
 | 
      
         | 922 |  |  |         *tmp_precede = must_precede;
 | 
      
         | 923 |  |  |       else                      /* step == -1.  */
 | 
      
         | 924 |  |  |         *tmp_follow = must_follow;
 | 
      
         | 925 |  |  |     }
 | 
      
         | 926 |  |  |   if (c == end - step)
 | 
      
         | 927 |  |  |     {
 | 
      
         | 928 |  |  |       if (step == 1)
 | 
      
         | 929 |  |  |         *tmp_follow = must_follow;
 | 
      
         | 930 |  |  |       else                      /* step == -1.  */
 | 
      
         | 931 |  |  |         *tmp_precede = must_precede;
 | 
      
         | 932 |  |  |     }
 | 
      
         | 933 |  |  |  
 | 
      
         | 934 |  |  | }
 | 
      
         | 935 |  |  |  
 | 
      
         | 936 |  |  | /* Return True if the branch can be moved to row ii-1 while
 | 
      
         | 937 |  |  |    normalizing the partial schedule PS to start from cycle zero and thus
 | 
      
         | 938 |  |  |    optimize the SC.  Otherwise return False.  */
 | 
      
         | 939 |  |  | static bool
 | 
      
         | 940 |  |  | optimize_sc (partial_schedule_ptr ps, ddg_ptr g)
 | 
      
         | 941 |  |  | {
 | 
      
         | 942 |  |  |   int amount = PS_MIN_CYCLE (ps);
 | 
      
         | 943 |  |  |   sbitmap sched_nodes = sbitmap_alloc (g->num_nodes);
 | 
      
         | 944 |  |  |   int start, end, step;
 | 
      
         | 945 |  |  |   int ii = ps->ii;
 | 
      
         | 946 |  |  |   bool ok = false;
 | 
      
         | 947 |  |  |   int stage_count, stage_count_curr;
 | 
      
         | 948 |  |  |  
 | 
      
         | 949 |  |  |   /* Compare the SC after normalization and SC after bringing the branch
 | 
      
         | 950 |  |  |      to row ii-1.  If they are equal just bail out.  */
 | 
      
         | 951 |  |  |   stage_count = calculate_stage_count (ps, amount);
 | 
      
         | 952 |  |  |   stage_count_curr =
 | 
      
         | 953 |  |  |     calculate_stage_count (ps, SCHED_TIME (g->closing_branch->cuid) - (ii - 1));
 | 
      
         | 954 |  |  |  
 | 
      
         | 955 |  |  |   if (stage_count == stage_count_curr)
 | 
      
         | 956 |  |  |     {
 | 
      
         | 957 |  |  |       if (dump_file)
 | 
      
         | 958 |  |  |         fprintf (dump_file, "SMS SC already optimized.\n");
 | 
      
         | 959 |  |  |  
 | 
      
         | 960 |  |  |       ok = false;
 | 
      
         | 961 |  |  |       goto clear;
 | 
      
         | 962 |  |  |     }
 | 
      
         | 963 |  |  |  
 | 
      
         | 964 |  |  |   if (dump_file)
 | 
      
         | 965 |  |  |     {
 | 
      
         | 966 |  |  |       fprintf (dump_file, "SMS Trying to optimize branch location\n");
 | 
      
         | 967 |  |  |       fprintf (dump_file, "SMS partial schedule before trial:\n");
 | 
      
         | 968 |  |  |       print_partial_schedule (ps, dump_file);
 | 
      
         | 969 |  |  |     }
 | 
      
         | 970 |  |  |  
 | 
      
         | 971 |  |  |   /* First, normalize the partial scheduling.  */
 | 
      
         | 972 |  |  |   reset_sched_times (ps, amount);
 | 
      
         | 973 |  |  |   rotate_partial_schedule (ps, amount);
 | 
      
         | 974 |  |  |   if (dump_file)
 | 
      
         | 975 |  |  |     {
 | 
      
         | 976 |  |  |       fprintf (dump_file,
 | 
      
         | 977 |  |  |                "SMS partial schedule after normalization (ii, %d, SC %d):\n",
 | 
      
         | 978 |  |  |                ii, stage_count);
 | 
      
         | 979 |  |  |       print_partial_schedule (ps, dump_file);
 | 
      
         | 980 |  |  |     }
 | 
      
         | 981 |  |  |  
 | 
      
         | 982 |  |  |   if (SMODULO (SCHED_TIME (g->closing_branch->cuid), ii) == ii - 1)
 | 
      
         | 983 |  |  |     {
 | 
      
         | 984 |  |  |       ok = true;
 | 
      
         | 985 |  |  |       goto clear;
 | 
      
         | 986 |  |  |     }
 | 
      
         | 987 |  |  |  
 | 
      
         | 988 |  |  |   sbitmap_ones (sched_nodes);
 | 
      
         | 989 |  |  |  
 | 
      
         | 990 |  |  |   /* Calculate the new placement of the branch.  It should be in row
 | 
      
         | 991 |  |  |      ii-1 and fall into it's scheduling window.  */
 | 
      
         | 992 |  |  |   if (get_sched_window (ps, g->closing_branch, sched_nodes, ii, &start,
 | 
      
         | 993 |  |  |                         &step, &end) == 0)
 | 
      
         | 994 |  |  |     {
 | 
      
         | 995 |  |  |       bool success;
 | 
      
         | 996 |  |  |       ps_insn_ptr next_ps_i;
 | 
      
         | 997 |  |  |       int branch_cycle = SCHED_TIME (g->closing_branch->cuid);
 | 
      
         | 998 |  |  |       int row = SMODULO (branch_cycle, ps->ii);
 | 
      
         | 999 |  |  |       int num_splits = 0;
 | 
      
         | 1000 |  |  |       sbitmap must_precede, must_follow, tmp_precede, tmp_follow;
 | 
      
         | 1001 |  |  |       int c;
 | 
      
         | 1002 |  |  |  
 | 
      
         | 1003 |  |  |       if (dump_file)
 | 
      
         | 1004 |  |  |         fprintf (dump_file, "\nTrying to schedule node %d "
 | 
      
         | 1005 |  |  |                  "INSN = %d  in (%d .. %d) step %d\n",
 | 
      
         | 1006 |  |  |                  g->closing_branch->cuid,
 | 
      
         | 1007 |  |  |                  (INSN_UID (g->closing_branch->insn)), start, end, step);
 | 
      
         | 1008 |  |  |  
 | 
      
         | 1009 |  |  |       gcc_assert ((step > 0 && start < end) || (step < 0 && start > end));
 | 
      
         | 1010 |  |  |       if (step == 1)
 | 
      
         | 1011 |  |  |         {
 | 
      
         | 1012 |  |  |           c = start + ii - SMODULO (start, ii) - 1;
 | 
      
         | 1013 |  |  |           gcc_assert (c >= start);
 | 
      
         | 1014 |  |  |           if (c >= end)
 | 
      
         | 1015 |  |  |             {
 | 
      
         | 1016 |  |  |               ok = false;
 | 
      
         | 1017 |  |  |               if (dump_file)
 | 
      
         | 1018 |  |  |                 fprintf (dump_file,
 | 
      
         | 1019 |  |  |                          "SMS failed to schedule branch at cycle: %d\n", c);
 | 
      
         | 1020 |  |  |               goto clear;
 | 
      
         | 1021 |  |  |             }
 | 
      
         | 1022 |  |  |         }
 | 
      
         | 1023 |  |  |       else
 | 
      
         | 1024 |  |  |         {
 | 
      
         | 1025 |  |  |           c = start - SMODULO (start, ii) - 1;
 | 
      
         | 1026 |  |  |           gcc_assert (c <= start);
 | 
      
         | 1027 |  |  |  
 | 
      
         | 1028 |  |  |           if (c <= end)
 | 
      
         | 1029 |  |  |             {
 | 
      
         | 1030 |  |  |               if (dump_file)
 | 
      
         | 1031 |  |  |                 fprintf (dump_file,
 | 
      
         | 1032 |  |  |                          "SMS failed to schedule branch at cycle: %d\n", c);
 | 
      
         | 1033 |  |  |               ok = false;
 | 
      
         | 1034 |  |  |               goto clear;
 | 
      
         | 1035 |  |  |             }
 | 
      
         | 1036 |  |  |         }
 | 
      
         | 1037 |  |  |  
 | 
      
         | 1038 |  |  |       must_precede = sbitmap_alloc (g->num_nodes);
 | 
      
         | 1039 |  |  |       must_follow = sbitmap_alloc (g->num_nodes);
 | 
      
         | 1040 |  |  |  
 | 
      
         | 1041 |  |  |       /* Try to schedule the branch is it's new cycle.  */
 | 
      
         | 1042 |  |  |       calculate_must_precede_follow (g->closing_branch, start, end,
 | 
      
         | 1043 |  |  |                                      step, ii, sched_nodes,
 | 
      
         | 1044 |  |  |                                      must_precede, must_follow);
 | 
      
         | 1045 |  |  |  
 | 
      
         | 1046 |  |  |       set_must_precede_follow (&tmp_follow, must_follow, &tmp_precede,
 | 
      
         | 1047 |  |  |                                must_precede, c, start, end, step);
 | 
      
         | 1048 |  |  |  
 | 
      
         | 1049 |  |  |       /* Find the element in the partial schedule related to the closing
 | 
      
         | 1050 |  |  |          branch so we can remove it from it's current cycle.  */
 | 
      
         | 1051 |  |  |       for (next_ps_i = ps->rows[row];
 | 
      
         | 1052 |  |  |            next_ps_i; next_ps_i = next_ps_i->next_in_row)
 | 
      
         | 1053 |  |  |         if (next_ps_i->id == g->closing_branch->cuid)
 | 
      
         | 1054 |  |  |           break;
 | 
      
         | 1055 |  |  |  
 | 
      
         | 1056 |  |  |       remove_node_from_ps (ps, next_ps_i);
 | 
      
         | 1057 |  |  |       success =
 | 
      
         | 1058 |  |  |         try_scheduling_node_in_cycle (ps, g->closing_branch->cuid, c,
 | 
      
         | 1059 |  |  |                                       sched_nodes, &num_splits,
 | 
      
         | 1060 |  |  |                                       tmp_precede, tmp_follow);
 | 
      
         | 1061 |  |  |       gcc_assert (num_splits == 0);
 | 
      
         | 1062 |  |  |       if (!success)
 | 
      
         | 1063 |  |  |         {
 | 
      
         | 1064 |  |  |           if (dump_file)
 | 
      
         | 1065 |  |  |             fprintf (dump_file,
 | 
      
         | 1066 |  |  |                      "SMS failed to schedule branch at cycle: %d, "
 | 
      
         | 1067 |  |  |                      "bringing it back to cycle %d\n", c, branch_cycle);
 | 
      
         | 1068 |  |  |  
 | 
      
         | 1069 |  |  |           /* The branch was failed to be placed in row ii - 1.
 | 
      
         | 1070 |  |  |              Put it back in it's original place in the partial
 | 
      
         | 1071 |  |  |              schedualing.  */
 | 
      
         | 1072 |  |  |           set_must_precede_follow (&tmp_follow, must_follow, &tmp_precede,
 | 
      
         | 1073 |  |  |                                    must_precede, branch_cycle, start, end,
 | 
      
         | 1074 |  |  |                                    step);
 | 
      
         | 1075 |  |  |           success =
 | 
      
         | 1076 |  |  |             try_scheduling_node_in_cycle (ps, g->closing_branch->cuid,
 | 
      
         | 1077 |  |  |                                           branch_cycle, sched_nodes,
 | 
      
         | 1078 |  |  |                                           &num_splits, tmp_precede,
 | 
      
         | 1079 |  |  |                                           tmp_follow);
 | 
      
         | 1080 |  |  |           gcc_assert (success && (num_splits == 0));
 | 
      
         | 1081 |  |  |           ok = false;
 | 
      
         | 1082 |  |  |         }
 | 
      
         | 1083 |  |  |       else
 | 
      
         | 1084 |  |  |         {
 | 
      
         | 1085 |  |  |           /* The branch is placed in row ii - 1.  */
 | 
      
         | 1086 |  |  |           if (dump_file)
 | 
      
         | 1087 |  |  |             fprintf (dump_file,
 | 
      
         | 1088 |  |  |                      "SMS success in moving branch to cycle %d\n", c);
 | 
      
         | 1089 |  |  |  
 | 
      
         | 1090 |  |  |           update_node_sched_params (g->closing_branch->cuid, ii, c,
 | 
      
         | 1091 |  |  |                                     PS_MIN_CYCLE (ps));
 | 
      
         | 1092 |  |  |           ok = true;
 | 
      
         | 1093 |  |  |         }
 | 
      
         | 1094 |  |  |  
 | 
      
         | 1095 |  |  |       free (must_precede);
 | 
      
         | 1096 |  |  |       free (must_follow);
 | 
      
         | 1097 |  |  |     }
 | 
      
         | 1098 |  |  |  
 | 
      
         | 1099 |  |  | clear:
 | 
      
         | 1100 |  |  |   free (sched_nodes);
 | 
      
         | 1101 |  |  |   return ok;
 | 
      
         | 1102 |  |  | }
 | 
      
         | 1103 |  |  |  
 | 
      
         | 1104 |  |  | static void
 | 
      
         | 1105 |  |  | duplicate_insns_of_cycles (partial_schedule_ptr ps, int from_stage,
 | 
      
         | 1106 |  |  |                            int to_stage, rtx count_reg)
 | 
      
         | 1107 |  |  | {
 | 
      
         | 1108 |  |  |   int row;
 | 
      
         | 1109 |  |  |   ps_insn_ptr ps_ij;
 | 
      
         | 1110 |  |  |  
 | 
      
         | 1111 |  |  |   for (row = 0; row < ps->ii; row++)
 | 
      
         | 1112 |  |  |     for (ps_ij = ps->rows[row]; ps_ij; ps_ij = ps_ij->next_in_row)
 | 
      
         | 1113 |  |  |       {
 | 
      
         | 1114 |  |  |         int u = ps_ij->id;
 | 
      
         | 1115 |  |  |         int first_u, last_u;
 | 
      
         | 1116 |  |  |         rtx u_insn;
 | 
      
         | 1117 |  |  |  
 | 
      
         | 1118 |  |  |         /* Do not duplicate any insn which refers to count_reg as it
 | 
      
         | 1119 |  |  |            belongs to the control part.
 | 
      
         | 1120 |  |  |            The closing branch is scheduled as well and thus should
 | 
      
         | 1121 |  |  |            be ignored.
 | 
      
         | 1122 |  |  |            TODO: This should be done by analyzing the control part of
 | 
      
         | 1123 |  |  |            the loop.  */
 | 
      
         | 1124 |  |  |         u_insn = ps_rtl_insn (ps, u);
 | 
      
         | 1125 |  |  |         if (reg_mentioned_p (count_reg, u_insn)
 | 
      
         | 1126 |  |  |             || JUMP_P (u_insn))
 | 
      
         | 1127 |  |  |           continue;
 | 
      
         | 1128 |  |  |  
 | 
      
         | 1129 |  |  |         first_u = SCHED_STAGE (u);
 | 
      
         | 1130 |  |  |         last_u = first_u + ps_num_consecutive_stages (ps, u) - 1;
 | 
      
         | 1131 |  |  |         if (from_stage <= last_u && to_stage >= first_u)
 | 
      
         | 1132 |  |  |           {
 | 
      
         | 1133 |  |  |             if (u < ps->g->num_nodes)
 | 
      
         | 1134 |  |  |               duplicate_insn_chain (ps_first_note (ps, u), u_insn);
 | 
      
         | 1135 |  |  |             else
 | 
      
         | 1136 |  |  |               emit_insn (copy_rtx (PATTERN (u_insn)));
 | 
      
         | 1137 |  |  |           }
 | 
      
         | 1138 |  |  |       }
 | 
      
         | 1139 |  |  | }
 | 
      
         | 1140 |  |  |  
 | 
      
         | 1141 |  |  |  
 | 
      
         | 1142 |  |  | /* Generate the instructions (including reg_moves) for prolog & epilog.  */
 | 
      
         | 1143 |  |  | static void
 | 
      
         | 1144 |  |  | generate_prolog_epilog (partial_schedule_ptr ps, struct loop *loop,
 | 
      
         | 1145 |  |  |                         rtx count_reg, rtx count_init)
 | 
      
         | 1146 |  |  | {
 | 
      
         | 1147 |  |  |   int i;
 | 
      
         | 1148 |  |  |   int last_stage = PS_STAGE_COUNT (ps) - 1;
 | 
      
         | 1149 |  |  |   edge e;
 | 
      
         | 1150 |  |  |  
 | 
      
         | 1151 |  |  |   /* Generate the prolog, inserting its insns on the loop-entry edge.  */
 | 
      
         | 1152 |  |  |   start_sequence ();
 | 
      
         | 1153 |  |  |  
 | 
      
         | 1154 |  |  |   if (!count_init)
 | 
      
         | 1155 |  |  |     {
 | 
      
         | 1156 |  |  |       /* Generate instructions at the beginning of the prolog to
 | 
      
         | 1157 |  |  |          adjust the loop count by STAGE_COUNT.  If loop count is constant
 | 
      
         | 1158 |  |  |          (count_init), this constant is adjusted by STAGE_COUNT in
 | 
      
         | 1159 |  |  |          generate_prolog_epilog function.  */
 | 
      
         | 1160 |  |  |       rtx sub_reg = NULL_RTX;
 | 
      
         | 1161 |  |  |  
 | 
      
         | 1162 |  |  |       sub_reg = expand_simple_binop (GET_MODE (count_reg), MINUS,
 | 
      
         | 1163 |  |  |                                      count_reg, GEN_INT (last_stage),
 | 
      
         | 1164 |  |  |                                      count_reg, 1, OPTAB_DIRECT);
 | 
      
         | 1165 |  |  |       gcc_assert (REG_P (sub_reg));
 | 
      
         | 1166 |  |  |       if (REGNO (sub_reg) != REGNO (count_reg))
 | 
      
         | 1167 |  |  |         emit_move_insn (count_reg, sub_reg);
 | 
      
         | 1168 |  |  |     }
 | 
      
         | 1169 |  |  |  
 | 
      
         | 1170 |  |  |   for (i = 0; i < last_stage; i++)
 | 
      
         | 1171 |  |  |     duplicate_insns_of_cycles (ps, 0, i, count_reg);
 | 
      
         | 1172 |  |  |  
 | 
      
         | 1173 |  |  |   /* Put the prolog on the entry edge.  */
 | 
      
         | 1174 |  |  |   e = loop_preheader_edge (loop);
 | 
      
         | 1175 |  |  |   split_edge_and_insert (e, get_insns ());
 | 
      
         | 1176 |  |  |   if (!flag_resched_modulo_sched)
 | 
      
         | 1177 |  |  |     e->dest->flags |= BB_DISABLE_SCHEDULE;
 | 
      
         | 1178 |  |  |  
 | 
      
         | 1179 |  |  |   end_sequence ();
 | 
      
         | 1180 |  |  |  
 | 
      
         | 1181 |  |  |   /* Generate the epilog, inserting its insns on the loop-exit edge.  */
 | 
      
         | 1182 |  |  |   start_sequence ();
 | 
      
         | 1183 |  |  |  
 | 
      
         | 1184 |  |  |   for (i = 0; i < last_stage; i++)
 | 
      
         | 1185 |  |  |     duplicate_insns_of_cycles (ps, i + 1, last_stage, count_reg);
 | 
      
         | 1186 |  |  |  
 | 
      
         | 1187 |  |  |   /* Put the epilogue on the exit edge.  */
 | 
      
         | 1188 |  |  |   gcc_assert (single_exit (loop));
 | 
      
         | 1189 |  |  |   e = single_exit (loop);
 | 
      
         | 1190 |  |  |   split_edge_and_insert (e, get_insns ());
 | 
      
         | 1191 |  |  |   if (!flag_resched_modulo_sched)
 | 
      
         | 1192 |  |  |     e->dest->flags |= BB_DISABLE_SCHEDULE;
 | 
      
         | 1193 |  |  |  
 | 
      
         | 1194 |  |  |   end_sequence ();
 | 
      
         | 1195 |  |  | }
 | 
      
         | 1196 |  |  |  
 | 
      
         | 1197 |  |  | /* Mark LOOP as software pipelined so the later
 | 
      
         | 1198 |  |  |    scheduling passes don't touch it.  */
 | 
      
         | 1199 |  |  | static void
 | 
      
         | 1200 |  |  | mark_loop_unsched (struct loop *loop)
 | 
      
         | 1201 |  |  | {
 | 
      
         | 1202 |  |  |   unsigned i;
 | 
      
         | 1203 |  |  |   basic_block *bbs = get_loop_body (loop);
 | 
      
         | 1204 |  |  |  
 | 
      
         | 1205 |  |  |   for (i = 0; i < loop->num_nodes; i++)
 | 
      
         | 1206 |  |  |     bbs[i]->flags |= BB_DISABLE_SCHEDULE;
 | 
      
         | 1207 |  |  |  
 | 
      
         | 1208 |  |  |   free (bbs);
 | 
      
         | 1209 |  |  | }
 | 
      
         | 1210 |  |  |  
 | 
      
         | 1211 |  |  | /* Return true if all the BBs of the loop are empty except the
 | 
      
         | 1212 |  |  |    loop header.  */
 | 
      
         | 1213 |  |  | static bool
 | 
      
         | 1214 |  |  | loop_single_full_bb_p (struct loop *loop)
 | 
      
         | 1215 |  |  | {
 | 
      
         | 1216 |  |  |   unsigned i;
 | 
      
         | 1217 |  |  |   basic_block *bbs = get_loop_body (loop);
 | 
      
         | 1218 |  |  |  
 | 
      
         | 1219 |  |  |   for (i = 0; i < loop->num_nodes ; i++)
 | 
      
         | 1220 |  |  |     {
 | 
      
         | 1221 |  |  |       rtx head, tail;
 | 
      
         | 1222 |  |  |       bool empty_bb = true;
 | 
      
         | 1223 |  |  |  
 | 
      
         | 1224 |  |  |       if (bbs[i] == loop->header)
 | 
      
         | 1225 |  |  |         continue;
 | 
      
         | 1226 |  |  |  
 | 
      
         | 1227 |  |  |       /* Make sure that basic blocks other than the header
 | 
      
         | 1228 |  |  |          have only notes labels or jumps.  */
 | 
      
         | 1229 |  |  |       get_ebb_head_tail (bbs[i], bbs[i], &head, &tail);
 | 
      
         | 1230 |  |  |       for (; head != NEXT_INSN (tail); head = NEXT_INSN (head))
 | 
      
         | 1231 |  |  |         {
 | 
      
         | 1232 |  |  |           if (NOTE_P (head) || LABEL_P (head)
 | 
      
         | 1233 |  |  |               || (INSN_P (head) && (DEBUG_INSN_P (head) || JUMP_P (head))))
 | 
      
         | 1234 |  |  |             continue;
 | 
      
         | 1235 |  |  |           empty_bb = false;
 | 
      
         | 1236 |  |  |           break;
 | 
      
         | 1237 |  |  |         }
 | 
      
         | 1238 |  |  |  
 | 
      
         | 1239 |  |  |       if (! empty_bb)
 | 
      
         | 1240 |  |  |         {
 | 
      
         | 1241 |  |  |           free (bbs);
 | 
      
         | 1242 |  |  |           return false;
 | 
      
         | 1243 |  |  |         }
 | 
      
         | 1244 |  |  |     }
 | 
      
         | 1245 |  |  |   free (bbs);
 | 
      
         | 1246 |  |  |   return true;
 | 
      
         | 1247 |  |  | }
 | 
      
         | 1248 |  |  |  
 | 
      
         | 1249 |  |  | /* Dump file:line from INSN's location info to dump_file.  */
 | 
      
         | 1250 |  |  |  
 | 
      
         | 1251 |  |  | static void
 | 
      
         | 1252 |  |  | dump_insn_locator (rtx insn)
 | 
      
         | 1253 |  |  | {
 | 
      
         | 1254 |  |  |   if (dump_file && INSN_LOCATOR (insn))
 | 
      
         | 1255 |  |  |     {
 | 
      
         | 1256 |  |  |       const char *file = insn_file (insn);
 | 
      
         | 1257 |  |  |       if (file)
 | 
      
         | 1258 |  |  |         fprintf (dump_file, " %s:%i", file, insn_line (insn));
 | 
      
         | 1259 |  |  |     }
 | 
      
         | 1260 |  |  | }
 | 
      
         | 1261 |  |  |  
 | 
      
         | 1262 |  |  | /* A simple loop from SMS point of view; it is a loop that is composed of
 | 
      
         | 1263 |  |  |    either a single basic block or two BBs - a header and a latch.  */
 | 
      
         | 1264 |  |  | #define SIMPLE_SMS_LOOP_P(loop) ((loop->num_nodes < 3 )                     \
 | 
      
         | 1265 |  |  |                                   && (EDGE_COUNT (loop->latch->preds) == 1) \
 | 
      
         | 1266 |  |  |                                   && (EDGE_COUNT (loop->latch->succs) == 1))
 | 
      
         | 1267 |  |  |  
 | 
      
         | 1268 |  |  | /* Return true if the loop is in its canonical form and false if not.
 | 
      
         | 1269 |  |  |    i.e. SIMPLE_SMS_LOOP_P and have one preheader block, and single exit.  */
 | 
      
         | 1270 |  |  | static bool
 | 
      
         | 1271 |  |  | loop_canon_p (struct loop *loop)
 | 
      
         | 1272 |  |  | {
 | 
      
         | 1273 |  |  |  
 | 
      
         | 1274 |  |  |   if (loop->inner || !loop_outer (loop))
 | 
      
         | 1275 |  |  |   {
 | 
      
         | 1276 |  |  |     if (dump_file)
 | 
      
         | 1277 |  |  |       fprintf (dump_file, "SMS loop inner or !loop_outer\n");
 | 
      
         | 1278 |  |  |     return false;
 | 
      
         | 1279 |  |  |   }
 | 
      
         | 1280 |  |  |  
 | 
      
         | 1281 |  |  |   if (!single_exit (loop))
 | 
      
         | 1282 |  |  |     {
 | 
      
         | 1283 |  |  |       if (dump_file)
 | 
      
         | 1284 |  |  |         {
 | 
      
         | 1285 |  |  |           rtx insn = BB_END (loop->header);
 | 
      
         | 1286 |  |  |  
 | 
      
         | 1287 |  |  |           fprintf (dump_file, "SMS loop many exits");
 | 
      
         | 1288 |  |  |           dump_insn_locator (insn);
 | 
      
         | 1289 |  |  |           fprintf (dump_file, "\n");
 | 
      
         | 1290 |  |  |         }
 | 
      
         | 1291 |  |  |       return false;
 | 
      
         | 1292 |  |  |     }
 | 
      
         | 1293 |  |  |  
 | 
      
         | 1294 |  |  |   if (! SIMPLE_SMS_LOOP_P (loop) && ! loop_single_full_bb_p (loop))
 | 
      
         | 1295 |  |  |     {
 | 
      
         | 1296 |  |  |       if (dump_file)
 | 
      
         | 1297 |  |  |         {
 | 
      
         | 1298 |  |  |           rtx insn = BB_END (loop->header);
 | 
      
         | 1299 |  |  |  
 | 
      
         | 1300 |  |  |           fprintf (dump_file, "SMS loop many BBs.");
 | 
      
         | 1301 |  |  |           dump_insn_locator (insn);
 | 
      
         | 1302 |  |  |           fprintf (dump_file, "\n");
 | 
      
         | 1303 |  |  |         }
 | 
      
         | 1304 |  |  |       return false;
 | 
      
         | 1305 |  |  |     }
 | 
      
         | 1306 |  |  |  
 | 
      
         | 1307 |  |  |     return true;
 | 
      
         | 1308 |  |  | }
 | 
      
         | 1309 |  |  |  
 | 
      
         | 1310 |  |  | /* If there are more than one entry for the loop,
 | 
      
         | 1311 |  |  |    make it one by splitting the first entry edge and
 | 
      
         | 1312 |  |  |    redirecting the others to the new BB.  */
 | 
      
         | 1313 |  |  | static void
 | 
      
         | 1314 |  |  | canon_loop (struct loop *loop)
 | 
      
         | 1315 |  |  | {
 | 
      
         | 1316 |  |  |   edge e;
 | 
      
         | 1317 |  |  |   edge_iterator i;
 | 
      
         | 1318 |  |  |  
 | 
      
         | 1319 |  |  |   /* Avoid annoying special cases of edges going to exit
 | 
      
         | 1320 |  |  |      block.  */
 | 
      
         | 1321 |  |  |   FOR_EACH_EDGE (e, i, EXIT_BLOCK_PTR->preds)
 | 
      
         | 1322 |  |  |     if ((e->flags & EDGE_FALLTHRU) && (EDGE_COUNT (e->src->succs) > 1))
 | 
      
         | 1323 |  |  |       split_edge (e);
 | 
      
         | 1324 |  |  |  
 | 
      
         | 1325 |  |  |   if (loop->latch == loop->header
 | 
      
         | 1326 |  |  |       || EDGE_COUNT (loop->latch->succs) > 1)
 | 
      
         | 1327 |  |  |     {
 | 
      
         | 1328 |  |  |       FOR_EACH_EDGE (e, i, loop->header->preds)
 | 
      
         | 1329 |  |  |         if (e->src == loop->latch)
 | 
      
         | 1330 |  |  |           break;
 | 
      
         | 1331 |  |  |       split_edge (e);
 | 
      
         | 1332 |  |  |     }
 | 
      
         | 1333 |  |  | }
 | 
      
         | 1334 |  |  |  
 | 
      
         | 1335 |  |  | /* Setup infos.  */
 | 
      
         | 1336 |  |  | static void
 | 
      
         | 1337 |  |  | setup_sched_infos (void)
 | 
      
         | 1338 |  |  | {
 | 
      
         | 1339 |  |  |   memcpy (&sms_common_sched_info, &haifa_common_sched_info,
 | 
      
         | 1340 |  |  |           sizeof (sms_common_sched_info));
 | 
      
         | 1341 |  |  |   sms_common_sched_info.sched_pass_id = SCHED_SMS_PASS;
 | 
      
         | 1342 |  |  |   common_sched_info = &sms_common_sched_info;
 | 
      
         | 1343 |  |  |  
 | 
      
         | 1344 |  |  |   sched_deps_info = &sms_sched_deps_info;
 | 
      
         | 1345 |  |  |   current_sched_info = &sms_sched_info;
 | 
      
         | 1346 |  |  | }
 | 
      
         | 1347 |  |  |  
 | 
      
         | 1348 |  |  | /* Probability in % that the sms-ed loop rolls enough so that optimized
 | 
      
         | 1349 |  |  |    version may be entered.  Just a guess.  */
 | 
      
         | 1350 |  |  | #define PROB_SMS_ENOUGH_ITERATIONS 80
 | 
      
         | 1351 |  |  |  
 | 
      
         | 1352 |  |  | /* Used to calculate the upper bound of ii.  */
 | 
      
         | 1353 |  |  | #define MAXII_FACTOR 2
 | 
      
         | 1354 |  |  |  
 | 
      
         | 1355 |  |  | /* Main entry point, perform SMS scheduling on the loops of the function
 | 
      
         | 1356 |  |  |    that consist of single basic blocks.  */
 | 
      
         | 1357 |  |  | static void
 | 
      
         | 1358 |  |  | sms_schedule (void)
 | 
      
         | 1359 |  |  | {
 | 
      
         | 1360 |  |  |   rtx insn;
 | 
      
         | 1361 |  |  |   ddg_ptr *g_arr, g;
 | 
      
         | 1362 |  |  |   int * node_order;
 | 
      
         | 1363 |  |  |   int maxii, max_asap;
 | 
      
         | 1364 |  |  |   loop_iterator li;
 | 
      
         | 1365 |  |  |   partial_schedule_ptr ps;
 | 
      
         | 1366 |  |  |   basic_block bb = NULL;
 | 
      
         | 1367 |  |  |   struct loop *loop;
 | 
      
         | 1368 |  |  |   basic_block condition_bb = NULL;
 | 
      
         | 1369 |  |  |   edge latch_edge;
 | 
      
         | 1370 |  |  |   gcov_type trip_count = 0;
 | 
      
         | 1371 |  |  |  
 | 
      
         | 1372 |  |  |   loop_optimizer_init (LOOPS_HAVE_PREHEADERS
 | 
      
         | 1373 |  |  |                        | LOOPS_HAVE_RECORDED_EXITS);
 | 
      
         | 1374 |  |  |   if (number_of_loops () <= 1)
 | 
      
         | 1375 |  |  |     {
 | 
      
         | 1376 |  |  |       loop_optimizer_finalize ();
 | 
      
         | 1377 |  |  |       return;  /* There are no loops to schedule.  */
 | 
      
         | 1378 |  |  |     }
 | 
      
         | 1379 |  |  |  
 | 
      
         | 1380 |  |  |   /* Initialize issue_rate.  */
 | 
      
         | 1381 |  |  |   if (targetm.sched.issue_rate)
 | 
      
         | 1382 |  |  |     {
 | 
      
         | 1383 |  |  |       int temp = reload_completed;
 | 
      
         | 1384 |  |  |  
 | 
      
         | 1385 |  |  |       reload_completed = 1;
 | 
      
         | 1386 |  |  |       issue_rate = targetm.sched.issue_rate ();
 | 
      
         | 1387 |  |  |       reload_completed = temp;
 | 
      
         | 1388 |  |  |     }
 | 
      
         | 1389 |  |  |   else
 | 
      
         | 1390 |  |  |     issue_rate = 1;
 | 
      
         | 1391 |  |  |  
 | 
      
         | 1392 |  |  |   /* Initialize the scheduler.  */
 | 
      
         | 1393 |  |  |   setup_sched_infos ();
 | 
      
         | 1394 |  |  |   haifa_sched_init ();
 | 
      
         | 1395 |  |  |  
 | 
      
         | 1396 |  |  |   /* Allocate memory to hold the DDG array one entry for each loop.
 | 
      
         | 1397 |  |  |      We use loop->num as index into this array.  */
 | 
      
         | 1398 |  |  |   g_arr = XCNEWVEC (ddg_ptr, number_of_loops ());
 | 
      
         | 1399 |  |  |  
 | 
      
         | 1400 |  |  |   if (dump_file)
 | 
      
         | 1401 |  |  |   {
 | 
      
         | 1402 |  |  |     fprintf (dump_file, "\n\nSMS analysis phase\n");
 | 
      
         | 1403 |  |  |     fprintf (dump_file, "===================\n\n");
 | 
      
         | 1404 |  |  |   }
 | 
      
         | 1405 |  |  |  
 | 
      
         | 1406 |  |  |   /* Build DDGs for all the relevant loops and hold them in G_ARR
 | 
      
         | 1407 |  |  |      indexed by the loop index.  */
 | 
      
         | 1408 |  |  |   FOR_EACH_LOOP (li, loop, 0)
 | 
      
         | 1409 |  |  |     {
 | 
      
         | 1410 |  |  |       rtx head, tail;
 | 
      
         | 1411 |  |  |       rtx count_reg;
 | 
      
         | 1412 |  |  |  
 | 
      
         | 1413 |  |  |       /* For debugging.  */
 | 
      
         | 1414 |  |  |       if (dbg_cnt (sms_sched_loop) == false)
 | 
      
         | 1415 |  |  |         {
 | 
      
         | 1416 |  |  |           if (dump_file)
 | 
      
         | 1417 |  |  |             fprintf (dump_file, "SMS reached max limit... \n");
 | 
      
         | 1418 |  |  |  
 | 
      
         | 1419 |  |  |           break;
 | 
      
         | 1420 |  |  |         }
 | 
      
         | 1421 |  |  |  
 | 
      
         | 1422 |  |  |       if (dump_file)
 | 
      
         | 1423 |  |  |         {
 | 
      
         | 1424 |  |  |           rtx insn = BB_END (loop->header);
 | 
      
         | 1425 |  |  |  
 | 
      
         | 1426 |  |  |           fprintf (dump_file, "SMS loop num: %d", loop->num);
 | 
      
         | 1427 |  |  |           dump_insn_locator (insn);
 | 
      
         | 1428 |  |  |           fprintf (dump_file, "\n");
 | 
      
         | 1429 |  |  |         }
 | 
      
         | 1430 |  |  |  
 | 
      
         | 1431 |  |  |       if (! loop_canon_p (loop))
 | 
      
         | 1432 |  |  |         continue;
 | 
      
         | 1433 |  |  |  
 | 
      
         | 1434 |  |  |       if (! loop_single_full_bb_p (loop))
 | 
      
         | 1435 |  |  |       {
 | 
      
         | 1436 |  |  |         if (dump_file)
 | 
      
         | 1437 |  |  |           fprintf (dump_file, "SMS not loop_single_full_bb_p\n");
 | 
      
         | 1438 |  |  |         continue;
 | 
      
         | 1439 |  |  |       }
 | 
      
         | 1440 |  |  |  
 | 
      
         | 1441 |  |  |       bb = loop->header;
 | 
      
         | 1442 |  |  |  
 | 
      
         | 1443 |  |  |       get_ebb_head_tail (bb, bb, &head, &tail);
 | 
      
         | 1444 |  |  |       latch_edge = loop_latch_edge (loop);
 | 
      
         | 1445 |  |  |       gcc_assert (single_exit (loop));
 | 
      
         | 1446 |  |  |       if (single_exit (loop)->count)
 | 
      
         | 1447 |  |  |         trip_count = latch_edge->count / single_exit (loop)->count;
 | 
      
         | 1448 |  |  |  
 | 
      
         | 1449 |  |  |       /* Perform SMS only on loops that their average count is above threshold.  */
 | 
      
         | 1450 |  |  |  
 | 
      
         | 1451 |  |  |       if ( latch_edge->count
 | 
      
         | 1452 |  |  |           && (latch_edge->count < single_exit (loop)->count * SMS_LOOP_AVERAGE_COUNT_THRESHOLD))
 | 
      
         | 1453 |  |  |         {
 | 
      
         | 1454 |  |  |           if (dump_file)
 | 
      
         | 1455 |  |  |             {
 | 
      
         | 1456 |  |  |               dump_insn_locator (tail);
 | 
      
         | 1457 |  |  |               fprintf (dump_file, "\nSMS single-bb-loop\n");
 | 
      
         | 1458 |  |  |               if (profile_info && flag_branch_probabilities)
 | 
      
         | 1459 |  |  |                 {
 | 
      
         | 1460 |  |  |                   fprintf (dump_file, "SMS loop-count ");
 | 
      
         | 1461 |  |  |                   fprintf (dump_file, HOST_WIDEST_INT_PRINT_DEC,
 | 
      
         | 1462 |  |  |                            (HOST_WIDEST_INT) bb->count);
 | 
      
         | 1463 |  |  |                   fprintf (dump_file, "\n");
 | 
      
         | 1464 |  |  |                   fprintf (dump_file, "SMS trip-count ");
 | 
      
         | 1465 |  |  |                   fprintf (dump_file, HOST_WIDEST_INT_PRINT_DEC,
 | 
      
         | 1466 |  |  |                            (HOST_WIDEST_INT) trip_count);
 | 
      
         | 1467 |  |  |                   fprintf (dump_file, "\n");
 | 
      
         | 1468 |  |  |                   fprintf (dump_file, "SMS profile-sum-max ");
 | 
      
         | 1469 |  |  |                   fprintf (dump_file, HOST_WIDEST_INT_PRINT_DEC,
 | 
      
         | 1470 |  |  |                            (HOST_WIDEST_INT) profile_info->sum_max);
 | 
      
         | 1471 |  |  |                   fprintf (dump_file, "\n");
 | 
      
         | 1472 |  |  |                 }
 | 
      
         | 1473 |  |  |             }
 | 
      
         | 1474 |  |  |           continue;
 | 
      
         | 1475 |  |  |         }
 | 
      
         | 1476 |  |  |  
 | 
      
         | 1477 |  |  |       /* Make sure this is a doloop.  */
 | 
      
         | 1478 |  |  |       if ( !(count_reg = doloop_register_get (head, tail)))
 | 
      
         | 1479 |  |  |       {
 | 
      
         | 1480 |  |  |         if (dump_file)
 | 
      
         | 1481 |  |  |           fprintf (dump_file, "SMS doloop_register_get failed\n");
 | 
      
         | 1482 |  |  |         continue;
 | 
      
         | 1483 |  |  |       }
 | 
      
         | 1484 |  |  |  
 | 
      
         | 1485 |  |  |       /* Don't handle BBs with calls or barriers
 | 
      
         | 1486 |  |  |          or !single_set with the exception of instructions that include
 | 
      
         | 1487 |  |  |          count_reg---these instructions are part of the control part
 | 
      
         | 1488 |  |  |          that do-loop recognizes.
 | 
      
         | 1489 |  |  |          ??? Should handle insns defining subregs.  */
 | 
      
         | 1490 |  |  |      for (insn = head; insn != NEXT_INSN (tail); insn = NEXT_INSN (insn))
 | 
      
         | 1491 |  |  |       {
 | 
      
         | 1492 |  |  |          rtx set;
 | 
      
         | 1493 |  |  |  
 | 
      
         | 1494 |  |  |         if (CALL_P (insn)
 | 
      
         | 1495 |  |  |             || BARRIER_P (insn)
 | 
      
         | 1496 |  |  |             || (NONDEBUG_INSN_P (insn) && !JUMP_P (insn)
 | 
      
         | 1497 |  |  |                 && !single_set (insn) && GET_CODE (PATTERN (insn)) != USE
 | 
      
         | 1498 |  |  |                 && !reg_mentioned_p (count_reg, insn))
 | 
      
         | 1499 |  |  |             || (INSN_P (insn) && (set = single_set (insn))
 | 
      
         | 1500 |  |  |                 && GET_CODE (SET_DEST (set)) == SUBREG))
 | 
      
         | 1501 |  |  |         break;
 | 
      
         | 1502 |  |  |       }
 | 
      
         | 1503 |  |  |  
 | 
      
         | 1504 |  |  |       if (insn != NEXT_INSN (tail))
 | 
      
         | 1505 |  |  |         {
 | 
      
         | 1506 |  |  |           if (dump_file)
 | 
      
         | 1507 |  |  |             {
 | 
      
         | 1508 |  |  |               if (CALL_P (insn))
 | 
      
         | 1509 |  |  |                 fprintf (dump_file, "SMS loop-with-call\n");
 | 
      
         | 1510 |  |  |               else if (BARRIER_P (insn))
 | 
      
         | 1511 |  |  |                 fprintf (dump_file, "SMS loop-with-barrier\n");
 | 
      
         | 1512 |  |  |               else if ((NONDEBUG_INSN_P (insn) && !JUMP_P (insn)
 | 
      
         | 1513 |  |  |                 && !single_set (insn) && GET_CODE (PATTERN (insn)) != USE))
 | 
      
         | 1514 |  |  |                 fprintf (dump_file, "SMS loop-with-not-single-set\n");
 | 
      
         | 1515 |  |  |               else
 | 
      
         | 1516 |  |  |                fprintf (dump_file, "SMS loop with subreg in lhs\n");
 | 
      
         | 1517 |  |  |               print_rtl_single (dump_file, insn);
 | 
      
         | 1518 |  |  |             }
 | 
      
         | 1519 |  |  |  
 | 
      
         | 1520 |  |  |           continue;
 | 
      
         | 1521 |  |  |         }
 | 
      
         | 1522 |  |  |  
 | 
      
         | 1523 |  |  |       /* Always schedule the closing branch with the rest of the
 | 
      
         | 1524 |  |  |          instructions. The branch is rotated to be in row ii-1 at the
 | 
      
         | 1525 |  |  |          end of the scheduling procedure to make sure it's the last
 | 
      
         | 1526 |  |  |          instruction in the iteration.  */
 | 
      
         | 1527 |  |  |       if (! (g = create_ddg (bb, 1)))
 | 
      
         | 1528 |  |  |         {
 | 
      
         | 1529 |  |  |           if (dump_file)
 | 
      
         | 1530 |  |  |             fprintf (dump_file, "SMS create_ddg failed\n");
 | 
      
         | 1531 |  |  |           continue;
 | 
      
         | 1532 |  |  |         }
 | 
      
         | 1533 |  |  |  
 | 
      
         | 1534 |  |  |       g_arr[loop->num] = g;
 | 
      
         | 1535 |  |  |       if (dump_file)
 | 
      
         | 1536 |  |  |         fprintf (dump_file, "...OK\n");
 | 
      
         | 1537 |  |  |  
 | 
      
         | 1538 |  |  |     }
 | 
      
         | 1539 |  |  |   if (dump_file)
 | 
      
         | 1540 |  |  |   {
 | 
      
         | 1541 |  |  |     fprintf (dump_file, "\nSMS transformation phase\n");
 | 
      
         | 1542 |  |  |     fprintf (dump_file, "=========================\n\n");
 | 
      
         | 1543 |  |  |   }
 | 
      
         | 1544 |  |  |  
 | 
      
         | 1545 |  |  |   /* We don't want to perform SMS on new loops - created by versioning.  */
 | 
      
         | 1546 |  |  |   FOR_EACH_LOOP (li, loop, 0)
 | 
      
         | 1547 |  |  |     {
 | 
      
         | 1548 |  |  |       rtx head, tail;
 | 
      
         | 1549 |  |  |       rtx count_reg, count_init;
 | 
      
         | 1550 |  |  |       int mii, rec_mii, stage_count, min_cycle;
 | 
      
         | 1551 |  |  |       HOST_WIDEST_INT loop_count = 0;
 | 
      
         | 1552 |  |  |       bool opt_sc_p;
 | 
      
         | 1553 |  |  |  
 | 
      
         | 1554 |  |  |       if (! (g = g_arr[loop->num]))
 | 
      
         | 1555 |  |  |         continue;
 | 
      
         | 1556 |  |  |  
 | 
      
         | 1557 |  |  |       if (dump_file)
 | 
      
         | 1558 |  |  |         {
 | 
      
         | 1559 |  |  |           rtx insn = BB_END (loop->header);
 | 
      
         | 1560 |  |  |  
 | 
      
         | 1561 |  |  |           fprintf (dump_file, "SMS loop num: %d", loop->num);
 | 
      
         | 1562 |  |  |           dump_insn_locator (insn);
 | 
      
         | 1563 |  |  |           fprintf (dump_file, "\n");
 | 
      
         | 1564 |  |  |  
 | 
      
         | 1565 |  |  |           print_ddg (dump_file, g);
 | 
      
         | 1566 |  |  |         }
 | 
      
         | 1567 |  |  |  
 | 
      
         | 1568 |  |  |       get_ebb_head_tail (loop->header, loop->header, &head, &tail);
 | 
      
         | 1569 |  |  |  
 | 
      
         | 1570 |  |  |       latch_edge = loop_latch_edge (loop);
 | 
      
         | 1571 |  |  |       gcc_assert (single_exit (loop));
 | 
      
         | 1572 |  |  |       if (single_exit (loop)->count)
 | 
      
         | 1573 |  |  |         trip_count = latch_edge->count / single_exit (loop)->count;
 | 
      
         | 1574 |  |  |  
 | 
      
         | 1575 |  |  |       if (dump_file)
 | 
      
         | 1576 |  |  |         {
 | 
      
         | 1577 |  |  |           dump_insn_locator (tail);
 | 
      
         | 1578 |  |  |           fprintf (dump_file, "\nSMS single-bb-loop\n");
 | 
      
         | 1579 |  |  |           if (profile_info && flag_branch_probabilities)
 | 
      
         | 1580 |  |  |             {
 | 
      
         | 1581 |  |  |               fprintf (dump_file, "SMS loop-count ");
 | 
      
         | 1582 |  |  |               fprintf (dump_file, HOST_WIDEST_INT_PRINT_DEC,
 | 
      
         | 1583 |  |  |                        (HOST_WIDEST_INT) bb->count);
 | 
      
         | 1584 |  |  |               fprintf (dump_file, "\n");
 | 
      
         | 1585 |  |  |               fprintf (dump_file, "SMS profile-sum-max ");
 | 
      
         | 1586 |  |  |               fprintf (dump_file, HOST_WIDEST_INT_PRINT_DEC,
 | 
      
         | 1587 |  |  |                        (HOST_WIDEST_INT) profile_info->sum_max);
 | 
      
         | 1588 |  |  |               fprintf (dump_file, "\n");
 | 
      
         | 1589 |  |  |             }
 | 
      
         | 1590 |  |  |           fprintf (dump_file, "SMS doloop\n");
 | 
      
         | 1591 |  |  |           fprintf (dump_file, "SMS built-ddg %d\n", g->num_nodes);
 | 
      
         | 1592 |  |  |           fprintf (dump_file, "SMS num-loads %d\n", g->num_loads);
 | 
      
         | 1593 |  |  |           fprintf (dump_file, "SMS num-stores %d\n", g->num_stores);
 | 
      
         | 1594 |  |  |         }
 | 
      
         | 1595 |  |  |  
 | 
      
         | 1596 |  |  |  
 | 
      
         | 1597 |  |  |       /* In case of th loop have doloop register it gets special
 | 
      
         | 1598 |  |  |          handling.  */
 | 
      
         | 1599 |  |  |       count_init = NULL_RTX;
 | 
      
         | 1600 |  |  |       if ((count_reg = doloop_register_get (head, tail)))
 | 
      
         | 1601 |  |  |         {
 | 
      
         | 1602 |  |  |           basic_block pre_header;
 | 
      
         | 1603 |  |  |  
 | 
      
         | 1604 |  |  |           pre_header = loop_preheader_edge (loop)->src;
 | 
      
         | 1605 |  |  |           count_init = const_iteration_count (count_reg, pre_header,
 | 
      
         | 1606 |  |  |                                               &loop_count);
 | 
      
         | 1607 |  |  |         }
 | 
      
         | 1608 |  |  |       gcc_assert (count_reg);
 | 
      
         | 1609 |  |  |  
 | 
      
         | 1610 |  |  |       if (dump_file && count_init)
 | 
      
         | 1611 |  |  |         {
 | 
      
         | 1612 |  |  |           fprintf (dump_file, "SMS const-doloop ");
 | 
      
         | 1613 |  |  |           fprintf (dump_file, HOST_WIDEST_INT_PRINT_DEC,
 | 
      
         | 1614 |  |  |                      loop_count);
 | 
      
         | 1615 |  |  |           fprintf (dump_file, "\n");
 | 
      
         | 1616 |  |  |         }
 | 
      
         | 1617 |  |  |  
 | 
      
         | 1618 |  |  |       node_order = XNEWVEC (int, g->num_nodes);
 | 
      
         | 1619 |  |  |  
 | 
      
         | 1620 |  |  |       mii = 1; /* Need to pass some estimate of mii.  */
 | 
      
         | 1621 |  |  |       rec_mii = sms_order_nodes (g, mii, node_order, &max_asap);
 | 
      
         | 1622 |  |  |       mii = MAX (res_MII (g), rec_mii);
 | 
      
         | 1623 |  |  |       maxii = MAX (max_asap, MAXII_FACTOR * mii);
 | 
      
         | 1624 |  |  |  
 | 
      
         | 1625 |  |  |       if (dump_file)
 | 
      
         | 1626 |  |  |         fprintf (dump_file, "SMS iis %d %d %d (rec_mii, mii, maxii)\n",
 | 
      
         | 1627 |  |  |                  rec_mii, mii, maxii);
 | 
      
         | 1628 |  |  |  
 | 
      
         | 1629 |  |  |       for (;;)
 | 
      
         | 1630 |  |  |         {
 | 
      
         | 1631 |  |  |           set_node_sched_params (g);
 | 
      
         | 1632 |  |  |  
 | 
      
         | 1633 |  |  |           stage_count = 0;
 | 
      
         | 1634 |  |  |           opt_sc_p = false;
 | 
      
         | 1635 |  |  |           ps = sms_schedule_by_order (g, mii, maxii, node_order);
 | 
      
         | 1636 |  |  |  
 | 
      
         | 1637 |  |  |           if (ps)
 | 
      
         | 1638 |  |  |             {
 | 
      
         | 1639 |  |  |               /* Try to achieve optimized SC by normalizing the partial
 | 
      
         | 1640 |  |  |                  schedule (having the cycles start from cycle zero).
 | 
      
         | 1641 |  |  |                  The branch location must be placed in row ii-1 in the
 | 
      
         | 1642 |  |  |                  final scheduling.      If failed, shift all instructions to
 | 
      
         | 1643 |  |  |                  position the branch in row ii-1.  */
 | 
      
         | 1644 |  |  |               opt_sc_p = optimize_sc (ps, g);
 | 
      
         | 1645 |  |  |               if (opt_sc_p)
 | 
      
         | 1646 |  |  |                 stage_count = calculate_stage_count (ps, 0);
 | 
      
         | 1647 |  |  |               else
 | 
      
         | 1648 |  |  |                 {
 | 
      
         | 1649 |  |  |                   /* Bring the branch to cycle ii-1.  */
 | 
      
         | 1650 |  |  |                   int amount = (SCHED_TIME (g->closing_branch->cuid)
 | 
      
         | 1651 |  |  |                                 - (ps->ii - 1));
 | 
      
         | 1652 |  |  |  
 | 
      
         | 1653 |  |  |                   if (dump_file)
 | 
      
         | 1654 |  |  |                     fprintf (dump_file, "SMS schedule branch at cycle ii-1\n");
 | 
      
         | 1655 |  |  |  
 | 
      
         | 1656 |  |  |                   stage_count = calculate_stage_count (ps, amount);
 | 
      
         | 1657 |  |  |                 }
 | 
      
         | 1658 |  |  |  
 | 
      
         | 1659 |  |  |               gcc_assert (stage_count >= 1);
 | 
      
         | 1660 |  |  |             }
 | 
      
         | 1661 |  |  |  
 | 
      
         | 1662 |  |  |           /* The default value of PARAM_SMS_MIN_SC is 2 as stage count of
 | 
      
         | 1663 |  |  |              1 means that there is no interleaving between iterations thus
 | 
      
         | 1664 |  |  |              we let the scheduling passes do the job in this case.  */
 | 
      
         | 1665 |  |  |           if (stage_count < PARAM_VALUE (PARAM_SMS_MIN_SC)
 | 
      
         | 1666 |  |  |               || (count_init && (loop_count <= stage_count))
 | 
      
         | 1667 |  |  |               || (flag_branch_probabilities && (trip_count <= stage_count)))
 | 
      
         | 1668 |  |  |             {
 | 
      
         | 1669 |  |  |               if (dump_file)
 | 
      
         | 1670 |  |  |                 {
 | 
      
         | 1671 |  |  |                   fprintf (dump_file, "SMS failed... \n");
 | 
      
         | 1672 |  |  |                   fprintf (dump_file, "SMS sched-failed (stage-count=%d,"
 | 
      
         | 1673 |  |  |                            " loop-count=", stage_count);
 | 
      
         | 1674 |  |  |                   fprintf (dump_file, HOST_WIDEST_INT_PRINT_DEC, loop_count);
 | 
      
         | 1675 |  |  |                   fprintf (dump_file, ", trip-count=");
 | 
      
         | 1676 |  |  |                   fprintf (dump_file, HOST_WIDEST_INT_PRINT_DEC, trip_count);
 | 
      
         | 1677 |  |  |                   fprintf (dump_file, ")\n");
 | 
      
         | 1678 |  |  |                 }
 | 
      
         | 1679 |  |  |               break;
 | 
      
         | 1680 |  |  |             }
 | 
      
         | 1681 |  |  |  
 | 
      
         | 1682 |  |  |           if (!opt_sc_p)
 | 
      
         | 1683 |  |  |             {
 | 
      
         | 1684 |  |  |               /* Rotate the partial schedule to have the branch in row ii-1.  */
 | 
      
         | 1685 |  |  |               int amount = SCHED_TIME (g->closing_branch->cuid) - (ps->ii - 1);
 | 
      
         | 1686 |  |  |  
 | 
      
         | 1687 |  |  |               reset_sched_times (ps, amount);
 | 
      
         | 1688 |  |  |               rotate_partial_schedule (ps, amount);
 | 
      
         | 1689 |  |  |             }
 | 
      
         | 1690 |  |  |  
 | 
      
         | 1691 |  |  |           set_columns_for_ps (ps);
 | 
      
         | 1692 |  |  |  
 | 
      
         | 1693 |  |  |           min_cycle = PS_MIN_CYCLE (ps) - SMODULO (PS_MIN_CYCLE (ps), ps->ii);
 | 
      
         | 1694 |  |  |           if (!schedule_reg_moves (ps))
 | 
      
         | 1695 |  |  |             {
 | 
      
         | 1696 |  |  |               mii = ps->ii + 1;
 | 
      
         | 1697 |  |  |               free_partial_schedule (ps);
 | 
      
         | 1698 |  |  |               continue;
 | 
      
         | 1699 |  |  |             }
 | 
      
         | 1700 |  |  |  
 | 
      
         | 1701 |  |  |           /* Moves that handle incoming values might have been added
 | 
      
         | 1702 |  |  |              to a new first stage.  Bump the stage count if so.
 | 
      
         | 1703 |  |  |  
 | 
      
         | 1704 |  |  |              ??? Perhaps we could consider rotating the schedule here
 | 
      
         | 1705 |  |  |              instead?  */
 | 
      
         | 1706 |  |  |           if (PS_MIN_CYCLE (ps) < min_cycle)
 | 
      
         | 1707 |  |  |             {
 | 
      
         | 1708 |  |  |               reset_sched_times (ps, 0);
 | 
      
         | 1709 |  |  |               stage_count++;
 | 
      
         | 1710 |  |  |             }
 | 
      
         | 1711 |  |  |  
 | 
      
         | 1712 |  |  |           /* The stage count should now be correct without rotation.  */
 | 
      
         | 1713 |  |  |           gcc_checking_assert (stage_count == calculate_stage_count (ps, 0));
 | 
      
         | 1714 |  |  |           PS_STAGE_COUNT (ps) = stage_count;
 | 
      
         | 1715 |  |  |  
 | 
      
         | 1716 |  |  |           canon_loop (loop);
 | 
      
         | 1717 |  |  |  
 | 
      
         | 1718 |  |  |           if (dump_file)
 | 
      
         | 1719 |  |  |             {
 | 
      
         | 1720 |  |  |               dump_insn_locator (tail);
 | 
      
         | 1721 |  |  |               fprintf (dump_file, " SMS succeeded %d %d (with ii, sc)\n",
 | 
      
         | 1722 |  |  |                        ps->ii, stage_count);
 | 
      
         | 1723 |  |  |               print_partial_schedule (ps, dump_file);
 | 
      
         | 1724 |  |  |             }
 | 
      
         | 1725 |  |  |  
 | 
      
         | 1726 |  |  |           /* case the BCT count is not known , Do loop-versioning */
 | 
      
         | 1727 |  |  |           if (count_reg && ! count_init)
 | 
      
         | 1728 |  |  |             {
 | 
      
         | 1729 |  |  |               rtx comp_rtx = gen_rtx_fmt_ee (GT, VOIDmode, count_reg,
 | 
      
         | 1730 |  |  |                                              GEN_INT(stage_count));
 | 
      
         | 1731 |  |  |               unsigned prob = (PROB_SMS_ENOUGH_ITERATIONS
 | 
      
         | 1732 |  |  |                                * REG_BR_PROB_BASE) / 100;
 | 
      
         | 1733 |  |  |  
 | 
      
         | 1734 |  |  |               loop_version (loop, comp_rtx, &condition_bb,
 | 
      
         | 1735 |  |  |                             prob, prob, REG_BR_PROB_BASE - prob,
 | 
      
         | 1736 |  |  |                             true);
 | 
      
         | 1737 |  |  |              }
 | 
      
         | 1738 |  |  |  
 | 
      
         | 1739 |  |  |           /* Set new iteration count of loop kernel.  */
 | 
      
         | 1740 |  |  |           if (count_reg && count_init)
 | 
      
         | 1741 |  |  |             SET_SRC (single_set (count_init)) = GEN_INT (loop_count
 | 
      
         | 1742 |  |  |                                                      - stage_count + 1);
 | 
      
         | 1743 |  |  |  
 | 
      
         | 1744 |  |  |           /* Now apply the scheduled kernel to the RTL of the loop.  */
 | 
      
         | 1745 |  |  |           permute_partial_schedule (ps, g->closing_branch->first_note);
 | 
      
         | 1746 |  |  |  
 | 
      
         | 1747 |  |  |           /* Mark this loop as software pipelined so the later
 | 
      
         | 1748 |  |  |              scheduling passes don't touch it.  */
 | 
      
         | 1749 |  |  |           if (! flag_resched_modulo_sched)
 | 
      
         | 1750 |  |  |             mark_loop_unsched (loop);
 | 
      
         | 1751 |  |  |  
 | 
      
         | 1752 |  |  |           /* The life-info is not valid any more.  */
 | 
      
         | 1753 |  |  |           df_set_bb_dirty (g->bb);
 | 
      
         | 1754 |  |  |  
 | 
      
         | 1755 |  |  |           apply_reg_moves (ps);
 | 
      
         | 1756 |  |  |           if (dump_file)
 | 
      
         | 1757 |  |  |             print_node_sched_params (dump_file, g->num_nodes, ps);
 | 
      
         | 1758 |  |  |           /* Generate prolog and epilog.  */
 | 
      
         | 1759 |  |  |           generate_prolog_epilog (ps, loop, count_reg, count_init);
 | 
      
         | 1760 |  |  |           break;
 | 
      
         | 1761 |  |  |         }
 | 
      
         | 1762 |  |  |  
 | 
      
         | 1763 |  |  |       free_partial_schedule (ps);
 | 
      
         | 1764 |  |  |       VEC_free (node_sched_params, heap, node_sched_param_vec);
 | 
      
         | 1765 |  |  |       free (node_order);
 | 
      
         | 1766 |  |  |       free_ddg (g);
 | 
      
         | 1767 |  |  |     }
 | 
      
         | 1768 |  |  |  
 | 
      
         | 1769 |  |  |   free (g_arr);
 | 
      
         | 1770 |  |  |  
 | 
      
         | 1771 |  |  |   /* Release scheduler data, needed until now because of DFA.  */
 | 
      
         | 1772 |  |  |   haifa_sched_finish ();
 | 
      
         | 1773 |  |  |   loop_optimizer_finalize ();
 | 
      
         | 1774 |  |  | }
 | 
      
         | 1775 |  |  |  
 | 
      
         | 1776 |  |  | /* The SMS scheduling algorithm itself
 | 
      
         | 1777 |  |  |    -----------------------------------
 | 
      
         | 1778 |  |  |    Input: 'O' an ordered list of insns of a loop.
 | 
      
         | 1779 |  |  |    Output: A scheduling of the loop - kernel, prolog, and epilogue.
 | 
      
         | 1780 |  |  |  
 | 
      
         | 1781 |  |  |    'Q' is the empty Set
 | 
      
         | 1782 |  |  |    'PS' is the partial schedule; it holds the currently scheduled nodes with
 | 
      
         | 1783 |  |  |         their cycle/slot.
 | 
      
         | 1784 |  |  |    'PSP' previously scheduled predecessors.
 | 
      
         | 1785 |  |  |    'PSS' previously scheduled successors.
 | 
      
         | 1786 |  |  |    't(u)' the cycle where u is scheduled.
 | 
      
         | 1787 |  |  |    'l(u)' is the latency of u.
 | 
      
         | 1788 |  |  |    'd(v,u)' is the dependence distance from v to u.
 | 
      
         | 1789 |  |  |    'ASAP(u)' the earliest time at which u could be scheduled as computed in
 | 
      
         | 1790 |  |  |              the node ordering phase.
 | 
      
         | 1791 |  |  |    'check_hardware_resources_conflicts(u, PS, c)'
 | 
      
         | 1792 |  |  |                              run a trace around cycle/slot through DFA model
 | 
      
         | 1793 |  |  |                              to check resource conflicts involving instruction u
 | 
      
         | 1794 |  |  |                              at cycle c given the partial schedule PS.
 | 
      
         | 1795 |  |  |    'add_to_partial_schedule_at_time(u, PS, c)'
 | 
      
         | 1796 |  |  |                              Add the node/instruction u to the partial schedule
 | 
      
         | 1797 |  |  |                              PS at time c.
 | 
      
         | 1798 |  |  |    'calculate_register_pressure(PS)'
 | 
      
         | 1799 |  |  |                              Given a schedule of instructions, calculate the register
 | 
      
         | 1800 |  |  |                              pressure it implies.  One implementation could be the
 | 
      
         | 1801 |  |  |                              maximum number of overlapping live ranges.
 | 
      
         | 1802 |  |  |    'maxRP' The maximum allowed register pressure, it is usually derived from the number
 | 
      
         | 1803 |  |  |            registers available in the hardware.
 | 
      
         | 1804 |  |  |  
 | 
      
         | 1805 |  |  |    1. II = MII.
 | 
      
         | 1806 |  |  |    2. PS = empty list
 | 
      
         | 1807 |  |  |    3. for each node u in O in pre-computed order
 | 
      
         | 1808 |  |  |    4.   if (PSP(u) != Q && PSS(u) == Q) then
 | 
      
         | 1809 |  |  |    5.     Early_start(u) = max ( t(v) + l(v) - d(v,u)*II ) over all every v in PSP(u).
 | 
      
         | 1810 |  |  |    6.     start = Early_start; end = Early_start + II - 1; step = 1
 | 
      
         | 1811 |  |  |    11.  else if (PSP(u) == Q && PSS(u) != Q) then
 | 
      
         | 1812 |  |  |    12.      Late_start(u) = min ( t(v) - l(v) + d(v,u)*II ) over all every v in PSS(u).
 | 
      
         | 1813 |  |  |    13.     start = Late_start; end = Late_start - II + 1; step = -1
 | 
      
         | 1814 |  |  |    14.  else if (PSP(u) != Q && PSS(u) != Q) then
 | 
      
         | 1815 |  |  |    15.     Early_start(u) = max ( t(v) + l(v) - d(v,u)*II ) over all every v in PSP(u).
 | 
      
         | 1816 |  |  |    16.     Late_start(u) = min ( t(v) - l(v) + d(v,u)*II ) over all every v in PSS(u).
 | 
      
         | 1817 |  |  |    17.     start = Early_start;
 | 
      
         | 1818 |  |  |    18.     end = min(Early_start + II - 1 , Late_start);
 | 
      
         | 1819 |  |  |    19.     step = 1
 | 
      
         | 1820 |  |  |    20.     else "if (PSP(u) == Q && PSS(u) == Q)"
 | 
      
         | 1821 |  |  |    21.    start = ASAP(u); end = start + II - 1; step = 1
 | 
      
         | 1822 |  |  |    22.  endif
 | 
      
         | 1823 |  |  |  
 | 
      
         | 1824 |  |  |    23.  success = false
 | 
      
         | 1825 |  |  |    24.  for (c = start ; c != end ; c += step)
 | 
      
         | 1826 |  |  |    25.     if check_hardware_resources_conflicts(u, PS, c) then
 | 
      
         | 1827 |  |  |    26.       add_to_partial_schedule_at_time(u, PS, c)
 | 
      
         | 1828 |  |  |    27.       success = true
 | 
      
         | 1829 |  |  |    28.       break
 | 
      
         | 1830 |  |  |    29.     endif
 | 
      
         | 1831 |  |  |    30.  endfor
 | 
      
         | 1832 |  |  |    31.  if (success == false) then
 | 
      
         | 1833 |  |  |    32.    II = II + 1
 | 
      
         | 1834 |  |  |    33.    if (II > maxII) then
 | 
      
         | 1835 |  |  |    34.       finish - failed to schedule
 | 
      
         | 1836 |  |  |    35.   endif
 | 
      
         | 1837 |  |  |    36.    goto 2.
 | 
      
         | 1838 |  |  |    37.  endif
 | 
      
         | 1839 |  |  |    38. endfor
 | 
      
         | 1840 |  |  |    39. if (calculate_register_pressure(PS) > maxRP) then
 | 
      
         | 1841 |  |  |    40.    goto 32.
 | 
      
         | 1842 |  |  |    41. endif
 | 
      
         | 1843 |  |  |    42. compute epilogue & prologue
 | 
      
         | 1844 |  |  |    43. finish - succeeded to schedule
 | 
      
         | 1845 |  |  |  
 | 
      
         | 1846 |  |  |    ??? The algorithm restricts the scheduling window to II cycles.
 | 
      
         | 1847 |  |  |    In rare cases, it may be better to allow windows of II+1 cycles.
 | 
      
         | 1848 |  |  |    The window would then start and end on the same row, but with
 | 
      
         | 1849 |  |  |    different "must precede" and "must follow" requirements.  */
 | 
      
         | 1850 |  |  |  
 | 
      
         | 1851 |  |  | /* A limit on the number of cycles that resource conflicts can span.  ??? Should
 | 
      
         | 1852 |  |  |    be provided by DFA, and be dependent on the type of insn scheduled.  Currently
 | 
      
         | 1853 |  |  |    set to 0 to save compile time.  */
 | 
      
         | 1854 |  |  | #define DFA_HISTORY SMS_DFA_HISTORY
 | 
      
         | 1855 |  |  |  
 | 
      
         | 1856 |  |  | /* A threshold for the number of repeated unsuccessful attempts to insert
 | 
      
         | 1857 |  |  |    an empty row, before we flush the partial schedule and start over.  */
 | 
      
         | 1858 |  |  | #define MAX_SPLIT_NUM 10
 | 
      
         | 1859 |  |  | /* Given the partial schedule PS, this function calculates and returns the
 | 
      
         | 1860 |  |  |    cycles in which we can schedule the node with the given index I.
 | 
      
         | 1861 |  |  |    NOTE: Here we do the backtracking in SMS, in some special cases. We have
 | 
      
         | 1862 |  |  |    noticed that there are several cases in which we fail    to SMS the loop
 | 
      
         | 1863 |  |  |    because the sched window of a node is empty    due to tight data-deps. In
 | 
      
         | 1864 |  |  |    such cases we want to unschedule    some of the predecessors/successors
 | 
      
         | 1865 |  |  |    until we get non-empty    scheduling window.  It returns -1 if the
 | 
      
         | 1866 |  |  |    scheduling window is empty and zero otherwise.  */
 | 
      
         | 1867 |  |  |  
 | 
      
         | 1868 |  |  | static int
 | 
      
         | 1869 |  |  | get_sched_window (partial_schedule_ptr ps, ddg_node_ptr u_node,
 | 
      
         | 1870 |  |  |                   sbitmap sched_nodes, int ii, int *start_p, int *step_p,
 | 
      
         | 1871 |  |  |                   int *end_p)
 | 
      
         | 1872 |  |  | {
 | 
      
         | 1873 |  |  |   int start, step, end;
 | 
      
         | 1874 |  |  |   int early_start, late_start;
 | 
      
         | 1875 |  |  |   ddg_edge_ptr e;
 | 
      
         | 1876 |  |  |   sbitmap psp = sbitmap_alloc (ps->g->num_nodes);
 | 
      
         | 1877 |  |  |   sbitmap pss = sbitmap_alloc (ps->g->num_nodes);
 | 
      
         | 1878 |  |  |   sbitmap u_node_preds = NODE_PREDECESSORS (u_node);
 | 
      
         | 1879 |  |  |   sbitmap u_node_succs = NODE_SUCCESSORS (u_node);
 | 
      
         | 1880 |  |  |   int psp_not_empty;
 | 
      
         | 1881 |  |  |   int pss_not_empty;
 | 
      
         | 1882 |  |  |   int count_preds;
 | 
      
         | 1883 |  |  |   int count_succs;
 | 
      
         | 1884 |  |  |  
 | 
      
         | 1885 |  |  |   /* 1. compute sched window for u (start, end, step).  */
 | 
      
         | 1886 |  |  |   sbitmap_zero (psp);
 | 
      
         | 1887 |  |  |   sbitmap_zero (pss);
 | 
      
         | 1888 |  |  |   psp_not_empty = sbitmap_a_and_b_cg (psp, u_node_preds, sched_nodes);
 | 
      
         | 1889 |  |  |   pss_not_empty = sbitmap_a_and_b_cg (pss, u_node_succs, sched_nodes);
 | 
      
         | 1890 |  |  |  
 | 
      
         | 1891 |  |  |   /* We first compute a forward range (start <= end), then decide whether
 | 
      
         | 1892 |  |  |      to reverse it.  */
 | 
      
         | 1893 |  |  |   early_start = INT_MIN;
 | 
      
         | 1894 |  |  |   late_start = INT_MAX;
 | 
      
         | 1895 |  |  |   start = INT_MIN;
 | 
      
         | 1896 |  |  |   end = INT_MAX;
 | 
      
         | 1897 |  |  |   step = 1;
 | 
      
         | 1898 |  |  |  
 | 
      
         | 1899 |  |  |   count_preds = 0;
 | 
      
         | 1900 |  |  |   count_succs = 0;
 | 
      
         | 1901 |  |  |  
 | 
      
         | 1902 |  |  |   if (dump_file && (psp_not_empty || pss_not_empty))
 | 
      
         | 1903 |  |  |     {
 | 
      
         | 1904 |  |  |       fprintf (dump_file, "\nAnalyzing dependencies for node %d (INSN %d)"
 | 
      
         | 1905 |  |  |                "; ii = %d\n\n", u_node->cuid, INSN_UID (u_node->insn), ii);
 | 
      
         | 1906 |  |  |       fprintf (dump_file, "%11s %11s %11s %11s %5s\n",
 | 
      
         | 1907 |  |  |                "start", "early start", "late start", "end", "time");
 | 
      
         | 1908 |  |  |       fprintf (dump_file, "=========== =========== =========== ==========="
 | 
      
         | 1909 |  |  |                " =====\n");
 | 
      
         | 1910 |  |  |     }
 | 
      
         | 1911 |  |  |   /* Calculate early_start and limit end.  Both bounds are inclusive.  */
 | 
      
         | 1912 |  |  |   if (psp_not_empty)
 | 
      
         | 1913 |  |  |     for (e = u_node->in; e != 0; e = e->next_in)
 | 
      
         | 1914 |  |  |       {
 | 
      
         | 1915 |  |  |         int v = e->src->cuid;
 | 
      
         | 1916 |  |  |  
 | 
      
         | 1917 |  |  |         if (TEST_BIT (sched_nodes, v))
 | 
      
         | 1918 |  |  |           {
 | 
      
         | 1919 |  |  |             int p_st = SCHED_TIME (v);
 | 
      
         | 1920 |  |  |             int earliest = p_st + e->latency - (e->distance * ii);
 | 
      
         | 1921 |  |  |             int latest = (e->data_type == MEM_DEP ? p_st + ii - 1 : INT_MAX);
 | 
      
         | 1922 |  |  |  
 | 
      
         | 1923 |  |  |             if (dump_file)
 | 
      
         | 1924 |  |  |               {
 | 
      
         | 1925 |  |  |                 fprintf (dump_file, "%11s %11d %11s %11d %5d",
 | 
      
         | 1926 |  |  |                          "", earliest, "", latest, p_st);
 | 
      
         | 1927 |  |  |                 print_ddg_edge (dump_file, e);
 | 
      
         | 1928 |  |  |                 fprintf (dump_file, "\n");
 | 
      
         | 1929 |  |  |               }
 | 
      
         | 1930 |  |  |  
 | 
      
         | 1931 |  |  |             early_start = MAX (early_start, earliest);
 | 
      
         | 1932 |  |  |             end = MIN (end, latest);
 | 
      
         | 1933 |  |  |  
 | 
      
         | 1934 |  |  |             if (e->type == TRUE_DEP && e->data_type == REG_DEP)
 | 
      
         | 1935 |  |  |               count_preds++;
 | 
      
         | 1936 |  |  |           }
 | 
      
         | 1937 |  |  |       }
 | 
      
         | 1938 |  |  |  
 | 
      
         | 1939 |  |  |   /* Calculate late_start and limit start.  Both bounds are inclusive.  */
 | 
      
         | 1940 |  |  |   if (pss_not_empty)
 | 
      
         | 1941 |  |  |     for (e = u_node->out; e != 0; e = e->next_out)
 | 
      
         | 1942 |  |  |       {
 | 
      
         | 1943 |  |  |         int v = e->dest->cuid;
 | 
      
         | 1944 |  |  |  
 | 
      
         | 1945 |  |  |         if (TEST_BIT (sched_nodes, v))
 | 
      
         | 1946 |  |  |           {
 | 
      
         | 1947 |  |  |             int s_st = SCHED_TIME (v);
 | 
      
         | 1948 |  |  |             int earliest = (e->data_type == MEM_DEP ? s_st - ii + 1 : INT_MIN);
 | 
      
         | 1949 |  |  |             int latest = s_st - e->latency + (e->distance * ii);
 | 
      
         | 1950 |  |  |  
 | 
      
         | 1951 |  |  |             if (dump_file)
 | 
      
         | 1952 |  |  |               {
 | 
      
         | 1953 |  |  |                 fprintf (dump_file, "%11d %11s %11d %11s %5d",
 | 
      
         | 1954 |  |  |                          earliest, "", latest, "", s_st);
 | 
      
         | 1955 |  |  |                 print_ddg_edge (dump_file, e);
 | 
      
         | 1956 |  |  |                 fprintf (dump_file, "\n");
 | 
      
         | 1957 |  |  |               }
 | 
      
         | 1958 |  |  |  
 | 
      
         | 1959 |  |  |             start = MAX (start, earliest);
 | 
      
         | 1960 |  |  |             late_start = MIN (late_start, latest);
 | 
      
         | 1961 |  |  |  
 | 
      
         | 1962 |  |  |             if (e->type == TRUE_DEP && e->data_type == REG_DEP)
 | 
      
         | 1963 |  |  |               count_succs++;
 | 
      
         | 1964 |  |  |           }
 | 
      
         | 1965 |  |  |       }
 | 
      
         | 1966 |  |  |  
 | 
      
         | 1967 |  |  |   if (dump_file && (psp_not_empty || pss_not_empty))
 | 
      
         | 1968 |  |  |     {
 | 
      
         | 1969 |  |  |       fprintf (dump_file, "----------- ----------- ----------- -----------"
 | 
      
         | 1970 |  |  |                " -----\n");
 | 
      
         | 1971 |  |  |       fprintf (dump_file, "%11d %11d %11d %11d %5s %s\n",
 | 
      
         | 1972 |  |  |                start, early_start, late_start, end, "",
 | 
      
         | 1973 |  |  |                "(max, max, min, min)");
 | 
      
         | 1974 |  |  |     }
 | 
      
         | 1975 |  |  |  
 | 
      
         | 1976 |  |  |   /* Get a target scheduling window no bigger than ii.  */
 | 
      
         | 1977 |  |  |   if (early_start == INT_MIN && late_start == INT_MAX)
 | 
      
         | 1978 |  |  |     early_start = NODE_ASAP (u_node);
 | 
      
         | 1979 |  |  |   else if (early_start == INT_MIN)
 | 
      
         | 1980 |  |  |     early_start = late_start - (ii - 1);
 | 
      
         | 1981 |  |  |   late_start = MIN (late_start, early_start + (ii - 1));
 | 
      
         | 1982 |  |  |  
 | 
      
         | 1983 |  |  |   /* Apply memory dependence limits.  */
 | 
      
         | 1984 |  |  |   start = MAX (start, early_start);
 | 
      
         | 1985 |  |  |   end = MIN (end, late_start);
 | 
      
         | 1986 |  |  |  
 | 
      
         | 1987 |  |  |   if (dump_file && (psp_not_empty || pss_not_empty))
 | 
      
         | 1988 |  |  |     fprintf (dump_file, "%11s %11d %11d %11s %5s final window\n",
 | 
      
         | 1989 |  |  |              "", start, end, "", "");
 | 
      
         | 1990 |  |  |  
 | 
      
         | 1991 |  |  |   /* If there are at least as many successors as predecessors, schedule the
 | 
      
         | 1992 |  |  |      node close to its successors.  */
 | 
      
         | 1993 |  |  |   if (pss_not_empty && count_succs >= count_preds)
 | 
      
         | 1994 |  |  |     {
 | 
      
         | 1995 |  |  |       int tmp = end;
 | 
      
         | 1996 |  |  |       end = start;
 | 
      
         | 1997 |  |  |       start = tmp;
 | 
      
         | 1998 |  |  |       step = -1;
 | 
      
         | 1999 |  |  |     }
 | 
      
         | 2000 |  |  |  
 | 
      
         | 2001 |  |  |   /* Now that we've finalized the window, make END an exclusive rather
 | 
      
         | 2002 |  |  |      than an inclusive bound.  */
 | 
      
         | 2003 |  |  |   end += step;
 | 
      
         | 2004 |  |  |  
 | 
      
         | 2005 |  |  |   *start_p = start;
 | 
      
         | 2006 |  |  |   *step_p = step;
 | 
      
         | 2007 |  |  |   *end_p = end;
 | 
      
         | 2008 |  |  |   sbitmap_free (psp);
 | 
      
         | 2009 |  |  |   sbitmap_free (pss);
 | 
      
         | 2010 |  |  |  
 | 
      
         | 2011 |  |  |   if ((start >= end && step == 1) || (start <= end && step == -1))
 | 
      
         | 2012 |  |  |     {
 | 
      
         | 2013 |  |  |       if (dump_file)
 | 
      
         | 2014 |  |  |         fprintf (dump_file, "\nEmpty window: start=%d, end=%d, step=%d\n",
 | 
      
         | 2015 |  |  |                  start, end, step);
 | 
      
         | 2016 |  |  |       return -1;
 | 
      
         | 2017 |  |  |     }
 | 
      
         | 2018 |  |  |  
 | 
      
         | 2019 |  |  |   return 0;
 | 
      
         | 2020 |  |  | }
 | 
      
         | 2021 |  |  |  
 | 
      
         | 2022 |  |  | /* Calculate MUST_PRECEDE/MUST_FOLLOW bitmaps of U_NODE; which is the
 | 
      
         | 2023 |  |  |    node currently been scheduled.  At the end of the calculation
 | 
      
         | 2024 |  |  |    MUST_PRECEDE/MUST_FOLLOW contains all predecessors/successors of
 | 
      
         | 2025 |  |  |    U_NODE which are (1) already scheduled in the first/last row of
 | 
      
         | 2026 |  |  |    U_NODE's scheduling window, (2) whose dependence inequality with U
 | 
      
         | 2027 |  |  |    becomes an equality when U is scheduled in this same row, and (3)
 | 
      
         | 2028 |  |  |    whose dependence latency is zero.
 | 
      
         | 2029 |  |  |  
 | 
      
         | 2030 |  |  |    The first and last rows are calculated using the following parameters:
 | 
      
         | 2031 |  |  |    START/END rows - The cycles that begins/ends the traversal on the window;
 | 
      
         | 2032 |  |  |    searching for an empty cycle to schedule U_NODE.
 | 
      
         | 2033 |  |  |    STEP - The direction in which we traverse the window.
 | 
      
         | 2034 |  |  |    II - The initiation interval.  */
 | 
      
         | 2035 |  |  |  
 | 
      
         | 2036 |  |  | static void
 | 
      
         | 2037 |  |  | calculate_must_precede_follow (ddg_node_ptr u_node, int start, int end,
 | 
      
         | 2038 |  |  |                                int step, int ii, sbitmap sched_nodes,
 | 
      
         | 2039 |  |  |                                sbitmap must_precede, sbitmap must_follow)
 | 
      
         | 2040 |  |  | {
 | 
      
         | 2041 |  |  |   ddg_edge_ptr e;
 | 
      
         | 2042 |  |  |   int first_cycle_in_window, last_cycle_in_window;
 | 
      
         | 2043 |  |  |  
 | 
      
         | 2044 |  |  |   gcc_assert (must_precede && must_follow);
 | 
      
         | 2045 |  |  |  
 | 
      
         | 2046 |  |  |   /* Consider the following scheduling window:
 | 
      
         | 2047 |  |  |      {first_cycle_in_window, first_cycle_in_window+1, ...,
 | 
      
         | 2048 |  |  |      last_cycle_in_window}.  If step is 1 then the following will be
 | 
      
         | 2049 |  |  |      the order we traverse the window: {start=first_cycle_in_window,
 | 
      
         | 2050 |  |  |      first_cycle_in_window+1, ..., end=last_cycle_in_window+1},
 | 
      
         | 2051 |  |  |      or {start=last_cycle_in_window, last_cycle_in_window-1, ...,
 | 
      
         | 2052 |  |  |      end=first_cycle_in_window-1} if step is -1.  */
 | 
      
         | 2053 |  |  |   first_cycle_in_window = (step == 1) ? start : end - step;
 | 
      
         | 2054 |  |  |   last_cycle_in_window = (step == 1) ? end - step : start;
 | 
      
         | 2055 |  |  |  
 | 
      
         | 2056 |  |  |   sbitmap_zero (must_precede);
 | 
      
         | 2057 |  |  |   sbitmap_zero (must_follow);
 | 
      
         | 2058 |  |  |  
 | 
      
         | 2059 |  |  |   if (dump_file)
 | 
      
         | 2060 |  |  |     fprintf (dump_file, "\nmust_precede: ");
 | 
      
         | 2061 |  |  |  
 | 
      
         | 2062 |  |  |   /* Instead of checking if:
 | 
      
         | 2063 |  |  |       (SMODULO (SCHED_TIME (e->src), ii) == first_row_in_window)
 | 
      
         | 2064 |  |  |       && ((SCHED_TIME (e->src) + e->latency - (e->distance * ii)) ==
 | 
      
         | 2065 |  |  |              first_cycle_in_window)
 | 
      
         | 2066 |  |  |       && e->latency == 0
 | 
      
         | 2067 |  |  |      we use the fact that latency is non-negative:
 | 
      
         | 2068 |  |  |       SCHED_TIME (e->src) - (e->distance * ii) <=
 | 
      
         | 2069 |  |  |       SCHED_TIME (e->src) + e->latency - (e->distance * ii)) <=
 | 
      
         | 2070 |  |  |       first_cycle_in_window
 | 
      
         | 2071 |  |  |      and check only if
 | 
      
         | 2072 |  |  |       SCHED_TIME (e->src) - (e->distance * ii) == first_cycle_in_window  */
 | 
      
         | 2073 |  |  |   for (e = u_node->in; e != 0; e = e->next_in)
 | 
      
         | 2074 |  |  |     if (TEST_BIT (sched_nodes, e->src->cuid)
 | 
      
         | 2075 |  |  |         && ((SCHED_TIME (e->src->cuid) - (e->distance * ii)) ==
 | 
      
         | 2076 |  |  |              first_cycle_in_window))
 | 
      
         | 2077 |  |  |       {
 | 
      
         | 2078 |  |  |         if (dump_file)
 | 
      
         | 2079 |  |  |           fprintf (dump_file, "%d ", e->src->cuid);
 | 
      
         | 2080 |  |  |  
 | 
      
         | 2081 |  |  |         SET_BIT (must_precede, e->src->cuid);
 | 
      
         | 2082 |  |  |       }
 | 
      
         | 2083 |  |  |  
 | 
      
         | 2084 |  |  |   if (dump_file)
 | 
      
         | 2085 |  |  |     fprintf (dump_file, "\nmust_follow: ");
 | 
      
         | 2086 |  |  |  
 | 
      
         | 2087 |  |  |   /* Instead of checking if:
 | 
      
         | 2088 |  |  |       (SMODULO (SCHED_TIME (e->dest), ii) == last_row_in_window)
 | 
      
         | 2089 |  |  |       && ((SCHED_TIME (e->dest) - e->latency + (e->distance * ii)) ==
 | 
      
         | 2090 |  |  |              last_cycle_in_window)
 | 
      
         | 2091 |  |  |       && e->latency == 0
 | 
      
         | 2092 |  |  |      we use the fact that latency is non-negative:
 | 
      
         | 2093 |  |  |       SCHED_TIME (e->dest) + (e->distance * ii) >=
 | 
      
         | 2094 |  |  |       SCHED_TIME (e->dest) - e->latency + (e->distance * ii)) >=
 | 
      
         | 2095 |  |  |       last_cycle_in_window
 | 
      
         | 2096 |  |  |      and check only if
 | 
      
         | 2097 |  |  |       SCHED_TIME (e->dest) + (e->distance * ii) == last_cycle_in_window  */
 | 
      
         | 2098 |  |  |   for (e = u_node->out; e != 0; e = e->next_out)
 | 
      
         | 2099 |  |  |     if (TEST_BIT (sched_nodes, e->dest->cuid)
 | 
      
         | 2100 |  |  |         && ((SCHED_TIME (e->dest->cuid) + (e->distance * ii)) ==
 | 
      
         | 2101 |  |  |              last_cycle_in_window))
 | 
      
         | 2102 |  |  |       {
 | 
      
         | 2103 |  |  |         if (dump_file)
 | 
      
         | 2104 |  |  |           fprintf (dump_file, "%d ", e->dest->cuid);
 | 
      
         | 2105 |  |  |  
 | 
      
         | 2106 |  |  |         SET_BIT (must_follow, e->dest->cuid);
 | 
      
         | 2107 |  |  |       }
 | 
      
         | 2108 |  |  |  
 | 
      
         | 2109 |  |  |   if (dump_file)
 | 
      
         | 2110 |  |  |     fprintf (dump_file, "\n");
 | 
      
         | 2111 |  |  | }
 | 
      
         | 2112 |  |  |  
 | 
      
         | 2113 |  |  | /* Return 1 if U_NODE can be scheduled in CYCLE.  Use the following
 | 
      
         | 2114 |  |  |    parameters to decide if that's possible:
 | 
      
         | 2115 |  |  |    PS - The partial schedule.
 | 
      
         | 2116 |  |  |    U - The serial number of U_NODE.
 | 
      
         | 2117 |  |  |    NUM_SPLITS - The number of row splits made so far.
 | 
      
         | 2118 |  |  |    MUST_PRECEDE - The nodes that must precede U_NODE. (only valid at
 | 
      
         | 2119 |  |  |    the first row of the scheduling window)
 | 
      
         | 2120 |  |  |    MUST_FOLLOW - The nodes that must follow U_NODE. (only valid at the
 | 
      
         | 2121 |  |  |    last row of the scheduling window)  */
 | 
      
         | 2122 |  |  |  
 | 
      
         | 2123 |  |  | static bool
 | 
      
         | 2124 |  |  | try_scheduling_node_in_cycle (partial_schedule_ptr ps,
 | 
      
         | 2125 |  |  |                               int u, int cycle, sbitmap sched_nodes,
 | 
      
         | 2126 |  |  |                               int *num_splits, sbitmap must_precede,
 | 
      
         | 2127 |  |  |                               sbitmap must_follow)
 | 
      
         | 2128 |  |  | {
 | 
      
         | 2129 |  |  |   ps_insn_ptr psi;
 | 
      
         | 2130 |  |  |   bool success = 0;
 | 
      
         | 2131 |  |  |  
 | 
      
         | 2132 |  |  |   verify_partial_schedule (ps, sched_nodes);
 | 
      
         | 2133 |  |  |   psi = ps_add_node_check_conflicts (ps, u, cycle, must_precede, must_follow);
 | 
      
         | 2134 |  |  |   if (psi)
 | 
      
         | 2135 |  |  |     {
 | 
      
         | 2136 |  |  |       SCHED_TIME (u) = cycle;
 | 
      
         | 2137 |  |  |       SET_BIT (sched_nodes, u);
 | 
      
         | 2138 |  |  |       success = 1;
 | 
      
         | 2139 |  |  |       *num_splits = 0;
 | 
      
         | 2140 |  |  |       if (dump_file)
 | 
      
         | 2141 |  |  |         fprintf (dump_file, "Scheduled w/o split in %d\n", cycle);
 | 
      
         | 2142 |  |  |  
 | 
      
         | 2143 |  |  |     }
 | 
      
         | 2144 |  |  |  
 | 
      
         | 2145 |  |  |   return success;
 | 
      
         | 2146 |  |  | }
 | 
      
         | 2147 |  |  |  
 | 
      
         | 2148 |  |  | /* This function implements the scheduling algorithm for SMS according to the
 | 
      
         | 2149 |  |  |    above algorithm.  */
 | 
      
         | 2150 |  |  | static partial_schedule_ptr
 | 
      
         | 2151 |  |  | sms_schedule_by_order (ddg_ptr g, int mii, int maxii, int *nodes_order)
 | 
      
         | 2152 |  |  | {
 | 
      
         | 2153 |  |  |   int ii = mii;
 | 
      
         | 2154 |  |  |   int i, c, success, num_splits = 0;
 | 
      
         | 2155 |  |  |   int flush_and_start_over = true;
 | 
      
         | 2156 |  |  |   int num_nodes = g->num_nodes;
 | 
      
         | 2157 |  |  |   int start, end, step; /* Place together into one struct?  */
 | 
      
         | 2158 |  |  |   sbitmap sched_nodes = sbitmap_alloc (num_nodes);
 | 
      
         | 2159 |  |  |   sbitmap must_precede = sbitmap_alloc (num_nodes);
 | 
      
         | 2160 |  |  |   sbitmap must_follow = sbitmap_alloc (num_nodes);
 | 
      
         | 2161 |  |  |   sbitmap tobe_scheduled = sbitmap_alloc (num_nodes);
 | 
      
         | 2162 |  |  |  
 | 
      
         | 2163 |  |  |   partial_schedule_ptr ps = create_partial_schedule (ii, g, DFA_HISTORY);
 | 
      
         | 2164 |  |  |  
 | 
      
         | 2165 |  |  |   sbitmap_ones (tobe_scheduled);
 | 
      
         | 2166 |  |  |   sbitmap_zero (sched_nodes);
 | 
      
         | 2167 |  |  |  
 | 
      
         | 2168 |  |  |   while (flush_and_start_over && (ii < maxii))
 | 
      
         | 2169 |  |  |     {
 | 
      
         | 2170 |  |  |  
 | 
      
         | 2171 |  |  |       if (dump_file)
 | 
      
         | 2172 |  |  |         fprintf (dump_file, "Starting with ii=%d\n", ii);
 | 
      
         | 2173 |  |  |       flush_and_start_over = false;
 | 
      
         | 2174 |  |  |       sbitmap_zero (sched_nodes);
 | 
      
         | 2175 |  |  |  
 | 
      
         | 2176 |  |  |       for (i = 0; i < num_nodes; i++)
 | 
      
         | 2177 |  |  |         {
 | 
      
         | 2178 |  |  |           int u = nodes_order[i];
 | 
      
         | 2179 |  |  |           ddg_node_ptr u_node = &ps->g->nodes[u];
 | 
      
         | 2180 |  |  |           rtx insn = u_node->insn;
 | 
      
         | 2181 |  |  |  
 | 
      
         | 2182 |  |  |           if (!NONDEBUG_INSN_P (insn))
 | 
      
         | 2183 |  |  |             {
 | 
      
         | 2184 |  |  |               RESET_BIT (tobe_scheduled, u);
 | 
      
         | 2185 |  |  |               continue;
 | 
      
         | 2186 |  |  |             }
 | 
      
         | 2187 |  |  |  
 | 
      
         | 2188 |  |  |           if (TEST_BIT (sched_nodes, u))
 | 
      
         | 2189 |  |  |             continue;
 | 
      
         | 2190 |  |  |  
 | 
      
         | 2191 |  |  |           /* Try to get non-empty scheduling window.  */
 | 
      
         | 2192 |  |  |          success = 0;
 | 
      
         | 2193 |  |  |          if (get_sched_window (ps, u_node, sched_nodes, ii, &start,
 | 
      
         | 2194 |  |  |                                 &step, &end) == 0)
 | 
      
         | 2195 |  |  |             {
 | 
      
         | 2196 |  |  |               if (dump_file)
 | 
      
         | 2197 |  |  |                 fprintf (dump_file, "\nTrying to schedule node %d "
 | 
      
         | 2198 |  |  |                          "INSN = %d  in (%d .. %d) step %d\n", u, (INSN_UID
 | 
      
         | 2199 |  |  |                         (g->nodes[u].insn)), start, end, step);
 | 
      
         | 2200 |  |  |  
 | 
      
         | 2201 |  |  |               gcc_assert ((step > 0 && start < end)
 | 
      
         | 2202 |  |  |                           || (step < 0 && start > end));
 | 
      
         | 2203 |  |  |  
 | 
      
         | 2204 |  |  |               calculate_must_precede_follow (u_node, start, end, step, ii,
 | 
      
         | 2205 |  |  |                                              sched_nodes, must_precede,
 | 
      
         | 2206 |  |  |                                              must_follow);
 | 
      
         | 2207 |  |  |  
 | 
      
         | 2208 |  |  |               for (c = start; c != end; c += step)
 | 
      
         | 2209 |  |  |                 {
 | 
      
         | 2210 |  |  |                   sbitmap tmp_precede, tmp_follow;
 | 
      
         | 2211 |  |  |  
 | 
      
         | 2212 |  |  |                   set_must_precede_follow (&tmp_follow, must_follow,
 | 
      
         | 2213 |  |  |                                            &tmp_precede, must_precede,
 | 
      
         | 2214 |  |  |                                            c, start, end, step);
 | 
      
         | 2215 |  |  |                   success =
 | 
      
         | 2216 |  |  |                     try_scheduling_node_in_cycle (ps, u, c,
 | 
      
         | 2217 |  |  |                                                   sched_nodes,
 | 
      
         | 2218 |  |  |                                                   &num_splits, tmp_precede,
 | 
      
         | 2219 |  |  |                                                   tmp_follow);
 | 
      
         | 2220 |  |  |                   if (success)
 | 
      
         | 2221 |  |  |                     break;
 | 
      
         | 2222 |  |  |                 }
 | 
      
         | 2223 |  |  |  
 | 
      
         | 2224 |  |  |               verify_partial_schedule (ps, sched_nodes);
 | 
      
         | 2225 |  |  |             }
 | 
      
         | 2226 |  |  |             if (!success)
 | 
      
         | 2227 |  |  |             {
 | 
      
         | 2228 |  |  |               int split_row;
 | 
      
         | 2229 |  |  |  
 | 
      
         | 2230 |  |  |               if (ii++ == maxii)
 | 
      
         | 2231 |  |  |                 break;
 | 
      
         | 2232 |  |  |  
 | 
      
         | 2233 |  |  |               if (num_splits >= MAX_SPLIT_NUM)
 | 
      
         | 2234 |  |  |                 {
 | 
      
         | 2235 |  |  |                   num_splits = 0;
 | 
      
         | 2236 |  |  |                   flush_and_start_over = true;
 | 
      
         | 2237 |  |  |                   verify_partial_schedule (ps, sched_nodes);
 | 
      
         | 2238 |  |  |                   reset_partial_schedule (ps, ii);
 | 
      
         | 2239 |  |  |                   verify_partial_schedule (ps, sched_nodes);
 | 
      
         | 2240 |  |  |                   break;
 | 
      
         | 2241 |  |  |                 }
 | 
      
         | 2242 |  |  |  
 | 
      
         | 2243 |  |  |               num_splits++;
 | 
      
         | 2244 |  |  |               /* The scheduling window is exclusive of 'end'
 | 
      
         | 2245 |  |  |                  whereas compute_split_window() expects an inclusive,
 | 
      
         | 2246 |  |  |                  ordered range.  */
 | 
      
         | 2247 |  |  |               if (step == 1)
 | 
      
         | 2248 |  |  |                 split_row = compute_split_row (sched_nodes, start, end - 1,
 | 
      
         | 2249 |  |  |                                                ps->ii, u_node);
 | 
      
         | 2250 |  |  |               else
 | 
      
         | 2251 |  |  |                 split_row = compute_split_row (sched_nodes, end + 1, start,
 | 
      
         | 2252 |  |  |                                                ps->ii, u_node);
 | 
      
         | 2253 |  |  |  
 | 
      
         | 2254 |  |  |               ps_insert_empty_row (ps, split_row, sched_nodes);
 | 
      
         | 2255 |  |  |               i--;              /* Go back and retry node i.  */
 | 
      
         | 2256 |  |  |  
 | 
      
         | 2257 |  |  |               if (dump_file)
 | 
      
         | 2258 |  |  |                 fprintf (dump_file, "num_splits=%d\n", num_splits);
 | 
      
         | 2259 |  |  |             }
 | 
      
         | 2260 |  |  |  
 | 
      
         | 2261 |  |  |           /* ??? If (success), check register pressure estimates.  */
 | 
      
         | 2262 |  |  |         }                       /* Continue with next node.  */
 | 
      
         | 2263 |  |  |     }                           /* While flush_and_start_over.  */
 | 
      
         | 2264 |  |  |   if (ii >= maxii)
 | 
      
         | 2265 |  |  |     {
 | 
      
         | 2266 |  |  |       free_partial_schedule (ps);
 | 
      
         | 2267 |  |  |       ps = NULL;
 | 
      
         | 2268 |  |  |     }
 | 
      
         | 2269 |  |  |   else
 | 
      
         | 2270 |  |  |     gcc_assert (sbitmap_equal (tobe_scheduled, sched_nodes));
 | 
      
         | 2271 |  |  |  
 | 
      
         | 2272 |  |  |   sbitmap_free (sched_nodes);
 | 
      
         | 2273 |  |  |   sbitmap_free (must_precede);
 | 
      
         | 2274 |  |  |   sbitmap_free (must_follow);
 | 
      
         | 2275 |  |  |   sbitmap_free (tobe_scheduled);
 | 
      
         | 2276 |  |  |  
 | 
      
         | 2277 |  |  |   return ps;
 | 
      
         | 2278 |  |  | }
 | 
      
         | 2279 |  |  |  
 | 
      
         | 2280 |  |  | /* This function inserts a new empty row into PS at the position
 | 
      
         | 2281 |  |  |    according to SPLITROW, keeping all already scheduled instructions
 | 
      
         | 2282 |  |  |    intact and updating their SCHED_TIME and cycle accordingly.  */
 | 
      
         | 2283 |  |  | static void
 | 
      
         | 2284 |  |  | ps_insert_empty_row (partial_schedule_ptr ps, int split_row,
 | 
      
         | 2285 |  |  |                      sbitmap sched_nodes)
 | 
      
         | 2286 |  |  | {
 | 
      
         | 2287 |  |  |   ps_insn_ptr crr_insn;
 | 
      
         | 2288 |  |  |   ps_insn_ptr *rows_new;
 | 
      
         | 2289 |  |  |   int ii = ps->ii;
 | 
      
         | 2290 |  |  |   int new_ii = ii + 1;
 | 
      
         | 2291 |  |  |   int row;
 | 
      
         | 2292 |  |  |   int *rows_length_new;
 | 
      
         | 2293 |  |  |  
 | 
      
         | 2294 |  |  |   verify_partial_schedule (ps, sched_nodes);
 | 
      
         | 2295 |  |  |  
 | 
      
         | 2296 |  |  |   /* We normalize sched_time and rotate ps to have only non-negative sched
 | 
      
         | 2297 |  |  |      times, for simplicity of updating cycles after inserting new row.  */
 | 
      
         | 2298 |  |  |   split_row -= ps->min_cycle;
 | 
      
         | 2299 |  |  |   split_row = SMODULO (split_row, ii);
 | 
      
         | 2300 |  |  |   if (dump_file)
 | 
      
         | 2301 |  |  |     fprintf (dump_file, "split_row=%d\n", split_row);
 | 
      
         | 2302 |  |  |  
 | 
      
         | 2303 |  |  |   reset_sched_times (ps, PS_MIN_CYCLE (ps));
 | 
      
         | 2304 |  |  |   rotate_partial_schedule (ps, PS_MIN_CYCLE (ps));
 | 
      
         | 2305 |  |  |  
 | 
      
         | 2306 |  |  |   rows_new = (ps_insn_ptr *) xcalloc (new_ii, sizeof (ps_insn_ptr));
 | 
      
         | 2307 |  |  |   rows_length_new = (int *) xcalloc (new_ii, sizeof (int));
 | 
      
         | 2308 |  |  |   for (row = 0; row < split_row; row++)
 | 
      
         | 2309 |  |  |     {
 | 
      
         | 2310 |  |  |       rows_new[row] = ps->rows[row];
 | 
      
         | 2311 |  |  |       rows_length_new[row] = ps->rows_length[row];
 | 
      
         | 2312 |  |  |       ps->rows[row] = NULL;
 | 
      
         | 2313 |  |  |       for (crr_insn = rows_new[row];
 | 
      
         | 2314 |  |  |            crr_insn; crr_insn = crr_insn->next_in_row)
 | 
      
         | 2315 |  |  |         {
 | 
      
         | 2316 |  |  |           int u = crr_insn->id;
 | 
      
         | 2317 |  |  |           int new_time = SCHED_TIME (u) + (SCHED_TIME (u) / ii);
 | 
      
         | 2318 |  |  |  
 | 
      
         | 2319 |  |  |           SCHED_TIME (u) = new_time;
 | 
      
         | 2320 |  |  |           crr_insn->cycle = new_time;
 | 
      
         | 2321 |  |  |           SCHED_ROW (u) = new_time % new_ii;
 | 
      
         | 2322 |  |  |           SCHED_STAGE (u) = new_time / new_ii;
 | 
      
         | 2323 |  |  |         }
 | 
      
         | 2324 |  |  |  
 | 
      
         | 2325 |  |  |     }
 | 
      
         | 2326 |  |  |  
 | 
      
         | 2327 |  |  |   rows_new[split_row] = NULL;
 | 
      
         | 2328 |  |  |  
 | 
      
         | 2329 |  |  |   for (row = split_row; row < ii; row++)
 | 
      
         | 2330 |  |  |     {
 | 
      
         | 2331 |  |  |       rows_new[row + 1] = ps->rows[row];
 | 
      
         | 2332 |  |  |       rows_length_new[row + 1] = ps->rows_length[row];
 | 
      
         | 2333 |  |  |       ps->rows[row] = NULL;
 | 
      
         | 2334 |  |  |       for (crr_insn = rows_new[row + 1];
 | 
      
         | 2335 |  |  |            crr_insn; crr_insn = crr_insn->next_in_row)
 | 
      
         | 2336 |  |  |         {
 | 
      
         | 2337 |  |  |           int u = crr_insn->id;
 | 
      
         | 2338 |  |  |           int new_time = SCHED_TIME (u) + (SCHED_TIME (u) / ii) + 1;
 | 
      
         | 2339 |  |  |  
 | 
      
         | 2340 |  |  |           SCHED_TIME (u) = new_time;
 | 
      
         | 2341 |  |  |           crr_insn->cycle = new_time;
 | 
      
         | 2342 |  |  |           SCHED_ROW (u) = new_time % new_ii;
 | 
      
         | 2343 |  |  |           SCHED_STAGE (u) = new_time / new_ii;
 | 
      
         | 2344 |  |  |         }
 | 
      
         | 2345 |  |  |     }
 | 
      
         | 2346 |  |  |  
 | 
      
         | 2347 |  |  |   /* Updating ps.  */
 | 
      
         | 2348 |  |  |   ps->min_cycle = ps->min_cycle + ps->min_cycle / ii
 | 
      
         | 2349 |  |  |     + (SMODULO (ps->min_cycle, ii) >= split_row ? 1 : 0);
 | 
      
         | 2350 |  |  |   ps->max_cycle = ps->max_cycle + ps->max_cycle / ii
 | 
      
         | 2351 |  |  |     + (SMODULO (ps->max_cycle, ii) >= split_row ? 1 : 0);
 | 
      
         | 2352 |  |  |   free (ps->rows);
 | 
      
         | 2353 |  |  |   ps->rows = rows_new;
 | 
      
         | 2354 |  |  |   free (ps->rows_length);
 | 
      
         | 2355 |  |  |   ps->rows_length = rows_length_new;
 | 
      
         | 2356 |  |  |   ps->ii = new_ii;
 | 
      
         | 2357 |  |  |   gcc_assert (ps->min_cycle >= 0);
 | 
      
         | 2358 |  |  |  
 | 
      
         | 2359 |  |  |   verify_partial_schedule (ps, sched_nodes);
 | 
      
         | 2360 |  |  |  
 | 
      
         | 2361 |  |  |   if (dump_file)
 | 
      
         | 2362 |  |  |     fprintf (dump_file, "min_cycle=%d, max_cycle=%d\n", ps->min_cycle,
 | 
      
         | 2363 |  |  |              ps->max_cycle);
 | 
      
         | 2364 |  |  | }
 | 
      
         | 2365 |  |  |  
 | 
      
         | 2366 |  |  | /* Given U_NODE which is the node that failed to be scheduled; LOW and
 | 
      
         | 2367 |  |  |    UP which are the boundaries of it's scheduling window; compute using
 | 
      
         | 2368 |  |  |    SCHED_NODES and II a row in the partial schedule that can be split
 | 
      
         | 2369 |  |  |    which will separate a critical predecessor from a critical successor
 | 
      
         | 2370 |  |  |    thereby expanding the window, and return it.  */
 | 
      
         | 2371 |  |  | static int
 | 
      
         | 2372 |  |  | compute_split_row (sbitmap sched_nodes, int low, int up, int ii,
 | 
      
         | 2373 |  |  |                    ddg_node_ptr u_node)
 | 
      
         | 2374 |  |  | {
 | 
      
         | 2375 |  |  |   ddg_edge_ptr e;
 | 
      
         | 2376 |  |  |   int lower = INT_MIN, upper = INT_MAX;
 | 
      
         | 2377 |  |  |   int crit_pred = -1;
 | 
      
         | 2378 |  |  |   int crit_succ = -1;
 | 
      
         | 2379 |  |  |   int crit_cycle;
 | 
      
         | 2380 |  |  |  
 | 
      
         | 2381 |  |  |   for (e = u_node->in; e != 0; e = e->next_in)
 | 
      
         | 2382 |  |  |     {
 | 
      
         | 2383 |  |  |       int v = e->src->cuid;
 | 
      
         | 2384 |  |  |  
 | 
      
         | 2385 |  |  |       if (TEST_BIT (sched_nodes, v)
 | 
      
         | 2386 |  |  |           && (low == SCHED_TIME (v) + e->latency - (e->distance * ii)))
 | 
      
         | 2387 |  |  |         if (SCHED_TIME (v) > lower)
 | 
      
         | 2388 |  |  |           {
 | 
      
         | 2389 |  |  |             crit_pred = v;
 | 
      
         | 2390 |  |  |             lower = SCHED_TIME (v);
 | 
      
         | 2391 |  |  |           }
 | 
      
         | 2392 |  |  |     }
 | 
      
         | 2393 |  |  |  
 | 
      
         | 2394 |  |  |   if (crit_pred >= 0)
 | 
      
         | 2395 |  |  |     {
 | 
      
         | 2396 |  |  |       crit_cycle = SCHED_TIME (crit_pred) + 1;
 | 
      
         | 2397 |  |  |       return SMODULO (crit_cycle, ii);
 | 
      
         | 2398 |  |  |     }
 | 
      
         | 2399 |  |  |  
 | 
      
         | 2400 |  |  |   for (e = u_node->out; e != 0; e = e->next_out)
 | 
      
         | 2401 |  |  |     {
 | 
      
         | 2402 |  |  |       int v = e->dest->cuid;
 | 
      
         | 2403 |  |  |  
 | 
      
         | 2404 |  |  |       if (TEST_BIT (sched_nodes, v)
 | 
      
         | 2405 |  |  |           && (up == SCHED_TIME (v) - e->latency + (e->distance * ii)))
 | 
      
         | 2406 |  |  |         if (SCHED_TIME (v) < upper)
 | 
      
         | 2407 |  |  |           {
 | 
      
         | 2408 |  |  |             crit_succ = v;
 | 
      
         | 2409 |  |  |             upper = SCHED_TIME (v);
 | 
      
         | 2410 |  |  |           }
 | 
      
         | 2411 |  |  |     }
 | 
      
         | 2412 |  |  |  
 | 
      
         | 2413 |  |  |   if (crit_succ >= 0)
 | 
      
         | 2414 |  |  |     {
 | 
      
         | 2415 |  |  |       crit_cycle = SCHED_TIME (crit_succ);
 | 
      
         | 2416 |  |  |       return SMODULO (crit_cycle, ii);
 | 
      
         | 2417 |  |  |     }
 | 
      
         | 2418 |  |  |  
 | 
      
         | 2419 |  |  |   if (dump_file)
 | 
      
         | 2420 |  |  |     fprintf (dump_file, "Both crit_pred and crit_succ are NULL\n");
 | 
      
         | 2421 |  |  |  
 | 
      
         | 2422 |  |  |   return SMODULO ((low + up + 1) / 2, ii);
 | 
      
         | 2423 |  |  | }
 | 
      
         | 2424 |  |  |  
 | 
      
         | 2425 |  |  | static void
 | 
      
         | 2426 |  |  | verify_partial_schedule (partial_schedule_ptr ps, sbitmap sched_nodes)
 | 
      
         | 2427 |  |  | {
 | 
      
         | 2428 |  |  |   int row;
 | 
      
         | 2429 |  |  |   ps_insn_ptr crr_insn;
 | 
      
         | 2430 |  |  |  
 | 
      
         | 2431 |  |  |   for (row = 0; row < ps->ii; row++)
 | 
      
         | 2432 |  |  |     {
 | 
      
         | 2433 |  |  |       int length = 0;
 | 
      
         | 2434 |  |  |  
 | 
      
         | 2435 |  |  |       for (crr_insn = ps->rows[row]; crr_insn; crr_insn = crr_insn->next_in_row)
 | 
      
         | 2436 |  |  |         {
 | 
      
         | 2437 |  |  |           int u = crr_insn->id;
 | 
      
         | 2438 |  |  |  
 | 
      
         | 2439 |  |  |           length++;
 | 
      
         | 2440 |  |  |           gcc_assert (TEST_BIT (sched_nodes, u));
 | 
      
         | 2441 |  |  |           /* ??? Test also that all nodes of sched_nodes are in ps, perhaps by
 | 
      
         | 2442 |  |  |              popcount (sched_nodes) == number of insns in ps.  */
 | 
      
         | 2443 |  |  |           gcc_assert (SCHED_TIME (u) >= ps->min_cycle);
 | 
      
         | 2444 |  |  |           gcc_assert (SCHED_TIME (u) <= ps->max_cycle);
 | 
      
         | 2445 |  |  |         }
 | 
      
         | 2446 |  |  |  
 | 
      
         | 2447 |  |  |       gcc_assert (ps->rows_length[row] == length);
 | 
      
         | 2448 |  |  |     }
 | 
      
         | 2449 |  |  | }
 | 
      
         | 2450 |  |  |  
 | 
      
         | 2451 |  |  |  
 | 
      
         | 2452 |  |  | /* This page implements the algorithm for ordering the nodes of a DDG
 | 
      
         | 2453 |  |  |    for modulo scheduling, activated through the
 | 
      
         | 2454 |  |  |    "int sms_order_nodes (ddg_ptr, int mii, int * result)" API.  */
 | 
      
         | 2455 |  |  |  
 | 
      
         | 2456 |  |  | #define ORDER_PARAMS(x) ((struct node_order_params *) (x)->aux.info)
 | 
      
         | 2457 |  |  | #define ASAP(x) (ORDER_PARAMS ((x))->asap)
 | 
      
         | 2458 |  |  | #define ALAP(x) (ORDER_PARAMS ((x))->alap)
 | 
      
         | 2459 |  |  | #define HEIGHT(x) (ORDER_PARAMS ((x))->height)
 | 
      
         | 2460 |  |  | #define MOB(x) (ALAP ((x)) - ASAP ((x)))
 | 
      
         | 2461 |  |  | #define DEPTH(x) (ASAP ((x)))
 | 
      
         | 2462 |  |  |  
 | 
      
         | 2463 |  |  | typedef struct node_order_params * nopa;
 | 
      
         | 2464 |  |  |  
 | 
      
         | 2465 |  |  | static void order_nodes_of_sccs (ddg_all_sccs_ptr, int * result);
 | 
      
         | 2466 |  |  | static int order_nodes_in_scc (ddg_ptr, sbitmap, sbitmap, int*, int);
 | 
      
         | 2467 |  |  | static nopa  calculate_order_params (ddg_ptr, int, int *);
 | 
      
         | 2468 |  |  | static int find_max_asap (ddg_ptr, sbitmap);
 | 
      
         | 2469 |  |  | static int find_max_hv_min_mob (ddg_ptr, sbitmap);
 | 
      
         | 2470 |  |  | static int find_max_dv_min_mob (ddg_ptr, sbitmap);
 | 
      
         | 2471 |  |  |  
 | 
      
         | 2472 |  |  | enum sms_direction {BOTTOMUP, TOPDOWN};
 | 
      
         | 2473 |  |  |  
 | 
      
         | 2474 |  |  | struct node_order_params
 | 
      
         | 2475 |  |  | {
 | 
      
         | 2476 |  |  |   int asap;
 | 
      
         | 2477 |  |  |   int alap;
 | 
      
         | 2478 |  |  |   int height;
 | 
      
         | 2479 |  |  | };
 | 
      
         | 2480 |  |  |  
 | 
      
         | 2481 |  |  | /* Check if NODE_ORDER contains a permutation of 0 .. NUM_NODES-1.  */
 | 
      
         | 2482 |  |  | static void
 | 
      
         | 2483 |  |  | check_nodes_order (int *node_order, int num_nodes)
 | 
      
         | 2484 |  |  | {
 | 
      
         | 2485 |  |  |   int i;
 | 
      
         | 2486 |  |  |   sbitmap tmp = sbitmap_alloc (num_nodes);
 | 
      
         | 2487 |  |  |  
 | 
      
         | 2488 |  |  |   sbitmap_zero (tmp);
 | 
      
         | 2489 |  |  |  
 | 
      
         | 2490 |  |  |   if (dump_file)
 | 
      
         | 2491 |  |  |     fprintf (dump_file, "SMS final nodes order: \n");
 | 
      
         | 2492 |  |  |  
 | 
      
         | 2493 |  |  |   for (i = 0; i < num_nodes; i++)
 | 
      
         | 2494 |  |  |     {
 | 
      
         | 2495 |  |  |       int u = node_order[i];
 | 
      
         | 2496 |  |  |  
 | 
      
         | 2497 |  |  |       if (dump_file)
 | 
      
         | 2498 |  |  |         fprintf (dump_file, "%d ", u);
 | 
      
         | 2499 |  |  |       gcc_assert (u < num_nodes && u >= 0 && !TEST_BIT (tmp, u));
 | 
      
         | 2500 |  |  |  
 | 
      
         | 2501 |  |  |       SET_BIT (tmp, u);
 | 
      
         | 2502 |  |  |     }
 | 
      
         | 2503 |  |  |  
 | 
      
         | 2504 |  |  |   if (dump_file)
 | 
      
         | 2505 |  |  |     fprintf (dump_file, "\n");
 | 
      
         | 2506 |  |  |  
 | 
      
         | 2507 |  |  |   sbitmap_free (tmp);
 | 
      
         | 2508 |  |  | }
 | 
      
         | 2509 |  |  |  
 | 
      
         | 2510 |  |  | /* Order the nodes of G for scheduling and pass the result in
 | 
      
         | 2511 |  |  |    NODE_ORDER.  Also set aux.count of each node to ASAP.
 | 
      
         | 2512 |  |  |    Put maximal ASAP to PMAX_ASAP.  Return the recMII for the given DDG.  */
 | 
      
         | 2513 |  |  | static int
 | 
      
         | 2514 |  |  | sms_order_nodes (ddg_ptr g, int mii, int * node_order, int *pmax_asap)
 | 
      
         | 2515 |  |  | {
 | 
      
         | 2516 |  |  |   int i;
 | 
      
         | 2517 |  |  |   int rec_mii = 0;
 | 
      
         | 2518 |  |  |   ddg_all_sccs_ptr sccs = create_ddg_all_sccs (g);
 | 
      
         | 2519 |  |  |  
 | 
      
         | 2520 |  |  |   nopa nops = calculate_order_params (g, mii, pmax_asap);
 | 
      
         | 2521 |  |  |  
 | 
      
         | 2522 |  |  |   if (dump_file)
 | 
      
         | 2523 |  |  |     print_sccs (dump_file, sccs, g);
 | 
      
         | 2524 |  |  |  
 | 
      
         | 2525 |  |  |   order_nodes_of_sccs (sccs, node_order);
 | 
      
         | 2526 |  |  |  
 | 
      
         | 2527 |  |  |   if (sccs->num_sccs > 0)
 | 
      
         | 2528 |  |  |     /* First SCC has the largest recurrence_length.  */
 | 
      
         | 2529 |  |  |     rec_mii = sccs->sccs[0]->recurrence_length;
 | 
      
         | 2530 |  |  |  
 | 
      
         | 2531 |  |  |   /* Save ASAP before destroying node_order_params.  */
 | 
      
         | 2532 |  |  |   for (i = 0; i < g->num_nodes; i++)
 | 
      
         | 2533 |  |  |     {
 | 
      
         | 2534 |  |  |       ddg_node_ptr v = &g->nodes[i];
 | 
      
         | 2535 |  |  |       v->aux.count = ASAP (v);
 | 
      
         | 2536 |  |  |     }
 | 
      
         | 2537 |  |  |  
 | 
      
         | 2538 |  |  |   free (nops);
 | 
      
         | 2539 |  |  |   free_ddg_all_sccs (sccs);
 | 
      
         | 2540 |  |  |   check_nodes_order (node_order, g->num_nodes);
 | 
      
         | 2541 |  |  |  
 | 
      
         | 2542 |  |  |   return rec_mii;
 | 
      
         | 2543 |  |  | }
 | 
      
         | 2544 |  |  |  
 | 
      
         | 2545 |  |  | static void
 | 
      
         | 2546 |  |  | order_nodes_of_sccs (ddg_all_sccs_ptr all_sccs, int * node_order)
 | 
      
         | 2547 |  |  | {
 | 
      
         | 2548 |  |  |   int i, pos = 0;
 | 
      
         | 2549 |  |  |   ddg_ptr g = all_sccs->ddg;
 | 
      
         | 2550 |  |  |   int num_nodes = g->num_nodes;
 | 
      
         | 2551 |  |  |   sbitmap prev_sccs = sbitmap_alloc (num_nodes);
 | 
      
         | 2552 |  |  |   sbitmap on_path = sbitmap_alloc (num_nodes);
 | 
      
         | 2553 |  |  |   sbitmap tmp = sbitmap_alloc (num_nodes);
 | 
      
         | 2554 |  |  |   sbitmap ones = sbitmap_alloc (num_nodes);
 | 
      
         | 2555 |  |  |  
 | 
      
         | 2556 |  |  |   sbitmap_zero (prev_sccs);
 | 
      
         | 2557 |  |  |   sbitmap_ones (ones);
 | 
      
         | 2558 |  |  |  
 | 
      
         | 2559 |  |  |   /* Perform the node ordering starting from the SCC with the highest recMII.
 | 
      
         | 2560 |  |  |      For each SCC order the nodes according to their ASAP/ALAP/HEIGHT etc.  */
 | 
      
         | 2561 |  |  |   for (i = 0; i < all_sccs->num_sccs; i++)
 | 
      
         | 2562 |  |  |     {
 | 
      
         | 2563 |  |  |       ddg_scc_ptr scc = all_sccs->sccs[i];
 | 
      
         | 2564 |  |  |  
 | 
      
         | 2565 |  |  |       /* Add nodes on paths from previous SCCs to the current SCC.  */
 | 
      
         | 2566 |  |  |       find_nodes_on_paths (on_path, g, prev_sccs, scc->nodes);
 | 
      
         | 2567 |  |  |       sbitmap_a_or_b (tmp, scc->nodes, on_path);
 | 
      
         | 2568 |  |  |  
 | 
      
         | 2569 |  |  |       /* Add nodes on paths from the current SCC to previous SCCs.  */
 | 
      
         | 2570 |  |  |       find_nodes_on_paths (on_path, g, scc->nodes, prev_sccs);
 | 
      
         | 2571 |  |  |       sbitmap_a_or_b (tmp, tmp, on_path);
 | 
      
         | 2572 |  |  |  
 | 
      
         | 2573 |  |  |       /* Remove nodes of previous SCCs from current extended SCC.  */
 | 
      
         | 2574 |  |  |       sbitmap_difference (tmp, tmp, prev_sccs);
 | 
      
         | 2575 |  |  |  
 | 
      
         | 2576 |  |  |       pos = order_nodes_in_scc (g, prev_sccs, tmp, node_order, pos);
 | 
      
         | 2577 |  |  |       /* Above call to order_nodes_in_scc updated prev_sccs |= tmp.  */
 | 
      
         | 2578 |  |  |     }
 | 
      
         | 2579 |  |  |  
 | 
      
         | 2580 |  |  |   /* Handle the remaining nodes that do not belong to any scc.  Each call
 | 
      
         | 2581 |  |  |      to order_nodes_in_scc handles a single connected component.  */
 | 
      
         | 2582 |  |  |   while (pos < g->num_nodes)
 | 
      
         | 2583 |  |  |     {
 | 
      
         | 2584 |  |  |       sbitmap_difference (tmp, ones, prev_sccs);
 | 
      
         | 2585 |  |  |       pos = order_nodes_in_scc (g, prev_sccs, tmp, node_order, pos);
 | 
      
         | 2586 |  |  |     }
 | 
      
         | 2587 |  |  |   sbitmap_free (prev_sccs);
 | 
      
         | 2588 |  |  |   sbitmap_free (on_path);
 | 
      
         | 2589 |  |  |   sbitmap_free (tmp);
 | 
      
         | 2590 |  |  |   sbitmap_free (ones);
 | 
      
         | 2591 |  |  | }
 | 
      
         | 2592 |  |  |  
 | 
      
         | 2593 |  |  | /* MII is needed if we consider backarcs (that do not close recursive cycles).  */
 | 
      
         | 2594 |  |  | static struct node_order_params *
 | 
      
         | 2595 |  |  | calculate_order_params (ddg_ptr g, int mii ATTRIBUTE_UNUSED, int *pmax_asap)
 | 
      
         | 2596 |  |  | {
 | 
      
         | 2597 |  |  |   int u;
 | 
      
         | 2598 |  |  |   int max_asap;
 | 
      
         | 2599 |  |  |   int num_nodes = g->num_nodes;
 | 
      
         | 2600 |  |  |   ddg_edge_ptr e;
 | 
      
         | 2601 |  |  |   /* Allocate a place to hold ordering params for each node in the DDG.  */
 | 
      
         | 2602 |  |  |   nopa node_order_params_arr;
 | 
      
         | 2603 |  |  |  
 | 
      
         | 2604 |  |  |   /* Initialize of ASAP/ALAP/HEIGHT to zero.  */
 | 
      
         | 2605 |  |  |   node_order_params_arr = (nopa) xcalloc (num_nodes,
 | 
      
         | 2606 |  |  |                                           sizeof (struct node_order_params));
 | 
      
         | 2607 |  |  |  
 | 
      
         | 2608 |  |  |   /* Set the aux pointer of each node to point to its order_params structure.  */
 | 
      
         | 2609 |  |  |   for (u = 0; u < num_nodes; u++)
 | 
      
         | 2610 |  |  |     g->nodes[u].aux.info = &node_order_params_arr[u];
 | 
      
         | 2611 |  |  |  
 | 
      
         | 2612 |  |  |   /* Disregarding a backarc from each recursive cycle to obtain a DAG,
 | 
      
         | 2613 |  |  |      calculate ASAP, ALAP, mobility, distance, and height for each node
 | 
      
         | 2614 |  |  |      in the dependence (direct acyclic) graph.  */
 | 
      
         | 2615 |  |  |  
 | 
      
         | 2616 |  |  |   /* We assume that the nodes in the array are in topological order.  */
 | 
      
         | 2617 |  |  |  
 | 
      
         | 2618 |  |  |   max_asap = 0;
 | 
      
         | 2619 |  |  |   for (u = 0; u < num_nodes; u++)
 | 
      
         | 2620 |  |  |     {
 | 
      
         | 2621 |  |  |       ddg_node_ptr u_node = &g->nodes[u];
 | 
      
         | 2622 |  |  |  
 | 
      
         | 2623 |  |  |       ASAP (u_node) = 0;
 | 
      
         | 2624 |  |  |       for (e = u_node->in; e; e = e->next_in)
 | 
      
         | 2625 |  |  |         if (e->distance == 0)
 | 
      
         | 2626 |  |  |           ASAP (u_node) = MAX (ASAP (u_node),
 | 
      
         | 2627 |  |  |                                ASAP (e->src) + e->latency);
 | 
      
         | 2628 |  |  |       max_asap = MAX (max_asap, ASAP (u_node));
 | 
      
         | 2629 |  |  |     }
 | 
      
         | 2630 |  |  |  
 | 
      
         | 2631 |  |  |   for (u = num_nodes - 1; u > -1; u--)
 | 
      
         | 2632 |  |  |     {
 | 
      
         | 2633 |  |  |       ddg_node_ptr u_node = &g->nodes[u];
 | 
      
         | 2634 |  |  |  
 | 
      
         | 2635 |  |  |       ALAP (u_node) = max_asap;
 | 
      
         | 2636 |  |  |       HEIGHT (u_node) = 0;
 | 
      
         | 2637 |  |  |       for (e = u_node->out; e; e = e->next_out)
 | 
      
         | 2638 |  |  |         if (e->distance == 0)
 | 
      
         | 2639 |  |  |           {
 | 
      
         | 2640 |  |  |             ALAP (u_node) = MIN (ALAP (u_node),
 | 
      
         | 2641 |  |  |                                  ALAP (e->dest) - e->latency);
 | 
      
         | 2642 |  |  |             HEIGHT (u_node) = MAX (HEIGHT (u_node),
 | 
      
         | 2643 |  |  |                                    HEIGHT (e->dest) + e->latency);
 | 
      
         | 2644 |  |  |           }
 | 
      
         | 2645 |  |  |     }
 | 
      
         | 2646 |  |  |   if (dump_file)
 | 
      
         | 2647 |  |  |   {
 | 
      
         | 2648 |  |  |     fprintf (dump_file, "\nOrder params\n");
 | 
      
         | 2649 |  |  |     for (u = 0; u < num_nodes; u++)
 | 
      
         | 2650 |  |  |       {
 | 
      
         | 2651 |  |  |         ddg_node_ptr u_node = &g->nodes[u];
 | 
      
         | 2652 |  |  |  
 | 
      
         | 2653 |  |  |         fprintf (dump_file, "node %d, ASAP: %d, ALAP: %d, HEIGHT: %d\n", u,
 | 
      
         | 2654 |  |  |                  ASAP (u_node), ALAP (u_node), HEIGHT (u_node));
 | 
      
         | 2655 |  |  |       }
 | 
      
         | 2656 |  |  |   }
 | 
      
         | 2657 |  |  |  
 | 
      
         | 2658 |  |  |   *pmax_asap = max_asap;
 | 
      
         | 2659 |  |  |   return node_order_params_arr;
 | 
      
         | 2660 |  |  | }
 | 
      
         | 2661 |  |  |  
 | 
      
         | 2662 |  |  | static int
 | 
      
         | 2663 |  |  | find_max_asap (ddg_ptr g, sbitmap nodes)
 | 
      
         | 2664 |  |  | {
 | 
      
         | 2665 |  |  |   unsigned int u = 0;
 | 
      
         | 2666 |  |  |   int max_asap = -1;
 | 
      
         | 2667 |  |  |   int result = -1;
 | 
      
         | 2668 |  |  |   sbitmap_iterator sbi;
 | 
      
         | 2669 |  |  |  
 | 
      
         | 2670 |  |  |   EXECUTE_IF_SET_IN_SBITMAP (nodes, 0, u, sbi)
 | 
      
         | 2671 |  |  |     {
 | 
      
         | 2672 |  |  |       ddg_node_ptr u_node = &g->nodes[u];
 | 
      
         | 2673 |  |  |  
 | 
      
         | 2674 |  |  |       if (max_asap < ASAP (u_node))
 | 
      
         | 2675 |  |  |         {
 | 
      
         | 2676 |  |  |           max_asap = ASAP (u_node);
 | 
      
         | 2677 |  |  |           result = u;
 | 
      
         | 2678 |  |  |         }
 | 
      
         | 2679 |  |  |     }
 | 
      
         | 2680 |  |  |   return result;
 | 
      
         | 2681 |  |  | }
 | 
      
         | 2682 |  |  |  
 | 
      
         | 2683 |  |  | static int
 | 
      
         | 2684 |  |  | find_max_hv_min_mob (ddg_ptr g, sbitmap nodes)
 | 
      
         | 2685 |  |  | {
 | 
      
         | 2686 |  |  |   unsigned int u = 0;
 | 
      
         | 2687 |  |  |   int max_hv = -1;
 | 
      
         | 2688 |  |  |   int min_mob = INT_MAX;
 | 
      
         | 2689 |  |  |   int result = -1;
 | 
      
         | 2690 |  |  |   sbitmap_iterator sbi;
 | 
      
         | 2691 |  |  |  
 | 
      
         | 2692 |  |  |   EXECUTE_IF_SET_IN_SBITMAP (nodes, 0, u, sbi)
 | 
      
         | 2693 |  |  |     {
 | 
      
         | 2694 |  |  |       ddg_node_ptr u_node = &g->nodes[u];
 | 
      
         | 2695 |  |  |  
 | 
      
         | 2696 |  |  |       if (max_hv < HEIGHT (u_node))
 | 
      
         | 2697 |  |  |         {
 | 
      
         | 2698 |  |  |           max_hv = HEIGHT (u_node);
 | 
      
         | 2699 |  |  |           min_mob = MOB (u_node);
 | 
      
         | 2700 |  |  |           result = u;
 | 
      
         | 2701 |  |  |         }
 | 
      
         | 2702 |  |  |       else if ((max_hv == HEIGHT (u_node))
 | 
      
         | 2703 |  |  |                && (min_mob > MOB (u_node)))
 | 
      
         | 2704 |  |  |         {
 | 
      
         | 2705 |  |  |           min_mob = MOB (u_node);
 | 
      
         | 2706 |  |  |           result = u;
 | 
      
         | 2707 |  |  |         }
 | 
      
         | 2708 |  |  |     }
 | 
      
         | 2709 |  |  |   return result;
 | 
      
         | 2710 |  |  | }
 | 
      
         | 2711 |  |  |  
 | 
      
         | 2712 |  |  | static int
 | 
      
         | 2713 |  |  | find_max_dv_min_mob (ddg_ptr g, sbitmap nodes)
 | 
      
         | 2714 |  |  | {
 | 
      
         | 2715 |  |  |   unsigned int u = 0;
 | 
      
         | 2716 |  |  |   int max_dv = -1;
 | 
      
         | 2717 |  |  |   int min_mob = INT_MAX;
 | 
      
         | 2718 |  |  |   int result = -1;
 | 
      
         | 2719 |  |  |   sbitmap_iterator sbi;
 | 
      
         | 2720 |  |  |  
 | 
      
         | 2721 |  |  |   EXECUTE_IF_SET_IN_SBITMAP (nodes, 0, u, sbi)
 | 
      
         | 2722 |  |  |     {
 | 
      
         | 2723 |  |  |       ddg_node_ptr u_node = &g->nodes[u];
 | 
      
         | 2724 |  |  |  
 | 
      
         | 2725 |  |  |       if (max_dv < DEPTH (u_node))
 | 
      
         | 2726 |  |  |         {
 | 
      
         | 2727 |  |  |           max_dv = DEPTH (u_node);
 | 
      
         | 2728 |  |  |           min_mob = MOB (u_node);
 | 
      
         | 2729 |  |  |           result = u;
 | 
      
         | 2730 |  |  |         }
 | 
      
         | 2731 |  |  |       else if ((max_dv == DEPTH (u_node))
 | 
      
         | 2732 |  |  |                && (min_mob > MOB (u_node)))
 | 
      
         | 2733 |  |  |         {
 | 
      
         | 2734 |  |  |           min_mob = MOB (u_node);
 | 
      
         | 2735 |  |  |           result = u;
 | 
      
         | 2736 |  |  |         }
 | 
      
         | 2737 |  |  |     }
 | 
      
         | 2738 |  |  |   return result;
 | 
      
         | 2739 |  |  | }
 | 
      
         | 2740 |  |  |  
 | 
      
         | 2741 |  |  | /* Places the nodes of SCC into the NODE_ORDER array starting
 | 
      
         | 2742 |  |  |    at position POS, according to the SMS ordering algorithm.
 | 
      
         | 2743 |  |  |    NODES_ORDERED (in&out parameter) holds the bitset of all nodes in
 | 
      
         | 2744 |  |  |    the NODE_ORDER array, starting from position zero.  */
 | 
      
         | 2745 |  |  | static int
 | 
      
         | 2746 |  |  | order_nodes_in_scc (ddg_ptr g, sbitmap nodes_ordered, sbitmap scc,
 | 
      
         | 2747 |  |  |                     int * node_order, int pos)
 | 
      
         | 2748 |  |  | {
 | 
      
         | 2749 |  |  |   enum sms_direction dir;
 | 
      
         | 2750 |  |  |   int num_nodes = g->num_nodes;
 | 
      
         | 2751 |  |  |   sbitmap workset = sbitmap_alloc (num_nodes);
 | 
      
         | 2752 |  |  |   sbitmap tmp = sbitmap_alloc (num_nodes);
 | 
      
         | 2753 |  |  |   sbitmap zero_bitmap = sbitmap_alloc (num_nodes);
 | 
      
         | 2754 |  |  |   sbitmap predecessors = sbitmap_alloc (num_nodes);
 | 
      
         | 2755 |  |  |   sbitmap successors = sbitmap_alloc (num_nodes);
 | 
      
         | 2756 |  |  |  
 | 
      
         | 2757 |  |  |   sbitmap_zero (predecessors);
 | 
      
         | 2758 |  |  |   find_predecessors (predecessors, g, nodes_ordered);
 | 
      
         | 2759 |  |  |  
 | 
      
         | 2760 |  |  |   sbitmap_zero (successors);
 | 
      
         | 2761 |  |  |   find_successors (successors, g, nodes_ordered);
 | 
      
         | 2762 |  |  |  
 | 
      
         | 2763 |  |  |   sbitmap_zero (tmp);
 | 
      
         | 2764 |  |  |   if (sbitmap_a_and_b_cg (tmp, predecessors, scc))
 | 
      
         | 2765 |  |  |     {
 | 
      
         | 2766 |  |  |       sbitmap_copy (workset, tmp);
 | 
      
         | 2767 |  |  |       dir = BOTTOMUP;
 | 
      
         | 2768 |  |  |     }
 | 
      
         | 2769 |  |  |   else if (sbitmap_a_and_b_cg (tmp, successors, scc))
 | 
      
         | 2770 |  |  |     {
 | 
      
         | 2771 |  |  |       sbitmap_copy (workset, tmp);
 | 
      
         | 2772 |  |  |       dir = TOPDOWN;
 | 
      
         | 2773 |  |  |     }
 | 
      
         | 2774 |  |  |   else
 | 
      
         | 2775 |  |  |     {
 | 
      
         | 2776 |  |  |       int u;
 | 
      
         | 2777 |  |  |  
 | 
      
         | 2778 |  |  |       sbitmap_zero (workset);
 | 
      
         | 2779 |  |  |       if ((u = find_max_asap (g, scc)) >= 0)
 | 
      
         | 2780 |  |  |         SET_BIT (workset, u);
 | 
      
         | 2781 |  |  |       dir = BOTTOMUP;
 | 
      
         | 2782 |  |  |     }
 | 
      
         | 2783 |  |  |  
 | 
      
         | 2784 |  |  |   sbitmap_zero (zero_bitmap);
 | 
      
         | 2785 |  |  |   while (!sbitmap_equal (workset, zero_bitmap))
 | 
      
         | 2786 |  |  |     {
 | 
      
         | 2787 |  |  |       int v;
 | 
      
         | 2788 |  |  |       ddg_node_ptr v_node;
 | 
      
         | 2789 |  |  |       sbitmap v_node_preds;
 | 
      
         | 2790 |  |  |       sbitmap v_node_succs;
 | 
      
         | 2791 |  |  |  
 | 
      
         | 2792 |  |  |       if (dir == TOPDOWN)
 | 
      
         | 2793 |  |  |         {
 | 
      
         | 2794 |  |  |           while (!sbitmap_equal (workset, zero_bitmap))
 | 
      
         | 2795 |  |  |             {
 | 
      
         | 2796 |  |  |               v = find_max_hv_min_mob (g, workset);
 | 
      
         | 2797 |  |  |               v_node = &g->nodes[v];
 | 
      
         | 2798 |  |  |               node_order[pos++] = v;
 | 
      
         | 2799 |  |  |               v_node_succs = NODE_SUCCESSORS (v_node);
 | 
      
         | 2800 |  |  |               sbitmap_a_and_b (tmp, v_node_succs, scc);
 | 
      
         | 2801 |  |  |  
 | 
      
         | 2802 |  |  |               /* Don't consider the already ordered successors again.  */
 | 
      
         | 2803 |  |  |               sbitmap_difference (tmp, tmp, nodes_ordered);
 | 
      
         | 2804 |  |  |               sbitmap_a_or_b (workset, workset, tmp);
 | 
      
         | 2805 |  |  |               RESET_BIT (workset, v);
 | 
      
         | 2806 |  |  |               SET_BIT (nodes_ordered, v);
 | 
      
         | 2807 |  |  |             }
 | 
      
         | 2808 |  |  |           dir = BOTTOMUP;
 | 
      
         | 2809 |  |  |           sbitmap_zero (predecessors);
 | 
      
         | 2810 |  |  |           find_predecessors (predecessors, g, nodes_ordered);
 | 
      
         | 2811 |  |  |           sbitmap_a_and_b (workset, predecessors, scc);
 | 
      
         | 2812 |  |  |         }
 | 
      
         | 2813 |  |  |       else
 | 
      
         | 2814 |  |  |         {
 | 
      
         | 2815 |  |  |           while (!sbitmap_equal (workset, zero_bitmap))
 | 
      
         | 2816 |  |  |             {
 | 
      
         | 2817 |  |  |               v = find_max_dv_min_mob (g, workset);
 | 
      
         | 2818 |  |  |               v_node = &g->nodes[v];
 | 
      
         | 2819 |  |  |               node_order[pos++] = v;
 | 
      
         | 2820 |  |  |               v_node_preds = NODE_PREDECESSORS (v_node);
 | 
      
         | 2821 |  |  |               sbitmap_a_and_b (tmp, v_node_preds, scc);
 | 
      
         | 2822 |  |  |  
 | 
      
         | 2823 |  |  |               /* Don't consider the already ordered predecessors again.  */
 | 
      
         | 2824 |  |  |               sbitmap_difference (tmp, tmp, nodes_ordered);
 | 
      
         | 2825 |  |  |               sbitmap_a_or_b (workset, workset, tmp);
 | 
      
         | 2826 |  |  |               RESET_BIT (workset, v);
 | 
      
         | 2827 |  |  |               SET_BIT (nodes_ordered, v);
 | 
      
         | 2828 |  |  |             }
 | 
      
         | 2829 |  |  |           dir = TOPDOWN;
 | 
      
         | 2830 |  |  |           sbitmap_zero (successors);
 | 
      
         | 2831 |  |  |           find_successors (successors, g, nodes_ordered);
 | 
      
         | 2832 |  |  |           sbitmap_a_and_b (workset, successors, scc);
 | 
      
         | 2833 |  |  |         }
 | 
      
         | 2834 |  |  |     }
 | 
      
         | 2835 |  |  |   sbitmap_free (tmp);
 | 
      
         | 2836 |  |  |   sbitmap_free (workset);
 | 
      
         | 2837 |  |  |   sbitmap_free (zero_bitmap);
 | 
      
         | 2838 |  |  |   sbitmap_free (predecessors);
 | 
      
         | 2839 |  |  |   sbitmap_free (successors);
 | 
      
         | 2840 |  |  |   return pos;
 | 
      
         | 2841 |  |  | }
 | 
      
         | 2842 |  |  |  
 | 
      
         | 2843 |  |  |  
 | 
      
         | 2844 |  |  | /* This page contains functions for manipulating partial-schedules during
 | 
      
         | 2845 |  |  |    modulo scheduling.  */
 | 
      
         | 2846 |  |  |  
 | 
      
         | 2847 |  |  | /* Create a partial schedule and allocate a memory to hold II rows.  */
 | 
      
         | 2848 |  |  |  
 | 
      
         | 2849 |  |  | static partial_schedule_ptr
 | 
      
         | 2850 |  |  | create_partial_schedule (int ii, ddg_ptr g, int history)
 | 
      
         | 2851 |  |  | {
 | 
      
         | 2852 |  |  |   partial_schedule_ptr ps = XNEW (struct partial_schedule);
 | 
      
         | 2853 |  |  |   ps->rows = (ps_insn_ptr *) xcalloc (ii, sizeof (ps_insn_ptr));
 | 
      
         | 2854 |  |  |   ps->rows_length = (int *) xcalloc (ii, sizeof (int));
 | 
      
         | 2855 |  |  |   ps->reg_moves = NULL;
 | 
      
         | 2856 |  |  |   ps->ii = ii;
 | 
      
         | 2857 |  |  |   ps->history = history;
 | 
      
         | 2858 |  |  |   ps->min_cycle = INT_MAX;
 | 
      
         | 2859 |  |  |   ps->max_cycle = INT_MIN;
 | 
      
         | 2860 |  |  |   ps->g = g;
 | 
      
         | 2861 |  |  |  
 | 
      
         | 2862 |  |  |   return ps;
 | 
      
         | 2863 |  |  | }
 | 
      
         | 2864 |  |  |  
 | 
      
         | 2865 |  |  | /* Free the PS_INSNs in rows array of the given partial schedule.
 | 
      
         | 2866 |  |  |    ??? Consider caching the PS_INSN's.  */
 | 
      
         | 2867 |  |  | static void
 | 
      
         | 2868 |  |  | free_ps_insns (partial_schedule_ptr ps)
 | 
      
         | 2869 |  |  | {
 | 
      
         | 2870 |  |  |   int i;
 | 
      
         | 2871 |  |  |  
 | 
      
         | 2872 |  |  |   for (i = 0; i < ps->ii; i++)
 | 
      
         | 2873 |  |  |     {
 | 
      
         | 2874 |  |  |       while (ps->rows[i])
 | 
      
         | 2875 |  |  |         {
 | 
      
         | 2876 |  |  |           ps_insn_ptr ps_insn = ps->rows[i]->next_in_row;
 | 
      
         | 2877 |  |  |  
 | 
      
         | 2878 |  |  |           free (ps->rows[i]);
 | 
      
         | 2879 |  |  |           ps->rows[i] = ps_insn;
 | 
      
         | 2880 |  |  |         }
 | 
      
         | 2881 |  |  |       ps->rows[i] = NULL;
 | 
      
         | 2882 |  |  |     }
 | 
      
         | 2883 |  |  | }
 | 
      
         | 2884 |  |  |  
 | 
      
         | 2885 |  |  | /* Free all the memory allocated to the partial schedule.  */
 | 
      
         | 2886 |  |  |  
 | 
      
         | 2887 |  |  | static void
 | 
      
         | 2888 |  |  | free_partial_schedule (partial_schedule_ptr ps)
 | 
      
         | 2889 |  |  | {
 | 
      
         | 2890 |  |  |   ps_reg_move_info *move;
 | 
      
         | 2891 |  |  |   unsigned int i;
 | 
      
         | 2892 |  |  |  
 | 
      
         | 2893 |  |  |   if (!ps)
 | 
      
         | 2894 |  |  |     return;
 | 
      
         | 2895 |  |  |  
 | 
      
         | 2896 |  |  |   FOR_EACH_VEC_ELT (ps_reg_move_info, ps->reg_moves, i, move)
 | 
      
         | 2897 |  |  |     sbitmap_free (move->uses);
 | 
      
         | 2898 |  |  |   VEC_free (ps_reg_move_info, heap, ps->reg_moves);
 | 
      
         | 2899 |  |  |  
 | 
      
         | 2900 |  |  |   free_ps_insns (ps);
 | 
      
         | 2901 |  |  |   free (ps->rows);
 | 
      
         | 2902 |  |  |   free (ps->rows_length);
 | 
      
         | 2903 |  |  |   free (ps);
 | 
      
         | 2904 |  |  | }
 | 
      
         | 2905 |  |  |  
 | 
      
         | 2906 |  |  | /* Clear the rows array with its PS_INSNs, and create a new one with
 | 
      
         | 2907 |  |  |    NEW_II rows.  */
 | 
      
         | 2908 |  |  |  
 | 
      
         | 2909 |  |  | static void
 | 
      
         | 2910 |  |  | reset_partial_schedule (partial_schedule_ptr ps, int new_ii)
 | 
      
         | 2911 |  |  | {
 | 
      
         | 2912 |  |  |   if (!ps)
 | 
      
         | 2913 |  |  |     return;
 | 
      
         | 2914 |  |  |   free_ps_insns (ps);
 | 
      
         | 2915 |  |  |   if (new_ii == ps->ii)
 | 
      
         | 2916 |  |  |     return;
 | 
      
         | 2917 |  |  |   ps->rows = (ps_insn_ptr *) xrealloc (ps->rows, new_ii
 | 
      
         | 2918 |  |  |                                                  * sizeof (ps_insn_ptr));
 | 
      
         | 2919 |  |  |   memset (ps->rows, 0, new_ii * sizeof (ps_insn_ptr));
 | 
      
         | 2920 |  |  |   ps->rows_length = (int *) xrealloc (ps->rows_length, new_ii * sizeof (int));
 | 
      
         | 2921 |  |  |   memset (ps->rows_length, 0, new_ii * sizeof (int));
 | 
      
         | 2922 |  |  |   ps->ii = new_ii;
 | 
      
         | 2923 |  |  |   ps->min_cycle = INT_MAX;
 | 
      
         | 2924 |  |  |   ps->max_cycle = INT_MIN;
 | 
      
         | 2925 |  |  | }
 | 
      
         | 2926 |  |  |  
 | 
      
         | 2927 |  |  | /* Prints the partial schedule as an ii rows array, for each rows
 | 
      
         | 2928 |  |  |    print the ids of the insns in it.  */
 | 
      
         | 2929 |  |  | void
 | 
      
         | 2930 |  |  | print_partial_schedule (partial_schedule_ptr ps, FILE *dump)
 | 
      
         | 2931 |  |  | {
 | 
      
         | 2932 |  |  |   int i;
 | 
      
         | 2933 |  |  |  
 | 
      
         | 2934 |  |  |   for (i = 0; i < ps->ii; i++)
 | 
      
         | 2935 |  |  |     {
 | 
      
         | 2936 |  |  |       ps_insn_ptr ps_i = ps->rows[i];
 | 
      
         | 2937 |  |  |  
 | 
      
         | 2938 |  |  |       fprintf (dump, "\n[ROW %d ]: ", i);
 | 
      
         | 2939 |  |  |       while (ps_i)
 | 
      
         | 2940 |  |  |         {
 | 
      
         | 2941 |  |  |           rtx insn = ps_rtl_insn (ps, ps_i->id);
 | 
      
         | 2942 |  |  |  
 | 
      
         | 2943 |  |  |           if (JUMP_P (insn))
 | 
      
         | 2944 |  |  |             fprintf (dump, "%d (branch), ", INSN_UID (insn));
 | 
      
         | 2945 |  |  |           else
 | 
      
         | 2946 |  |  |             fprintf (dump, "%d, ", INSN_UID (insn));
 | 
      
         | 2947 |  |  |  
 | 
      
         | 2948 |  |  |           ps_i = ps_i->next_in_row;
 | 
      
         | 2949 |  |  |         }
 | 
      
         | 2950 |  |  |     }
 | 
      
         | 2951 |  |  | }
 | 
      
         | 2952 |  |  |  
 | 
      
         | 2953 |  |  | /* Creates an object of PS_INSN and initializes it to the given parameters.  */
 | 
      
         | 2954 |  |  | static ps_insn_ptr
 | 
      
         | 2955 |  |  | create_ps_insn (int id, int cycle)
 | 
      
         | 2956 |  |  | {
 | 
      
         | 2957 |  |  |   ps_insn_ptr ps_i = XNEW (struct ps_insn);
 | 
      
         | 2958 |  |  |  
 | 
      
         | 2959 |  |  |   ps_i->id = id;
 | 
      
         | 2960 |  |  |   ps_i->next_in_row = NULL;
 | 
      
         | 2961 |  |  |   ps_i->prev_in_row = NULL;
 | 
      
         | 2962 |  |  |   ps_i->cycle = cycle;
 | 
      
         | 2963 |  |  |  
 | 
      
         | 2964 |  |  |   return ps_i;
 | 
      
         | 2965 |  |  | }
 | 
      
         | 2966 |  |  |  
 | 
      
         | 2967 |  |  |  
 | 
      
         | 2968 |  |  | /* Removes the given PS_INSN from the partial schedule.  */
 | 
      
         | 2969 |  |  | static void
 | 
      
         | 2970 |  |  | remove_node_from_ps (partial_schedule_ptr ps, ps_insn_ptr ps_i)
 | 
      
         | 2971 |  |  | {
 | 
      
         | 2972 |  |  |   int row;
 | 
      
         | 2973 |  |  |  
 | 
      
         | 2974 |  |  |   gcc_assert (ps && ps_i);
 | 
      
         | 2975 |  |  |  
 | 
      
         | 2976 |  |  |   row = SMODULO (ps_i->cycle, ps->ii);
 | 
      
         | 2977 |  |  |   if (! ps_i->prev_in_row)
 | 
      
         | 2978 |  |  |     {
 | 
      
         | 2979 |  |  |       gcc_assert (ps_i == ps->rows[row]);
 | 
      
         | 2980 |  |  |       ps->rows[row] = ps_i->next_in_row;
 | 
      
         | 2981 |  |  |       if (ps->rows[row])
 | 
      
         | 2982 |  |  |         ps->rows[row]->prev_in_row = NULL;
 | 
      
         | 2983 |  |  |     }
 | 
      
         | 2984 |  |  |   else
 | 
      
         | 2985 |  |  |     {
 | 
      
         | 2986 |  |  |       ps_i->prev_in_row->next_in_row = ps_i->next_in_row;
 | 
      
         | 2987 |  |  |       if (ps_i->next_in_row)
 | 
      
         | 2988 |  |  |         ps_i->next_in_row->prev_in_row = ps_i->prev_in_row;
 | 
      
         | 2989 |  |  |     }
 | 
      
         | 2990 |  |  |  
 | 
      
         | 2991 |  |  |   ps->rows_length[row] -= 1;
 | 
      
         | 2992 |  |  |   free (ps_i);
 | 
      
         | 2993 |  |  |   return;
 | 
      
         | 2994 |  |  | }
 | 
      
         | 2995 |  |  |  
 | 
      
         | 2996 |  |  | /* Unlike what literature describes for modulo scheduling (which focuses
 | 
      
         | 2997 |  |  |    on VLIW machines) the order of the instructions inside a cycle is
 | 
      
         | 2998 |  |  |    important.  Given the bitmaps MUST_FOLLOW and MUST_PRECEDE we know
 | 
      
         | 2999 |  |  |    where the current instruction should go relative to the already
 | 
      
         | 3000 |  |  |    scheduled instructions in the given cycle.  Go over these
 | 
      
         | 3001 |  |  |    instructions and find the first possible column to put it in.  */
 | 
      
         | 3002 |  |  | static bool
 | 
      
         | 3003 |  |  | ps_insn_find_column (partial_schedule_ptr ps, ps_insn_ptr ps_i,
 | 
      
         | 3004 |  |  |                      sbitmap must_precede, sbitmap must_follow)
 | 
      
         | 3005 |  |  | {
 | 
      
         | 3006 |  |  |   ps_insn_ptr next_ps_i;
 | 
      
         | 3007 |  |  |   ps_insn_ptr first_must_follow = NULL;
 | 
      
         | 3008 |  |  |   ps_insn_ptr last_must_precede = NULL;
 | 
      
         | 3009 |  |  |   ps_insn_ptr last_in_row = NULL;
 | 
      
         | 3010 |  |  |   int row;
 | 
      
         | 3011 |  |  |  
 | 
      
         | 3012 |  |  |   if (! ps_i)
 | 
      
         | 3013 |  |  |     return false;
 | 
      
         | 3014 |  |  |  
 | 
      
         | 3015 |  |  |   row = SMODULO (ps_i->cycle, ps->ii);
 | 
      
         | 3016 |  |  |  
 | 
      
         | 3017 |  |  |   /* Find the first must follow and the last must precede
 | 
      
         | 3018 |  |  |      and insert the node immediately after the must precede
 | 
      
         | 3019 |  |  |      but make sure that it there is no must follow after it.  */
 | 
      
         | 3020 |  |  |   for (next_ps_i = ps->rows[row];
 | 
      
         | 3021 |  |  |        next_ps_i;
 | 
      
         | 3022 |  |  |        next_ps_i = next_ps_i->next_in_row)
 | 
      
         | 3023 |  |  |     {
 | 
      
         | 3024 |  |  |       if (must_follow
 | 
      
         | 3025 |  |  |           && TEST_BIT (must_follow, next_ps_i->id)
 | 
      
         | 3026 |  |  |           && ! first_must_follow)
 | 
      
         | 3027 |  |  |         first_must_follow = next_ps_i;
 | 
      
         | 3028 |  |  |       if (must_precede && TEST_BIT (must_precede, next_ps_i->id))
 | 
      
         | 3029 |  |  |         {
 | 
      
         | 3030 |  |  |           /* If we have already met a node that must follow, then
 | 
      
         | 3031 |  |  |              there is no possible column.  */
 | 
      
         | 3032 |  |  |           if (first_must_follow)
 | 
      
         | 3033 |  |  |             return false;
 | 
      
         | 3034 |  |  |           else
 | 
      
         | 3035 |  |  |             last_must_precede = next_ps_i;
 | 
      
         | 3036 |  |  |         }
 | 
      
         | 3037 |  |  |       /* The closing branch must be the last in the row.  */
 | 
      
         | 3038 |  |  |       if (must_precede
 | 
      
         | 3039 |  |  |           && TEST_BIT (must_precede, next_ps_i->id)
 | 
      
         | 3040 |  |  |           && JUMP_P (ps_rtl_insn (ps, next_ps_i->id)))
 | 
      
         | 3041 |  |  |         return false;
 | 
      
         | 3042 |  |  |  
 | 
      
         | 3043 |  |  |        last_in_row = next_ps_i;
 | 
      
         | 3044 |  |  |     }
 | 
      
         | 3045 |  |  |  
 | 
      
         | 3046 |  |  |   /* The closing branch is scheduled as well.  Make sure there is no
 | 
      
         | 3047 |  |  |      dependent instruction after it as the branch should be the last
 | 
      
         | 3048 |  |  |      instruction in the row.  */
 | 
      
         | 3049 |  |  |   if (JUMP_P (ps_rtl_insn (ps, ps_i->id)))
 | 
      
         | 3050 |  |  |     {
 | 
      
         | 3051 |  |  |       if (first_must_follow)
 | 
      
         | 3052 |  |  |         return false;
 | 
      
         | 3053 |  |  |       if (last_in_row)
 | 
      
         | 3054 |  |  |         {
 | 
      
         | 3055 |  |  |           /* Make the branch the last in the row.  New instructions
 | 
      
         | 3056 |  |  |              will be inserted at the beginning of the row or after the
 | 
      
         | 3057 |  |  |              last must_precede instruction thus the branch is guaranteed
 | 
      
         | 3058 |  |  |              to remain the last instruction in the row.  */
 | 
      
         | 3059 |  |  |           last_in_row->next_in_row = ps_i;
 | 
      
         | 3060 |  |  |           ps_i->prev_in_row = last_in_row;
 | 
      
         | 3061 |  |  |           ps_i->next_in_row = NULL;
 | 
      
         | 3062 |  |  |         }
 | 
      
         | 3063 |  |  |       else
 | 
      
         | 3064 |  |  |         ps->rows[row] = ps_i;
 | 
      
         | 3065 |  |  |       return true;
 | 
      
         | 3066 |  |  |     }
 | 
      
         | 3067 |  |  |  
 | 
      
         | 3068 |  |  |   /* Now insert the node after INSERT_AFTER_PSI.  */
 | 
      
         | 3069 |  |  |  
 | 
      
         | 3070 |  |  |   if (! last_must_precede)
 | 
      
         | 3071 |  |  |     {
 | 
      
         | 3072 |  |  |       ps_i->next_in_row = ps->rows[row];
 | 
      
         | 3073 |  |  |       ps_i->prev_in_row = NULL;
 | 
      
         | 3074 |  |  |       if (ps_i->next_in_row)
 | 
      
         | 3075 |  |  |         ps_i->next_in_row->prev_in_row = ps_i;
 | 
      
         | 3076 |  |  |       ps->rows[row] = ps_i;
 | 
      
         | 3077 |  |  |     }
 | 
      
         | 3078 |  |  |   else
 | 
      
         | 3079 |  |  |     {
 | 
      
         | 3080 |  |  |       ps_i->next_in_row = last_must_precede->next_in_row;
 | 
      
         | 3081 |  |  |       last_must_precede->next_in_row = ps_i;
 | 
      
         | 3082 |  |  |       ps_i->prev_in_row = last_must_precede;
 | 
      
         | 3083 |  |  |       if (ps_i->next_in_row)
 | 
      
         | 3084 |  |  |         ps_i->next_in_row->prev_in_row = ps_i;
 | 
      
         | 3085 |  |  |     }
 | 
      
         | 3086 |  |  |  
 | 
      
         | 3087 |  |  |   return true;
 | 
      
         | 3088 |  |  | }
 | 
      
         | 3089 |  |  |  
 | 
      
         | 3090 |  |  | /* Advances the PS_INSN one column in its current row; returns false
 | 
      
         | 3091 |  |  |    in failure and true in success.  Bit N is set in MUST_FOLLOW if
 | 
      
         | 3092 |  |  |    the node with cuid N must be come after the node pointed to by
 | 
      
         | 3093 |  |  |    PS_I when scheduled in the same cycle.  */
 | 
      
         | 3094 |  |  | static int
 | 
      
         | 3095 |  |  | ps_insn_advance_column (partial_schedule_ptr ps, ps_insn_ptr ps_i,
 | 
      
         | 3096 |  |  |                         sbitmap must_follow)
 | 
      
         | 3097 |  |  | {
 | 
      
         | 3098 |  |  |   ps_insn_ptr prev, next;
 | 
      
         | 3099 |  |  |   int row;
 | 
      
         | 3100 |  |  |  
 | 
      
         | 3101 |  |  |   if (!ps || !ps_i)
 | 
      
         | 3102 |  |  |     return false;
 | 
      
         | 3103 |  |  |  
 | 
      
         | 3104 |  |  |   row = SMODULO (ps_i->cycle, ps->ii);
 | 
      
         | 3105 |  |  |  
 | 
      
         | 3106 |  |  |   if (! ps_i->next_in_row)
 | 
      
         | 3107 |  |  |     return false;
 | 
      
         | 3108 |  |  |  
 | 
      
         | 3109 |  |  |   /* Check if next_in_row is dependent on ps_i, both having same sched
 | 
      
         | 3110 |  |  |      times (typically ANTI_DEP).  If so, ps_i cannot skip over it.  */
 | 
      
         | 3111 |  |  |   if (must_follow && TEST_BIT (must_follow, ps_i->next_in_row->id))
 | 
      
         | 3112 |  |  |     return false;
 | 
      
         | 3113 |  |  |  
 | 
      
         | 3114 |  |  |   /* Advance PS_I over its next_in_row in the doubly linked list.  */
 | 
      
         | 3115 |  |  |   prev = ps_i->prev_in_row;
 | 
      
         | 3116 |  |  |   next = ps_i->next_in_row;
 | 
      
         | 3117 |  |  |  
 | 
      
         | 3118 |  |  |   if (ps_i == ps->rows[row])
 | 
      
         | 3119 |  |  |     ps->rows[row] = next;
 | 
      
         | 3120 |  |  |  
 | 
      
         | 3121 |  |  |   ps_i->next_in_row = next->next_in_row;
 | 
      
         | 3122 |  |  |  
 | 
      
         | 3123 |  |  |   if (next->next_in_row)
 | 
      
         | 3124 |  |  |     next->next_in_row->prev_in_row = ps_i;
 | 
      
         | 3125 |  |  |  
 | 
      
         | 3126 |  |  |   next->next_in_row = ps_i;
 | 
      
         | 3127 |  |  |   ps_i->prev_in_row = next;
 | 
      
         | 3128 |  |  |  
 | 
      
         | 3129 |  |  |   next->prev_in_row = prev;
 | 
      
         | 3130 |  |  |   if (prev)
 | 
      
         | 3131 |  |  |     prev->next_in_row = next;
 | 
      
         | 3132 |  |  |  
 | 
      
         | 3133 |  |  |   return true;
 | 
      
         | 3134 |  |  | }
 | 
      
         | 3135 |  |  |  
 | 
      
         | 3136 |  |  | /* Inserts a DDG_NODE to the given partial schedule at the given cycle.
 | 
      
         | 3137 |  |  |    Returns 0 if this is not possible and a PS_INSN otherwise.  Bit N is
 | 
      
         | 3138 |  |  |    set in MUST_PRECEDE/MUST_FOLLOW if the node with cuid N must be come
 | 
      
         | 3139 |  |  |    before/after (respectively) the node pointed to by PS_I when scheduled
 | 
      
         | 3140 |  |  |    in the same cycle.  */
 | 
      
         | 3141 |  |  | static ps_insn_ptr
 | 
      
         | 3142 |  |  | add_node_to_ps (partial_schedule_ptr ps, int id, int cycle,
 | 
      
         | 3143 |  |  |                 sbitmap must_precede, sbitmap must_follow)
 | 
      
         | 3144 |  |  | {
 | 
      
         | 3145 |  |  |   ps_insn_ptr ps_i;
 | 
      
         | 3146 |  |  |   int row = SMODULO (cycle, ps->ii);
 | 
      
         | 3147 |  |  |  
 | 
      
         | 3148 |  |  |   if (ps->rows_length[row] >= issue_rate)
 | 
      
         | 3149 |  |  |     return NULL;
 | 
      
         | 3150 |  |  |  
 | 
      
         | 3151 |  |  |   ps_i = create_ps_insn (id, cycle);
 | 
      
         | 3152 |  |  |  
 | 
      
         | 3153 |  |  |   /* Finds and inserts PS_I according to MUST_FOLLOW and
 | 
      
         | 3154 |  |  |      MUST_PRECEDE.  */
 | 
      
         | 3155 |  |  |   if (! ps_insn_find_column (ps, ps_i, must_precede, must_follow))
 | 
      
         | 3156 |  |  |     {
 | 
      
         | 3157 |  |  |       free (ps_i);
 | 
      
         | 3158 |  |  |       return NULL;
 | 
      
         | 3159 |  |  |     }
 | 
      
         | 3160 |  |  |  
 | 
      
         | 3161 |  |  |   ps->rows_length[row] += 1;
 | 
      
         | 3162 |  |  |   return ps_i;
 | 
      
         | 3163 |  |  | }
 | 
      
         | 3164 |  |  |  
 | 
      
         | 3165 |  |  | /* Advance time one cycle.  Assumes DFA is being used.  */
 | 
      
         | 3166 |  |  | static void
 | 
      
         | 3167 |  |  | advance_one_cycle (void)
 | 
      
         | 3168 |  |  | {
 | 
      
         | 3169 |  |  |   if (targetm.sched.dfa_pre_cycle_insn)
 | 
      
         | 3170 |  |  |     state_transition (curr_state,
 | 
      
         | 3171 |  |  |                       targetm.sched.dfa_pre_cycle_insn ());
 | 
      
         | 3172 |  |  |  
 | 
      
         | 3173 |  |  |   state_transition (curr_state, NULL);
 | 
      
         | 3174 |  |  |  
 | 
      
         | 3175 |  |  |   if (targetm.sched.dfa_post_cycle_insn)
 | 
      
         | 3176 |  |  |     state_transition (curr_state,
 | 
      
         | 3177 |  |  |                       targetm.sched.dfa_post_cycle_insn ());
 | 
      
         | 3178 |  |  | }
 | 
      
         | 3179 |  |  |  
 | 
      
         | 3180 |  |  |  
 | 
      
         | 3181 |  |  |  
 | 
      
         | 3182 |  |  | /* Checks if PS has resource conflicts according to DFA, starting from
 | 
      
         | 3183 |  |  |    FROM cycle to TO cycle; returns true if there are conflicts and false
 | 
      
         | 3184 |  |  |    if there are no conflicts.  Assumes DFA is being used.  */
 | 
      
         | 3185 |  |  | static int
 | 
      
         | 3186 |  |  | ps_has_conflicts (partial_schedule_ptr ps, int from, int to)
 | 
      
         | 3187 |  |  | {
 | 
      
         | 3188 |  |  |   int cycle;
 | 
      
         | 3189 |  |  |  
 | 
      
         | 3190 |  |  |   state_reset (curr_state);
 | 
      
         | 3191 |  |  |  
 | 
      
         | 3192 |  |  |   for (cycle = from; cycle <= to; cycle++)
 | 
      
         | 3193 |  |  |     {
 | 
      
         | 3194 |  |  |       ps_insn_ptr crr_insn;
 | 
      
         | 3195 |  |  |       /* Holds the remaining issue slots in the current row.  */
 | 
      
         | 3196 |  |  |       int can_issue_more = issue_rate;
 | 
      
         | 3197 |  |  |  
 | 
      
         | 3198 |  |  |       /* Walk through the DFA for the current row.  */
 | 
      
         | 3199 |  |  |       for (crr_insn = ps->rows[SMODULO (cycle, ps->ii)];
 | 
      
         | 3200 |  |  |            crr_insn;
 | 
      
         | 3201 |  |  |            crr_insn = crr_insn->next_in_row)
 | 
      
         | 3202 |  |  |         {
 | 
      
         | 3203 |  |  |           rtx insn = ps_rtl_insn (ps, crr_insn->id);
 | 
      
         | 3204 |  |  |  
 | 
      
         | 3205 |  |  |           if (!NONDEBUG_INSN_P (insn))
 | 
      
         | 3206 |  |  |             continue;
 | 
      
         | 3207 |  |  |  
 | 
      
         | 3208 |  |  |           /* Check if there is room for the current insn.  */
 | 
      
         | 3209 |  |  |           if (!can_issue_more || state_dead_lock_p (curr_state))
 | 
      
         | 3210 |  |  |             return true;
 | 
      
         | 3211 |  |  |  
 | 
      
         | 3212 |  |  |           /* Update the DFA state and return with failure if the DFA found
 | 
      
         | 3213 |  |  |              resource conflicts.  */
 | 
      
         | 3214 |  |  |           if (state_transition (curr_state, insn) >= 0)
 | 
      
         | 3215 |  |  |             return true;
 | 
      
         | 3216 |  |  |  
 | 
      
         | 3217 |  |  |           if (targetm.sched.variable_issue)
 | 
      
         | 3218 |  |  |             can_issue_more =
 | 
      
         | 3219 |  |  |               targetm.sched.variable_issue (sched_dump, sched_verbose,
 | 
      
         | 3220 |  |  |                                             insn, can_issue_more);
 | 
      
         | 3221 |  |  |           /* A naked CLOBBER or USE generates no instruction, so don't
 | 
      
         | 3222 |  |  |              let them consume issue slots.  */
 | 
      
         | 3223 |  |  |           else if (GET_CODE (PATTERN (insn)) != USE
 | 
      
         | 3224 |  |  |                    && GET_CODE (PATTERN (insn)) != CLOBBER)
 | 
      
         | 3225 |  |  |             can_issue_more--;
 | 
      
         | 3226 |  |  |         }
 | 
      
         | 3227 |  |  |  
 | 
      
         | 3228 |  |  |       /* Advance the DFA to the next cycle.  */
 | 
      
         | 3229 |  |  |       advance_one_cycle ();
 | 
      
         | 3230 |  |  |     }
 | 
      
         | 3231 |  |  |   return false;
 | 
      
         | 3232 |  |  | }
 | 
      
         | 3233 |  |  |  
 | 
      
         | 3234 |  |  | /* Checks if the given node causes resource conflicts when added to PS at
 | 
      
         | 3235 |  |  |    cycle C.  If not the node is added to PS and returned; otherwise zero
 | 
      
         | 3236 |  |  |    is returned.  Bit N is set in MUST_PRECEDE/MUST_FOLLOW if the node with
 | 
      
         | 3237 |  |  |    cuid N must be come before/after (respectively) the node pointed to by
 | 
      
         | 3238 |  |  |    PS_I when scheduled in the same cycle.  */
 | 
      
         | 3239 |  |  | ps_insn_ptr
 | 
      
         | 3240 |  |  | ps_add_node_check_conflicts (partial_schedule_ptr ps, int n,
 | 
      
         | 3241 |  |  |                              int c, sbitmap must_precede,
 | 
      
         | 3242 |  |  |                              sbitmap must_follow)
 | 
      
         | 3243 |  |  | {
 | 
      
         | 3244 |  |  |   int has_conflicts = 0;
 | 
      
         | 3245 |  |  |   ps_insn_ptr ps_i;
 | 
      
         | 3246 |  |  |  
 | 
      
         | 3247 |  |  |   /* First add the node to the PS, if this succeeds check for
 | 
      
         | 3248 |  |  |      conflicts, trying different issue slots in the same row.  */
 | 
      
         | 3249 |  |  |   if (! (ps_i = add_node_to_ps (ps, n, c, must_precede, must_follow)))
 | 
      
         | 3250 |  |  |     return NULL; /* Failed to insert the node at the given cycle.  */
 | 
      
         | 3251 |  |  |  
 | 
      
         | 3252 |  |  |   has_conflicts = ps_has_conflicts (ps, c, c)
 | 
      
         | 3253 |  |  |                   || (ps->history > 0
 | 
      
         | 3254 |  |  |                       && ps_has_conflicts (ps,
 | 
      
         | 3255 |  |  |                                            c - ps->history,
 | 
      
         | 3256 |  |  |                                            c + ps->history));
 | 
      
         | 3257 |  |  |  
 | 
      
         | 3258 |  |  |   /* Try different issue slots to find one that the given node can be
 | 
      
         | 3259 |  |  |      scheduled in without conflicts.  */
 | 
      
         | 3260 |  |  |   while (has_conflicts)
 | 
      
         | 3261 |  |  |     {
 | 
      
         | 3262 |  |  |       if (! ps_insn_advance_column (ps, ps_i, must_follow))
 | 
      
         | 3263 |  |  |         break;
 | 
      
         | 3264 |  |  |       has_conflicts = ps_has_conflicts (ps, c, c)
 | 
      
         | 3265 |  |  |                       || (ps->history > 0
 | 
      
         | 3266 |  |  |                           && ps_has_conflicts (ps,
 | 
      
         | 3267 |  |  |                                                c - ps->history,
 | 
      
         | 3268 |  |  |                                                c + ps->history));
 | 
      
         | 3269 |  |  |     }
 | 
      
         | 3270 |  |  |  
 | 
      
         | 3271 |  |  |   if (has_conflicts)
 | 
      
         | 3272 |  |  |     {
 | 
      
         | 3273 |  |  |       remove_node_from_ps (ps, ps_i);
 | 
      
         | 3274 |  |  |       return NULL;
 | 
      
         | 3275 |  |  |     }
 | 
      
         | 3276 |  |  |  
 | 
      
         | 3277 |  |  |   ps->min_cycle = MIN (ps->min_cycle, c);
 | 
      
         | 3278 |  |  |   ps->max_cycle = MAX (ps->max_cycle, c);
 | 
      
         | 3279 |  |  |   return ps_i;
 | 
      
         | 3280 |  |  | }
 | 
      
         | 3281 |  |  |  
 | 
      
         | 3282 |  |  | /* Calculate the stage count of the partial schedule PS.  The calculation
 | 
      
         | 3283 |  |  |    takes into account the rotation amount passed in ROTATION_AMOUNT.  */
 | 
      
         | 3284 |  |  | int
 | 
      
         | 3285 |  |  | calculate_stage_count (partial_schedule_ptr ps, int rotation_amount)
 | 
      
         | 3286 |  |  | {
 | 
      
         | 3287 |  |  |   int new_min_cycle = PS_MIN_CYCLE (ps) - rotation_amount;
 | 
      
         | 3288 |  |  |   int new_max_cycle = PS_MAX_CYCLE (ps) - rotation_amount;
 | 
      
         | 3289 |  |  |   int stage_count = CALC_STAGE_COUNT (-1, new_min_cycle, ps->ii);
 | 
      
         | 3290 |  |  |  
 | 
      
         | 3291 |  |  |   /* The calculation of stage count is done adding the number of stages
 | 
      
         | 3292 |  |  |      before cycle zero and after cycle zero.  */
 | 
      
         | 3293 |  |  |   stage_count += CALC_STAGE_COUNT (new_max_cycle, 0, ps->ii);
 | 
      
         | 3294 |  |  |  
 | 
      
         | 3295 |  |  |   return stage_count;
 | 
      
         | 3296 |  |  | }
 | 
      
         | 3297 |  |  |  
 | 
      
         | 3298 |  |  | /* Rotate the rows of PS such that insns scheduled at time
 | 
      
         | 3299 |  |  |    START_CYCLE will appear in row 0.  Updates max/min_cycles.  */
 | 
      
         | 3300 |  |  | void
 | 
      
         | 3301 |  |  | rotate_partial_schedule (partial_schedule_ptr ps, int start_cycle)
 | 
      
         | 3302 |  |  | {
 | 
      
         | 3303 |  |  |   int i, row, backward_rotates;
 | 
      
         | 3304 |  |  |   int last_row = ps->ii - 1;
 | 
      
         | 3305 |  |  |  
 | 
      
         | 3306 |  |  |   if (start_cycle == 0)
 | 
      
         | 3307 |  |  |     return;
 | 
      
         | 3308 |  |  |  
 | 
      
         | 3309 |  |  |   backward_rotates = SMODULO (start_cycle, ps->ii);
 | 
      
         | 3310 |  |  |  
 | 
      
         | 3311 |  |  |   /* Revisit later and optimize this into a single loop.  */
 | 
      
         | 3312 |  |  |   for (i = 0; i < backward_rotates; i++)
 | 
      
         | 3313 |  |  |     {
 | 
      
         | 3314 |  |  |       ps_insn_ptr first_row = ps->rows[0];
 | 
      
         | 3315 |  |  |       int first_row_length = ps->rows_length[0];
 | 
      
         | 3316 |  |  |  
 | 
      
         | 3317 |  |  |       for (row = 0; row < last_row; row++)
 | 
      
         | 3318 |  |  |         {
 | 
      
         | 3319 |  |  |           ps->rows[row] = ps->rows[row + 1];
 | 
      
         | 3320 |  |  |           ps->rows_length[row] = ps->rows_length[row + 1];
 | 
      
         | 3321 |  |  |         }
 | 
      
         | 3322 |  |  |  
 | 
      
         | 3323 |  |  |       ps->rows[last_row] = first_row;
 | 
      
         | 3324 |  |  |       ps->rows_length[last_row] = first_row_length;
 | 
      
         | 3325 |  |  |     }
 | 
      
         | 3326 |  |  |  
 | 
      
         | 3327 |  |  |   ps->max_cycle -= start_cycle;
 | 
      
         | 3328 |  |  |   ps->min_cycle -= start_cycle;
 | 
      
         | 3329 |  |  | }
 | 
      
         | 3330 |  |  |  
 | 
      
         | 3331 |  |  | #endif /* INSN_SCHEDULING */
 | 
      
         | 3332 |  |  |  
 | 
      
         | 3333 |  |  | static bool
 | 
      
         | 3334 |  |  | gate_handle_sms (void)
 | 
      
         | 3335 |  |  | {
 | 
      
         | 3336 |  |  |   return (optimize > 0 && flag_modulo_sched);
 | 
      
         | 3337 |  |  | }
 | 
      
         | 3338 |  |  |  
 | 
      
         | 3339 |  |  |  
 | 
      
         | 3340 |  |  | /* Run instruction scheduler.  */
 | 
      
         | 3341 |  |  | /* Perform SMS module scheduling.  */
 | 
      
         | 3342 |  |  | static unsigned int
 | 
      
         | 3343 |  |  | rest_of_handle_sms (void)
 | 
      
         | 3344 |  |  | {
 | 
      
         | 3345 |  |  | #ifdef INSN_SCHEDULING
 | 
      
         | 3346 |  |  |   basic_block bb;
 | 
      
         | 3347 |  |  |  
 | 
      
         | 3348 |  |  |   /* Collect loop information to be used in SMS.  */
 | 
      
         | 3349 |  |  |   cfg_layout_initialize (0);
 | 
      
         | 3350 |  |  |   sms_schedule ();
 | 
      
         | 3351 |  |  |  
 | 
      
         | 3352 |  |  |   /* Update the life information, because we add pseudos.  */
 | 
      
         | 3353 |  |  |   max_regno = max_reg_num ();
 | 
      
         | 3354 |  |  |  
 | 
      
         | 3355 |  |  |   /* Finalize layout changes.  */
 | 
      
         | 3356 |  |  |   FOR_EACH_BB (bb)
 | 
      
         | 3357 |  |  |     if (bb->next_bb != EXIT_BLOCK_PTR)
 | 
      
         | 3358 |  |  |       bb->aux = bb->next_bb;
 | 
      
         | 3359 |  |  |   free_dominance_info (CDI_DOMINATORS);
 | 
      
         | 3360 |  |  |   cfg_layout_finalize ();
 | 
      
         | 3361 |  |  | #endif /* INSN_SCHEDULING */
 | 
      
         | 3362 |  |  |   return 0;
 | 
      
         | 3363 |  |  | }
 | 
      
         | 3364 |  |  |  
 | 
      
         | 3365 |  |  | struct rtl_opt_pass pass_sms =
 | 
      
         | 3366 |  |  | {
 | 
      
         | 3367 |  |  |  {
 | 
      
         | 3368 |  |  |   RTL_PASS,
 | 
      
         | 3369 |  |  |   "sms",                                /* name */
 | 
      
         | 3370 |  |  |   gate_handle_sms,                      /* gate */
 | 
      
         | 3371 |  |  |   rest_of_handle_sms,                   /* execute */
 | 
      
         | 3372 |  |  |   NULL,                                 /* sub */
 | 
      
         | 3373 |  |  |   NULL,                                 /* next */
 | 
      
         | 3374 |  |  |   0,                                    /* static_pass_number */
 | 
      
         | 3375 |  |  |   TV_SMS,                               /* tv_id */
 | 
      
         | 3376 |  |  |   0,                                    /* properties_required */
 | 
      
         | 3377 |  |  |   0,                                    /* properties_provided */
 | 
      
         | 3378 |  |  |   0,                                    /* properties_destroyed */
 | 
      
         | 3379 |  |  |   0,                                    /* todo_flags_start */
 | 
      
         | 3380 |  |  |   TODO_df_finish
 | 
      
         | 3381 |  |  |     | TODO_verify_flow
 | 
      
         | 3382 |  |  |     | TODO_verify_rtl_sharing
 | 
      
         | 3383 |  |  |     | TODO_ggc_collect                  /* todo_flags_finish */
 | 
      
         | 3384 |  |  |  }
 | 
      
         | 3385 |  |  | };
 |