OpenCores
URL https://opencores.org/ocsvn/openrisc/openrisc/trunk

Subversion Repositories openrisc

[/] [openrisc/] [trunk/] [gnu-dev/] [or1k-gcc/] [gcc/] [sel-sched.c] - Blame information for rev 746

Go to most recent revision | Details | Compare with Previous | View Log

Line No. Rev Author Line
1 684 jeremybenn
/* Instruction scheduling pass.  Selective scheduler and pipeliner.
2
   Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011
3
   Free Software Foundation, Inc.
4
 
5
This file is part of GCC.
6
 
7
GCC is free software; you can redistribute it and/or modify it under
8
the terms of the GNU General Public License as published by the Free
9
Software Foundation; either version 3, or (at your option) any later
10
version.
11
 
12
GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13
WARRANTY; without even the implied warranty of MERCHANTABILITY or
14
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
15
for more details.
16
 
17
You should have received a copy of the GNU General Public License
18
along with GCC; see the file COPYING3.  If not see
19
<http://www.gnu.org/licenses/>.  */
20
 
21
#include "config.h"
22
#include "system.h"
23
#include "coretypes.h"
24
#include "tm.h"
25
#include "rtl-error.h"
26
#include "tm_p.h"
27
#include "hard-reg-set.h"
28
#include "regs.h"
29
#include "function.h"
30
#include "flags.h"
31
#include "insn-config.h"
32
#include "insn-attr.h"
33
#include "except.h"
34
#include "recog.h"
35
#include "params.h"
36
#include "target.h"
37
#include "output.h"
38
#include "timevar.h"
39
#include "tree-pass.h"
40
#include "sched-int.h"
41
#include "ggc.h"
42
#include "tree.h"
43
#include "vec.h"
44
#include "langhooks.h"
45
#include "rtlhooks-def.h"
46
#include "output.h"
47
#include "emit-rtl.h"
48
 
49
#ifdef INSN_SCHEDULING
50
#include "sel-sched-ir.h"
51
#include "sel-sched-dump.h"
52
#include "sel-sched.h"
53
#include "dbgcnt.h"
54
 
55
/* Implementation of selective scheduling approach.
56
   The below implementation follows the original approach with the following
57
   changes:
58
 
59
   o the scheduler works after register allocation (but can be also tuned
60
   to work before RA);
61
   o some instructions are not copied or register renamed;
62
   o conditional jumps are not moved with code duplication;
63
   o several jumps in one parallel group are not supported;
64
   o when pipelining outer loops, code motion through inner loops
65
   is not supported;
66
   o control and data speculation are supported;
67
   o some improvements for better compile time/performance were made.
68
 
69
   Terminology
70
   ===========
71
 
72
   A vinsn, or virtual insn, is an insn with additional data characterizing
73
   insn pattern, such as LHS, RHS, register sets used/set/clobbered, etc.
74
   Vinsns also act as smart pointers to save memory by reusing them in
75
   different expressions.  A vinsn is described by vinsn_t type.
76
 
77
   An expression is a vinsn with additional data characterizing its properties
78
   at some point in the control flow graph.  The data may be its usefulness,
79
   priority, speculative status, whether it was renamed/subsituted, etc.
80
   An expression is described by expr_t type.
81
 
82
   Availability set (av_set) is a set of expressions at a given control flow
83
   point. It is represented as av_set_t.  The expressions in av sets are kept
84
   sorted in the terms of expr_greater_p function.  It allows to truncate
85
   the set while leaving the best expressions.
86
 
87
   A fence is a point through which code motion is prohibited.  On each step,
88
   we gather a parallel group of insns at a fence.  It is possible to have
89
   multiple fences. A fence is represented via fence_t.
90
 
91
   A boundary is the border between the fence group and the rest of the code.
92
   Currently, we never have more than one boundary per fence, as we finalize
93
   the fence group when a jump is scheduled. A boundary is represented
94
   via bnd_t.
95
 
96
   High-level overview
97
   ===================
98
 
99
   The scheduler finds regions to schedule, schedules each one, and finalizes.
100
   The regions are formed starting from innermost loops, so that when the inner
101
   loop is pipelined, its prologue can be scheduled together with yet unprocessed
102
   outer loop. The rest of acyclic regions are found using extend_rgns:
103
   the blocks that are not yet allocated to any regions are traversed in top-down
104
   order, and a block is added to a region to which all its predecessors belong;
105
   otherwise, the block starts its own region.
106
 
107
   The main scheduling loop (sel_sched_region_2) consists of just
108
   scheduling on each fence and updating fences.  For each fence,
109
   we fill a parallel group of insns (fill_insns) until some insns can be added.
110
   First, we compute available exprs (av-set) at the boundary of the current
111
   group.  Second, we choose the best expression from it.  If the stall is
112
   required to schedule any of the expressions, we advance the current cycle
113
   appropriately.  So, the final group does not exactly correspond to a VLIW
114
   word.  Third, we move the chosen expression to the boundary (move_op)
115
   and update the intermediate av sets and liveness sets.  We quit fill_insns
116
   when either no insns left for scheduling or we have scheduled enough insns
117
   so we feel like advancing a scheduling point.
118
 
119
   Computing available expressions
120
   ===============================
121
 
122
   The computation (compute_av_set) is a bottom-up traversal.  At each insn,
123
   we're moving the union of its successors' sets through it via
124
   moveup_expr_set.  The dependent expressions are removed.  Local
125
   transformations (substitution, speculation) are applied to move more
126
   exprs.  Then the expr corresponding to the current insn is added.
127
   The result is saved on each basic block header.
128
 
129
   When traversing the CFG, we're moving down for no more than max_ws insns.
130
   Also, we do not move down to ineligible successors (is_ineligible_successor),
131
   which include moving along a back-edge, moving to already scheduled code,
132
   and moving to another fence.  The first two restrictions are lifted during
133
   pipelining, which allows us to move insns along a back-edge.  We always have
134
   an acyclic region for scheduling because we forbid motion through fences.
135
 
136
   Choosing the best expression
137
   ============================
138
 
139
   We sort the final availability set via sel_rank_for_schedule, then we remove
140
   expressions which are not yet ready (tick_check_p) or which dest registers
141
   cannot be used.  For some of them, we choose another register via
142
   find_best_reg.  To do this, we run find_used_regs to calculate the set of
143
   registers which cannot be used.  The find_used_regs function performs
144
   a traversal of code motion paths for an expr.  We consider for renaming
145
   only registers which are from the same regclass as the original one and
146
   using which does not interfere with any live ranges.  Finally, we convert
147
   the resulting set to the ready list format and use max_issue and reorder*
148
   hooks similarly to the Haifa scheduler.
149
 
150
   Scheduling the best expression
151
   ==============================
152
 
153
   We run the move_op routine to perform the same type of code motion paths
154
   traversal as in find_used_regs.  (These are working via the same driver,
155
   code_motion_path_driver.)  When moving down the CFG, we look for original
156
   instruction that gave birth to a chosen expression.  We undo
157
   the transformations performed on an expression via the history saved in it.
158
   When found, we remove the instruction or leave a reg-reg copy/speculation
159
   check if needed.  On a way up, we insert bookkeeping copies at each join
160
   point.  If a copy is not needed, it will be removed later during this
161
   traversal.  We update the saved av sets and liveness sets on the way up, too.
162
 
163
   Finalizing the schedule
164
   =======================
165
 
166
   When pipelining, we reschedule the blocks from which insns were pipelined
167
   to get a tighter schedule.  On Itanium, we also perform bundling via
168
   the same routine from ia64.c.
169
 
170
   Dependence analysis changes
171
   ===========================
172
 
173
   We augmented the sched-deps.c with hooks that get called when a particular
174
   dependence is found in a particular part of an insn.  Using these hooks, we
175
   can do several actions such as: determine whether an insn can be moved through
176
   another (has_dependence_p, moveup_expr); find out whether an insn can be
177
   scheduled on the current cycle (tick_check_p); find out registers that
178
   are set/used/clobbered by an insn and find out all the strange stuff that
179
   restrict its movement, like SCHED_GROUP_P or CANT_MOVE (done in
180
   init_global_and_expr_for_insn).
181
 
182
   Initialization changes
183
   ======================
184
 
185
   There are parts of haifa-sched.c, sched-deps.c, and sched-rgn.c that are
186
   reused in all of the schedulers.  We have split up the initialization of data
187
   of such parts into different functions prefixed with scheduler type and
188
   postfixed with the type of data initialized: {,sel_,haifa_}sched_{init,finish},
189
   sched_rgn_init/finish, sched_deps_init/finish, sched_init_{luids/bbs}, etc.
190
   The same splitting is done with current_sched_info structure:
191
   dependence-related parts are in sched_deps_info, common part is in
192
   common_sched_info, and haifa/sel/etc part is in current_sched_info.
193
 
194
   Target contexts
195
   ===============
196
 
197
   As we now have multiple-point scheduling, this would not work with backends
198
   which save some of the scheduler state to use it in the target hooks.
199
   For this purpose, we introduce a concept of target contexts, which
200
   encapsulate such information.  The backend should implement simple routines
201
   of allocating/freeing/setting such a context.  The scheduler calls these
202
   as target hooks and handles the target context as an opaque pointer (similar
203
   to the DFA state type, state_t).
204
 
205
   Various speedups
206
   ================
207
 
208
   As the correct data dependence graph is not supported during scheduling (which
209
   is to be changed in mid-term), we cache as much of the dependence analysis
210
   results as possible to avoid reanalyzing.  This includes: bitmap caches on
211
   each insn in stream of the region saying yes/no for a query with a pair of
212
   UIDs; hashtables with the previously done transformations on each insn in
213
   stream; a vector keeping a history of transformations on each expr.
214
 
215
   Also, we try to minimize the dependence context used on each fence to check
216
   whether the given expression is ready for scheduling by removing from it
217
   insns that are definitely completed the execution.  The results of
218
   tick_check_p checks are also cached in a vector on each fence.
219
 
220
   We keep a valid liveness set on each insn in a region to avoid the high
221
   cost of recomputation on large basic blocks.
222
 
223
   Finally, we try to minimize the number of needed updates to the availability
224
   sets.  The updates happen in two cases: when fill_insns terminates,
225
   we advance all fences and increase the stage number to show that the region
226
   has changed and the sets are to be recomputed; and when the next iteration
227
   of a loop in fill_insns happens (but this one reuses the saved av sets
228
   on bb headers.)  Thus, we try to break the fill_insns loop only when
229
   "significant" number of insns from the current scheduling window was
230
   scheduled.  This should be made a target param.
231
 
232
 
233
   TODO: correctly support the data dependence graph at all stages and get rid
234
   of all caches.  This should speed up the scheduler.
235
   TODO: implement moving cond jumps with bookkeeping copies on both targets.
236
   TODO: tune the scheduler before RA so it does not create too much pseudos.
237
 
238
 
239
   References:
240
   S.-M. Moon and K. Ebcioglu. Parallelizing nonnumerical code with
241
   selective scheduling and software pipelining.
242
   ACM TOPLAS, Vol 19, No. 6, pages 853--898, Nov. 1997.
243
 
244
   Andrey Belevantsev, Maxim Kuvyrkov, Vladimir Makarov, Dmitry Melnik,
245
   and Dmitry Zhurikhin.  An interblock VLIW-targeted instruction scheduler
246
   for GCC. In Proceedings of GCC Developers' Summit 2006.
247
 
248
   Arutyun Avetisyan, Andrey Belevantsev, and Dmitry Melnik.  GCC Instruction
249
   Scheduler and Software Pipeliner on the Itanium Platform.   EPIC-7 Workshop.
250
   http://rogue.colorado.edu/EPIC7/.
251
 
252
*/
253
 
254
/* True when pipelining is enabled.  */
255
bool pipelining_p;
256
 
257
/* True if bookkeeping is enabled.  */
258
bool bookkeeping_p;
259
 
260
/* Maximum number of insns that are eligible for renaming.  */
261
int max_insns_to_rename;
262
 
263
 
264
/* Definitions of local types and macros.  */
265
 
266
/* Represents possible outcomes of moving an expression through an insn.  */
267
enum MOVEUP_EXPR_CODE
268
  {
269
    /* The expression is not changed.  */
270
    MOVEUP_EXPR_SAME,
271
 
272
    /* Not changed, but requires a new destination register.  */
273
    MOVEUP_EXPR_AS_RHS,
274
 
275
    /* Cannot be moved.  */
276
    MOVEUP_EXPR_NULL,
277
 
278
    /* Changed (substituted or speculated).  */
279
    MOVEUP_EXPR_CHANGED
280
  };
281
 
282
/* The container to be passed into rtx search & replace functions.  */
283
struct rtx_search_arg
284
{
285
  /* What we are searching for.  */
286
  rtx x;
287
 
288
  /* The occurence counter.  */
289
  int n;
290
};
291
 
292
typedef struct rtx_search_arg *rtx_search_arg_p;
293
 
294
/* This struct contains precomputed hard reg sets that are needed when
295
   computing registers available for renaming.  */
296
struct hard_regs_data
297
{
298
  /* For every mode, this stores registers available for use with
299
     that mode.  */
300
  HARD_REG_SET regs_for_mode[NUM_MACHINE_MODES];
301
 
302
  /* True when regs_for_mode[mode] is initialized.  */
303
  bool regs_for_mode_ok[NUM_MACHINE_MODES];
304
 
305
  /* For every register, it has regs that are ok to rename into it.
306
     The register in question is always set.  If not, this means
307
     that the whole set is not computed yet.  */
308
  HARD_REG_SET regs_for_rename[FIRST_PSEUDO_REGISTER];
309
 
310
  /* For every mode, this stores registers not available due to
311
     call clobbering.  */
312
  HARD_REG_SET regs_for_call_clobbered[NUM_MACHINE_MODES];
313
 
314
  /* All registers that are used or call used.  */
315
  HARD_REG_SET regs_ever_used;
316
 
317
#ifdef STACK_REGS
318
  /* Stack registers.  */
319
  HARD_REG_SET stack_regs;
320
#endif
321
};
322
 
323
/* Holds the results of computation of available for renaming and
324
   unavailable hard registers.  */
325
struct reg_rename
326
{
327
  /* These are unavailable due to calls crossing, globalness, etc.  */
328
  HARD_REG_SET unavailable_hard_regs;
329
 
330
  /* These are *available* for renaming.  */
331
  HARD_REG_SET available_for_renaming;
332
 
333
  /* Whether this code motion path crosses a call.  */
334
  bool crosses_call;
335
};
336
 
337
/* A global structure that contains the needed information about harg
338
   regs.  */
339
static struct hard_regs_data sel_hrd;
340
 
341
 
342
/* This structure holds local data used in code_motion_path_driver hooks on
343
   the same or adjacent levels of recursion.  Here we keep those parameters
344
   that are not used in code_motion_path_driver routine itself, but only in
345
   its hooks.  Moreover, all parameters that can be modified in hooks are
346
   in this structure, so all other parameters passed explicitly to hooks are
347
   read-only.  */
348
struct cmpd_local_params
349
{
350
  /* Local params used in move_op_* functions.  */
351
 
352
  /* Edges for bookkeeping generation.  */
353
  edge e1, e2;
354
 
355
  /* C_EXPR merged from all successors and locally allocated temporary C_EXPR.  */
356
  expr_t c_expr_merged, c_expr_local;
357
 
358
  /* Local params used in fur_* functions.  */
359
  /* Copy of the ORIGINAL_INSN list, stores the original insns already
360
     found before entering the current level of code_motion_path_driver.  */
361
  def_list_t old_original_insns;
362
 
363
  /* Local params used in move_op_* functions.  */
364
  /* True when we have removed last insn in the block which was
365
     also a boundary.  Do not update anything or create bookkeeping copies.  */
366
  BOOL_BITFIELD removed_last_insn : 1;
367
};
368
 
369
/* Stores the static parameters for move_op_* calls.  */
370
struct moveop_static_params
371
{
372
  /* Destination register.  */
373
  rtx dest;
374
 
375
  /* Current C_EXPR.  */
376
  expr_t c_expr;
377
 
378
  /* An UID of expr_vliw which is to be moved up.  If we find other exprs,
379
     they are to be removed.  */
380
  int uid;
381
 
382
#ifdef ENABLE_CHECKING
383
  /* This is initialized to the insn on which the driver stopped its traversal.  */
384
  insn_t failed_insn;
385
#endif
386
 
387
  /* True if we scheduled an insn with different register.  */
388
  bool was_renamed;
389
};
390
 
391
/* Stores the static parameters for fur_* calls.  */
392
struct fur_static_params
393
{
394
  /* Set of registers unavailable on the code motion path.  */
395
  regset used_regs;
396
 
397
  /* Pointer to the list of original insns definitions.  */
398
  def_list_t *original_insns;
399
 
400
  /* True if a code motion path contains a CALL insn.  */
401
  bool crosses_call;
402
};
403
 
404
typedef struct fur_static_params *fur_static_params_p;
405
typedef struct cmpd_local_params *cmpd_local_params_p;
406
typedef struct moveop_static_params *moveop_static_params_p;
407
 
408
/* Set of hooks and parameters that determine behaviour specific to
409
   move_op or find_used_regs functions.  */
410
struct code_motion_path_driver_info_def
411
{
412
  /* Called on enter to the basic block.  */
413
  int (*on_enter) (insn_t, cmpd_local_params_p, void *, bool);
414
 
415
  /* Called when original expr is found.  */
416
  void (*orig_expr_found) (insn_t, expr_t, cmpd_local_params_p, void *);
417
 
418
  /* Called while descending current basic block if current insn is not
419
     the original EXPR we're searching for.  */
420
  bool (*orig_expr_not_found) (insn_t, av_set_t, void *);
421
 
422
  /* Function to merge C_EXPRes from different successors.  */
423
  void (*merge_succs) (insn_t, insn_t, int, cmpd_local_params_p, void *);
424
 
425
  /* Function to finalize merge from different successors and possibly
426
     deallocate temporary data structures used for merging.  */
427
  void (*after_merge_succs) (cmpd_local_params_p, void *);
428
 
429
  /* Called on the backward stage of recursion to do moveup_expr.
430
     Used only with move_op_*.  */
431
  void (*ascend) (insn_t, void *);
432
 
433
  /* Called on the ascending pass, before returning from the current basic
434
     block or from the whole traversal.  */
435
  void (*at_first_insn) (insn_t, cmpd_local_params_p, void *);
436
 
437
  /* When processing successors in move_op we need only descend into
438
     SUCCS_NORMAL successors, while in find_used_regs we need SUCCS_ALL.  */
439
  int succ_flags;
440
 
441
  /* The routine name to print in dumps ("move_op" of "find_used_regs").  */
442
  const char *routine_name;
443
};
444
 
445
/* Global pointer to current hooks, either points to MOVE_OP_HOOKS or
446
   FUR_HOOKS.  */
447
struct code_motion_path_driver_info_def *code_motion_path_driver_info;
448
 
449
/* Set of hooks for performing move_op and find_used_regs routines with
450
   code_motion_path_driver.  */
451
extern struct code_motion_path_driver_info_def move_op_hooks, fur_hooks;
452
 
453
/* True if/when we want to emulate Haifa scheduler in the common code.
454
   This is used in sched_rgn_local_init and in various places in
455
   sched-deps.c.  */
456
int sched_emulate_haifa_p;
457
 
458
/* GLOBAL_LEVEL is used to discard information stored in basic block headers
459
   av_sets.  Av_set of bb header is valid if its (bb header's) level is equal
460
   to GLOBAL_LEVEL.  And invalid if lesser.  This is primarily used to advance
461
   scheduling window.  */
462
int global_level;
463
 
464
/* Current fences.  */
465
flist_t fences;
466
 
467
/* True when separable insns should be scheduled as RHSes.  */
468
static bool enable_schedule_as_rhs_p;
469
 
470
/* Used in verify_target_availability to assert that target reg is reported
471
   unavailabile by both TARGET_UNAVAILABLE and find_used_regs only if
472
   we haven't scheduled anything on the previous fence.
473
   if scheduled_something_on_previous_fence is true, TARGET_UNAVAILABLE can
474
   have more conservative value than the one returned by the
475
   find_used_regs, thus we shouldn't assert that these values are equal.  */
476
static bool scheduled_something_on_previous_fence;
477
 
478
/* All newly emitted insns will have their uids greater than this value.  */
479
static int first_emitted_uid;
480
 
481
/* Set of basic blocks that are forced to start new ebbs.  This is a subset
482
   of all the ebb heads.  */
483
static bitmap_head _forced_ebb_heads;
484
bitmap_head *forced_ebb_heads = &_forced_ebb_heads;
485
 
486
/* Blocks that need to be rescheduled after pipelining.  */
487
bitmap blocks_to_reschedule = NULL;
488
 
489
/* True when the first lv set should be ignored when updating liveness.  */
490
static bool ignore_first = false;
491
 
492
/* Number of insns max_issue has initialized data structures for.  */
493
static int max_issue_size = 0;
494
 
495
/* Whether we can issue more instructions.  */
496
static int can_issue_more;
497
 
498
/* Maximum software lookahead window size, reduced when rescheduling after
499
   pipelining.  */
500
static int max_ws;
501
 
502
/* Number of insns scheduled in current region.  */
503
static int num_insns_scheduled;
504
 
505
/* A vector of expressions is used to be able to sort them.  */
506
DEF_VEC_P(expr_t);
507
DEF_VEC_ALLOC_P(expr_t,heap);
508
static VEC(expr_t, heap) *vec_av_set = NULL;
509
 
510
/* A vector of vinsns is used to hold temporary lists of vinsns.  */
511
DEF_VEC_P(vinsn_t);
512
DEF_VEC_ALLOC_P(vinsn_t,heap);
513
typedef VEC(vinsn_t, heap) *vinsn_vec_t;
514
 
515
/* This vector has the exprs which may still present in av_sets, but actually
516
   can't be moved up due to bookkeeping created during code motion to another
517
   fence.  See comment near the call to update_and_record_unavailable_insns
518
   for the detailed explanations.  */
519
static vinsn_vec_t vec_bookkeeping_blocked_vinsns = NULL;
520
 
521
/* This vector has vinsns which are scheduled with renaming on the first fence
522
   and then seen on the second.  For expressions with such vinsns, target
523
   availability information may be wrong.  */
524
static vinsn_vec_t vec_target_unavailable_vinsns = NULL;
525
 
526
/* Vector to store temporary nops inserted in move_op to prevent removal
527
   of empty bbs.  */
528
DEF_VEC_P(insn_t);
529
DEF_VEC_ALLOC_P(insn_t,heap);
530
static VEC(insn_t, heap) *vec_temp_moveop_nops = NULL;
531
 
532
/* These bitmaps record original instructions scheduled on the current
533
   iteration and bookkeeping copies created by them.  */
534
static bitmap current_originators = NULL;
535
static bitmap current_copies = NULL;
536
 
537
/* This bitmap marks the blocks visited by code_motion_path_driver so we don't
538
   visit them afterwards.  */
539
static bitmap code_motion_visited_blocks = NULL;
540
 
541
/* Variables to accumulate different statistics.  */
542
 
543
/* The number of bookkeeping copies created.  */
544
static int stat_bookkeeping_copies;
545
 
546
/* The number of insns that required bookkeeiping for their scheduling.  */
547
static int stat_insns_needed_bookkeeping;
548
 
549
/* The number of insns that got renamed.  */
550
static int stat_renamed_scheduled;
551
 
552
/* The number of substitutions made during scheduling.  */
553
static int stat_substitutions_total;
554
 
555
 
556
/* Forward declarations of static functions.  */
557
static bool rtx_ok_for_substitution_p (rtx, rtx);
558
static int sel_rank_for_schedule (const void *, const void *);
559
static av_set_t find_sequential_best_exprs (bnd_t, expr_t, bool);
560
static basic_block find_block_for_bookkeeping (edge e1, edge e2, bool lax);
561
 
562
static rtx get_dest_from_orig_ops (av_set_t);
563
static basic_block generate_bookkeeping_insn (expr_t, edge, edge);
564
static bool find_used_regs (insn_t, av_set_t, regset, struct reg_rename *,
565
                            def_list_t *);
566
static bool move_op (insn_t, av_set_t, expr_t, rtx, expr_t, bool*);
567
static int code_motion_path_driver (insn_t, av_set_t, ilist_t,
568
                                    cmpd_local_params_p, void *);
569
static void sel_sched_region_1 (void);
570
static void sel_sched_region_2 (int);
571
static av_set_t compute_av_set_inside_bb (insn_t, ilist_t, int, bool);
572
 
573
static void debug_state (state_t);
574
 
575
 
576
/* Functions that work with fences.  */
577
 
578
/* Advance one cycle on FENCE.  */
579
static void
580
advance_one_cycle (fence_t fence)
581
{
582
  unsigned i;
583
  int cycle;
584
  rtx insn;
585
 
586
  advance_state (FENCE_STATE (fence));
587
  cycle = ++FENCE_CYCLE (fence);
588
  FENCE_ISSUED_INSNS (fence) = 0;
589
  FENCE_STARTS_CYCLE_P (fence) = 1;
590
  can_issue_more = issue_rate;
591
  FENCE_ISSUE_MORE (fence) = can_issue_more;
592
 
593
  for (i = 0; VEC_iterate (rtx, FENCE_EXECUTING_INSNS (fence), i, insn); )
594
    {
595
      if (INSN_READY_CYCLE (insn) < cycle)
596
        {
597
          remove_from_deps (FENCE_DC (fence), insn);
598
          VEC_unordered_remove (rtx, FENCE_EXECUTING_INSNS (fence), i);
599
          continue;
600
        }
601
      i++;
602
    }
603
  if (sched_verbose >= 2)
604
    {
605
      sel_print ("Finished a cycle.  Current cycle = %d\n", FENCE_CYCLE (fence));
606
      debug_state (FENCE_STATE (fence));
607
    }
608
}
609
 
610
/* Returns true when SUCC in a fallthru bb of INSN, possibly
611
   skipping empty basic blocks.  */
612
static bool
613
in_fallthru_bb_p (rtx insn, rtx succ)
614
{
615
  basic_block bb = BLOCK_FOR_INSN (insn);
616
  edge e;
617
 
618
  if (bb == BLOCK_FOR_INSN (succ))
619
    return true;
620
 
621
  e = find_fallthru_edge_from (bb);
622
  if (e)
623
    bb = e->dest;
624
  else
625
    return false;
626
 
627
  while (sel_bb_empty_p (bb))
628
    bb = bb->next_bb;
629
 
630
  return bb == BLOCK_FOR_INSN (succ);
631
}
632
 
633
/* Construct successor fences from OLD_FENCEs and put them in NEW_FENCES.
634
   When a successor will continue a ebb, transfer all parameters of a fence
635
   to the new fence.  ORIG_MAX_SEQNO is the maximal seqno before this round
636
   of scheduling helping to distinguish between the old and the new code.  */
637
static void
638
extract_new_fences_from (flist_t old_fences, flist_tail_t new_fences,
639
                         int orig_max_seqno)
640
{
641
  bool was_here_p = false;
642
  insn_t insn = NULL_RTX;
643
  insn_t succ;
644
  succ_iterator si;
645
  ilist_iterator ii;
646
  fence_t fence = FLIST_FENCE (old_fences);
647
  basic_block bb;
648
 
649
  /* Get the only element of FENCE_BNDS (fence).  */
650
  FOR_EACH_INSN (insn, ii, FENCE_BNDS (fence))
651
    {
652
      gcc_assert (!was_here_p);
653
      was_here_p = true;
654
    }
655
  gcc_assert (was_here_p && insn != NULL_RTX);
656
 
657
  /* When in the "middle" of the block, just move this fence
658
     to the new list.  */
659
  bb = BLOCK_FOR_INSN (insn);
660
  if (! sel_bb_end_p (insn)
661
      || (single_succ_p (bb)
662
          && single_pred_p (single_succ (bb))))
663
    {
664
      insn_t succ;
665
 
666
      succ = (sel_bb_end_p (insn)
667
              ? sel_bb_head (single_succ (bb))
668
              : NEXT_INSN (insn));
669
 
670
      if (INSN_SEQNO (succ) > 0
671
          && INSN_SEQNO (succ) <= orig_max_seqno
672
          && INSN_SCHED_TIMES (succ) <= 0)
673
        {
674
          FENCE_INSN (fence) = succ;
675
          move_fence_to_fences (old_fences, new_fences);
676
 
677
          if (sched_verbose >= 1)
678
            sel_print ("Fence %d continues as %d[%d] (state continue)\n",
679
                       INSN_UID (insn), INSN_UID (succ), BLOCK_NUM (succ));
680
        }
681
      return;
682
    }
683
 
684
  /* Otherwise copy fence's structures to (possibly) multiple successors.  */
685
  FOR_EACH_SUCC_1 (succ, si, insn, SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
686
    {
687
      int seqno = INSN_SEQNO (succ);
688
 
689
      if (0 < seqno && seqno <= orig_max_seqno
690
          && (pipelining_p || INSN_SCHED_TIMES (succ) <= 0))
691
        {
692
          bool b = (in_same_ebb_p (insn, succ)
693
                    || in_fallthru_bb_p (insn, succ));
694
 
695
          if (sched_verbose >= 1)
696
            sel_print ("Fence %d continues as %d[%d] (state %s)\n",
697
                       INSN_UID (insn), INSN_UID (succ),
698
                       BLOCK_NUM (succ), b ? "continue" : "reset");
699
 
700
          if (b)
701
            add_dirty_fence_to_fences (new_fences, succ, fence);
702
          else
703
            {
704
              /* Mark block of the SUCC as head of the new ebb.  */
705
              bitmap_set_bit (forced_ebb_heads, BLOCK_NUM (succ));
706
              add_clean_fence_to_fences (new_fences, succ, fence);
707
            }
708
        }
709
    }
710
}
711
 
712
 
713
/* Functions to support substitution.  */
714
 
715
/* Returns whether INSN with dependence status DS is eligible for
716
   substitution, i.e. it's a copy operation x := y, and RHS that is
717
   moved up through this insn should be substituted.  */
718
static bool
719
can_substitute_through_p (insn_t insn, ds_t ds)
720
{
721
  /* We can substitute only true dependencies.  */
722
  if ((ds & DEP_OUTPUT)
723
      || (ds & DEP_ANTI)
724
      || ! INSN_RHS (insn)
725
      || ! INSN_LHS (insn))
726
    return false;
727
 
728
  /* Now we just need to make sure the INSN_RHS consists of only one
729
     simple REG rtx.  */
730
  if (REG_P (INSN_LHS (insn))
731
      && REG_P (INSN_RHS (insn)))
732
    return true;
733
  return false;
734
}
735
 
736
/* Substitute all occurences of INSN's destination in EXPR' vinsn with INSN's
737
   source (if INSN is eligible for substitution).  Returns TRUE if
738
   substitution was actually performed, FALSE otherwise.  Substitution might
739
   be not performed because it's either EXPR' vinsn doesn't contain INSN's
740
   destination or the resulting insn is invalid for the target machine.
741
   When UNDO is true, perform unsubstitution instead (the difference is in
742
   the part of rtx on which validate_replace_rtx is called).  */
743
static bool
744
substitute_reg_in_expr (expr_t expr, insn_t insn, bool undo)
745
{
746
  rtx *where;
747
  bool new_insn_valid;
748
  vinsn_t *vi = &EXPR_VINSN (expr);
749
  bool has_rhs = VINSN_RHS (*vi) != NULL;
750
  rtx old, new_rtx;
751
 
752
  /* Do not try to replace in SET_DEST.  Although we'll choose new
753
     register for the RHS, we don't want to change RHS' original reg.
754
     If the insn is not SET, we may still be able to substitute something
755
     in it, and if we're here (don't have deps), it doesn't write INSN's
756
     dest.  */
757
  where = (has_rhs
758
           ? &VINSN_RHS (*vi)
759
           : &PATTERN (VINSN_INSN_RTX (*vi)));
760
  old = undo ? INSN_RHS (insn) : INSN_LHS (insn);
761
 
762
  /* Substitute if INSN has a form of x:=y and LHS(INSN) occurs in *VI.  */
763
  if (rtx_ok_for_substitution_p (old, *where))
764
    {
765
      rtx new_insn;
766
      rtx *where_replace;
767
 
768
      /* We should copy these rtxes before substitution.  */
769
      new_rtx = copy_rtx (undo ? INSN_LHS (insn) : INSN_RHS (insn));
770
      new_insn = create_copy_of_insn_rtx (VINSN_INSN_RTX (*vi));
771
 
772
      /* Where we'll replace.
773
         WHERE_REPLACE should point inside NEW_INSN, so INSN_RHS couldn't be
774
         used instead of SET_SRC.  */
775
      where_replace = (has_rhs
776
                       ? &SET_SRC (PATTERN (new_insn))
777
                       : &PATTERN (new_insn));
778
 
779
      new_insn_valid
780
        = validate_replace_rtx_part_nosimplify (old, new_rtx, where_replace,
781
                                                new_insn);
782
 
783
      /* ??? Actually, constrain_operands result depends upon choice of
784
         destination register.  E.g. if we allow single register to be an rhs,
785
         and if we try to move dx=ax(as rhs) through ax=dx, we'll result
786
         in invalid insn dx=dx, so we'll loose this rhs here.
787
         Just can't come up with significant testcase for this, so just
788
         leaving it for now.  */
789
      if (new_insn_valid)
790
        {
791
          change_vinsn_in_expr (expr,
792
                                create_vinsn_from_insn_rtx (new_insn, false));
793
 
794
          /* Do not allow clobbering the address register of speculative
795
             insns.  */
796
          if ((EXPR_SPEC_DONE_DS (expr) & SPECULATIVE)
797
              && register_unavailable_p (VINSN_REG_USES (EXPR_VINSN (expr)),
798
                                         expr_dest_reg (expr)))
799
            EXPR_TARGET_AVAILABLE (expr) = false;
800
 
801
          return true;
802
        }
803
      else
804
        return false;
805
    }
806
  else
807
    return false;
808
}
809
 
810
/* Helper function for count_occurences_equiv.  */
811
static int
812
count_occurrences_1 (rtx *cur_rtx, void *arg)
813
{
814
  rtx_search_arg_p p = (rtx_search_arg_p) arg;
815
 
816
  if (REG_P (*cur_rtx) && REGNO (*cur_rtx) == REGNO (p->x))
817
    {
818
      /* Bail out if mode is different or more than one register is used.  */
819
      if (GET_MODE (*cur_rtx) != GET_MODE (p->x)
820
          || (HARD_REGISTER_P (*cur_rtx)
821
              && hard_regno_nregs[REGNO(*cur_rtx)][GET_MODE (*cur_rtx)] > 1))
822
        {
823
          p->n = 0;
824
          return 1;
825
        }
826
 
827
      p->n++;
828
 
829
      /* Do not traverse subexprs.  */
830
      return -1;
831
    }
832
 
833
  if (GET_CODE (*cur_rtx) == SUBREG
834
      && (!REG_P (SUBREG_REG (*cur_rtx))
835
          || REGNO (SUBREG_REG (*cur_rtx)) == REGNO (p->x)))
836
    {
837
      /* ??? Do not support substituting regs inside subregs.  In that case,
838
         simplify_subreg will be called by validate_replace_rtx, and
839
         unsubstitution will fail later.  */
840
      p->n = 0;
841
      return 1;
842
    }
843
 
844
  /* Continue search.  */
845
  return 0;
846
}
847
 
848
/* Return the number of places WHAT appears within WHERE.
849
   Bail out when we found a reference occupying several hard registers.  */
850
static int
851
count_occurrences_equiv (rtx what, rtx where)
852
{
853
  struct rtx_search_arg arg;
854
 
855
  gcc_assert (REG_P (what));
856
  arg.x = what;
857
  arg.n = 0;
858
 
859
  for_each_rtx (&where, &count_occurrences_1, (void *) &arg);
860
 
861
  return arg.n;
862
}
863
 
864
/* Returns TRUE if WHAT is found in WHERE rtx tree.  */
865
static bool
866
rtx_ok_for_substitution_p (rtx what, rtx where)
867
{
868
  return (count_occurrences_equiv (what, where) > 0);
869
}
870
 
871
 
872
/* Functions to support register renaming.  */
873
 
874
/* Substitute VI's set source with REGNO.  Returns newly created pattern
875
   that has REGNO as its source.  */
876
static rtx
877
create_insn_rtx_with_rhs (vinsn_t vi, rtx rhs_rtx)
878
{
879
  rtx lhs_rtx;
880
  rtx pattern;
881
  rtx insn_rtx;
882
 
883
  lhs_rtx = copy_rtx (VINSN_LHS (vi));
884
 
885
  pattern = gen_rtx_SET (VOIDmode, lhs_rtx, rhs_rtx);
886
  insn_rtx = create_insn_rtx_from_pattern (pattern, NULL_RTX);
887
 
888
  return insn_rtx;
889
}
890
 
891
/* Returns whether INSN's src can be replaced with register number
892
   NEW_SRC_REG. E.g. the following insn is valid for i386:
893
 
894
    (insn:HI 2205 6585 2207 727 ../../gcc/libiberty/regex.c:3337
895
      (set (mem/s:QI (plus:SI (plus:SI (reg/f:SI 7 sp)
896
                        (reg:SI 0 ax [orig:770 c1 ] [770]))
897
                    (const_int 288 [0x120])) [0 str S1 A8])
898
            (const_int 0 [0x0])) 43 {*movqi_1} (nil)
899
        (nil))
900
 
901
  But if we change (const_int 0 [0x0]) to (reg:QI 4 si), it will be invalid
902
  because of operand constraints:
903
 
904
    (define_insn "*movqi_1"
905
      [(set (match_operand:QI 0 "nonimmediate_operand" "=q,q ,q ,r,r ,?r,m")
906
            (match_operand:QI 1 "general_operand"      " q,qn,qm,q,rn,qm,qn")
907
            )]
908
 
909
  So do constrain_operands here, before choosing NEW_SRC_REG as best
910
  reg for rhs.  */
911
 
912
static bool
913
replace_src_with_reg_ok_p (insn_t insn, rtx new_src_reg)
914
{
915
  vinsn_t vi = INSN_VINSN (insn);
916
  enum machine_mode mode;
917
  rtx dst_loc;
918
  bool res;
919
 
920
  gcc_assert (VINSN_SEPARABLE_P (vi));
921
 
922
  get_dest_and_mode (insn, &dst_loc, &mode);
923
  gcc_assert (mode == GET_MODE (new_src_reg));
924
 
925
  if (REG_P (dst_loc) && REGNO (new_src_reg) == REGNO (dst_loc))
926
    return true;
927
 
928
  /* See whether SET_SRC can be replaced with this register.  */
929
  validate_change (insn, &SET_SRC (PATTERN (insn)), new_src_reg, 1);
930
  res = verify_changes (0);
931
  cancel_changes (0);
932
 
933
  return res;
934
}
935
 
936
/* Returns whether INSN still be valid after replacing it's DEST with
937
   register NEW_REG.  */
938
static bool
939
replace_dest_with_reg_ok_p (insn_t insn, rtx new_reg)
940
{
941
  vinsn_t vi = INSN_VINSN (insn);
942
  bool res;
943
 
944
  /* We should deal here only with separable insns.  */
945
  gcc_assert (VINSN_SEPARABLE_P (vi));
946
  gcc_assert (GET_MODE (VINSN_LHS (vi)) == GET_MODE (new_reg));
947
 
948
  /* See whether SET_DEST can be replaced with this register.  */
949
  validate_change (insn, &SET_DEST (PATTERN (insn)), new_reg, 1);
950
  res = verify_changes (0);
951
  cancel_changes (0);
952
 
953
  return res;
954
}
955
 
956
/* Create a pattern with rhs of VI and lhs of LHS_RTX.  */
957
static rtx
958
create_insn_rtx_with_lhs (vinsn_t vi, rtx lhs_rtx)
959
{
960
  rtx rhs_rtx;
961
  rtx pattern;
962
  rtx insn_rtx;
963
 
964
  rhs_rtx = copy_rtx (VINSN_RHS (vi));
965
 
966
  pattern = gen_rtx_SET (VOIDmode, lhs_rtx, rhs_rtx);
967
  insn_rtx = create_insn_rtx_from_pattern (pattern, NULL_RTX);
968
 
969
  return insn_rtx;
970
}
971
 
972
/* Substitute lhs in the given expression EXPR for the register with number
973
   NEW_REGNO.  SET_DEST may be arbitrary rtx, not only register.  */
974
static void
975
replace_dest_with_reg_in_expr (expr_t expr, rtx new_reg)
976
{
977
  rtx insn_rtx;
978
  vinsn_t vinsn;
979
 
980
  insn_rtx = create_insn_rtx_with_lhs (EXPR_VINSN (expr), new_reg);
981
  vinsn = create_vinsn_from_insn_rtx (insn_rtx, false);
982
 
983
  change_vinsn_in_expr (expr, vinsn);
984
  EXPR_WAS_RENAMED (expr) = 1;
985
  EXPR_TARGET_AVAILABLE (expr) = 1;
986
}
987
 
988
/* Returns whether VI writes either one of the USED_REGS registers or,
989
   if a register is a hard one, one of the UNAVAILABLE_HARD_REGS registers.  */
990
static bool
991
vinsn_writes_one_of_regs_p (vinsn_t vi, regset used_regs,
992
                            HARD_REG_SET unavailable_hard_regs)
993
{
994
  unsigned regno;
995
  reg_set_iterator rsi;
996
 
997
  EXECUTE_IF_SET_IN_REG_SET (VINSN_REG_SETS (vi), 0, regno, rsi)
998
    {
999
      if (REGNO_REG_SET_P (used_regs, regno))
1000
        return true;
1001
      if (HARD_REGISTER_NUM_P (regno)
1002
          && TEST_HARD_REG_BIT (unavailable_hard_regs, regno))
1003
        return true;
1004
    }
1005
 
1006
  EXECUTE_IF_SET_IN_REG_SET (VINSN_REG_CLOBBERS (vi), 0, regno, rsi)
1007
    {
1008
      if (REGNO_REG_SET_P (used_regs, regno))
1009
        return true;
1010
      if (HARD_REGISTER_NUM_P (regno)
1011
          && TEST_HARD_REG_BIT (unavailable_hard_regs, regno))
1012
        return true;
1013
    }
1014
 
1015
  return false;
1016
}
1017
 
1018
/* Returns register class of the output register in INSN.
1019
   Returns NO_REGS for call insns because some targets have constraints on
1020
   destination register of a call insn.
1021
 
1022
   Code adopted from regrename.c::build_def_use.  */
1023
static enum reg_class
1024
get_reg_class (rtx insn)
1025
{
1026
  int alt, i, n_ops;
1027
 
1028
  extract_insn (insn);
1029
  if (! constrain_operands (1))
1030
    fatal_insn_not_found (insn);
1031
  preprocess_constraints ();
1032
  alt = which_alternative;
1033
  n_ops = recog_data.n_operands;
1034
 
1035
  for (i = 0; i < n_ops; ++i)
1036
    {
1037
      int matches = recog_op_alt[i][alt].matches;
1038
      if (matches >= 0)
1039
        recog_op_alt[i][alt].cl = recog_op_alt[matches][alt].cl;
1040
    }
1041
 
1042
  if (asm_noperands (PATTERN (insn)) > 0)
1043
    {
1044
      for (i = 0; i < n_ops; i++)
1045
        if (recog_data.operand_type[i] == OP_OUT)
1046
          {
1047
            rtx *loc = recog_data.operand_loc[i];
1048
            rtx op = *loc;
1049
            enum reg_class cl = recog_op_alt[i][alt].cl;
1050
 
1051
            if (REG_P (op)
1052
                && REGNO (op) == ORIGINAL_REGNO (op))
1053
              continue;
1054
 
1055
            return cl;
1056
          }
1057
    }
1058
  else if (!CALL_P (insn))
1059
    {
1060
      for (i = 0; i < n_ops + recog_data.n_dups; i++)
1061
       {
1062
         int opn = i < n_ops ? i : recog_data.dup_num[i - n_ops];
1063
         enum reg_class cl = recog_op_alt[opn][alt].cl;
1064
 
1065
         if (recog_data.operand_type[opn] == OP_OUT ||
1066
             recog_data.operand_type[opn] == OP_INOUT)
1067
           return cl;
1068
       }
1069
    }
1070
 
1071
/*  Insns like
1072
    (insn (set (reg:CCZ 17 flags) (compare:CCZ ...)))
1073
    may result in returning NO_REGS, cause flags is written implicitly through
1074
    CMP insn, which has no OP_OUT | OP_INOUT operands.  */
1075
  return NO_REGS;
1076
}
1077
 
1078
#ifdef HARD_REGNO_RENAME_OK
1079
/* Calculate HARD_REGNO_RENAME_OK data for REGNO.  */
1080
static void
1081
init_hard_regno_rename (int regno)
1082
{
1083
  int cur_reg;
1084
 
1085
  SET_HARD_REG_BIT (sel_hrd.regs_for_rename[regno], regno);
1086
 
1087
  for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1088
    {
1089
      /* We are not interested in renaming in other regs.  */
1090
      if (!TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, cur_reg))
1091
        continue;
1092
 
1093
      if (HARD_REGNO_RENAME_OK (regno, cur_reg))
1094
        SET_HARD_REG_BIT (sel_hrd.regs_for_rename[regno], cur_reg);
1095
    }
1096
}
1097
#endif
1098
 
1099
/* A wrapper around HARD_REGNO_RENAME_OK that will look into the hard regs
1100
   data first.  */
1101
static inline bool
1102
sel_hard_regno_rename_ok (int from ATTRIBUTE_UNUSED, int to ATTRIBUTE_UNUSED)
1103
{
1104
#ifdef HARD_REGNO_RENAME_OK
1105
  /* Check whether this is all calculated.  */
1106
  if (TEST_HARD_REG_BIT (sel_hrd.regs_for_rename[from], from))
1107
    return TEST_HARD_REG_BIT (sel_hrd.regs_for_rename[from], to);
1108
 
1109
  init_hard_regno_rename (from);
1110
 
1111
  return TEST_HARD_REG_BIT (sel_hrd.regs_for_rename[from], to);
1112
#else
1113
  return true;
1114
#endif
1115
}
1116
 
1117
/* Calculate set of registers that are capable of holding MODE.  */
1118
static void
1119
init_regs_for_mode (enum machine_mode mode)
1120
{
1121
  int cur_reg;
1122
 
1123
  CLEAR_HARD_REG_SET (sel_hrd.regs_for_mode[mode]);
1124
  CLEAR_HARD_REG_SET (sel_hrd.regs_for_call_clobbered[mode]);
1125
 
1126
  for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1127
    {
1128
      int nregs = hard_regno_nregs[cur_reg][mode];
1129
      int i;
1130
 
1131
      for (i = nregs - 1; i >= 0; --i)
1132
        if (fixed_regs[cur_reg + i]
1133
                || global_regs[cur_reg + i]
1134
            /* Can't use regs which aren't saved by
1135
               the prologue.  */
1136
            || !TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, cur_reg + i)
1137
            /* Can't use regs with non-null REG_BASE_VALUE, because adjusting
1138
               it affects aliasing globally and invalidates all AV sets.  */
1139
            || get_reg_base_value (cur_reg + i)
1140
#ifdef LEAF_REGISTERS
1141
            /* We can't use a non-leaf register if we're in a
1142
               leaf function.  */
1143
            || (current_function_is_leaf
1144
                && !LEAF_REGISTERS[cur_reg + i])
1145
#endif
1146
            )
1147
          break;
1148
 
1149
      if (i >= 0)
1150
        continue;
1151
 
1152
      /* See whether it accepts all modes that occur in
1153
         original insns.  */
1154
      if (! HARD_REGNO_MODE_OK (cur_reg, mode))
1155
        continue;
1156
 
1157
      if (HARD_REGNO_CALL_PART_CLOBBERED (cur_reg, mode))
1158
        SET_HARD_REG_BIT (sel_hrd.regs_for_call_clobbered[mode],
1159
                          cur_reg);
1160
 
1161
      /* If the CUR_REG passed all the checks above,
1162
         then it's ok.  */
1163
      SET_HARD_REG_BIT (sel_hrd.regs_for_mode[mode], cur_reg);
1164
    }
1165
 
1166
  sel_hrd.regs_for_mode_ok[mode] = true;
1167
}
1168
 
1169
/* Init all register sets gathered in HRD.  */
1170
static void
1171
init_hard_regs_data (void)
1172
{
1173
  int cur_reg = 0;
1174
  int cur_mode = 0;
1175
 
1176
  CLEAR_HARD_REG_SET (sel_hrd.regs_ever_used);
1177
  for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1178
    if (df_regs_ever_live_p (cur_reg) || call_used_regs[cur_reg])
1179
      SET_HARD_REG_BIT (sel_hrd.regs_ever_used, cur_reg);
1180
 
1181
  /* Initialize registers that are valid based on mode when this is
1182
     really needed.  */
1183
  for (cur_mode = 0; cur_mode < NUM_MACHINE_MODES; cur_mode++)
1184
    sel_hrd.regs_for_mode_ok[cur_mode] = false;
1185
 
1186
  /* Mark that all HARD_REGNO_RENAME_OK is not calculated.  */
1187
  for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++)
1188
    CLEAR_HARD_REG_SET (sel_hrd.regs_for_rename[cur_reg]);
1189
 
1190
#ifdef STACK_REGS
1191
  CLEAR_HARD_REG_SET (sel_hrd.stack_regs);
1192
 
1193
  for (cur_reg = FIRST_STACK_REG; cur_reg <= LAST_STACK_REG; cur_reg++)
1194
    SET_HARD_REG_BIT (sel_hrd.stack_regs, cur_reg);
1195
#endif
1196
}
1197
 
1198
/* Mark hardware regs in REG_RENAME_P that are not suitable
1199
   for renaming rhs in INSN due to hardware restrictions (register class,
1200
   modes compatibility etc).  This doesn't affect original insn's dest reg,
1201
   if it isn't in USED_REGS.  DEF is a definition insn of rhs for which the
1202
   destination register is sought.  LHS (DEF->ORIG_INSN) may be REG or MEM.
1203
   Registers that are in used_regs are always marked in
1204
   unavailable_hard_regs as well.  */
1205
 
1206
static void
1207
mark_unavailable_hard_regs (def_t def, struct reg_rename *reg_rename_p,
1208
                            regset used_regs ATTRIBUTE_UNUSED)
1209
{
1210
  enum machine_mode mode;
1211
  enum reg_class cl = NO_REGS;
1212
  rtx orig_dest;
1213
  unsigned cur_reg, regno;
1214
  hard_reg_set_iterator hrsi;
1215
 
1216
  gcc_assert (GET_CODE (PATTERN (def->orig_insn)) == SET);
1217
  gcc_assert (reg_rename_p);
1218
 
1219
  orig_dest = SET_DEST (PATTERN (def->orig_insn));
1220
 
1221
  /* We have decided not to rename 'mem = something;' insns, as 'something'
1222
     is usually a register.  */
1223
  if (!REG_P (orig_dest))
1224
    return;
1225
 
1226
  regno = REGNO (orig_dest);
1227
 
1228
  /* If before reload, don't try to work with pseudos.  */
1229
  if (!reload_completed && !HARD_REGISTER_NUM_P (regno))
1230
    return;
1231
 
1232
  if (reload_completed)
1233
    cl = get_reg_class (def->orig_insn);
1234
 
1235
  /* Stop if the original register is one of the fixed_regs, global_regs or
1236
     frame pointer, or we could not discover its class.  */
1237
  if (fixed_regs[regno]
1238
      || global_regs[regno]
1239
#if !HARD_FRAME_POINTER_IS_FRAME_POINTER
1240
      || (frame_pointer_needed && regno == HARD_FRAME_POINTER_REGNUM)
1241
#else
1242
      || (frame_pointer_needed && regno == FRAME_POINTER_REGNUM)
1243
#endif
1244
      || (reload_completed && cl == NO_REGS))
1245
    {
1246
      SET_HARD_REG_SET (reg_rename_p->unavailable_hard_regs);
1247
 
1248
      /* Give a chance for original register, if it isn't in used_regs.  */
1249
      if (!def->crosses_call)
1250
        CLEAR_HARD_REG_BIT (reg_rename_p->unavailable_hard_regs, regno);
1251
 
1252
      return;
1253
    }
1254
 
1255
  /* If something allocated on stack in this function, mark frame pointer
1256
     register unavailable, considering also modes.
1257
     FIXME: it is enough to do this once per all original defs.  */
1258
  if (frame_pointer_needed)
1259
    {
1260
      add_to_hard_reg_set (&reg_rename_p->unavailable_hard_regs,
1261
                           Pmode, FRAME_POINTER_REGNUM);
1262
 
1263
      if (!HARD_FRAME_POINTER_IS_FRAME_POINTER)
1264
        add_to_hard_reg_set (&reg_rename_p->unavailable_hard_regs,
1265
                             Pmode, HARD_FRAME_POINTER_IS_FRAME_POINTER);
1266
    }
1267
 
1268
#ifdef STACK_REGS
1269
  /* For the stack registers the presence of FIRST_STACK_REG in USED_REGS
1270
     is equivalent to as if all stack regs were in this set.
1271
     I.e. no stack register can be renamed, and even if it's an original
1272
     register here we make sure it won't be lifted over it's previous def
1273
     (it's previous def will appear as if it's a FIRST_STACK_REG def.
1274
     The HARD_REGNO_RENAME_OK covers other cases in condition below.  */
1275
  if (IN_RANGE (REGNO (orig_dest), FIRST_STACK_REG, LAST_STACK_REG)
1276
      && REGNO_REG_SET_P (used_regs, FIRST_STACK_REG))
1277
    IOR_HARD_REG_SET (reg_rename_p->unavailable_hard_regs,
1278
                      sel_hrd.stack_regs);
1279
#endif
1280
 
1281
  /* If there's a call on this path, make regs from call_used_reg_set
1282
     unavailable.  */
1283
  if (def->crosses_call)
1284
    IOR_HARD_REG_SET (reg_rename_p->unavailable_hard_regs,
1285
                      call_used_reg_set);
1286
 
1287
  /* Stop here before reload: we need FRAME_REGS, STACK_REGS, and crosses_call,
1288
     but not register classes.  */
1289
  if (!reload_completed)
1290
    return;
1291
 
1292
  /* Leave regs as 'available' only from the current
1293
     register class.  */
1294
  COPY_HARD_REG_SET (reg_rename_p->available_for_renaming,
1295
                     reg_class_contents[cl]);
1296
 
1297
  mode = GET_MODE (orig_dest);
1298
 
1299
  /* Leave only registers available for this mode.  */
1300
  if (!sel_hrd.regs_for_mode_ok[mode])
1301
    init_regs_for_mode (mode);
1302
  AND_HARD_REG_SET (reg_rename_p->available_for_renaming,
1303
                    sel_hrd.regs_for_mode[mode]);
1304
 
1305
  /* Exclude registers that are partially call clobbered.  */
1306
  if (def->crosses_call
1307
      && ! HARD_REGNO_CALL_PART_CLOBBERED (regno, mode))
1308
    AND_COMPL_HARD_REG_SET (reg_rename_p->available_for_renaming,
1309
                            sel_hrd.regs_for_call_clobbered[mode]);
1310
 
1311
  /* Leave only those that are ok to rename.  */
1312
  EXECUTE_IF_SET_IN_HARD_REG_SET (reg_rename_p->available_for_renaming,
1313
                                  0, cur_reg, hrsi)
1314
    {
1315
      int nregs;
1316
      int i;
1317
 
1318
      nregs = hard_regno_nregs[cur_reg][mode];
1319
      gcc_assert (nregs > 0);
1320
 
1321
      for (i = nregs - 1; i >= 0; --i)
1322
        if (! sel_hard_regno_rename_ok (regno + i, cur_reg + i))
1323
          break;
1324
 
1325
      if (i >= 0)
1326
        CLEAR_HARD_REG_BIT (reg_rename_p->available_for_renaming,
1327
                            cur_reg);
1328
    }
1329
 
1330
  AND_COMPL_HARD_REG_SET (reg_rename_p->available_for_renaming,
1331
                          reg_rename_p->unavailable_hard_regs);
1332
 
1333
  /* Regno is always ok from the renaming part of view, but it really
1334
     could be in *unavailable_hard_regs already, so set it here instead
1335
     of there.  */
1336
  SET_HARD_REG_BIT (reg_rename_p->available_for_renaming, regno);
1337
}
1338
 
1339
/* reg_rename_tick[REG1] > reg_rename_tick[REG2] if REG1 was chosen as the
1340
   best register more recently than REG2.  */
1341
static int reg_rename_tick[FIRST_PSEUDO_REGISTER];
1342
 
1343
/* Indicates the number of times renaming happened before the current one.  */
1344
static int reg_rename_this_tick;
1345
 
1346
/* Choose the register among free, that is suitable for storing
1347
   the rhs value.
1348
 
1349
   ORIGINAL_INSNS is the list of insns where the operation (rhs)
1350
   originally appears.  There could be multiple original operations
1351
   for single rhs since we moving it up and merging along different
1352
   paths.
1353
 
1354
   Some code is adapted from regrename.c (regrename_optimize).
1355
   If original register is available, function returns it.
1356
   Otherwise it performs the checks, so the new register should
1357
   comply with the following:
1358
    - it should not violate any live ranges (such registers are in
1359
      REG_RENAME_P->available_for_renaming set);
1360
    - it should not be in the HARD_REGS_USED regset;
1361
    - it should be in the class compatible with original uses;
1362
    - it should not be clobbered through reference with different mode;
1363
    - if we're in the leaf function, then the new register should
1364
      not be in the LEAF_REGISTERS;
1365
    - etc.
1366
 
1367
   If several registers meet the conditions, the register with smallest
1368
   tick is returned to achieve more even register allocation.
1369
 
1370
   If original register seems to be ok, we set *IS_ORIG_REG_P_PTR to true.
1371
 
1372
   If no register satisfies the above conditions, NULL_RTX is returned.  */
1373
static rtx
1374
choose_best_reg_1 (HARD_REG_SET hard_regs_used,
1375
                   struct reg_rename *reg_rename_p,
1376
                   def_list_t original_insns, bool *is_orig_reg_p_ptr)
1377
{
1378
  int best_new_reg;
1379
  unsigned cur_reg;
1380
  enum machine_mode mode = VOIDmode;
1381
  unsigned regno, i, n;
1382
  hard_reg_set_iterator hrsi;
1383
  def_list_iterator di;
1384
  def_t def;
1385
 
1386
  /* If original register is available, return it.  */
1387
  *is_orig_reg_p_ptr = true;
1388
 
1389
  FOR_EACH_DEF (def, di, original_insns)
1390
    {
1391
      rtx orig_dest = SET_DEST (PATTERN (def->orig_insn));
1392
 
1393
      gcc_assert (REG_P (orig_dest));
1394
 
1395
      /* Check that all original operations have the same mode.
1396
         This is done for the next loop; if we'd return from this
1397
         loop, we'd check only part of them, but in this case
1398
         it doesn't matter.  */
1399
      if (mode == VOIDmode)
1400
        mode = GET_MODE (orig_dest);
1401
      gcc_assert (mode == GET_MODE (orig_dest));
1402
 
1403
      regno = REGNO (orig_dest);
1404
      for (i = 0, n = hard_regno_nregs[regno][mode]; i < n; i++)
1405
        if (TEST_HARD_REG_BIT (hard_regs_used, regno + i))
1406
          break;
1407
 
1408
      /* All hard registers are available.  */
1409
      if (i == n)
1410
        {
1411
          gcc_assert (mode != VOIDmode);
1412
 
1413
          /* Hard registers should not be shared.  */
1414
          return gen_rtx_REG (mode, regno);
1415
        }
1416
    }
1417
 
1418
  *is_orig_reg_p_ptr = false;
1419
  best_new_reg = -1;
1420
 
1421
  /* Among all available regs choose the register that was
1422
     allocated earliest.  */
1423
  EXECUTE_IF_SET_IN_HARD_REG_SET (reg_rename_p->available_for_renaming,
1424
                                  0, cur_reg, hrsi)
1425
    if (! TEST_HARD_REG_BIT (hard_regs_used, cur_reg))
1426
      {
1427
        /* Check that all hard regs for mode are available.  */
1428
        for (i = 1, n = hard_regno_nregs[cur_reg][mode]; i < n; i++)
1429
          if (TEST_HARD_REG_BIT (hard_regs_used, cur_reg + i)
1430
              || !TEST_HARD_REG_BIT (reg_rename_p->available_for_renaming,
1431
                                     cur_reg + i))
1432
            break;
1433
 
1434
        if (i < n)
1435
          continue;
1436
 
1437
        /* All hard registers are available.  */
1438
        if (best_new_reg < 0
1439
            || reg_rename_tick[cur_reg] < reg_rename_tick[best_new_reg])
1440
          {
1441
            best_new_reg = cur_reg;
1442
 
1443
            /* Return immediately when we know there's no better reg.  */
1444
            if (! reg_rename_tick[best_new_reg])
1445
              break;
1446
          }
1447
      }
1448
 
1449
  if (best_new_reg >= 0)
1450
    {
1451
      /* Use the check from the above loop.  */
1452
      gcc_assert (mode != VOIDmode);
1453
      return gen_rtx_REG (mode, best_new_reg);
1454
    }
1455
 
1456
  return NULL_RTX;
1457
}
1458
 
1459
/* A wrapper around choose_best_reg_1 () to verify that we make correct
1460
   assumptions about available registers in the function.  */
1461
static rtx
1462
choose_best_reg (HARD_REG_SET hard_regs_used, struct reg_rename *reg_rename_p,
1463
                 def_list_t original_insns, bool *is_orig_reg_p_ptr)
1464
{
1465
  rtx best_reg = choose_best_reg_1 (hard_regs_used, reg_rename_p,
1466
                                    original_insns, is_orig_reg_p_ptr);
1467
 
1468
  /* FIXME loop over hard_regno_nregs here.  */
1469
  gcc_assert (best_reg == NULL_RTX
1470
              || TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, REGNO (best_reg)));
1471
 
1472
  return best_reg;
1473
}
1474
 
1475
/* Choose the pseudo register for storing rhs value.  As this is supposed
1476
   to work before reload, we return either the original register or make
1477
   the new one.  The parameters are the same that in choose_nest_reg_1
1478
   functions, except that USED_REGS may contain pseudos.
1479
   If we work with hard regs, check also REG_RENAME_P->UNAVAILABLE_HARD_REGS.
1480
 
1481
   TODO: take into account register pressure while doing this.  Up to this
1482
   moment, this function would never return NULL for pseudos, but we should
1483
   not rely on this.  */
1484
static rtx
1485
choose_best_pseudo_reg (regset used_regs,
1486
                        struct reg_rename *reg_rename_p,
1487
                        def_list_t original_insns, bool *is_orig_reg_p_ptr)
1488
{
1489
  def_list_iterator i;
1490
  def_t def;
1491
  enum machine_mode mode = VOIDmode;
1492
  bool bad_hard_regs = false;
1493
 
1494
  /* We should not use this after reload.  */
1495
  gcc_assert (!reload_completed);
1496
 
1497
  /* If original register is available, return it.  */
1498
  *is_orig_reg_p_ptr = true;
1499
 
1500
  FOR_EACH_DEF (def, i, original_insns)
1501
    {
1502
      rtx dest = SET_DEST (PATTERN (def->orig_insn));
1503
      int orig_regno;
1504
 
1505
      gcc_assert (REG_P (dest));
1506
 
1507
      /* Check that all original operations have the same mode.  */
1508
      if (mode == VOIDmode)
1509
        mode = GET_MODE (dest);
1510
      else
1511
        gcc_assert (mode == GET_MODE (dest));
1512
      orig_regno = REGNO (dest);
1513
 
1514
      if (!REGNO_REG_SET_P (used_regs, orig_regno))
1515
        {
1516
          if (orig_regno < FIRST_PSEUDO_REGISTER)
1517
            {
1518
              gcc_assert (df_regs_ever_live_p (orig_regno));
1519
 
1520
              /* For hard registers, we have to check hardware imposed
1521
                 limitations (frame/stack registers, calls crossed).  */
1522
              if (!TEST_HARD_REG_BIT (reg_rename_p->unavailable_hard_regs,
1523
                                      orig_regno))
1524
                {
1525
                  /* Don't let register cross a call if it doesn't already
1526
                     cross one.  This condition is written in accordance with
1527
                     that in sched-deps.c sched_analyze_reg().  */
1528
                  if (!reg_rename_p->crosses_call
1529
                      || REG_N_CALLS_CROSSED (orig_regno) > 0)
1530
                    return gen_rtx_REG (mode, orig_regno);
1531
                }
1532
 
1533
              bad_hard_regs = true;
1534
            }
1535
          else
1536
            return dest;
1537
        }
1538
     }
1539
 
1540
  *is_orig_reg_p_ptr = false;
1541
 
1542
  /* We had some original hard registers that couldn't be used.
1543
     Those were likely special.  Don't try to create a pseudo.  */
1544
  if (bad_hard_regs)
1545
    return NULL_RTX;
1546
 
1547
  /* We haven't found a register from original operations.  Get a new one.
1548
     FIXME: control register pressure somehow.  */
1549
  {
1550
    rtx new_reg = gen_reg_rtx (mode);
1551
 
1552
    gcc_assert (mode != VOIDmode);
1553
 
1554
    max_regno = max_reg_num ();
1555
    maybe_extend_reg_info_p ();
1556
    REG_N_CALLS_CROSSED (REGNO (new_reg)) = reg_rename_p->crosses_call ? 1 : 0;
1557
 
1558
    return new_reg;
1559
  }
1560
}
1561
 
1562
/* True when target of EXPR is available due to EXPR_TARGET_AVAILABLE,
1563
   USED_REGS and REG_RENAME_P->UNAVAILABLE_HARD_REGS.  */
1564
static void
1565
verify_target_availability (expr_t expr, regset used_regs,
1566
                            struct reg_rename *reg_rename_p)
1567
{
1568
  unsigned n, i, regno;
1569
  enum machine_mode mode;
1570
  bool target_available, live_available, hard_available;
1571
 
1572
  if (!REG_P (EXPR_LHS (expr)) || EXPR_TARGET_AVAILABLE (expr) < 0)
1573
    return;
1574
 
1575
  regno = expr_dest_regno (expr);
1576
  mode = GET_MODE (EXPR_LHS (expr));
1577
  target_available = EXPR_TARGET_AVAILABLE (expr) == 1;
1578
  n = HARD_REGISTER_NUM_P (regno) ? hard_regno_nregs[regno][mode] : 1;
1579
 
1580
  live_available = hard_available = true;
1581
  for (i = 0; i < n; i++)
1582
    {
1583
      if (bitmap_bit_p (used_regs, regno + i))
1584
        live_available = false;
1585
      if (TEST_HARD_REG_BIT (reg_rename_p->unavailable_hard_regs, regno + i))
1586
        hard_available = false;
1587
    }
1588
 
1589
  /* When target is not available, it may be due to hard register
1590
     restrictions, e.g. crosses calls, so we check hard_available too.  */
1591
  if (target_available)
1592
    gcc_assert (live_available);
1593
  else
1594
    /* Check only if we haven't scheduled something on the previous fence,
1595
       cause due to MAX_SOFTWARE_LOOKAHEAD_WINDOW_SIZE issues
1596
       and having more than one fence, we may end having targ_un in a block
1597
       in which successors target register is actually available.
1598
 
1599
       The last condition handles the case when a dependence from a call insn
1600
       was created in sched-deps.c for insns with destination registers that
1601
       never crossed a call before, but do cross one after our code motion.
1602
 
1603
       FIXME: in the latter case, we just uselessly called find_used_regs,
1604
       because we can't move this expression with any other register
1605
       as well.  */
1606
    gcc_assert (scheduled_something_on_previous_fence || !live_available
1607
                || !hard_available
1608
                || (!reload_completed && reg_rename_p->crosses_call
1609
                    && REG_N_CALLS_CROSSED (regno) == 0));
1610
}
1611
 
1612
/* Collect unavailable registers due to liveness for EXPR from BNDS
1613
   into USED_REGS.  Save additional information about available
1614
   registers and unavailable due to hardware restriction registers
1615
   into REG_RENAME_P structure.  Save original insns into ORIGINAL_INSNS
1616
   list.  */
1617
static void
1618
collect_unavailable_regs_from_bnds (expr_t expr, blist_t bnds, regset used_regs,
1619
                                    struct reg_rename *reg_rename_p,
1620
                                    def_list_t *original_insns)
1621
{
1622
  for (; bnds; bnds = BLIST_NEXT (bnds))
1623
    {
1624
      bool res;
1625
      av_set_t orig_ops = NULL;
1626
      bnd_t bnd = BLIST_BND (bnds);
1627
 
1628
      /* If the chosen best expr doesn't belong to current boundary,
1629
         skip it.  */
1630
      if (!av_set_is_in_p (BND_AV1 (bnd), EXPR_VINSN (expr)))
1631
        continue;
1632
 
1633
      /* Put in ORIG_OPS all exprs from this boundary that became
1634
         RES on top.  */
1635
      orig_ops = find_sequential_best_exprs (bnd, expr, false);
1636
 
1637
      /* Compute used regs and OR it into the USED_REGS.  */
1638
      res = find_used_regs (BND_TO (bnd), orig_ops, used_regs,
1639
                            reg_rename_p, original_insns);
1640
 
1641
      /* FIXME: the assert is true until we'd have several boundaries.  */
1642
      gcc_assert (res);
1643
      av_set_clear (&orig_ops);
1644
    }
1645
}
1646
 
1647
/* Return TRUE if it is possible to replace LHSes of ORIG_INSNS with BEST_REG.
1648
   If BEST_REG is valid, replace LHS of EXPR with it.  */
1649
static bool
1650
try_replace_dest_reg (ilist_t orig_insns, rtx best_reg, expr_t expr)
1651
{
1652
  /* Try whether we'll be able to generate the insn
1653
     'dest := best_reg' at the place of the original operation.  */
1654
  for (; orig_insns; orig_insns = ILIST_NEXT (orig_insns))
1655
    {
1656
      insn_t orig_insn = DEF_LIST_DEF (orig_insns)->orig_insn;
1657
 
1658
      gcc_assert (EXPR_SEPARABLE_P (INSN_EXPR (orig_insn)));
1659
 
1660
      if (REGNO (best_reg) != REGNO (INSN_LHS (orig_insn))
1661
          && (! replace_src_with_reg_ok_p (orig_insn, best_reg)
1662
              || ! replace_dest_with_reg_ok_p (orig_insn, best_reg)))
1663
        return false;
1664
    }
1665
 
1666
  /* Make sure that EXPR has the right destination
1667
     register.  */
1668
  if (expr_dest_regno (expr) != REGNO (best_reg))
1669
    replace_dest_with_reg_in_expr (expr, best_reg);
1670
  else
1671
    EXPR_TARGET_AVAILABLE (expr) = 1;
1672
 
1673
  return true;
1674
}
1675
 
1676
/* Select and assign best register to EXPR searching from BNDS.
1677
   Set *IS_ORIG_REG_P to TRUE if original register was selected.
1678
   Return FALSE if no register can be chosen, which could happen when:
1679
   * EXPR_SEPARABLE_P is true but we were unable to find suitable register;
1680
   * EXPR_SEPARABLE_P is false but the insn sets/clobbers one of the registers
1681
     that are used on the moving path.  */
1682
static bool
1683
find_best_reg_for_expr (expr_t expr, blist_t bnds, bool *is_orig_reg_p)
1684
{
1685
  static struct reg_rename reg_rename_data;
1686
 
1687
  regset used_regs;
1688
  def_list_t original_insns = NULL;
1689
  bool reg_ok;
1690
 
1691
  *is_orig_reg_p = false;
1692
 
1693
  /* Don't bother to do anything if this insn doesn't set any registers.  */
1694
  if (bitmap_empty_p (VINSN_REG_SETS (EXPR_VINSN (expr)))
1695
      && bitmap_empty_p (VINSN_REG_CLOBBERS (EXPR_VINSN (expr))))
1696
    return true;
1697
 
1698
  used_regs = get_clear_regset_from_pool ();
1699
  CLEAR_HARD_REG_SET (reg_rename_data.unavailable_hard_regs);
1700
 
1701
  collect_unavailable_regs_from_bnds (expr, bnds, used_regs, &reg_rename_data,
1702
                                      &original_insns);
1703
 
1704
#ifdef ENABLE_CHECKING
1705
  /* If after reload, make sure we're working with hard regs here.  */
1706
  if (reload_completed)
1707
    {
1708
      reg_set_iterator rsi;
1709
      unsigned i;
1710
 
1711
      EXECUTE_IF_SET_IN_REG_SET (used_regs, FIRST_PSEUDO_REGISTER, i, rsi)
1712
        gcc_unreachable ();
1713
    }
1714
#endif
1715
 
1716
  if (EXPR_SEPARABLE_P (expr))
1717
    {
1718
      rtx best_reg = NULL_RTX;
1719
      /* Check that we have computed availability of a target register
1720
         correctly.  */
1721
      verify_target_availability (expr, used_regs, &reg_rename_data);
1722
 
1723
      /* Turn everything in hard regs after reload.  */
1724
      if (reload_completed)
1725
        {
1726
          HARD_REG_SET hard_regs_used;
1727
          REG_SET_TO_HARD_REG_SET (hard_regs_used, used_regs);
1728
 
1729
          /* Join hard registers unavailable due to register class
1730
             restrictions and live range intersection.  */
1731
          IOR_HARD_REG_SET (hard_regs_used,
1732
                            reg_rename_data.unavailable_hard_regs);
1733
 
1734
          best_reg = choose_best_reg (hard_regs_used, &reg_rename_data,
1735
                                      original_insns, is_orig_reg_p);
1736
        }
1737
      else
1738
        best_reg = choose_best_pseudo_reg (used_regs, &reg_rename_data,
1739
                                           original_insns, is_orig_reg_p);
1740
 
1741
      if (!best_reg)
1742
        reg_ok = false;
1743
      else if (*is_orig_reg_p)
1744
        {
1745
          /* In case of unification BEST_REG may be different from EXPR's LHS
1746
             when EXPR's LHS is unavailable, and there is another LHS among
1747
             ORIGINAL_INSNS.  */
1748
          reg_ok = try_replace_dest_reg (original_insns, best_reg, expr);
1749
        }
1750
      else
1751
        {
1752
          /* Forbid renaming of low-cost insns.  */
1753
          if (sel_vinsn_cost (EXPR_VINSN (expr)) < 2)
1754
            reg_ok = false;
1755
          else
1756
            reg_ok = try_replace_dest_reg (original_insns, best_reg, expr);
1757
        }
1758
    }
1759
  else
1760
    {
1761
      /* If !EXPR_SCHEDULE_AS_RHS (EXPR), just make sure INSN doesn't set
1762
         any of the HARD_REGS_USED set.  */
1763
      if (vinsn_writes_one_of_regs_p (EXPR_VINSN (expr), used_regs,
1764
                                      reg_rename_data.unavailable_hard_regs))
1765
        {
1766
          reg_ok = false;
1767
          gcc_assert (EXPR_TARGET_AVAILABLE (expr) <= 0);
1768
        }
1769
      else
1770
        {
1771
          reg_ok = true;
1772
          gcc_assert (EXPR_TARGET_AVAILABLE (expr) != 0);
1773
        }
1774
    }
1775
 
1776
  ilist_clear (&original_insns);
1777
  return_regset_to_pool (used_regs);
1778
 
1779
  return reg_ok;
1780
}
1781
 
1782
 
1783
/* Return true if dependence described by DS can be overcomed.  */
1784
static bool
1785
can_speculate_dep_p (ds_t ds)
1786
{
1787
  if (spec_info == NULL)
1788
    return false;
1789
 
1790
  /* Leave only speculative data.  */
1791
  ds &= SPECULATIVE;
1792
 
1793
  if (ds == 0)
1794
    return false;
1795
 
1796
  {
1797
    /* FIXME: make sched-deps.c produce only those non-hard dependencies,
1798
       that we can overcome.  */
1799
    ds_t spec_mask = spec_info->mask;
1800
 
1801
    if ((ds & spec_mask) != ds)
1802
      return false;
1803
  }
1804
 
1805
  if (ds_weak (ds) < spec_info->data_weakness_cutoff)
1806
    return false;
1807
 
1808
  return true;
1809
}
1810
 
1811
/* Get a speculation check instruction.
1812
   C_EXPR is a speculative expression,
1813
   CHECK_DS describes speculations that should be checked,
1814
   ORIG_INSN is the original non-speculative insn in the stream.  */
1815
static insn_t
1816
create_speculation_check (expr_t c_expr, ds_t check_ds, insn_t orig_insn)
1817
{
1818
  rtx check_pattern;
1819
  rtx insn_rtx;
1820
  insn_t insn;
1821
  basic_block recovery_block;
1822
  rtx label;
1823
 
1824
  /* Create a recovery block if target is going to emit branchy check, or if
1825
     ORIG_INSN was speculative already.  */
1826
  if (targetm.sched.needs_block_p (check_ds)
1827
      || EXPR_SPEC_DONE_DS (INSN_EXPR (orig_insn)) != 0)
1828
    {
1829
      recovery_block = sel_create_recovery_block (orig_insn);
1830
      label = BB_HEAD (recovery_block);
1831
    }
1832
  else
1833
    {
1834
      recovery_block = NULL;
1835
      label = NULL_RTX;
1836
    }
1837
 
1838
  /* Get pattern of the check.  */
1839
  check_pattern = targetm.sched.gen_spec_check (EXPR_INSN_RTX (c_expr), label,
1840
                                                check_ds);
1841
 
1842
  gcc_assert (check_pattern != NULL);
1843
 
1844
  /* Emit check.  */
1845
  insn_rtx = create_insn_rtx_from_pattern (check_pattern, label);
1846
 
1847
  insn = sel_gen_insn_from_rtx_after (insn_rtx, INSN_EXPR (orig_insn),
1848
                                      INSN_SEQNO (orig_insn), orig_insn);
1849
 
1850
  /* Make check to be non-speculative.  */
1851
  EXPR_SPEC_DONE_DS (INSN_EXPR (insn)) = 0;
1852
  INSN_SPEC_CHECKED_DS (insn) = check_ds;
1853
 
1854
  /* Decrease priority of check by difference of load/check instruction
1855
     latencies.  */
1856
  EXPR_PRIORITY (INSN_EXPR (insn)) -= (sel_vinsn_cost (INSN_VINSN (orig_insn))
1857
                                       - sel_vinsn_cost (INSN_VINSN (insn)));
1858
 
1859
  /* Emit copy of original insn (though with replaced target register,
1860
     if needed) to the recovery block.  */
1861
  if (recovery_block != NULL)
1862
    {
1863
      rtx twin_rtx;
1864
 
1865
      twin_rtx = copy_rtx (PATTERN (EXPR_INSN_RTX (c_expr)));
1866
      twin_rtx = create_insn_rtx_from_pattern (twin_rtx, NULL_RTX);
1867
      sel_gen_recovery_insn_from_rtx_after (twin_rtx,
1868
                                            INSN_EXPR (orig_insn),
1869
                                            INSN_SEQNO (insn),
1870
                                            bb_note (recovery_block));
1871
    }
1872
 
1873
  /* If we've generated a data speculation check, make sure
1874
     that all the bookkeeping instruction we'll create during
1875
     this move_op () will allocate an ALAT entry so that the
1876
     check won't fail.
1877
     In case of control speculation we must convert C_EXPR to control
1878
     speculative mode, because failing to do so will bring us an exception
1879
     thrown by the non-control-speculative load.  */
1880
  check_ds = ds_get_max_dep_weak (check_ds);
1881
  speculate_expr (c_expr, check_ds);
1882
 
1883
  return insn;
1884
}
1885
 
1886
/* True when INSN is a "regN = regN" copy.  */
1887
static bool
1888
identical_copy_p (rtx insn)
1889
{
1890
  rtx lhs, rhs, pat;
1891
 
1892
  pat = PATTERN (insn);
1893
 
1894
  if (GET_CODE (pat) != SET)
1895
    return false;
1896
 
1897
  lhs = SET_DEST (pat);
1898
  if (!REG_P (lhs))
1899
    return false;
1900
 
1901
  rhs = SET_SRC (pat);
1902
  if (!REG_P (rhs))
1903
    return false;
1904
 
1905
  return REGNO (lhs) == REGNO (rhs);
1906
}
1907
 
1908
/* Undo all transformations on *AV_PTR that were done when
1909
   moving through INSN.  */
1910
static void
1911
undo_transformations (av_set_t *av_ptr, rtx insn)
1912
{
1913
  av_set_iterator av_iter;
1914
  expr_t expr;
1915
  av_set_t new_set = NULL;
1916
 
1917
  /* First, kill any EXPR that uses registers set by an insn.  This is
1918
     required for correctness.  */
1919
  FOR_EACH_EXPR_1 (expr, av_iter, av_ptr)
1920
    if (!sched_insns_conditions_mutex_p (insn, EXPR_INSN_RTX (expr))
1921
        && bitmap_intersect_p (INSN_REG_SETS (insn),
1922
                               VINSN_REG_USES (EXPR_VINSN (expr)))
1923
        /* When an insn looks like 'r1 = r1', we could substitute through
1924
           it, but the above condition will still hold.  This happened with
1925
           gcc.c-torture/execute/961125-1.c.  */
1926
        && !identical_copy_p (insn))
1927
      {
1928
        if (sched_verbose >= 6)
1929
          sel_print ("Expr %d removed due to use/set conflict\n",
1930
                     INSN_UID (EXPR_INSN_RTX (expr)));
1931
        av_set_iter_remove (&av_iter);
1932
      }
1933
 
1934
  /* Undo transformations looking at the history vector.  */
1935
  FOR_EACH_EXPR (expr, av_iter, *av_ptr)
1936
    {
1937
      int index = find_in_history_vect (EXPR_HISTORY_OF_CHANGES (expr),
1938
                                        insn, EXPR_VINSN (expr), true);
1939
 
1940
      if (index >= 0)
1941
        {
1942
          expr_history_def *phist;
1943
 
1944
          phist = VEC_index (expr_history_def,
1945
                             EXPR_HISTORY_OF_CHANGES (expr),
1946
                             index);
1947
 
1948
          switch (phist->type)
1949
            {
1950
            case TRANS_SPECULATION:
1951
              {
1952
                ds_t old_ds, new_ds;
1953
 
1954
                /* Compute the difference between old and new speculative
1955
                   statuses: that's what we need to check.
1956
                   Earlier we used to assert that the status will really
1957
                   change.  This no longer works because only the probability
1958
                   bits in the status may have changed during compute_av_set,
1959
                   and in the case of merging different probabilities of the
1960
                   same speculative status along different paths we do not
1961
                   record this in the history vector.  */
1962
                old_ds = phist->spec_ds;
1963
                new_ds = EXPR_SPEC_DONE_DS (expr);
1964
 
1965
                old_ds &= SPECULATIVE;
1966
                new_ds &= SPECULATIVE;
1967
                new_ds &= ~old_ds;
1968
 
1969
                EXPR_SPEC_TO_CHECK_DS (expr) |= new_ds;
1970
                break;
1971
              }
1972
            case TRANS_SUBSTITUTION:
1973
              {
1974
                expr_def _tmp_expr, *tmp_expr = &_tmp_expr;
1975
                vinsn_t new_vi;
1976
                bool add = true;
1977
 
1978
                new_vi = phist->old_expr_vinsn;
1979
 
1980
                gcc_assert (VINSN_SEPARABLE_P (new_vi)
1981
                            == EXPR_SEPARABLE_P (expr));
1982
                copy_expr (tmp_expr, expr);
1983
 
1984
                if (vinsn_equal_p (phist->new_expr_vinsn,
1985
                                   EXPR_VINSN (tmp_expr)))
1986
                  change_vinsn_in_expr (tmp_expr, new_vi);
1987
                else
1988
                  /* This happens when we're unsubstituting on a bookkeeping
1989
                     copy, which was in turn substituted.  The history is wrong
1990
                     in this case.  Do it the hard way.  */
1991
                  add = substitute_reg_in_expr (tmp_expr, insn, true);
1992
                if (add)
1993
                  av_set_add (&new_set, tmp_expr);
1994
                clear_expr (tmp_expr);
1995
                break;
1996
              }
1997
            default:
1998
              gcc_unreachable ();
1999
            }
2000
        }
2001
 
2002
    }
2003
 
2004
  av_set_union_and_clear (av_ptr, &new_set, NULL);
2005
}
2006
 
2007
 
2008
/* Moveup_* helpers for code motion and computing av sets.  */
2009
 
2010
/* Propagates EXPR inside an insn group through THROUGH_INSN.
2011
   The difference from the below function is that only substitution is
2012
   performed.  */
2013
static enum MOVEUP_EXPR_CODE
2014
moveup_expr_inside_insn_group (expr_t expr, insn_t through_insn)
2015
{
2016
  vinsn_t vi = EXPR_VINSN (expr);
2017
  ds_t *has_dep_p;
2018
  ds_t full_ds;
2019
 
2020
  /* Do this only inside insn group.  */
2021
  gcc_assert (INSN_SCHED_CYCLE (through_insn) > 0);
2022
 
2023
  full_ds = has_dependence_p (expr, through_insn, &has_dep_p);
2024
  if (full_ds == 0)
2025
    return MOVEUP_EXPR_SAME;
2026
 
2027
  /* Substitution is the possible choice in this case.  */
2028
  if (has_dep_p[DEPS_IN_RHS])
2029
    {
2030
      /* Can't substitute UNIQUE VINSNs.  */
2031
      gcc_assert (!VINSN_UNIQUE_P (vi));
2032
 
2033
      if (can_substitute_through_p (through_insn,
2034
                                    has_dep_p[DEPS_IN_RHS])
2035
          && substitute_reg_in_expr (expr, through_insn, false))
2036
        {
2037
          EXPR_WAS_SUBSTITUTED (expr) = true;
2038
          return MOVEUP_EXPR_CHANGED;
2039
        }
2040
 
2041
      /* Don't care about this, as even true dependencies may be allowed
2042
         in an insn group.  */
2043
      return MOVEUP_EXPR_SAME;
2044
    }
2045
 
2046
  /* This can catch output dependencies in COND_EXECs.  */
2047
  if (has_dep_p[DEPS_IN_INSN])
2048
    return MOVEUP_EXPR_NULL;
2049
 
2050
  /* This is either an output or an anti dependence, which usually have
2051
     a zero latency.  Allow this here, if we'd be wrong, tick_check_p
2052
     will fix this.  */
2053
  gcc_assert (has_dep_p[DEPS_IN_LHS]);
2054
  return MOVEUP_EXPR_AS_RHS;
2055
}
2056
 
2057
/* True when a trapping EXPR cannot be moved through THROUGH_INSN.  */
2058
#define CANT_MOVE_TRAPPING(expr, through_insn)                \
2059
  (VINSN_MAY_TRAP_P (EXPR_VINSN (expr))                       \
2060
   && !sel_insn_has_single_succ_p ((through_insn), SUCCS_ALL) \
2061
   && !sel_insn_is_speculation_check (through_insn))
2062
 
2063
/* True when a conflict on a target register was found during moveup_expr.  */
2064
static bool was_target_conflict = false;
2065
 
2066
/* Return true when moving a debug INSN across THROUGH_INSN will
2067
   create a bookkeeping block.  We don't want to create such blocks,
2068
   for they would cause codegen differences between compilations with
2069
   and without debug info.  */
2070
 
2071
static bool
2072
moving_insn_creates_bookkeeping_block_p (insn_t insn,
2073
                                         insn_t through_insn)
2074
{
2075
  basic_block bbi, bbt;
2076
  edge e1, e2;
2077
  edge_iterator ei1, ei2;
2078
 
2079
  if (!bookkeeping_can_be_created_if_moved_through_p (through_insn))
2080
    {
2081
      if (sched_verbose >= 9)
2082
        sel_print ("no bookkeeping required: ");
2083
      return FALSE;
2084
    }
2085
 
2086
  bbi = BLOCK_FOR_INSN (insn);
2087
 
2088
  if (EDGE_COUNT (bbi->preds) == 1)
2089
    {
2090
      if (sched_verbose >= 9)
2091
        sel_print ("only one pred edge: ");
2092
      return TRUE;
2093
    }
2094
 
2095
  bbt = BLOCK_FOR_INSN (through_insn);
2096
 
2097
  FOR_EACH_EDGE (e1, ei1, bbt->succs)
2098
    {
2099
      FOR_EACH_EDGE (e2, ei2, bbi->preds)
2100
        {
2101
          if (find_block_for_bookkeeping (e1, e2, TRUE))
2102
            {
2103
              if (sched_verbose >= 9)
2104
                sel_print ("found existing block: ");
2105
              return FALSE;
2106
            }
2107
        }
2108
    }
2109
 
2110
  if (sched_verbose >= 9)
2111
    sel_print ("would create bookkeeping block: ");
2112
 
2113
  return TRUE;
2114
}
2115
 
2116
/* Modifies EXPR so it can be moved through the THROUGH_INSN,
2117
   performing necessary transformations.  Record the type of transformation
2118
   made in PTRANS_TYPE, when it is not NULL.  When INSIDE_INSN_GROUP,
2119
   permit all dependencies except true ones, and try to remove those
2120
   too via forward substitution.  All cases when a non-eliminable
2121
   non-zero cost dependency exists inside an insn group will be fixed
2122
   in tick_check_p instead.  */
2123
static enum MOVEUP_EXPR_CODE
2124
moveup_expr (expr_t expr, insn_t through_insn, bool inside_insn_group,
2125
            enum local_trans_type *ptrans_type)
2126
{
2127
  vinsn_t vi = EXPR_VINSN (expr);
2128
  insn_t insn = VINSN_INSN_RTX (vi);
2129
  bool was_changed = false;
2130
  bool as_rhs = false;
2131
  ds_t *has_dep_p;
2132
  ds_t full_ds;
2133
 
2134
  /* ??? We use dependencies of non-debug insns on debug insns to
2135
     indicate that the debug insns need to be reset if the non-debug
2136
     insn is pulled ahead of it.  It's hard to figure out how to
2137
     introduce such a notion in sel-sched, but it already fails to
2138
     support debug insns in other ways, so we just go ahead and
2139
     let the deug insns go corrupt for now.  */
2140
  if (DEBUG_INSN_P (through_insn) && !DEBUG_INSN_P (insn))
2141
    return MOVEUP_EXPR_SAME;
2142
 
2143
  /* When inside_insn_group, delegate to the helper.  */
2144
  if (inside_insn_group)
2145
    return moveup_expr_inside_insn_group (expr, through_insn);
2146
 
2147
  /* Deal with unique insns and control dependencies.  */
2148
  if (VINSN_UNIQUE_P (vi))
2149
    {
2150
      /* We can move jumps without side-effects or jumps that are
2151
         mutually exclusive with instruction THROUGH_INSN (all in cases
2152
         dependencies allow to do so and jump is not speculative).  */
2153
      if (control_flow_insn_p (insn))
2154
        {
2155
          basic_block fallthru_bb;
2156
 
2157
          /* Do not move checks and do not move jumps through other
2158
             jumps.  */
2159
          if (control_flow_insn_p (through_insn)
2160
              || sel_insn_is_speculation_check (insn))
2161
            return MOVEUP_EXPR_NULL;
2162
 
2163
          /* Don't move jumps through CFG joins.  */
2164
          if (bookkeeping_can_be_created_if_moved_through_p (through_insn))
2165
            return MOVEUP_EXPR_NULL;
2166
 
2167
          /* The jump should have a clear fallthru block, and
2168
             this block should be in the current region.  */
2169
          if ((fallthru_bb = fallthru_bb_of_jump (insn)) == NULL
2170
              || ! in_current_region_p (fallthru_bb))
2171
            return MOVEUP_EXPR_NULL;
2172
 
2173
          /* And it should be mutually exclusive with through_insn.  */
2174
          if (! sched_insns_conditions_mutex_p (insn, through_insn)
2175
              && ! DEBUG_INSN_P (through_insn))
2176
            return MOVEUP_EXPR_NULL;
2177
        }
2178
 
2179
      /* Don't move what we can't move.  */
2180
      if (EXPR_CANT_MOVE (expr)
2181
          && BLOCK_FOR_INSN (through_insn) != BLOCK_FOR_INSN (insn))
2182
        return MOVEUP_EXPR_NULL;
2183
 
2184
      /* Don't move SCHED_GROUP instruction through anything.
2185
         If we don't force this, then it will be possible to start
2186
         scheduling a sched_group before all its dependencies are
2187
         resolved.
2188
         ??? Haifa deals with this issue by delaying the SCHED_GROUP
2189
         as late as possible through rank_for_schedule.  */
2190
      if (SCHED_GROUP_P (insn))
2191
        return MOVEUP_EXPR_NULL;
2192
    }
2193
  else
2194
    gcc_assert (!control_flow_insn_p (insn));
2195
 
2196
  /* Don't move debug insns if this would require bookkeeping.  */
2197
  if (DEBUG_INSN_P (insn)
2198
      && BLOCK_FOR_INSN (through_insn) != BLOCK_FOR_INSN (insn)
2199
      && moving_insn_creates_bookkeeping_block_p (insn, through_insn))
2200
    return MOVEUP_EXPR_NULL;
2201
 
2202
  /* Deal with data dependencies.  */
2203
  was_target_conflict = false;
2204
  full_ds = has_dependence_p (expr, through_insn, &has_dep_p);
2205
  if (full_ds == 0)
2206
    {
2207
      if (!CANT_MOVE_TRAPPING (expr, through_insn))
2208
        return MOVEUP_EXPR_SAME;
2209
    }
2210
  else
2211
    {
2212
      /* We can move UNIQUE insn up only as a whole and unchanged,
2213
         so it shouldn't have any dependencies.  */
2214
      if (VINSN_UNIQUE_P (vi))
2215
        return MOVEUP_EXPR_NULL;
2216
    }
2217
 
2218
  if (full_ds != 0 && can_speculate_dep_p (full_ds))
2219
    {
2220
      int res;
2221
 
2222
      res = speculate_expr (expr, full_ds);
2223
      if (res >= 0)
2224
        {
2225
          /* Speculation was successful.  */
2226
          full_ds = 0;
2227
          was_changed = (res > 0);
2228
          if (res == 2)
2229
            was_target_conflict = true;
2230
          if (ptrans_type)
2231
            *ptrans_type = TRANS_SPECULATION;
2232
          sel_clear_has_dependence ();
2233
        }
2234
    }
2235
 
2236
  if (has_dep_p[DEPS_IN_INSN])
2237
    /* We have some dependency that cannot be discarded.  */
2238
    return MOVEUP_EXPR_NULL;
2239
 
2240
  if (has_dep_p[DEPS_IN_LHS])
2241
    {
2242
      /* Only separable insns can be moved up with the new register.
2243
         Anyways, we should mark that the original register is
2244
         unavailable.  */
2245
      if (!enable_schedule_as_rhs_p || !EXPR_SEPARABLE_P (expr))
2246
        return MOVEUP_EXPR_NULL;
2247
 
2248
      EXPR_TARGET_AVAILABLE (expr) = false;
2249
      was_target_conflict = true;
2250
      as_rhs = true;
2251
    }
2252
 
2253
  /* At this point we have either separable insns, that will be lifted
2254
     up only as RHSes, or non-separable insns with no dependency in lhs.
2255
     If dependency is in RHS, then try to perform substitution and move up
2256
     substituted RHS:
2257
 
2258
      Ex. 1:                              Ex.2
2259
        y = x;                              y = x;
2260
        z = y*2;                            y = y*2;
2261
 
2262
    In Ex.1 y*2 can be substituted for x*2 and the whole operation can be
2263
    moved above y=x assignment as z=x*2.
2264
 
2265
    In Ex.2 y*2 also can be substituted for x*2, but only the right hand
2266
    side can be moved because of the output dependency.  The operation was
2267
    cropped to its rhs above.  */
2268
  if (has_dep_p[DEPS_IN_RHS])
2269
    {
2270
      ds_t *rhs_dsp = &has_dep_p[DEPS_IN_RHS];
2271
 
2272
      /* Can't substitute UNIQUE VINSNs.  */
2273
      gcc_assert (!VINSN_UNIQUE_P (vi));
2274
 
2275
      if (can_speculate_dep_p (*rhs_dsp))
2276
        {
2277
          int res;
2278
 
2279
          res = speculate_expr (expr, *rhs_dsp);
2280
          if (res >= 0)
2281
            {
2282
              /* Speculation was successful.  */
2283
              *rhs_dsp = 0;
2284
              was_changed = (res > 0);
2285
              if (res == 2)
2286
                was_target_conflict = true;
2287
              if (ptrans_type)
2288
                *ptrans_type = TRANS_SPECULATION;
2289
            }
2290
          else
2291
            return MOVEUP_EXPR_NULL;
2292
        }
2293
      else if (can_substitute_through_p (through_insn,
2294
                                         *rhs_dsp)
2295
               && substitute_reg_in_expr (expr, through_insn, false))
2296
        {
2297
          /* ??? We cannot perform substitution AND speculation on the same
2298
             insn.  */
2299
          gcc_assert (!was_changed);
2300
          was_changed = true;
2301
          if (ptrans_type)
2302
            *ptrans_type = TRANS_SUBSTITUTION;
2303
          EXPR_WAS_SUBSTITUTED (expr) = true;
2304
        }
2305
      else
2306
        return MOVEUP_EXPR_NULL;
2307
    }
2308
 
2309
  /* Don't move trapping insns through jumps.
2310
     This check should be at the end to give a chance to control speculation
2311
     to perform its duties.  */
2312
  if (CANT_MOVE_TRAPPING (expr, through_insn))
2313
    return MOVEUP_EXPR_NULL;
2314
 
2315
  return (was_changed
2316
          ? MOVEUP_EXPR_CHANGED
2317
          : (as_rhs
2318
             ? MOVEUP_EXPR_AS_RHS
2319
             : MOVEUP_EXPR_SAME));
2320
}
2321
 
2322
/* Try to look at bitmap caches for EXPR and INSN pair, return true
2323
   if successful.  When INSIDE_INSN_GROUP, also try ignore dependencies
2324
   that can exist within a parallel group.  Write to RES the resulting
2325
   code for moveup_expr.  */
2326
static bool
2327
try_bitmap_cache (expr_t expr, insn_t insn,
2328
                  bool inside_insn_group,
2329
                  enum MOVEUP_EXPR_CODE *res)
2330
{
2331
  int expr_uid = INSN_UID (EXPR_INSN_RTX (expr));
2332
 
2333
  /* First check whether we've analyzed this situation already.  */
2334
  if (bitmap_bit_p (INSN_ANALYZED_DEPS (insn), expr_uid))
2335
    {
2336
      if (bitmap_bit_p (INSN_FOUND_DEPS (insn), expr_uid))
2337
        {
2338
          if (sched_verbose >= 6)
2339
            sel_print ("removed (cached)\n");
2340
          *res = MOVEUP_EXPR_NULL;
2341
          return true;
2342
        }
2343
      else
2344
        {
2345
          if (sched_verbose >= 6)
2346
            sel_print ("unchanged (cached)\n");
2347
          *res = MOVEUP_EXPR_SAME;
2348
          return true;
2349
        }
2350
    }
2351
  else if (bitmap_bit_p (INSN_FOUND_DEPS (insn), expr_uid))
2352
    {
2353
      if (inside_insn_group)
2354
        {
2355
          if (sched_verbose >= 6)
2356
            sel_print ("unchanged (as RHS, cached, inside insn group)\n");
2357
          *res = MOVEUP_EXPR_SAME;
2358
          return true;
2359
 
2360
        }
2361
      else
2362
        EXPR_TARGET_AVAILABLE (expr) = false;
2363
 
2364
      /* This is the only case when propagation result can change over time,
2365
         as we can dynamically switch off scheduling as RHS.  In this case,
2366
         just check the flag to reach the correct decision.  */
2367
      if (enable_schedule_as_rhs_p)
2368
        {
2369
          if (sched_verbose >= 6)
2370
            sel_print ("unchanged (as RHS, cached)\n");
2371
          *res = MOVEUP_EXPR_AS_RHS;
2372
          return true;
2373
        }
2374
      else
2375
        {
2376
          if (sched_verbose >= 6)
2377
            sel_print ("removed (cached as RHS, but renaming"
2378
                       " is now disabled)\n");
2379
          *res = MOVEUP_EXPR_NULL;
2380
          return true;
2381
        }
2382
    }
2383
 
2384
  return false;
2385
}
2386
 
2387
/* Try to look at bitmap caches for EXPR and INSN pair, return true
2388
   if successful.  Write to RES the resulting code for moveup_expr.  */
2389
static bool
2390
try_transformation_cache (expr_t expr, insn_t insn,
2391
                          enum MOVEUP_EXPR_CODE *res)
2392
{
2393
  struct transformed_insns *pti
2394
    = (struct transformed_insns *)
2395
    htab_find_with_hash (INSN_TRANSFORMED_INSNS (insn),
2396
                         &EXPR_VINSN (expr),
2397
                         VINSN_HASH_RTX (EXPR_VINSN (expr)));
2398
  if (pti)
2399
    {
2400
      /* This EXPR was already moved through this insn and was
2401
         changed as a result.  Fetch the proper data from
2402
         the hashtable.  */
2403
      insert_in_history_vect (&EXPR_HISTORY_OF_CHANGES (expr),
2404
                              INSN_UID (insn), pti->type,
2405
                              pti->vinsn_old, pti->vinsn_new,
2406
                              EXPR_SPEC_DONE_DS (expr));
2407
 
2408
      if (INSN_IN_STREAM_P (VINSN_INSN_RTX (pti->vinsn_new)))
2409
        pti->vinsn_new = vinsn_copy (pti->vinsn_new, true);
2410
      change_vinsn_in_expr (expr, pti->vinsn_new);
2411
      if (pti->was_target_conflict)
2412
        EXPR_TARGET_AVAILABLE (expr) = false;
2413
      if (pti->type == TRANS_SPECULATION)
2414
        {
2415
          EXPR_SPEC_DONE_DS (expr) = pti->ds;
2416
          EXPR_NEEDS_SPEC_CHECK_P (expr) |= pti->needs_check;
2417
        }
2418
 
2419
      if (sched_verbose >= 6)
2420
        {
2421
          sel_print ("changed (cached): ");
2422
          dump_expr (expr);
2423
          sel_print ("\n");
2424
        }
2425
 
2426
      *res = MOVEUP_EXPR_CHANGED;
2427
      return true;
2428
    }
2429
 
2430
  return false;
2431
}
2432
 
2433
/* Update bitmap caches on INSN with result RES of propagating EXPR.  */
2434
static void
2435
update_bitmap_cache (expr_t expr, insn_t insn, bool inside_insn_group,
2436
                     enum MOVEUP_EXPR_CODE res)
2437
{
2438
  int expr_uid = INSN_UID (EXPR_INSN_RTX (expr));
2439
 
2440
  /* Do not cache result of propagating jumps through an insn group,
2441
     as it is always true, which is not useful outside the group.  */
2442
  if (inside_insn_group)
2443
    return;
2444
 
2445
  if (res == MOVEUP_EXPR_NULL)
2446
    {
2447
      bitmap_set_bit (INSN_ANALYZED_DEPS (insn), expr_uid);
2448
      bitmap_set_bit (INSN_FOUND_DEPS (insn), expr_uid);
2449
    }
2450
  else if (res == MOVEUP_EXPR_SAME)
2451
    {
2452
      bitmap_set_bit (INSN_ANALYZED_DEPS (insn), expr_uid);
2453
      bitmap_clear_bit (INSN_FOUND_DEPS (insn), expr_uid);
2454
    }
2455
  else if (res == MOVEUP_EXPR_AS_RHS)
2456
    {
2457
      bitmap_clear_bit (INSN_ANALYZED_DEPS (insn), expr_uid);
2458
      bitmap_set_bit (INSN_FOUND_DEPS (insn), expr_uid);
2459
    }
2460
  else
2461
    gcc_unreachable ();
2462
}
2463
 
2464
/* Update hashtable on INSN with changed EXPR, old EXPR_OLD_VINSN
2465
   and transformation type TRANS_TYPE.  */
2466
static void
2467
update_transformation_cache (expr_t expr, insn_t insn,
2468
                             bool inside_insn_group,
2469
                             enum local_trans_type trans_type,
2470
                             vinsn_t expr_old_vinsn)
2471
{
2472
  struct transformed_insns *pti;
2473
 
2474
  if (inside_insn_group)
2475
    return;
2476
 
2477
  pti = XNEW (struct transformed_insns);
2478
  pti->vinsn_old = expr_old_vinsn;
2479
  pti->vinsn_new = EXPR_VINSN (expr);
2480
  pti->type = trans_type;
2481
  pti->was_target_conflict = was_target_conflict;
2482
  pti->ds = EXPR_SPEC_DONE_DS (expr);
2483
  pti->needs_check = EXPR_NEEDS_SPEC_CHECK_P (expr);
2484
  vinsn_attach (pti->vinsn_old);
2485
  vinsn_attach (pti->vinsn_new);
2486
  *((struct transformed_insns **)
2487
    htab_find_slot_with_hash (INSN_TRANSFORMED_INSNS (insn),
2488
                              pti, VINSN_HASH_RTX (expr_old_vinsn),
2489
                              INSERT)) = pti;
2490
}
2491
 
2492
/* Same as moveup_expr, but first looks up the result of
2493
   transformation in caches.  */
2494
static enum MOVEUP_EXPR_CODE
2495
moveup_expr_cached (expr_t expr, insn_t insn, bool inside_insn_group)
2496
{
2497
  enum MOVEUP_EXPR_CODE res;
2498
  bool got_answer = false;
2499
 
2500
  if (sched_verbose >= 6)
2501
    {
2502
      sel_print ("Moving ");
2503
      dump_expr (expr);
2504
      sel_print (" through %d: ", INSN_UID (insn));
2505
    }
2506
 
2507
  if (DEBUG_INSN_P (EXPR_INSN_RTX (expr))
2508
      && (sel_bb_head (BLOCK_FOR_INSN (EXPR_INSN_RTX (expr)))
2509
          == EXPR_INSN_RTX (expr)))
2510
    /* Don't use cached information for debug insns that are heads of
2511
       basic blocks.  */;
2512
  else if (try_bitmap_cache (expr, insn, inside_insn_group, &res))
2513
    /* When inside insn group, we do not want remove stores conflicting
2514
       with previosly issued loads.  */
2515
    got_answer = ! inside_insn_group || res != MOVEUP_EXPR_NULL;
2516
  else if (try_transformation_cache (expr, insn, &res))
2517
    got_answer = true;
2518
 
2519
  if (! got_answer)
2520
    {
2521
      /* Invoke moveup_expr and record the results.  */
2522
      vinsn_t expr_old_vinsn = EXPR_VINSN (expr);
2523
      ds_t expr_old_spec_ds = EXPR_SPEC_DONE_DS (expr);
2524
      int expr_uid = INSN_UID (VINSN_INSN_RTX (expr_old_vinsn));
2525
      bool unique_p = VINSN_UNIQUE_P (expr_old_vinsn);
2526
      enum local_trans_type trans_type = TRANS_SUBSTITUTION;
2527
 
2528
      /* ??? Invent something better than this.  We can't allow old_vinsn
2529
         to go, we need it for the history vector.  */
2530
      vinsn_attach (expr_old_vinsn);
2531
 
2532
      res = moveup_expr (expr, insn, inside_insn_group,
2533
                         &trans_type);
2534
      switch (res)
2535
        {
2536
        case MOVEUP_EXPR_NULL:
2537
          update_bitmap_cache (expr, insn, inside_insn_group, res);
2538
          if (sched_verbose >= 6)
2539
            sel_print ("removed\n");
2540
          break;
2541
 
2542
        case MOVEUP_EXPR_SAME:
2543
          update_bitmap_cache (expr, insn, inside_insn_group, res);
2544
          if (sched_verbose >= 6)
2545
            sel_print ("unchanged\n");
2546
          break;
2547
 
2548
        case MOVEUP_EXPR_AS_RHS:
2549
          gcc_assert (!unique_p || inside_insn_group);
2550
          update_bitmap_cache (expr, insn, inside_insn_group, res);
2551
          if (sched_verbose >= 6)
2552
            sel_print ("unchanged (as RHS)\n");
2553
          break;
2554
 
2555
        case MOVEUP_EXPR_CHANGED:
2556
          gcc_assert (INSN_UID (EXPR_INSN_RTX (expr)) != expr_uid
2557
                      || EXPR_SPEC_DONE_DS (expr) != expr_old_spec_ds);
2558
          insert_in_history_vect (&EXPR_HISTORY_OF_CHANGES (expr),
2559
                                  INSN_UID (insn), trans_type,
2560
                                  expr_old_vinsn, EXPR_VINSN (expr),
2561
                                  expr_old_spec_ds);
2562
          update_transformation_cache (expr, insn, inside_insn_group,
2563
                                       trans_type, expr_old_vinsn);
2564
          if (sched_verbose >= 6)
2565
            {
2566
              sel_print ("changed: ");
2567
              dump_expr (expr);
2568
              sel_print ("\n");
2569
            }
2570
          break;
2571
        default:
2572
          gcc_unreachable ();
2573
        }
2574
 
2575
      vinsn_detach (expr_old_vinsn);
2576
    }
2577
 
2578
  return res;
2579
}
2580
 
2581
/* Moves an av set AVP up through INSN, performing necessary
2582
   transformations.  */
2583
static void
2584
moveup_set_expr (av_set_t *avp, insn_t insn, bool inside_insn_group)
2585
{
2586
  av_set_iterator i;
2587
  expr_t expr;
2588
 
2589
  FOR_EACH_EXPR_1 (expr, i, avp)
2590
    {
2591
 
2592
      switch (moveup_expr_cached (expr, insn, inside_insn_group))
2593
        {
2594
        case MOVEUP_EXPR_SAME:
2595
        case MOVEUP_EXPR_AS_RHS:
2596
          break;
2597
 
2598
        case MOVEUP_EXPR_NULL:
2599
          av_set_iter_remove (&i);
2600
          break;
2601
 
2602
        case MOVEUP_EXPR_CHANGED:
2603
          expr = merge_with_other_exprs (avp, &i, expr);
2604
          break;
2605
 
2606
        default:
2607
          gcc_unreachable ();
2608
        }
2609
    }
2610
}
2611
 
2612
/* Moves AVP set along PATH.  */
2613
static void
2614
moveup_set_inside_insn_group (av_set_t *avp, ilist_t path)
2615
{
2616
  int last_cycle;
2617
 
2618
  if (sched_verbose >= 6)
2619
    sel_print ("Moving expressions up in the insn group...\n");
2620
  if (! path)
2621
    return;
2622
  last_cycle = INSN_SCHED_CYCLE (ILIST_INSN (path));
2623
  while (path
2624
         && INSN_SCHED_CYCLE (ILIST_INSN (path)) == last_cycle)
2625
    {
2626
      moveup_set_expr (avp, ILIST_INSN (path), true);
2627
      path = ILIST_NEXT (path);
2628
    }
2629
}
2630
 
2631
/* Returns true if after moving EXPR along PATH it equals to EXPR_VLIW.  */
2632
static bool
2633
equal_after_moveup_path_p (expr_t expr, ilist_t path, expr_t expr_vliw)
2634
{
2635
  expr_def _tmp, *tmp = &_tmp;
2636
  int last_cycle;
2637
  bool res = true;
2638
 
2639
  copy_expr_onside (tmp, expr);
2640
  last_cycle = path ? INSN_SCHED_CYCLE (ILIST_INSN (path)) : 0;
2641
  while (path
2642
         && res
2643
         && INSN_SCHED_CYCLE (ILIST_INSN (path)) == last_cycle)
2644
    {
2645
      res = (moveup_expr_cached (tmp, ILIST_INSN (path), true)
2646
             != MOVEUP_EXPR_NULL);
2647
      path = ILIST_NEXT (path);
2648
    }
2649
 
2650
  if (res)
2651
    {
2652
      vinsn_t tmp_vinsn = EXPR_VINSN (tmp);
2653
      vinsn_t expr_vliw_vinsn = EXPR_VINSN (expr_vliw);
2654
 
2655
      if (tmp_vinsn != expr_vliw_vinsn)
2656
        res = vinsn_equal_p (tmp_vinsn, expr_vliw_vinsn);
2657
    }
2658
 
2659
  clear_expr (tmp);
2660
  return res;
2661
}
2662
 
2663
 
2664
/* Functions that compute av and lv sets.  */
2665
 
2666
/* Returns true if INSN is not a downward continuation of the given path P in
2667
   the current stage.  */
2668
static bool
2669
is_ineligible_successor (insn_t insn, ilist_t p)
2670
{
2671
  insn_t prev_insn;
2672
 
2673
  /* Check if insn is not deleted.  */
2674
  if (PREV_INSN (insn) && NEXT_INSN (PREV_INSN (insn)) != insn)
2675
    gcc_unreachable ();
2676
  else if (NEXT_INSN (insn) && PREV_INSN (NEXT_INSN (insn)) != insn)
2677
    gcc_unreachable ();
2678
 
2679
  /* If it's the first insn visited, then the successor is ok.  */
2680
  if (!p)
2681
    return false;
2682
 
2683
  prev_insn = ILIST_INSN (p);
2684
 
2685
  if (/* a backward edge.  */
2686
      INSN_SEQNO (insn) < INSN_SEQNO (prev_insn)
2687
      /* is already visited.  */
2688
      || (INSN_SEQNO (insn) == INSN_SEQNO (prev_insn)
2689
          && (ilist_is_in_p (p, insn)
2690
              /* We can reach another fence here and still seqno of insn
2691
                 would be equal to seqno of prev_insn.  This is possible
2692
                 when prev_insn is a previously created bookkeeping copy.
2693
                 In that case it'd get a seqno of insn.  Thus, check here
2694
                 whether insn is in current fence too.  */
2695
              || IN_CURRENT_FENCE_P (insn)))
2696
      /* Was already scheduled on this round.  */
2697
      || (INSN_SEQNO (insn) > INSN_SEQNO (prev_insn)
2698
          && IN_CURRENT_FENCE_P (insn))
2699
      /* An insn from another fence could also be
2700
         scheduled earlier even if this insn is not in
2701
         a fence list right now.  Check INSN_SCHED_CYCLE instead.  */
2702
      || (!pipelining_p
2703
          && INSN_SCHED_TIMES (insn) > 0))
2704
    return true;
2705
  else
2706
    return false;
2707
}
2708
 
2709
/* Computes the av_set below the last bb insn INSN, doing all the 'dirty work'
2710
   of handling multiple successors and properly merging its av_sets.  P is
2711
   the current path traversed.  WS is the size of lookahead window.
2712
   Return the av set computed.  */
2713
static av_set_t
2714
compute_av_set_at_bb_end (insn_t insn, ilist_t p, int ws)
2715
{
2716
  struct succs_info *sinfo;
2717
  av_set_t expr_in_all_succ_branches = NULL;
2718
  int is;
2719
  insn_t succ, zero_succ = NULL;
2720
  av_set_t av1 = NULL;
2721
 
2722
  gcc_assert (sel_bb_end_p (insn));
2723
 
2724
  /* Find different kind of successors needed for correct computing of
2725
     SPEC and TARGET_AVAILABLE attributes.  */
2726
  sinfo = compute_succs_info (insn, SUCCS_NORMAL);
2727
 
2728
  /* Debug output.  */
2729
  if (sched_verbose >= 6)
2730
    {
2731
      sel_print ("successors of bb end (%d): ", INSN_UID (insn));
2732
      dump_insn_vector (sinfo->succs_ok);
2733
      sel_print ("\n");
2734
      if (sinfo->succs_ok_n != sinfo->all_succs_n)
2735
        sel_print ("real successors num: %d\n", sinfo->all_succs_n);
2736
    }
2737
 
2738
  /* Add insn to the tail of current path.  */
2739
  ilist_add (&p, insn);
2740
 
2741
  FOR_EACH_VEC_ELT (rtx, sinfo->succs_ok, is, succ)
2742
    {
2743
      av_set_t succ_set;
2744
 
2745
      /* We will edit SUCC_SET and EXPR_SPEC field of its elements.  */
2746
      succ_set = compute_av_set_inside_bb (succ, p, ws, true);
2747
 
2748
      av_set_split_usefulness (succ_set,
2749
                               VEC_index (int, sinfo->probs_ok, is),
2750
                               sinfo->all_prob);
2751
 
2752
      if (sinfo->all_succs_n > 1)
2753
        {
2754
          /* Find EXPR'es that came from *all* successors and save them
2755
             into expr_in_all_succ_branches.  This set will be used later
2756
             for calculating speculation attributes of EXPR'es.  */
2757
          if (is == 0)
2758
            {
2759
              expr_in_all_succ_branches = av_set_copy (succ_set);
2760
 
2761
              /* Remember the first successor for later. */
2762
              zero_succ = succ;
2763
            }
2764
          else
2765
            {
2766
              av_set_iterator i;
2767
              expr_t expr;
2768
 
2769
              FOR_EACH_EXPR_1 (expr, i, &expr_in_all_succ_branches)
2770
                if (!av_set_is_in_p (succ_set, EXPR_VINSN (expr)))
2771
                  av_set_iter_remove (&i);
2772
            }
2773
        }
2774
 
2775
      /* Union the av_sets.  Check liveness restrictions on target registers
2776
         in special case of two successors.  */
2777
      if (sinfo->succs_ok_n == 2 && is == 1)
2778
        {
2779
          basic_block bb0 = BLOCK_FOR_INSN (zero_succ);
2780
          basic_block bb1 = BLOCK_FOR_INSN (succ);
2781
 
2782
          gcc_assert (BB_LV_SET_VALID_P (bb0) && BB_LV_SET_VALID_P (bb1));
2783
          av_set_union_and_live (&av1, &succ_set,
2784
                                 BB_LV_SET (bb0),
2785
                                 BB_LV_SET (bb1),
2786
                                 insn);
2787
        }
2788
      else
2789
        av_set_union_and_clear (&av1, &succ_set, insn);
2790
    }
2791
 
2792
  /* Check liveness restrictions via hard way when there are more than
2793
     two successors.  */
2794
  if (sinfo->succs_ok_n > 2)
2795
    FOR_EACH_VEC_ELT (rtx, sinfo->succs_ok, is, succ)
2796
      {
2797
        basic_block succ_bb = BLOCK_FOR_INSN (succ);
2798
 
2799
        gcc_assert (BB_LV_SET_VALID_P (succ_bb));
2800
        mark_unavailable_targets (av1, BB_AV_SET (succ_bb),
2801
                                  BB_LV_SET (succ_bb));
2802
      }
2803
 
2804
  /* Finally, check liveness restrictions on paths leaving the region.  */
2805
  if (sinfo->all_succs_n > sinfo->succs_ok_n)
2806
    FOR_EACH_VEC_ELT (rtx, sinfo->succs_other, is, succ)
2807
      mark_unavailable_targets
2808
        (av1, NULL, BB_LV_SET (BLOCK_FOR_INSN (succ)));
2809
 
2810
  if (sinfo->all_succs_n > 1)
2811
    {
2812
      av_set_iterator i;
2813
      expr_t expr;
2814
 
2815
      /* Increase the spec attribute of all EXPR'es that didn't come
2816
         from all successors.  */
2817
      FOR_EACH_EXPR (expr, i, av1)
2818
        if (!av_set_is_in_p (expr_in_all_succ_branches, EXPR_VINSN (expr)))
2819
          EXPR_SPEC (expr)++;
2820
 
2821
      av_set_clear (&expr_in_all_succ_branches);
2822
 
2823
      /* Do not move conditional branches through other
2824
         conditional branches.  So, remove all conditional
2825
         branches from av_set if current operator is a conditional
2826
         branch.  */
2827
      av_set_substract_cond_branches (&av1);
2828
    }
2829
 
2830
  ilist_remove (&p);
2831
  free_succs_info (sinfo);
2832
 
2833
  if (sched_verbose >= 6)
2834
    {
2835
      sel_print ("av_succs (%d): ", INSN_UID (insn));
2836
      dump_av_set (av1);
2837
      sel_print ("\n");
2838
    }
2839
 
2840
  return av1;
2841
}
2842
 
2843
/* This function computes av_set for the FIRST_INSN by dragging valid
2844
   av_set through all basic block insns either from the end of basic block
2845
   (computed using compute_av_set_at_bb_end) or from the insn on which
2846
   MAX_WS was exceeded.  It uses compute_av_set_at_bb_end to compute av_set
2847
   below the basic block and handling conditional branches.
2848
   FIRST_INSN - the basic block head, P - path consisting of the insns
2849
   traversed on the way to the FIRST_INSN (the path is sparse, only bb heads
2850
   and bb ends are added to the path), WS - current window size,
2851
   NEED_COPY_P - true if we'll make a copy of av_set before returning it.  */
2852
static av_set_t
2853
compute_av_set_inside_bb (insn_t first_insn, ilist_t p, int ws,
2854
                          bool need_copy_p)
2855
{
2856
  insn_t cur_insn;
2857
  int end_ws = ws;
2858
  insn_t bb_end = sel_bb_end (BLOCK_FOR_INSN (first_insn));
2859
  insn_t after_bb_end = NEXT_INSN (bb_end);
2860
  insn_t last_insn;
2861
  av_set_t av = NULL;
2862
  basic_block cur_bb = BLOCK_FOR_INSN (first_insn);
2863
 
2864
  /* Return NULL if insn is not on the legitimate downward path.  */
2865
  if (is_ineligible_successor (first_insn, p))
2866
    {
2867
      if (sched_verbose >= 6)
2868
        sel_print ("Insn %d is ineligible_successor\n", INSN_UID (first_insn));
2869
 
2870
      return NULL;
2871
    }
2872
 
2873
  /* If insn already has valid av(insn) computed, just return it.  */
2874
  if (AV_SET_VALID_P (first_insn))
2875
    {
2876
      av_set_t av_set;
2877
 
2878
      if (sel_bb_head_p (first_insn))
2879
        av_set = BB_AV_SET (BLOCK_FOR_INSN (first_insn));
2880
      else
2881
        av_set = NULL;
2882
 
2883
      if (sched_verbose >= 6)
2884
        {
2885
          sel_print ("Insn %d has a valid av set: ", INSN_UID (first_insn));
2886
          dump_av_set (av_set);
2887
          sel_print ("\n");
2888
        }
2889
 
2890
      return need_copy_p ? av_set_copy (av_set) : av_set;
2891
    }
2892
 
2893
  ilist_add (&p, first_insn);
2894
 
2895
  /* As the result after this loop have completed, in LAST_INSN we'll
2896
     have the insn which has valid av_set to start backward computation
2897
     from: it either will be NULL because on it the window size was exceeded
2898
     or other valid av_set as returned by compute_av_set for the last insn
2899
     of the basic block.  */
2900
  for (last_insn = first_insn; last_insn != after_bb_end;
2901
       last_insn = NEXT_INSN (last_insn))
2902
    {
2903
      /* We may encounter valid av_set not only on bb_head, but also on
2904
         those insns on which previously MAX_WS was exceeded.  */
2905
      if (AV_SET_VALID_P (last_insn))
2906
        {
2907
          if (sched_verbose >= 6)
2908
            sel_print ("Insn %d has a valid empty av set\n", INSN_UID (last_insn));
2909
          break;
2910
        }
2911
 
2912
      /* The special case: the last insn of the BB may be an
2913
         ineligible_successor due to its SEQ_NO that was set on
2914
         it as a bookkeeping.  */
2915
      if (last_insn != first_insn
2916
          && is_ineligible_successor (last_insn, p))
2917
        {
2918
          if (sched_verbose >= 6)
2919
            sel_print ("Insn %d is ineligible_successor\n", INSN_UID (last_insn));
2920
          break;
2921
        }
2922
 
2923
      if (DEBUG_INSN_P (last_insn))
2924
        continue;
2925
 
2926
      if (end_ws > max_ws)
2927
        {
2928
          /* We can reach max lookahead size at bb_header, so clean av_set
2929
             first.  */
2930
          INSN_WS_LEVEL (last_insn) = global_level;
2931
 
2932
          if (sched_verbose >= 6)
2933
            sel_print ("Insn %d is beyond the software lookahead window size\n",
2934
                       INSN_UID (last_insn));
2935
          break;
2936
        }
2937
 
2938
      end_ws++;
2939
    }
2940
 
2941
  /* Get the valid av_set into AV above the LAST_INSN to start backward
2942
     computation from.  It either will be empty av_set or av_set computed from
2943
     the successors on the last insn of the current bb.  */
2944
  if (last_insn != after_bb_end)
2945
    {
2946
      av = NULL;
2947
 
2948
      /* This is needed only to obtain av_sets that are identical to
2949
         those computed by the old compute_av_set version.  */
2950
      if (last_insn == first_insn && !INSN_NOP_P (last_insn))
2951
        av_set_add (&av, INSN_EXPR (last_insn));
2952
    }
2953
  else
2954
    /* END_WS is always already increased by 1 if LAST_INSN == AFTER_BB_END.  */
2955
    av = compute_av_set_at_bb_end (bb_end, p, end_ws);
2956
 
2957
  /* Compute av_set in AV starting from below the LAST_INSN up to
2958
     location above the FIRST_INSN.  */
2959
  for (cur_insn = PREV_INSN (last_insn); cur_insn != PREV_INSN (first_insn);
2960
       cur_insn = PREV_INSN (cur_insn))
2961
    if (!INSN_NOP_P (cur_insn))
2962
      {
2963
        expr_t expr;
2964
 
2965
        moveup_set_expr (&av, cur_insn, false);
2966
 
2967
        /* If the expression for CUR_INSN is already in the set,
2968
           replace it by the new one.  */
2969
        expr = av_set_lookup (av, INSN_VINSN (cur_insn));
2970
        if (expr != NULL)
2971
          {
2972
            clear_expr (expr);
2973
            copy_expr (expr, INSN_EXPR (cur_insn));
2974
          }
2975
        else
2976
          av_set_add (&av, INSN_EXPR (cur_insn));
2977
      }
2978
 
2979
  /* Clear stale bb_av_set.  */
2980
  if (sel_bb_head_p (first_insn))
2981
    {
2982
      av_set_clear (&BB_AV_SET (cur_bb));
2983
      BB_AV_SET (cur_bb) = need_copy_p ? av_set_copy (av) : av;
2984
      BB_AV_LEVEL (cur_bb) = global_level;
2985
    }
2986
 
2987
  if (sched_verbose >= 6)
2988
    {
2989
      sel_print ("Computed av set for insn %d: ", INSN_UID (first_insn));
2990
      dump_av_set (av);
2991
      sel_print ("\n");
2992
    }
2993
 
2994
  ilist_remove (&p);
2995
  return av;
2996
}
2997
 
2998
/* Compute av set before INSN.
2999
   INSN - the current operation (actual rtx INSN)
3000
   P - the current path, which is list of insns visited so far
3001
   WS - software lookahead window size.
3002
   UNIQUE_P - TRUE, if returned av_set will be changed, hence
3003
   if we want to save computed av_set in s_i_d, we should make a copy of it.
3004
 
3005
   In the resulting set we will have only expressions that don't have delay
3006
   stalls and nonsubstitutable dependences.  */
3007
static av_set_t
3008
compute_av_set (insn_t insn, ilist_t p, int ws, bool unique_p)
3009
{
3010
  return compute_av_set_inside_bb (insn, p, ws, unique_p);
3011
}
3012
 
3013
/* Propagate a liveness set LV through INSN.  */
3014
static void
3015
propagate_lv_set (regset lv, insn_t insn)
3016
{
3017
  gcc_assert (INSN_P (insn));
3018
 
3019
  if (INSN_NOP_P (insn))
3020
    return;
3021
 
3022
  df_simulate_one_insn_backwards (BLOCK_FOR_INSN (insn), insn, lv);
3023
}
3024
 
3025
/* Return livness set at the end of BB.  */
3026
static regset
3027
compute_live_after_bb (basic_block bb)
3028
{
3029
  edge e;
3030
  edge_iterator ei;
3031
  regset lv = get_clear_regset_from_pool ();
3032
 
3033
  gcc_assert (!ignore_first);
3034
 
3035
  FOR_EACH_EDGE (e, ei, bb->succs)
3036
    if (sel_bb_empty_p (e->dest))
3037
      {
3038
        if (! BB_LV_SET_VALID_P (e->dest))
3039
          {
3040
            gcc_unreachable ();
3041
            gcc_assert (BB_LV_SET (e->dest) == NULL);
3042
            BB_LV_SET (e->dest) = compute_live_after_bb (e->dest);
3043
            BB_LV_SET_VALID_P (e->dest) = true;
3044
          }
3045
        IOR_REG_SET (lv, BB_LV_SET (e->dest));
3046
      }
3047
    else
3048
      IOR_REG_SET (lv, compute_live (sel_bb_head (e->dest)));
3049
 
3050
  return lv;
3051
}
3052
 
3053
/* Compute the set of all live registers at the point before INSN and save
3054
   it at INSN if INSN is bb header.  */
3055
regset
3056
compute_live (insn_t insn)
3057
{
3058
  basic_block bb = BLOCK_FOR_INSN (insn);
3059
  insn_t final, temp;
3060
  regset lv;
3061
 
3062
  /* Return the valid set if we're already on it.  */
3063
  if (!ignore_first)
3064
    {
3065
      regset src = NULL;
3066
 
3067
      if (sel_bb_head_p (insn) && BB_LV_SET_VALID_P (bb))
3068
        src = BB_LV_SET (bb);
3069
      else
3070
        {
3071
          gcc_assert (in_current_region_p (bb));
3072
          if (INSN_LIVE_VALID_P (insn))
3073
            src = INSN_LIVE (insn);
3074
        }
3075
 
3076
      if (src)
3077
        {
3078
          lv = get_regset_from_pool ();
3079
          COPY_REG_SET (lv, src);
3080
 
3081
          if (sel_bb_head_p (insn) && ! BB_LV_SET_VALID_P (bb))
3082
            {
3083
              COPY_REG_SET (BB_LV_SET (bb), lv);
3084
              BB_LV_SET_VALID_P (bb) = true;
3085
            }
3086
 
3087
          return_regset_to_pool (lv);
3088
          return lv;
3089
        }
3090
    }
3091
 
3092
  /* We've skipped the wrong lv_set.  Don't skip the right one.  */
3093
  ignore_first = false;
3094
  gcc_assert (in_current_region_p (bb));
3095
 
3096
  /* Find a valid LV set in this block or below, if needed.
3097
     Start searching from the next insn: either ignore_first is true, or
3098
     INSN doesn't have a correct live set.  */
3099
  temp = NEXT_INSN (insn);
3100
  final = NEXT_INSN (BB_END (bb));
3101
  while (temp != final && ! INSN_LIVE_VALID_P (temp))
3102
    temp = NEXT_INSN (temp);
3103
  if (temp == final)
3104
    {
3105
      lv = compute_live_after_bb (bb);
3106
      temp = PREV_INSN (temp);
3107
    }
3108
  else
3109
    {
3110
      lv = get_regset_from_pool ();
3111
      COPY_REG_SET (lv, INSN_LIVE (temp));
3112
    }
3113
 
3114
  /* Put correct lv sets on the insns which have bad sets.  */
3115
  final = PREV_INSN (insn);
3116
  while (temp != final)
3117
    {
3118
      propagate_lv_set (lv, temp);
3119
      COPY_REG_SET (INSN_LIVE (temp), lv);
3120
      INSN_LIVE_VALID_P (temp) = true;
3121
      temp = PREV_INSN (temp);
3122
    }
3123
 
3124
  /* Also put it in a BB.  */
3125
  if (sel_bb_head_p (insn))
3126
    {
3127
      basic_block bb = BLOCK_FOR_INSN (insn);
3128
 
3129
      COPY_REG_SET (BB_LV_SET (bb), lv);
3130
      BB_LV_SET_VALID_P (bb) = true;
3131
    }
3132
 
3133
  /* We return LV to the pool, but will not clear it there.  Thus we can
3134
     legimatelly use LV till the next use of regset_pool_get ().  */
3135
  return_regset_to_pool (lv);
3136
  return lv;
3137
}
3138
 
3139
/* Update liveness sets for INSN.  */
3140
static inline void
3141
update_liveness_on_insn (rtx insn)
3142
{
3143
  ignore_first = true;
3144
  compute_live (insn);
3145
}
3146
 
3147
/* Compute liveness below INSN and write it into REGS.  */
3148
static inline void
3149
compute_live_below_insn (rtx insn, regset regs)
3150
{
3151
  rtx succ;
3152
  succ_iterator si;
3153
 
3154
  FOR_EACH_SUCC_1 (succ, si, insn, SUCCS_ALL)
3155
    IOR_REG_SET (regs, compute_live (succ));
3156
}
3157
 
3158
/* Update the data gathered in av and lv sets starting from INSN.  */
3159
static void
3160
update_data_sets (rtx insn)
3161
{
3162
  update_liveness_on_insn (insn);
3163
  if (sel_bb_head_p (insn))
3164
    {
3165
      gcc_assert (AV_LEVEL (insn) != 0);
3166
      BB_AV_LEVEL (BLOCK_FOR_INSN (insn)) = -1;
3167
      compute_av_set (insn, NULL, 0, 0);
3168
    }
3169
}
3170
 
3171
 
3172
/* Helper for move_op () and find_used_regs ().
3173
   Return speculation type for which a check should be created on the place
3174
   of INSN.  EXPR is one of the original ops we are searching for.  */
3175
static ds_t
3176
get_spec_check_type_for_insn (insn_t insn, expr_t expr)
3177
{
3178
  ds_t to_check_ds;
3179
  ds_t already_checked_ds = EXPR_SPEC_DONE_DS (INSN_EXPR (insn));
3180
 
3181
  to_check_ds = EXPR_SPEC_TO_CHECK_DS (expr);
3182
 
3183
  if (targetm.sched.get_insn_checked_ds)
3184
    already_checked_ds |= targetm.sched.get_insn_checked_ds (insn);
3185
 
3186
  if (spec_info != NULL
3187
      && (spec_info->flags & SEL_SCHED_SPEC_DONT_CHECK_CONTROL))
3188
    already_checked_ds |= BEGIN_CONTROL;
3189
 
3190
  already_checked_ds = ds_get_speculation_types (already_checked_ds);
3191
 
3192
  to_check_ds &= ~already_checked_ds;
3193
 
3194
  return to_check_ds;
3195
}
3196
 
3197
/* Find the set of registers that are unavailable for storing expres
3198
   while moving ORIG_OPS up on the path starting from INSN due to
3199
   liveness (USED_REGS) or hardware restrictions (REG_RENAME_P).
3200
 
3201
   All the original operations found during the traversal are saved in the
3202
   ORIGINAL_INSNS list.
3203
 
3204
   REG_RENAME_P denotes the set of hardware registers that
3205
   can not be used with renaming due to the register class restrictions,
3206
   mode restrictions and other (the register we'll choose should be
3207
   compatible class with the original uses, shouldn't be in call_used_regs,
3208
   should be HARD_REGNO_RENAME_OK etc).
3209
 
3210
   Returns TRUE if we've found all original insns, FALSE otherwise.
3211
 
3212
   This function utilizes code_motion_path_driver (formerly find_used_regs_1)
3213
   to traverse the code motion paths.  This helper function finds registers
3214
   that are not available for storing expres while moving ORIG_OPS up on the
3215
   path starting from INSN.  A register considered as used on the moving path,
3216
   if one of the following conditions is not satisfied:
3217
 
3218
      (1) a register not set or read on any path from xi to an instance of
3219
          the original operation,
3220
      (2) not among the live registers of the point immediately following the
3221
          first original operation on a given downward path, except for the
3222
          original target register of the operation,
3223
      (3) not live on the other path of any conditional branch that is passed
3224
          by the operation, in case original operations are not present on
3225
          both paths of the conditional branch.
3226
 
3227
   All the original operations found during the traversal are saved in the
3228
   ORIGINAL_INSNS list.
3229
 
3230
   REG_RENAME_P->CROSSES_CALL is true, if there is a call insn on the path
3231
   from INSN to original insn. In this case CALL_USED_REG_SET will be added
3232
   to unavailable hard regs at the point original operation is found.  */
3233
 
3234
static bool
3235
find_used_regs (insn_t insn, av_set_t orig_ops, regset used_regs,
3236
                struct reg_rename  *reg_rename_p, def_list_t *original_insns)
3237
{
3238
  def_list_iterator i;
3239
  def_t def;
3240
  int res;
3241
  bool needs_spec_check_p = false;
3242
  expr_t expr;
3243
  av_set_iterator expr_iter;
3244
  struct fur_static_params sparams;
3245
  struct cmpd_local_params lparams;
3246
 
3247
  /* We haven't visited any blocks yet.  */
3248
  bitmap_clear (code_motion_visited_blocks);
3249
 
3250
  /* Init parameters for code_motion_path_driver.  */
3251
  sparams.crosses_call = false;
3252
  sparams.original_insns = original_insns;
3253
  sparams.used_regs = used_regs;
3254
 
3255
  /* Set the appropriate hooks and data.  */
3256
  code_motion_path_driver_info = &fur_hooks;
3257
 
3258
  res = code_motion_path_driver (insn, orig_ops, NULL, &lparams, &sparams);
3259
 
3260
  reg_rename_p->crosses_call |= sparams.crosses_call;
3261
 
3262
  gcc_assert (res == 1);
3263
  gcc_assert (original_insns && *original_insns);
3264
 
3265
  /* ??? We calculate whether an expression needs a check when computing
3266
     av sets.  This information is not as precise as it could be due to
3267
     merging this bit in merge_expr.  We can do better in find_used_regs,
3268
     but we want to avoid multiple traversals of the same code motion
3269
     paths.  */
3270
  FOR_EACH_EXPR (expr, expr_iter, orig_ops)
3271
    needs_spec_check_p |= EXPR_NEEDS_SPEC_CHECK_P (expr);
3272
 
3273
  /* Mark hardware regs in REG_RENAME_P that are not suitable
3274
     for renaming expr in INSN due to hardware restrictions (register class,
3275
     modes compatibility etc).  */
3276
  FOR_EACH_DEF (def, i, *original_insns)
3277
    {
3278
      vinsn_t vinsn = INSN_VINSN (def->orig_insn);
3279
 
3280
      if (VINSN_SEPARABLE_P (vinsn))
3281
        mark_unavailable_hard_regs (def, reg_rename_p, used_regs);
3282
 
3283
      /* Do not allow clobbering of ld.[sa] address in case some of the
3284
         original operations need a check.  */
3285
      if (needs_spec_check_p)
3286
        IOR_REG_SET (used_regs, VINSN_REG_USES (vinsn));
3287
    }
3288
 
3289
  return true;
3290
}
3291
 
3292
 
3293
/* Functions to choose the best insn from available ones.  */
3294
 
3295
/* Adjusts the priority for EXPR using the backend *_adjust_priority hook.  */
3296
static int
3297
sel_target_adjust_priority (expr_t expr)
3298
{
3299
  int priority = EXPR_PRIORITY (expr);
3300
  int new_priority;
3301
 
3302
  if (targetm.sched.adjust_priority)
3303
    new_priority = targetm.sched.adjust_priority (EXPR_INSN_RTX (expr), priority);
3304
  else
3305
    new_priority = priority;
3306
 
3307
  /* If the priority has changed, adjust EXPR_PRIORITY_ADJ accordingly.  */
3308
  EXPR_PRIORITY_ADJ (expr) = new_priority - EXPR_PRIORITY (expr);
3309
 
3310
  gcc_assert (EXPR_PRIORITY_ADJ (expr) >= 0);
3311
 
3312
  if (sched_verbose >= 4)
3313
    sel_print ("sel_target_adjust_priority: insn %d,  %d+%d = %d.\n",
3314
               INSN_UID (EXPR_INSN_RTX (expr)), EXPR_PRIORITY (expr),
3315
               EXPR_PRIORITY_ADJ (expr), new_priority);
3316
 
3317
  return new_priority;
3318
}
3319
 
3320
/* Rank two available exprs for schedule.  Never return 0 here.  */
3321
static int
3322
sel_rank_for_schedule (const void *x, const void *y)
3323
{
3324
  expr_t tmp = *(const expr_t *) y;
3325
  expr_t tmp2 = *(const expr_t *) x;
3326
  insn_t tmp_insn, tmp2_insn;
3327
  vinsn_t tmp_vinsn, tmp2_vinsn;
3328
  int val;
3329
 
3330
  tmp_vinsn = EXPR_VINSN (tmp);
3331
  tmp2_vinsn = EXPR_VINSN (tmp2);
3332
  tmp_insn = EXPR_INSN_RTX (tmp);
3333
  tmp2_insn = EXPR_INSN_RTX (tmp2);
3334
 
3335
  /* Schedule debug insns as early as possible.  */
3336
  if (DEBUG_INSN_P (tmp_insn) && !DEBUG_INSN_P (tmp2_insn))
3337
    return -1;
3338
  else if (DEBUG_INSN_P (tmp2_insn))
3339
    return 1;
3340
 
3341
  /* Prefer SCHED_GROUP_P insns to any others.  */
3342
  if (SCHED_GROUP_P (tmp_insn) != SCHED_GROUP_P (tmp2_insn))
3343
    {
3344
      if (VINSN_UNIQUE_P (tmp_vinsn) && VINSN_UNIQUE_P (tmp2_vinsn))
3345
        return SCHED_GROUP_P (tmp2_insn) ? 1 : -1;
3346
 
3347
      /* Now uniqueness means SCHED_GROUP_P is set, because schedule groups
3348
         cannot be cloned.  */
3349
      if (VINSN_UNIQUE_P (tmp2_vinsn))
3350
        return 1;
3351
      return -1;
3352
    }
3353
 
3354
  /* Discourage scheduling of speculative checks.  */
3355
  val = (sel_insn_is_speculation_check (tmp_insn)
3356
         - sel_insn_is_speculation_check (tmp2_insn));
3357
  if (val)
3358
    return val;
3359
 
3360
  /* Prefer not scheduled insn over scheduled one.  */
3361
  if (EXPR_SCHED_TIMES (tmp) > 0 || EXPR_SCHED_TIMES (tmp2) > 0)
3362
    {
3363
      val = EXPR_SCHED_TIMES (tmp) - EXPR_SCHED_TIMES (tmp2);
3364
      if (val)
3365
        return val;
3366
    }
3367
 
3368
  /* Prefer jump over non-jump instruction.  */
3369
  if (control_flow_insn_p (tmp_insn) && !control_flow_insn_p (tmp2_insn))
3370
    return -1;
3371
  else if (control_flow_insn_p (tmp2_insn) && !control_flow_insn_p (tmp_insn))
3372
    return 1;
3373
 
3374
  /* Prefer an expr with greater priority.  */
3375
  if (EXPR_USEFULNESS (tmp) != 0 && EXPR_USEFULNESS (tmp2) != 0)
3376
    {
3377
      int p2 = EXPR_PRIORITY (tmp2) + EXPR_PRIORITY_ADJ (tmp2),
3378
          p1 = EXPR_PRIORITY (tmp) + EXPR_PRIORITY_ADJ (tmp);
3379
 
3380
      val = p2 * EXPR_USEFULNESS (tmp2) - p1 * EXPR_USEFULNESS (tmp);
3381
    }
3382
  else
3383
    val = EXPR_PRIORITY (tmp2) - EXPR_PRIORITY (tmp)
3384
          + EXPR_PRIORITY_ADJ (tmp2) - EXPR_PRIORITY_ADJ (tmp);
3385
  if (val)
3386
    return val;
3387
 
3388
  if (spec_info != NULL && spec_info->mask != 0)
3389
    /* This code was taken from haifa-sched.c: rank_for_schedule ().  */
3390
    {
3391
      ds_t ds1, ds2;
3392
      dw_t dw1, dw2;
3393
      int dw;
3394
 
3395
      ds1 = EXPR_SPEC_DONE_DS (tmp);
3396
      if (ds1)
3397
        dw1 = ds_weak (ds1);
3398
      else
3399
        dw1 = NO_DEP_WEAK;
3400
 
3401
      ds2 = EXPR_SPEC_DONE_DS (tmp2);
3402
      if (ds2)
3403
        dw2 = ds_weak (ds2);
3404
      else
3405
        dw2 = NO_DEP_WEAK;
3406
 
3407
      dw = dw2 - dw1;
3408
      if (dw > (NO_DEP_WEAK / 8) || dw < -(NO_DEP_WEAK / 8))
3409
        return dw;
3410
    }
3411
 
3412
  /* Prefer an old insn to a bookkeeping insn.  */
3413
  if (INSN_UID (tmp_insn) < first_emitted_uid
3414
      && INSN_UID (tmp2_insn) >= first_emitted_uid)
3415
    return -1;
3416
  if (INSN_UID (tmp_insn) >= first_emitted_uid
3417
      && INSN_UID (tmp2_insn) < first_emitted_uid)
3418
    return 1;
3419
 
3420
  /* Prefer an insn with smaller UID, as a last resort.
3421
     We can't safely use INSN_LUID as it is defined only for those insns
3422
     that are in the stream.  */
3423
  return INSN_UID (tmp_insn) - INSN_UID (tmp2_insn);
3424
}
3425
 
3426
/* Filter out expressions from av set pointed to by AV_PTR
3427
   that are pipelined too many times.  */
3428
static void
3429
process_pipelined_exprs (av_set_t *av_ptr)
3430
{
3431
  expr_t expr;
3432
  av_set_iterator si;
3433
 
3434
  /* Don't pipeline already pipelined code as that would increase
3435
     number of unnecessary register moves.  */
3436
  FOR_EACH_EXPR_1 (expr, si, av_ptr)
3437
    {
3438
      if (EXPR_SCHED_TIMES (expr)
3439
          >= PARAM_VALUE (PARAM_SELSCHED_MAX_SCHED_TIMES))
3440
        av_set_iter_remove (&si);
3441
    }
3442
}
3443
 
3444
/* Filter speculative insns from AV_PTR if we don't want them.  */
3445
static void
3446
process_spec_exprs (av_set_t *av_ptr)
3447
{
3448
  bool try_data_p = true;
3449
  bool try_control_p = true;
3450
  expr_t expr;
3451
  av_set_iterator si;
3452
 
3453
  if (spec_info == NULL)
3454
    return;
3455
 
3456
  /* Scan *AV_PTR to find out if we want to consider speculative
3457
     instructions for scheduling.  */
3458
  FOR_EACH_EXPR_1 (expr, si, av_ptr)
3459
    {
3460
      ds_t ds;
3461
 
3462
      ds = EXPR_SPEC_DONE_DS (expr);
3463
 
3464
      /* The probability of a success is too low - don't speculate.  */
3465
      if ((ds & SPECULATIVE)
3466
          && (ds_weak (ds) < spec_info->data_weakness_cutoff
3467
              || EXPR_USEFULNESS (expr) < spec_info->control_weakness_cutoff
3468
              || (pipelining_p && false
3469
                  && (ds & DATA_SPEC)
3470
                  && (ds & CONTROL_SPEC))))
3471
        {
3472
          av_set_iter_remove (&si);
3473
          continue;
3474
        }
3475
 
3476
      if ((spec_info->flags & PREFER_NON_DATA_SPEC)
3477
          && !(ds & BEGIN_DATA))
3478
        try_data_p = false;
3479
 
3480
      if ((spec_info->flags & PREFER_NON_CONTROL_SPEC)
3481
          && !(ds & BEGIN_CONTROL))
3482
        try_control_p = false;
3483
    }
3484
 
3485
  FOR_EACH_EXPR_1 (expr, si, av_ptr)
3486
    {
3487
      ds_t ds;
3488
 
3489
      ds = EXPR_SPEC_DONE_DS (expr);
3490
 
3491
      if (ds & SPECULATIVE)
3492
        {
3493
          if ((ds & BEGIN_DATA) && !try_data_p)
3494
            /* We don't want any data speculative instructions right
3495
               now.  */
3496
            av_set_iter_remove (&si);
3497
 
3498
          if ((ds & BEGIN_CONTROL) && !try_control_p)
3499
            /* We don't want any control speculative instructions right
3500
               now.  */
3501
            av_set_iter_remove (&si);
3502
        }
3503
    }
3504
}
3505
 
3506
/* Search for any use-like insns in AV_PTR and decide on scheduling
3507
   them.  Return one when found, and NULL otherwise.
3508
   Note that we check here whether a USE could be scheduled to avoid
3509
   an infinite loop later.  */
3510
static expr_t
3511
process_use_exprs (av_set_t *av_ptr)
3512
{
3513
  expr_t expr;
3514
  av_set_iterator si;
3515
  bool uses_present_p = false;
3516
  bool try_uses_p = true;
3517
 
3518
  FOR_EACH_EXPR_1 (expr, si, av_ptr)
3519
    {
3520
      /* This will also initialize INSN_CODE for later use.  */
3521
      if (recog_memoized (EXPR_INSN_RTX (expr)) < 0)
3522
        {
3523
          /* If we have a USE in *AV_PTR that was not scheduled yet,
3524
             do so because it will do good only.  */
3525
          if (EXPR_SCHED_TIMES (expr) <= 0)
3526
            {
3527
              if (EXPR_TARGET_AVAILABLE (expr) == 1)
3528
                return expr;
3529
 
3530
              av_set_iter_remove (&si);
3531
            }
3532
          else
3533
            {
3534
              gcc_assert (pipelining_p);
3535
 
3536
              uses_present_p = true;
3537
            }
3538
        }
3539
      else
3540
        try_uses_p = false;
3541
    }
3542
 
3543
  if (uses_present_p)
3544
    {
3545
      /* If we don't want to schedule any USEs right now and we have some
3546
           in *AV_PTR, remove them, else just return the first one found.  */
3547
      if (!try_uses_p)
3548
        {
3549
          FOR_EACH_EXPR_1 (expr, si, av_ptr)
3550
            if (INSN_CODE (EXPR_INSN_RTX (expr)) < 0)
3551
              av_set_iter_remove (&si);
3552
        }
3553
      else
3554
        {
3555
          FOR_EACH_EXPR_1 (expr, si, av_ptr)
3556
            {
3557
              gcc_assert (INSN_CODE (EXPR_INSN_RTX (expr)) < 0);
3558
 
3559
              if (EXPR_TARGET_AVAILABLE (expr) == 1)
3560
                return expr;
3561
 
3562
              av_set_iter_remove (&si);
3563
            }
3564
        }
3565
    }
3566
 
3567
  return NULL;
3568
}
3569
 
3570
/* Lookup EXPR in VINSN_VEC and return TRUE if found.  */
3571
static bool
3572
vinsn_vec_has_expr_p (vinsn_vec_t vinsn_vec, expr_t expr)
3573
{
3574
  vinsn_t vinsn;
3575
  int n;
3576
 
3577
  FOR_EACH_VEC_ELT (vinsn_t, vinsn_vec, n, vinsn)
3578
    if (VINSN_SEPARABLE_P (vinsn))
3579
      {
3580
        if (vinsn_equal_p (vinsn, EXPR_VINSN (expr)))
3581
          return true;
3582
      }
3583
    else
3584
      {
3585
        /* For non-separable instructions, the blocking insn can have
3586
           another pattern due to substitution, and we can't choose
3587
           different register as in the above case.  Check all registers
3588
           being written instead.  */
3589
        if (bitmap_intersect_p (VINSN_REG_SETS (vinsn),
3590
                                VINSN_REG_SETS (EXPR_VINSN (expr))))
3591
          return true;
3592
      }
3593
 
3594
  return false;
3595
}
3596
 
3597
#ifdef ENABLE_CHECKING
3598
/* Return true if either of expressions from ORIG_OPS can be blocked
3599
   by previously created bookkeeping code.  STATIC_PARAMS points to static
3600
   parameters of move_op.  */
3601
static bool
3602
av_set_could_be_blocked_by_bookkeeping_p (av_set_t orig_ops, void *static_params)
3603
{
3604
  expr_t expr;
3605
  av_set_iterator iter;
3606
  moveop_static_params_p sparams;
3607
 
3608
  /* This checks that expressions in ORIG_OPS are not blocked by bookkeeping
3609
     created while scheduling on another fence.  */
3610
  FOR_EACH_EXPR (expr, iter, orig_ops)
3611
    if (vinsn_vec_has_expr_p (vec_bookkeeping_blocked_vinsns, expr))
3612
      return true;
3613
 
3614
  gcc_assert (code_motion_path_driver_info == &move_op_hooks);
3615
  sparams = (moveop_static_params_p) static_params;
3616
 
3617
  /* Expressions can be also blocked by bookkeeping created during current
3618
     move_op.  */
3619
  if (bitmap_bit_p (current_copies, INSN_UID (sparams->failed_insn)))
3620
    FOR_EACH_EXPR (expr, iter, orig_ops)
3621
      if (moveup_expr_cached (expr, sparams->failed_insn, false) != MOVEUP_EXPR_NULL)
3622
        return true;
3623
 
3624
  /* Expressions in ORIG_OPS may have wrong destination register due to
3625
     renaming.  Check with the right register instead.  */
3626
  if (sparams->dest && REG_P (sparams->dest))
3627
    {
3628
      rtx reg = sparams->dest;
3629
      vinsn_t failed_vinsn = INSN_VINSN (sparams->failed_insn);
3630
 
3631
      if (register_unavailable_p (VINSN_REG_SETS (failed_vinsn), reg)
3632
          || register_unavailable_p (VINSN_REG_USES (failed_vinsn), reg)
3633
          || register_unavailable_p (VINSN_REG_CLOBBERS (failed_vinsn), reg))
3634
        return true;
3635
    }
3636
 
3637
  return false;
3638
}
3639
#endif
3640
 
3641
/* Clear VINSN_VEC and detach vinsns.  */
3642
static void
3643
vinsn_vec_clear (vinsn_vec_t *vinsn_vec)
3644
{
3645
  unsigned len = VEC_length (vinsn_t, *vinsn_vec);
3646
  if (len > 0)
3647
    {
3648
      vinsn_t vinsn;
3649
      int n;
3650
 
3651
      FOR_EACH_VEC_ELT (vinsn_t, *vinsn_vec, n, vinsn)
3652
        vinsn_detach (vinsn);
3653
      VEC_block_remove (vinsn_t, *vinsn_vec, 0, len);
3654
    }
3655
}
3656
 
3657
/* Add the vinsn of EXPR to the VINSN_VEC.  */
3658
static void
3659
vinsn_vec_add (vinsn_vec_t *vinsn_vec, expr_t expr)
3660
{
3661
  vinsn_attach (EXPR_VINSN (expr));
3662
  VEC_safe_push (vinsn_t, heap, *vinsn_vec, EXPR_VINSN (expr));
3663
}
3664
 
3665
/* Free the vector representing blocked expressions.  */
3666
static void
3667
vinsn_vec_free (vinsn_vec_t *vinsn_vec)
3668
{
3669
  if (*vinsn_vec)
3670
    VEC_free (vinsn_t, heap, *vinsn_vec);
3671
}
3672
 
3673
/* Increase EXPR_PRIORITY_ADJ for INSN by AMOUNT.  */
3674
 
3675
void sel_add_to_insn_priority (rtx insn, int amount)
3676
{
3677
  EXPR_PRIORITY_ADJ (INSN_EXPR (insn)) += amount;
3678
 
3679
  if (sched_verbose >= 2)
3680
    sel_print ("sel_add_to_insn_priority: insn %d, by %d (now %d+%d).\n",
3681
               INSN_UID (insn), amount, EXPR_PRIORITY (INSN_EXPR (insn)),
3682
               EXPR_PRIORITY_ADJ (INSN_EXPR (insn)));
3683
}
3684
 
3685
/* Turn AV into a vector, filter inappropriate insns and sort it.  Return
3686
   true if there is something to schedule.  BNDS and FENCE are current
3687
   boundaries and fence, respectively.  If we need to stall for some cycles
3688
   before an expr from AV would become available, write this number to
3689
   *PNEED_STALL.  */
3690
static bool
3691
fill_vec_av_set (av_set_t av, blist_t bnds, fence_t fence,
3692
                 int *pneed_stall)
3693
{
3694
  av_set_iterator si;
3695
  expr_t expr;
3696
  int sched_next_worked = 0, stalled, n;
3697
  static int av_max_prio, est_ticks_till_branch;
3698
  int min_need_stall = -1;
3699
  deps_t dc = BND_DC (BLIST_BND (bnds));
3700
 
3701
  /* Bail out early when the ready list contained only USEs/CLOBBERs that are
3702
     already scheduled.  */
3703
  if (av == NULL)
3704
    return false;
3705
 
3706
  /* Empty vector from the previous stuff.  */
3707
  if (VEC_length (expr_t, vec_av_set) > 0)
3708
    VEC_block_remove (expr_t, vec_av_set, 0, VEC_length (expr_t, vec_av_set));
3709
 
3710
  /* Turn the set into a vector for sorting and call sel_target_adjust_priority
3711
     for each insn.  */
3712
  gcc_assert (VEC_empty (expr_t, vec_av_set));
3713
  FOR_EACH_EXPR (expr, si, av)
3714
    {
3715
      VEC_safe_push (expr_t, heap, vec_av_set, expr);
3716
 
3717
      gcc_assert (EXPR_PRIORITY_ADJ (expr) == 0 || *pneed_stall);
3718
 
3719
      /* Adjust priority using target backend hook.  */
3720
      sel_target_adjust_priority (expr);
3721
    }
3722
 
3723
  /* Sort the vector.  */
3724
  VEC_qsort (expr_t, vec_av_set, sel_rank_for_schedule);
3725
 
3726
  /* We record maximal priority of insns in av set for current instruction
3727
     group.  */
3728
  if (FENCE_STARTS_CYCLE_P (fence))
3729
    av_max_prio = est_ticks_till_branch = INT_MIN;
3730
 
3731
  /* Filter out inappropriate expressions.  Loop's direction is reversed to
3732
     visit "best" instructions first.  We assume that VEC_unordered_remove
3733
     moves last element in place of one being deleted.  */
3734
  for (n = VEC_length (expr_t, vec_av_set) - 1, stalled = 0; n >= 0; n--)
3735
    {
3736
      expr_t expr = VEC_index (expr_t, vec_av_set, n);
3737
      insn_t insn = EXPR_INSN_RTX (expr);
3738
      signed char target_available;
3739
      bool is_orig_reg_p = true;
3740
      int need_cycles, new_prio;
3741
 
3742
      /* Don't allow any insns other than from SCHED_GROUP if we have one.  */
3743
      if (FENCE_SCHED_NEXT (fence) && insn != FENCE_SCHED_NEXT (fence))
3744
        {
3745
          VEC_unordered_remove (expr_t, vec_av_set, n);
3746
          continue;
3747
        }
3748
 
3749
      /* Set number of sched_next insns (just in case there
3750
         could be several).  */
3751
      if (FENCE_SCHED_NEXT (fence))
3752
        sched_next_worked++;
3753
 
3754
      /* Check all liveness requirements and try renaming.
3755
         FIXME: try to minimize calls to this.  */
3756
      target_available = EXPR_TARGET_AVAILABLE (expr);
3757
 
3758
      /* If insn was already scheduled on the current fence,
3759
         set TARGET_AVAILABLE to -1 no matter what expr's attribute says.  */
3760
      if (vinsn_vec_has_expr_p (vec_target_unavailable_vinsns, expr))
3761
        target_available = -1;
3762
 
3763
      /* If the availability of the EXPR is invalidated by the insertion of
3764
         bookkeeping earlier, make sure that we won't choose this expr for
3765
         scheduling if it's not separable, and if it is separable, then
3766
         we have to recompute the set of available registers for it.  */
3767
      if (vinsn_vec_has_expr_p (vec_bookkeeping_blocked_vinsns, expr))
3768
        {
3769
          VEC_unordered_remove (expr_t, vec_av_set, n);
3770
          if (sched_verbose >= 4)
3771
            sel_print ("Expr %d is blocked by bookkeeping inserted earlier\n",
3772
                       INSN_UID (insn));
3773
          continue;
3774
        }
3775
 
3776
      if (target_available == true)
3777
        {
3778
          /* Do nothing -- we can use an existing register.  */
3779
          is_orig_reg_p = EXPR_SEPARABLE_P (expr);
3780
        }
3781
      else if (/* Non-separable instruction will never
3782
                  get another register. */
3783
               (target_available == false
3784
                && !EXPR_SEPARABLE_P (expr))
3785
               /* Don't try to find a register for low-priority expression.  */
3786
               || (int) VEC_length (expr_t, vec_av_set) - 1 - n >= max_insns_to_rename
3787
               /* ??? FIXME: Don't try to rename data speculation.  */
3788
               || (EXPR_SPEC_DONE_DS (expr) & BEGIN_DATA)
3789
               || ! find_best_reg_for_expr (expr, bnds, &is_orig_reg_p))
3790
        {
3791
          VEC_unordered_remove (expr_t, vec_av_set, n);
3792
          if (sched_verbose >= 4)
3793
            sel_print ("Expr %d has no suitable target register\n",
3794
                       INSN_UID (insn));
3795
          continue;
3796
        }
3797
 
3798
      /* Filter expressions that need to be renamed or speculated when
3799
         pipelining, because compensating register copies or speculation
3800
         checks are likely to be placed near the beginning of the loop,
3801
         causing a stall.  */
3802
      if (pipelining_p && EXPR_ORIG_SCHED_CYCLE (expr) > 0
3803
          && (!is_orig_reg_p || EXPR_SPEC_DONE_DS (expr) != 0))
3804
        {
3805
          /* Estimation of number of cycles until loop branch for
3806
             renaming/speculation to be successful.  */
3807
          int need_n_ticks_till_branch = sel_vinsn_cost (EXPR_VINSN (expr));
3808
 
3809
          if ((int) current_loop_nest->ninsns < 9)
3810
            {
3811
              VEC_unordered_remove (expr_t, vec_av_set, n);
3812
              if (sched_verbose >= 4)
3813
                sel_print ("Pipelining expr %d will likely cause stall\n",
3814
                           INSN_UID (insn));
3815
              continue;
3816
            }
3817
 
3818
          if ((int) current_loop_nest->ninsns - num_insns_scheduled
3819
              < need_n_ticks_till_branch * issue_rate / 2
3820
              && est_ticks_till_branch < need_n_ticks_till_branch)
3821
             {
3822
               VEC_unordered_remove (expr_t, vec_av_set, n);
3823
               if (sched_verbose >= 4)
3824
                 sel_print ("Pipelining expr %d will likely cause stall\n",
3825
                            INSN_UID (insn));
3826
               continue;
3827
             }
3828
        }
3829
 
3830
      /* We want to schedule speculation checks as late as possible.  Discard
3831
         them from av set if there are instructions with higher priority.  */
3832
      if (sel_insn_is_speculation_check (insn)
3833
          && EXPR_PRIORITY (expr) < av_max_prio)
3834
        {
3835
          stalled++;
3836
          min_need_stall = min_need_stall < 0 ? 1 : MIN (min_need_stall, 1);
3837
          VEC_unordered_remove (expr_t, vec_av_set, n);
3838
          if (sched_verbose >= 4)
3839
            sel_print ("Delaying speculation check %d until its first use\n",
3840
                       INSN_UID (insn));
3841
          continue;
3842
        }
3843
 
3844
      /* Ignore EXPRs available from pipelining to update AV_MAX_PRIO.  */
3845
      if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0)
3846
        av_max_prio = MAX (av_max_prio, EXPR_PRIORITY (expr));
3847
 
3848
      /* Don't allow any insns whose data is not yet ready.
3849
         Check first whether we've already tried them and failed.  */
3850
      if (INSN_UID (insn) < FENCE_READY_TICKS_SIZE (fence))
3851
        {
3852
          need_cycles = (FENCE_READY_TICKS (fence)[INSN_UID (insn)]
3853
                         - FENCE_CYCLE (fence));
3854
          if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0)
3855
            est_ticks_till_branch = MAX (est_ticks_till_branch,
3856
                                         EXPR_PRIORITY (expr) + need_cycles);
3857
 
3858
          if (need_cycles > 0)
3859
            {
3860
              stalled++;
3861
              min_need_stall = (min_need_stall < 0
3862
                                ? need_cycles
3863
                                : MIN (min_need_stall, need_cycles));
3864
              VEC_unordered_remove (expr_t, vec_av_set, n);
3865
 
3866
              if (sched_verbose >= 4)
3867
                sel_print ("Expr %d is not ready until cycle %d (cached)\n",
3868
                           INSN_UID (insn),
3869
                           FENCE_READY_TICKS (fence)[INSN_UID (insn)]);
3870
              continue;
3871
            }
3872
        }
3873
 
3874
      /* Now resort to dependence analysis to find whether EXPR might be
3875
         stalled due to dependencies from FENCE's context.  */
3876
      need_cycles = tick_check_p (expr, dc, fence);
3877
      new_prio = EXPR_PRIORITY (expr) + EXPR_PRIORITY_ADJ (expr) + need_cycles;
3878
 
3879
      if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0)
3880
        est_ticks_till_branch = MAX (est_ticks_till_branch,
3881
                                     new_prio);
3882
 
3883
      if (need_cycles > 0)
3884
        {
3885
          if (INSN_UID (insn) >= FENCE_READY_TICKS_SIZE (fence))
3886
            {
3887
              int new_size = INSN_UID (insn) * 3 / 2;
3888
 
3889
              FENCE_READY_TICKS (fence)
3890
                = (int *) xrecalloc (FENCE_READY_TICKS (fence),
3891
                                     new_size, FENCE_READY_TICKS_SIZE (fence),
3892
                                     sizeof (int));
3893
            }
3894
          FENCE_READY_TICKS (fence)[INSN_UID (insn)]
3895
            = FENCE_CYCLE (fence) + need_cycles;
3896
 
3897
          stalled++;
3898
          min_need_stall = (min_need_stall < 0
3899
                            ? need_cycles
3900
                            : MIN (min_need_stall, need_cycles));
3901
 
3902
          VEC_unordered_remove (expr_t, vec_av_set, n);
3903
 
3904
          if (sched_verbose >= 4)
3905
            sel_print ("Expr %d is not ready yet until cycle %d\n",
3906
                       INSN_UID (insn),
3907
                       FENCE_READY_TICKS (fence)[INSN_UID (insn)]);
3908
          continue;
3909
        }
3910
 
3911
      if (sched_verbose >= 4)
3912
        sel_print ("Expr %d is ok\n", INSN_UID (insn));
3913
      min_need_stall = 0;
3914
    }
3915
 
3916
  /* Clear SCHED_NEXT.  */
3917
  if (FENCE_SCHED_NEXT (fence))
3918
    {
3919
      gcc_assert (sched_next_worked == 1);
3920
      FENCE_SCHED_NEXT (fence) = NULL_RTX;
3921
    }
3922
 
3923
  /* No need to stall if this variable was not initialized.  */
3924
  if (min_need_stall < 0)
3925
    min_need_stall = 0;
3926
 
3927
  if (VEC_empty (expr_t, vec_av_set))
3928
    {
3929
      /* We need to set *pneed_stall here, because later we skip this code
3930
         when ready list is empty.  */
3931
      *pneed_stall = min_need_stall;
3932
      return false;
3933
    }
3934
  else
3935
    gcc_assert (min_need_stall == 0);
3936
 
3937
  /* Sort the vector.  */
3938
  VEC_qsort (expr_t, vec_av_set, sel_rank_for_schedule);
3939
 
3940
  if (sched_verbose >= 4)
3941
    {
3942
      sel_print ("Total ready exprs: %d, stalled: %d\n",
3943
                 VEC_length (expr_t, vec_av_set), stalled);
3944
      sel_print ("Sorted av set (%d): ", VEC_length (expr_t, vec_av_set));
3945
      FOR_EACH_VEC_ELT (expr_t, vec_av_set, n, expr)
3946
        dump_expr (expr);
3947
      sel_print ("\n");
3948
    }
3949
 
3950
  *pneed_stall = 0;
3951
  return true;
3952
}
3953
 
3954
/* Convert a vectored and sorted av set to the ready list that
3955
   the rest of the backend wants to see.  */
3956
static void
3957
convert_vec_av_set_to_ready (void)
3958
{
3959
  int n;
3960
  expr_t expr;
3961
 
3962
  /* Allocate and fill the ready list from the sorted vector.  */
3963
  ready.n_ready = VEC_length (expr_t, vec_av_set);
3964
  ready.first = ready.n_ready - 1;
3965
 
3966
  gcc_assert (ready.n_ready > 0);
3967
 
3968
  if (ready.n_ready > max_issue_size)
3969
    {
3970
      max_issue_size = ready.n_ready;
3971
      sched_extend_ready_list (ready.n_ready);
3972
    }
3973
 
3974
  FOR_EACH_VEC_ELT (expr_t, vec_av_set, n, expr)
3975
    {
3976
      vinsn_t vi = EXPR_VINSN (expr);
3977
      insn_t insn = VINSN_INSN_RTX (vi);
3978
 
3979
      ready_try[n] = 0;
3980
      ready.vec[n] = insn;
3981
    }
3982
}
3983
 
3984
/* Initialize ready list from *AV_PTR for the max_issue () call.
3985
   If any unrecognizable insn found in *AV_PTR, return it (and skip
3986
   max_issue).  BND and FENCE are current boundary and fence,
3987
   respectively.  If we need to stall for some cycles before an expr
3988
   from *AV_PTR would become available, write this number to *PNEED_STALL.  */
3989
static expr_t
3990
fill_ready_list (av_set_t *av_ptr, blist_t bnds, fence_t fence,
3991
                 int *pneed_stall)
3992
{
3993
  expr_t expr;
3994
 
3995
  /* We do not support multiple boundaries per fence.  */
3996
  gcc_assert (BLIST_NEXT (bnds) == NULL);
3997
 
3998
  /* Process expressions required special handling, i.e.  pipelined,
3999
     speculative and recog() < 0 expressions first.  */
4000
  process_pipelined_exprs (av_ptr);
4001
  process_spec_exprs (av_ptr);
4002
 
4003
  /* A USE could be scheduled immediately.  */
4004
  expr = process_use_exprs (av_ptr);
4005
  if (expr)
4006
    {
4007
      *pneed_stall = 0;
4008
      return expr;
4009
    }
4010
 
4011
  /* Turn the av set to a vector for sorting.  */
4012
  if (! fill_vec_av_set (*av_ptr, bnds, fence, pneed_stall))
4013
    {
4014
      ready.n_ready = 0;
4015
      return NULL;
4016
    }
4017
 
4018
  /* Build the final ready list.  */
4019
  convert_vec_av_set_to_ready ();
4020
  return NULL;
4021
}
4022
 
4023
/* Wrapper for dfa_new_cycle ().  Returns TRUE if cycle was advanced.  */
4024
static bool
4025
sel_dfa_new_cycle (insn_t insn, fence_t fence)
4026
{
4027
  int last_scheduled_cycle = FENCE_LAST_SCHEDULED_INSN (fence)
4028
                             ? INSN_SCHED_CYCLE (FENCE_LAST_SCHEDULED_INSN (fence))
4029
                             : FENCE_CYCLE (fence) - 1;
4030
  bool res = false;
4031
  int sort_p = 0;
4032
 
4033
  if (!targetm.sched.dfa_new_cycle)
4034
    return false;
4035
 
4036
  memcpy (curr_state, FENCE_STATE (fence), dfa_state_size);
4037
 
4038
  while (!sort_p && targetm.sched.dfa_new_cycle (sched_dump, sched_verbose,
4039
                                                 insn, last_scheduled_cycle,
4040
                                                 FENCE_CYCLE (fence), &sort_p))
4041
    {
4042
      memcpy (FENCE_STATE (fence), curr_state, dfa_state_size);
4043
      advance_one_cycle (fence);
4044
      memcpy (curr_state, FENCE_STATE (fence), dfa_state_size);
4045
      res = true;
4046
    }
4047
 
4048
  return res;
4049
}
4050
 
4051
/* Invoke reorder* target hooks on the ready list.  Return the number of insns
4052
   we can issue.  FENCE is the current fence.  */
4053
static int
4054
invoke_reorder_hooks (fence_t fence)
4055
{
4056
  int issue_more;
4057
  bool ran_hook = false;
4058
 
4059
  /* Call the reorder hook at the beginning of the cycle, and call
4060
     the reorder2 hook in the middle of the cycle.  */
4061
  if (FENCE_ISSUED_INSNS (fence) == 0)
4062
    {
4063
      if (targetm.sched.reorder
4064
          && !SCHED_GROUP_P (ready_element (&ready, 0))
4065
          && ready.n_ready > 1)
4066
        {
4067
          /* Don't give reorder the most prioritized insn as it can break
4068
             pipelining.  */
4069
          if (pipelining_p)
4070
            --ready.n_ready;
4071
 
4072
          issue_more
4073
            = targetm.sched.reorder (sched_dump, sched_verbose,
4074
                                     ready_lastpos (&ready),
4075
                                     &ready.n_ready, FENCE_CYCLE (fence));
4076
 
4077
          if (pipelining_p)
4078
            ++ready.n_ready;
4079
 
4080
          ran_hook = true;
4081
        }
4082
      else
4083
        /* Initialize can_issue_more for variable_issue.  */
4084
        issue_more = issue_rate;
4085
    }
4086
  else if (targetm.sched.reorder2
4087
           && !SCHED_GROUP_P (ready_element (&ready, 0)))
4088
    {
4089
      if (ready.n_ready == 1)
4090
        issue_more =
4091
          targetm.sched.reorder2 (sched_dump, sched_verbose,
4092
                                  ready_lastpos (&ready),
4093
                                  &ready.n_ready, FENCE_CYCLE (fence));
4094
      else
4095
        {
4096
          if (pipelining_p)
4097
            --ready.n_ready;
4098
 
4099
          issue_more =
4100
            targetm.sched.reorder2 (sched_dump, sched_verbose,
4101
                                    ready.n_ready
4102
                                    ? ready_lastpos (&ready) : NULL,
4103
                                    &ready.n_ready, FENCE_CYCLE (fence));
4104
 
4105
          if (pipelining_p)
4106
            ++ready.n_ready;
4107
        }
4108
 
4109
      ran_hook = true;
4110
    }
4111
  else
4112
    issue_more = FENCE_ISSUE_MORE (fence);
4113
 
4114
  /* Ensure that ready list and vec_av_set are in line with each other,
4115
     i.e. vec_av_set[i] == ready_element (&ready, i).  */
4116
  if (issue_more && ran_hook)
4117
    {
4118
      int i, j, n;
4119
      rtx *arr = ready.vec;
4120
      expr_t *vec = VEC_address (expr_t, vec_av_set);
4121
 
4122
      for (i = 0, n = ready.n_ready; i < n; i++)
4123
        if (EXPR_INSN_RTX (vec[i]) != arr[i])
4124
          {
4125
            expr_t tmp;
4126
 
4127
            for (j = i; j < n; j++)
4128
              if (EXPR_INSN_RTX (vec[j]) == arr[i])
4129
                break;
4130
            gcc_assert (j < n);
4131
 
4132
            tmp = vec[i];
4133
            vec[i] = vec[j];
4134
            vec[j] = tmp;
4135
          }
4136
    }
4137
 
4138
  return issue_more;
4139
}
4140
 
4141
/* Return an EXPR correponding to INDEX element of ready list, if
4142
   FOLLOW_READY_ELEMENT is true (i.e., an expr of
4143
   ready_element (&ready, INDEX) will be returned), and to INDEX element of
4144
   ready.vec otherwise.  */
4145
static inline expr_t
4146
find_expr_for_ready (int index, bool follow_ready_element)
4147
{
4148
  expr_t expr;
4149
  int real_index;
4150
 
4151
  real_index = follow_ready_element ? ready.first - index : index;
4152
 
4153
  expr = VEC_index (expr_t, vec_av_set, real_index);
4154
  gcc_assert (ready.vec[real_index] == EXPR_INSN_RTX (expr));
4155
 
4156
  return expr;
4157
}
4158
 
4159
/* Calculate insns worth trying via lookahead_guard hook.  Return a number
4160
   of such insns found.  */
4161
static int
4162
invoke_dfa_lookahead_guard (void)
4163
{
4164
  int i, n;
4165
  bool have_hook
4166
    = targetm.sched.first_cycle_multipass_dfa_lookahead_guard != NULL;
4167
 
4168
  if (sched_verbose >= 2)
4169
    sel_print ("ready after reorder: ");
4170
 
4171
  for (i = 0, n = 0; i < ready.n_ready; i++)
4172
    {
4173
      expr_t expr;
4174
      insn_t insn;
4175
      int r;
4176
 
4177
      /* In this loop insn is Ith element of the ready list given by
4178
         ready_element, not Ith element of ready.vec.  */
4179
      insn = ready_element (&ready, i);
4180
 
4181
      if (! have_hook || i == 0)
4182
        r = 0;
4183
      else
4184
        r = !targetm.sched.first_cycle_multipass_dfa_lookahead_guard (insn);
4185
 
4186
      gcc_assert (INSN_CODE (insn) >= 0);
4187
 
4188
      /* Only insns with ready_try = 0 can get here
4189
         from fill_ready_list.  */
4190
      gcc_assert (ready_try [i] == 0);
4191
      ready_try[i] = r;
4192
      if (!r)
4193
        n++;
4194
 
4195
      expr = find_expr_for_ready (i, true);
4196
 
4197
      if (sched_verbose >= 2)
4198
        {
4199
          dump_vinsn (EXPR_VINSN (expr));
4200
          sel_print (":%d; ", ready_try[i]);
4201
        }
4202
    }
4203
 
4204
  if (sched_verbose >= 2)
4205
    sel_print ("\n");
4206
  return n;
4207
}
4208
 
4209
/* Calculate the number of privileged insns and return it.  */
4210
static int
4211
calculate_privileged_insns (void)
4212
{
4213
  expr_t cur_expr, min_spec_expr = NULL;
4214
  int privileged_n = 0, i;
4215
 
4216
  for (i = 0; i < ready.n_ready; i++)
4217
    {
4218
      if (ready_try[i])
4219
        continue;
4220
 
4221
      if (! min_spec_expr)
4222
        min_spec_expr = find_expr_for_ready (i, true);
4223
 
4224
      cur_expr = find_expr_for_ready (i, true);
4225
 
4226
      if (EXPR_SPEC (cur_expr) > EXPR_SPEC (min_spec_expr))
4227
        break;
4228
 
4229
      ++privileged_n;
4230
    }
4231
 
4232
  if (i == ready.n_ready)
4233
    privileged_n = 0;
4234
 
4235
  if (sched_verbose >= 2)
4236
    sel_print ("privileged_n: %d insns with SPEC %d\n",
4237
               privileged_n, privileged_n ? EXPR_SPEC (min_spec_expr) : -1);
4238
  return privileged_n;
4239
}
4240
 
4241
/* Call the rest of the hooks after the choice was made.  Return
4242
   the number of insns that still can be issued given that the current
4243
   number is ISSUE_MORE.  FENCE and BEST_INSN are the current fence
4244
   and the insn chosen for scheduling, respectively.  */
4245
static int
4246
invoke_aftermath_hooks (fence_t fence, rtx best_insn, int issue_more)
4247
{
4248
  gcc_assert (INSN_P (best_insn));
4249
 
4250
  /* First, call dfa_new_cycle, and then variable_issue, if available.  */
4251
  sel_dfa_new_cycle (best_insn, fence);
4252
 
4253
  if (targetm.sched.variable_issue)
4254
    {
4255
      memcpy (curr_state, FENCE_STATE (fence), dfa_state_size);
4256
      issue_more =
4257
        targetm.sched.variable_issue (sched_dump, sched_verbose, best_insn,
4258
                                      issue_more);
4259
      memcpy (FENCE_STATE (fence), curr_state, dfa_state_size);
4260
    }
4261
  else if (GET_CODE (PATTERN (best_insn)) != USE
4262
           && GET_CODE (PATTERN (best_insn)) != CLOBBER)
4263
    issue_more--;
4264
 
4265
  return issue_more;
4266
}
4267
 
4268
/* Estimate the cost of issuing INSN on DFA state STATE.  */
4269
static int
4270
estimate_insn_cost (rtx insn, state_t state)
4271
{
4272
  static state_t temp = NULL;
4273
  int cost;
4274
 
4275
  if (!temp)
4276
    temp = xmalloc (dfa_state_size);
4277
 
4278
  memcpy (temp, state, dfa_state_size);
4279
  cost = state_transition (temp, insn);
4280
 
4281
  if (cost < 0)
4282
    return 0;
4283
  else if (cost == 0)
4284
    return 1;
4285
  return cost;
4286
}
4287
 
4288
/* Return the cost of issuing EXPR on the FENCE as estimated by DFA.
4289
   This function properly handles ASMs, USEs etc.  */
4290
static int
4291
get_expr_cost (expr_t expr, fence_t fence)
4292
{
4293
  rtx insn = EXPR_INSN_RTX (expr);
4294
 
4295
  if (recog_memoized (insn) < 0)
4296
    {
4297
      if (!FENCE_STARTS_CYCLE_P (fence)
4298
          && INSN_ASM_P (insn))
4299
        /* This is asm insn which is tryed to be issued on the
4300
           cycle not first.  Issue it on the next cycle.  */
4301
        return 1;
4302
      else
4303
        /* A USE insn, or something else we don't need to
4304
           understand.  We can't pass these directly to
4305
           state_transition because it will trigger a
4306
           fatal error for unrecognizable insns.  */
4307
        return 0;
4308
    }
4309
  else
4310
    return estimate_insn_cost (insn, FENCE_STATE (fence));
4311
}
4312
 
4313
/* Find the best insn for scheduling, either via max_issue or just take
4314
   the most prioritized available.  */
4315
static int
4316
choose_best_insn (fence_t fence, int privileged_n, int *index)
4317
{
4318
  int can_issue = 0;
4319
 
4320
  if (dfa_lookahead > 0)
4321
    {
4322
      cycle_issued_insns = FENCE_ISSUED_INSNS (fence);
4323
      /* TODO: pass equivalent of first_cycle_insn_p to max_issue ().  */
4324
      can_issue = max_issue (&ready, privileged_n,
4325
                             FENCE_STATE (fence), true, index);
4326
      if (sched_verbose >= 2)
4327
        sel_print ("max_issue: we can issue %d insns, already did %d insns\n",
4328
                   can_issue, FENCE_ISSUED_INSNS (fence));
4329
    }
4330
  else
4331
    {
4332
      /* We can't use max_issue; just return the first available element.  */
4333
      int i;
4334
 
4335
      for (i = 0; i < ready.n_ready; i++)
4336
        {
4337
          expr_t expr = find_expr_for_ready (i, true);
4338
 
4339
          if (get_expr_cost (expr, fence) < 1)
4340
            {
4341
              can_issue = can_issue_more;
4342
              *index = i;
4343
 
4344
              if (sched_verbose >= 2)
4345
                sel_print ("using %dth insn from the ready list\n", i + 1);
4346
 
4347
              break;
4348
            }
4349
        }
4350
 
4351
      if (i == ready.n_ready)
4352
        {
4353
          can_issue = 0;
4354
          *index = -1;
4355
        }
4356
    }
4357
 
4358
  return can_issue;
4359
}
4360
 
4361
/* Choose the best expr from *AV_VLIW_PTR and a suitable register for it.
4362
   BNDS and FENCE are current boundaries and scheduling fence respectively.
4363
   Return the expr found and NULL if nothing can be issued atm.
4364
   Write to PNEED_STALL the number of cycles to stall if no expr was found.  */
4365
static expr_t
4366
find_best_expr (av_set_t *av_vliw_ptr, blist_t bnds, fence_t fence,
4367
                int *pneed_stall)
4368
{
4369
  expr_t best;
4370
 
4371
  /* Choose the best insn for scheduling via:
4372
     1) sorting the ready list based on priority;
4373
     2) calling the reorder hook;
4374
     3) calling max_issue.  */
4375
  best = fill_ready_list (av_vliw_ptr, bnds, fence, pneed_stall);
4376
  if (best == NULL && ready.n_ready > 0)
4377
    {
4378
      int privileged_n, index;
4379
 
4380
      can_issue_more = invoke_reorder_hooks (fence);
4381
      if (can_issue_more > 0)
4382
        {
4383
          /* Try choosing the best insn until we find one that is could be
4384
             scheduled due to liveness restrictions on its destination register.
4385
             In the future, we'd like to choose once and then just probe insns
4386
             in the order of their priority.  */
4387
          invoke_dfa_lookahead_guard ();
4388
          privileged_n = calculate_privileged_insns ();
4389
          can_issue_more = choose_best_insn (fence, privileged_n, &index);
4390
          if (can_issue_more)
4391
            best = find_expr_for_ready (index, true);
4392
        }
4393
      /* We had some available insns, so if we can't issue them,
4394
         we have a stall.  */
4395
      if (can_issue_more == 0)
4396
        {
4397
          best = NULL;
4398
          *pneed_stall = 1;
4399
        }
4400
    }
4401
 
4402
  if (best != NULL)
4403
    {
4404
      can_issue_more = invoke_aftermath_hooks (fence, EXPR_INSN_RTX (best),
4405
                                               can_issue_more);
4406
      if (targetm.sched.variable_issue
4407
          && can_issue_more == 0)
4408
        *pneed_stall = 1;
4409
    }
4410
 
4411
  if (sched_verbose >= 2)
4412
    {
4413
      if (best != NULL)
4414
        {
4415
          sel_print ("Best expression (vliw form): ");
4416
          dump_expr (best);
4417
          sel_print ("; cycle %d\n", FENCE_CYCLE (fence));
4418
        }
4419
      else
4420
        sel_print ("No best expr found!\n");
4421
    }
4422
 
4423
  return best;
4424
}
4425
 
4426
 
4427
/* Functions that implement the core of the scheduler.  */
4428
 
4429
 
4430
/* Emit an instruction from EXPR with SEQNO and VINSN after
4431
   PLACE_TO_INSERT.  */
4432
static insn_t
4433
emit_insn_from_expr_after (expr_t expr, vinsn_t vinsn, int seqno,
4434
                           insn_t place_to_insert)
4435
{
4436
  /* This assert fails when we have identical instructions
4437
     one of which dominates the other.  In this case move_op ()
4438
     finds the first instruction and doesn't search for second one.
4439
     The solution would be to compute av_set after the first found
4440
     insn and, if insn present in that set, continue searching.
4441
     For now we workaround this issue in move_op.  */
4442
  gcc_assert (!INSN_IN_STREAM_P (EXPR_INSN_RTX (expr)));
4443
 
4444
  if (EXPR_WAS_RENAMED (expr))
4445
    {
4446
      unsigned regno = expr_dest_regno (expr);
4447
 
4448
      if (HARD_REGISTER_NUM_P (regno))
4449
        {
4450
          df_set_regs_ever_live (regno, true);
4451
          reg_rename_tick[regno] = ++reg_rename_this_tick;
4452
        }
4453
    }
4454
 
4455
  return sel_gen_insn_from_expr_after (expr, vinsn, seqno,
4456
                                       place_to_insert);
4457
}
4458
 
4459
/* Return TRUE if BB can hold bookkeeping code.  */
4460
static bool
4461
block_valid_for_bookkeeping_p (basic_block bb)
4462
{
4463
  insn_t bb_end = BB_END (bb);
4464
 
4465
  if (!in_current_region_p (bb) || EDGE_COUNT (bb->succs) > 1)
4466
    return false;
4467
 
4468
  if (INSN_P (bb_end))
4469
    {
4470
      if (INSN_SCHED_TIMES (bb_end) > 0)
4471
        return false;
4472
    }
4473
  else
4474
    gcc_assert (NOTE_INSN_BASIC_BLOCK_P (bb_end));
4475
 
4476
  return true;
4477
}
4478
 
4479
/* Attempt to find a block that can hold bookkeeping code for path(s) incoming
4480
   into E2->dest, except from E1->src (there may be a sequence of empty basic
4481
   blocks between E1->src and E2->dest).  Return found block, or NULL if new
4482
   one must be created.  If LAX holds, don't assume there is a simple path
4483
   from E1->src to E2->dest.  */
4484
static basic_block
4485
find_block_for_bookkeeping (edge e1, edge e2, bool lax)
4486
{
4487
  basic_block candidate_block = NULL;
4488
  edge e;
4489
 
4490
  /* Loop over edges from E1 to E2, inclusive.  */
4491
  for (e = e1; !lax || e->dest != EXIT_BLOCK_PTR; e = EDGE_SUCC (e->dest, 0))
4492
    {
4493
      if (EDGE_COUNT (e->dest->preds) == 2)
4494
        {
4495
          if (candidate_block == NULL)
4496
            candidate_block = (EDGE_PRED (e->dest, 0) == e
4497
                               ? EDGE_PRED (e->dest, 1)->src
4498
                               : EDGE_PRED (e->dest, 0)->src);
4499
          else
4500
            /* Found additional edge leading to path from e1 to e2
4501
               from aside.  */
4502
            return NULL;
4503
        }
4504
      else if (EDGE_COUNT (e->dest->preds) > 2)
4505
        /* Several edges leading to path from e1 to e2 from aside.  */
4506
        return NULL;
4507
 
4508
      if (e == e2)
4509
        return ((!lax || candidate_block)
4510
                && block_valid_for_bookkeeping_p (candidate_block)
4511
                ? candidate_block
4512
                : NULL);
4513
 
4514
      if (lax && EDGE_COUNT (e->dest->succs) != 1)
4515
        return NULL;
4516
    }
4517
 
4518
  if (lax)
4519
    return NULL;
4520
 
4521
  gcc_unreachable ();
4522
}
4523
 
4524
/* Create new basic block for bookkeeping code for path(s) incoming into
4525
   E2->dest, except from E1->src.  Return created block.  */
4526
static basic_block
4527
create_block_for_bookkeeping (edge e1, edge e2)
4528
{
4529
  basic_block new_bb, bb = e2->dest;
4530
 
4531
  /* Check that we don't spoil the loop structure.  */
4532
  if (current_loop_nest)
4533
    {
4534
      basic_block latch = current_loop_nest->latch;
4535
 
4536
      /* We do not split header.  */
4537
      gcc_assert (e2->dest != current_loop_nest->header);
4538
 
4539
      /* We do not redirect the only edge to the latch block.  */
4540
      gcc_assert (e1->dest != latch
4541
                  || !single_pred_p (latch)
4542
                  || e1 != single_pred_edge (latch));
4543
    }
4544
 
4545
  /* Split BB to insert BOOK_INSN there.  */
4546
  new_bb = sched_split_block (bb, NULL);
4547
 
4548
  /* Move note_list from the upper bb.  */
4549
  gcc_assert (BB_NOTE_LIST (new_bb) == NULL_RTX);
4550
  BB_NOTE_LIST (new_bb) = BB_NOTE_LIST (bb);
4551
  BB_NOTE_LIST (bb) = NULL_RTX;
4552
 
4553
  gcc_assert (e2->dest == bb);
4554
 
4555
  /* Skip block for bookkeeping copy when leaving E1->src.  */
4556
  if (e1->flags & EDGE_FALLTHRU)
4557
    sel_redirect_edge_and_branch_force (e1, new_bb);
4558
  else
4559
    sel_redirect_edge_and_branch (e1, new_bb);
4560
 
4561
  gcc_assert (e1->dest == new_bb);
4562
  gcc_assert (sel_bb_empty_p (bb));
4563
 
4564
  /* To keep basic block numbers in sync between debug and non-debug
4565
     compilations, we have to rotate blocks here.  Consider that we
4566
     started from (a,b)->d, (c,d)->e, and d contained only debug
4567
     insns.  It would have been removed before if the debug insns
4568
     weren't there, so we'd have split e rather than d.  So what we do
4569
     now is to swap the block numbers of new_bb and
4570
     single_succ(new_bb) == e, so that the insns that were in e before
4571
     get the new block number.  */
4572
 
4573
  if (MAY_HAVE_DEBUG_INSNS)
4574
    {
4575
      basic_block succ;
4576
      insn_t insn = sel_bb_head (new_bb);
4577
      insn_t last;
4578
 
4579
      if (DEBUG_INSN_P (insn)
4580
          && single_succ_p (new_bb)
4581
          && (succ = single_succ (new_bb))
4582
          && succ != EXIT_BLOCK_PTR
4583
          && DEBUG_INSN_P ((last = sel_bb_end (new_bb))))
4584
        {
4585
          while (insn != last && (DEBUG_INSN_P (insn) || NOTE_P (insn)))
4586
            insn = NEXT_INSN (insn);
4587
 
4588
          if (insn == last)
4589
            {
4590
              sel_global_bb_info_def gbi;
4591
              sel_region_bb_info_def rbi;
4592
              int i;
4593
 
4594
              if (sched_verbose >= 2)
4595
                sel_print ("Swapping block ids %i and %i\n",
4596
                           new_bb->index, succ->index);
4597
 
4598
              i = new_bb->index;
4599
              new_bb->index = succ->index;
4600
              succ->index = i;
4601
 
4602
              SET_BASIC_BLOCK (new_bb->index, new_bb);
4603
              SET_BASIC_BLOCK (succ->index, succ);
4604
 
4605
              memcpy (&gbi, SEL_GLOBAL_BB_INFO (new_bb), sizeof (gbi));
4606
              memcpy (SEL_GLOBAL_BB_INFO (new_bb), SEL_GLOBAL_BB_INFO (succ),
4607
                      sizeof (gbi));
4608
              memcpy (SEL_GLOBAL_BB_INFO (succ), &gbi, sizeof (gbi));
4609
 
4610
              memcpy (&rbi, SEL_REGION_BB_INFO (new_bb), sizeof (rbi));
4611
              memcpy (SEL_REGION_BB_INFO (new_bb), SEL_REGION_BB_INFO (succ),
4612
                      sizeof (rbi));
4613
              memcpy (SEL_REGION_BB_INFO (succ), &rbi, sizeof (rbi));
4614
 
4615
              i = BLOCK_TO_BB (new_bb->index);
4616
              BLOCK_TO_BB (new_bb->index) = BLOCK_TO_BB (succ->index);
4617
              BLOCK_TO_BB (succ->index) = i;
4618
 
4619
              i = CONTAINING_RGN (new_bb->index);
4620
              CONTAINING_RGN (new_bb->index) = CONTAINING_RGN (succ->index);
4621
              CONTAINING_RGN (succ->index) = i;
4622
 
4623
              for (i = 0; i < current_nr_blocks; i++)
4624
                if (BB_TO_BLOCK (i) == succ->index)
4625
                  BB_TO_BLOCK (i) = new_bb->index;
4626
                else if (BB_TO_BLOCK (i) == new_bb->index)
4627
                  BB_TO_BLOCK (i) = succ->index;
4628
 
4629
              FOR_BB_INSNS (new_bb, insn)
4630
                if (INSN_P (insn))
4631
                  EXPR_ORIG_BB_INDEX (INSN_EXPR (insn)) = new_bb->index;
4632
 
4633
              FOR_BB_INSNS (succ, insn)
4634
                if (INSN_P (insn))
4635
                  EXPR_ORIG_BB_INDEX (INSN_EXPR (insn)) = succ->index;
4636
 
4637
              if (bitmap_clear_bit (code_motion_visited_blocks, new_bb->index))
4638
                bitmap_set_bit (code_motion_visited_blocks, succ->index);
4639
 
4640
              gcc_assert (LABEL_P (BB_HEAD (new_bb))
4641
                          && LABEL_P (BB_HEAD (succ)));
4642
 
4643
              if (sched_verbose >= 4)
4644
                sel_print ("Swapping code labels %i and %i\n",
4645
                           CODE_LABEL_NUMBER (BB_HEAD (new_bb)),
4646
                           CODE_LABEL_NUMBER (BB_HEAD (succ)));
4647
 
4648
              i = CODE_LABEL_NUMBER (BB_HEAD (new_bb));
4649
              CODE_LABEL_NUMBER (BB_HEAD (new_bb))
4650
                = CODE_LABEL_NUMBER (BB_HEAD (succ));
4651
              CODE_LABEL_NUMBER (BB_HEAD (succ)) = i;
4652
            }
4653
        }
4654
    }
4655
 
4656
  return bb;
4657
}
4658
 
4659
/* Return insn after which we must insert bookkeeping code for path(s) incoming
4660
   into E2->dest, except from E1->src.  If the returned insn immediately
4661
   precedes a fence, assign that fence to *FENCE_TO_REWIND.  */
4662
static insn_t
4663
find_place_for_bookkeeping (edge e1, edge e2, fence_t *fence_to_rewind)
4664
{
4665
  insn_t place_to_insert;
4666
  /* Find a basic block that can hold bookkeeping.  If it can be found, do not
4667
     create new basic block, but insert bookkeeping there.  */
4668
  basic_block book_block = find_block_for_bookkeeping (e1, e2, FALSE);
4669
 
4670
  if (book_block)
4671
    {
4672
      place_to_insert = BB_END (book_block);
4673
 
4674
      /* Don't use a block containing only debug insns for
4675
         bookkeeping, this causes scheduling differences between debug
4676
         and non-debug compilations, for the block would have been
4677
         removed already.  */
4678
      if (DEBUG_INSN_P (place_to_insert))
4679
        {
4680
          rtx insn = sel_bb_head (book_block);
4681
 
4682
          while (insn != place_to_insert &&
4683
                 (DEBUG_INSN_P (insn) || NOTE_P (insn)))
4684
            insn = NEXT_INSN (insn);
4685
 
4686
          if (insn == place_to_insert)
4687
            book_block = NULL;
4688
        }
4689
    }
4690
 
4691
  if (!book_block)
4692
    {
4693
      book_block = create_block_for_bookkeeping (e1, e2);
4694
      place_to_insert = BB_END (book_block);
4695
      if (sched_verbose >= 9)
4696
        sel_print ("New block is %i, split from bookkeeping block %i\n",
4697
                   EDGE_SUCC (book_block, 0)->dest->index, book_block->index);
4698
    }
4699
  else
4700
    {
4701
      if (sched_verbose >= 9)
4702
        sel_print ("Pre-existing bookkeeping block is %i\n", book_block->index);
4703
    }
4704
 
4705
  *fence_to_rewind = NULL;
4706
  /* If basic block ends with a jump, insert bookkeeping code right before it.
4707
     Notice if we are crossing a fence when taking PREV_INSN.  */
4708
  if (INSN_P (place_to_insert) && control_flow_insn_p (place_to_insert))
4709
    {
4710
      *fence_to_rewind = flist_lookup (fences, place_to_insert);
4711
      place_to_insert = PREV_INSN (place_to_insert);
4712
    }
4713
 
4714
  return place_to_insert;
4715
}
4716
 
4717
/* Find a proper seqno for bookkeeing insn inserted at PLACE_TO_INSERT
4718
   for JOIN_POINT.   */
4719
static int
4720
find_seqno_for_bookkeeping (insn_t place_to_insert, insn_t join_point)
4721
{
4722
  int seqno;
4723
  rtx next;
4724
 
4725
  /* Check if we are about to insert bookkeeping copy before a jump, and use
4726
     jump's seqno for the copy; otherwise, use JOIN_POINT's seqno.  */
4727
  next = NEXT_INSN (place_to_insert);
4728
  if (INSN_P (next)
4729
      && JUMP_P (next)
4730
      && BLOCK_FOR_INSN (next) == BLOCK_FOR_INSN (place_to_insert))
4731
    {
4732
      gcc_assert (INSN_SCHED_TIMES (next) == 0);
4733
      seqno = INSN_SEQNO (next);
4734
    }
4735
  else if (INSN_SEQNO (join_point) > 0)
4736
    seqno = INSN_SEQNO (join_point);
4737
  else
4738
    {
4739
      seqno = get_seqno_by_preds (place_to_insert);
4740
 
4741
      /* Sometimes the fences can move in such a way that there will be
4742
         no instructions with positive seqno around this bookkeeping.
4743
         This means that there will be no way to get to it by a regular
4744
         fence movement.  Never mind because we pick up such pieces for
4745
         rescheduling anyways, so any positive value will do for now.  */
4746
      if (seqno < 0)
4747
        {
4748
          gcc_assert (pipelining_p);
4749
          seqno = 1;
4750
        }
4751
    }
4752
 
4753
  gcc_assert (seqno > 0);
4754
  return seqno;
4755
}
4756
 
4757
/* Insert bookkeeping copy of C_EXPS's insn after PLACE_TO_INSERT, assigning
4758
   NEW_SEQNO to it.  Return created insn.  */
4759
static insn_t
4760
emit_bookkeeping_insn (insn_t place_to_insert, expr_t c_expr, int new_seqno)
4761
{
4762
  rtx new_insn_rtx = create_copy_of_insn_rtx (EXPR_INSN_RTX (c_expr));
4763
 
4764
  vinsn_t new_vinsn
4765
    = create_vinsn_from_insn_rtx (new_insn_rtx,
4766
                                  VINSN_UNIQUE_P (EXPR_VINSN (c_expr)));
4767
 
4768
  insn_t new_insn = emit_insn_from_expr_after (c_expr, new_vinsn, new_seqno,
4769
                                               place_to_insert);
4770
 
4771
  INSN_SCHED_TIMES (new_insn) = 0;
4772
  bitmap_set_bit (current_copies, INSN_UID (new_insn));
4773
 
4774
  return new_insn;
4775
}
4776
 
4777
/* Generate a bookkeeping copy of C_EXPR's insn for path(s) incoming into to
4778
   E2->dest, except from E1->src (there may be a sequence of empty blocks
4779
   between E1->src and E2->dest).  Return block containing the copy.
4780
   All scheduler data is initialized for the newly created insn.  */
4781
static basic_block
4782
generate_bookkeeping_insn (expr_t c_expr, edge e1, edge e2)
4783
{
4784
  insn_t join_point, place_to_insert, new_insn;
4785
  int new_seqno;
4786
  bool need_to_exchange_data_sets;
4787
  fence_t fence_to_rewind;
4788
 
4789
  if (sched_verbose >= 4)
4790
    sel_print ("Generating bookkeeping insn (%d->%d)\n", e1->src->index,
4791
               e2->dest->index);
4792
 
4793
  join_point = sel_bb_head (e2->dest);
4794
  place_to_insert = find_place_for_bookkeeping (e1, e2, &fence_to_rewind);
4795
  new_seqno = find_seqno_for_bookkeeping (place_to_insert, join_point);
4796
  need_to_exchange_data_sets
4797
    = sel_bb_empty_p (BLOCK_FOR_INSN (place_to_insert));
4798
 
4799
  new_insn = emit_bookkeeping_insn (place_to_insert, c_expr, new_seqno);
4800
 
4801
  if (fence_to_rewind)
4802
    FENCE_INSN (fence_to_rewind) = new_insn;
4803
 
4804
  /* When inserting bookkeeping insn in new block, av sets should be
4805
     following: old basic block (that now holds bookkeeping) data sets are
4806
     the same as was before generation of bookkeeping, and new basic block
4807
     (that now hold all other insns of old basic block) data sets are
4808
     invalid.  So exchange data sets for these basic blocks as sel_split_block
4809
     mistakenly exchanges them in this case.  Cannot do it earlier because
4810
     when single instruction is added to new basic block it should hold NULL
4811
     lv_set.  */
4812
  if (need_to_exchange_data_sets)
4813
    exchange_data_sets (BLOCK_FOR_INSN (new_insn),
4814
                        BLOCK_FOR_INSN (join_point));
4815
 
4816
  stat_bookkeeping_copies++;
4817
  return BLOCK_FOR_INSN (new_insn);
4818
}
4819
 
4820
/* Remove from AV_PTR all insns that may need bookkeeping when scheduling
4821
   on FENCE, but we are unable to copy them.  */
4822
static void
4823
remove_insns_that_need_bookkeeping (fence_t fence, av_set_t *av_ptr)
4824
{
4825
  expr_t expr;
4826
  av_set_iterator i;
4827
 
4828
  /*  An expression does not need bookkeeping if it is available on all paths
4829
      from current block to original block and current block dominates
4830
      original block.  We check availability on all paths by examining
4831
      EXPR_SPEC; this is not equivalent, because it may be positive even
4832
      if expr is available on all paths (but if expr is not available on
4833
      any path, EXPR_SPEC will be positive).  */
4834
 
4835
  FOR_EACH_EXPR_1 (expr, i, av_ptr)
4836
    {
4837
      if (!control_flow_insn_p (EXPR_INSN_RTX (expr))
4838
          && (!bookkeeping_p || VINSN_UNIQUE_P (EXPR_VINSN (expr)))
4839
          && (EXPR_SPEC (expr)
4840
              || !EXPR_ORIG_BB_INDEX (expr)
4841
              || !dominated_by_p (CDI_DOMINATORS,
4842
                                  BASIC_BLOCK (EXPR_ORIG_BB_INDEX (expr)),
4843
                                  BLOCK_FOR_INSN (FENCE_INSN (fence)))))
4844
        {
4845
          if (sched_verbose >= 4)
4846
            sel_print ("Expr %d removed because it would need bookkeeping, which "
4847
                       "cannot be created\n", INSN_UID (EXPR_INSN_RTX (expr)));
4848
          av_set_iter_remove (&i);
4849
        }
4850
    }
4851
}
4852
 
4853
/* Moving conditional jump through some instructions.
4854
 
4855
   Consider example:
4856
 
4857
       ...                     <- current scheduling point
4858
       NOTE BASIC BLOCK:       <- bb header
4859
       (p8)  add r14=r14+0x9;;
4860
       (p8)  mov [r14]=r23
4861
       (!p8) jump L1;;
4862
       NOTE BASIC BLOCK:
4863
       ...
4864
 
4865
   We can schedule jump one cycle earlier, than mov, because they cannot be
4866
   executed together as their predicates are mutually exclusive.
4867
 
4868
   This is done in this way: first, new fallthrough basic block is created
4869
   after jump (it is always can be done, because there already should be a
4870
   fallthrough block, where control flow goes in case of predicate being true -
4871
   in our example; otherwise there should be a dependence between those
4872
   instructions and jump and we cannot schedule jump right now);
4873
   next, all instructions between jump and current scheduling point are moved
4874
   to this new block.  And the result is this:
4875
 
4876
      NOTE BASIC BLOCK:
4877
      (!p8) jump L1           <- current scheduling point
4878
      NOTE BASIC BLOCK:       <- bb header
4879
      (p8)  add r14=r14+0x9;;
4880
      (p8)  mov [r14]=r23
4881
      NOTE BASIC BLOCK:
4882
      ...
4883
*/
4884
static void
4885
move_cond_jump (rtx insn, bnd_t bnd)
4886
{
4887
  edge ft_edge;
4888
  basic_block block_from, block_next, block_new, block_bnd, bb;
4889
  rtx next, prev, link, head;
4890
 
4891
  block_from = BLOCK_FOR_INSN (insn);
4892
  block_bnd = BLOCK_FOR_INSN (BND_TO (bnd));
4893
  prev = BND_TO (bnd);
4894
 
4895
#ifdef ENABLE_CHECKING
4896
  /* Moving of jump should not cross any other jumps or beginnings of new
4897
     basic blocks.  The only exception is when we move a jump through
4898
     mutually exclusive insns along fallthru edges.  */
4899
  if (block_from != block_bnd)
4900
    {
4901
      bb = block_from;
4902
      for (link = PREV_INSN (insn); link != PREV_INSN (prev);
4903
           link = PREV_INSN (link))
4904
        {
4905
          if (INSN_P (link))
4906
            gcc_assert (sched_insns_conditions_mutex_p (insn, link));
4907
          if (BLOCK_FOR_INSN (link) && BLOCK_FOR_INSN (link) != bb)
4908
            {
4909
              gcc_assert (single_pred (bb) == BLOCK_FOR_INSN (link));
4910
              bb = BLOCK_FOR_INSN (link);
4911
            }
4912
        }
4913
    }
4914
#endif
4915
 
4916
  /* Jump is moved to the boundary.  */
4917
  next = PREV_INSN (insn);
4918
  BND_TO (bnd) = insn;
4919
 
4920
  ft_edge = find_fallthru_edge_from (block_from);
4921
  block_next = ft_edge->dest;
4922
  /* There must be a fallthrough block (or where should go
4923
  control flow in case of false jump predicate otherwise?).  */
4924
  gcc_assert (block_next);
4925
 
4926
  /* Create new empty basic block after source block.  */
4927
  block_new = sel_split_edge (ft_edge);
4928
  gcc_assert (block_new->next_bb == block_next
4929
              && block_from->next_bb == block_new);
4930
 
4931
  /* Move all instructions except INSN to BLOCK_NEW.  */
4932
  bb = block_bnd;
4933
  head = BB_HEAD (block_new);
4934
  while (bb != block_from->next_bb)
4935
    {
4936
      rtx from, to;
4937
      from = bb == block_bnd ? prev : sel_bb_head (bb);
4938
      to = bb == block_from ? next : sel_bb_end (bb);
4939
 
4940
      /* The jump being moved can be the first insn in the block.
4941
         In this case we don't have to move anything in this block.  */
4942
      if (NEXT_INSN (to) != from)
4943
        {
4944
          reorder_insns (from, to, head);
4945
 
4946
          for (link = to; link != head; link = PREV_INSN (link))
4947
            EXPR_ORIG_BB_INDEX (INSN_EXPR (link)) = block_new->index;
4948
          head = to;
4949
        }
4950
 
4951
      /* Cleanup possibly empty blocks left.  */
4952
      block_next = bb->next_bb;
4953
      if (bb != block_from)
4954
        tidy_control_flow (bb, false);
4955
      bb = block_next;
4956
    }
4957
 
4958
  /* Assert there is no jump to BLOCK_NEW, only fallthrough edge.  */
4959
  gcc_assert (NOTE_INSN_BASIC_BLOCK_P (BB_HEAD (block_new)));
4960
 
4961
  gcc_assert (!sel_bb_empty_p (block_from)
4962
              && !sel_bb_empty_p (block_new));
4963
 
4964
  /* Update data sets for BLOCK_NEW to represent that INSN and
4965
     instructions from the other branch of INSN is no longer
4966
     available at BLOCK_NEW.  */
4967
  BB_AV_LEVEL (block_new) = global_level;
4968
  gcc_assert (BB_LV_SET (block_new) == NULL);
4969
  BB_LV_SET (block_new) = get_clear_regset_from_pool ();
4970
  update_data_sets (sel_bb_head (block_new));
4971
 
4972
  /* INSN is a new basic block header - so prepare its data
4973
     structures and update availability and liveness sets.  */
4974
  update_data_sets (insn);
4975
 
4976
  if (sched_verbose >= 4)
4977
    sel_print ("Moving jump %d\n", INSN_UID (insn));
4978
}
4979
 
4980
/* Remove nops generated during move_op for preventing removal of empty
4981
   basic blocks.  */
4982
static void
4983
remove_temp_moveop_nops (bool full_tidying)
4984
{
4985
  int i;
4986
  insn_t insn;
4987
 
4988
  FOR_EACH_VEC_ELT (insn_t, vec_temp_moveop_nops, i, insn)
4989
    {
4990
      gcc_assert (INSN_NOP_P (insn));
4991
      return_nop_to_pool (insn, full_tidying);
4992
    }
4993
 
4994
  /* Empty the vector.  */
4995
  if (VEC_length (insn_t, vec_temp_moveop_nops) > 0)
4996
    VEC_block_remove (insn_t, vec_temp_moveop_nops, 0,
4997
                      VEC_length (insn_t, vec_temp_moveop_nops));
4998
}
4999
 
5000
/* Records the maximal UID before moving up an instruction.  Used for
5001
   distinguishing between bookkeeping copies and original insns.  */
5002
static int max_uid_before_move_op = 0;
5003
 
5004
/* Remove from AV_VLIW_P all instructions but next when debug counter
5005
   tells us so.  Next instruction is fetched from BNDS.  */
5006
static void
5007
remove_insns_for_debug (blist_t bnds, av_set_t *av_vliw_p)
5008
{
5009
  if (! dbg_cnt (sel_sched_insn_cnt))
5010
    /* Leave only the next insn in av_vliw.  */
5011
    {
5012
      av_set_iterator av_it;
5013
      expr_t expr;
5014
      bnd_t bnd = BLIST_BND (bnds);
5015
      insn_t next = BND_TO (bnd);
5016
 
5017
      gcc_assert (BLIST_NEXT (bnds) == NULL);
5018
 
5019
      FOR_EACH_EXPR_1 (expr, av_it, av_vliw_p)
5020
        if (EXPR_INSN_RTX (expr) != next)
5021
          av_set_iter_remove (&av_it);
5022
    }
5023
}
5024
 
5025
/* Compute available instructions on BNDS.  FENCE is the current fence.  Write
5026
   the computed set to *AV_VLIW_P.  */
5027
static void
5028
compute_av_set_on_boundaries (fence_t fence, blist_t bnds, av_set_t *av_vliw_p)
5029
{
5030
  if (sched_verbose >= 2)
5031
    {
5032
      sel_print ("Boundaries: ");
5033
      dump_blist (bnds);
5034
      sel_print ("\n");
5035
    }
5036
 
5037
  for (; bnds; bnds = BLIST_NEXT (bnds))
5038
    {
5039
      bnd_t bnd = BLIST_BND (bnds);
5040
      av_set_t av1_copy;
5041
      insn_t bnd_to = BND_TO (bnd);
5042
 
5043
      /* Rewind BND->TO to the basic block header in case some bookkeeping
5044
         instructions were inserted before BND->TO and it needs to be
5045
         adjusted.  */
5046
      if (sel_bb_head_p (bnd_to))
5047
        gcc_assert (INSN_SCHED_TIMES (bnd_to) == 0);
5048
      else
5049
        while (INSN_SCHED_TIMES (PREV_INSN (bnd_to)) == 0)
5050
          {
5051
            bnd_to = PREV_INSN (bnd_to);
5052
            if (sel_bb_head_p (bnd_to))
5053
              break;
5054
          }
5055
 
5056
      if (BND_TO (bnd) != bnd_to)
5057
        {
5058
          gcc_assert (FENCE_INSN (fence) == BND_TO (bnd));
5059
          FENCE_INSN (fence) = bnd_to;
5060
          BND_TO (bnd) = bnd_to;
5061
        }
5062
 
5063
      av_set_clear (&BND_AV (bnd));
5064
      BND_AV (bnd) = compute_av_set (BND_TO (bnd), NULL, 0, true);
5065
 
5066
      av_set_clear (&BND_AV1 (bnd));
5067
      BND_AV1 (bnd) = av_set_copy (BND_AV (bnd));
5068
 
5069
      moveup_set_inside_insn_group (&BND_AV1 (bnd), NULL);
5070
 
5071
      av1_copy = av_set_copy (BND_AV1 (bnd));
5072
      av_set_union_and_clear (av_vliw_p, &av1_copy, NULL);
5073
    }
5074
 
5075
  if (sched_verbose >= 2)
5076
    {
5077
      sel_print ("Available exprs (vliw form): ");
5078
      dump_av_set (*av_vliw_p);
5079
      sel_print ("\n");
5080
    }
5081
}
5082
 
5083
/* Calculate the sequential av set on BND corresponding to the EXPR_VLIW
5084
   expression.  When FOR_MOVEOP is true, also replace the register of
5085
   expressions found with the register from EXPR_VLIW.  */
5086
static av_set_t
5087
find_sequential_best_exprs (bnd_t bnd, expr_t expr_vliw, bool for_moveop)
5088
{
5089
  av_set_t expr_seq = NULL;
5090
  expr_t expr;
5091
  av_set_iterator i;
5092
 
5093
  FOR_EACH_EXPR (expr, i, BND_AV (bnd))
5094
    {
5095
      if (equal_after_moveup_path_p (expr, NULL, expr_vliw))
5096
        {
5097
          if (for_moveop)
5098
            {
5099
              /* The sequential expression has the right form to pass
5100
                 to move_op except when renaming happened.  Put the
5101
                 correct register in EXPR then.  */
5102
              if (EXPR_SEPARABLE_P (expr) && REG_P (EXPR_LHS (expr)))
5103
                {
5104
                  if (expr_dest_regno (expr) != expr_dest_regno (expr_vliw))
5105
                    {
5106
                      replace_dest_with_reg_in_expr (expr, EXPR_LHS (expr_vliw));
5107
                      stat_renamed_scheduled++;
5108
                    }
5109
                  /* Also put the correct TARGET_AVAILABLE bit on the expr.
5110
                     This is needed when renaming came up with original
5111
                     register.  */
5112
                  else if (EXPR_TARGET_AVAILABLE (expr)
5113
                           != EXPR_TARGET_AVAILABLE (expr_vliw))
5114
                    {
5115
                      gcc_assert (EXPR_TARGET_AVAILABLE (expr_vliw) == 1);
5116
                      EXPR_TARGET_AVAILABLE (expr) = 1;
5117
                    }
5118
                }
5119
              if (EXPR_WAS_SUBSTITUTED (expr))
5120
                stat_substitutions_total++;
5121
            }
5122
 
5123
          av_set_add (&expr_seq, expr);
5124
 
5125
          /* With substitution inside insn group, it is possible
5126
             that more than one expression in expr_seq will correspond
5127
             to expr_vliw.  In this case, choose one as the attempt to
5128
             move both leads to miscompiles.  */
5129
          break;
5130
        }
5131
    }
5132
 
5133
  if (for_moveop && sched_verbose >= 2)
5134
    {
5135
      sel_print ("Best expression(s) (sequential form): ");
5136
      dump_av_set (expr_seq);
5137
      sel_print ("\n");
5138
    }
5139
 
5140
  return expr_seq;
5141
}
5142
 
5143
 
5144
/* Move nop to previous block.  */
5145
static void ATTRIBUTE_UNUSED
5146
move_nop_to_previous_block (insn_t nop, basic_block prev_bb)
5147
{
5148
  insn_t prev_insn, next_insn, note;
5149
 
5150
  gcc_assert (sel_bb_head_p (nop)
5151
              && prev_bb == BLOCK_FOR_INSN (nop)->prev_bb);
5152
  note = bb_note (BLOCK_FOR_INSN (nop));
5153
  prev_insn = sel_bb_end (prev_bb);
5154
  next_insn = NEXT_INSN (nop);
5155
  gcc_assert (prev_insn != NULL_RTX
5156
              && PREV_INSN (note) == prev_insn);
5157
 
5158
  NEXT_INSN (prev_insn) = nop;
5159
  PREV_INSN (nop) = prev_insn;
5160
 
5161
  PREV_INSN (note) = nop;
5162
  NEXT_INSN (note) = next_insn;
5163
 
5164
  NEXT_INSN (nop) = note;
5165
  PREV_INSN (next_insn) = note;
5166
 
5167
  BB_END (prev_bb) = nop;
5168
  BLOCK_FOR_INSN (nop) = prev_bb;
5169
}
5170
 
5171
/* Prepare a place to insert the chosen expression on BND.  */
5172
static insn_t
5173
prepare_place_to_insert (bnd_t bnd)
5174
{
5175
  insn_t place_to_insert;
5176
 
5177
  /* Init place_to_insert before calling move_op, as the later
5178
     can possibly remove BND_TO (bnd).  */
5179
  if (/* If this is not the first insn scheduled.  */
5180
      BND_PTR (bnd))
5181
    {
5182
      /* Add it after last scheduled.  */
5183
      place_to_insert = ILIST_INSN (BND_PTR (bnd));
5184
      if (DEBUG_INSN_P (place_to_insert))
5185
        {
5186
          ilist_t l = BND_PTR (bnd);
5187
          while ((l = ILIST_NEXT (l)) &&
5188
                 DEBUG_INSN_P (ILIST_INSN (l)))
5189
            ;
5190
          if (!l)
5191
            place_to_insert = NULL;
5192
        }
5193
    }
5194
  else
5195
    place_to_insert = NULL;
5196
 
5197
  if (!place_to_insert)
5198
    {
5199
      /* Add it before BND_TO.  The difference is in the
5200
         basic block, where INSN will be added.  */
5201
      place_to_insert = get_nop_from_pool (BND_TO (bnd));
5202
      gcc_assert (BLOCK_FOR_INSN (place_to_insert)
5203
                  == BLOCK_FOR_INSN (BND_TO (bnd)));
5204
    }
5205
 
5206
  return place_to_insert;
5207
}
5208
 
5209
/* Find original instructions for EXPR_SEQ and move it to BND boundary.
5210
   Return the expression to emit in C_EXPR.  */
5211
static bool
5212
move_exprs_to_boundary (bnd_t bnd, expr_t expr_vliw,
5213
                        av_set_t expr_seq, expr_t c_expr)
5214
{
5215
  bool b, should_move;
5216
  unsigned book_uid;
5217
  bitmap_iterator bi;
5218
  int n_bookkeeping_copies_before_moveop;
5219
 
5220
  /* Make a move.  This call will remove the original operation,
5221
     insert all necessary bookkeeping instructions and update the
5222
     data sets.  After that all we have to do is add the operation
5223
     at before BND_TO (BND).  */
5224
  n_bookkeeping_copies_before_moveop = stat_bookkeeping_copies;
5225
  max_uid_before_move_op = get_max_uid ();
5226
  bitmap_clear (current_copies);
5227
  bitmap_clear (current_originators);
5228
 
5229
  b = move_op (BND_TO (bnd), expr_seq, expr_vliw,
5230
               get_dest_from_orig_ops (expr_seq), c_expr, &should_move);
5231
 
5232
  /* We should be able to find the expression we've chosen for
5233
     scheduling.  */
5234
  gcc_assert (b);
5235
 
5236
  if (stat_bookkeeping_copies > n_bookkeeping_copies_before_moveop)
5237
    stat_insns_needed_bookkeeping++;
5238
 
5239
  EXECUTE_IF_SET_IN_BITMAP (current_copies, 0, book_uid, bi)
5240
    {
5241
      unsigned uid;
5242
      bitmap_iterator bi;
5243
 
5244
      /* We allocate these bitmaps lazily.  */
5245
      if (! INSN_ORIGINATORS_BY_UID (book_uid))
5246
        INSN_ORIGINATORS_BY_UID (book_uid) = BITMAP_ALLOC (NULL);
5247
 
5248
      bitmap_copy (INSN_ORIGINATORS_BY_UID (book_uid),
5249
                   current_originators);
5250
 
5251
      /* Transitively add all originators' originators.  */
5252
      EXECUTE_IF_SET_IN_BITMAP (current_originators, 0, uid, bi)
5253
       if (INSN_ORIGINATORS_BY_UID (uid))
5254
         bitmap_ior_into (INSN_ORIGINATORS_BY_UID (book_uid),
5255
                          INSN_ORIGINATORS_BY_UID (uid));
5256
    }
5257
 
5258
  return should_move;
5259
}
5260
 
5261
 
5262
/* Debug a DFA state as an array of bytes.  */
5263
static void
5264
debug_state (state_t state)
5265
{
5266
  unsigned char *p;
5267
  unsigned int i, size = dfa_state_size;
5268
 
5269
  sel_print ("state (%u):", size);
5270
  for (i = 0, p = (unsigned char *) state; i < size; i++)
5271
    sel_print (" %d", p[i]);
5272
  sel_print ("\n");
5273
}
5274
 
5275
/* Advance state on FENCE with INSN.  Return true if INSN is
5276
   an ASM, and we should advance state once more.  */
5277
static bool
5278
advance_state_on_fence (fence_t fence, insn_t insn)
5279
{
5280
  bool asm_p;
5281
 
5282
  if (recog_memoized (insn) >= 0)
5283
    {
5284
      int res;
5285
      state_t temp_state = alloca (dfa_state_size);
5286
 
5287
      gcc_assert (!INSN_ASM_P (insn));
5288
      asm_p = false;
5289
 
5290
      memcpy (temp_state, FENCE_STATE (fence), dfa_state_size);
5291
      res = state_transition (FENCE_STATE (fence), insn);
5292
      gcc_assert (res < 0);
5293
 
5294
      if (memcmp (temp_state, FENCE_STATE (fence), dfa_state_size))
5295
        {
5296
          FENCE_ISSUED_INSNS (fence)++;
5297
 
5298
          /* We should never issue more than issue_rate insns.  */
5299
          if (FENCE_ISSUED_INSNS (fence) > issue_rate)
5300
            gcc_unreachable ();
5301
        }
5302
    }
5303
  else
5304
    {
5305
      /* This could be an ASM insn which we'd like to schedule
5306
         on the next cycle.  */
5307
      asm_p = INSN_ASM_P (insn);
5308
      if (!FENCE_STARTS_CYCLE_P (fence) && asm_p)
5309
        advance_one_cycle (fence);
5310
    }
5311
 
5312
  if (sched_verbose >= 2)
5313
    debug_state (FENCE_STATE (fence));
5314
  if (!DEBUG_INSN_P (insn))
5315
    FENCE_STARTS_CYCLE_P (fence) = 0;
5316
  FENCE_ISSUE_MORE (fence) = can_issue_more;
5317
  return asm_p;
5318
}
5319
 
5320
/* Update FENCE on which INSN was scheduled and this INSN, too.  NEED_STALL
5321
   is nonzero if we need to stall after issuing INSN.  */
5322
static void
5323
update_fence_and_insn (fence_t fence, insn_t insn, int need_stall)
5324
{
5325
  bool asm_p;
5326
 
5327
  /* First, reflect that something is scheduled on this fence.  */
5328
  asm_p = advance_state_on_fence (fence, insn);
5329
  FENCE_LAST_SCHEDULED_INSN (fence) = insn;
5330
  VEC_safe_push (rtx, gc, FENCE_EXECUTING_INSNS (fence), insn);
5331
  if (SCHED_GROUP_P (insn))
5332
    {
5333
      FENCE_SCHED_NEXT (fence) = INSN_SCHED_NEXT (insn);
5334
      SCHED_GROUP_P (insn) = 0;
5335
    }
5336
  else
5337
    FENCE_SCHED_NEXT (fence) = NULL_RTX;
5338
  if (INSN_UID (insn) < FENCE_READY_TICKS_SIZE (fence))
5339
    FENCE_READY_TICKS (fence) [INSN_UID (insn)] = 0;
5340
 
5341
  /* Set instruction scheduling info.  This will be used in bundling,
5342
     pipelining, tick computations etc.  */
5343
  ++INSN_SCHED_TIMES (insn);
5344
  EXPR_TARGET_AVAILABLE (INSN_EXPR (insn)) = true;
5345
  EXPR_ORIG_SCHED_CYCLE (INSN_EXPR (insn)) = FENCE_CYCLE (fence);
5346
  INSN_AFTER_STALL_P (insn) = FENCE_AFTER_STALL_P (fence);
5347
  INSN_SCHED_CYCLE (insn) = FENCE_CYCLE (fence);
5348
 
5349
  /* This does not account for adjust_cost hooks, just add the biggest
5350
     constant the hook may add to the latency.  TODO: make this
5351
     a target dependent constant.  */
5352
  INSN_READY_CYCLE (insn)
5353
    = INSN_SCHED_CYCLE (insn) + (INSN_CODE (insn) < 0
5354
                                 ? 1
5355
                                 : maximal_insn_latency (insn) + 1);
5356
 
5357
  /* Change these fields last, as they're used above.  */
5358
  FENCE_AFTER_STALL_P (fence) = 0;
5359
  if (asm_p || need_stall)
5360
    advance_one_cycle (fence);
5361
 
5362
  /* Indicate that we've scheduled something on this fence.  */
5363
  FENCE_SCHEDULED_P (fence) = true;
5364
  scheduled_something_on_previous_fence = true;
5365
 
5366
  /* Print debug information when insn's fields are updated.  */
5367
  if (sched_verbose >= 2)
5368
    {
5369
      sel_print ("Scheduling insn: ");
5370
      dump_insn_1 (insn, 1);
5371
      sel_print ("\n");
5372
    }
5373
}
5374
 
5375
/* Update boundary BND (and, if needed, FENCE) with INSN, remove the
5376
   old boundary from BNDSP, add new boundaries to BNDS_TAIL_P and
5377
   return it.  */
5378
static blist_t *
5379
update_boundaries (fence_t fence, bnd_t bnd, insn_t insn, blist_t *bndsp,
5380
                   blist_t *bnds_tailp)
5381
{
5382
  succ_iterator si;
5383
  insn_t succ;
5384
 
5385
  advance_deps_context (BND_DC (bnd), insn);
5386
  FOR_EACH_SUCC_1 (succ, si, insn,
5387
                   SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
5388
    {
5389
      ilist_t ptr = ilist_copy (BND_PTR (bnd));
5390
 
5391
      ilist_add (&ptr, insn);
5392
 
5393
      if (DEBUG_INSN_P (insn) && sel_bb_end_p (insn)
5394
          && is_ineligible_successor (succ, ptr))
5395
        {
5396
          ilist_clear (&ptr);
5397
          continue;
5398
        }
5399
 
5400
      if (FENCE_INSN (fence) == insn && !sel_bb_end_p (insn))
5401
        {
5402
          if (sched_verbose >= 9)
5403
            sel_print ("Updating fence insn from %i to %i\n",
5404
                       INSN_UID (insn), INSN_UID (succ));
5405
          FENCE_INSN (fence) = succ;
5406
        }
5407
      blist_add (bnds_tailp, succ, ptr, BND_DC (bnd));
5408
      bnds_tailp = &BLIST_NEXT (*bnds_tailp);
5409
    }
5410
 
5411
  blist_remove (bndsp);
5412
  return bnds_tailp;
5413
}
5414
 
5415
/* Schedule EXPR_VLIW on BND.  Return the insn emitted.  */
5416
static insn_t
5417
schedule_expr_on_boundary (bnd_t bnd, expr_t expr_vliw, int seqno)
5418
{
5419
  av_set_t expr_seq;
5420
  expr_t c_expr = XALLOCA (expr_def);
5421
  insn_t place_to_insert;
5422
  insn_t insn;
5423
  bool should_move;
5424
 
5425
  expr_seq = find_sequential_best_exprs (bnd, expr_vliw, true);
5426
 
5427
  /* In case of scheduling a jump skipping some other instructions,
5428
     prepare CFG.  After this, jump is at the boundary and can be
5429
     scheduled as usual insn by MOVE_OP.  */
5430
  if (vinsn_cond_branch_p (EXPR_VINSN (expr_vliw)))
5431
    {
5432
      insn = EXPR_INSN_RTX (expr_vliw);
5433
 
5434
      /* Speculative jumps are not handled.  */
5435
      if (insn != BND_TO (bnd)
5436
          && !sel_insn_is_speculation_check (insn))
5437
        move_cond_jump (insn, bnd);
5438
    }
5439
 
5440
  /* Find a place for C_EXPR to schedule.  */
5441
  place_to_insert = prepare_place_to_insert (bnd);
5442
  should_move = move_exprs_to_boundary (bnd, expr_vliw, expr_seq, c_expr);
5443
  clear_expr (c_expr);
5444
 
5445
  /* Add the instruction.  The corner case to care about is when
5446
     the expr_seq set has more than one expr, and we chose the one that
5447
     is not equal to expr_vliw.  Then expr_vliw may be insn in stream, and
5448
     we can't use it.  Generate the new vinsn.  */
5449
  if (INSN_IN_STREAM_P (EXPR_INSN_RTX (expr_vliw)))
5450
    {
5451
      vinsn_t vinsn_new;
5452
 
5453
      vinsn_new = vinsn_copy (EXPR_VINSN (expr_vliw), false);
5454
      change_vinsn_in_expr (expr_vliw, vinsn_new);
5455
      should_move = false;
5456
    }
5457
  if (should_move)
5458
    insn = sel_move_insn (expr_vliw, seqno, place_to_insert);
5459
  else
5460
    insn = emit_insn_from_expr_after (expr_vliw, NULL, seqno,
5461
                                      place_to_insert);
5462
 
5463
  /* Return the nops generated for preserving of data sets back
5464
     into pool.  */
5465
  if (INSN_NOP_P (place_to_insert))
5466
    return_nop_to_pool (place_to_insert, !DEBUG_INSN_P (insn));
5467
  remove_temp_moveop_nops (!DEBUG_INSN_P (insn));
5468
 
5469
  av_set_clear (&expr_seq);
5470
 
5471
  /* Save the expression scheduled so to reset target availability if we'll
5472
     meet it later on the same fence.  */
5473
  if (EXPR_WAS_RENAMED (expr_vliw))
5474
    vinsn_vec_add (&vec_target_unavailable_vinsns, INSN_EXPR (insn));
5475
 
5476
  /* Check that the recent movement didn't destroyed loop
5477
     structure.  */
5478
  gcc_assert (!pipelining_p
5479
              || current_loop_nest == NULL
5480
              || loop_latch_edge (current_loop_nest));
5481
  return insn;
5482
}
5483
 
5484
/* Stall for N cycles on FENCE.  */
5485
static void
5486
stall_for_cycles (fence_t fence, int n)
5487
{
5488
  int could_more;
5489
 
5490
  could_more = n > 1 || FENCE_ISSUED_INSNS (fence) < issue_rate;
5491
  while (n--)
5492
    advance_one_cycle (fence);
5493
  if (could_more)
5494
    FENCE_AFTER_STALL_P (fence) = 1;
5495
}
5496
 
5497
/* Gather a parallel group of insns at FENCE and assign their seqno
5498
   to SEQNO.  All scheduled insns are gathered in SCHEDULED_INSNS_TAILPP
5499
   list for later recalculation of seqnos.  */
5500
static void
5501
fill_insns (fence_t fence, int seqno, ilist_t **scheduled_insns_tailpp)
5502
{
5503
  blist_t bnds = NULL, *bnds_tailp;
5504
  av_set_t av_vliw = NULL;
5505
  insn_t insn = FENCE_INSN (fence);
5506
 
5507
  if (sched_verbose >= 2)
5508
    sel_print ("Starting fill_insns for insn %d, cycle %d\n",
5509
               INSN_UID (insn), FENCE_CYCLE (fence));
5510
 
5511
  blist_add (&bnds, insn, NULL, FENCE_DC (fence));
5512
  bnds_tailp = &BLIST_NEXT (bnds);
5513
  set_target_context (FENCE_TC (fence));
5514
  can_issue_more = FENCE_ISSUE_MORE (fence);
5515
  target_bb = INSN_BB (insn);
5516
 
5517
  /* Do while we can add any operation to the current group.  */
5518
  do
5519
    {
5520
      blist_t *bnds_tailp1, *bndsp;
5521
      expr_t expr_vliw;
5522
      int need_stall = false;
5523
      int was_stall = 0, scheduled_insns = 0;
5524
      int max_insns = pipelining_p ? issue_rate : 2 * issue_rate;
5525
      int max_stall = pipelining_p ? 1 : 3;
5526
      bool last_insn_was_debug = false;
5527
      bool was_debug_bb_end_p = false;
5528
 
5529
      compute_av_set_on_boundaries (fence, bnds, &av_vliw);
5530
      remove_insns_that_need_bookkeeping (fence, &av_vliw);
5531
      remove_insns_for_debug (bnds, &av_vliw);
5532
 
5533
      /* Return early if we have nothing to schedule.  */
5534
      if (av_vliw == NULL)
5535
        break;
5536
 
5537
      /* Choose the best expression and, if needed, destination register
5538
         for it.  */
5539
      do
5540
        {
5541
          expr_vliw = find_best_expr (&av_vliw, bnds, fence, &need_stall);
5542
          if (! expr_vliw && need_stall)
5543
            {
5544
              /* All expressions required a stall.  Do not recompute av sets
5545
                 as we'll get the same answer (modulo the insns between
5546
                 the fence and its boundary, which will not be available for
5547
                 pipelining).
5548
                 If we are going to stall for too long, break to recompute av
5549
                 sets and bring more insns for pipelining.  */
5550
              was_stall++;
5551
              if (need_stall <= 3)
5552
                stall_for_cycles (fence, need_stall);
5553
              else
5554
                {
5555
                  stall_for_cycles (fence, 1);
5556
                  break;
5557
                }
5558
            }
5559
        }
5560
      while (! expr_vliw && need_stall);
5561
 
5562
      /* Now either we've selected expr_vliw or we have nothing to schedule.  */
5563
      if (!expr_vliw)
5564
        {
5565
          av_set_clear (&av_vliw);
5566
          break;
5567
        }
5568
 
5569
      bndsp = &bnds;
5570
      bnds_tailp1 = bnds_tailp;
5571
 
5572
      do
5573
        /* This code will be executed only once until we'd have several
5574
           boundaries per fence.  */
5575
        {
5576
          bnd_t bnd = BLIST_BND (*bndsp);
5577
 
5578
          if (!av_set_is_in_p (BND_AV1 (bnd), EXPR_VINSN (expr_vliw)))
5579
            {
5580
              bndsp = &BLIST_NEXT (*bndsp);
5581
              continue;
5582
            }
5583
 
5584
          insn = schedule_expr_on_boundary (bnd, expr_vliw, seqno);
5585
          last_insn_was_debug = DEBUG_INSN_P (insn);
5586
          if (last_insn_was_debug)
5587
            was_debug_bb_end_p = (insn == BND_TO (bnd) && sel_bb_end_p (insn));
5588
          update_fence_and_insn (fence, insn, need_stall);
5589
          bnds_tailp = update_boundaries (fence, bnd, insn, bndsp, bnds_tailp);
5590
 
5591
          /* Add insn to the list of scheduled on this cycle instructions.  */
5592
          ilist_add (*scheduled_insns_tailpp, insn);
5593
          *scheduled_insns_tailpp = &ILIST_NEXT (**scheduled_insns_tailpp);
5594
        }
5595
      while (*bndsp != *bnds_tailp1);
5596
 
5597
      av_set_clear (&av_vliw);
5598
      if (!last_insn_was_debug)
5599
        scheduled_insns++;
5600
 
5601
      /* We currently support information about candidate blocks only for
5602
         one 'target_bb' block.  Hence we can't schedule after jump insn,
5603
         as this will bring two boundaries and, hence, necessity to handle
5604
         information for two or more blocks concurrently.  */
5605
      if ((last_insn_was_debug ? was_debug_bb_end_p : sel_bb_end_p (insn))
5606
          || (was_stall
5607
              && (was_stall >= max_stall
5608
                  || scheduled_insns >= max_insns)))
5609
        break;
5610
    }
5611
  while (bnds);
5612
 
5613
  gcc_assert (!FENCE_BNDS (fence));
5614
 
5615
  /* Update boundaries of the FENCE.  */
5616
  while (bnds)
5617
    {
5618
      ilist_t ptr = BND_PTR (BLIST_BND (bnds));
5619
 
5620
      if (ptr)
5621
        {
5622
          insn = ILIST_INSN (ptr);
5623
 
5624
          if (!ilist_is_in_p (FENCE_BNDS (fence), insn))
5625
            ilist_add (&FENCE_BNDS (fence), insn);
5626
        }
5627
 
5628
      blist_remove (&bnds);
5629
    }
5630
 
5631
  /* Update target context on the fence.  */
5632
  reset_target_context (FENCE_TC (fence), false);
5633
}
5634
 
5635
/* All exprs in ORIG_OPS must have the same destination register or memory.
5636
   Return that destination.  */
5637
static rtx
5638
get_dest_from_orig_ops (av_set_t orig_ops)
5639
{
5640
  rtx dest = NULL_RTX;
5641
  av_set_iterator av_it;
5642
  expr_t expr;
5643
  bool first_p = true;
5644
 
5645
  FOR_EACH_EXPR (expr, av_it, orig_ops)
5646
    {
5647
      rtx x = EXPR_LHS (expr);
5648
 
5649
      if (first_p)
5650
        {
5651
          first_p = false;
5652
          dest = x;
5653
        }
5654
      else
5655
        gcc_assert (dest == x
5656
                    || (dest != NULL_RTX && x != NULL_RTX
5657
                        && rtx_equal_p (dest, x)));
5658
    }
5659
 
5660
  return dest;
5661
}
5662
 
5663
/* Update data sets for the bookkeeping block and record those expressions
5664
   which become no longer available after inserting this bookkeeping.  */
5665
static void
5666
update_and_record_unavailable_insns (basic_block book_block)
5667
{
5668
  av_set_iterator i;
5669
  av_set_t old_av_set = NULL;
5670
  expr_t cur_expr;
5671
  rtx bb_end = sel_bb_end (book_block);
5672
 
5673
  /* First, get correct liveness in the bookkeeping block.  The problem is
5674
     the range between the bookeeping insn and the end of block.  */
5675
  update_liveness_on_insn (bb_end);
5676
  if (control_flow_insn_p (bb_end))
5677
    update_liveness_on_insn (PREV_INSN (bb_end));
5678
 
5679
  /* If there's valid av_set on BOOK_BLOCK, then there might exist another
5680
     fence above, where we may choose to schedule an insn which is
5681
     actually blocked from moving up with the bookkeeping we create here.  */
5682
  if (AV_SET_VALID_P (sel_bb_head (book_block)))
5683
    {
5684
      old_av_set = av_set_copy (BB_AV_SET (book_block));
5685
      update_data_sets (sel_bb_head (book_block));
5686
 
5687
      /* Traverse all the expressions in the old av_set and check whether
5688
         CUR_EXPR is in new AV_SET.  */
5689
      FOR_EACH_EXPR (cur_expr, i, old_av_set)
5690
        {
5691
          expr_t new_expr = av_set_lookup (BB_AV_SET (book_block),
5692
                                           EXPR_VINSN (cur_expr));
5693
 
5694
          if (! new_expr
5695
              /* In this case, we can just turn off the E_T_A bit, but we can't
5696
                 represent this information with the current vector.  */
5697
              || EXPR_TARGET_AVAILABLE (new_expr)
5698
                 != EXPR_TARGET_AVAILABLE (cur_expr))
5699
            /* Unfortunately, the below code could be also fired up on
5700
               separable insns.
5701
               FIXME: add an example of how this could happen.  */
5702
            vinsn_vec_add (&vec_bookkeeping_blocked_vinsns, cur_expr);
5703
        }
5704
 
5705
      av_set_clear (&old_av_set);
5706
    }
5707
}
5708
 
5709
/* The main effect of this function is that sparams->c_expr is merged
5710
   with (or copied to) lparams->c_expr_merged.  If there's only one successor,
5711
   we avoid merging anything by copying sparams->c_expr to lparams->c_expr_merged.
5712
   lparams->c_expr_merged is copied back to sparams->c_expr after all
5713
   successors has been traversed.  lparams->c_expr_local is an expr allocated
5714
   on stack in the caller function, and is used if there is more than one
5715
   successor.
5716
 
5717
   SUCC is one of the SUCCS_NORMAL successors of INSN,
5718
   MOVEOP_DRV_CALL_RES is the result of call code_motion_path_driver on succ,
5719
   LPARAMS and STATIC_PARAMS contain the parameters described above.  */
5720
static void
5721
move_op_merge_succs (insn_t insn ATTRIBUTE_UNUSED,
5722
                     insn_t succ ATTRIBUTE_UNUSED,
5723
                     int moveop_drv_call_res,
5724
                     cmpd_local_params_p lparams, void *static_params)
5725
{
5726
  moveop_static_params_p sparams = (moveop_static_params_p) static_params;
5727
 
5728
  /* Nothing to do, if original expr wasn't found below.  */
5729
  if (moveop_drv_call_res != 1)
5730
    return;
5731
 
5732
  /* If this is a first successor.  */
5733
  if (!lparams->c_expr_merged)
5734
    {
5735
      lparams->c_expr_merged = sparams->c_expr;
5736
      sparams->c_expr = lparams->c_expr_local;
5737
    }
5738
  else
5739
    {
5740
      /* We must merge all found expressions to get reasonable
5741
         EXPR_SPEC_DONE_DS for the resulting insn.  If we don't
5742
         do so then we can first find the expr with epsilon
5743
         speculation success probability and only then with the
5744
         good probability.  As a result the insn will get epsilon
5745
         probability and will never be scheduled because of
5746
         weakness_cutoff in find_best_expr.
5747
 
5748
         We call merge_expr_data here instead of merge_expr
5749
         because due to speculation C_EXPR and X may have the
5750
         same insns with different speculation types.  And as of
5751
         now such insns are considered non-equal.
5752
 
5753
         However, EXPR_SCHED_TIMES is different -- we must get
5754
         SCHED_TIMES from a real insn, not a bookkeeping copy.
5755
         We force this here.  Instead, we may consider merging
5756
         SCHED_TIMES to the maximum instead of minimum in the
5757
         below function.  */
5758
      int old_times = EXPR_SCHED_TIMES (lparams->c_expr_merged);
5759
 
5760
      merge_expr_data (lparams->c_expr_merged, sparams->c_expr, NULL);
5761
      if (EXPR_SCHED_TIMES (sparams->c_expr) == 0)
5762
        EXPR_SCHED_TIMES (lparams->c_expr_merged) = old_times;
5763
 
5764
      clear_expr (sparams->c_expr);
5765
    }
5766
}
5767
 
5768
/*  Add used regs for the successor SUCC into SPARAMS->USED_REGS.
5769
 
5770
   SUCC is one of the SUCCS_NORMAL successors of INSN,
5771
   MOVEOP_DRV_CALL_RES is the result of call code_motion_path_driver on succ or 0,
5772
     if SUCC is one of SUCCS_BACK or SUCCS_OUT.
5773
   STATIC_PARAMS contain USED_REGS set.  */
5774
static void
5775
fur_merge_succs (insn_t insn ATTRIBUTE_UNUSED, insn_t succ,
5776
                 int moveop_drv_call_res,
5777
                 cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
5778
                 void *static_params)
5779
{
5780
  regset succ_live;
5781
  fur_static_params_p sparams = (fur_static_params_p) static_params;
5782
 
5783
  /* Here we compute live regsets only for branches that do not lie
5784
     on the code motion paths.  These branches correspond to value
5785
     MOVEOP_DRV_CALL_RES==0 and include SUCCS_BACK and SUCCS_OUT, though
5786
     for such branches code_motion_path_driver is not called.  */
5787
  if (moveop_drv_call_res != 0)
5788
    return;
5789
 
5790
  /* Mark all registers that do not meet the following condition:
5791
     (3) not live on the other path of any conditional branch
5792
     that is passed by the operation, in case original
5793
     operations are not present on both paths of the
5794
     conditional branch.  */
5795
  succ_live = compute_live (succ);
5796
  IOR_REG_SET (sparams->used_regs, succ_live);
5797
}
5798
 
5799
/* This function is called after the last successor.  Copies LP->C_EXPR_MERGED
5800
   into SP->CEXPR.  */
5801
static void
5802
move_op_after_merge_succs (cmpd_local_params_p lp, void *sparams)
5803
{
5804
  moveop_static_params_p sp = (moveop_static_params_p) sparams;
5805
 
5806
  sp->c_expr = lp->c_expr_merged;
5807
}
5808
 
5809
/* Track bookkeeping copies created, insns scheduled, and blocks for
5810
   rescheduling when INSN is found by move_op.  */
5811
static void
5812
track_scheduled_insns_and_blocks (rtx insn)
5813
{
5814
  /* Even if this insn can be a copy that will be removed during current move_op,
5815
     we still need to count it as an originator.  */
5816
  bitmap_set_bit (current_originators, INSN_UID (insn));
5817
 
5818
  if (!bitmap_clear_bit (current_copies, INSN_UID (insn)))
5819
    {
5820
      /* Note that original block needs to be rescheduled, as we pulled an
5821
         instruction out of it.  */
5822
      if (INSN_SCHED_TIMES (insn) > 0)
5823
        bitmap_set_bit (blocks_to_reschedule, BLOCK_FOR_INSN (insn)->index);
5824
      else if (INSN_UID (insn) < first_emitted_uid && !DEBUG_INSN_P (insn))
5825
        num_insns_scheduled++;
5826
    }
5827
 
5828
  /* For instructions we must immediately remove insn from the
5829
     stream, so subsequent update_data_sets () won't include this
5830
     insn into av_set.
5831
     For expr we must make insn look like "INSN_REG (insn) := c_expr".  */
5832
  if (INSN_UID (insn) > max_uid_before_move_op)
5833
    stat_bookkeeping_copies--;
5834
}
5835
 
5836
/* Emit a register-register copy for INSN if needed.  Return true if
5837
   emitted one.  PARAMS is the move_op static parameters.  */
5838
static bool
5839
maybe_emit_renaming_copy (rtx insn,
5840
                          moveop_static_params_p params)
5841
{
5842
  bool insn_emitted  = false;
5843
  rtx cur_reg;
5844
 
5845
  /* Bail out early when expression can not be renamed at all.  */
5846
  if (!EXPR_SEPARABLE_P (params->c_expr))
5847
    return false;
5848
 
5849
  cur_reg = expr_dest_reg (params->c_expr);
5850
  gcc_assert (cur_reg && params->dest && REG_P (params->dest));
5851
 
5852
  /* If original operation has expr and the register chosen for
5853
     that expr is not original operation's dest reg, substitute
5854
     operation's right hand side with the register chosen.  */
5855
  if (REGNO (params->dest) != REGNO (cur_reg))
5856
    {
5857
      insn_t reg_move_insn, reg_move_insn_rtx;
5858
 
5859
      reg_move_insn_rtx = create_insn_rtx_with_rhs (INSN_VINSN (insn),
5860
                                                    params->dest);
5861
      reg_move_insn = sel_gen_insn_from_rtx_after (reg_move_insn_rtx,
5862
                                                   INSN_EXPR (insn),
5863
                                                   INSN_SEQNO (insn),
5864
                                                   insn);
5865
      EXPR_SPEC_DONE_DS (INSN_EXPR (reg_move_insn)) = 0;
5866
      replace_dest_with_reg_in_expr (params->c_expr, params->dest);
5867
 
5868
      insn_emitted = true;
5869
      params->was_renamed = true;
5870
    }
5871
 
5872
  return insn_emitted;
5873
}
5874
 
5875
/* Emit a speculative check for INSN speculated as EXPR if needed.
5876
   Return true if we've  emitted one.  PARAMS is the move_op static
5877
   parameters.  */
5878
static bool
5879
maybe_emit_speculative_check (rtx insn, expr_t expr,
5880
                              moveop_static_params_p params)
5881
{
5882
  bool insn_emitted = false;
5883
  insn_t x;
5884
  ds_t check_ds;
5885
 
5886
  check_ds = get_spec_check_type_for_insn (insn, expr);
5887
  if (check_ds != 0)
5888
    {
5889
      /* A speculation check should be inserted.  */
5890
      x = create_speculation_check (params->c_expr, check_ds, insn);
5891
      insn_emitted = true;
5892
    }
5893
  else
5894
    {
5895
      EXPR_SPEC_DONE_DS (INSN_EXPR (insn)) = 0;
5896
      x = insn;
5897
    }
5898
 
5899
  gcc_assert (EXPR_SPEC_DONE_DS (INSN_EXPR (x)) == 0
5900
              && EXPR_SPEC_TO_CHECK_DS (INSN_EXPR (x)) == 0);
5901
  return insn_emitted;
5902
}
5903
 
5904
/* Handle transformations that leave an insn in place of original
5905
   insn such as renaming/speculation.  Return true if one of such
5906
   transformations actually happened, and we have emitted this insn.  */
5907
static bool
5908
handle_emitting_transformations (rtx insn, expr_t expr,
5909
                                 moveop_static_params_p params)
5910
{
5911
  bool insn_emitted = false;
5912
 
5913
  insn_emitted = maybe_emit_renaming_copy (insn, params);
5914
  insn_emitted |= maybe_emit_speculative_check (insn, expr, params);
5915
 
5916
  return insn_emitted;
5917
}
5918
 
5919
/* If INSN is the only insn in the basic block (not counting JUMP,
5920
   which may be a jump to next insn, and DEBUG_INSNs), we want to
5921
   leave a NOP there till the return to fill_insns.  */
5922
 
5923
static bool
5924
need_nop_to_preserve_insn_bb (rtx insn)
5925
{
5926
  insn_t bb_head, bb_end, bb_next, in_next;
5927
  basic_block bb = BLOCK_FOR_INSN (insn);
5928
 
5929
  bb_head = sel_bb_head (bb);
5930
  bb_end = sel_bb_end (bb);
5931
 
5932
  if (bb_head == bb_end)
5933
    return true;
5934
 
5935
  while (bb_head != bb_end && DEBUG_INSN_P (bb_head))
5936
    bb_head = NEXT_INSN (bb_head);
5937
 
5938
  if (bb_head == bb_end)
5939
    return true;
5940
 
5941
  while (bb_head != bb_end && DEBUG_INSN_P (bb_end))
5942
    bb_end = PREV_INSN (bb_end);
5943
 
5944
  if (bb_head == bb_end)
5945
    return true;
5946
 
5947
  bb_next = NEXT_INSN (bb_head);
5948
  while (bb_next != bb_end && DEBUG_INSN_P (bb_next))
5949
    bb_next = NEXT_INSN (bb_next);
5950
 
5951
  if (bb_next == bb_end && JUMP_P (bb_end))
5952
    return true;
5953
 
5954
  in_next = NEXT_INSN (insn);
5955
  while (DEBUG_INSN_P (in_next))
5956
    in_next = NEXT_INSN (in_next);
5957
 
5958
  if (IN_CURRENT_FENCE_P (in_next))
5959
    return true;
5960
 
5961
  return false;
5962
}
5963
 
5964
/* Remove INSN from stream.  When ONLY_DISCONNECT is true, its data
5965
   is not removed but reused when INSN is re-emitted.  */
5966
static void
5967
remove_insn_from_stream (rtx insn, bool only_disconnect)
5968
{
5969
  /* If there's only one insn in the BB, make sure that a nop is
5970
     inserted into it, so the basic block won't disappear when we'll
5971
     delete INSN below with sel_remove_insn. It should also survive
5972
     till the return to fill_insns.  */
5973
  if (need_nop_to_preserve_insn_bb (insn))
5974
    {
5975
      insn_t nop = get_nop_from_pool (insn);
5976
      gcc_assert (INSN_NOP_P (nop));
5977
      VEC_safe_push (insn_t, heap, vec_temp_moveop_nops, nop);
5978
    }
5979
 
5980
  sel_remove_insn (insn, only_disconnect, false);
5981
}
5982
 
5983
/* This function is called when original expr is found.
5984
   INSN - current insn traversed, EXPR - the corresponding expr found.
5985
   LPARAMS is the local parameters of code modion driver, STATIC_PARAMS
5986
   is static parameters of move_op.  */
5987
static void
5988
move_op_orig_expr_found (insn_t insn, expr_t expr,
5989
                         cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
5990
                         void *static_params)
5991
{
5992
  bool only_disconnect, insn_emitted;
5993
  moveop_static_params_p params = (moveop_static_params_p) static_params;
5994
 
5995
  copy_expr_onside (params->c_expr, INSN_EXPR (insn));
5996
  track_scheduled_insns_and_blocks (insn);
5997
  insn_emitted = handle_emitting_transformations (insn, expr, params);
5998
  only_disconnect = (params->uid == INSN_UID (insn)
5999
                     && ! insn_emitted  && ! EXPR_WAS_CHANGED (expr));
6000
 
6001
  /* Mark that we've disconnected an insn.  */
6002
  if (only_disconnect)
6003
    params->uid = -1;
6004
  remove_insn_from_stream (insn, only_disconnect);
6005
}
6006
 
6007
/* The function is called when original expr is found.
6008
   INSN - current insn traversed, EXPR - the corresponding expr found,
6009
   crosses_call and original_insns in STATIC_PARAMS are updated.  */
6010
static void
6011
fur_orig_expr_found (insn_t insn, expr_t expr ATTRIBUTE_UNUSED,
6012
                     cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
6013
                     void *static_params)
6014
{
6015
  fur_static_params_p params = (fur_static_params_p) static_params;
6016
  regset tmp;
6017
 
6018
  if (CALL_P (insn))
6019
    params->crosses_call = true;
6020
 
6021
  def_list_add (params->original_insns, insn, params->crosses_call);
6022
 
6023
  /* Mark the registers that do not meet the following condition:
6024
    (2) not among the live registers of the point
6025
        immediately following the first original operation on
6026
        a given downward path, except for the original target
6027
        register of the operation.  */
6028
  tmp = get_clear_regset_from_pool ();
6029
  compute_live_below_insn (insn, tmp);
6030
  AND_COMPL_REG_SET (tmp, INSN_REG_SETS (insn));
6031
  AND_COMPL_REG_SET (tmp, INSN_REG_CLOBBERS (insn));
6032
  IOR_REG_SET (params->used_regs, tmp);
6033
  return_regset_to_pool (tmp);
6034
 
6035
  /* (*1) We need to add to USED_REGS registers that are read by
6036
     INSN's lhs. This may lead to choosing wrong src register.
6037
     E.g. (scheduling const expr enabled):
6038
 
6039
        429: ax=0x0     <- Can't use AX for this expr (0x0)
6040
        433: dx=[bp-0x18]
6041
        427: [ax+dx+0x1]=ax
6042
          REG_DEAD: ax
6043
        168: di=dx
6044
          REG_DEAD: dx
6045
     */
6046
  /* FIXME: see comment above and enable MEM_P
6047
     in vinsn_separable_p.  */
6048
  gcc_assert (!VINSN_SEPARABLE_P (INSN_VINSN (insn))
6049
              || !MEM_P (INSN_LHS (insn)));
6050
}
6051
 
6052
/* This function is called on the ascending pass, before returning from
6053
   current basic block.  */
6054
static void
6055
move_op_at_first_insn (insn_t insn, cmpd_local_params_p lparams,
6056
                       void *static_params)
6057
{
6058
  moveop_static_params_p sparams = (moveop_static_params_p) static_params;
6059
  basic_block book_block = NULL;
6060
 
6061
  /* When we have removed the boundary insn for scheduling, which also
6062
     happened to be the end insn in its bb, we don't need to update sets.  */
6063
  if (!lparams->removed_last_insn
6064
      && lparams->e1
6065
      && sel_bb_head_p (insn))
6066
    {
6067
      /* We should generate bookkeeping code only if we are not at the
6068
         top level of the move_op.  */
6069
      if (sel_num_cfg_preds_gt_1 (insn))
6070
        book_block = generate_bookkeeping_insn (sparams->c_expr,
6071
                                                lparams->e1, lparams->e2);
6072
      /* Update data sets for the current insn.  */
6073
      update_data_sets (insn);
6074
    }
6075
 
6076
  /* If bookkeeping code was inserted, we need to update av sets of basic
6077
     block that received bookkeeping.  After generation of bookkeeping insn,
6078
     bookkeeping block does not contain valid av set because we are not following
6079
     the original algorithm in every detail with regards to e.g. renaming
6080
     simple reg-reg copies.  Consider example:
6081
 
6082
     bookkeeping block           scheduling fence
6083
     \            /
6084
      \    join  /
6085
       ----------
6086
       |        |
6087
       ----------
6088
      /           \
6089
     /             \
6090
     r1 := r2          r1 := r3
6091
 
6092
     We try to schedule insn "r1 := r3" on the current
6093
     scheduling fence.  Also, note that av set of bookkeeping block
6094
     contain both insns "r1 := r2" and "r1 := r3".  When the insn has
6095
     been scheduled, the CFG is as follows:
6096
 
6097
     r1 := r3               r1 := r3
6098
     bookkeeping block           scheduling fence
6099
     \            /
6100
      \    join  /
6101
       ----------
6102
       |        |
6103
       ----------
6104
      /          \
6105
     /            \
6106
     r1 := r2
6107
 
6108
     Here, insn "r1 := r3" was scheduled at the current scheduling point
6109
     and bookkeeping code was generated at the bookeeping block.  This
6110
     way insn "r1 := r2" is no longer available as a whole instruction
6111
     (but only as expr) ahead of insn "r1 := r3" in bookkeeping block.
6112
     This situation is handled by calling update_data_sets.
6113
 
6114
     Since update_data_sets is called only on the bookkeeping block, and
6115
     it also may have predecessors with av_sets, containing instructions that
6116
     are no longer available, we save all such expressions that become
6117
     unavailable during data sets update on the bookkeeping block in
6118
     VEC_BOOKKEEPING_BLOCKED_VINSNS.  Later we avoid selecting such
6119
     expressions for scheduling.  This allows us to avoid recomputation of
6120
     av_sets outside the code motion path.  */
6121
 
6122
  if (book_block)
6123
    update_and_record_unavailable_insns (book_block);
6124
 
6125
  /* If INSN was previously marked for deletion, it's time to do it.  */
6126
  if (lparams->removed_last_insn)
6127
    insn = PREV_INSN (insn);
6128
 
6129
  /* Do not tidy control flow at the topmost moveop, as we can erroneously
6130
     kill a block with a single nop in which the insn should be emitted.  */
6131
  if (lparams->e1)
6132
    tidy_control_flow (BLOCK_FOR_INSN (insn), true);
6133
}
6134
 
6135
/* This function is called on the ascending pass, before returning from the
6136
   current basic block.  */
6137
static void
6138
fur_at_first_insn (insn_t insn,
6139
                   cmpd_local_params_p lparams ATTRIBUTE_UNUSED,
6140
                   void *static_params ATTRIBUTE_UNUSED)
6141
{
6142
  gcc_assert (!sel_bb_head_p (insn) || AV_SET_VALID_P (insn)
6143
              || AV_LEVEL (insn) == -1);
6144
}
6145
 
6146
/* Called on the backward stage of recursion to call moveup_expr for insn
6147
   and sparams->c_expr.  */
6148
static void
6149
move_op_ascend (insn_t insn, void *static_params)
6150
{
6151
  enum MOVEUP_EXPR_CODE res;
6152
  moveop_static_params_p sparams = (moveop_static_params_p) static_params;
6153
 
6154
  if (! INSN_NOP_P (insn))
6155
    {
6156
      res = moveup_expr_cached (sparams->c_expr, insn, false);
6157
      gcc_assert (res != MOVEUP_EXPR_NULL);
6158
    }
6159
 
6160
  /* Update liveness for this insn as it was invalidated.  */
6161
  update_liveness_on_insn (insn);
6162
}
6163
 
6164
/* This function is called on enter to the basic block.
6165
   Returns TRUE if this block already have been visited and
6166
   code_motion_path_driver should return 1, FALSE otherwise.  */
6167
static int
6168
fur_on_enter (insn_t insn ATTRIBUTE_UNUSED, cmpd_local_params_p local_params,
6169
              void *static_params, bool visited_p)
6170
{
6171
  fur_static_params_p sparams = (fur_static_params_p) static_params;
6172
 
6173
  if (visited_p)
6174
    {
6175
      /* If we have found something below this block, there should be at
6176
         least one insn in ORIGINAL_INSNS.  */
6177
      gcc_assert (*sparams->original_insns);
6178
 
6179
      /* Adjust CROSSES_CALL, since we may have come to this block along
6180
         different path.  */
6181
      DEF_LIST_DEF (*sparams->original_insns)->crosses_call
6182
          |= sparams->crosses_call;
6183
    }
6184
  else
6185
    local_params->old_original_insns = *sparams->original_insns;
6186
 
6187
  return 1;
6188
}
6189
 
6190
/* Same as above but for move_op.   */
6191
static int
6192
move_op_on_enter (insn_t insn ATTRIBUTE_UNUSED,
6193
                  cmpd_local_params_p local_params ATTRIBUTE_UNUSED,
6194
                  void *static_params ATTRIBUTE_UNUSED, bool visited_p)
6195
{
6196
  if (visited_p)
6197
    return -1;
6198
  return 1;
6199
}
6200
 
6201
/* This function is called while descending current basic block if current
6202
   insn is not the original EXPR we're searching for.
6203
 
6204
   Return value: FALSE, if code_motion_path_driver should perform a local
6205
                        cleanup and return 0 itself;
6206
                 TRUE, if code_motion_path_driver should continue.  */
6207
static bool
6208
move_op_orig_expr_not_found (insn_t insn, av_set_t orig_ops ATTRIBUTE_UNUSED,
6209
                            void *static_params)
6210
{
6211
  moveop_static_params_p sparams = (moveop_static_params_p) static_params;
6212
 
6213
#ifdef ENABLE_CHECKING
6214
  sparams->failed_insn = insn;
6215
#endif
6216
 
6217
  /* If we're scheduling separate expr, in order to generate correct code
6218
     we need to stop the search at bookkeeping code generated with the
6219
     same destination register or memory.  */
6220
  if (lhs_of_insn_equals_to_dest_p (insn, sparams->dest))
6221
    return false;
6222
  return true;
6223
}
6224
 
6225
/* This function is called while descending current basic block if current
6226
   insn is not the original EXPR we're searching for.
6227
 
6228
   Return value: TRUE (code_motion_path_driver should continue).  */
6229
static bool
6230
fur_orig_expr_not_found (insn_t insn, av_set_t orig_ops, void *static_params)
6231
{
6232
  bool mutexed;
6233
  expr_t r;
6234
  av_set_iterator avi;
6235
  fur_static_params_p sparams = (fur_static_params_p) static_params;
6236
 
6237
  if (CALL_P (insn))
6238
    sparams->crosses_call = true;
6239
  else if (DEBUG_INSN_P (insn))
6240
    return true;
6241
 
6242
  /* If current insn we are looking at cannot be executed together
6243
     with original insn, then we can skip it safely.
6244
 
6245
     Example: ORIG_OPS = { (p6) r14 = sign_extend (r15); }
6246
              INSN = (!p6) r14 = r14 + 1;
6247
 
6248
     Here we can schedule ORIG_OP with lhs = r14, though only
6249
     looking at the set of used and set registers of INSN we must
6250
     forbid it.  So, add set/used in INSN registers to the
6251
     untouchable set only if there is an insn in ORIG_OPS that can
6252
     affect INSN.  */
6253
  mutexed = true;
6254
  FOR_EACH_EXPR (r, avi, orig_ops)
6255
    if (!sched_insns_conditions_mutex_p (insn, EXPR_INSN_RTX (r)))
6256
      {
6257
        mutexed = false;
6258
        break;
6259
      }
6260
 
6261
  /* Mark all registers that do not meet the following condition:
6262
     (1) Not set or read on any path from xi to an instance of the
6263
         original operation.  */
6264
  if (!mutexed)
6265
    {
6266
      IOR_REG_SET (sparams->used_regs, INSN_REG_SETS (insn));
6267
      IOR_REG_SET (sparams->used_regs, INSN_REG_USES (insn));
6268
      IOR_REG_SET (sparams->used_regs, INSN_REG_CLOBBERS (insn));
6269
    }
6270
 
6271
  return true;
6272
}
6273
 
6274
/* Hooks and data to perform move_op operations with code_motion_path_driver.  */
6275
struct code_motion_path_driver_info_def move_op_hooks = {
6276
  move_op_on_enter,
6277
  move_op_orig_expr_found,
6278
  move_op_orig_expr_not_found,
6279
  move_op_merge_succs,
6280
  move_op_after_merge_succs,
6281
  move_op_ascend,
6282
  move_op_at_first_insn,
6283
  SUCCS_NORMAL,
6284
  "move_op"
6285
};
6286
 
6287
/* Hooks and data to perform find_used_regs operations
6288
   with code_motion_path_driver.  */
6289
struct code_motion_path_driver_info_def fur_hooks = {
6290
  fur_on_enter,
6291
  fur_orig_expr_found,
6292
  fur_orig_expr_not_found,
6293
  fur_merge_succs,
6294
  NULL, /* fur_after_merge_succs */
6295
  NULL, /* fur_ascend */
6296
  fur_at_first_insn,
6297
  SUCCS_ALL,
6298
  "find_used_regs"
6299
};
6300
 
6301
/* Traverse all successors of INSN.  For each successor that is SUCCS_NORMAL
6302
   code_motion_path_driver is called recursively.  Original operation
6303
   was found at least on one path that is starting with one of INSN's
6304
   successors (this fact is asserted).  ORIG_OPS is expressions we're looking
6305
   for, PATH is the path we've traversed, STATIC_PARAMS is the parameters
6306
   of either move_op or find_used_regs depending on the caller.
6307
 
6308
   Return 0 if we haven't found expression, 1 if we found it, -1 if we don't
6309
   know for sure at this point.  */
6310
static int
6311
code_motion_process_successors (insn_t insn, av_set_t orig_ops,
6312
                                ilist_t path, void *static_params)
6313
{
6314
  int res = 0;
6315
  succ_iterator succ_i;
6316
  rtx succ;
6317
  basic_block bb;
6318
  int old_index;
6319
  unsigned old_succs;
6320
 
6321
  struct cmpd_local_params lparams;
6322
  expr_def _x;
6323
 
6324
  lparams.c_expr_local = &_x;
6325
  lparams.c_expr_merged = NULL;
6326
 
6327
  /* We need to process only NORMAL succs for move_op, and collect live
6328
     registers from ALL branches (including those leading out of the
6329
     region) for find_used_regs.
6330
 
6331
     In move_op, there can be a case when insn's bb number has changed
6332
     due to created bookkeeping.  This happens very rare, as we need to
6333
     move expression from the beginning to the end of the same block.
6334
     Rescan successors in this case.  */
6335
 
6336
 rescan:
6337
  bb = BLOCK_FOR_INSN (insn);
6338
  old_index = bb->index;
6339
  old_succs = EDGE_COUNT (bb->succs);
6340
 
6341
  FOR_EACH_SUCC_1 (succ, succ_i, insn, code_motion_path_driver_info->succ_flags)
6342
    {
6343
      int b;
6344
 
6345
      lparams.e1 = succ_i.e1;
6346
      lparams.e2 = succ_i.e2;
6347
 
6348
      /* Go deep into recursion only for NORMAL edges (non-backedges within the
6349
         current region).  */
6350
      if (succ_i.current_flags == SUCCS_NORMAL)
6351
        b = code_motion_path_driver (succ, orig_ops, path, &lparams,
6352
                                     static_params);
6353
      else
6354
        b = 0;
6355
 
6356
      /* Merge c_expres found or unify live register sets from different
6357
         successors.  */
6358
      code_motion_path_driver_info->merge_succs (insn, succ, b, &lparams,
6359
                                                 static_params);
6360
      if (b == 1)
6361
        res = b;
6362
      else if (b == -1 && res != 1)
6363
        res = b;
6364
 
6365
      /* We have simplified the control flow below this point.  In this case,
6366
         the iterator becomes invalid.  We need to try again.  */
6367
      if (BLOCK_FOR_INSN (insn)->index != old_index
6368
          || EDGE_COUNT (bb->succs) != old_succs)
6369
        {
6370
          insn = sel_bb_end (BLOCK_FOR_INSN (insn));
6371
          goto rescan;
6372
        }
6373
    }
6374
 
6375
#ifdef ENABLE_CHECKING
6376
  /* Here, RES==1 if original expr was found at least for one of the
6377
     successors.  After the loop, RES may happen to have zero value
6378
     only if at some point the expr searched is present in av_set, but is
6379
     not found below.  In most cases, this situation is an error.
6380
     The exception is when the original operation is blocked by
6381
     bookkeeping generated for another fence or for another path in current
6382
     move_op.  */
6383
  gcc_assert (res == 1
6384
              || (res == 0
6385
                  && av_set_could_be_blocked_by_bookkeeping_p (orig_ops,
6386
                                                               static_params))
6387
              || res == -1);
6388
#endif
6389
 
6390
  /* Merge data, clean up, etc.  */
6391
  if (res != -1 && code_motion_path_driver_info->after_merge_succs)
6392
    code_motion_path_driver_info->after_merge_succs (&lparams, static_params);
6393
 
6394
  return res;
6395
}
6396
 
6397
 
6398
/* Perform a cleanup when the driver is about to terminate.  ORIG_OPS_P
6399
   is the pointer to the av set with expressions we were looking for,
6400
   PATH_P is the pointer to the traversed path.  */
6401
static inline void
6402
code_motion_path_driver_cleanup (av_set_t *orig_ops_p, ilist_t *path_p)
6403
{
6404
  ilist_remove (path_p);
6405
  av_set_clear (orig_ops_p);
6406
}
6407
 
6408
/* The driver function that implements move_op or find_used_regs
6409
   functionality dependent whether code_motion_path_driver_INFO is set to
6410
   &MOVE_OP_HOOKS or &FUR_HOOKS.  This function implements the common parts
6411
   of code (CFG traversal etc) that are shared among both functions.  INSN
6412
   is the insn we're starting the search from, ORIG_OPS are the expressions
6413
   we're searching for, PATH is traversed path, LOCAL_PARAMS_IN are local
6414
   parameters of the driver, and STATIC_PARAMS are static parameters of
6415
   the caller.
6416
 
6417
   Returns whether original instructions were found.  Note that top-level
6418
   code_motion_path_driver always returns true.  */
6419
static int
6420
code_motion_path_driver (insn_t insn, av_set_t orig_ops, ilist_t path,
6421
                         cmpd_local_params_p local_params_in,
6422
                         void *static_params)
6423
{
6424
  expr_t expr = NULL;
6425
  basic_block bb = BLOCK_FOR_INSN (insn);
6426
  insn_t first_insn, bb_tail, before_first;
6427
  bool removed_last_insn = false;
6428
 
6429
  if (sched_verbose >= 6)
6430
    {
6431
      sel_print ("%s (", code_motion_path_driver_info->routine_name);
6432
      dump_insn (insn);
6433
      sel_print (",");
6434
      dump_av_set (orig_ops);
6435
      sel_print (")\n");
6436
    }
6437
 
6438
  gcc_assert (orig_ops);
6439
 
6440
  /* If no original operations exist below this insn, return immediately.  */
6441
  if (is_ineligible_successor (insn, path))
6442
    {
6443
      if (sched_verbose >= 6)
6444
        sel_print ("Insn %d is ineligible successor\n", INSN_UID (insn));
6445
      return false;
6446
    }
6447
 
6448
  /* The block can have invalid av set, in which case it was created earlier
6449
     during move_op.  Return immediately.  */
6450
  if (sel_bb_head_p (insn))
6451
    {
6452
      if (! AV_SET_VALID_P (insn))
6453
        {
6454
          if (sched_verbose >= 6)
6455
            sel_print ("Returned from block %d as it had invalid av set\n",
6456
                       bb->index);
6457
          return false;
6458
        }
6459
 
6460
      if (bitmap_bit_p (code_motion_visited_blocks, bb->index))
6461
        {
6462
          /* We have already found an original operation on this branch, do not
6463
             go any further and just return TRUE here.  If we don't stop here,
6464
             function can have exponential behaviour even on the small code
6465
             with many different paths (e.g. with data speculation and
6466
             recovery blocks).  */
6467
          if (sched_verbose >= 6)
6468
            sel_print ("Block %d already visited in this traversal\n", bb->index);
6469
          if (code_motion_path_driver_info->on_enter)
6470
            return code_motion_path_driver_info->on_enter (insn,
6471
                                                           local_params_in,
6472
                                                           static_params,
6473
                                                           true);
6474
        }
6475
    }
6476
 
6477
  if (code_motion_path_driver_info->on_enter)
6478
    code_motion_path_driver_info->on_enter (insn, local_params_in,
6479
                                            static_params, false);
6480
  orig_ops = av_set_copy (orig_ops);
6481
 
6482
  /* Filter the orig_ops set.  */
6483
  if (AV_SET_VALID_P (insn))
6484
    av_set_code_motion_filter (&orig_ops, AV_SET (insn));
6485
 
6486
  /* If no more original ops, return immediately.  */
6487
  if (!orig_ops)
6488
    {
6489
      if (sched_verbose >= 6)
6490
        sel_print ("No intersection with av set of block %d\n", bb->index);
6491
      return false;
6492
    }
6493
 
6494
  /* For non-speculative insns we have to leave only one form of the
6495
     original operation, because if we don't, we may end up with
6496
     different C_EXPRes and, consequently, with bookkeepings for different
6497
     expression forms along the same code motion path.  That may lead to
6498
     generation of incorrect code.  So for each code motion we stick to
6499
     the single form of the instruction,  except for speculative insns
6500
     which we need to keep in different forms with all speculation
6501
     types.  */
6502
  av_set_leave_one_nonspec (&orig_ops);
6503
 
6504
  /* It is not possible that all ORIG_OPS are filtered out.  */
6505
  gcc_assert (orig_ops);
6506
 
6507
  /* It is enough to place only heads and tails of visited basic blocks into
6508
     the PATH.  */
6509
  ilist_add (&path, insn);
6510
  first_insn = insn;
6511
  bb_tail = sel_bb_end (bb);
6512
 
6513
  /* Descend the basic block in search of the original expr; this part
6514
     corresponds to the part of the original move_op procedure executed
6515
     before the recursive call.  */
6516
  for (;;)
6517
    {
6518
      /* Look at the insn and decide if it could be an ancestor of currently
6519
         scheduling operation.  If it is so, then the insn "dest = op" could
6520
         either be replaced with "dest = reg", because REG now holds the result
6521
         of OP, or just removed, if we've scheduled the insn as a whole.
6522
 
6523
         If this insn doesn't contain currently scheduling OP, then proceed
6524
         with searching and look at its successors.  Operations we're searching
6525
         for could have changed when moving up through this insn via
6526
         substituting.  In this case, perform unsubstitution on them first.
6527
 
6528
         When traversing the DAG below this insn is finished, insert
6529
         bookkeeping code, if the insn is a joint point, and remove
6530
         leftovers.  */
6531
 
6532
      expr = av_set_lookup (orig_ops, INSN_VINSN (insn));
6533
      if (expr)
6534
        {
6535
          insn_t last_insn = PREV_INSN (insn);
6536
 
6537
          /* We have found the original operation.   */
6538
          if (sched_verbose >= 6)
6539
            sel_print ("Found original operation at insn %d\n", INSN_UID (insn));
6540
 
6541
          code_motion_path_driver_info->orig_expr_found
6542
            (insn, expr, local_params_in, static_params);
6543
 
6544
          /* Step back, so on the way back we'll start traversing from the
6545
             previous insn (or we'll see that it's bb_note and skip that
6546
             loop).  */
6547
          if (insn == first_insn)
6548
            {
6549
              first_insn = NEXT_INSN (last_insn);
6550
              removed_last_insn = sel_bb_end_p (last_insn);
6551
            }
6552
          insn = last_insn;
6553
          break;
6554
        }
6555
      else
6556
        {
6557
          /* We haven't found the original expr, continue descending the basic
6558
             block.  */
6559
          if (code_motion_path_driver_info->orig_expr_not_found
6560
              (insn, orig_ops, static_params))
6561
            {
6562
              /* Av set ops could have been changed when moving through this
6563
                 insn.  To find them below it, we have to un-substitute them.  */
6564
              undo_transformations (&orig_ops, insn);
6565
            }
6566
          else
6567
            {
6568
              /* Clean up and return, if the hook tells us to do so.  It may
6569
                 happen if we've encountered the previously created
6570
                 bookkeeping.  */
6571
              code_motion_path_driver_cleanup (&orig_ops, &path);
6572
              return -1;
6573
            }
6574
 
6575
          gcc_assert (orig_ops);
6576
        }
6577
 
6578
      /* Stop at insn if we got to the end of BB.  */
6579
      if (insn == bb_tail)
6580
        break;
6581
 
6582
      insn = NEXT_INSN (insn);
6583
    }
6584
 
6585
  /* Here INSN either points to the insn before the original insn (may be
6586
     bb_note, if original insn was a bb_head) or to the bb_end.  */
6587
  if (!expr)
6588
    {
6589
      int res;
6590
      rtx last_insn = PREV_INSN (insn);
6591
      bool added_to_path;
6592
 
6593
      gcc_assert (insn == sel_bb_end (bb));
6594
 
6595
      /* Add bb tail to PATH (but it doesn't make any sense if it's a bb_head -
6596
         it's already in PATH then).  */
6597
      if (insn != first_insn)
6598
        {
6599
          ilist_add (&path, insn);
6600
          added_to_path = true;
6601
        }
6602
      else
6603
        added_to_path = false;
6604
 
6605
      /* Process_successors should be able to find at least one
6606
         successor for which code_motion_path_driver returns TRUE.  */
6607
      res = code_motion_process_successors (insn, orig_ops,
6608
                                            path, static_params);
6609
 
6610
      /* Jump in the end of basic block could have been removed or replaced
6611
         during code_motion_process_successors, so recompute insn as the
6612
         last insn in bb.  */
6613
      if (NEXT_INSN (last_insn) != insn)
6614
        {
6615
          insn = sel_bb_end (bb);
6616
          first_insn = sel_bb_head (bb);
6617
        }
6618
 
6619
      /* Remove bb tail from path.  */
6620
      if (added_to_path)
6621
        ilist_remove (&path);
6622
 
6623
      if (res != 1)
6624
        {
6625
          /* This is the case when one of the original expr is no longer available
6626
             due to bookkeeping created on this branch with the same register.
6627
             In the original algorithm, which doesn't have update_data_sets call
6628
             on a bookkeeping block, it would simply result in returning
6629
             FALSE when we've encountered a previously generated bookkeeping
6630
             insn in moveop_orig_expr_not_found.  */
6631
          code_motion_path_driver_cleanup (&orig_ops, &path);
6632
          return res;
6633
        }
6634
    }
6635
 
6636
  /* Don't need it any more.  */
6637
  av_set_clear (&orig_ops);
6638
 
6639
  /* Backward pass: now, when we have C_EXPR computed, we'll drag it to
6640
     the beginning of the basic block.  */
6641
  before_first = PREV_INSN (first_insn);
6642
  while (insn != before_first)
6643
    {
6644
      if (code_motion_path_driver_info->ascend)
6645
        code_motion_path_driver_info->ascend (insn, static_params);
6646
 
6647
      insn = PREV_INSN (insn);
6648
    }
6649
 
6650
  /* Now we're at the bb head.  */
6651
  insn = first_insn;
6652
  ilist_remove (&path);
6653
  local_params_in->removed_last_insn = removed_last_insn;
6654
  code_motion_path_driver_info->at_first_insn (insn, local_params_in, static_params);
6655
 
6656
  /* This should be the very last operation as at bb head we could change
6657
     the numbering by creating bookkeeping blocks.  */
6658
  if (removed_last_insn)
6659
    insn = PREV_INSN (insn);
6660
  bitmap_set_bit (code_motion_visited_blocks, BLOCK_FOR_INSN (insn)->index);
6661
  return true;
6662
}
6663
 
6664
/* Move up the operations from ORIG_OPS set traversing the dag starting
6665
   from INSN.  PATH represents the edges traversed so far.
6666
   DEST is the register chosen for scheduling the current expr.  Insert
6667
   bookkeeping code in the join points.  EXPR_VLIW is the chosen expression,
6668
   C_EXPR is how it looks like at the given cfg point.
6669
   Set *SHOULD_MOVE to indicate whether we have only disconnected
6670
   one of the insns found.
6671
 
6672
   Returns whether original instructions were found, which is asserted
6673
   to be true in the caller.  */
6674
static bool
6675
move_op (insn_t insn, av_set_t orig_ops, expr_t expr_vliw,
6676
         rtx dest, expr_t c_expr, bool *should_move)
6677
{
6678
  struct moveop_static_params sparams;
6679
  struct cmpd_local_params lparams;
6680
  int res;
6681
 
6682
  /* Init params for code_motion_path_driver.  */
6683
  sparams.dest = dest;
6684
  sparams.c_expr = c_expr;
6685
  sparams.uid = INSN_UID (EXPR_INSN_RTX (expr_vliw));
6686
#ifdef ENABLE_CHECKING
6687
  sparams.failed_insn = NULL;
6688
#endif
6689
  sparams.was_renamed = false;
6690
  lparams.e1 = NULL;
6691
 
6692
  /* We haven't visited any blocks yet.  */
6693
  bitmap_clear (code_motion_visited_blocks);
6694
 
6695
  /* Set appropriate hooks and data.  */
6696
  code_motion_path_driver_info = &move_op_hooks;
6697
  res = code_motion_path_driver (insn, orig_ops, NULL, &lparams, &sparams);
6698
 
6699
  gcc_assert (res != -1);
6700
 
6701
  if (sparams.was_renamed)
6702
    EXPR_WAS_RENAMED (expr_vliw) = true;
6703
 
6704
  *should_move = (sparams.uid == -1);
6705
 
6706
  return res;
6707
}
6708
 
6709
 
6710
/* Functions that work with regions.  */
6711
 
6712
/* Current number of seqno used in init_seqno and init_seqno_1.  */
6713
static int cur_seqno;
6714
 
6715
/* A helper for init_seqno.  Traverse the region starting from BB and
6716
   compute seqnos for visited insns, marking visited bbs in VISITED_BBS.
6717
   Clear visited blocks from BLOCKS_TO_RESCHEDULE.  */
6718
static void
6719
init_seqno_1 (basic_block bb, sbitmap visited_bbs, bitmap blocks_to_reschedule)
6720
{
6721
  int bbi = BLOCK_TO_BB (bb->index);
6722
  insn_t insn, note = bb_note (bb);
6723
  insn_t succ_insn;
6724
  succ_iterator si;
6725
 
6726
  SET_BIT (visited_bbs, bbi);
6727
  if (blocks_to_reschedule)
6728
    bitmap_clear_bit (blocks_to_reschedule, bb->index);
6729
 
6730
  FOR_EACH_SUCC_1 (succ_insn, si, BB_END (bb),
6731
                   SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
6732
    {
6733
      basic_block succ = BLOCK_FOR_INSN (succ_insn);
6734
      int succ_bbi = BLOCK_TO_BB (succ->index);
6735
 
6736
      gcc_assert (in_current_region_p (succ));
6737
 
6738
      if (!TEST_BIT (visited_bbs, succ_bbi))
6739
        {
6740
          gcc_assert (succ_bbi > bbi);
6741
 
6742
          init_seqno_1 (succ, visited_bbs, blocks_to_reschedule);
6743
        }
6744
      else if (blocks_to_reschedule)
6745
        bitmap_set_bit (forced_ebb_heads, succ->index);
6746
    }
6747
 
6748
  for (insn = BB_END (bb); insn != note; insn = PREV_INSN (insn))
6749
    INSN_SEQNO (insn) = cur_seqno--;
6750
}
6751
 
6752
/* Initialize seqnos for the current region.  BLOCKS_TO_RESCHEDULE contains
6753
   blocks on which we're rescheduling when pipelining, FROM is the block where
6754
   traversing region begins (it may not be the head of the region when
6755
   pipelining, but the head of the loop instead).
6756
 
6757
   Returns the maximal seqno found.  */
6758
static int
6759
init_seqno (bitmap blocks_to_reschedule, basic_block from)
6760
{
6761
  sbitmap visited_bbs;
6762
  bitmap_iterator bi;
6763
  unsigned bbi;
6764
 
6765
  visited_bbs = sbitmap_alloc (current_nr_blocks);
6766
 
6767
  if (blocks_to_reschedule)
6768
    {
6769
      sbitmap_ones (visited_bbs);
6770
      EXECUTE_IF_SET_IN_BITMAP (blocks_to_reschedule, 0, bbi, bi)
6771
        {
6772
          gcc_assert (BLOCK_TO_BB (bbi) < current_nr_blocks);
6773
          RESET_BIT (visited_bbs, BLOCK_TO_BB (bbi));
6774
        }
6775
    }
6776
  else
6777
    {
6778
      sbitmap_zero (visited_bbs);
6779
      from = EBB_FIRST_BB (0);
6780
    }
6781
 
6782
  cur_seqno = sched_max_luid - 1;
6783
  init_seqno_1 (from, visited_bbs, blocks_to_reschedule);
6784
 
6785
  /* cur_seqno may be positive if the number of instructions is less than
6786
     sched_max_luid - 1 (when rescheduling or if some instructions have been
6787
     removed by the call to purge_empty_blocks in sel_sched_region_1).  */
6788
  gcc_assert (cur_seqno >= 0);
6789
 
6790
  sbitmap_free (visited_bbs);
6791
  return sched_max_luid - 1;
6792
}
6793
 
6794
/* Initialize scheduling parameters for current region.  */
6795
static void
6796
sel_setup_region_sched_flags (void)
6797
{
6798
  enable_schedule_as_rhs_p = 1;
6799
  bookkeeping_p = 1;
6800
  pipelining_p = (bookkeeping_p
6801
                  && (flag_sel_sched_pipelining != 0)
6802
                  && current_loop_nest != NULL
6803
                  && loop_has_exit_edges (current_loop_nest));
6804
  max_insns_to_rename = PARAM_VALUE (PARAM_SELSCHED_INSNS_TO_RENAME);
6805
  max_ws = MAX_WS;
6806
}
6807
 
6808
/* Return true if all basic blocks of current region are empty.  */
6809
static bool
6810
current_region_empty_p (void)
6811
{
6812
  int i;
6813
  for (i = 0; i < current_nr_blocks; i++)
6814
    if (! sel_bb_empty_p (BASIC_BLOCK (BB_TO_BLOCK (i))))
6815
      return false;
6816
 
6817
  return true;
6818
}
6819
 
6820
/* Prepare and verify loop nest for pipelining.  */
6821
static void
6822
setup_current_loop_nest (int rgn, bb_vec_t *bbs)
6823
{
6824
  current_loop_nest = get_loop_nest_for_rgn (rgn);
6825
 
6826
  if (!current_loop_nest)
6827
    return;
6828
 
6829
  /* If this loop has any saved loop preheaders from nested loops,
6830
     add these basic blocks to the current region.  */
6831
  sel_add_loop_preheaders (bbs);
6832
 
6833
  /* Check that we're starting with a valid information.  */
6834
  gcc_assert (loop_latch_edge (current_loop_nest));
6835
  gcc_assert (LOOP_MARKED_FOR_PIPELINING_P (current_loop_nest));
6836
}
6837
 
6838
/* Compute instruction priorities for current region.  */
6839
static void
6840
sel_compute_priorities (int rgn)
6841
{
6842
  sched_rgn_compute_dependencies (rgn);
6843
 
6844
  /* Compute insn priorities in haifa style.  Then free haifa style
6845
     dependencies that we've calculated for this.  */
6846
  compute_priorities ();
6847
 
6848
  if (sched_verbose >= 5)
6849
    debug_rgn_dependencies (0);
6850
 
6851
  free_rgn_deps ();
6852
}
6853
 
6854
/* Init scheduling data for RGN.  Returns true when this region should not
6855
   be scheduled.  */
6856
static bool
6857
sel_region_init (int rgn)
6858
{
6859
  int i;
6860
  bb_vec_t bbs;
6861
 
6862
  rgn_setup_region (rgn);
6863
 
6864
  /* Even if sched_is_disabled_for_current_region_p() is true, we still
6865
     do region initialization here so the region can be bundled correctly,
6866
     but we'll skip the scheduling in sel_sched_region ().  */
6867
  if (current_region_empty_p ())
6868
    return true;
6869
 
6870
  bbs = VEC_alloc (basic_block, heap, current_nr_blocks);
6871
 
6872
  for (i = 0; i < current_nr_blocks; i++)
6873
    VEC_quick_push (basic_block, bbs, BASIC_BLOCK (BB_TO_BLOCK (i)));
6874
 
6875
  sel_init_bbs (bbs);
6876
 
6877
  if (flag_sel_sched_pipelining)
6878
    setup_current_loop_nest (rgn, &bbs);
6879
 
6880
  sel_setup_region_sched_flags ();
6881
 
6882
  /* Initialize luids and dependence analysis which both sel-sched and haifa
6883
     need.  */
6884
  sched_init_luids (bbs);
6885
  sched_deps_init (false);
6886
 
6887
  /* Initialize haifa data.  */
6888
  rgn_setup_sched_infos ();
6889
  sel_set_sched_flags ();
6890
  haifa_init_h_i_d (bbs);
6891
 
6892
  sel_compute_priorities (rgn);
6893
  init_deps_global ();
6894
 
6895
  /* Main initialization.  */
6896
  sel_setup_sched_infos ();
6897
  sel_init_global_and_expr (bbs);
6898
 
6899
  VEC_free (basic_block, heap, bbs);
6900
 
6901
  blocks_to_reschedule = BITMAP_ALLOC (NULL);
6902
 
6903
  /* Init correct liveness sets on each instruction of a single-block loop.
6904
     This is the only situation when we can't update liveness when calling
6905
     compute_live for the first insn of the loop.  */
6906
  if (current_loop_nest)
6907
    {
6908
      int header = (sel_is_loop_preheader_p (BASIC_BLOCK (BB_TO_BLOCK (0)))
6909
                    ? 1
6910
                    : 0);
6911
 
6912
      if (current_nr_blocks == header + 1)
6913
        update_liveness_on_insn
6914
          (sel_bb_head (BASIC_BLOCK (BB_TO_BLOCK (header))));
6915
    }
6916
 
6917
  /* Set hooks so that no newly generated insn will go out unnoticed.  */
6918
  sel_register_cfg_hooks ();
6919
 
6920
  /* !!! We call target.sched.init () for the whole region, but we invoke
6921
     targetm.sched.finish () for every ebb.  */
6922
  if (targetm.sched.init)
6923
    /* None of the arguments are actually used in any target.  */
6924
    targetm.sched.init (sched_dump, sched_verbose, -1);
6925
 
6926
  first_emitted_uid = get_max_uid () + 1;
6927
  preheader_removed = false;
6928
 
6929
  /* Reset register allocation ticks array.  */
6930
  memset (reg_rename_tick, 0, sizeof reg_rename_tick);
6931
  reg_rename_this_tick = 0;
6932
 
6933
  bitmap_initialize (forced_ebb_heads, 0);
6934
  bitmap_clear (forced_ebb_heads);
6935
 
6936
  setup_nop_vinsn ();
6937
  current_copies = BITMAP_ALLOC (NULL);
6938
  current_originators = BITMAP_ALLOC (NULL);
6939
  code_motion_visited_blocks = BITMAP_ALLOC (NULL);
6940
 
6941
  return false;
6942
}
6943
 
6944
/* Simplify insns after the scheduling.  */
6945
static void
6946
simplify_changed_insns (void)
6947
{
6948
  int i;
6949
 
6950
  for (i = 0; i < current_nr_blocks; i++)
6951
    {
6952
      basic_block bb = BASIC_BLOCK (BB_TO_BLOCK (i));
6953
      rtx insn;
6954
 
6955
      FOR_BB_INSNS (bb, insn)
6956
        if (INSN_P (insn))
6957
          {
6958
            expr_t expr = INSN_EXPR (insn);
6959
 
6960
            if (EXPR_WAS_SUBSTITUTED (expr))
6961
              validate_simplify_insn (insn);
6962
          }
6963
    }
6964
}
6965
 
6966
/* Find boundaries of the EBB starting from basic block BB, marking blocks of
6967
   this EBB in SCHEDULED_BLOCKS and appropriately filling in HEAD, TAIL,
6968
   PREV_HEAD, and NEXT_TAIL fields of CURRENT_SCHED_INFO structure.  */
6969
static void
6970
find_ebb_boundaries (basic_block bb, bitmap scheduled_blocks)
6971
{
6972
  insn_t head, tail;
6973
  basic_block bb1 = bb;
6974
  if (sched_verbose >= 2)
6975
    sel_print ("Finishing schedule in bbs: ");
6976
 
6977
  do
6978
    {
6979
      bitmap_set_bit (scheduled_blocks, BLOCK_TO_BB (bb1->index));
6980
 
6981
      if (sched_verbose >= 2)
6982
        sel_print ("%d; ", bb1->index);
6983
    }
6984
  while (!bb_ends_ebb_p (bb1) && (bb1 = bb_next_bb (bb1)));
6985
 
6986
  if (sched_verbose >= 2)
6987
    sel_print ("\n");
6988
 
6989
  get_ebb_head_tail (bb, bb1, &head, &tail);
6990
 
6991
  current_sched_info->head = head;
6992
  current_sched_info->tail = tail;
6993
  current_sched_info->prev_head = PREV_INSN (head);
6994
  current_sched_info->next_tail = NEXT_INSN (tail);
6995
}
6996
 
6997
/* Regenerate INSN_SCHED_CYCLEs for insns of current EBB.  */
6998
static void
6999
reset_sched_cycles_in_current_ebb (void)
7000
{
7001
  int last_clock = 0;
7002
  int haifa_last_clock = -1;
7003
  int haifa_clock = 0;
7004
  int issued_insns = 0;
7005
  insn_t insn;
7006
 
7007
  if (targetm.sched.init)
7008
    {
7009
      /* None of the arguments are actually used in any target.
7010
         NB: We should have md_reset () hook for cases like this.  */
7011
      targetm.sched.init (sched_dump, sched_verbose, -1);
7012
    }
7013
 
7014
  state_reset (curr_state);
7015
  advance_state (curr_state);
7016
 
7017
  for (insn = current_sched_info->head;
7018
       insn != current_sched_info->next_tail;
7019
       insn = NEXT_INSN (insn))
7020
    {
7021
      int cost, haifa_cost;
7022
      int sort_p;
7023
      bool asm_p, real_insn, after_stall, all_issued;
7024
      int clock;
7025
 
7026
      if (!INSN_P (insn))
7027
        continue;
7028
 
7029
      asm_p = false;
7030
      real_insn = recog_memoized (insn) >= 0;
7031
      clock = INSN_SCHED_CYCLE (insn);
7032
 
7033
      cost = clock - last_clock;
7034
 
7035
      /* Initialize HAIFA_COST.  */
7036
      if (! real_insn)
7037
        {
7038
          asm_p = INSN_ASM_P (insn);
7039
 
7040
          if (asm_p)
7041
            /* This is asm insn which *had* to be scheduled first
7042
               on the cycle.  */
7043
            haifa_cost = 1;
7044
          else
7045
            /* This is a use/clobber insn.  It should not change
7046
               cost.  */
7047
            haifa_cost = 0;
7048
        }
7049
      else
7050
        haifa_cost = estimate_insn_cost (insn, curr_state);
7051
 
7052
      /* Stall for whatever cycles we've stalled before.  */
7053
      after_stall = 0;
7054
      if (INSN_AFTER_STALL_P (insn) && cost > haifa_cost)
7055
        {
7056
          haifa_cost = cost;
7057
          after_stall = 1;
7058
        }
7059
      all_issued = issued_insns == issue_rate;
7060
      if (haifa_cost == 0 && all_issued)
7061
        haifa_cost = 1;
7062
      if (haifa_cost > 0)
7063
        {
7064
          int i = 0;
7065
 
7066
          while (haifa_cost--)
7067
            {
7068
              advance_state (curr_state);
7069
              issued_insns = 0;
7070
              i++;
7071
 
7072
              if (sched_verbose >= 2)
7073
                {
7074
                  sel_print ("advance_state (state_transition)\n");
7075
                  debug_state (curr_state);
7076
                }
7077
 
7078
              /* The DFA may report that e.g. insn requires 2 cycles to be
7079
                 issued, but on the next cycle it says that insn is ready
7080
                 to go.  Check this here.  */
7081
              if (!after_stall
7082
                  && real_insn
7083
                  && haifa_cost > 0
7084
                  && estimate_insn_cost (insn, curr_state) == 0)
7085
                break;
7086
 
7087
              /* When the data dependency stall is longer than the DFA stall,
7088
                 and when we have issued exactly issue_rate insns and stalled,
7089
                 it could be that after this longer stall the insn will again
7090
                 become unavailable  to the DFA restrictions.  Looks strange
7091
                 but happens e.g. on x86-64.  So recheck DFA on the last
7092
                 iteration.  */
7093
              if ((after_stall || all_issued)
7094
                  && real_insn
7095
                  && haifa_cost == 0)
7096
                haifa_cost = estimate_insn_cost (insn, curr_state);
7097
            }
7098
 
7099
          haifa_clock += i;
7100
          if (sched_verbose >= 2)
7101
            sel_print ("haifa clock: %d\n", haifa_clock);
7102
        }
7103
      else
7104
        gcc_assert (haifa_cost == 0);
7105
 
7106
      if (sched_verbose >= 2)
7107
        sel_print ("Haifa cost for insn %d: %d\n", INSN_UID (insn), haifa_cost);
7108
 
7109
      if (targetm.sched.dfa_new_cycle)
7110
        while (targetm.sched.dfa_new_cycle (sched_dump, sched_verbose, insn,
7111
                                            haifa_last_clock, haifa_clock,
7112
                                            &sort_p))
7113
          {
7114
            advance_state (curr_state);
7115
            issued_insns = 0;
7116
            haifa_clock++;
7117
            if (sched_verbose >= 2)
7118
              {
7119
                sel_print ("advance_state (dfa_new_cycle)\n");
7120
                debug_state (curr_state);
7121
                sel_print ("haifa clock: %d\n", haifa_clock + 1);
7122
              }
7123
          }
7124
 
7125
      if (real_insn)
7126
        {
7127
          cost = state_transition (curr_state, insn);
7128
          issued_insns++;
7129
 
7130
          if (sched_verbose >= 2)
7131
            {
7132
              sel_print ("scheduled insn %d, clock %d\n", INSN_UID (insn),
7133
                         haifa_clock + 1);
7134
              debug_state (curr_state);
7135
            }
7136
          gcc_assert (cost < 0);
7137
        }
7138
 
7139
      if (targetm.sched.variable_issue)
7140
        targetm.sched.variable_issue (sched_dump, sched_verbose, insn, 0);
7141
 
7142
      INSN_SCHED_CYCLE (insn) = haifa_clock;
7143
 
7144
      last_clock = clock;
7145
      haifa_last_clock = haifa_clock;
7146
    }
7147
}
7148
 
7149
/* Put TImode markers on insns starting a new issue group.  */
7150
static void
7151
put_TImodes (void)
7152
{
7153
  int last_clock = -1;
7154
  insn_t insn;
7155
 
7156
  for (insn = current_sched_info->head; insn != current_sched_info->next_tail;
7157
       insn = NEXT_INSN (insn))
7158
    {
7159
      int cost, clock;
7160
 
7161
      if (!INSN_P (insn))
7162
        continue;
7163
 
7164
      clock = INSN_SCHED_CYCLE (insn);
7165
      cost = (last_clock == -1) ? 1 : clock - last_clock;
7166
 
7167
      gcc_assert (cost >= 0);
7168
 
7169
      if (issue_rate > 1
7170
          && GET_CODE (PATTERN (insn)) != USE
7171
          && GET_CODE (PATTERN (insn)) != CLOBBER)
7172
        {
7173
          if (reload_completed && cost > 0)
7174
            PUT_MODE (insn, TImode);
7175
 
7176
          last_clock = clock;
7177
        }
7178
 
7179
      if (sched_verbose >= 2)
7180
        sel_print ("Cost for insn %d is %d\n", INSN_UID (insn), cost);
7181
    }
7182
}
7183
 
7184
/* Perform MD_FINISH on EBBs comprising current region.  When
7185
   RESET_SCHED_CYCLES_P is true, run a pass emulating the scheduler
7186
   to produce correct sched cycles on insns.  */
7187
static void
7188
sel_region_target_finish (bool reset_sched_cycles_p)
7189
{
7190
  int i;
7191
  bitmap scheduled_blocks = BITMAP_ALLOC (NULL);
7192
 
7193
  for (i = 0; i < current_nr_blocks; i++)
7194
    {
7195
      if (bitmap_bit_p (scheduled_blocks, i))
7196
        continue;
7197
 
7198
      /* While pipelining outer loops, skip bundling for loop
7199
         preheaders.  Those will be rescheduled in the outer loop.  */
7200
      if (sel_is_loop_preheader_p (EBB_FIRST_BB (i)))
7201
        continue;
7202
 
7203
      find_ebb_boundaries (EBB_FIRST_BB (i), scheduled_blocks);
7204
 
7205
      if (no_real_insns_p (current_sched_info->head, current_sched_info->tail))
7206
        continue;
7207
 
7208
      if (reset_sched_cycles_p)
7209
        reset_sched_cycles_in_current_ebb ();
7210
 
7211
      if (targetm.sched.init)
7212
        targetm.sched.init (sched_dump, sched_verbose, -1);
7213
 
7214
      put_TImodes ();
7215
 
7216
      if (targetm.sched.finish)
7217
        {
7218
          targetm.sched.finish (sched_dump, sched_verbose);
7219
 
7220
          /* Extend luids so that insns generated by the target will
7221
             get zero luid.  */
7222
          sched_extend_luids ();
7223
        }
7224
    }
7225
 
7226
  BITMAP_FREE (scheduled_blocks);
7227
}
7228
 
7229
/* Free the scheduling data for the current region.  When RESET_SCHED_CYCLES_P
7230
   is true, make an additional pass emulating scheduler to get correct insn
7231
   cycles for md_finish calls.  */
7232
static void
7233
sel_region_finish (bool reset_sched_cycles_p)
7234
{
7235
  simplify_changed_insns ();
7236
  sched_finish_ready_list ();
7237
  free_nop_pool ();
7238
 
7239
  /* Free the vectors.  */
7240
  if (vec_av_set)
7241
    VEC_free (expr_t, heap, vec_av_set);
7242
  BITMAP_FREE (current_copies);
7243
  BITMAP_FREE (current_originators);
7244
  BITMAP_FREE (code_motion_visited_blocks);
7245
  vinsn_vec_free (&vec_bookkeeping_blocked_vinsns);
7246
  vinsn_vec_free (&vec_target_unavailable_vinsns);
7247
 
7248
  /* If LV_SET of the region head should be updated, do it now because
7249
     there will be no other chance.  */
7250
  {
7251
    succ_iterator si;
7252
    insn_t insn;
7253
 
7254
    FOR_EACH_SUCC_1 (insn, si, bb_note (EBB_FIRST_BB (0)),
7255
                     SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS)
7256
      {
7257
        basic_block bb = BLOCK_FOR_INSN (insn);
7258
 
7259
        if (!BB_LV_SET_VALID_P (bb))
7260
          compute_live (insn);
7261
      }
7262
  }
7263
 
7264
  /* Emulate the Haifa scheduler for bundling.  */
7265
  if (reload_completed)
7266
    sel_region_target_finish (reset_sched_cycles_p);
7267
 
7268
  sel_finish_global_and_expr ();
7269
 
7270
  bitmap_clear (forced_ebb_heads);
7271
 
7272
  free_nop_vinsn ();
7273
 
7274
  finish_deps_global ();
7275
  sched_finish_luids ();
7276
  VEC_free (haifa_deps_insn_data_def, heap, h_d_i_d);
7277
 
7278
  sel_finish_bbs ();
7279
  BITMAP_FREE (blocks_to_reschedule);
7280
 
7281
  sel_unregister_cfg_hooks ();
7282
 
7283
  max_issue_size = 0;
7284
}
7285
 
7286
 
7287
/* Functions that implement the scheduler driver.  */
7288
 
7289
/* Schedule a parallel instruction group on each of FENCES.  MAX_SEQNO
7290
   is the current maximum seqno.  SCHEDULED_INSNS_TAILPP is the list
7291
   of insns scheduled -- these would be postprocessed later.  */
7292
static void
7293
schedule_on_fences (flist_t fences, int max_seqno,
7294
                    ilist_t **scheduled_insns_tailpp)
7295
{
7296
  flist_t old_fences = fences;
7297
 
7298
  if (sched_verbose >= 1)
7299
    {
7300
      sel_print ("\nScheduling on fences: ");
7301
      dump_flist (fences);
7302
      sel_print ("\n");
7303
    }
7304
 
7305
  scheduled_something_on_previous_fence = false;
7306
  for (; fences; fences = FLIST_NEXT (fences))
7307
    {
7308
      fence_t fence = NULL;
7309
      int seqno = 0;
7310
      flist_t fences2;
7311
      bool first_p = true;
7312
 
7313
      /* Choose the next fence group to schedule.
7314
         The fact that insn can be scheduled only once
7315
         on the cycle is guaranteed by two properties:
7316
         1. seqnos of parallel groups decrease with each iteration.
7317
         2. If is_ineligible_successor () sees the larger seqno, it
7318
         checks if candidate insn is_in_current_fence_p ().  */
7319
      for (fences2 = old_fences; fences2; fences2 = FLIST_NEXT (fences2))
7320
        {
7321
          fence_t f = FLIST_FENCE (fences2);
7322
 
7323
          if (!FENCE_PROCESSED_P (f))
7324
            {
7325
              int i = INSN_SEQNO (FENCE_INSN (f));
7326
 
7327
              if (first_p || i > seqno)
7328
                {
7329
                  seqno = i;
7330
                  fence = f;
7331
                  first_p = false;
7332
                }
7333
              else
7334
                /* ??? Seqnos of different groups should be different.  */
7335
                gcc_assert (1 || i != seqno);
7336
            }
7337
        }
7338
 
7339
      gcc_assert (fence);
7340
 
7341
      /* As FENCE is nonnull, SEQNO is initialized.  */
7342
      seqno -= max_seqno + 1;
7343
      fill_insns (fence, seqno, scheduled_insns_tailpp);
7344
      FENCE_PROCESSED_P (fence) = true;
7345
    }
7346
 
7347
  /* All av_sets are invalidated by GLOBAL_LEVEL increase, thus we
7348
     don't need to keep bookkeeping-invalidated and target-unavailable
7349
     vinsns any more.  */
7350
  vinsn_vec_clear (&vec_bookkeeping_blocked_vinsns);
7351
  vinsn_vec_clear (&vec_target_unavailable_vinsns);
7352
}
7353
 
7354
/* Calculate MIN_SEQNO and MAX_SEQNO.  */
7355
static void
7356
find_min_max_seqno (flist_t fences, int *min_seqno, int *max_seqno)
7357
{
7358
  *min_seqno = *max_seqno = INSN_SEQNO (FENCE_INSN (FLIST_FENCE (fences)));
7359
 
7360
  /* The first element is already processed.  */
7361
  while ((fences = FLIST_NEXT (fences)))
7362
    {
7363
      int seqno = INSN_SEQNO (FENCE_INSN (FLIST_FENCE (fences)));
7364
 
7365
      if (*min_seqno > seqno)
7366
        *min_seqno = seqno;
7367
      else if (*max_seqno < seqno)
7368
        *max_seqno = seqno;
7369
    }
7370
}
7371
 
7372
/* Calculate new fences from FENCES.  */
7373
static flist_t
7374
calculate_new_fences (flist_t fences, int orig_max_seqno)
7375
{
7376
  flist_t old_fences = fences;
7377
  struct flist_tail_def _new_fences, *new_fences = &_new_fences;
7378
 
7379
  flist_tail_init (new_fences);
7380
  for (; fences; fences = FLIST_NEXT (fences))
7381
    {
7382
      fence_t fence = FLIST_FENCE (fences);
7383
      insn_t insn;
7384
 
7385
      if (!FENCE_BNDS (fence))
7386
        {
7387
          /* This fence doesn't have any successors.  */
7388
          if (!FENCE_SCHEDULED_P (fence))
7389
            {
7390
              /* Nothing was scheduled on this fence.  */
7391
              int seqno;
7392
 
7393
              insn = FENCE_INSN (fence);
7394
              seqno = INSN_SEQNO (insn);
7395
              gcc_assert (seqno > 0 && seqno <= orig_max_seqno);
7396
 
7397
              if (sched_verbose >= 1)
7398
                sel_print ("Fence %d[%d] has not changed\n",
7399
                           INSN_UID (insn),
7400
                           BLOCK_NUM (insn));
7401
              move_fence_to_fences (fences, new_fences);
7402
            }
7403
        }
7404
      else
7405
        extract_new_fences_from (fences, new_fences, orig_max_seqno);
7406
    }
7407
 
7408
  flist_clear (&old_fences);
7409
  return FLIST_TAIL_HEAD (new_fences);
7410
}
7411
 
7412
/* Update seqnos of insns given by PSCHEDULED_INSNS.  MIN_SEQNO and MAX_SEQNO
7413
   are the miminum and maximum seqnos of the group, HIGHEST_SEQNO_IN_USE is
7414
   the highest seqno used in a region.  Return the updated highest seqno.  */
7415
static int
7416
update_seqnos_and_stage (int min_seqno, int max_seqno,
7417
                         int highest_seqno_in_use,
7418
                         ilist_t *pscheduled_insns)
7419
{
7420
  int new_hs;
7421
  ilist_iterator ii;
7422
  insn_t insn;
7423
 
7424
  /* Actually, new_hs is the seqno of the instruction, that was
7425
     scheduled first (i.e. it is the first one in SCHEDULED_INSNS).  */
7426
  if (*pscheduled_insns)
7427
    {
7428
      new_hs = (INSN_SEQNO (ILIST_INSN (*pscheduled_insns))
7429
                + highest_seqno_in_use + max_seqno - min_seqno + 2);
7430
      gcc_assert (new_hs > highest_seqno_in_use);
7431
    }
7432
  else
7433
    new_hs = highest_seqno_in_use;
7434
 
7435
  FOR_EACH_INSN (insn, ii, *pscheduled_insns)
7436
    {
7437
      gcc_assert (INSN_SEQNO (insn) < 0);
7438
      INSN_SEQNO (insn) += highest_seqno_in_use + max_seqno - min_seqno + 2;
7439
      gcc_assert (INSN_SEQNO (insn) <= new_hs);
7440
 
7441
      /* When not pipelining, purge unneeded insn info on the scheduled insns.
7442
         For example, having reg_last array of INSN_DEPS_CONTEXT in memory may
7443
         require > 1GB of memory e.g. on limit-fnargs.c.  */
7444
      if (! pipelining_p)
7445
        free_data_for_scheduled_insn (insn);
7446
    }
7447
 
7448
  ilist_clear (pscheduled_insns);
7449
  global_level++;
7450
 
7451
  return new_hs;
7452
}
7453
 
7454
/* The main driver for scheduling a region.  This function is responsible
7455
   for correct propagation of fences (i.e. scheduling points) and creating
7456
   a group of parallel insns at each of them.  It also supports
7457
   pipelining.  ORIG_MAX_SEQNO is the maximal seqno before this pass
7458
   of scheduling.  */
7459
static void
7460
sel_sched_region_2 (int orig_max_seqno)
7461
{
7462
  int highest_seqno_in_use = orig_max_seqno;
7463
 
7464
  stat_bookkeeping_copies = 0;
7465
  stat_insns_needed_bookkeeping = 0;
7466
  stat_renamed_scheduled = 0;
7467
  stat_substitutions_total = 0;
7468
  num_insns_scheduled = 0;
7469
 
7470
  while (fences)
7471
    {
7472
      int min_seqno, max_seqno;
7473
      ilist_t scheduled_insns = NULL;
7474
      ilist_t *scheduled_insns_tailp = &scheduled_insns;
7475
 
7476
      find_min_max_seqno (fences, &min_seqno, &max_seqno);
7477
      schedule_on_fences (fences, max_seqno, &scheduled_insns_tailp);
7478
      fences = calculate_new_fences (fences, orig_max_seqno);
7479
      highest_seqno_in_use = update_seqnos_and_stage (min_seqno, max_seqno,
7480
                                                      highest_seqno_in_use,
7481
                                                      &scheduled_insns);
7482
    }
7483
 
7484
  if (sched_verbose >= 1)
7485
    sel_print ("Scheduled %d bookkeeping copies, %d insns needed "
7486
               "bookkeeping, %d insns renamed, %d insns substituted\n",
7487
               stat_bookkeeping_copies,
7488
               stat_insns_needed_bookkeeping,
7489
               stat_renamed_scheduled,
7490
               stat_substitutions_total);
7491
}
7492
 
7493
/* Schedule a region.  When pipelining, search for possibly never scheduled
7494
   bookkeeping code and schedule it.  Reschedule pipelined code without
7495
   pipelining after.  */
7496
static void
7497
sel_sched_region_1 (void)
7498
{
7499
  int orig_max_seqno;
7500
 
7501
  /* Remove empty blocks that might be in the region from the beginning.  */
7502
  purge_empty_blocks ();
7503
 
7504
  orig_max_seqno = init_seqno (NULL, NULL);
7505
  gcc_assert (orig_max_seqno >= 1);
7506
 
7507
  /* When pipelining outer loops, create fences on the loop header,
7508
     not preheader.  */
7509
  fences = NULL;
7510
  if (current_loop_nest)
7511
    init_fences (BB_END (EBB_FIRST_BB (0)));
7512
  else
7513
    init_fences (bb_note (EBB_FIRST_BB (0)));
7514
  global_level = 1;
7515
 
7516
  sel_sched_region_2 (orig_max_seqno);
7517
 
7518
  gcc_assert (fences == NULL);
7519
 
7520
  if (pipelining_p)
7521
    {
7522
      int i;
7523
      basic_block bb;
7524
      struct flist_tail_def _new_fences;
7525
      flist_tail_t new_fences = &_new_fences;
7526
      bool do_p = true;
7527
 
7528
      pipelining_p = false;
7529
      max_ws = MIN (max_ws, issue_rate * 3 / 2);
7530
      bookkeeping_p = false;
7531
      enable_schedule_as_rhs_p = false;
7532
 
7533
      /* Schedule newly created code, that has not been scheduled yet.  */
7534
      do_p = true;
7535
 
7536
      while (do_p)
7537
        {
7538
          do_p = false;
7539
 
7540
          for (i = 0; i < current_nr_blocks; i++)
7541
            {
7542
              basic_block bb = EBB_FIRST_BB (i);
7543
 
7544
              if (bitmap_bit_p (blocks_to_reschedule, bb->index))
7545
                {
7546
                  if (! bb_ends_ebb_p (bb))
7547
                    bitmap_set_bit (blocks_to_reschedule, bb_next_bb (bb)->index);
7548
                  if (sel_bb_empty_p (bb))
7549
                    {
7550
                      bitmap_clear_bit (blocks_to_reschedule, bb->index);
7551
                      continue;
7552
                    }
7553
                  clear_outdated_rtx_info (bb);
7554
                  if (sel_insn_is_speculation_check (BB_END (bb))
7555
                      && JUMP_P (BB_END (bb)))
7556
                    bitmap_set_bit (blocks_to_reschedule,
7557
                                    BRANCH_EDGE (bb)->dest->index);
7558
                }
7559
              else if (! sel_bb_empty_p (bb)
7560
                       && INSN_SCHED_TIMES (sel_bb_head (bb)) <= 0)
7561
                bitmap_set_bit (blocks_to_reschedule, bb->index);
7562
            }
7563
 
7564
          for (i = 0; i < current_nr_blocks; i++)
7565
            {
7566
              bb = EBB_FIRST_BB (i);
7567
 
7568
              /* While pipelining outer loops, skip bundling for loop
7569
                 preheaders.  Those will be rescheduled in the outer
7570
                 loop.  */
7571
              if (sel_is_loop_preheader_p (bb))
7572
                {
7573
                  clear_outdated_rtx_info (bb);
7574
                  continue;
7575
                }
7576
 
7577
              if (bitmap_bit_p (blocks_to_reschedule, bb->index))
7578
                {
7579
                  flist_tail_init (new_fences);
7580
 
7581
                  orig_max_seqno = init_seqno (blocks_to_reschedule, bb);
7582
 
7583
                  /* Mark BB as head of the new ebb.  */
7584
                  bitmap_set_bit (forced_ebb_heads, bb->index);
7585
 
7586
                  gcc_assert (fences == NULL);
7587
 
7588
                  init_fences (bb_note (bb));
7589
 
7590
                  sel_sched_region_2 (orig_max_seqno);
7591
 
7592
                  do_p = true;
7593
                  break;
7594
                }
7595
            }
7596
        }
7597
    }
7598
}
7599
 
7600
/* Schedule the RGN region.  */
7601
void
7602
sel_sched_region (int rgn)
7603
{
7604
  bool schedule_p;
7605
  bool reset_sched_cycles_p;
7606
 
7607
  if (sel_region_init (rgn))
7608
    return;
7609
 
7610
  if (sched_verbose >= 1)
7611
    sel_print ("Scheduling region %d\n", rgn);
7612
 
7613
  schedule_p = (!sched_is_disabled_for_current_region_p ()
7614
                && dbg_cnt (sel_sched_region_cnt));
7615
  reset_sched_cycles_p = pipelining_p;
7616
  if (schedule_p)
7617
    sel_sched_region_1 ();
7618
  else
7619
    /* Force initialization of INSN_SCHED_CYCLEs for correct bundling.  */
7620
    reset_sched_cycles_p = true;
7621
 
7622
  sel_region_finish (reset_sched_cycles_p);
7623
}
7624
 
7625
/* Perform global init for the scheduler.  */
7626
static void
7627
sel_global_init (void)
7628
{
7629
  calculate_dominance_info (CDI_DOMINATORS);
7630
  alloc_sched_pools ();
7631
 
7632
  /* Setup the infos for sched_init.  */
7633
  sel_setup_sched_infos ();
7634
  setup_sched_dump ();
7635
 
7636
  sched_rgn_init (false);
7637
  sched_init ();
7638
 
7639
  sched_init_bbs ();
7640
  /* Reset AFTER_RECOVERY if it has been set by the 1st scheduler pass.  */
7641
  after_recovery = 0;
7642
  can_issue_more = issue_rate;
7643
 
7644
  sched_extend_target ();
7645
  sched_deps_init (true);
7646
  setup_nop_and_exit_insns ();
7647
  sel_extend_global_bb_info ();
7648
  init_lv_sets ();
7649
  init_hard_regs_data ();
7650
}
7651
 
7652
/* Free the global data of the scheduler.  */
7653
static void
7654
sel_global_finish (void)
7655
{
7656
  free_bb_note_pool ();
7657
  free_lv_sets ();
7658
  sel_finish_global_bb_info ();
7659
 
7660
  free_regset_pool ();
7661
  free_nop_and_exit_insns ();
7662
 
7663
  sched_rgn_finish ();
7664
  sched_deps_finish ();
7665
  sched_finish ();
7666
 
7667
  if (current_loops)
7668
    sel_finish_pipelining ();
7669
 
7670
  free_sched_pools ();
7671
  free_dominance_info (CDI_DOMINATORS);
7672
}
7673
 
7674
/* Return true when we need to skip selective scheduling.  Used for debugging.  */
7675
bool
7676
maybe_skip_selective_scheduling (void)
7677
{
7678
  return ! dbg_cnt (sel_sched_cnt);
7679
}
7680
 
7681
/* The entry point.  */
7682
void
7683
run_selective_scheduling (void)
7684
{
7685
  int rgn;
7686
 
7687
  if (n_basic_blocks == NUM_FIXED_BLOCKS)
7688
    return;
7689
 
7690
  sel_global_init ();
7691
 
7692
  for (rgn = 0; rgn < nr_regions; rgn++)
7693
    sel_sched_region (rgn);
7694
 
7695
  sel_global_finish ();
7696
}
7697
 
7698
#endif

powered by: WebSVN 2.1.0

© copyright 1999-2024 OpenCores.org, equivalent to Oliscience, all rights reserved. OpenCores®, registered trademark.