/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc.
|
/* Copyright (C) 2005, 2008, 2009 Free Software Foundation, Inc.
|
Contributed by Richard Henderson <rth@redhat.com>.
|
Contributed by Richard Henderson <rth@redhat.com>.
|
|
|
This file is part of the GNU OpenMP Library (libgomp).
|
This file is part of the GNU OpenMP Library (libgomp).
|
|
|
Libgomp is free software; you can redistribute it and/or modify it
|
Libgomp is free software; you can redistribute it and/or modify it
|
under the terms of the GNU General Public License as published by
|
under the terms of the GNU General Public License as published by
|
the Free Software Foundation; either version 3, or (at your option)
|
the Free Software Foundation; either version 3, or (at your option)
|
any later version.
|
any later version.
|
|
|
Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
|
Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
more details.
|
more details.
|
|
|
Under Section 7 of GPL version 3, you are granted additional
|
Under Section 7 of GPL version 3, you are granted additional
|
permissions described in the GCC Runtime Library Exception, version
|
permissions described in the GCC Runtime Library Exception, version
|
3.1, as published by the Free Software Foundation.
|
3.1, as published by the Free Software Foundation.
|
|
|
You should have received a copy of the GNU General Public License and
|
You should have received a copy of the GNU General Public License and
|
a copy of the GCC Runtime Library Exception along with this program;
|
a copy of the GCC Runtime Library Exception along with this program;
|
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
<http://www.gnu.org/licenses/>. */
|
<http://www.gnu.org/licenses/>. */
|
|
|
/* This file handles the LOOP (FOR/DO) construct. */
|
/* This file handles the LOOP (FOR/DO) construct. */
|
|
|
#include <limits.h>
|
#include <limits.h>
|
#include <stdlib.h>
|
#include <stdlib.h>
|
#include "libgomp.h"
|
#include "libgomp.h"
|
|
|
typedef unsigned long long gomp_ull;
|
typedef unsigned long long gomp_ull;
|
|
|
/* Initialize the given work share construct from the given arguments. */
|
/* Initialize the given work share construct from the given arguments. */
|
|
|
static inline void
|
static inline void
|
gomp_loop_ull_init (struct gomp_work_share *ws, bool up, gomp_ull start,
|
gomp_loop_ull_init (struct gomp_work_share *ws, bool up, gomp_ull start,
|
gomp_ull end, gomp_ull incr, enum gomp_schedule_type sched,
|
gomp_ull end, gomp_ull incr, enum gomp_schedule_type sched,
|
gomp_ull chunk_size)
|
gomp_ull chunk_size)
|
{
|
{
|
ws->sched = sched;
|
ws->sched = sched;
|
ws->chunk_size_ull = chunk_size;
|
ws->chunk_size_ull = chunk_size;
|
/* Canonicalize loops that have zero iterations to ->next == ->end. */
|
/* Canonicalize loops that have zero iterations to ->next == ->end. */
|
ws->end_ull = ((up && start > end) || (!up && start < end))
|
ws->end_ull = ((up && start > end) || (!up && start < end))
|
? start : end;
|
? start : end;
|
ws->incr_ull = incr;
|
ws->incr_ull = incr;
|
ws->next_ull = start;
|
ws->next_ull = start;
|
ws->mode = 0;
|
ws->mode = 0;
|
if (sched == GFS_DYNAMIC)
|
if (sched == GFS_DYNAMIC)
|
{
|
{
|
ws->chunk_size_ull *= incr;
|
ws->chunk_size_ull *= incr;
|
|
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
{
|
{
|
/* For dynamic scheduling prepare things to make each iteration
|
/* For dynamic scheduling prepare things to make each iteration
|
faster. */
|
faster. */
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_team *team = thr->ts.team;
|
struct gomp_team *team = thr->ts.team;
|
long nthreads = team ? team->nthreads : 1;
|
long nthreads = team ? team->nthreads : 1;
|
|
|
if (__builtin_expect (up, 1))
|
if (__builtin_expect (up, 1))
|
{
|
{
|
/* Cheap overflow protection. */
|
/* Cheap overflow protection. */
|
if (__builtin_expect ((nthreads | ws->chunk_size_ull)
|
if (__builtin_expect ((nthreads | ws->chunk_size_ull)
|
< 1ULL << (sizeof (gomp_ull)
|
< 1ULL << (sizeof (gomp_ull)
|
* __CHAR_BIT__ / 2 - 1), 1))
|
* __CHAR_BIT__ / 2 - 1), 1))
|
ws->mode = ws->end_ull < (__LONG_LONG_MAX__ * 2ULL + 1
|
ws->mode = ws->end_ull < (__LONG_LONG_MAX__ * 2ULL + 1
|
- (nthreads + 1) * ws->chunk_size_ull);
|
- (nthreads + 1) * ws->chunk_size_ull);
|
}
|
}
|
/* Cheap overflow protection. */
|
/* Cheap overflow protection. */
|
else if (__builtin_expect ((nthreads | -ws->chunk_size_ull)
|
else if (__builtin_expect ((nthreads | -ws->chunk_size_ull)
|
< 1ULL << (sizeof (gomp_ull)
|
< 1ULL << (sizeof (gomp_ull)
|
* __CHAR_BIT__ / 2 - 1), 1))
|
* __CHAR_BIT__ / 2 - 1), 1))
|
ws->mode = ws->end_ull > ((nthreads + 1) * -ws->chunk_size_ull
|
ws->mode = ws->end_ull > ((nthreads + 1) * -ws->chunk_size_ull
|
- (__LONG_LONG_MAX__ * 2ULL + 1));
|
- (__LONG_LONG_MAX__ * 2ULL + 1));
|
}
|
}
|
#endif
|
#endif
|
}
|
}
|
if (!up)
|
if (!up)
|
ws->mode |= 2;
|
ws->mode |= 2;
|
}
|
}
|
|
|
/* The *_start routines are called when first encountering a loop construct
|
/* The *_start routines are called when first encountering a loop construct
|
that is not bound directly to a parallel construct. The first thread
|
that is not bound directly to a parallel construct. The first thread
|
that arrives will create the work-share construct; subsequent threads
|
that arrives will create the work-share construct; subsequent threads
|
will see the construct exists and allocate work from it.
|
will see the construct exists and allocate work from it.
|
|
|
START, END, INCR are the bounds of the loop; due to the restrictions of
|
START, END, INCR are the bounds of the loop; due to the restrictions of
|
OpenMP, these values must be the same in every thread. This is not
|
OpenMP, these values must be the same in every thread. This is not
|
verified (nor is it entirely verifiable, since START is not necessarily
|
verified (nor is it entirely verifiable, since START is not necessarily
|
retained intact in the work-share data structure). CHUNK_SIZE is the
|
retained intact in the work-share data structure). CHUNK_SIZE is the
|
scheduling parameter; again this must be identical in all threads.
|
scheduling parameter; again this must be identical in all threads.
|
|
|
Returns true if there's any work for this thread to perform. If so,
|
Returns true if there's any work for this thread to perform. If so,
|
*ISTART and *IEND are filled with the bounds of the iteration block
|
*ISTART and *IEND are filled with the bounds of the iteration block
|
allocated to this thread. Returns false if all work was assigned to
|
allocated to this thread. Returns false if all work was assigned to
|
other threads prior to this thread's arrival. */
|
other threads prior to this thread's arrival. */
|
|
|
static bool
|
static bool
|
gomp_loop_ull_static_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_loop_ull_static_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull *istart, gomp_ull *iend)
|
gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_thread *thr = gomp_thread ();
|
|
|
thr->ts.static_trip = 0;
|
thr->ts.static_trip = 0;
|
if (gomp_work_share_start (false))
|
if (gomp_work_share_start (false))
|
{
|
{
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
GFS_STATIC, chunk_size);
|
GFS_STATIC, chunk_size);
|
gomp_work_share_init_done ();
|
gomp_work_share_init_done ();
|
}
|
}
|
|
|
return !gomp_iter_ull_static_next (istart, iend);
|
return !gomp_iter_ull_static_next (istart, iend);
|
}
|
}
|
|
|
static bool
|
static bool
|
gomp_loop_ull_dynamic_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_loop_ull_dynamic_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull *istart, gomp_ull *iend)
|
gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_thread *thr = gomp_thread ();
|
bool ret;
|
bool ret;
|
|
|
if (gomp_work_share_start (false))
|
if (gomp_work_share_start (false))
|
{
|
{
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
GFS_DYNAMIC, chunk_size);
|
GFS_DYNAMIC, chunk_size);
|
gomp_work_share_init_done ();
|
gomp_work_share_init_done ();
|
}
|
}
|
|
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
ret = gomp_iter_ull_dynamic_next (istart, iend);
|
ret = gomp_iter_ull_dynamic_next (istart, iend);
|
#else
|
#else
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
|
ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
#endif
|
#endif
|
|
|
return ret;
|
return ret;
|
}
|
}
|
|
|
static bool
|
static bool
|
gomp_loop_ull_guided_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_loop_ull_guided_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull *istart, gomp_ull *iend)
|
gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_thread *thr = gomp_thread ();
|
bool ret;
|
bool ret;
|
|
|
if (gomp_work_share_start (false))
|
if (gomp_work_share_start (false))
|
{
|
{
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
GFS_GUIDED, chunk_size);
|
GFS_GUIDED, chunk_size);
|
gomp_work_share_init_done ();
|
gomp_work_share_init_done ();
|
}
|
}
|
|
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
ret = gomp_iter_ull_guided_next (istart, iend);
|
ret = gomp_iter_ull_guided_next (istart, iend);
|
#else
|
#else
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
ret = gomp_iter_ull_guided_next_locked (istart, iend);
|
ret = gomp_iter_ull_guided_next_locked (istart, iend);
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
#endif
|
#endif
|
|
|
return ret;
|
return ret;
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_runtime_start (bool up, gomp_ull start, gomp_ull end,
|
GOMP_loop_ull_runtime_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_ull incr, gomp_ull *istart, gomp_ull *iend)
|
gomp_ull incr, gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
struct gomp_task_icv *icv = gomp_icv (false);
|
struct gomp_task_icv *icv = gomp_icv (false);
|
switch (icv->run_sched_var)
|
switch (icv->run_sched_var)
|
{
|
{
|
case GFS_STATIC:
|
case GFS_STATIC:
|
return gomp_loop_ull_static_start (up, start, end, incr,
|
return gomp_loop_ull_static_start (up, start, end, incr,
|
icv->run_sched_modifier,
|
icv->run_sched_modifier,
|
istart, iend);
|
istart, iend);
|
case GFS_DYNAMIC:
|
case GFS_DYNAMIC:
|
return gomp_loop_ull_dynamic_start (up, start, end, incr,
|
return gomp_loop_ull_dynamic_start (up, start, end, incr,
|
icv->run_sched_modifier,
|
icv->run_sched_modifier,
|
istart, iend);
|
istart, iend);
|
case GFS_GUIDED:
|
case GFS_GUIDED:
|
return gomp_loop_ull_guided_start (up, start, end, incr,
|
return gomp_loop_ull_guided_start (up, start, end, incr,
|
icv->run_sched_modifier,
|
icv->run_sched_modifier,
|
istart, iend);
|
istart, iend);
|
case GFS_AUTO:
|
case GFS_AUTO:
|
/* For now map to schedule(static), later on we could play with feedback
|
/* For now map to schedule(static), later on we could play with feedback
|
driven choice. */
|
driven choice. */
|
return gomp_loop_ull_static_start (up, start, end, incr,
|
return gomp_loop_ull_static_start (up, start, end, incr,
|
0, istart, iend);
|
0, istart, iend);
|
default:
|
default:
|
abort ();
|
abort ();
|
}
|
}
|
}
|
}
|
|
|
/* The *_ordered_*_start routines are similar. The only difference is that
|
/* The *_ordered_*_start routines are similar. The only difference is that
|
this work-share construct is initialized to expect an ORDERED section. */
|
this work-share construct is initialized to expect an ORDERED section. */
|
|
|
static bool
|
static bool
|
gomp_loop_ull_ordered_static_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_loop_ull_ordered_static_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull *istart, gomp_ull *iend)
|
gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_thread *thr = gomp_thread ();
|
|
|
thr->ts.static_trip = 0;
|
thr->ts.static_trip = 0;
|
if (gomp_work_share_start (true))
|
if (gomp_work_share_start (true))
|
{
|
{
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
GFS_STATIC, chunk_size);
|
GFS_STATIC, chunk_size);
|
gomp_ordered_static_init ();
|
gomp_ordered_static_init ();
|
gomp_work_share_init_done ();
|
gomp_work_share_init_done ();
|
}
|
}
|
|
|
return !gomp_iter_ull_static_next (istart, iend);
|
return !gomp_iter_ull_static_next (istart, iend);
|
}
|
}
|
|
|
static bool
|
static bool
|
gomp_loop_ull_ordered_dynamic_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_loop_ull_ordered_dynamic_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull *istart, gomp_ull *iend)
|
gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_thread *thr = gomp_thread ();
|
bool ret;
|
bool ret;
|
|
|
if (gomp_work_share_start (true))
|
if (gomp_work_share_start (true))
|
{
|
{
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
GFS_DYNAMIC, chunk_size);
|
GFS_DYNAMIC, chunk_size);
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
gomp_work_share_init_done ();
|
gomp_work_share_init_done ();
|
}
|
}
|
else
|
else
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
|
ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
|
ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
|
if (ret)
|
if (ret)
|
gomp_ordered_first ();
|
gomp_ordered_first ();
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
|
return ret;
|
return ret;
|
}
|
}
|
|
|
static bool
|
static bool
|
gomp_loop_ull_ordered_guided_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_loop_ull_ordered_guided_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull *istart, gomp_ull *iend)
|
gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_thread *thr = gomp_thread ();
|
bool ret;
|
bool ret;
|
|
|
if (gomp_work_share_start (true))
|
if (gomp_work_share_start (true))
|
{
|
{
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
|
GFS_GUIDED, chunk_size);
|
GFS_GUIDED, chunk_size);
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
gomp_work_share_init_done ();
|
gomp_work_share_init_done ();
|
}
|
}
|
else
|
else
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
|
|
ret = gomp_iter_ull_guided_next_locked (istart, iend);
|
ret = gomp_iter_ull_guided_next_locked (istart, iend);
|
if (ret)
|
if (ret)
|
gomp_ordered_first ();
|
gomp_ordered_first ();
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
|
return ret;
|
return ret;
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_ordered_runtime_start (bool up, gomp_ull start, gomp_ull end,
|
GOMP_loop_ull_ordered_runtime_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_ull incr, gomp_ull *istart,
|
gomp_ull incr, gomp_ull *istart,
|
gomp_ull *iend)
|
gomp_ull *iend)
|
{
|
{
|
struct gomp_task_icv *icv = gomp_icv (false);
|
struct gomp_task_icv *icv = gomp_icv (false);
|
switch (icv->run_sched_var)
|
switch (icv->run_sched_var)
|
{
|
{
|
case GFS_STATIC:
|
case GFS_STATIC:
|
return gomp_loop_ull_ordered_static_start (up, start, end, incr,
|
return gomp_loop_ull_ordered_static_start (up, start, end, incr,
|
icv->run_sched_modifier,
|
icv->run_sched_modifier,
|
istart, iend);
|
istart, iend);
|
case GFS_DYNAMIC:
|
case GFS_DYNAMIC:
|
return gomp_loop_ull_ordered_dynamic_start (up, start, end, incr,
|
return gomp_loop_ull_ordered_dynamic_start (up, start, end, incr,
|
icv->run_sched_modifier,
|
icv->run_sched_modifier,
|
istart, iend);
|
istart, iend);
|
case GFS_GUIDED:
|
case GFS_GUIDED:
|
return gomp_loop_ull_ordered_guided_start (up, start, end, incr,
|
return gomp_loop_ull_ordered_guided_start (up, start, end, incr,
|
icv->run_sched_modifier,
|
icv->run_sched_modifier,
|
istart, iend);
|
istart, iend);
|
case GFS_AUTO:
|
case GFS_AUTO:
|
/* For now map to schedule(static), later on we could play with feedback
|
/* For now map to schedule(static), later on we could play with feedback
|
driven choice. */
|
driven choice. */
|
return gomp_loop_ull_ordered_static_start (up, start, end, incr,
|
return gomp_loop_ull_ordered_static_start (up, start, end, incr,
|
0, istart, iend);
|
0, istart, iend);
|
default:
|
default:
|
abort ();
|
abort ();
|
}
|
}
|
}
|
}
|
|
|
/* The *_next routines are called when the thread completes processing of
|
/* The *_next routines are called when the thread completes processing of
|
the iteration block currently assigned to it. If the work-share
|
the iteration block currently assigned to it. If the work-share
|
construct is bound directly to a parallel construct, then the iteration
|
construct is bound directly to a parallel construct, then the iteration
|
bounds may have been set up before the parallel. In which case, this
|
bounds may have been set up before the parallel. In which case, this
|
may be the first iteration for the thread.
|
may be the first iteration for the thread.
|
|
|
Returns true if there is work remaining to be performed; *ISTART and
|
Returns true if there is work remaining to be performed; *ISTART and
|
*IEND are filled with a new iteration block. Returns false if all work
|
*IEND are filled with a new iteration block. Returns false if all work
|
has been assigned. */
|
has been assigned. */
|
|
|
static bool
|
static bool
|
gomp_loop_ull_static_next (gomp_ull *istart, gomp_ull *iend)
|
gomp_loop_ull_static_next (gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
return !gomp_iter_ull_static_next (istart, iend);
|
return !gomp_iter_ull_static_next (istart, iend);
|
}
|
}
|
|
|
static bool
|
static bool
|
gomp_loop_ull_dynamic_next (gomp_ull *istart, gomp_ull *iend)
|
gomp_loop_ull_dynamic_next (gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
bool ret;
|
bool ret;
|
|
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
ret = gomp_iter_ull_dynamic_next (istart, iend);
|
ret = gomp_iter_ull_dynamic_next (istart, iend);
|
#else
|
#else
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_thread *thr = gomp_thread ();
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
|
ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
#endif
|
#endif
|
|
|
return ret;
|
return ret;
|
}
|
}
|
|
|
static bool
|
static bool
|
gomp_loop_ull_guided_next (gomp_ull *istart, gomp_ull *iend)
|
gomp_loop_ull_guided_next (gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
bool ret;
|
bool ret;
|
|
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
#if defined HAVE_SYNC_BUILTINS && defined __LP64__
|
ret = gomp_iter_ull_guided_next (istart, iend);
|
ret = gomp_iter_ull_guided_next (istart, iend);
|
#else
|
#else
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_thread *thr = gomp_thread ();
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
ret = gomp_iter_ull_guided_next_locked (istart, iend);
|
ret = gomp_iter_ull_guided_next_locked (istart, iend);
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
#endif
|
#endif
|
|
|
return ret;
|
return ret;
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_runtime_next (gomp_ull *istart, gomp_ull *iend)
|
GOMP_loop_ull_runtime_next (gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_thread *thr = gomp_thread ();
|
|
|
switch (thr->ts.work_share->sched)
|
switch (thr->ts.work_share->sched)
|
{
|
{
|
case GFS_STATIC:
|
case GFS_STATIC:
|
case GFS_AUTO:
|
case GFS_AUTO:
|
return gomp_loop_ull_static_next (istart, iend);
|
return gomp_loop_ull_static_next (istart, iend);
|
case GFS_DYNAMIC:
|
case GFS_DYNAMIC:
|
return gomp_loop_ull_dynamic_next (istart, iend);
|
return gomp_loop_ull_dynamic_next (istart, iend);
|
case GFS_GUIDED:
|
case GFS_GUIDED:
|
return gomp_loop_ull_guided_next (istart, iend);
|
return gomp_loop_ull_guided_next (istart, iend);
|
default:
|
default:
|
abort ();
|
abort ();
|
}
|
}
|
}
|
}
|
|
|
/* The *_ordered_*_next routines are called when the thread completes
|
/* The *_ordered_*_next routines are called when the thread completes
|
processing of the iteration block currently assigned to it.
|
processing of the iteration block currently assigned to it.
|
|
|
Returns true if there is work remaining to be performed; *ISTART and
|
Returns true if there is work remaining to be performed; *ISTART and
|
*IEND are filled with a new iteration block. Returns false if all work
|
*IEND are filled with a new iteration block. Returns false if all work
|
has been assigned. */
|
has been assigned. */
|
|
|
static bool
|
static bool
|
gomp_loop_ull_ordered_static_next (gomp_ull *istart, gomp_ull *iend)
|
gomp_loop_ull_ordered_static_next (gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_thread *thr = gomp_thread ();
|
int test;
|
int test;
|
|
|
gomp_ordered_sync ();
|
gomp_ordered_sync ();
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
test = gomp_iter_ull_static_next (istart, iend);
|
test = gomp_iter_ull_static_next (istart, iend);
|
if (test >= 0)
|
if (test >= 0)
|
gomp_ordered_static_next ();
|
gomp_ordered_static_next ();
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
|
return test == 0;
|
return test == 0;
|
}
|
}
|
|
|
static bool
|
static bool
|
gomp_loop_ull_ordered_dynamic_next (gomp_ull *istart, gomp_ull *iend)
|
gomp_loop_ull_ordered_dynamic_next (gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_thread *thr = gomp_thread ();
|
bool ret;
|
bool ret;
|
|
|
gomp_ordered_sync ();
|
gomp_ordered_sync ();
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
|
ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
|
if (ret)
|
if (ret)
|
gomp_ordered_next ();
|
gomp_ordered_next ();
|
else
|
else
|
gomp_ordered_last ();
|
gomp_ordered_last ();
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
|
return ret;
|
return ret;
|
}
|
}
|
|
|
static bool
|
static bool
|
gomp_loop_ull_ordered_guided_next (gomp_ull *istart, gomp_ull *iend)
|
gomp_loop_ull_ordered_guided_next (gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_thread *thr = gomp_thread ();
|
bool ret;
|
bool ret;
|
|
|
gomp_ordered_sync ();
|
gomp_ordered_sync ();
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
gomp_mutex_lock (&thr->ts.work_share->lock);
|
ret = gomp_iter_ull_guided_next_locked (istart, iend);
|
ret = gomp_iter_ull_guided_next_locked (istart, iend);
|
if (ret)
|
if (ret)
|
gomp_ordered_next ();
|
gomp_ordered_next ();
|
else
|
else
|
gomp_ordered_last ();
|
gomp_ordered_last ();
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
gomp_mutex_unlock (&thr->ts.work_share->lock);
|
|
|
return ret;
|
return ret;
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_ordered_runtime_next (gomp_ull *istart, gomp_ull *iend)
|
GOMP_loop_ull_ordered_runtime_next (gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
struct gomp_thread *thr = gomp_thread ();
|
struct gomp_thread *thr = gomp_thread ();
|
|
|
switch (thr->ts.work_share->sched)
|
switch (thr->ts.work_share->sched)
|
{
|
{
|
case GFS_STATIC:
|
case GFS_STATIC:
|
case GFS_AUTO:
|
case GFS_AUTO:
|
return gomp_loop_ull_ordered_static_next (istart, iend);
|
return gomp_loop_ull_ordered_static_next (istart, iend);
|
case GFS_DYNAMIC:
|
case GFS_DYNAMIC:
|
return gomp_loop_ull_ordered_dynamic_next (istart, iend);
|
return gomp_loop_ull_ordered_dynamic_next (istart, iend);
|
case GFS_GUIDED:
|
case GFS_GUIDED:
|
return gomp_loop_ull_ordered_guided_next (istart, iend);
|
return gomp_loop_ull_ordered_guided_next (istart, iend);
|
default:
|
default:
|
abort ();
|
abort ();
|
}
|
}
|
}
|
}
|
|
|
/* We use static functions above so that we're sure that the "runtime"
|
/* We use static functions above so that we're sure that the "runtime"
|
function can defer to the proper routine without interposition. We
|
function can defer to the proper routine without interposition. We
|
export the static function with a strong alias when possible, or with
|
export the static function with a strong alias when possible, or with
|
a wrapper function otherwise. */
|
a wrapper function otherwise. */
|
|
|
#ifdef HAVE_ATTRIBUTE_ALIAS
|
#ifdef HAVE_ATTRIBUTE_ALIAS
|
extern __typeof(gomp_loop_ull_static_start) GOMP_loop_ull_static_start
|
extern __typeof(gomp_loop_ull_static_start) GOMP_loop_ull_static_start
|
__attribute__((alias ("gomp_loop_ull_static_start")));
|
__attribute__((alias ("gomp_loop_ull_static_start")));
|
extern __typeof(gomp_loop_ull_dynamic_start) GOMP_loop_ull_dynamic_start
|
extern __typeof(gomp_loop_ull_dynamic_start) GOMP_loop_ull_dynamic_start
|
__attribute__((alias ("gomp_loop_ull_dynamic_start")));
|
__attribute__((alias ("gomp_loop_ull_dynamic_start")));
|
extern __typeof(gomp_loop_ull_guided_start) GOMP_loop_ull_guided_start
|
extern __typeof(gomp_loop_ull_guided_start) GOMP_loop_ull_guided_start
|
__attribute__((alias ("gomp_loop_ull_guided_start")));
|
__attribute__((alias ("gomp_loop_ull_guided_start")));
|
|
|
extern __typeof(gomp_loop_ull_ordered_static_start) GOMP_loop_ull_ordered_static_start
|
extern __typeof(gomp_loop_ull_ordered_static_start) GOMP_loop_ull_ordered_static_start
|
__attribute__((alias ("gomp_loop_ull_ordered_static_start")));
|
__attribute__((alias ("gomp_loop_ull_ordered_static_start")));
|
extern __typeof(gomp_loop_ull_ordered_dynamic_start) GOMP_loop_ull_ordered_dynamic_start
|
extern __typeof(gomp_loop_ull_ordered_dynamic_start) GOMP_loop_ull_ordered_dynamic_start
|
__attribute__((alias ("gomp_loop_ull_ordered_dynamic_start")));
|
__attribute__((alias ("gomp_loop_ull_ordered_dynamic_start")));
|
extern __typeof(gomp_loop_ull_ordered_guided_start) GOMP_loop_ull_ordered_guided_start
|
extern __typeof(gomp_loop_ull_ordered_guided_start) GOMP_loop_ull_ordered_guided_start
|
__attribute__((alias ("gomp_loop_ull_ordered_guided_start")));
|
__attribute__((alias ("gomp_loop_ull_ordered_guided_start")));
|
|
|
extern __typeof(gomp_loop_ull_static_next) GOMP_loop_ull_static_next
|
extern __typeof(gomp_loop_ull_static_next) GOMP_loop_ull_static_next
|
__attribute__((alias ("gomp_loop_ull_static_next")));
|
__attribute__((alias ("gomp_loop_ull_static_next")));
|
extern __typeof(gomp_loop_ull_dynamic_next) GOMP_loop_ull_dynamic_next
|
extern __typeof(gomp_loop_ull_dynamic_next) GOMP_loop_ull_dynamic_next
|
__attribute__((alias ("gomp_loop_ull_dynamic_next")));
|
__attribute__((alias ("gomp_loop_ull_dynamic_next")));
|
extern __typeof(gomp_loop_ull_guided_next) GOMP_loop_ull_guided_next
|
extern __typeof(gomp_loop_ull_guided_next) GOMP_loop_ull_guided_next
|
__attribute__((alias ("gomp_loop_ull_guided_next")));
|
__attribute__((alias ("gomp_loop_ull_guided_next")));
|
|
|
extern __typeof(gomp_loop_ull_ordered_static_next) GOMP_loop_ull_ordered_static_next
|
extern __typeof(gomp_loop_ull_ordered_static_next) GOMP_loop_ull_ordered_static_next
|
__attribute__((alias ("gomp_loop_ull_ordered_static_next")));
|
__attribute__((alias ("gomp_loop_ull_ordered_static_next")));
|
extern __typeof(gomp_loop_ull_ordered_dynamic_next) GOMP_loop_ull_ordered_dynamic_next
|
extern __typeof(gomp_loop_ull_ordered_dynamic_next) GOMP_loop_ull_ordered_dynamic_next
|
__attribute__((alias ("gomp_loop_ull_ordered_dynamic_next")));
|
__attribute__((alias ("gomp_loop_ull_ordered_dynamic_next")));
|
extern __typeof(gomp_loop_ull_ordered_guided_next) GOMP_loop_ull_ordered_guided_next
|
extern __typeof(gomp_loop_ull_ordered_guided_next) GOMP_loop_ull_ordered_guided_next
|
__attribute__((alias ("gomp_loop_ull_ordered_guided_next")));
|
__attribute__((alias ("gomp_loop_ull_ordered_guided_next")));
|
#else
|
#else
|
bool
|
bool
|
GOMP_loop_ull_static_start (bool up, gomp_ull start, gomp_ull end,
|
GOMP_loop_ull_static_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull *istart, gomp_ull *iend)
|
gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
return gomp_loop_ull_static_start (up, start, end, incr, chunk_size, istart,
|
return gomp_loop_ull_static_start (up, start, end, incr, chunk_size, istart,
|
iend);
|
iend);
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_dynamic_start (bool up, gomp_ull start, gomp_ull end,
|
GOMP_loop_ull_dynamic_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull *istart, gomp_ull *iend)
|
gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
return gomp_loop_ull_dynamic_start (up, start, end, incr, chunk_size, istart,
|
return gomp_loop_ull_dynamic_start (up, start, end, incr, chunk_size, istart,
|
iend);
|
iend);
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_guided_start (bool up, gomp_ull start, gomp_ull end,
|
GOMP_loop_ull_guided_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull *istart, gomp_ull *iend)
|
gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
return gomp_loop_ull_guided_start (up, start, end, incr, chunk_size, istart,
|
return gomp_loop_ull_guided_start (up, start, end, incr, chunk_size, istart,
|
iend);
|
iend);
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_ordered_static_start (bool up, gomp_ull start, gomp_ull end,
|
GOMP_loop_ull_ordered_static_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull *istart, gomp_ull *iend)
|
gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
return gomp_loop_ull_ordered_static_start (up, start, end, incr, chunk_size,
|
return gomp_loop_ull_ordered_static_start (up, start, end, incr, chunk_size,
|
istart, iend);
|
istart, iend);
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_ordered_dynamic_start (bool up, gomp_ull start, gomp_ull end,
|
GOMP_loop_ull_ordered_dynamic_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull *istart, gomp_ull *iend)
|
gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
return gomp_loop_ull_ordered_dynamic_start (up, start, end, incr, chunk_size,
|
return gomp_loop_ull_ordered_dynamic_start (up, start, end, incr, chunk_size,
|
istart, iend);
|
istart, iend);
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_ordered_guided_start (bool up, gomp_ull start, gomp_ull end,
|
GOMP_loop_ull_ordered_guided_start (bool up, gomp_ull start, gomp_ull end,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull incr, gomp_ull chunk_size,
|
gomp_ull *istart, gomp_ull *iend)
|
gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
return gomp_loop_ull_ordered_guided_start (up, start, end, incr, chunk_size,
|
return gomp_loop_ull_ordered_guided_start (up, start, end, incr, chunk_size,
|
istart, iend);
|
istart, iend);
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_static_next (gomp_ull *istart, gomp_ull *iend)
|
GOMP_loop_ull_static_next (gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
return gomp_loop_ull_static_next (istart, iend);
|
return gomp_loop_ull_static_next (istart, iend);
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_dynamic_next (gomp_ull *istart, gomp_ull *iend)
|
GOMP_loop_ull_dynamic_next (gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
return gomp_loop_ull_dynamic_next (istart, iend);
|
return gomp_loop_ull_dynamic_next (istart, iend);
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_guided_next (gomp_ull *istart, gomp_ull *iend)
|
GOMP_loop_ull_guided_next (gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
return gomp_loop_ull_guided_next (istart, iend);
|
return gomp_loop_ull_guided_next (istart, iend);
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_ordered_static_next (gomp_ull *istart, gomp_ull *iend)
|
GOMP_loop_ull_ordered_static_next (gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
return gomp_loop_ull_ordered_static_next (istart, iend);
|
return gomp_loop_ull_ordered_static_next (istart, iend);
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_ordered_dynamic_next (gomp_ull *istart, gomp_ull *iend)
|
GOMP_loop_ull_ordered_dynamic_next (gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
return gomp_loop_ull_ordered_dynamic_next (istart, iend);
|
return gomp_loop_ull_ordered_dynamic_next (istart, iend);
|
}
|
}
|
|
|
bool
|
bool
|
GOMP_loop_ull_ordered_guided_next (gomp_ull *istart, gomp_ull *iend)
|
GOMP_loop_ull_ordered_guided_next (gomp_ull *istart, gomp_ull *iend)
|
{
|
{
|
return gomp_loop_ull_ordered_guided_next (istart, iend);
|
return gomp_loop_ull_ordered_guided_next (istart, iend);
|
}
|
}
|
#endif
|
#endif
|
|
|
