Subversion Repositories openrisc

// Copyright 2009 The Go Authors. All rights reserved.

// Use of this source code is governed by a BSD-style

// license that can be found in the LICENSE file.

#include <limits.h>

#include <stdlib.h>

#include <pthread.h>

#include <unistd.h>

#include "config.h"

#include "runtime.h"

#include "arch.h"

#include "defs.h"

#include "malloc.h"

#include "go-defer.h"

#ifdef USING_SPLIT_STACK

/* FIXME: These are not declared anywhere.  */

extern void __splitstack_getcontext(void *context[10]);

extern void __splitstack_setcontext(void *context[10]);

extern void *__splitstack_makecontext(size_t, void *context[10], size_t *);

extern void * __splitstack_resetcontext(void *context[10], size_t *);

extern void *__splitstack_find(void *, void *, size_t *, void **, void **,

                               void **);

extern void __splitstack_block_signals (int *, int *);

extern void __splitstack_block_signals_context (void *context[10], int *,

                                                int *);

#endif

#if defined(USING_SPLIT_STACK) && defined(LINKER_SUPPORTS_SPLIT_STACK)

# ifdef PTHREAD_STACK_MIN

#  define StackMin PTHREAD_STACK_MIN

# else

#  define StackMin 8192

# endif

#else

# define StackMin 2 * 1024 * 1024

#endif

static void schedule(G*);

typedef struct Sched Sched;

M       runtime_m0;

G       runtime_g0;     // idle goroutine for m0

#ifdef __rtems__

#define __thread

#endif

static __thread G *g;

static __thread M *m;

#ifndef SETCONTEXT_CLOBBERS_TLS

static inline void

initcontext(void)

{

}

static inline void

fixcontext(ucontext_t *c __attribute__ ((unused)))

{

}

# else

# if defined(__x86_64__) && defined(__sun__)

// x86_64 Solaris 10 and 11 have a bug: setcontext switches the %fs

// register to that of the thread which called getcontext.  The effect

// is that the address of all __thread variables changes.  This bug

// also affects pthread_self() and pthread_getspecific.  We work

// around it by clobbering the context field directly to keep %fs the

// same.

static __thread greg_t fs;

static inline void

initcontext(void)

{

        ucontext_t c;

        getcontext(&c);

        fs = c.uc_mcontext.gregs[REG_FSBASE];

}

static inline void

fixcontext(ucontext_t* c)

{

        c->uc_mcontext.gregs[REG_FSBASE] = fs;

}

# else

#  error unknown case for SETCONTEXT_CLOBBERS_TLS

# endif

#endif

// We can not always refer to the TLS variables directly.  The

// compiler will call tls_get_addr to get the address of the variable,

// and it may hold it in a register across a call to schedule.  When

// we get back from the call we may be running in a different thread,

// in which case the register now points to the TLS variable for a

// different thread.  We use non-inlinable functions to avoid this

// when necessary.

G* runtime_g(void) __attribute__ ((noinline, no_split_stack));

G*

runtime_g(void)

{

        return g;

}

M* runtime_m(void) __attribute__ ((noinline, no_split_stack));

M*

runtime_m(void)

{

        return m;

}

int32   runtime_gcwaiting;

// Go scheduler

//

// The go scheduler's job is to match ready-to-run goroutines (`g's)

// with waiting-for-work schedulers (`m's).  If there are ready g's

// and no waiting m's, ready() will start a new m running in a new

// OS thread, so that all ready g's can run simultaneously, up to a limit.

// For now, m's never go away.

//

// By default, Go keeps only one kernel thread (m) running user code

// at a single time; other threads may be blocked in the operating system.

// Setting the environment variable $GOMAXPROCS or calling

// runtime.GOMAXPROCS() will change the number of user threads

// allowed to execute simultaneously.  $GOMAXPROCS is thus an

// approximation of the maximum number of cores to use.

//

// Even a program that can run without deadlock in a single process

// might use more m's if given the chance.  For example, the prime

// sieve will use as many m's as there are primes (up to runtime_sched.mmax),

// allowing different stages of the pipeline to execute in parallel.

// We could revisit this choice, only kicking off new m's for blocking

// system calls, but that would limit the amount of parallel computation

// that go would try to do.

//

// In general, one could imagine all sorts of refinements to the

// scheduler, but the goal now is just to get something working on

// Linux and OS X.

struct Sched {

        Lock;

        G *gfree;       // available g's (status == Gdead)

        int32 goidgen;

        G *ghead;       // g's waiting to run

        G *gtail;

        int32 gwait;    // number of g's waiting to run

        int32 gcount;   // number of g's that are alive

        int32 grunning; // number of g's running on cpu or in syscall

        M *mhead;       // m's waiting for work

        int32 mwait;    // number of m's waiting for work

        int32 mcount;   // number of m's that have been created

        volatile uint32 atomic; // atomic scheduling word (see below)

        int32 profilehz;        // cpu profiling rate

        bool init;  // running initialization

        bool lockmain;  // init called runtime.LockOSThread

        Note    stopped;        // one g can set waitstop and wait here for m's to stop

};

// The atomic word in sched is an atomic uint32 that

// holds these fields.

//

//      [15 bits] mcpu          number of m's executing on cpu

//      [15 bits] mcpumax       max number of m's allowed on cpu

//      [1 bit] waitstop        some g is waiting on stopped

//      [1 bit] gwaiting        gwait != 0

//

// These fields are the information needed by entersyscall

// and exitsyscall to decide whether to coordinate with the

// scheduler.  Packing them into a single machine word lets

// them use a fast path with a single atomic read/write and

// no lock/unlock.  This greatly reduces contention in

// syscall- or cgo-heavy multithreaded programs.

//

// Except for entersyscall and exitsyscall, the manipulations

// to these fields only happen while holding the schedlock,

// so the routines holding schedlock only need to worry about

// what entersyscall and exitsyscall do, not the other routines

// (which also use the schedlock).

//

// In particular, entersyscall and exitsyscall only read mcpumax,

// waitstop, and gwaiting.  They never write them.  Thus, writes to those

// fields can be done (holding schedlock) without fear of write conflicts.

// There may still be logic conflicts: for example, the set of waitstop must

// be conditioned on mcpu >= mcpumax or else the wait may be a

// spurious sleep.  The Promela model in proc.p verifies these accesses.

enum {

        mcpuWidth = 15,

        mcpuMask = (1<<mcpuWidth) - 1,

        mcpuShift = 0,

        mcpumaxShift = mcpuShift + mcpuWidth,

        waitstopShift = mcpumaxShift + mcpuWidth,

        gwaitingShift = waitstopShift+1,

        // The max value of GOMAXPROCS is constrained

        // by the max value we can store in the bit fields

        // of the atomic word.  Reserve a few high values

        // so that we can detect accidental decrement

        // beyond zero.

        maxgomaxprocs = mcpuMask - 10,

};

#define atomic_mcpu(v)          (((v)>>mcpuShift)&mcpuMask)

#define atomic_mcpumax(v)       (((v)>>mcpumaxShift)&mcpuMask)

#define atomic_waitstop(v)      (((v)>>waitstopShift)&1)

#define atomic_gwaiting(v)      (((v)>>gwaitingShift)&1)

Sched runtime_sched;

int32 runtime_gomaxprocs;

bool runtime_singleproc;

static bool canaddmcpu(void);

// An m that is waiting for notewakeup(&m->havenextg).  This may

// only be accessed while the scheduler lock is held.  This is used to

// minimize the number of times we call notewakeup while the scheduler

// lock is held, since the m will normally move quickly to lock the

// scheduler itself, producing lock contention.

static M* mwakeup;

// Scheduling helpers.  Sched must be locked.

static void gput(G*);   // put/get on ghead/gtail

static G* gget(void);

static void mput(M*);   // put/get on mhead

static M* mget(G*);

static void gfput(G*);  // put/get on gfree

static G* gfget(void);

static void matchmg(void);      // match m's to g's

static void readylocked(G*);    // ready, but sched is locked

static void mnextg(M*, G*);

static void mcommoninit(M*);

void

setmcpumax(uint32 n)

{

        uint32 v, w;

        for(;;) {

                v = runtime_sched.atomic;

                w = v;

                w &= ~(mcpuMask<<mcpumaxShift);

                w |= n<<mcpumaxShift;

                if(runtime_cas(&runtime_sched.atomic, v, w))

                        break;

        }

}

// First function run by a new goroutine.  This replaces gogocall.

static void

kickoff(void)

{

        void (*fn)(void*);

        fn = (void (*)(void*))(g->entry);

        fn(g->param);

        runtime_goexit();

}

// Switch context to a different goroutine.  This is like longjmp.

static void runtime_gogo(G*) __attribute__ ((noinline));

static void

runtime_gogo(G* newg)

{

#ifdef USING_SPLIT_STACK

        __splitstack_setcontext(&newg->stack_context[0]);

#endif

        g = newg;

        newg->fromgogo = true;

        fixcontext(&newg->context);

        setcontext(&newg->context);

        runtime_throw("gogo setcontext returned");

}

// Save context and call fn passing g as a parameter.  This is like

// setjmp.  Because getcontext always returns 0, unlike setjmp, we use

// g->fromgogo as a code.  It will be true if we got here via

// setcontext.  g == nil the first time this is called in a new m.

static void runtime_mcall(void (*)(G*)) __attribute__ ((noinline));

static void

runtime_mcall(void (*pfn)(G*))

{

        M *mp;

        G *gp;

#ifndef USING_SPLIT_STACK

        int i;

#endif

        // Ensure that all registers are on the stack for the garbage

        // collector.

        __builtin_unwind_init();

        mp = m;

        gp = g;

        if(gp == mp->g0)

                runtime_throw("runtime: mcall called on m->g0 stack");

        if(gp != nil) {

#ifdef USING_SPLIT_STACK

                __splitstack_getcontext(&g->stack_context[0]);

#else

                gp->gcnext_sp = &i;

#endif

                gp->fromgogo = false;

                getcontext(&gp->context);

                // When we return from getcontext, we may be running

                // in a new thread.  That means that m and g may have

                // changed.  They are global variables so we will

                // reload them, but the addresses of m and g may be

                // cached in our local stack frame, and those

                // addresses may be wrong.  Call functions to reload

                // the values for this thread.

                mp = runtime_m();

                gp = runtime_g();

        }

        if (gp == nil || !gp->fromgogo) {

#ifdef USING_SPLIT_STACK

                __splitstack_setcontext(&mp->g0->stack_context[0]);

#endif

                mp->g0->entry = (byte*)pfn;

                mp->g0->param = gp;

                // It's OK to set g directly here because this case

                // can not occur if we got here via a setcontext to

                // the getcontext call just above.

                g = mp->g0;

                fixcontext(&mp->g0->context);

                setcontext(&mp->g0->context);

                runtime_throw("runtime: mcall function returned");

        }

}

// The bootstrap sequence is:

//

//      call osinit

//      call schedinit

//      make & queue new G

//      call runtime_mstart

//

// The new G calls runtime_main.

void

runtime_schedinit(void)

{

        int32 n;

        const byte *p;

        m = &runtime_m0;

        g = &runtime_g0;

        m->g0 = g;

        m->curg = g;

        g->m = m;

        initcontext();

        m->nomemprof++;

        runtime_mallocinit();

        mcommoninit(m);

        runtime_goargs();

        runtime_goenvs();

        // For debugging:

        // Allocate internal symbol table representation now,

        // so that we don't need to call malloc when we crash.

        // runtime_findfunc(0);

        runtime_gomaxprocs = 1;

        p = runtime_getenv("GOMAXPROCS");

        if(p != nil && (n = runtime_atoi(p)) != 0) {

                if(n > maxgomaxprocs)

                        n = maxgomaxprocs;

                runtime_gomaxprocs = n;

        }

        setmcpumax(runtime_gomaxprocs);

        runtime_singleproc = runtime_gomaxprocs == 1;

        canaddmcpu();   // mcpu++ to account for bootstrap m

        m->helpgc = 1;  // flag to tell schedule() to mcpu--

        runtime_sched.grunning++;

        // Can not enable GC until all roots are registered.

        // mstats.enablegc = 1;

        m->nomemprof--;

}

extern void main_init(void) __asm__ ("__go_init_main");

extern void main_main(void) __asm__ ("main.main");

// The main goroutine.

void

runtime_main(void)

{

        // Lock the main goroutine onto this, the main OS thread,

        // during initialization.  Most programs won't care, but a few

        // do require certain calls to be made by the main thread.

        // Those can arrange for main.main to run in the main thread

        // by calling runtime.LockOSThread during initialization

        // to preserve the lock.

        runtime_LockOSThread();

        runtime_sched.init = true;

        main_init();

        runtime_sched.init = false;

        if(!runtime_sched.lockmain)

                runtime_UnlockOSThread();

        // For gccgo we have to wait until after main is initialized

        // to enable GC, because initializing main registers the GC

        // roots.

        mstats.enablegc = 1;

        main_main();

        runtime_exit(0);

        for(;;)

                *(int32*)0 = 0;

}

// Lock the scheduler.

static void

schedlock(void)

{

        runtime_lock(&runtime_sched);

}

// Unlock the scheduler.

static void

schedunlock(void)

{

        M *m;

        m = mwakeup;

        mwakeup = nil;

        runtime_unlock(&runtime_sched);

        if(m != nil)

                runtime_notewakeup(&m->havenextg);

}

void

runtime_goexit(void)

{

        g->status = Gmoribund;

        runtime_gosched();

}

void

runtime_goroutineheader(G *g)

{

        const char *status;

        switch(g->status) {

        case Gidle:

                status = "idle";

                break;

        case Grunnable:

                status = "runnable";

                break;

        case Grunning:

                status = "running";

                break;

        case Gsyscall:

                status = "syscall";

                break;

        case Gwaiting:

                if(g->waitreason)

                        status = g->waitreason;

                else

                        status = "waiting";

                break;

        case Gmoribund:

                status = "moribund";

                break;

        default:

                status = "???";

                break;

        }

        runtime_printf("goroutine %d [%s]:\n", g->goid, status);

}

void

runtime_tracebackothers(G *me)

{

        G *g;

        for(g = runtime_allg; g != nil; g = g->alllink) {

                if(g == me || g->status == Gdead)

                        continue;

                runtime_printf("\n");

                runtime_goroutineheader(g);

                // runtime_traceback(g->sched.pc, g->sched.sp, 0, g);

        }

}

// Mark this g as m's idle goroutine.

// This functionality might be used in environments where programs

// are limited to a single thread, to simulate a select-driven

// network server.  It is not exposed via the standard runtime API.

void

runtime_idlegoroutine(void)

{

        if(g->idlem != nil)

                runtime_throw("g is already an idle goroutine");

        g->idlem = m;

}

static void

mcommoninit(M *m)

{

        // Add to runtime_allm so garbage collector doesn't free m

        // when it is just in a register or thread-local storage.

        m->alllink = runtime_allm;

        // runtime_Cgocalls() iterates over allm w/o schedlock,

        // so we need to publish it safely.

        runtime_atomicstorep((void**)&runtime_allm, m);

        m->id = runtime_sched.mcount++;

        m->fastrand = 0x49f6428aUL + m->id + runtime_cputicks();

        if(m->mcache == nil)

                m->mcache = runtime_allocmcache();

}

// Try to increment mcpu.  Report whether succeeded.

static bool

canaddmcpu(void)

{

        uint32 v;

        for(;;) {

                v = runtime_sched.atomic;

                if(atomic_mcpu(v) >= atomic_mcpumax(v))

                        return 0;

                if(runtime_cas(&runtime_sched.atomic, v, v+(1<<mcpuShift)))

                        return 1;

        }

}

// Put on `g' queue.  Sched must be locked.

static void

gput(G *g)

{

        M *m;

        // If g is wired, hand it off directly.

        if((m = g->lockedm) != nil && canaddmcpu()) {

                mnextg(m, g);

                return;

        }

        // If g is the idle goroutine for an m, hand it off.

        if(g->idlem != nil) {

                if(g->idlem->idleg != nil) {

                        runtime_printf("m%d idle out of sync: g%d g%d\n",

                                g->idlem->id,

                                g->idlem->idleg->goid, g->goid);

                        runtime_throw("runtime: double idle");

                }

                g->idlem->idleg = g;

                return;

        }

        g->schedlink = nil;

        if(runtime_sched.ghead == nil)

                runtime_sched.ghead = g;

        else

                runtime_sched.gtail->schedlink = g;

        runtime_sched.gtail = g;

        // increment gwait.

        // if it transitions to nonzero, set atomic gwaiting bit.

        if(runtime_sched.gwait++ == 0)

                runtime_xadd(&runtime_sched.atomic, 1<<gwaitingShift);

}

// Report whether gget would return something.

static bool

haveg(void)

{

        return runtime_sched.ghead != nil || m->idleg != nil;

}

// Get from `g' queue.  Sched must be locked.

static G*

gget(void)

{

        G *g;

        g = runtime_sched.ghead;

        if(g){

                runtime_sched.ghead = g->schedlink;

                if(runtime_sched.ghead == nil)

                        runtime_sched.gtail = nil;

                // decrement gwait.

                // if it transitions to zero, clear atomic gwaiting bit.

                if(--runtime_sched.gwait == 0)

                        runtime_xadd(&runtime_sched.atomic, -1<<gwaitingShift);

        } else if(m->idleg != nil) {

                g = m->idleg;

                m->idleg = nil;

        }

        return g;

}

// Put on `m' list.  Sched must be locked.

static void

mput(M *m)

{

        m->schedlink = runtime_sched.mhead;

        runtime_sched.mhead = m;

        runtime_sched.mwait++;

}

// Get an `m' to run `g'.  Sched must be locked.

static M*

mget(G *g)

{

        M *m;

        // if g has its own m, use it.

        if(g && (m = g->lockedm) != nil)

                return m;

        // otherwise use general m pool.

        if((m = runtime_sched.mhead) != nil){

                runtime_sched.mhead = m->schedlink;

                runtime_sched.mwait--;

        }

        return m;

}

// Mark g ready to run.

void

runtime_ready(G *g)

{

        schedlock();

        readylocked(g);

        schedunlock();

}

// Mark g ready to run.  Sched is already locked.

// G might be running already and about to stop.

// The sched lock protects g->status from changing underfoot.

static void

readylocked(G *g)

{

        if(g->m){

                // Running on another machine.

                // Ready it when it stops.

                g->readyonstop = 1;

                return;

        }

        // Mark runnable.

        if(g->status == Grunnable || g->status == Grunning) {

                runtime_printf("goroutine %d has status %d\n", g->goid, g->status);

                runtime_throw("bad g->status in ready");

        }

        g->status = Grunnable;

        gput(g);

        matchmg();

}

// Same as readylocked but a different symbol so that

// debuggers can set a breakpoint here and catch all

// new goroutines.

static void

newprocreadylocked(G *g)

{

        readylocked(g);

}

// Pass g to m for running.

// Caller has already incremented mcpu.

static void

mnextg(M *m, G *g)

{

        runtime_sched.grunning++;

        m->nextg = g;

        if(m->waitnextg) {

                m->waitnextg = 0;

                if(mwakeup != nil)

                        runtime_notewakeup(&mwakeup->havenextg);

                mwakeup = m;