Subversion Repositories or1k

/*

 *

 *

 * SMP support for ppc.

 *

 * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great

 * deal of code from the sparc and intel versions.

 *

 * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>

 *

 * PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and

 * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com

 *

 *      This program is free software; you can redistribute it and/or

 *      modify it under the terms of the GNU General Public License

 *      as published by the Free Software Foundation; either version

 *      2 of the License, or (at your option) any later version.

 */

#include <linux/config.h>

#include <linux/kernel.h>

#include <linux/sched.h>

#include <linux/smp.h>

#include <linux/smp_lock.h>

#include <linux/interrupt.h>

#include <linux/kernel_stat.h>

#include <linux/delay.h>

#define __KERNEL_SYSCALLS__

#include <linux/unistd.h>

#include <linux/init.h>

#include <linux/mm.h>

#include <linux/spinlock.h>

#include <linux/cache.h>

#include <asm/ptrace.h>

#include <asm/atomic.h>

#include <asm/irq.h>

#include <asm/page.h>

#include <asm/pgtable.h>

#include <asm/hardirq.h>

#include <asm/softirq.h>

#include <asm/init.h>

#include <asm/io.h>

#include <asm/prom.h>

#include <asm/smp.h>

#include <asm/naca.h>

#include <asm/paca.h>

#include <asm/iSeries/LparData.h>

#include <asm/iSeries/HvCall.h>

#include <asm/iSeries/HvCallCfg.h>

#include <asm/time.h>

#include <asm/ppcdebug.h>

#include "open_pic.h"

#include <asm/machdep.h>

#include <asm/cputable.h>

#if defined(CONFIG_DUMP) || defined(CONFIG_DUMP_MODULE)

int (*dump_ipi_function_ptr)(struct pt_regs *);

#include <asm/dump.h>

#endif

#ifdef CONFIG_KDB

#include <linux/kdb.h>

#endif

int smp_threads_ready = 0;

volatile int smp_commenced = 0;

int smp_num_cpus = 1;

int smp_tb_synchronized = 0;

extern atomic_t ipi_recv;

extern atomic_t ipi_sent;

spinlock_t kernel_flag __cacheline_aligned = SPIN_LOCK_UNLOCKED;

cycles_t cacheflush_time;

static int max_cpus __initdata = NR_CPUS;

unsigned long cpu_online_map;

volatile unsigned long cpu_callin_map[NR_CPUS] = {0,};

#define TB_SYNC_PASSES 4

volatile unsigned long __initdata tb_sync_flag = 0;

volatile unsigned long __initdata tb_offset = 0;

extern unsigned char stab_array[];

int start_secondary(void *);

extern int cpu_idle(void *unused);

void smp_call_function_interrupt(void);

void smp_message_pass(int target, int msg, unsigned long data, int wait);

static unsigned long iSeries_smp_message[NR_CPUS];

void xics_setup_cpu(void);

void xics_cause_IPI(int cpu);

long h_register_vpa(unsigned long flags, unsigned long proc,

                    unsigned long vpa);

/*

 * XICS only has a single IPI, so encode the messages per CPU

 */

volatile unsigned long xics_ipi_message[NR_CPUS] = {0};

#define smp_message_pass(t,m,d,w) \

    do {     atomic_inc(&ipi_sent); \

             ppc_md.smp_message_pass((t),(m),(d),(w)); \

       } while(0)

#ifdef CONFIG_KDB

/* save regs here before calling kdb_ipi */

struct pt_regs *kdb_smp_regs[NR_CPUS];

/* called for each processor.. drop each into kdb. */

void smp_kdb_stop_proc(void)

{

    kdb_ipi(kdb_smp_regs[smp_processor_id()], NULL);

}

void smp_kdb_stop(void)

{

  int ret=0;

  ret =    smp_call_function(smp_kdb_stop_proc, NULL, 1, 0);

}

#endif

static inline void set_tb(unsigned int upper, unsigned int lower)

{

        mtspr(SPRN_TBWL, 0);

        mtspr(SPRN_TBWU, upper);

        mtspr(SPRN_TBWL, lower);

}

void iSeries_smp_message_recv( struct pt_regs * regs )

{

        int cpu = smp_processor_id();

        int msg;

        if ( smp_num_cpus < 2 )

                return;

        for ( msg = 0; msg < 4; ++msg )

                if ( test_and_clear_bit( msg, &iSeries_smp_message[cpu] ) )

                        smp_message_recv( msg, regs );

}

static void smp_iSeries_message_pass(int target, int msg, unsigned long data, int wait)

{

        int i;

        for (i = 0; i < smp_num_cpus; ++i) {

                if ( (target == MSG_ALL) ||

                    (target == i) ||

                    ((target == MSG_ALL_BUT_SELF) && (i != smp_processor_id())) ) {

                        set_bit( msg, &iSeries_smp_message[i] );

                        HvCall_sendIPI(&(paca[i]));

                }

        }

}

static int smp_iSeries_numProcs(void)

{

        unsigned np, i;

        struct ItLpPaca * lpPaca;

        np = 0;

        for (i=0; i < MAX_PACAS; ++i) {

                lpPaca = paca[i].xLpPacaPtr;

                if ( lpPaca->xDynProcStatus < 2 ) {

                        ++np;

                }

        }

        return np;

}

static int smp_iSeries_probe(void)

{

        unsigned i;

        unsigned np;

        struct ItLpPaca * lpPaca;

        np = 0;

        for (i=0; i < MAX_PACAS; ++i) {

                lpPaca = paca[i].xLpPacaPtr;

                if ( lpPaca->xDynProcStatus < 2 ) {

                        ++np;

                        paca[i].next_jiffy_update_tb = paca[0].next_jiffy_update_tb;

                }

        }

        smp_tb_synchronized = 1;

        return np;

}

static void smp_iSeries_kick_cpu(int nr)

{

        struct ItLpPaca * lpPaca;

        /* Verify we have a Paca for processor nr */

        if ( ( nr <= 0 ) ||

             ( nr >= MAX_PACAS ) )

                return;

        /* Verify that our partition has a processor nr */

        lpPaca = paca[nr].xLpPacaPtr;

        if ( lpPaca->xDynProcStatus >= 2 )

                return;

        /* The information for processor bringup must

         * be written out to main store before we release

         * the processor.

         */

        mb();

        /* The processor is currently spinning, waiting

         * for the xProcStart field to become non-zero

         * After we set xProcStart, the processor will

         * continue on to secondary_start in iSeries_head.S

         */

        paca[nr].xProcStart = 1;

}

static void smp_iSeries_setup_cpu(int nr)

{

}

/* This is called very early. */

void smp_init_iSeries(void)

{

        ppc_md.smp_message_pass = smp_iSeries_message_pass;

        ppc_md.smp_probe        = smp_iSeries_probe;

        ppc_md.smp_kick_cpu     = smp_iSeries_kick_cpu;

        ppc_md.smp_setup_cpu    = smp_iSeries_setup_cpu;

        systemcfg->processorCount       = smp_iSeries_numProcs();

}

static void

smp_openpic_message_pass(int target, int msg, unsigned long data, int wait)

{

        /* make sure we're sending something that translates to an IPI */

        if ( msg > 0x3 ){

                printk("SMP %d: smp_message_pass: unknown msg %d\n",

                       smp_processor_id(), msg);

                return;

        }

        switch ( target )

        {

        case MSG_ALL:

                openpic_cause_IPI(msg, 0xffffffff);

                break;

        case MSG_ALL_BUT_SELF:

                openpic_cause_IPI(msg,

                                  0xffffffff & ~(1 << smp_processor_id()));

                break;

        default:

                openpic_cause_IPI(msg, 1<<target);

                break;

        }

}

static int

smp_chrp_probe(void)

{

        if (systemcfg->processorCount > 1)

                openpic_request_IPIs();

        return systemcfg->processorCount;

}

static void

smp_kick_cpu(int nr)

{

        /* Verify we have a Paca for processor nr */

        if ( ( nr <= 0 ) ||

             ( nr >= MAX_PACAS ) )

                return;

        /* The information for processor bringup must

         * be written out to main store before we release

         * the processor.

         */

        mb();

        /* The processor is currently spinning, waiting

         * for the xProcStart field to become non-zero

         * After we set xProcStart, the processor will

         * continue on to secondary_start in iSeries_head.S

         */

        paca[nr].xProcStart = 1;

}

static void smp_space_timers( unsigned nr )

{

        unsigned long offset, i;

        offset = tb_ticks_per_jiffy / nr;

        for ( i=1; i<nr; ++i ) {

                paca[i].next_jiffy_update_tb = paca[i-1].next_jiffy_update_tb + offset;

        }

}

static void

smp_chrp_setup_cpu(int cpu_nr)

{

        static atomic_t ready = ATOMIC_INIT(1);

        static volatile int frozen = 0;

        if (systemcfg->platform == PLATFORM_PSERIES_LPAR) {

                /* timebases already synced under the hypervisor. */

                paca[cpu_nr].next_jiffy_update_tb = tb_last_stamp = get_tb();

                if (cpu_nr == 0) {

                        systemcfg->tb_orig_stamp = tb_last_stamp;

                        /* Should update naca->stamp_xsec.

                         * For now we leave it which means the time can be some

                         * number of msecs off until someone does a settimeofday()

                         */

                }

                smp_tb_synchronized = 1;

        } else {

                if (cpu_nr == 0) {

                        /* wait for all the others */

                        while (atomic_read(&ready) < smp_num_cpus)

                                barrier();

                        atomic_set(&ready, 1);

                        /* freeze the timebase */

                        rtas_call(rtas_token("freeze-time-base"), 0, 1, NULL);

                        mb();

                        frozen = 1;

                        set_tb(0, 0);

                        paca[0].next_jiffy_update_tb = 0;

                        smp_space_timers(smp_num_cpus);

                        while (atomic_read(&ready) < smp_num_cpus)

                                barrier();

                        /* thaw the timebase again */

                        rtas_call(rtas_token("thaw-time-base"), 0, 1, NULL);

                        mb();

                        frozen = 0;

                        tb_last_stamp = get_tb();

                        systemcfg->tb_orig_stamp = tb_last_stamp;

                        smp_tb_synchronized = 1;

                } else {

                        atomic_inc(&ready);

                        while (!frozen)

                                barrier();

                        set_tb(0, 0);

                        mb();

                        atomic_inc(&ready);

                        while (frozen)

                                barrier();

                }

        }

        if (OpenPIC_Addr) {

                do_openpic_setup_cpu();

        } else {

          if (cpu_nr > 0)

            xics_setup_cpu();

        }

}

static void

smp_xics_message_pass(int target, int msg, unsigned long data, int wait)

{

        int i;

        for (i = 0; i < smp_num_cpus; ++i) {

                if (target == MSG_ALL || target == i

                    || (target == MSG_ALL_BUT_SELF

                        && i != smp_processor_id())) {

                        set_bit(msg, &xics_ipi_message[i]);

                        mb();

                        xics_cause_IPI(i);

                }

        }

}

static int

smp_xics_probe(void)

{

        return systemcfg->processorCount;

}

/* This is called very early */

void smp_init_pSeries(void)

{

        if(naca->interrupt_controller == IC_OPEN_PIC) {

                ppc_md.smp_message_pass = smp_openpic_message_pass;

                ppc_md.smp_probe        = smp_chrp_probe;

                ppc_md.smp_kick_cpu     = smp_kick_cpu;

                ppc_md.smp_setup_cpu    = smp_chrp_setup_cpu;

        } else {

                ppc_md.smp_message_pass = smp_xics_message_pass;

                ppc_md.smp_probe        = smp_xics_probe;

                ppc_md.smp_kick_cpu     = smp_kick_cpu;

                ppc_md.smp_setup_cpu    = smp_chrp_setup_cpu;

        }

}

void smp_local_timer_interrupt(struct pt_regs * regs)

{

        if (!--(get_paca()->prof_counter)) {

                update_process_times(user_mode(regs));

                (get_paca()->prof_counter)=get_paca()->prof_multiplier;

        }

}

void smp_message_recv(int msg, struct pt_regs *regs)

{

        atomic_inc(&ipi_recv);

        switch( msg ) {

        case PPC_MSG_CALL_FUNCTION:

#ifdef CONFIG_KDB

                kdb_smp_regs[smp_processor_id()]=regs;

#endif

                smp_call_function_interrupt();

                break;

        case PPC_MSG_RESCHEDULE:

                current->need_resched = 1;

                break;

#ifdef CONFIG_XMON

        case PPC_MSG_XMON_BREAK:

                /* ToDo: need a nmi way to handle this.  Soft disable? */

#if defined(CONFIG_DUMP) || defined(CONFIG_DUMP_MODULE)

                if (dump_ipi_function_ptr) {

                        printk(KERN_ALERT "got dump ipi...\n");

                        dump_ipi_function_ptr(regs);

                } else

#endif

                        xmon(regs);

                break;

#endif /* CONFIG_XMON */

        default:

                printk("SMP %d: smp_message_recv(): unknown msg %d\n",

                       smp_processor_id(), msg);

                break;

        }

}

void smp_send_reschedule(int cpu)

{

        if ((systemcfg->platform & PLATFORM_LPAR) &&

            (paca[cpu].yielded == 1)) {

#ifdef CONFIG_PPC_ISERIES

                HvCall_sendLpProd(cpu);

#else

                prod_processor(cpu);

#endif

        } else {

        smp_message_pass(cpu, PPC_MSG_RESCHEDULE, 0, 0);

}

}

#ifdef CONFIG_XMON

void smp_send_xmon_break(int cpu)

{

        smp_message_pass(cpu, PPC_MSG_XMON_BREAK, 0, 0);

}

#endif /* CONFIG_XMON */

#if defined(CONFIG_DUMP) || defined(CONFIG_DUMP_MODULE)

void dump_send_ipi(int (*dump_ipi_callback)(struct pt_regs *))

{

        dump_ipi_function_ptr = dump_ipi_callback;

        if (dump_ipi_callback) {

                printk(KERN_ALERT "dump_send_ipi...\n");

                mb();

                smp_message_pass(MSG_ALL_BUT_SELF, PPC_MSG_XMON_BREAK, 0, 0);

        }

}

#endif

static void stop_this_cpu(void *dummy)

{

        __cli();

        while (1)

                ;

}

void smp_send_stop(void)

{

        smp_call_function(stop_this_cpu, NULL, 1, 0);

        smp_num_cpus = 1;

}

/*

 * Structure and data for smp_call_function(). This is designed to minimise

 * static memory requirements. It also looks cleaner.

 * Stolen from the i386 version.

 */

static spinlock_t call_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;

static struct call_data_struct {

        void (*func) (void *info);

        void *info;

        atomic_t started;

        atomic_t finished;

        int wait;

} *call_data;

/*

 * This function sends a 'generic call function' IPI to all other CPUs

 * in the system.

 *

 * [SUMMARY] Run a function on all other CPUs.

 * <func> The function to run. This must be fast and non-blocking.

 * <info> An arbitrary pointer to pass to the function.

 * <nonatomic> currently unused.

 * <wait> If true, wait (atomically) until function has completed on other CPUs.

 * [RETURNS] 0 on success, else a negative status code. Does not return until

 * remote CPUs are nearly ready to execute <<func>> or are or have executed.

 *

 * You must not call this function with disabled interrupts or from a

 * hardware interrupt handler or from a bottom half handler.

 */

int smp_call_function (void (*func) (void *info), void *info, int nonatomic,

                        int wait)

{

        struct call_data_struct data;

        int ret = -1, cpus = smp_num_cpus-1;

        int timeout;

        if (!cpus)

                return 0;

        data.func = func;

        data.info = info;

        atomic_set(&data.started, 0);

        data.wait = wait;

        if (wait)

                atomic_set(&data.finished, 0);

        spin_lock_bh(&call_lock);

        call_data = &data;

        /* Send a message to all other CPUs and wait for them to respond */

        smp_message_pass(MSG_ALL_BUT_SELF, PPC_MSG_CALL_FUNCTION, 0, 0);

        /* Wait for response */

        timeout = 8000000;

        while (atomic_read(&data.started) != cpus) {

                HMT_low();

                if (--timeout == 0) {

                        printk("smp_call_function on cpu %d: other cpus not responding (%d)\n",

                               smp_processor_id(), atomic_read(&data.started));

#ifdef CONFIG_XMON

                        xmon(0);

#endif

#ifdef CONFIG_KDB

                        kdb(KDB_REASON_CALL,0, (kdb_eframe_t) 0);

#endif

#ifdef CONFIG_PPC_ISERIES

                        HvCall_terminateMachineSrc();

#endif

                        goto out;

                }

                barrier();

                udelay(1);

        }

        if (wait) {

                timeout = 1000000;

                while (atomic_read(&data.finished) != cpus) {

                        HMT_low();

                        if (--timeout == 0) {

                                printk("smp_call_function on cpu %d: other cpus not finishing (%d/%d)\n",

                                       smp_processor_id(), atomic_read(&data.finished), atomic_read(&data.started));

#ifdef CONFIG_PPC_ISERIES

                                HvCall_terminateMachineSrc();

#endif

                                goto out;

                        }

                        barrier();

                        udelay(1);

                }

        }

        ret = 0;

 out:

        call_data = NULL;

        HMT_medium();

        spin_unlock_bh(&call_lock);

        return ret;

}

void smp_call_function_interrupt(void)

{

        void (*func) (void *info);

        void *info;

        int wait;

        /* call_data will be NULL if the sender timed out while

         * waiting on us to receive the call.

         */

        if (!call_data)

                return;

        func = call_data->func;

        info = call_data->info;

        wait = call_data->wait;

        /*

         * Notify initiating CPU that I've grabbed the data and am

         * about to execute the function

         */

        atomic_inc(&call_data->started);

        /*

         * At this point the info structure may be out of scope unless wait==1

         */

        (*func)(info);

        if (wait)

                atomic_inc(&call_data->finished);

}

extern unsigned long decr_overclock;

void __init smp_boot_cpus(void)

{

        extern struct current_set_struct current_set[];

        extern void __secondary_start_chrp(void);

        int i, cpu_nr;

        struct task_struct *p;

        unsigned long sp;

        printk("Entering SMP Mode...\n");

        PPCDBG(PPCDBG_SMP, "smp_boot_cpus: start.  NR_CPUS = 0x%lx\n", NR_CPUS);

        smp_num_cpus = 1;

        smp_store_cpu_info(0);

        cpu_online_map = 1UL;

        /*

         * assume for now that the first cpu booted is

         * cpu 0, the master -- Cort

         */

        cpu_callin_map[0] = 1;

        current->processor = 0;

        init_idle();

        for (i = 0; i < NR_CPUS; i++) {

                paca[i].prof_counter = 1;

                paca[i].prof_multiplier = 1;

                if(i != 0) {

                        /*

                         * Processor 0's segment table is statically

                         * initialized to real address STAB0_PHYS_ADDR.  The

                         * Other processor's tables are created and

                         * initialized here.

                         */

                        paca[i].xStab_data.virt = (unsigned long)&stab_array[PAGE_SIZE * (i-1)];

                        memset((void *)paca[i].xStab_data.virt, 0, PAGE_SIZE);

                        paca[i].xStab_data.real = __v2a(paca[i].xStab_data.virt);

                        paca[i].default_decr = tb_ticks_per_jiffy / decr_overclock;

                }

        }

        /*

         * XXX very rough, assumes 20 bus cycles to read a cache line,

         * timebase increments every 4 bus cycles, 32kB L1 data cache.

         */

        cacheflush_time = 5 * 1024;

        /* Probe arch for CPUs */

        cpu_nr = ppc_md.smp_probe();

        printk("Probe found %d CPUs\n", cpu_nr);

        /*

         * only check for cpus we know exist.  We keep the callin map

         * with cpus at the bottom -- Cort

         */

        if (cpu_nr > max_cpus)

                cpu_nr = max_cpus;

#ifdef CONFIG_PPC_ISERIES

        smp_space_timers( cpu_nr );

#endif

        printk("Waiting for %d CPUs\n", cpu_nr-1);

        for ( i = 1 ; i < cpu_nr; i++ ) {