per_cpu: Remove per_cpu__ prefix. Now that the return from alloc_percpu is compatible with the address of per-cpu vars, it makes sense to hand around the address of per-cpu variables. To make this sane, we remove the per_cpu__ prefix we used created to stop people accidentally using these vars directly. Now we have sparse, we can use that (next patch). Note that now DECLARE_PER_CPU vars are in the same namespace as everything else, we have to rename some of them. I probably missed some others. Signed-off-by: Rusty Russell --- arch/alpha/include/asm/percpu.h | 4 ++-- arch/cris/arch-v10/kernel/entry.S | 2 +- arch/cris/arch-v32/mm/mmu.S | 2 +- arch/ia64/include/asm/percpu.h | 4 ++-- arch/ia64/kernel/ia64_ksyms.c | 4 ++-- arch/ia64/mm/discontig.c | 2 +- arch/parisc/lib/fixup.S | 8 ++++---- arch/powerpc/platforms/pseries/hvCall.S | 2 +- arch/sparc64/kernel/rtrap.S | 8 ++++---- arch/x86/include/asm/percpu.h | 20 ++++++++++---------- arch/x86/include/asm/timer.h | 5 +++-- arch/x86/kernel/entry_64.S | 4 ++-- arch/x86/kernel/head_32.S | 2 +- arch/x86/kernel/head_64.S | 2 +- arch/x86/kernel/tsc.c | 4 ++-- arch/x86/xen/xen-asm_32.S | 4 ++-- include/asm-generic/percpu.h | 2 +- include/linux/percpu.h | 12 ++++++------ kernel/sched.c | 5 +++-- kernel/softirq.c | 4 ++-- kernel/softlockup.c | 12 ++++++------ mm/vmstat.c | 6 +++--- 22 files changed, 60 insertions(+), 58 deletions(-) diff --git a/arch/alpha/include/asm/percpu.h b/arch/alpha/include/asm/percpu.h --- a/arch/alpha/include/asm/percpu.h +++ b/arch/alpha/include/asm/percpu.h @@ -7,7 +7,7 @@ * Determine the real variable name from the name visible in the * kernel sources. */ -#define per_cpu_var(var) per_cpu__##var +#define per_cpu_var(var) var #ifdef CONFIG_SMP @@ -43,7 +43,7 @@ extern unsigned long __per_cpu_offset[NR unsigned long __ptr, tmp_gp; \ asm ( "br %1, 1f \n\ 1: ldgp %1, 0(%1) \n\ - ldq %0, per_cpu__" #var"(%1)\t!literal" \ + ldq %0, "#var"(%1)\t!literal" \ : "=&r"(__ptr), "=&r"(tmp_gp)); \ (typeof(&per_cpu_var(var)))(__ptr + (offset)); }) diff --git a/arch/cris/arch-v10/kernel/entry.S b/arch/cris/arch-v10/kernel/entry.S --- a/arch/cris/arch-v10/kernel/entry.S +++ b/arch/cris/arch-v10/kernel/entry.S @@ -358,7 +358,7 @@ 1: btstq 12, $r1 ; Refill? 1: btstq 12, $r1 ; Refill? bpl 2f lsrq 24, $r1 ; Get PGD index (bit 24-31) - move.d [per_cpu__current_pgd], $r0 ; PGD for the current process + move.d [current_pgd], $r0 ; PGD for the current process move.d [$r0+$r1.d], $r0 ; Get PMD beq 2f nop diff --git a/arch/cris/arch-v32/mm/mmu.S b/arch/cris/arch-v32/mm/mmu.S --- a/arch/cris/arch-v32/mm/mmu.S +++ b/arch/cris/arch-v32/mm/mmu.S @@ -115,7 +115,7 @@ 3: ; Probably not in a loop, continue no #ifdef CONFIG_SMP move $s7, $acr ; PGD #else - move.d per_cpu__current_pgd, $acr ; PGD + move.d current_pgd, $acr ; PGD #endif ; Look up PMD in PGD lsrq 24, $r0 ; Get PMD index into PGD (bit 24-31) diff --git a/arch/ia64/include/asm/percpu.h b/arch/ia64/include/asm/percpu.h --- a/arch/ia64/include/asm/percpu.h +++ b/arch/ia64/include/asm/percpu.h @@ -9,7 +9,7 @@ #define PERCPU_ENOUGH_ROOM PERCPU_PAGE_SIZE #ifdef __ASSEMBLY__ -# define THIS_CPU(var) (per_cpu__##var) /* use this to mark accesses to per-CPU variables... */ +# define THIS_CPU(var) (var) /* use this to mark accesses to per-CPU variables... */ #else /* !__ASSEMBLY__ */ @@ -39,7 +39,7 @@ extern void *per_cpu_init(void); * On the positive side, using __ia64_per_cpu_var() instead of __get_cpu_var() is slightly * more efficient. */ -#define __ia64_per_cpu_var(var) per_cpu__##var +#define __ia64_per_cpu_var(var) var #define read_percpu_var(var) (0, __ia64_per_cpu_var(var)) #include diff --git a/arch/ia64/kernel/ia64_ksyms.c b/arch/ia64/kernel/ia64_ksyms.c --- a/arch/ia64/kernel/ia64_ksyms.c +++ b/arch/ia64/kernel/ia64_ksyms.c @@ -29,9 +29,9 @@ EXPORT_SYMBOL(max_low_pfn); /* defined b #endif #include -EXPORT_SYMBOL(per_cpu__cpu_info); +EXPORT_SYMBOL(cpu_info); #ifdef CONFIG_SMP -EXPORT_SYMBOL(per_cpu__local_per_cpu_offset); +EXPORT_SYMBOL(local_per_cpu_offset); #endif #include diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c --- a/arch/ia64/mm/discontig.c +++ b/arch/ia64/mm/discontig.c @@ -360,7 +360,7 @@ static void __init initialize_pernode_da cpu = 0; node = node_cpuid[cpu].nid; cpu0_cpu_info = (struct cpuinfo_ia64 *)(__phys_per_cpu_start + - ((char *)&per_cpu__cpu_info - __per_cpu_start)); + ((char *)&cpu_info - __per_cpu_start)); cpu0_cpu_info->node_data = mem_data[node].node_data; } #endif /* CONFIG_SMP */ diff --git a/arch/parisc/lib/fixup.S b/arch/parisc/lib/fixup.S --- a/arch/parisc/lib/fixup.S +++ b/arch/parisc/lib/fixup.S @@ -36,8 +36,8 @@ #endif /* t2 = &__per_cpu_offset[smp_processor_id()]; */ LDREGX \t2(\t1),\t2 - addil LT%per_cpu__exception_data,%r27 - LDREG RT%per_cpu__exception_data(%r1),\t1 + addil LT%exception_data,%r27 + LDREG RT%exception_data(%r1),\t1 /* t1 = &__get_cpu_var(exception_data) */ add,l \t1,\t2,\t1 /* t1 = t1->fault_ip */ @@ -46,8 +46,8 @@ #else .macro get_fault_ip t1 t2 /* t1 = &__get_cpu_var(exception_data) */ - addil LT%per_cpu__exception_data,%r27 - LDREG RT%per_cpu__exception_data(%r1),\t2 + addil LT%exception_data,%r27 + LDREG RT%exception_data(%r1),\t2 /* t1 = t2->fault_ip */ LDREG EXCDATA_IP(\t2), \t1 .endm diff --git a/arch/powerpc/platforms/pseries/hvCall.S b/arch/powerpc/platforms/pseries/hvCall.S --- a/arch/powerpc/platforms/pseries/hvCall.S +++ b/arch/powerpc/platforms/pseries/hvCall.S @@ -55,7 +55,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_PURR); /* calculate address of stat structure r4 = opcode */ \ srdi r4,r4,2; /* index into array */ \ mulli r4,r4,HCALL_STAT_SIZE; \ - LOAD_REG_ADDR(r7, per_cpu__hcall_stats); \ + LOAD_REG_ADDR(r7, hcall_stats); \ add r4,r4,r7; \ ld r7,PACA_DATA_OFFSET(r13); /* per cpu offset */ \ add r4,r4,r7; \ diff --git a/arch/sparc64/kernel/rtrap.S b/arch/sparc64/kernel/rtrap.S --- a/arch/sparc64/kernel/rtrap.S +++ b/arch/sparc64/kernel/rtrap.S @@ -137,11 +137,11 @@ rtrap_irq: rtrap_irq: rtrap: #ifndef CONFIG_SMP - sethi %hi(per_cpu____cpu_data), %l0 - lduw [%l0 + %lo(per_cpu____cpu_data)], %l1 + sethi %hi(__cpu_data), %l0 + lduw [%l0 + %lo(__cpu_data)], %l1 #else - sethi %hi(per_cpu____cpu_data), %l0 - or %l0, %lo(per_cpu____cpu_data), %l0 + sethi %hi(__cpu_data), %l0 + or %l0, %lo(__cpu_data), %l0 lduw [%l0 + %g5], %l1 #endif cmp %l1, 0 diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -66,13 +66,13 @@ DECLARE_PER_CPU(struct x8664_pda, pda); */ #ifdef CONFIG_SMP #define PER_CPU(var, reg) \ - movl %fs:per_cpu__##this_cpu_off, reg; \ - lea per_cpu__##var(reg), reg -#define PER_CPU_VAR(var) %fs:per_cpu__##var + movl %fs:this_cpu_off, reg; \ + lea var(reg), reg +#define PER_CPU_VAR(var) %fs:var #else /* ! SMP */ #define PER_CPU(var, reg) \ - movl $per_cpu__##var, reg -#define PER_CPU_VAR(var) per_cpu__##var + movl $var, reg +#define PER_CPU_VAR(var) var #endif /* SMP */ #else /* ...!ASSEMBLY */ @@ -166,11 +166,11 @@ do { \ ret__; \ }) -#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var) -#define x86_write_percpu(var, val) percpu_to_op("mov", per_cpu__##var, val) -#define x86_add_percpu(var, val) percpu_to_op("add", per_cpu__##var, val) -#define x86_sub_percpu(var, val) percpu_to_op("sub", per_cpu__##var, val) -#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val) +#define x86_read_percpu(var) percpu_from_op("mov", var) +#define x86_write_percpu(var, val) percpu_to_op("mov", var, val) +#define x86_add_percpu(var, val) percpu_to_op("add", var, val) +#define x86_sub_percpu(var, val) percpu_to_op("sub", var, val) +#define x86_or_percpu(var, val) percpu_to_op("or", var, val) #endif /* !__ASSEMBLY__ */ #endif /* !CONFIG_X86_64 */ diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h --- a/arch/x86/include/asm/timer.h +++ b/arch/x86/include/asm/timer.h @@ -42,13 +42,14 @@ extern int no_timer_check; * -johnstul@us.ibm.com "math is hard, lets go shopping!" */ -DECLARE_PER_CPU(unsigned long, cyc2ns); +DECLARE_PER_CPU(unsigned long, percpu_cyc2ns); #define CYC2NS_SCALE_FACTOR 10 /* 2^10, carefully chosen */ static inline unsigned long long __cycles_2_ns(unsigned long long cyc) { - return cyc * per_cpu(cyc2ns, smp_processor_id()) >> CYC2NS_SCALE_FACTOR; + return cyc * per_cpu(percpu_cyc2ns, smp_processor_id()) >> + CYC2NS_SCALE_FACTOR; } static inline unsigned long long cycles_2_ns(unsigned long long cyc) diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -1073,9 +1073,9 @@ ENTRY(\sym) movq %rsp,%rdi /* pt_regs pointer */ xorl %esi,%esi /* no error code */ movq %gs:pda_data_offset, %rbp - subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp) + subq $EXCEPTION_STKSZ, init_tss + TSS_ist + (\ist - 1) * 8(%rbp) call \do_sym - addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp) + addq $EXCEPTION_STKSZ, init_tss + TSS_ist + (\ist - 1) * 8(%rbp) jmp paranoid_exit /* %ebx: no swapgs flag */ CFI_ENDPROC END(\sym) diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S --- a/arch/x86/kernel/head_32.S +++ b/arch/x86/kernel/head_32.S @@ -702,7 +702,7 @@ idt_descr: .word 0 # 32 bit align gdt_desc.address ENTRY(early_gdt_descr) .word GDT_ENTRIES*8-1 - .long per_cpu__gdt_page /* Overwritten for secondary CPUs */ + .long gdt_page /* Overwritten for secondary CPUs */ /* * The boot_gdt must mirror the equivalent in setup.S and is diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -401,7 +401,7 @@ NEXT_PAGE(level2_spare_pgt) .globl early_gdt_descr early_gdt_descr: .word GDT_ENTRIES*8-1 - .quad per_cpu__gdt_page + .quad gdt_page ENTRY(phys_base) /* This must match the first entry in level2_kernel_pgt */ diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -570,7 +570,7 @@ EXPORT_SYMBOL(recalibrate_cpu_khz); * -johnstul@us.ibm.com "math is hard, lets go shopping!" */ -DEFINE_PER_CPU(unsigned long, cyc2ns); +DEFINE_PER_CPU(unsigned long, percpu_cyc2ns); static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu) { @@ -580,7 +580,7 @@ static void set_cyc2ns_scale(unsigned lo local_irq_save(flags); sched_clock_idle_sleep_event(); - scale = &per_cpu(cyc2ns, cpu); + scale = &per_cpu(percpu_cyc2ns, cpu); rdtscll(tsc_now); ns_now = __cycles_2_ns(tsc_now); diff --git a/arch/x86/xen/xen-asm_32.S b/arch/x86/xen/xen-asm_32.S --- a/arch/x86/xen/xen-asm_32.S +++ b/arch/x86/xen/xen-asm_32.S @@ -164,9 +164,9 @@ ENTRY(xen_iret) GET_THREAD_INFO(%eax) movl TI_cpu(%eax),%eax movl __per_cpu_offset(,%eax,4),%eax - mov per_cpu__xen_vcpu(%eax),%eax + mov xen_vcpu(%eax),%eax #else - movl per_cpu__xen_vcpu, %eax + movl xen_vcpu, %eax #endif /* check IF state we're restoring */ diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h --- a/include/asm-generic/percpu.h +++ b/include/asm-generic/percpu.h @@ -7,7 +7,7 @@ * Determine the real variable name from the name visible in the * kernel sources. */ -#define per_cpu_var(var) per_cpu__##var +#define per_cpu_var(var) var #ifdef CONFIG_SMP diff --git a/include/linux/percpu.h b/include/linux/percpu.h --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -11,7 +11,7 @@ #ifdef CONFIG_SMP #define DEFINE_PER_CPU(type, name) \ __attribute__((__section__(".data.percpu"))) \ - PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name + PER_CPU_ATTRIBUTES __typeof__(type) name #ifdef MODULE #define SHARED_ALIGNED_SECTION ".data.percpu" @@ -21,15 +21,15 @@ #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ __attribute__((__section__(SHARED_ALIGNED_SECTION))) \ - PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name \ + PER_CPU_ATTRIBUTES __typeof__(type) name \ ____cacheline_aligned_in_smp #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \ __attribute__((__section__(".data.percpu.page_aligned"))) \ - PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name + PER_CPU_ATTRIBUTES __typeof__(type) name #else #define DEFINE_PER_CPU(type, name) \ - PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name + PER_CPU_ATTRIBUTES __typeof__(type) name #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ DEFINE_PER_CPU(type, name) @@ -38,8 +38,8 @@ DEFINE_PER_CPU(type, name) #endif -#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) -#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var) +#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(var) +#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(var) #ifndef PERCPU_ENOUGH_ROOM extern unsigned int percpu_reserve; diff --git a/kernel/sched.c b/kernel/sched.c --- a/kernel/sched.c +++ b/kernel/sched.c @@ -313,7 +313,8 @@ struct task_group root_task_group; /* Default task group's sched entity on each cpu */ static DEFINE_PER_CPU(struct sched_entity, init_sched_entity); /* Default task group's cfs_rq on each cpu */ -static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp; +static DEFINE_PER_CPU(struct cfs_rq, percpu_init_cfs_rq) + ____cacheline_aligned_in_smp; #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_RT_GROUP_SCHED @@ -8260,7 +8261,7 @@ void __init sched_init(void) * tasks in rq->cfs (i.e init_task_group->se[] != NULL). */ init_tg_cfs_entry(&init_task_group, - &per_cpu(init_cfs_rq, i), + &per_cpu(percpu_init_cfs_rq, i), &per_cpu(init_sched_entity, i), i, 1, root_task_group.se[i]); diff --git a/kernel/softirq.c b/kernel/softirq.c --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -615,7 +615,7 @@ void __init softirq_init(void) open_softirq(HI_SOFTIRQ, tasklet_hi_action); } -static int ksoftirqd(void * __bind_cpu) +static int run_ksoftirqd(void *__bind_cpu) { set_current_state(TASK_INTERRUPTIBLE); @@ -727,7 +727,7 @@ static int __cpuinit cpu_callback(struct switch (action) { case CPU_UP_PREPARE: case CPU_UP_PREPARE_FROZEN: - p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu); + p = kthread_create(run_ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu); if (IS_ERR(p)) { printk("ksoftirqd for %i failed\n", hotcpu); return NOTIFY_BAD; diff --git a/kernel/softlockup.c b/kernel/softlockup.c --- a/kernel/softlockup.c +++ b/kernel/softlockup.c @@ -95,15 +95,15 @@ void softlockup_tick(void) void softlockup_tick(void) { int this_cpu = smp_processor_id(); - unsigned long touch_timestamp = per_cpu(touch_timestamp, this_cpu); - unsigned long print_timestamp; + unsigned long touch_ts = per_cpu(touch_timestamp, this_cpu); + unsigned long print_ts; struct pt_regs *regs = get_irq_regs(); unsigned long now; /* Is detection switched off? */ if (!per_cpu(watchdog_task, this_cpu) || softlockup_thresh <= 0) { /* Be sure we don't false trigger if switched back on */ - if (touch_timestamp) + if (touch_ts) per_cpu(touch_timestamp, this_cpu) = 0; return; } @@ -113,10 +113,10 @@ void softlockup_tick(void) return; } - print_timestamp = per_cpu(print_timestamp, this_cpu); + print_ts = per_cpu(print_timestamp, this_cpu); /* report at most once a second */ - if (print_timestamp == touch_timestamp || did_panic) + if (print_ts == touch_timestamp || did_panic) return; /* do not print during early bootup: */ @@ -138,7 +138,7 @@ void softlockup_tick(void) if (now <= (touch_timestamp + softlockup_thresh)) return; - per_cpu(print_timestamp, this_cpu) = touch_timestamp; + per_cpu(print_ts, this_cpu) = touch_timestamp; spin_lock(&print_lock); printk(KERN_ERR "BUG: soft lockup - CPU#%d stuck for %lus! [%s:%d]\n", diff --git a/mm/vmstat.c b/mm/vmstat.c --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -903,10 +903,10 @@ static void vmstat_update(struct work_st static void __cpuinit start_cpu_timer(int cpu) { - struct delayed_work *vmstat_work = &per_cpu(vmstat_work, cpu); + struct delayed_work *vw = &per_cpu(vmstat_work, cpu); - INIT_DELAYED_WORK_DEFERRABLE(vmstat_work, vmstat_update); - schedule_delayed_work_on(cpu, vmstat_work, HZ + cpu); + INIT_DELAYED_WORK_DEFERRABLE(vw, vmstat_update); + schedule_delayed_work_on(cpu, vw, HZ + cpu); } /*