From: Rusty Russell Subject: cpumask: prepare for reduced cpumask_allocation. Thomas and Linus already made CONFIG_CPUMASK_OFFSTACK use a cpumask at the end of struct mm_struct, this just changes it into a bitmap and allocates it using cpumask_size(). This means it will shrink when cpumask_size() is changed to reflect nr_cpu_ids not NR_CPUS. Signed-off-by: Rusty Russell Cc: Thomas Gleixner Cc: Linus Torvalds Cc: Arnd Bergmann Cc: anton@samba.org Cc: KOSAKI Motohiro Cc: Mike Travis --- include/linux/mm_types.h | 21 +++++++++++++++++++-- kernel/fork.c | 13 +++---------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -386,14 +386,15 @@ struct mm_struct { pgtable_t pmd_huge_pte; /* protected by page_table_lock */ #endif #ifdef CONFIG_CPUMASK_OFFSTACK - struct cpumask cpumask_allocation; + /* This must be at the end! */ + DECLARE_BITMAP(cpumask_allocation, CONFIG_NR_CPUS); #endif }; static inline void mm_init_cpumask(struct mm_struct *mm) { #ifdef CONFIG_CPUMASK_OFFSTACK - mm->cpu_vm_mask_var = &mm->cpumask_allocation; + mm->cpu_vm_mask_var = to_cpumask(mm->cpumask_allocation); #endif } @@ -403,4 +404,20 @@ static inline cpumask_t *mm_cpumask(stru return mm->cpu_vm_mask_var; } +static inline size_t mm_struct_size(void) +{ +#ifdef CONFIG_CPUMASK_OFFSTACK + /* + * We reduce mm_struct allocations as cpu_vm_mask_var only + * needs cpumask_size() bytes. cpu_vm_mask must be a NR_CPUS + * bitmap at the end for this to work. + */ + BUILD_BUG_ON(offsetof(struct mm_struct, cpumask_allocation) + + BITS_TO_LONGS(CONFIG_NR_CPUS)*sizeof(long) + != sizeof(struct mm_struct)); + return offsetof(struct mm_struct, cpumask_allocation) + cpumask_size(); +#else + return sizeof(struct mm_struct); +#endif +} #endif /* _LINUX_MM_TYPES_H */ diff --git a/kernel/fork.c b/kernel/fork.c --- a/kernel/fork.c +++ b/kernel/fork.c @@ -521,7 +521,7 @@ struct mm_struct *mm_alloc(void) if (!mm) return NULL; - memset(mm, 0, sizeof(*mm)); + memset(mm, 0, mm_struct_size()); mm_init_cpumask(mm); return mm_init(mm, current); } @@ -746,7 +746,7 @@ struct mm_struct *dup_mm(struct task_str if (!mm) goto fail_nomem; - memcpy(mm, oldmm, sizeof(*mm)); + memcpy(mm, oldmm, mm_struct_size()); mm_init_cpumask(mm); /* Initializing for Swap token stuff */ @@ -1600,15 +1600,8 @@ void __init proc_caches_init(void) fs_cachep = kmem_cache_create("fs_cache", sizeof(struct fs_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL); - /* - * FIXME! The "sizeof(struct mm_struct)" currently includes the - * whole struct cpumask for the OFFSTACK case. We could change - * this to *only* allocate as much of it as required by the - * maximum number of CPU's we can ever have. The cpumask_allocation - * is at the end of the structure, exactly for that reason. - */ mm_cachep = kmem_cache_create("mm_struct", - sizeof(struct mm_struct), ARCH_MIN_MMSTRUCT_ALIGN, + mm_struct_size(), ARCH_MIN_MMSTRUCT_ALIGN, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_NOTRACK, NULL); vm_area_cachep = KMEM_CACHE(vm_area_struct, SLAB_PANIC); mmap_init();