cpumask: convert kernel/cpuset.c The biggest change here is that partition_sched_domains() and generate_sched_domains() used to use an array of cpumask_t's. This is no longer valid (since cpumasks are variable size, and will eventually be an undefined type), so it needs to be an array of pointers. In theory, it could be an array of cpumask_var_t's, but this is simple and we're already kmallocing them in this code anyway. Signed-off-by: Rusty Russell Signed-off-by: Mike Travis --- include/linux/cpuset.h | 13 ++-- include/linux/sched.h | 2 kernel/cpuset.c | 143 +++++++++++++++++++++++++++---------------------- kernel/sched.c | 36 +++++++----- 4 files changed, 112 insertions(+), 82 deletions(-) --- linux-2.6.28.orig/include/linux/cpuset.h +++ linux-2.6.28/include/linux/cpuset.h @@ -20,8 +20,8 @@ extern int number_of_cpusets; /* How man extern int cpuset_init_early(void); extern int cpuset_init(void); extern void cpuset_init_smp(void); -extern void cpuset_cpus_allowed(struct task_struct *p, cpumask_t *mask); -extern void cpuset_cpus_allowed_locked(struct task_struct *p, cpumask_t *mask); +extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); +extern void cpuset_cpus_allowed_locked(struct task_struct *p, struct cpumask *); extern nodemask_t cpuset_mems_allowed(struct task_struct *p); #define cpuset_current_mems_allowed (current->mems_allowed) void cpuset_init_current_mems_allowed(void); @@ -86,14 +86,15 @@ static inline int cpuset_init_early(void static inline int cpuset_init(void) { return 0; } static inline void cpuset_init_smp(void) {} -static inline void cpuset_cpus_allowed(struct task_struct *p, cpumask_t *mask) +static inline void cpuset_cpus_allowed(struct task_struct *p, + struct cpumask *mask) { - *mask = cpu_possible_map; + cpumask_copy(mask, cpu_possible_mask); } static inline void cpuset_cpus_allowed_locked(struct task_struct *p, - cpumask_t *mask) + struct cpumask *mask) { - *mask = cpu_possible_map; + cpumask_copy(mask, cpu_possible_mask); } static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) --- linux-2.6.28.orig/include/linux/sched.h +++ linux-2.6.28/include/linux/sched.h @@ -871,7 +871,7 @@ struct sched_domain { #endif }; -extern void partition_sched_domains(int ndoms_new, struct cpumask *doms_new, +extern void partition_sched_domains(int ndoms_new, struct cpumask *doms_new[], struct sched_domain_attr *dattr_new); extern int arch_reinit_sched_domains(void); --- linux-2.6.28.orig/kernel/cpuset.c +++ linux-2.6.28/kernel/cpuset.c @@ -83,7 +83,8 @@ struct cpuset { struct cgroup_subsys_state css; unsigned long flags; /* "unsigned long" so bitops work */ - cpumask_t cpus_allowed; /* CPUs allowed to tasks in cpuset */ + struct cpumask *cpus_allowed; /* CPUs allowed to tasks in cpuset */ + nodemask_t mems_allowed; /* Memory Nodes allowed to tasks */ struct cpuset *parent; /* my parent */ @@ -194,7 +195,7 @@ static int cpuset_mems_generation; static struct cpuset top_cpuset = { .flags = ((1 << CS_CPU_EXCLUSIVE) | (1 << CS_MEM_EXCLUSIVE)), - .cpus_allowed = CPU_MASK_ALL, + .cpus_allowed = cpu_all_mask, .mems_allowed = NODE_MASK_ALL, }; @@ -270,24 +271,24 @@ static struct file_system_type cpuset_fs * are online. If none are online, walk up the cpuset hierarchy * until we find one that does have some online cpus. If we get * all the way to the top and still haven't found any online cpus, - * return cpu_online_map. Or if passed a NULL cs from an exit'ing - * task, return cpu_online_map. + * return cpu_online_mask. Or if passed a NULL cs from an exit'ing + * task, return cpu_online_mask. * * One way or another, we guarantee to return some non-empty subset - * of cpu_online_map. + * of cpu_online_mask. * * Call with callback_mutex held. */ -static void guarantee_online_cpus(const struct cpuset *cs, cpumask_t *pmask) +static void guarantee_online_cpus(const struct cpuset *cs, struct cpumask *mask) { - while (cs && !cpus_intersects(cs->cpus_allowed, cpu_online_map)) + while (cs && !cpumask_intersects(cs->cpus_allowed, cpu_online_mask)) cs = cs->parent; if (cs) - cpus_and(*pmask, cs->cpus_allowed, cpu_online_map); + cpumask_and(mask, cs->cpus_allowed, cpu_online_mask); else - *pmask = cpu_online_map; - BUG_ON(!cpus_intersects(*pmask, cpu_online_map)); + cpumask_copy(mask, cpu_online_mask); + BUG_ON(!cpumask_intersects(mask, cpu_online_mask)); } /* @@ -402,7 +403,7 @@ void cpuset_update_task_memory_state(voi static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q) { - return cpus_subset(p->cpus_allowed, q->cpus_allowed) && + return cpumask_subset(p->cpus_allowed, q->cpus_allowed) && nodes_subset(p->mems_allowed, q->mems_allowed) && is_cpu_exclusive(p) <= is_cpu_exclusive(q) && is_mem_exclusive(p) <= is_mem_exclusive(q); @@ -457,7 +458,7 @@ static int validate_change(const struct c = cgroup_cs(cont); if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) && c != cur && - cpus_intersects(trial->cpus_allowed, c->cpus_allowed)) + cpumask_intersects(trial->cpus_allowed, c->cpus_allowed)) return -EINVAL; if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) && c != cur && @@ -467,7 +468,7 @@ static int validate_change(const struct /* Cpusets with tasks can't have empty cpus_allowed or mems_allowed */ if (cgroup_task_count(cur->css.cgroup)) { - if (cpus_empty(trial->cpus_allowed) || + if (cpumask_empty(trial->cpus_allowed) || nodes_empty(trial->mems_allowed)) { return -ENOSPC; } @@ -482,7 +483,7 @@ static int validate_change(const struct */ static int cpusets_overlap(struct cpuset *a, struct cpuset *b) { - return cpus_intersects(a->cpus_allowed, b->cpus_allowed); + return cpumask_intersects(a->cpus_allowed, b->cpus_allowed); } static void @@ -507,7 +508,7 @@ update_domain_attr_tree(struct sched_dom cp = list_first_entry(&q, struct cpuset, stack_list); list_del(q.next); - if (cpus_empty(cp->cpus_allowed)) + if (cpumask_empty(cp->cpus_allowed)) continue; if (is_sched_load_balance(cp)) @@ -574,7 +575,7 @@ update_domain_attr_tree(struct sched_dom * element of the partition (one sched domain) to be passed to * partition_sched_domains(). */ -static int generate_sched_domains(cpumask_t **domains, +static int generate_sched_domains(struct cpumask ***domains, struct sched_domain_attr **attributes) { LIST_HEAD(q); /* queue of cpusets to be scanned */ @@ -582,10 +583,10 @@ static int generate_sched_domains(cpumas struct cpuset **csa; /* array of all cpuset ptrs */ int csn; /* how many cpuset ptrs in csa so far */ int i, j, k; /* indices for partition finding loops */ - cpumask_t *doms; /* resulting partition; i.e. sched domains */ + struct cpumask **doms; /* resulting partition; i.e. sched domains */ struct sched_domain_attr *dattr; /* attributes for custom domains */ int ndoms; /* number of sched domains in result */ - int nslot; /* next empty doms[] cpumask_t slot */ + int nslot; /* next empty doms[] cpumask slot */ ndoms = 0; doms = NULL; @@ -594,7 +595,7 @@ static int generate_sched_domains(cpumas /* Special case for the 99% of systems with one, full, sched domain */ if (is_sched_load_balance(&top_cpuset)) { - doms = kmalloc(sizeof(cpumask_t), GFP_KERNEL); + doms = kmalloc(sizeof(struct cpumask *), GFP_KERNEL); if (!doms) goto done; @@ -622,7 +623,7 @@ static int generate_sched_domains(cpumas cp = list_first_entry(&q, struct cpuset, stack_list); list_del(q.next); - if (cpus_empty(cp->cpus_allowed)) + if (cpumask_empty(cp->cpus_allowed)) continue; /* @@ -673,11 +674,19 @@ restart: * Now we know how many domains to create. * Convert to and populate cpu masks. */ - doms = kmalloc(ndoms * sizeof(cpumask_t), GFP_KERNEL); + doms = kmalloc(ndoms * sizeof(struct cpumask *), GFP_KERNEL); if (!doms) { ndoms = 0; goto done; } + doms[0] = kmalloc(ndoms * cpumask_size(), GFP_KERNEL); + if (!doms[0]) { + kfree(doms); + ndoms = 0; + goto done; + } + for (i = 1; i < ndoms; i++) + doms[i] = (void *)doms[i-1] + cpumask_size(); /* * The rest of the code, including the scheduler, can deal with @@ -687,7 +696,7 @@ restart: for (nslot = 0, i = 0; i < csn; i++) { struct cpuset *a = csa[i]; - cpumask_t *dp; + struct cpumask *dp; int apn = a->pn; if (apn < 0) { @@ -695,7 +704,7 @@ restart: continue; } - dp = doms + nslot; + dp = doms[nslot]; if (nslot == ndoms) { static int warnings = 10; @@ -710,14 +719,14 @@ restart: continue; } - cpus_clear(*dp); + cpumask_clear(dp); if (dattr) *(dattr + nslot) = SD_ATTR_INIT; for (j = i; j < csn; j++) { struct cpuset *b = csa[j]; if (apn == b->pn) { - cpus_or(*dp, *dp, b->cpus_allowed); + cpumask_or(dp, dp, b->cpus_allowed); if (dattr) update_domain_attr_tree(dattr + nslot, b); @@ -750,7 +759,7 @@ done: static void do_rebuild_sched_domains(struct work_struct *unused) { struct sched_domain_attr *attr; - cpumask_t *doms; + struct cpumask **doms; int ndoms; get_online_cpus(); @@ -819,8 +828,8 @@ void rebuild_sched_domains(void) static int cpuset_test_cpumask(struct task_struct *tsk, struct cgroup_scanner *scan) { - return !cpus_equal(tsk->cpus_allowed, - (cgroup_cs(scan->cg))->cpus_allowed); + return !cpumask_equal(&tsk->cpus_allowed, + cgroup_cs(scan->cg)->cpus_allowed); } /** @@ -837,7 +846,7 @@ static int cpuset_test_cpumask(struct ta static void cpuset_change_cpumask(struct task_struct *tsk, struct cgroup_scanner *scan) { - set_cpus_allowed_ptr(tsk, &((cgroup_cs(scan->cg))->cpus_allowed)); + set_cpus_allowed_ptr(tsk, (cgroup_cs(scan->cg))->cpus_allowed); } /** @@ -876,7 +885,7 @@ static int update_cpumask(struct cpuset int retval; int is_load_balanced; - /* top_cpuset.cpus_allowed tracks cpu_online_map; it's read-only */ + /* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */ if (cs == &top_cpuset) return -EACCES; @@ -889,13 +898,13 @@ static int update_cpumask(struct cpuset * with tasks have cpus. */ if (!*buf) { - cpus_clear(trialcs.cpus_allowed); + cpumask_clear(trialcs.cpus_allowed); } else { - retval = cpulist_parse(buf, &trialcs.cpus_allowed); + retval = cpulist_parse(buf, trialcs.cpus_allowed); if (retval < 0) return retval; - if (!cpus_subset(trialcs.cpus_allowed, cpu_online_map)) + if (!cpumask_subset(trialcs.cpus_allowed, cpu_online_mask)) return -EINVAL; } retval = validate_change(cs, &trialcs); @@ -903,7 +912,7 @@ static int update_cpumask(struct cpuset return retval; /* Nothing to do if the cpus didn't change */ - if (cpus_equal(cs->cpus_allowed, trialcs.cpus_allowed)) + if (cpumask_equal(cs->cpus_allowed, trialcs.cpus_allowed)) return 0; retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL); @@ -1001,7 +1010,8 @@ static int update_tasks_nodemask(struct cpuset_being_rebound = cs; /* causes mpol_dup() rebind */ fudge = 10; /* spare mmarray[] slots */ - fudge += cpus_weight(cs->cpus_allowed); /* imagine one fork-bomb/cpu */ + fudge += cpumask_weight(cs->cpus_allowed); + /* imagine one fork-bomb/cpu */ retval = -ENOMEM; /* @@ -1151,7 +1161,8 @@ static int update_relax_domain_level(str if (val != cs->relax_domain_level) { cs->relax_domain_level = val; - if (!cpus_empty(cs->cpus_allowed) && is_sched_load_balance(cs)) + if (!cpumask_empty(cs->cpus_allowed) && + is_sched_load_balance(cs)) async_rebuild_sched_domains(); } @@ -1191,7 +1202,7 @@ static int update_flag(cpuset_flagbits_t cs->flags = trialcs.flags; mutex_unlock(&callback_mutex); - if (!cpus_empty(trialcs.cpus_allowed) && balance_flag_changed) + if (!cpumask_empty(trialcs.cpus_allowed) && balance_flag_changed) async_rebuild_sched_domains(); return 0; @@ -1301,15 +1312,15 @@ static int cpuset_can_attach(struct cgro { struct cpuset *cs = cgroup_cs(cont); - if (cpus_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed)) + if (cpumask_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed)) return -ENOSPC; if (tsk->flags & PF_THREAD_BOUND) { - cpumask_t mask; + bool equal; mutex_lock(&callback_mutex); - mask = cs->cpus_allowed; + equal = cpumask_equal(&tsk->cpus_allowed, cs->cpus_allowed); mutex_unlock(&callback_mutex); - if (!cpus_equal(tsk->cpus_allowed, mask)) + if (!equal) return -EINVAL; } @@ -1320,17 +1331,22 @@ static void cpuset_attach(struct cgroup_ struct cgroup *cont, struct cgroup *oldcont, struct task_struct *tsk) { - cpumask_t cpus; + cpumask_var_t cpus; nodemask_t from, to; struct mm_struct *mm; struct cpuset *cs = cgroup_cs(cont); struct cpuset *oldcs = cgroup_cs(oldcont); int err; + if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) { + printk(KERN_WARNING "cpuset_attach: out of memory!\n"); + return; + } mutex_lock(&callback_mutex); - guarantee_online_cpus(cs, &cpus); - err = set_cpus_allowed_ptr(tsk, &cpus); + guarantee_online_cpus(cs, cpus); + err = set_cpus_allowed_ptr(tsk, cpus); mutex_unlock(&callback_mutex); + free_cpumask_var(cpus); if (err) return; @@ -1471,13 +1487,13 @@ static int cpuset_write_resmask(struct c static int cpuset_sprintf_cpulist(char *page, struct cpuset *cs) { - cpumask_t mask; + int ret; mutex_lock(&callback_mutex); - mask = cs->cpus_allowed; + ret = cpulist_scnprintf(page, PAGE_SIZE, cs->cpus_allowed); mutex_unlock(&callback_mutex); - return cpulist_scnprintf(page, PAGE_SIZE, &mask); + return ret; } static int cpuset_sprintf_memlist(char *page, struct cpuset *cs) @@ -1747,7 +1763,7 @@ static struct cgroup_subsys_state *cpuse if (is_spread_slab(parent)) set_bit(CS_SPREAD_SLAB, &cs->flags); set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); - cpus_clear(cs->cpus_allowed); + cpumask_clear(cs->cpus_allowed); nodes_clear(cs->mems_allowed); cs->mems_generation = cpuset_mems_generation++; fmeter_init(&cs->fmeter); @@ -1812,7 +1828,7 @@ int __init cpuset_init(void) { int err = 0; - cpus_setall(top_cpuset.cpus_allowed); + cpumask_setall(top_cpuset.cpus_allowed); nodes_setall(top_cpuset.mems_allowed); fmeter_init(&top_cpuset.fmeter); @@ -1898,7 +1914,7 @@ static void remove_tasks_in_empty_cpuset * has online cpus, so can't be empty). */ parent = cs->parent; - while (cpus_empty(parent->cpus_allowed) || + while (cpumask_empty(parent->cpus_allowed) || nodes_empty(parent->mems_allowed)) parent = parent->parent; @@ -1939,7 +1955,7 @@ static void scan_for_empty_cpusets(struc } /* Continue past cpusets with all cpus, mems online */ - if (cpus_subset(cp->cpus_allowed, cpu_online_map) && + if (cpumask_subset(cp->cpus_allowed, cpu_online_mask) && nodes_subset(cp->mems_allowed, node_states[N_HIGH_MEMORY])) continue; @@ -1947,13 +1963,14 @@ static void scan_for_empty_cpusets(struc /* Remove offline cpus and mems from this cpuset. */ mutex_lock(&callback_mutex); - cpus_and(cp->cpus_allowed, cp->cpus_allowed, cpu_online_map); + cpumask_and(cp->cpus_allowed, + cp->cpus_allowed, cpu_online_mask); nodes_and(cp->mems_allowed, cp->mems_allowed, node_states[N_HIGH_MEMORY]); mutex_unlock(&callback_mutex); /* Move tasks from the empty cpuset to a parent */ - if (cpus_empty(cp->cpus_allowed) || + if (cpumask_empty(cp->cpus_allowed) || nodes_empty(cp->mems_allowed)) remove_tasks_in_empty_cpuset(cp); else { @@ -1970,7 +1987,7 @@ static void scan_for_empty_cpusets(struc * but making no active use of cpusets. * * This routine ensures that top_cpuset.cpus_allowed tracks - * cpu_online_map on each CPU hotplug (cpuhp) event. + * cpu_online_mask on each CPU hotplug (cpuhp) event. * * Called within get_online_cpus(). Needs to call cgroup_lock() * before calling generate_sched_domains(). @@ -1979,7 +1996,7 @@ static int cpuset_track_online_cpus(stru unsigned long phase, void *unused_cpu) { struct sched_domain_attr *attr; - cpumask_t *doms; + struct cpumask **doms; int ndoms; switch (phase) { @@ -1994,7 +2011,8 @@ static int cpuset_track_online_cpus(stru } cgroup_lock(); - top_cpuset.cpus_allowed = cpu_online_map; + /* We never manipulate top_cpuset, so this is safe. */ + top_cpuset.cpus_allowed = (struct cpumask *)cpu_online_mask; scan_for_empty_cpusets(&top_cpuset); ndoms = generate_sched_domains(&doms, &attr); cgroup_unlock(); @@ -2028,7 +2046,8 @@ void cpuset_track_online_nodes(void) void __init cpuset_init_smp(void) { - top_cpuset.cpus_allowed = cpu_online_map; + /* We never manipulate top_cpuset, so this is safe. */ + top_cpuset.cpus_allowed = (struct cpumask *)cpu_online_mask; top_cpuset.mems_allowed = node_states[N_HIGH_MEMORY]; hotcpu_notifier(cpuset_track_online_cpus, 0); @@ -2037,15 +2056,15 @@ void __init cpuset_init_smp(void) /** * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset. * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed. - * @pmask: pointer to cpumask_t variable to receive cpus_allowed set. + * @pmask: pointer to cpumask variable to receive cpus_allowed set. * - * Description: Returns the cpumask_t cpus_allowed of the cpuset + * Description: Returns the cpumask cpus_allowed of the cpuset * attached to the specified @tsk. Guaranteed to return some non-empty - * subset of cpu_online_map, even if this means going outside the + * subset of cpu_online_mask, even if this means going outside the * tasks cpuset. **/ -void cpuset_cpus_allowed(struct task_struct *tsk, cpumask_t *pmask) +void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask) { mutex_lock(&callback_mutex); cpuset_cpus_allowed_locked(tsk, pmask); @@ -2056,7 +2075,7 @@ void cpuset_cpus_allowed(struct task_str * cpuset_cpus_allowed_locked - return cpus_allowed mask from a tasks cpuset. * Must be called with callback_mutex held. **/ -void cpuset_cpus_allowed_locked(struct task_struct *tsk, cpumask_t *pmask) +void cpuset_cpus_allowed_locked(struct task_struct *tsk, struct cpumask *pmask) { task_lock(tsk); guarantee_online_cpus(task_cs(tsk), pmask); --- linux-2.6.28.orig/kernel/sched.c +++ linux-2.6.28/kernel/sched.c @@ -7678,7 +7678,7 @@ static int build_sched_domains(const str return __build_sched_domains(cpu_map, NULL); } -static struct cpumask *doms_cur; /* current sched domains */ +static struct cpumask **doms_cur; /* current sched domains */ static int ndoms_cur; /* number of sched domains in 'doms_cur' */ static struct sched_domain_attr *dattr_cur; /* attribues of custom domains in 'doms_cur' */ @@ -7706,12 +7706,20 @@ static int arch_init_sched_domains(const arch_update_cpu_topology(); ndoms_cur = 1; - doms_cur = kmalloc(cpumask_size(), GFP_KERNEL); + doms_cur = kmalloc(sizeof(doms_cur[0]), GFP_KERNEL); + if (doms_cur) { + doms_cur[0] = kmalloc(cpumask_size(), GFP_KERNEL); + if (!doms_cur[0]) { + kfree(doms_cur); + doms_cur = NULL; + } + } if (!doms_cur) - doms_cur = fallback_doms; - cpumask_andnot(doms_cur, cpu_map, to_cpumask(cpu_isolated_map)); + doms_cur = &fallback_doms; + + cpumask_andnot(doms_cur[0], cpu_map, to_cpumask(cpu_isolated_map)); dattr_cur = NULL; - err = build_sched_domains(doms_cur); + err = build_sched_domains(doms_cur[0]); register_sched_domain_sysctl(); return err; @@ -7783,7 +7791,7 @@ static int dattrs_equal(struct sched_dom * Call with hotplug lock held */ /* FIXME: Change to struct cpumask *doms_new[] */ -void partition_sched_domains(int ndoms_new, struct cpumask *doms_new, +void partition_sched_domains(int ndoms_new, struct cpumask *doms_new[], struct sched_domain_attr *dattr_new) { int i, j, n; @@ -7798,20 +7806,20 @@ void partition_sched_domains(int ndoms_n /* Destroy deleted domains */ for (i = 0; i < ndoms_cur; i++) { for (j = 0; j < n; j++) { - if (cpumask_equal(&doms_cur[i], &doms_new[j]) + if (cpumask_equal(doms_cur[i], doms_new[j]) && dattrs_equal(dattr_cur, i, dattr_new, j)) goto match1; } /* no match - a current sched domain not in new doms_new[] */ - detach_destroy_domains(doms_cur + i); + detach_destroy_domains(doms_cur[i]); match1: ; } if (doms_new == NULL) { ndoms_cur = 0; - doms_new = fallback_doms; - cpumask_andnot(&doms_new[0], + doms_new = &fallback_doms; + cpumask_andnot(doms_new[0], cpu_online_mask, to_cpumask(cpu_isolated_map)); dattr_new = NULL; } @@ -7819,20 +7827,22 @@ match1: /* Build new domains */ for (i = 0; i < ndoms_new; i++) { for (j = 0; j < ndoms_cur; j++) { - if (cpumask_equal(&doms_new[i], &doms_cur[j]) + if (cpumask_equal(doms_new[i], doms_cur[j]) && dattrs_equal(dattr_new, i, dattr_cur, j)) goto match2; } /* no match - add a new doms_new */ - __build_sched_domains(doms_new + i, + __build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL); match2: ; } /* Remember the new sched domains */ - if (doms_cur != fallback_doms) + if (doms_cur != &fallback_doms) { + kfree(doms_cur[0]); kfree(doms_cur); + } kfree(dattr_cur); /* kfree(NULL) is safe */ doms_cur = doms_new; dattr_cur = dattr_new;