alloc_percpu: big_percpu_alloc

Impact: New API

We never allowed the dynamic per-cpu allocator to use the per-cpu region,
waiting until that region became resizable.  That was over 5 years ago.

Christoph Lameter has demonstrated that small dynamic per-cpu allocations
make sense, and most allocations are small.  So we want to use the same
efficient per-cpu access paths for alloc_percpu() memory.

But some places really do allocate large amounts of percpu memory.  We
give them a separate API (equivalent to the current inefficient one).

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 include/linux/percpu.h |   24 ++++++++++++++++++++++++
 mm/allocpercpu.c       |   35 +++++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)

diff --git a/include/linux/percpu.h b/include/linux/percpu.h
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -113,4 +113,28 @@ static inline void percpu_free(void *__p
 #define free_percpu(ptr)	percpu_free((ptr))
 #define per_cpu_ptr(ptr, cpu)	percpu_ptr((ptr), (cpu))
 
+/* Big per-cpu allocations.  Less efficient, but if you're doing an unbounded
+ * number of allocations or large ones, you need these until we implement
+ * growing percpu regions. */
+#ifdef CONFIG_SMP
+extern void *big_alloc_percpu(unsigned long size);
+extern void big_free_percpu(const void *);
+
+#define big_per_cpu_ptr(bptr, cpu) ({		\
+	void **__bp = (void **)(bptr);		\
+	(__typeof__(bptr))__bp[(cpu)];		\
+})
+#else
+static inline void *big_alloc_percpu(unsigned long size)
+{
+	return kzalloc(size, GFP_KERNEL);
+}
+static inline void big_free_percpu(const void *bp)
+{
+	kfree(bp);
+}
+
+#define big_per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
+#endif /* !CONFIG_SMP */
+
 #endif /* __LINUX_PERCPU_H */
diff --git a/mm/allocpercpu.c b/mm/allocpercpu.c
--- a/mm/allocpercpu.c
+++ b/mm/allocpercpu.c
@@ -141,3 +141,38 @@ void percpu_free(void *__pdata)
 	kfree(__percpu_disguise(__pdata));
 }
 EXPORT_SYMBOL_GPL(percpu_free);
+
+void *big_alloc_percpu(unsigned long size)
+{
+	unsigned int cpu;
+	void **bp;
+
+	bp = kcalloc(sizeof(void *), nr_cpu_ids, GFP_KERNEL);
+	if (unlikely(!bp))
+		return NULL;
+
+	for_each_possible_cpu(cpu) {
+		bp[cpu] = kzalloc_node(size, GFP_KERNEL, cpu_to_node(cpu));
+		if (unlikely(!bp[cpu]))
+			goto fail;
+	}
+	return bp;
+
+fail:
+	/* kcalloc zeroes and kfree(NULL) is safe, so this works, */
+	big_free_percpu(bp);
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(big_alloc_percpu);
+
+void big_free_percpu(const void *_bp)
+{
+	void *const *bp = _bp;
+	unsigned int cpu;
+
+	if (likely(bp))
+		for_each_possible_cpu(cpu)
+			kfree(bp[cpu]);
+	kfree(bp);
+}
+EXPORT_SYMBOL_GPL(big_free_percpu);