cpumask: use percpu allocations instead of arrays in kernel/trace/ring_buffer.c Instead of allocating an nr_cpu_ids array, most places should be using percpu_alloc(). This doesn't waste space if cpu numbers aren't contiguous. Signed-off-by: Rusty Russell --- kernel/trace/ring_buffer.c | 69 +++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 36 deletions(-) diff -r 4c854df1e22f kernel/trace/ring_buffer.c --- a/kernel/trace/ring_buffer.c Thu Nov 20 00:38:37 2008 +1030 +++ b/kernel/trace/ring_buffer.c Thu Nov 20 00:49:09 2008 +1030 @@ -170,13 +170,12 @@ unsigned long size; unsigned pages; unsigned flags; - int cpus; cpumask_var_t cpumask; atomic_t record_disabled; struct mutex mutex; - struct ring_buffer_per_cpu **buffers; + struct ring_buffer_per_cpu **buffers_pcpu; }; struct ring_buffer_iter { @@ -357,7 +356,6 @@ struct ring_buffer *ring_buffer_alloc(unsigned long size, unsigned flags) { struct ring_buffer *buffer; - int bsize; int cpu; /* Paranoid! Optimizes out when all is well */ @@ -382,18 +380,14 @@ buffer->pages++; cpumask_copy(buffer->cpumask, cpu_possible_mask); - buffer->cpus = nr_cpu_ids; - - bsize = sizeof(void *) * nr_cpu_ids; - buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), - GFP_KERNEL); - if (!buffer->buffers) + buffer->buffers_pcpu = alloc_percpu(struct ring_buffer_per_cpu); + if (!buffer->buffers_pcpu) goto fail_free_cpumask; for_each_buffer_cpu(buffer, cpu) { - buffer->buffers[cpu] = + *per_cpu_ptr(buffer->buffers_pcpu, cpu) = rb_allocate_cpu_buffer(buffer, cpu); - if (!buffer->buffers[cpu]) + if (!*per_cpu_ptr(buffer->buffers_pcpu, cpu)) goto fail_free_buffers; } @@ -403,10 +397,11 @@ fail_free_buffers: for_each_buffer_cpu(buffer, cpu) { - if (buffer->buffers[cpu]) - rb_free_cpu_buffer(buffer->buffers[cpu]); + if (*per_cpu_ptr(buffer->buffers_pcpu, cpu)) + rb_free_cpu_buffer(*per_cpu_ptr(buffer->buffers_pcpu, + cpu)); } - kfree(buffer->buffers); + free_percpu(buffer->buffers_pcpu); fail_free_cpumask: free_cpumask_var(buffer->cpumask); @@ -426,7 +421,7 @@ int cpu; for_each_buffer_cpu(buffer, cpu) - rb_free_cpu_buffer(buffer->buffers[cpu]); + rb_free_cpu_buffer(*per_cpu_ptr(buffer->buffers_pcpu, cpu)); free_cpumask_var(buffer->cpumask); @@ -534,7 +529,7 @@ rm_pages = buffer->pages - nr_pages; for_each_buffer_cpu(buffer, cpu) { - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); rb_remove_pages(cpu_buffer, rm_pages); } goto out; @@ -567,7 +562,7 @@ } for_each_buffer_cpu(buffer, cpu) { - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); rb_insert_pages(cpu_buffer, &pages, new_pages); } @@ -1153,7 +1148,7 @@ if (!cpumask_test_cpu(cpu, buffer->cpumask)) goto out; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); if (atomic_read(&cpu_buffer->record_disabled)) goto out; @@ -1215,7 +1210,7 @@ struct ring_buffer_per_cpu *cpu_buffer; int cpu = raw_smp_processor_id(); - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); rb_commit(cpu_buffer, event); @@ -1268,7 +1263,7 @@ if (!cpumask_test_cpu(cpu, buffer->cpumask)) goto out; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); if (atomic_read(&cpu_buffer->record_disabled)) goto out; @@ -1350,7 +1345,7 @@ if (!cpumask_test_cpu(cpu, buffer->cpumask)) return; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); atomic_inc(&cpu_buffer->record_disabled); } @@ -1369,7 +1364,7 @@ if (!cpumask_test_cpu(cpu, buffer->cpumask)) return; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); atomic_dec(&cpu_buffer->record_disabled); } @@ -1385,7 +1380,7 @@ if (!cpumask_test_cpu(cpu, buffer->cpumask)) return 0; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); return cpu_buffer->entries; } @@ -1401,7 +1396,7 @@ if (!cpumask_test_cpu(cpu, buffer->cpumask)) return 0; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); return cpu_buffer->overrun; } @@ -1420,7 +1415,7 @@ /* if you care about this being correct, lock the buffer */ for_each_buffer_cpu(buffer, cpu) { - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); entries += cpu_buffer->entries; } @@ -1442,7 +1437,7 @@ /* if you care about this being correct, lock the buffer */ for_each_buffer_cpu(buffer, cpu) { - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); overruns += cpu_buffer->overrun; } @@ -1706,7 +1701,7 @@ if (!cpumask_test_cpu(cpu, buffer->cpumask)) return NULL; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); again: /* @@ -1849,7 +1844,7 @@ if (!event) return NULL; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); rb_advance_reader(cpu_buffer); return event; @@ -1881,7 +1876,7 @@ if (!iter) return NULL; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); iter->cpu_buffer = cpu_buffer; @@ -1970,11 +1965,13 @@ */ void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu) { - struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; + struct ring_buffer_per_cpu *cpu_buffer; unsigned long flags; if (!cpumask_test_cpu(cpu, buffer->cpumask)) return; + + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); spin_lock_irqsave(&cpu_buffer->lock, flags); @@ -2006,7 +2003,7 @@ /* yes this is racy, but if you don't like the race, lock the buffer */ for_each_buffer_cpu(buffer, cpu) { - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); if (!rb_per_cpu_empty(cpu_buffer)) return 0; } @@ -2025,7 +2022,7 @@ if (!cpumask_test_cpu(cpu, buffer->cpumask)) return 1; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *per_cpu_ptr(buffer->buffers_pcpu, cpu); return rb_per_cpu_empty(cpu_buffer); } @@ -2054,8 +2051,8 @@ buffer_a->pages != buffer_b->pages) return -EINVAL; - cpu_buffer_a = buffer_a->buffers[cpu]; - cpu_buffer_b = buffer_b->buffers[cpu]; + cpu_buffer_a = *per_cpu_ptr(buffer_a->buffers_pcpu, cpu); + cpu_buffer_b = *per_cpu_ptr(buffer_b->buffers_pcpu, cpu); /* * We can't do a synchronize_sched here because this @@ -2066,8 +2063,8 @@ atomic_inc(&cpu_buffer_a->record_disabled); atomic_inc(&cpu_buffer_b->record_disabled); - buffer_a->buffers[cpu] = cpu_buffer_b; - buffer_b->buffers[cpu] = cpu_buffer_a; + *per_cpu_ptr(buffer_a->buffers_pcpu, cpu) = cpu_buffer_b; + *per_cpu_ptr(buffer_b->buffers_pcpu, cpu) = cpu_buffer_a; cpu_buffer_b->buffer = buffer_a; cpu_buffer_a->buffer = buffer_b;