cpumask: use percpu allocations instead of arrays in kernel/trace/ring_buffer.c Instead of allocating an nr_cpu_ids array, most places should be using percpu_alloc(). This doesn't waste space if cpu numbers aren't contiguous. Signed-off-by: Rusty Russell --- kernel/trace/ring_buffer.c | 68 +++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 35 deletions(-) diff -r 88b5980eaf70 kernel/trace/ring_buffer.c --- a/kernel/trace/ring_buffer.c Thu Oct 23 00:39:36 2008 +1100 +++ b/kernel/trace/ring_buffer.c Thu Oct 23 00:39:45 2008 +1100 @@ -170,13 +170,12 @@ struct ring_buffer { unsigned long size; unsigned pages; unsigned flags; - int cpus; cpumask_t cpumask; atomic_t record_disabled; struct mutex mutex; - struct ring_buffer_per_cpu **buffers; + struct ring_buffer_per_cpu **buffers_pcpu; }; struct ring_buffer_iter { @@ -357,7 +356,6 @@ struct ring_buffer *ring_buffer_alloc(un struct ring_buffer *ring_buffer_alloc(unsigned long size, unsigned flags) { struct ring_buffer *buffer; - int bsize; int cpu; /* Paranoid! Optimizes out when all is well */ @@ -379,18 +377,15 @@ struct ring_buffer *ring_buffer_alloc(un buffer->pages++; buffer->cpumask = cpu_possible_map; - buffer->cpus = nr_cpu_ids; - bsize = sizeof(void *) * nr_cpu_ids; - buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), - GFP_KERNEL); - if (!buffer->buffers) + buffer->buffers_pcpu = alloc_percpu(struct ring_buffer_per_cpu); + if (!buffer->buffers_pcpu) goto fail_free_buffer; for_each_buffer_cpu(buffer, cpu) { - buffer->buffers[cpu] = + *percpu_ptr(buffer->buffers_pcpu, cpu) = rb_allocate_cpu_buffer(buffer, cpu); - if (!buffer->buffers[cpu]) + if (!*percpu_ptr(buffer->buffers_pcpu, cpu)) goto fail_free_buffers; } @@ -400,10 +395,11 @@ struct ring_buffer *ring_buffer_alloc(un fail_free_buffers: for_each_buffer_cpu(buffer, cpu) { - if (buffer->buffers[cpu]) - rb_free_cpu_buffer(buffer->buffers[cpu]); + if (*percpu_ptr(buffer->buffers_pcpu, cpu)) + rb_free_cpu_buffer(*percpu_ptr(buffer->buffers_pcpu, + cpu)); } - kfree(buffer->buffers); + free_percpu(buffer->buffers_pcpu); fail_free_buffer: kfree(buffer); @@ -420,7 +416,7 @@ ring_buffer_free(struct ring_buffer *buf int cpu; for_each_buffer_cpu(buffer, cpu) - rb_free_cpu_buffer(buffer->buffers[cpu]); + rb_free_cpu_buffer(*percpu_ptr(buffer->buffers_pcpu, cpu)); kfree(buffer); } @@ -526,7 +522,7 @@ int ring_buffer_resize(struct ring_buffe rm_pages = buffer->pages - nr_pages; for_each_buffer_cpu(buffer, cpu) { - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); rb_remove_pages(cpu_buffer, rm_pages); } goto out; @@ -559,7 +555,7 @@ int ring_buffer_resize(struct ring_buffe } for_each_buffer_cpu(buffer, cpu) { - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); rb_insert_pages(cpu_buffer, &pages, new_pages); } @@ -1128,7 +1124,7 @@ ring_buffer_lock_reserve(struct ring_buf if (!cpu_isset(cpu, buffer->cpumask)) goto out; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); if (atomic_read(&cpu_buffer->record_disabled)) goto out; @@ -1190,7 +1186,7 @@ int ring_buffer_unlock_commit(struct rin struct ring_buffer_per_cpu *cpu_buffer; int cpu = raw_smp_processor_id(); - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); rb_commit(cpu_buffer, event); @@ -1243,7 +1239,7 @@ int ring_buffer_write(struct ring_buffer if (!cpu_isset(cpu, buffer->cpumask)) goto out; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); if (atomic_read(&cpu_buffer->record_disabled)) goto out; @@ -1325,7 +1321,7 @@ void ring_buffer_record_disable_cpu(stru if (!cpu_isset(cpu, buffer->cpumask)) return; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); atomic_inc(&cpu_buffer->record_disabled); } @@ -1344,7 +1340,7 @@ void ring_buffer_record_enable_cpu(struc if (!cpu_isset(cpu, buffer->cpumask)) return; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); atomic_dec(&cpu_buffer->record_disabled); } @@ -1360,7 +1356,7 @@ unsigned long ring_buffer_entries_cpu(st if (!cpu_isset(cpu, buffer->cpumask)) return 0; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); return cpu_buffer->entries; } @@ -1376,7 +1372,7 @@ unsigned long ring_buffer_overrun_cpu(st if (!cpu_isset(cpu, buffer->cpumask)) return 0; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); return cpu_buffer->overrun; } @@ -1395,7 +1391,7 @@ unsigned long ring_buffer_entries(struct /* if you care about this being correct, lock the buffer */ for_each_buffer_cpu(buffer, cpu) { - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); entries += cpu_buffer->entries; } @@ -1417,7 +1413,7 @@ unsigned long ring_buffer_overruns(struc /* if you care about this being correct, lock the buffer */ for_each_buffer_cpu(buffer, cpu) { - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); overruns += cpu_buffer->overrun; } @@ -1667,7 +1663,7 @@ ring_buffer_peek(struct ring_buffer *buf if (!cpu_isset(cpu, buffer->cpumask)) return NULL; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); again: reader = rb_get_reader_page(cpu_buffer); @@ -1783,7 +1779,7 @@ ring_buffer_consume(struct ring_buffer * if (!event) return NULL; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); rb_advance_reader(cpu_buffer); return event; @@ -1815,7 +1811,7 @@ ring_buffer_read_start(struct ring_buffe if (!iter) return NULL; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); iter->cpu_buffer = cpu_buffer; @@ -1904,11 +1900,13 @@ rb_reset_cpu(struct ring_buffer_per_cpu */ void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu) { - struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; + struct ring_buffer_per_cpu *cpu_buffer; unsigned long flags; if (!cpu_isset(cpu, buffer->cpumask)) return; + + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); spin_lock_irqsave(&cpu_buffer->lock, flags); @@ -1940,7 +1938,7 @@ int ring_buffer_empty(struct ring_buffer /* yes this is racy, but if you don't like the race, lock the buffer */ for_each_buffer_cpu(buffer, cpu) { - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); if (!rb_per_cpu_empty(cpu_buffer)) return 0; } @@ -1959,7 +1957,7 @@ int ring_buffer_empty_cpu(struct ring_bu if (!cpu_isset(cpu, buffer->cpumask)) return 1; - cpu_buffer = buffer->buffers[cpu]; + cpu_buffer = *percpu_ptr(buffer->buffers_pcpu, cpu); return rb_per_cpu_empty(cpu_buffer); } @@ -1988,8 +1986,8 @@ int ring_buffer_swap_cpu(struct ring_buf buffer_a->pages != buffer_b->pages) return -EINVAL; - cpu_buffer_a = buffer_a->buffers[cpu]; - cpu_buffer_b = buffer_b->buffers[cpu]; + cpu_buffer_a = *percpu_ptr(buffer_a->buffers_pcpu, cpu); + cpu_buffer_b = *percpu_ptr(buffer_b->buffers_pcpu, cpu); /* * We can't do a synchronize_sched here because this @@ -2000,8 +1998,8 @@ int ring_buffer_swap_cpu(struct ring_buf atomic_inc(&cpu_buffer_a->record_disabled); atomic_inc(&cpu_buffer_b->record_disabled); - buffer_a->buffers[cpu] = cpu_buffer_b; - buffer_b->buffers[cpu] = cpu_buffer_a; + *percpu_ptr(buffer_a->buffers_pcpu, cpu) = cpu_buffer_b; + *percpu_ptr(buffer_b->buffers_pcpu, cpu) = cpu_buffer_a; cpu_buffer_b->buffer = buffer_a; cpu_buffer_a->buffer = buffer_b;