alloc_percpu: Move SNMP and ip_rt_acct to big_percpu If we look at percpu allocations to boot an x86/32 "allyesconfig" kernel, we see the following: File and line Number Size Total net/ipv4/af_inet.c:1287 21 2048 43008 net/ipv4/af_inet.c:1290 21 2048 43008 net/ipv4/af_inet.c:1287 48 128 6144 net/ipv4/af_inet.c:1290 48 128 6144 net/ipv4/route.c:3258 1 4096 4096 net/ipv4/af_inet.c:1287 1 288 288 net/ipv4/af_inet.c:1290 1 288 288 net/ipv4/af_inet.c:1287 1 256 256 net/ipv4/af_inet.c:1290 1 256 256 net/ipv4/af_inet.c:1287 1 104 104 net/ipv4/af_inet.c:1290 1 104 104 Subtotal: 103696 Total (including other callers untouched by this patch) 117228 So networking is chubby: worst case about 100k per cpu for SNMP stats. This can be reduced, but it's still 88% of the percpu memory, and IA64 has a hardcoded limit of 64k at the moment. That's why these two callers get moved to big_percpu_alloc. Note: ip_rt_acct could be simplified and made more efficient by making it a DEFINE_PER_CPU. But DaveM made some argument about image size and I have far too much respect to argue with BloatBoy :) Signed-off-by: Rusty Russell Cc: netdev@vger.kernel.org --- include/net/snmp.h | 12 ++++++------ net/ipv4/af_inet.c | 14 +++++++------- net/ipv4/ip_input.c | 3 ++- net/ipv4/route.c | 4 ++-- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/include/net/snmp.h b/include/net/snmp.h --- a/include/net/snmp.h +++ b/include/net/snmp.h @@ -137,27 +137,27 @@ struct linux_xfrm_mib { #define SNMP_STAT_USRPTR(name) (name[1]) #define SNMP_INC_STATS_BH(mib, field) \ - (per_cpu_ptr(mib[0], raw_smp_processor_id())->mibs[field]++) + (big_per_cpu_ptr(mib[0], raw_smp_processor_id())->mibs[field]++) #define SNMP_INC_STATS_USER(mib, field) \ do { \ - per_cpu_ptr(mib[1], get_cpu())->mibs[field]++; \ + big_per_cpu_ptr(mib[1], get_cpu())->mibs[field]++; \ put_cpu(); \ } while (0) #define SNMP_INC_STATS(mib, field) \ do { \ - per_cpu_ptr(mib[!in_softirq()], get_cpu())->mibs[field]++; \ + big_per_cpu_ptr(mib[!in_softirq()], get_cpu())->mibs[field]++; \ put_cpu(); \ } while (0) #define SNMP_DEC_STATS(mib, field) \ do { \ - per_cpu_ptr(mib[!in_softirq()], get_cpu())->mibs[field]--; \ + big_per_cpu_ptr(mib[!in_softirq()], get_cpu())->mibs[field]--; \ put_cpu(); \ } while (0) #define SNMP_ADD_STATS_BH(mib, field, addend) \ - (per_cpu_ptr(mib[0], raw_smp_processor_id())->mibs[field] += addend) + (big_per_cpu_ptr(mib[0], raw_smp_processor_id())->mibs[field] += addend) #define SNMP_ADD_STATS_USER(mib, field, addend) \ do { \ - per_cpu_ptr(mib[1], get_cpu())->mibs[field] += addend; \ + big_per_cpu_ptr(mib[1], get_cpu())->mibs[field] += addend; \ put_cpu(); \ } while (0) diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -1365,8 +1365,8 @@ unsigned long snmp_fold_field(void *mib[ int i; for_each_possible_cpu(i) { - res += *(((unsigned long *) per_cpu_ptr(mib[0], i)) + offt); - res += *(((unsigned long *) per_cpu_ptr(mib[1], i)) + offt); + res += *(((unsigned long *)big_per_cpu_ptr(mib[0], i)) + offt); + res += *(((unsigned long *)big_per_cpu_ptr(mib[1], i)) + offt); } return res; } @@ -1375,15 +1375,15 @@ int snmp_mib_init(void *ptr[2], size_t m int snmp_mib_init(void *ptr[2], size_t mibsize) { BUG_ON(ptr == NULL); - ptr[0] = __alloc_percpu(mibsize); + ptr[0] = big_alloc_percpu(mibsize); if (!ptr[0]) goto err0; - ptr[1] = __alloc_percpu(mibsize); + ptr[1] = big_alloc_percpu(mibsize); if (!ptr[1]) goto err1; return 0; err1: - free_percpu(ptr[0]); + big_free_percpu(ptr[0]); ptr[0] = NULL; err0: return -ENOMEM; @@ -1393,8 +1393,8 @@ void snmp_mib_free(void *ptr[2]) void snmp_mib_free(void *ptr[2]) { BUG_ON(ptr == NULL); - free_percpu(ptr[0]); - free_percpu(ptr[1]); + big_free_percpu(ptr[0]); + big_free_percpu(ptr[1]); ptr[0] = ptr[1] = NULL; } EXPORT_SYMBOL_GPL(snmp_mib_free); diff --git a/net/ipv4/ip_input.c b/net/ipv4/ip_input.c --- a/net/ipv4/ip_input.c +++ b/net/ipv4/ip_input.c @@ -345,7 +345,8 @@ static int ip_rcv_finish(struct sk_buff #ifdef CONFIG_NET_CLS_ROUTE if (unlikely(skb->dst->tclassid)) { - struct ip_rt_acct *st = per_cpu_ptr(ip_rt_acct, smp_processor_id()); + struct ip_rt_acct *st = big_per_cpu_ptr(ip_rt_acct, + smp_processor_id()); u32 idx = skb->dst->tclassid; st[idx&0xFF].o_packets++; st[idx&0xFF].o_bytes += skb->len; diff --git a/net/ipv4/route.c b/net/ipv4/route.c --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -543,7 +543,7 @@ static int ip_rt_acct_read(char *buffer, unsigned int j; u32 *src; - src = ((u32 *) per_cpu_ptr(ip_rt_acct, i)) + offset; + src = ((u32 *)big_per_cpu_ptr(ip_rt_acct, i)) + offset; for (j = 0; j < length/4; j++) dst[j] += src[j]; } @@ -3376,7 +3376,7 @@ int __init ip_rt_init(void) int rc = 0; #ifdef CONFIG_NET_CLS_ROUTE - ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct)); + ip_rt_acct = big_alloc_percpu(256 * sizeof(struct ip_rt_acct)); if (!ip_rt_acct) panic("IP: failed to allocate ip_rt_acct\n"); #endif