shell style¶
## Setup NIC queue cpu affinity
# eth0, eth1: separate rx and tx queues, should stay on the same core. They
# aren't used much, should probably just leave them default.
# for the rest, map queues to cpus, starting with the cpu that matches the
# eth# mod 8
# eth2: 48-55
echo 02 > /proc/irq/48/smp_affinity
echo 04 > /proc/irq/49/smp_affinity
echo 08 > /proc/irq/50/smp_affinity
echo 10 > /proc/irq/51/smp_affinity
echo 11 > /proc/irq/52/smp_affinity
echo 12 > /proc/irq/53/smp_affinity
echo 00 > /proc/irq/54/smp_affinity
echo 01 > /proc/irq/55/smp_affinity
# eth3: 57-64
echo 04 > /proc/irq/57/smp_affinity
echo 08 > /proc/irq/58/smp_affinity
echo 10 > /proc/irq/59/smp_affinity
echo 11 > /proc/irq/60/smp_affinity
echo 12 > /proc/irq/61/smp_affinity
echo 00 > /proc/irq/62/smp_affinity
echo 01 > /proc/irq/63/smp_affinity
echo 02 > /proc/irq/64/smp_affinity
# eth4: 69-76
echo 08 > /proc/irq/69/smp_affinity
echo 10 > /proc/irq/70/smp_affinity
echo 11 > /proc/irq/71/smp_affinity
echo 12 > /proc/irq/72/smp_affinity
echo 00 > /proc/irq/73/smp_affinity
echo 01 > /proc/irq/74/smp_affinity
echo 02 > /proc/irq/75/smp_affinity
echo 04 > /proc/irq/76/smp_affinity
# eth5: 78-85
echo 10 > /proc/irq/78/smp_affinity
echo 11 > /proc/irq/79/smp_affinity
echo 12 > /proc/irq/80/smp_affinity
echo 00 > /proc/irq/81/smp_affinity
echo 01 > /proc/irq/82/smp_affinity
echo 02 > /proc/irq/83/smp_affinity
echo 04 > /proc/irq/84/smp_affinity
echo 08 > /proc/irq/85/smp_affinity
# eth6: 87-94
echo 11 > /proc/irq/87/smp_affinity
echo 12 > /proc/irq/87/smp_affinity
echo 00 > /proc/irq/87/smp_affinity
echo 01 > /proc/irq/90/smp_affinity
echo 02 > /proc/irq/91/smp_affinity
echo 04 > /proc/irq/92/smp_affinity
echo 08 > /proc/irq/93/smp_affinity
echo 10 > /proc/irq/94/smp_affinity
# eth7: 97-104
echo 12 > /proc/irq/97/smp_affinity
echo 00 > /proc/irq/98/smp_affinity
echo 01 > /proc/irq/99/smp_affinity
echo 02 > /proc/irq/100/smp_affinity
echo 04 > /proc/irq/101/smp_affinity
echo 08 > /proc/irq/102/smp_affinity
echo 10 > /proc/irq/103/smp_affinity
echo 11 > /proc/irq/104/smp_affinity
# eth8: 106-113
echo 00 > /proc/irq/106/smp_affinity
echo 01 > /proc/irq/107/smp_affinity
echo 02 > /proc/irq/108/smp_affinity
echo 04 > /proc/irq/109/smp_affinity
echo 08 > /proc/irq/110/smp_affinity
echo 10 > /proc/irq/111/smp_affinity
echo 11 > /proc/irq/112/smp_affinity
echo 12 > /proc/irq/113/smp_affinity
# eth9: 115-122
echo 01 > /proc/irq/115/smp_affinity
echo 02 > /proc/irq/116/smp_affinity
echo 04 > /proc/irq/117/smp_affinity
echo 08 > /proc/irq/118/smp_affinity
echo 10 > /proc/irq/119/smp_affinity
echo 11 > /proc/irq/120/smp_affinity
echo 12 > /proc/irq/121/smp_affinity
echo 00 > /proc/irq/122/smp_affinity
sysctl.conf style¶
## Setup NIC queue cpu affinity
# eth0, eth1: separate rx and tx queues, should stay on the same core. They
# aren't used much, should probably just leave them default.
# for the rest, map queues to cpus, starting with the cpu that matches the
# eth# mod 8
# eth2: 48-55
irq.48.smp_affinity = 02
irq.49.smp_affinity = 04
irq.50.smp_affinity = 08
irq.51.smp_affinity = 10
irq.52.smp_affinity = 11
irq.53.smp_affinity = 12
irq.54.smp_affinity = 00
irq.55.smp_affinity = 01
# eth3: 57-64
irq.57.smp_affinity = 04
irq.58.smp_affinity = 08
irq.59.smp_affinity = 10
irq.60.smp_affinity = 11
irq.61.smp_affinity = 12
irq.62.smp_affinity = 00
irq.63.smp_affinity = 01
irq.64.smp_affinity = 02
# eth4: 69-76
irq.69.smp_affinity = 08
irq.70.smp_affinity = 10
irq.71.smp_affinity = 11
irq.72.smp_affinity = 12
irq.73.smp_affinity = 00
irq.74.smp_affinity = 01
irq.75.smp_affinity = 02
irq.76.smp_affinity = 04
# eth5: 78-85
irq.78.smp_affinity = 10
irq.79.smp_affinity = 11
irq.80.smp_affinity = 12
irq.81.smp_affinity = 00
irq.82.smp_affinity = 01
irq.83.smp_affinity = 02
irq.84.smp_affinity = 04
irq.85.smp_affinity = 08
# eth6: 87-94
irq.87.smp_affinity = 11
irq.87.smp_affinity = 12
irq.87.smp_affinity = 00
irq.90.smp_affinity = 01
irq.91.smp_affinity = 02
irq.92.smp_affinity = 04
irq.93.smp_affinity = 08
irq.94.smp_affinity = 10
# eth7: 97-104
irq.97.smp_affinity = 12
irq.98.smp_affinity = 00
irq.99.smp_affinity = 01
irq.100.smp_affinity = 02
irq.101.smp_affinity = 04
irq.102.smp_affinity = 08
irq.103.smp_affinity = 10
irq.104.smp_affinity = 11
# eth8: 106-113
irq.106.smp_affinity = 00
irq.107.smp_affinity = 01
irq.108.smp_affinity = 02
irq.109.smp_affinity = 04
irq.110.smp_affinity = 08
irq.111.smp_affinity = 10
irq.112.smp_affinity = 11
irq.113.smp_affinity = 12
# eth9: 115-122
irq.115.smp_affinity = 01
irq.116.smp_affinity = 02
irq.117.smp_affinity = 04
irq.118.smp_affinity = 08
irq.119.smp_affinity = 10
irq.120.smp_affinity = 11
irq.121.smp_affinity = 12
irq.122.smp_affinity = 00
program to convert from shell to sysctl.conf style¶
#!/usr/bin/perl -wn
if ( m/echo (\d*) > \/proc\/irq\/(\d*)\/smp_affinity/ ) {
print "irq.$2.smp_affinity = $1\n";
} else {
print;
}
Notes and Hints from others¶
- RE: the LRO+routing warning the igb README: LRO automatically gets disabled when you turn on forwarding. Confirmed off with ethtool.
- raj@HP
- isn’t sure if interrupts should be spread across threads or only real cores. Suggests
grep -e "core id" -e "processor" -e "physical id" /proc/cpuinfo</core> which seems to indicate that 0,4 : 1,5 : 2,6 , 3,7
** grep eth0 /proc/interrupts | awk -F “:” -v mask=1 ‘{printf(“echo %x > /proc/irq/%d/smp_affinity\n”,mask,$1);mask #= 2}’ | sh** if spreading the IRQs "works" for the problem, cool. if not though, or if you become curious and bored, something like "perf record -e cycles -a sleep 60" at a high load, followed by "perf report --sort symbol,dso" might be worthwhile<//code>
