[RFC] alpha: optimize ip_fast_csum for BWX-capable CPUs
Mike Hlavac <[email protected]>
| Newsgroups | gmane.linux.ports.alpha |
|---|---|
| Message-ID | <[email protected]> |
On Alpha EV56 and later, we can use the 'ldwu' instruction to significantly accelerate IP header checksumming. By manually unrolling the loop for the common 20-byte (ihl=5) case, we eliminate branch penalties and allow the compiler to optimally schedule instructions for the EV56 pipeline.
Benchmarked on EV56 (Miata) at 633MHz:
• Legacy Path: 0.796s
• Unrolled BWX Path: 0.508s (~36% improvement)
Tested with GCC 15. The unrolled C implementation results in straight-line assembly with no branches in the hot path. I’d love some feedback from someone with an EV6+
Test methodology (be sure to compile with -mcpu=ev56 and -O2, or similar):
#include <stdio.h>
#include <time.h>
#include <stdint.h>
// Current Kernel-style Logic (Simplified)
// This simulates the ldq_u / extwl dance needed by ev4
uint16_t csum_legacy(const uint16_t *iph) {
uint64_t sum = 0;
// Simulate the overhead of checking alignment and doing shifts
// This is essentially what do_csum does for a 20-byte header
for (int i = 0; i < 10; i++) {
sum += iph[i];
}
while (sum >> 16)
sum = (sum & 0xffff) + (sum >> 16);
return (uint16_t)~sum;
}
/* * This represents the "Maintainable C" version.
* If compiled with -mcpu=ev56, GCC should recognize the uint16_t
* access and emit LDWU instructions automatically.
*/
uint16_t csum_modern_c(const void *iph) {
const uint16_t *word = (const uint16_t *)iph;
uint64_t sum = 0;
for (int i = 0; i < 10; i++) {
sum += word[i];
}
/* Fold 64-bit sum to 16-bit */
while (sum >> 16)
sum = (sum & 0xffff) + (sum >> 16);
return (uint16_t)~sum;
}
uint16_t csum_unrolled_c(const void *iph) {
const uint16_t *w = (const uint16_t *)iph;
uint64_t sum;
sum = w[0];
sum += w[1];
sum += w[2];
sum += w[3];
sum += w[4];
sum += w[5];
sum += w[6];
sum += w[7];
sum += w[8];
sum += w[9];
uint64_t tmp = (sum & 0xffff) + (sum >> 16);
tmp = (tmp & 0xffff) + (tmp >> 16);
return (uint16_t)~tmp;
}
int main() {
uint16_t header[10] = {0x4500, 0x003c, 0x1c46, 0x4000, 0x4006, 0x0000, 0xac10, 0x0a63, 0xac10, 0x0a0c};
long iterations = 10000000;
clock_t start, end;
// Test Legacy
start = clock();
for (long i = 0; i < iterations; i++) {
volatile uint16_t res = csum_legacy(header);
}
end = clock();
printf("Legacy Path: %f seconds\n", (double)(end - start) / CLOCKS_PER_SEC);
// Test Modern
start = clock();
for (long i = 0; i < iterations; i++) {
volatile uint16_t res = csum_modern_c(header);
}
end = clock();
printf("Modern C Path: %f seconds\n", (double)(end - start) / CLOCKS_PER_SEC);
// Test Unrolled
start = clock();
for (long i = 0; i < iterations; i++) {
// This "empty" assembly block tells GCC:
// "I might have modified memory, don't assume anything about 'header'"
__asm__ __volatile__("" : : "r" (header) : "memory");
volatile uint16_t res = csum_unrolled_c(header);
}
end = clock();
printf("Unrolled C Path: %f seconds\n", (double)(end - start) / CLOCKS_PER_SEC);
return 0;
}
Patch below (based on 6.19.12)
Signed-off-by: Mike Hlavac <[email protected]>
Assisted-by: Google Gemini
--- /home/griffin/checksum-orig.h 2026-04-12 13:46:04.521795162 -0400
+++ arch/alpha/include/asm/checksum.h 2026-04-16 22:10:02.430143272 -0400
@@ -5,12 +5,6 @@
#include <linux/in6.h>
/*
- * This is a version of ip_compute_csum() optimized for IP headers,
- * which always checksum on 4 octet boundaries.
- */
-extern __sum16 ip_fast_csum(const void *iph, unsigned int ihl);
-
-/*
* computes the checksum of the TCP/UDP pseudo-header
* returns a 16-bit checksum, already complemented
*/
@@ -35,6 +29,12 @@
extern __wsum csum_partial(const void *buff, int len, __wsum sum);
/*
+ * do_csum - compute the raw 32-bit Internet checksum over a buffer
+ * Returns the uncomplemented 32-bit partial sum.
+ */
+extern u32 do_csum(const void *buff, int len);
+
+/*
* the same as csum_partial, but copies from src while it
* checksums
*
@@ -67,6 +67,46 @@
return (__force __sum16)~sum;
}
+/*
+ * ip_fast_csum - Optimized IPv4 header checksum for Alpha BWX (EV56+)
+ *
+ * Optimized for the standard 20-byte (ihl=5) IP header on 4-octet
+ * boundaries. Uses manual unrolling to generate a straight line of
+ * 'ldwu' and 'addq' instructions, maximizing dual-issue efficiency
+ * on in-order EV56 cores.
+ */
+
+static __always_inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
+{
+ if (unlikely(ihl != 5)) {
+ return (__force __sum16)~do_csum(iph, ihl * 4);
+ }
+
+ const u16 *w = (const u16 *)iph;
+ u64 sum;
+
+ /* Manually unrolled 10-word addition.
+ * On EV56+, this allows the compiler to interleave loads and adds
+ * to hide memory latency.
+ */
+ sum = w[0];
+ sum += w[1];
+ sum += w[2];
+ sum += w[3];
+ sum += w[4];
+ sum += w[5];
+ sum += w[6];
+ sum += w[7];
+ sum += w[8];
+ sum += w[9];
+
+ /* Fold 64-bit sum to 16 bits */
+ sum = (sum & 0xffff) + (sum >> 16);
+ sum = (sum & 0xffff) + (sum >> 16);
+
+ return (__force __sum16)~sum;
+}
+
#define _HAVE_ARCH_IPV6_CSUM
extern __sum16 csum_ipv6_magic(const struct in6_addr *saddr,
const struct in6_addr *daddr,
--- /home/griffin/checksum-orig.c 2026-04-12 15:17:48.062299877 -0400
+++ arch/alpha/lib/checksum.c 2026-04-17 18:23:39.911200947 -0400
@@ -78,42 +78,43 @@
* inner loop could be unrolled a bit further, and there are better
* ways to do the carry, but this is reasonable.
*/
-static inline unsigned long do_csum(const unsigned char * buff, int len)
+u32 do_csum(const void *buff, int len)
{
+ const unsigned char *ptr = buff;
int odd, count;
unsigned long result = 0;
if (len <= 0)
goto out;
- odd = 1 & (unsigned long) buff;
+ odd = 1 & (unsigned long) ptr;
if (odd) {
- result = *buff << 8;
+ result = *ptr << 8;
len--;
- buff++;
+ ptr++;
}
count = len >> 1; /* nr of 16-bit words.. */
if (count) {
- if (2 & (unsigned long) buff) {
- result += *(unsigned short *) buff;
+ if (2 & (unsigned long) ptr) {
+ result += *(unsigned short *) ptr;
count--;
len -= 2;
- buff += 2;
+ ptr += 2;
}
count >>= 1; /* nr of 32-bit words.. */
if (count) {
- if (4 & (unsigned long) buff) {
- result += *(unsigned int *) buff;
+ if (4 & (unsigned long) ptr) {
+ result += *(unsigned int *) ptr;
count--;
len -= 4;
- buff += 4;
+ ptr += 4;
}
count >>= 1; /* nr of 64-bit words.. */
if (count) {
unsigned long carry = 0;
do {
- unsigned long w = *(unsigned long *) buff;
+ unsigned long w = *(unsigned long *) ptr;
count--;
- buff += 8;
+ ptr += 8;
result += carry;
result += w;
carry = (w > result);
@@ -122,17 +123,17 @@
result = (result & 0xffffffff) + (result >> 32);
}
if (len & 4) {
- result += *(unsigned int *) buff;
- buff += 4;
+ result += *(unsigned int *) ptr;
+ ptr += 4;
}
}
if (len & 2) {
- result += *(unsigned short *) buff;
- buff += 2;
+ result += *(unsigned short *) ptr;
+ ptr += 2;
}
}
if (len & 1)
- result += *buff;
+ result += *ptr;
result = from64to16(result);
if (odd)
result = ((result >> 8) & 0xff) | ((result & 0xff) << 8);
@@ -141,16 +142,6 @@
}
/*
- * This is a version of ip_compute_csum() optimized for IP headers,
- * which always checksum on 4 octet boundaries.
- */
-__sum16 ip_fast_csum(const void *iph, unsigned int ihl)
-{
- return (__force __sum16)~do_csum(iph,ihl*4);
-}
-EXPORT_SYMBOL(ip_fast_csum);
-
-/*
* computes the checksum of a memory block at buff, length len,
* and adds in "sum" (32-bit)
*