Re: Help requested: Interpreting results from bench.pl

On Thu, Mar 12, 2026 at 10:48:34AM -0600, Karl Williamson via perl5-porters wrote:
> I've been told, and it may even be documented that the rows below IND_m are
> not significant; so don't even bother looking at them.  And it might be rows
> below plain 'IND' that aren't important.

My problem is that none of the values is a good direct indicator of
performance.

For reference, see the matrix multiplication example below.

Without arguments it runs the "dumb" implementation:

  $ perf stat ./matmult 2>&1 | grep task-clock
            667.59 msec task-clock                       #    0.999 CPUs utilized
  $ perf stat ./matmult smart 2>&1 | grep task-clock
            373.19 msec task-clock                       #    0.998 CPUs utilized

  $ valgrind --tool=cachegrind --cache-sim=yes ./matmult

  ==4011921== I refs:        6,477,837,655
  ==4011921== I1  misses:            1,249
  ==4011921== LLi misses:            1,232
  ==4011921== I1  miss rate:          0.00%
  ==4011921== LLi miss rate:          0.00%
  ==4011921== 
  ==4011921== D refs:        2,438,485,653  (1,605,672,547 rd   + 832,813,106 wr)
  ==4011921== D1  misses:      101,572,718  (  101,062,170 rd   +     510,548 wr)
  ==4011921== LLd misses:           16,384  (        1,019 rd   +      15,365 wr)
  ==4011921== D1  miss rate:           4.2% (          6.3%     +         0.1%  )
  ==4011921== LLd miss rate:           0.0% (          0.0%     +         0.0%  )
  ==4011921== 
  ==4011921== LL refs:         101,573,967  (  101,063,419 rd   +     510,548 wr)
  ==4011921== LL misses:            17,616  (        2,251 rd   +      15,365 wr)
  ==4011921== LL miss rate:            0.0% (          0.0%     +         0.0%  )
$ valgrind --tool=cachegrind --cache-sim=yes ./matmult smart
  ==4011954== I refs:        6,473,797,069
  ==4011954== I1  misses:            1,248
  ==4011954== LLi misses:            1,231
  ==4011954== I1  miss rate:          0.00%
  ==4011954== LLi miss rate:          0.00%
  ==4011954== 
  ==4011954== D refs:        3,234,445,061  (2,401,632,355 rd   + 832,812,706 wr)
  ==4011954== D1  misses:      101,532,923  (  101,022,370 rd   +     510,553 wr)
  ==4011954== LLd misses:           16,387  (        1,019 rd   +      15,368 wr)
  ==4011954== D1  miss rate:           3.1% (          4.2%     +         0.1%  )
  ==4011954== LLd miss rate:           0.0% (          0.0%     +         0.0%  )
  ==4011954== 
  ==4011954== LL refs:         101,534,171  (  101,023,618 rd   +     510,553 wr)
  ==4011954== LL misses:            17,618  (        2,250 rd   +      15,368 wr)
  ==4011954== LL miss rate:            0.0% (          0.0%     +         0.0%  )

Note that while the "smart" implementation is about twice as fast, the
cachegrind results are almost the same.

Asking perf stat for more details reveals the cause:

  $ perf stat -d ./matmult
  ...
     1,603,083,228      L1-dcache-loads                  #    2.392 G/sec                       (62.52%)
       104,022,669      L1-dcache-load-misses            #    6.49% of all L1-dcache accesses   (62.65%)
        52,203,679      LLC-loads                        #   77.900 M/sec                       (50.12%)
               571      LLC-load-misses                  #    0.00% of all LL-cache accesses    (50.02%)

  $ perf stat -d ./matmult smart

     2,390,381,021      L1-dcache-loads                  #    6.496 G/sec                       (64.13%)
       101,252,883      L1-dcache-load-misses            #    4.24% of all L1-dcache accesses   (63.31%)
         1,816,176      LLC-loads                        #    4.935 M/sec                       (49.18%)
               598      LLC-load-misses                  #    0.03% of all LL-cache accesses    (48.09%)

(perf stat highlights the L1-dcache-load-misses in the first result.)

So while cachegrind (which is what Porting/bench.pl uses) provides
useful information for tracking down performance issue causes, I don't
think it's so useful for benchmarking.

The reproducibility of the valgrind results is really tempting, but I
don't really trust it to reflect performance.

> There are people who say that this whole cachegrind output is not relevant
> to modern processors.  I plead ignorance.

https://valgrind.org/docs/manual/cg-manual.html#cache-sim

"The cache simulation approximates the hardware of an AMD Athlon CPU
circa 2002"

https://valgrind.org/docs/manual/cg-manual.html#branch-sim

"Cachegrind simulates branch predictors intended to be typical of
mainstream desktop/server processors of around 2004."

Tony

===

#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>

typedef struct {
  // only square because I'm lazy
  size_t d;
  double *values;
} matrix;

matrix *
new_matrix(size_t d) {
  // missing error checking that should be done in production
  matrix *result = malloc(sizeof(matrix));
  *result = (matrix){
    .d = d,
    .values = malloc(sizeof(double) * d * d)
  };

  return result;
}

void
free_matrix(matrix *m) {
  free(m->values);
  free(m);
}

void
clear_matrix(matrix *m) {
  size_t limit = m->d * m->d;
  double *values = m->values;
  for (size_t i = 0; i < limit; ++i)
    values[i] = 0.0;
}

#define M(matrix, row, column) \
  ((matrix)->values[(row) * (matrix)->d + (column)])

matrix *
random_matrix(size_t d) {
  matrix *m = new_matrix(d);

  size_t limit = d * d;
  for (size_t i = 0; i < limit; ++i)
    m->values[i] = drand48();
  
  return m;
}

// multiply a by b result in c
// typical cache unfriendly implementation
void
matmult_dumb(matrix *a, matrix *b, matrix *c) {
  size_t d = c->d;
  clear_matrix(c);
  for (size_t i = 0; i < d; ++i) {
    for (size_t j = 0; j < d; ++j) {
      for (size_t k = 0; k < d; ++k) {
        M(c, i, j) += M(a, i, k) * M(b, k, j);
      }
    }
  }
}

// multiply a by b result in c
void
matmult_smart(matrix *a, matrix *b, matrix *c) {
  size_t d = c->d;
  clear_matrix(c);
  for (size_t i = 0; i < d; ++i) {
    for (size_t k = 0; k < d; ++k) {
        for (size_t j = 0; j < d; ++j) {
        M(c, i, j) += M(a, i, k) * M(b, k, j);
      }
    }
  }
}

#define MSIZE 200

int
main(int argc, char **argv) {
  bool smart = argc > 1;

  matrix *c = new_matrix(MSIZE);
  matrix *a = random_matrix(MSIZE);
  matrix *b = random_matrix(MSIZE);

  if (smart) {
    for (int i = 0; i < 100; ++i) {
      matmult_smart(a, b, c);
    }
  }
  else {
    for (int i = 0; i < 100; ++i) {
      matmult_dumb(a, b, c);
    }
  }
  free_matrix(c);
  free_matrix(a);
  free_matrix(b);
}