git: a99d04f39dab - main - hwpmc: add RAPL energy-counter class (AMD + Intel)
Mitchell Horne <[email protected]>
| Newsgroups | gmane.os.freebsd.devel.cvs.src,gmane.os.freebsd.current.scm |
|---|---|
| Message-ID | <[email protected]> |
The branch main has been updated by mhorne: URL: https://cgit.FreeBSD.org/src/commit/?id=a99d04f39dab0eac88eb4f5af425aceaf9238207 commit a99d04f39dab0eac88eb4f5af425aceaf9238207 Author: Andre Silva <[email protected]> AuthorDate: 2026-07-28 14:50:54 +0000 Commit: Mitchell Horne <[email protected]> CommitDate: 2026-07-28 14:54:21 +0000 hwpmc: add RAPL energy-counter class (AMD + Intel) Add hwpmc_rapl.c/.h implementing PMC_CLASS_RAPL, a read-only system-scope class modeled on TSC and wired into x86 AMD and Intel MD init. A per-vendor MSR table covers AMD/Hygon and Intel; energy is reported in microjoules, with the Intel server 2^-16 J DRAM unit handled and 32-bit wraps recovered into a 64-bit accumulator. The overflow guard follows the PMC lifetime: armed on the first allocated PMC, callout_drain()d on the last release, and each tick only rendezvouses CPUs holding one. Per-CPU spin locks guard the accumulator against torn reads on i386. PMC_CAP_DOMWIDE lets pmcstat(8) allocate one counter per NUMA domain instead of per CPU. Reviewed by: mhorne, Ali Mashtizadeh <[email protected]> Sponsored by: AMD Differential Revision: https://reviews.freebsd.org/D58028 --- sys/amd64/include/pmc_mdep.h | 1 + sys/conf/files.x86 | 1 + sys/dev/hwpmc/hwpmc_amd.c | 17 +- sys/dev/hwpmc/hwpmc_intel.c | 9 + sys/dev/hwpmc/hwpmc_rapl.c | 612 +++++++++++++++++++++++++++++++++++++++++++ sys/dev/hwpmc/hwpmc_rapl.h | 28 ++ sys/dev/hwpmc/pmc_events.h | 14 +- sys/i386/include/pmc_mdep.h | 1 + sys/modules/hwpmc/Makefile | 1 + sys/sys/pmc.h | 7 +- 10 files changed, 683 insertions(+), 8 deletions(-) diff --git a/sys/amd64/include/pmc_mdep.h b/sys/amd64/include/pmc_mdep.h index c44dd066f827..2adc16c05cdd 100644 --- a/sys/amd64/include/pmc_mdep.h +++ b/sys/amd64/include/pmc_mdep.h @@ -42,6 +42,7 @@ struct pmc_mdep; #include <dev/hwpmc/hwpmc_amd.h> #include <dev/hwpmc/hwpmc_core.h> #include <dev/hwpmc/hwpmc_ibs.h> +#include <dev/hwpmc/hwpmc_rapl.h> #include <dev/hwpmc/hwpmc_tsc.h> #include <dev/hwpmc/hwpmc_uncore.h> diff --git a/sys/conf/files.x86 b/sys/conf/files.x86 index a9b9d8875ad1..d700041e01d0 100644 --- a/sys/conf/files.x86 +++ b/sys/conf/files.x86 @@ -118,6 +118,7 @@ dev/hwpmc/hwpmc_ibs.c optional hwpmc dev/hwpmc/hwpmc_intel.c optional hwpmc dev/hwpmc/hwpmc_core.c optional hwpmc dev/hwpmc/hwpmc_uncore.c optional hwpmc +dev/hwpmc/hwpmc_rapl.c optional hwpmc dev/hwpmc/hwpmc_tsc.c optional hwpmc dev/hwpmc/hwpmc_x86.c optional hwpmc dev/hyperv/hvsock/hv_sock.c optional hyperv diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c index 185ceff133b0..41e1eaad9d67 100644 --- a/sys/dev/hwpmc/hwpmc_amd.c +++ b/sys/dev/hwpmc/hwpmc_amd.c @@ -1083,12 +1083,13 @@ pmc_amd_initialize(void) /* * These processors have two or three classes of PMCs: the TSC, - * programmable PMCs, and AMD IBS. + * programmable PMCs, and AMD IBS. One extra class slot is reserved + * for the optional RAPL energy counters. */ if ((amd_feature2 & AMDID2_IBS) != 0) { - nclasses = 3; + nclasses = 4; } else { - nclasses = 2; + nclasses = 3; } pmc_mdep = pmc_mdep_alloc(nclasses); @@ -1134,12 +1135,17 @@ pmc_amd_initialize(void) PMCDBG0(MDP, INI, 0, "amd-initialize"); - if (nclasses >= 3) { + if ((amd_feature2 & AMDID2_IBS) != 0) { error = pmc_ibs_initialize(pmc_mdep, ncpus); if (error != 0) goto error; } + /* RAPL takes the reserved last slot; drop it if the probe fails. */ + error = pmc_rapl_initialize(pmc_mdep, ncpus, pmc_mdep->pmd_nclass - 1); + if (error != 0) + pmc_mdep->pmd_nclass--; + return (pmc_mdep); error: @@ -1155,6 +1161,9 @@ pmc_amd_finalize(struct pmc_mdep *md) { PMCDBG0(MDP, INI, 1, "amd-finalize"); + /* Safe even if the RAPL class was skipped at initialize time. */ + pmc_rapl_finalize(md); + pmc_tsc_finalize(md); for (int i = 0; i < pmc_cpu_max(); i++) diff --git a/sys/dev/hwpmc/hwpmc_intel.c b/sys/dev/hwpmc/hwpmc_intel.c index 4c8d0cd2e2bd..f2cca80bff21 100644 --- a/sys/dev/hwpmc/hwpmc_intel.c +++ b/sys/dev/hwpmc/hwpmc_intel.c @@ -280,6 +280,9 @@ pmc_intel_initialize(void) return (NULL); } + /* Reserve one extra class slot for the optional RAPL counters. */ + nclasses++; + /* Allocate base class and initialize machine dependent struct */ pmc_mdep = pmc_mdep_alloc(nclasses); @@ -334,6 +337,10 @@ pmc_intel_initialize(void) default: break; } + + if (error == 0 && + pmc_rapl_initialize(pmc_mdep, ncpus, pmc_mdep->pmd_nclass - 1) != 0) + pmc_mdep->pmd_nclass--; error: if (error) { pmc_mdep_free(pmc_mdep); @@ -346,6 +353,8 @@ pmc_intel_initialize(void) void pmc_intel_finalize(struct pmc_mdep *md) { + pmc_rapl_finalize(md); + pmc_tsc_finalize(md); pmc_core_finalize(md); diff --git a/sys/dev/hwpmc/hwpmc_rapl.c b/sys/dev/hwpmc/hwpmc_rapl.c new file mode 100644 index 000000000000..b4a105a86dbb --- /dev/null +++ b/sys/dev/hwpmc/hwpmc_rapl.c @@ -0,0 +1,612 @@ +/* + * Copyright (c) 2026 Advanced Micro Devices, Inc. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +/* + * AMD/Intel RAPL energy counters exposed as an hwpmc(4) PMC class. + * + * Read-only, system-scope (PMC_MODE_SC), 64-bit counters reporting + * microjoules. + */ + +#include <sys/param.h> +#include <sys/bus.h> +#include <sys/callout.h> +#include <sys/malloc.h> +#include <sys/mutex.h> +#include <sys/pmc.h> +#include <sys/pmckern.h> +#include <sys/priv.h> +#include <sys/proc.h> +#include <sys/smp.h> +#include <sys/systm.h> + +#include <machine/cpu.h> +#include <machine/cpufunc.h> +#include <machine/cputypes.h> +#include <machine/specialreg.h> + +#include <x86/x86_var.h> + +#include <dev/hwpmc/hwpmc_rapl.h> + +/* Energy counters are per-package/core domains, not per CPU: DOMWIDE. */ +#define RAPL_CAPS (PMC_CAP_READ | PMC_CAP_DOMWIDE) + +/* Worst-case package watts for sizing the guard timer. */ +#define RAPL_GUARD_WATT 1000 + +/* Guard interval clamp band (ms). */ +#define RAPL_GUARD_MIN_MS 10 +#define RAPL_GUARD_MAX_MS 60000 + +struct rapl_event { + enum pmc_event re_ev; + uint32_t re_msr; + uint32_t re_unit; /* unit shift: 1 tick = 1/2^unit J */ +}; + +struct rapl_value { + uint64_t rv_prev; /* last raw 32-bit MSR value */ + uint64_t rv_accum; + sbintime_t rv_prev_time; + bool rv_primed; +}; + +struct rapl_cpu { + struct pmc_hw rc_hw[RAPL_MAX_NPMCS]; + struct rapl_value rc_value[RAPL_MAX_NPMCS]; + struct mtx rc_mtx; + int rc_nalloc; /* allocated RAPL PMCs on this CPU */ +}; + +static struct rapl_cpu **rapl_pcpu; + +static struct rapl_event rapl_events[RAPL_MAX_NPMCS]; +static struct pmc_descr rapl_pmcdesc[RAPL_MAX_NPMCS]; +static int rapl_npmcs; +static int rapl_ri; + +static struct callout rapl_guard_callout; +static sbintime_t rapl_guard_sbt; +static int rapl_nalloc; +static cpuset_t rapl_cpus; /* CPUs with an allocated RAPL PMC */ + +static struct mtx rapl_alloc_mtx; + +/* Convert energy ticks to microjoules without overflowing uint64_t. */ +static uint64_t +rapl_raw_to_uj(uint64_t raw, uint32_t shift) +{ + uint64_t unit, whole, frac; + + unit = 1ULL << shift; + whole = raw / unit; + frac = raw % unit; + return (whole * 1000000ULL + (frac * 1000000ULL) / unit); +} + +/* Fold a 32-bit MSR reading into the 64-bit accumulator, can recover one wrap. */ +static void +rapl_update_delta(struct rapl_value *val, uint64_t cur) +{ + sbintime_t now = sbinuptime(); + uint64_t diff; + + cur &= UINT32_MAX; + if (!val->rv_primed) { + val->rv_prev = cur; + val->rv_prev_time = now; + val->rv_primed = true; + return; + } + /* Skip sub-ms re-samples; the next sample folds the full interval. */ + if (now - val->rv_prev_time < SBT_1MS) + return; + if (cur >= val->rv_prev) + diff = cur - val->rv_prev; + else + diff = (UINT32_MAX - val->rv_prev) + cur + 1; + val->rv_accum += diff; + val->rv_prev = cur; + val->rv_prev_time = now; +} + +/* Sample one row's MSR on the current CPU and return the folded accumulator. */ +static uint64_t +rapl_sample_row(int cpu, int ri) +{ + struct rapl_cpu *rc; + uint64_t accum, cur; + + rc = rapl_pcpu[cpu]; + KASSERT(rc != NULL, ("[rapl,%d] null pcpu state cpu %d", __LINE__, + cpu)); + mtx_lock_spin(&rc->rc_mtx); + if (rdmsr_safe(rapl_events[ri].re_msr, &cur) == 0) + rapl_update_delta(&rc->rc_value[ri], cur); + accum = rc->rc_value[ri].rv_accum; + mtx_unlock_spin(&rc->rc_mtx); + return (accum); +} + +/* Guard rendezvous handler: sample every row on this CPU. */ +static void +rapl_guard_handler(void *arg __unused) +{ + int cpu = curcpu; + int ri; + + for (ri = 0; ri < rapl_npmcs; ri++) + (void)rapl_sample_row(cpu, ri); +} + +static void rapl_guard_tick(void *arg); + +/* (Re)arm the guard callout. Caller holds rapl_alloc_mtx. */ +static void +rapl_guard_schedule(void) +{ + mtx_assert(&rapl_alloc_mtx, MA_OWNED); + callout_reset_sbt(&rapl_guard_callout, rapl_guard_sbt, + rapl_guard_sbt / 10, rapl_guard_tick, NULL, 0); +} + +/* Periodic overflow guard. */ +static void +rapl_guard_tick(void *arg __unused) +{ + cpuset_t cpus; + + mtx_lock(&rapl_alloc_mtx); + cpus = rapl_cpus; + mtx_unlock(&rapl_alloc_mtx); + + if (!CPU_EMPTY(&cpus)) + smp_rendezvous_cpus(cpus, smp_no_rendezvous_barrier, + rapl_guard_handler, smp_no_rendezvous_barrier, NULL); + + /* Keep firing while any RAPL PMC remains allocated. */ + mtx_lock(&rapl_alloc_mtx); + if (rapl_nalloc > 0) + rapl_guard_schedule(); + mtx_unlock(&rapl_alloc_mtx); +} + +static int +rapl_allocate_pmc(int cpu, int ri, struct pmc *pm __unused, + const struct pmc_op_pmcallocate *a) +{ + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[rapl,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < rapl_npmcs, + ("[rapl,%d] illegal row index %d", __LINE__, ri)); + + if (a->pm_class != PMC_CLASS_RAPL) + return (EINVAL); + + if (a->pm_mode != PMC_MODE_SC) + return (EINVAL); + + /* Power side channel (PLATYPUS): require privilege even if syspmcs bypass is set. */ + if (priv_check(curthread, PRIV_PMC_SYSTEM) != 0) + return (EPERM); + + /* Reject events this vendor does not expose (e.g. DRAM on AMD). */ + if (a->pm_ev != rapl_events[ri].re_ev) + return (EINVAL); + + /* Arm the guard on the first allocation (per-CPU and global). */ + mtx_lock(&rapl_alloc_mtx); + if (rapl_pcpu[cpu]->rc_nalloc++ == 0) + CPU_SET(cpu, &rapl_cpus); + if (rapl_nalloc++ == 0) + rapl_guard_schedule(); + mtx_unlock(&rapl_alloc_mtx); + + return (0); +} + +static int +rapl_config_pmc(int cpu, int ri, struct pmc *pm) +{ + struct pmc_hw *phw; + + PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[rapl,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < rapl_npmcs, + ("[rapl,%d] illegal row-index %d", __LINE__, ri)); + + phw = &rapl_pcpu[cpu]->rc_hw[ri]; + + KASSERT(pm == NULL || phw->phw_pmc == NULL, + ("[rapl,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__, + pm, phw->phw_pmc)); + + phw->phw_pmc = pm; + + return (0); +} + +static int +rapl_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) +{ + const struct pmc_descr *pd; + struct pmc_hw *phw; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[rapl,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < rapl_npmcs, + ("[rapl,%d] illegal row-index %d", __LINE__, ri)); + + phw = &rapl_pcpu[cpu]->rc_hw[ri]; + pd = &rapl_pmcdesc[ri]; + + strlcpy(pi->pm_name, pd->pd_name, sizeof(pi->pm_name)); + pi->pm_class = pd->pd_class; + + if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { + pi->pm_enabled = true; + *ppmc = phw->phw_pmc; + } else { + pi->pm_enabled = false; + *ppmc = NULL; + } + + return (0); +} + +static int +rapl_get_config(int cpu, int ri, struct pmc **ppm) +{ + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[rapl,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < rapl_npmcs, + ("[rapl,%d] illegal row-index %d", __LINE__, ri)); + + *ppm = rapl_pcpu[cpu]->rc_hw[ri].phw_pmc; + + return (0); +} + +static int +rapl_pcpu_init(struct pmc_mdep *md __unused, int cpu) +{ + struct pmc_cpu *pc; + struct rapl_cpu *rapl_pc; + int ri, n; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[rapl,%d] illegal cpu %d", __LINE__, cpu)); + KASSERT(rapl_pcpu, ("[rapl,%d] null pcpu", __LINE__)); + KASSERT(rapl_pcpu[cpu] == NULL, ("[rapl,%d] non-null per-cpu", + __LINE__)); + + rapl_pc = malloc(sizeof(struct rapl_cpu), M_PMC, M_WAITOK | M_ZERO); + mtx_init(&rapl_pc->rc_mtx, "rapl-cpu", NULL, MTX_SPIN); + + for (n = 0; n < rapl_npmcs; n++) + rapl_pc->rc_hw[n].phw_state = PMC_PHW_FLAG_IS_ENABLED | + PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n) | + PMC_PHW_FLAG_IS_SHAREABLE; + + rapl_pcpu[cpu] = rapl_pc; + + KASSERT(pmc_pcpu, ("[rapl,%d] null generic pcpu", __LINE__)); + + pc = pmc_pcpu[cpu]; + + KASSERT(pc, ("[rapl,%d] null generic per-cpu", __LINE__)); + + for (n = 0; n < rapl_npmcs; n++) { + ri = rapl_ri + n; + pc->pc_hwpmcs[ri] = &rapl_pc->rc_hw[n]; + } + + return (0); +} + +static int +rapl_pcpu_fini(struct pmc_mdep *md __unused, int cpu) +{ + struct pmc_cpu *pc; + int ri, n; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[rapl,%d] illegal cpu %d", __LINE__, cpu)); + KASSERT(rapl_pcpu[cpu] != NULL, ("[rapl,%d] null pcpu", __LINE__)); + KASSERT(rapl_pcpu[cpu]->rc_nalloc == 0, + ("[rapl,%d] %d PMCs still allocated on cpu %d", __LINE__, + rapl_pcpu[cpu]->rc_nalloc, cpu)); + + /* Last release already drained the guard, so no handler can race here. */ + mtx_destroy(&rapl_pcpu[cpu]->rc_mtx); + free(rapl_pcpu[cpu], M_PMC); + rapl_pcpu[cpu] = NULL; + + pc = pmc_pcpu[cpu]; + for (n = 0; n < rapl_npmcs; n++) { + ri = rapl_ri + n; + pc->pc_hwpmcs[ri] = NULL; + } + + return (0); +} + +static int +rapl_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) +{ + enum pmc_mode mode __diagused; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[rapl,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < rapl_npmcs, + ("[rapl,%d] illegal ri %d", __LINE__, ri)); + + mode = PMC_TO_MODE(pm); + + KASSERT(mode == PMC_MODE_SC, + ("[rapl,%d] illegal pmc mode %d", __LINE__, mode)); + + PMCDBG1(MDP,REA,1, "rapl-read id=%d", ri); + + /* Bound to cpu by hwpmc, so rdmsr reads this CPU's domain (see DOMWIDE). */ + *v = rapl_raw_to_uj(rapl_sample_row(cpu, ri), rapl_events[ri].re_unit); + + return (0); +} + +static int +rapl_release_pmc(int cpu, int ri, struct pmc *pmc __unused) +{ + struct pmc_hw *phw __diagused; + bool last; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[rapl,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < rapl_npmcs, + ("[rapl,%d] illegal row-index %d", __LINE__, ri)); + + phw = &rapl_pcpu[cpu]->rc_hw[ri]; + + KASSERT(phw->phw_pmc == NULL, + ("[rapl,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); + + mtx_lock(&rapl_alloc_mtx); + KASSERT(rapl_pcpu[cpu]->rc_nalloc > 0 && rapl_nalloc > 0, + ("[rapl,%d] release underflow", __LINE__)); + if (--rapl_pcpu[cpu]->rc_nalloc == 0) + CPU_CLR(cpu, &rapl_cpus); + last = (--rapl_nalloc == 0); + mtx_unlock(&rapl_alloc_mtx); + + /* Last release: drain the guard (sleepable here, mutex already dropped). */ + if (last) + callout_drain(&rapl_guard_callout); + + return (0); +} + +static int +rapl_start_pmc(int cpu __diagused, int ri __diagused, struct pmc *pm __unused) +{ + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[rapl,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < rapl_npmcs, + ("[rapl,%d] illegal row-index %d", __LINE__, ri)); + + return (0); /* RAPL counters are always running. */ +} + +static int +rapl_stop_pmc(int cpu __diagused, int ri __diagused, struct pmc *pm __unused) +{ + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[rapl,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < rapl_npmcs, + ("[rapl,%d] illegal row-index %d", __LINE__, ri)); + + return (0); /* RAPL counters cannot be stopped. */ +} + +static int +rapl_write_pmc(int cpu __diagused, int ri __diagused, struct pmc *pm __unused, + pmc_value_t v __unused) +{ + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[rapl,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < rapl_npmcs, + ("[rapl,%d] illegal row-index %d", __LINE__, ri)); + + /* Energy counters are not writable; refuse silently like TSC. */ + return (0); +} + +/* Fault-safe RAPL MSR presence probe on the current CPU. */ +static bool +rapl_msr_present(uint32_t msr) +{ + uint64_t v; + + return (rdmsr_safe(msr, &v) == 0); +} + +/* Append an event row to the table if its MSR responds on this hardware. */ +static void +rapl_add_event(enum pmc_event ev, uint32_t msr, uint32_t unit, + const char *name) +{ + + if (!rapl_msr_present(msr)) + return; + + rapl_events[rapl_npmcs].re_ev = ev; + rapl_events[rapl_npmcs].re_msr = msr; + rapl_events[rapl_npmcs].re_unit = unit; + rapl_pmcdesc[rapl_npmcs].pd_class = PMC_CLASS_RAPL; + rapl_pmcdesc[rapl_npmcs].pd_caps = RAPL_CAPS; + rapl_pmcdesc[rapl_npmcs].pd_width = 64; + strlcpy(rapl_pmcdesc[rapl_npmcs].pd_name, name, + sizeof(rapl_pmcdesc[rapl_npmcs].pd_name)); + rapl_npmcs++; +} + +/* Guard interval: half the worst-case wrap period at RAPL_GUARD_WATT. */ +static sbintime_t +rapl_compute_guard_sbt(uint32_t shift) +{ + uint64_t max_energy_uj, guard_ms; + + max_energy_uj = rapl_raw_to_uj(UINT32_MAX, shift); + guard_ms = max_energy_uj / (2000ULL * RAPL_GUARD_WATT); + if (guard_ms < RAPL_GUARD_MIN_MS) + guard_ms = RAPL_GUARD_MIN_MS; + else if (guard_ms > RAPL_GUARD_MAX_MS) + guard_ms = RAPL_GUARD_MAX_MS; + + return (guard_ms * SBT_1MS); +} + +/* Fixed 2^-16 J DRAM unit, not the ESU (HSX/KNL only -- not SPR/EMR/GNR). */ +static bool +rapl_intel_fixed_dram_unit(void) +{ + + if (CPUID_TO_FAMILY(cpu_id) != 0x6) + return (false); + + switch (CPUID_TO_MODEL(cpu_id)) { + case 0x3f: /* Haswell-EP */ + case 0x4f: /* Broadwell-EP */ + case 0x55: /* Skylake/Cascade Lake/Cooper Lake-SP */ + case 0x56: /* Broadwell-DE */ + case 0x57: /* Xeon Phi KNL */ + case 0x6a: /* Ice Lake-SP */ + case 0x6c: /* Ice Lake-D */ + case 0x85: /* Xeon Phi KNM */ + return (true); + default: + return (false); + } +} + +int +pmc_rapl_initialize(struct pmc_mdep *md, int maxcpu, int classindex) +{ + struct pmc_classdep *pcd; + uint32_t unit_msr, pkg_msr, cores_msr, dram_msr; + uint32_t esu, dram_unit, max_unit; + uint64_t unit_val; + int i; + + KASSERT(md != NULL, ("[rapl,%d] md is NULL", __LINE__)); + KASSERT(md->pmd_nclass >= 1, ("[rapl,%d] dubious md->nclass %d", + __LINE__, md->pmd_nclass)); + + /* Select the per-vendor MSR set. */ + switch (cpu_vendor_id) { + case CPU_VENDOR_AMD: + case CPU_VENDOR_HYGON: + unit_msr = MSR_AMD_RAPL_POWER_UNIT; + pkg_msr = MSR_AMD_PKG_ENERGY_STATUS; + cores_msr = MSR_AMD_CORE_ENERGY_STATUS; + dram_msr = 0; /* AMD has no DRAM domain */ + break; + case CPU_VENDOR_INTEL: + unit_msr = MSR_RAPL_POWER_UNIT; + pkg_msr = MSR_PKG_ENERGY_STATUS; + cores_msr = MSR_PP0_ENERGY_STATUS; + dram_msr = MSR_DRAM_ENERGY_STATUS; + break; + default: + return (ENXIO); + } + + /* Decode the energy unit. */ + unit_val = rdmsr(unit_msr); + esu = (unit_val >> 8) & 0x1f; + dram_unit = rapl_intel_fixed_dram_unit() ? 16 : esu; + + /* Build the event table from the MSRs that actually respond. */ + rapl_npmcs = 0; + rapl_add_event(PMC_EV_RAPL_ENERGY_PKG, pkg_msr, esu, + "RAPL_ENERGY_PKG"); + rapl_add_event(PMC_EV_RAPL_ENERGY_CORES, cores_msr, esu, + "RAPL_ENERGY_CORES"); + if (dram_msr != 0) + rapl_add_event(PMC_EV_RAPL_ENERGY_DRAM, dram_msr, dram_unit, + "RAPL_ENERGY_DRAM"); + + /* No RAPL energy MSR responded. */ + if (rapl_npmcs == 0) + return (ENXIO); + + /* Size the guard for the fastest-wrapping row (largest unit shift). */ + max_unit = 0; + for (i = 0; i < rapl_npmcs; i++) + max_unit = MAX(max_unit, rapl_events[i].re_unit); + rapl_guard_sbt = rapl_compute_guard_sbt(max_unit); + rapl_nalloc = 0; + CPU_ZERO(&rapl_cpus); + + mtx_init(&rapl_alloc_mtx, "rapl-alloc", NULL, MTX_DEF); + /* It does not need associated mutex, the handler locks itself. */ + callout_init(&rapl_guard_callout, 1); + + rapl_pcpu = malloc(sizeof(struct rapl_cpu *) * maxcpu, M_PMC, + M_ZERO | M_WAITOK); + + pcd = &md->pmd_classdep[classindex]; + + pcd->pcd_caps = RAPL_CAPS; + pcd->pcd_class = PMC_CLASS_RAPL; + pcd->pcd_num = rapl_npmcs; + pcd->pcd_ri = md->pmd_npmc; + pcd->pcd_width = 64; + + pcd->pcd_allocate_pmc = rapl_allocate_pmc; + pcd->pcd_config_pmc = rapl_config_pmc; + pcd->pcd_describe = rapl_describe; + pcd->pcd_get_config = rapl_get_config; + pcd->pcd_pcpu_init = rapl_pcpu_init; + pcd->pcd_pcpu_fini = rapl_pcpu_fini; + pcd->pcd_read_pmc = rapl_read_pmc; + pcd->pcd_release_pmc = rapl_release_pmc; + pcd->pcd_start_pmc = rapl_start_pmc; + pcd->pcd_stop_pmc = rapl_stop_pmc; + pcd->pcd_write_pmc = rapl_write_pmc; + + rapl_ri = md->pmd_npmc; + md->pmd_npmc += rapl_npmcs; + + return (0); +} + +void +pmc_rapl_finalize(struct pmc_mdep *md __unused) +{ + PMCDBG0(MDP, INI, 1, "rapl-finalize"); + + if (rapl_pcpu == NULL) + return; + + KASSERT(rapl_nalloc == 0, ("[rapl,%d] %d PMCs still allocated", + __LINE__, rapl_nalloc)); + for (int i = 0; i < pmc_cpu_max(); i++) + KASSERT(rapl_pcpu[i] == NULL, ("[rapl,%d] non-null pcpu cpu %d", + __LINE__, i)); + + mtx_destroy(&rapl_alloc_mtx); + + free(rapl_pcpu, M_PMC); + rapl_pcpu = NULL; +} diff --git a/sys/dev/hwpmc/hwpmc_rapl.h b/sys/dev/hwpmc/hwpmc_rapl.h new file mode 100644 index 000000000000..bc9f3dbdd952 --- /dev/null +++ b/sys/dev/hwpmc/hwpmc_rapl.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2026 Advanced Micro Devices, Inc. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +/* + * AMD/Intel RAPL energy counters exposed as an hwpmc(4) PMC class. + */ + +#ifndef _DEV_HWPMC_RAPL_H_ +#define _DEV_HWPMC_RAPL_H_ 1 + +#ifdef _KERNEL + +struct pmc_mdep; + +/* + * Max RAPL rows per CPU: package, "cores" and (Intel only) DRAM. The actual + * per-vendor count is determined at initialize time (pcd_num). + */ +#define RAPL_MAX_NPMCS 3 + +int pmc_rapl_initialize(struct pmc_mdep *_md, int _maxcpu, int _classindex); +void pmc_rapl_finalize(struct pmc_mdep *_md); + +#endif /* _KERNEL */ +#endif /* _DEV_HWPMC_RAPL_H_ */ diff --git a/sys/dev/hwpmc/pmc_events.h b/sys/dev/hwpmc/pmc_events.h index b22ec1919b32..f21abd0b77a7 100644 --- a/sys/dev/hwpmc/pmc_events.h +++ b/sys/dev/hwpmc/pmc_events.h @@ -53,6 +53,15 @@ #define __PMC_EV_ALIAS_TSC() \ __PMC_EV_ALIAS("cycles", TSC_TSC) +/* RAPL energy counters. */ +#define __PMC_EV_RAPL() \ + __PMC_EV(RAPL, ENERGY_PKG) \ + __PMC_EV(RAPL, ENERGY_CORES) \ + __PMC_EV(RAPL, ENERGY_DRAM) + +#define PMC_EV_RAPL_FIRST PMC_EV_RAPL_ENERGY_PKG +#define PMC_EV_RAPL_LAST PMC_EV_RAPL_ENERGY_DRAM + /* * Software events are dynamically defined. */ @@ -2428,6 +2437,7 @@ __PMC_EV_ALIAS("unhalted-reference-cycles", IAF_CPU_CLK_UNHALTED_REF) * 0x14500 0x0020 ARM DMC-620 clkdiv2 events * 0x14520 0x0080 ARM DMC-620 clk events * 0x14600 0x0100 ARM CMN-600 events + * 0x14700 0x0100 AMD/Intel RAPL energy events * 0x20000 0x1000 Software events */ #define __PMC_EVENTS() \ @@ -2454,7 +2464,9 @@ __PMC_EV_ALIAS("unhalted-reference-cycles", IAF_CPU_CLK_UNHALTED_REF) __PMC_EV_BLOCK(DMC620_PMU_C, 0x14520) \ __PMC_EV_DMC620_PMU_C() \ __PMC_EV_BLOCK(CMN600_PMU, 0x14600) \ - __PMC_EV_CMN600_PMU() + __PMC_EV_CMN600_PMU() \ + __PMC_EV_BLOCK(RAPL, 0x14700) \ + __PMC_EV_RAPL() #define PMC_EVENT_FIRST PMC_EV_TSC_TSC #define PMC_EVENT_LAST PMC_EV_SOFT_LAST diff --git a/sys/i386/include/pmc_mdep.h b/sys/i386/include/pmc_mdep.h index 618df8daebef..e8c500e4b81d 100644 --- a/sys/i386/include/pmc_mdep.h +++ b/sys/i386/include/pmc_mdep.h @@ -52,6 +52,7 @@ struct pmc_mdep; #include <dev/hwpmc/hwpmc_amd.h> #include <dev/hwpmc/hwpmc_ibs.h> #include <dev/hwpmc/hwpmc_core.h> +#include <dev/hwpmc/hwpmc_rapl.h> #include <dev/hwpmc/hwpmc_tsc.h> #include <dev/hwpmc/hwpmc_uncore.h> diff --git a/sys/modules/hwpmc/Makefile b/sys/modules/hwpmc/Makefile index 52d64fc9ec42..7d8918c862f8 100644 --- a/sys/modules/hwpmc/Makefile +++ b/sys/modules/hwpmc/Makefile @@ -24,6 +24,7 @@ SRCS+= hwpmc_amd.c \ hwpmc_core.c \ hwpmc_ibs.c \ hwpmc_intel.c \ + hwpmc_rapl.c \ hwpmc_tsc.c \ hwpmc_uncore.c \ hwpmc_x86.c diff --git a/sys/sys/pmc.h b/sys/sys/pmc.h index cf0382f66af3..dc2e9cd9109a 100644 --- a/sys/sys/pmc.h +++ b/sys/sys/pmc.h @@ -60,7 +60,7 @@ * The patch version is incremented for every bug fix. */ #define PMC_VERSION_MAJOR 0x0A -#define PMC_VERSION_MINOR 0x01 +#define PMC_VERSION_MINOR 0x02 #define PMC_VERSION_PATCH 0x0000 #define PMC_VERSION (PMC_VERSION_MAJOR << 24 | \ @@ -156,7 +156,8 @@ enum pmc_cputype { __PMC_CLASS(POWER8, 0x15, "IBM POWER8 class") \ __PMC_CLASS(DMC620_PMU_CD2, 0x16, "ARM DMC620 Memory Controller PMU CLKDIV2") \ __PMC_CLASS(DMC620_PMU_C, 0x17, "ARM DMC620 Memory Controller PMU CLK") \ - __PMC_CLASS(CMN600_PMU, 0x18, "Arm CoreLink CMN600 Coherent Mesh Network PMU") + __PMC_CLASS(CMN600_PMU, 0x18, "Arm CoreLink CMN600 Coherent Mesh Network PMU") \ + __PMC_CLASS(RAPL, 0x19, "AMD/Intel RAPL energy counters") enum pmc_class { #undef __PMC_CLASS @@ -165,7 +166,7 @@ enum pmc_class { }; #define PMC_CLASS_FIRST PMC_CLASS_TSC -#define PMC_CLASS_LAST PMC_CLASS_CMN600_PMU +#define PMC_CLASS_LAST PMC_CLASS_RAPL /* * A PMC can be in the following states: