[PATCH v2 01/10] Squashed 'scripts/nixos-flake/' content from commit b599d320
Daniel Gomez <[email protected]> Fri, 12 Jun 2026 12:55:13 +0200
| Newsgroups | dev.linux.lists.kdevops |
|---|---|
| Message-ID | <[email protected]> |
From: Daniel Gomez <[email protected]> git-subtree-dir: scripts/nixos-flake git-subtree-split: b599d320b12a3b320ab825e97d97d6bef6467c34 --- .editorconfig | 12 + .gitignore | 2 + CLAUDE.md | 244 ++++++++++ COPYING | 22 + LICENSE | 5 + LICENSES/preferred/copyleft-next-0.3.1 | 239 ++++++++++ README.md | 179 ++++++++ docs/design-decisions.md | 393 ++++++++++++++++ docs/usage.md | 802 +++++++++++++++++++++++++++++++++ docs/verifying.md | 72 +++ flake.lock | 27 ++ flake.nix | 239 ++++++++++ lib/toolchain.nix | 50 ++ modules/backends/imageless/default.nix | 216 +++++++++ modules/backends/imageless/user.nix | 24 + modules/backends/libvirt/default.nix | 77 ++++ modules/backends/libvirt/user.nix | 28 ++ modules/mounts/shares.nix | 49 ++ modules/mounts/storage.nix | 130 ++++++ modules/profiles/build-tools.nix | 24 + modules/profiles/controller.nix | 49 ++ modules/profiles/devel.nix | 125 +++++ modules/profiles/monitoring.nix | 464 +++++++++++++++++++ modules/testSuites/blktests.nix | 30 ++ modules/testSuites/fstests.nix | 370 +++++++++++++++ modules/testSuites/gitr.nix | 22 + modules/testSuites/ltp.nix | 34 ++ modules/testSuites/mmtests.nix | 24 + modules/testSuites/pynfs.nix | 22 + modules/testSuites/selftests.nix | 26 ++ modules/testSuites/sysbench.nix | 27 ++ modules/user-options.nix | 38 ++ modules/user.nix | 38 ++ overlays/default.nix | 17 + overlays/fio.nix | 36 ++ overlays/spdk.nix | 17 + overlays/xfstests.nix | 45 ++ pkgs/cpupower.nix | 65 +++ pkgs/damo.nix | 63 +++ pkgs/default.nix | 21 + pkgs/libbpf-tools.nix | 186 ++++++++ pkgs/nfstest.nix | 88 ++++ pkgs/pynfs.nix | 96 ++++ pkgs/xnvme.nix | 57 +++ templates/imageless/default.nix | 41 ++ templates/imageless/flake.nix | 70 +++ templates/libvirt/default.nix | 41 ++ templates/libvirt/flake.nix | 78 ++++ 48 files changed, 5024 insertions(+) diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..f15441ab --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..34554eca --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +result +configurations/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..d5f89c65 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,244 @@ +# CLAUDE.md + +## Project Overview + +Library flake of NixOS modules, overlays, and templates for +provisioning NixOS systems. Two backend modules differ in the +artifact each produces: `backends.imageless` (a system closure +that boots a VM directly — tmpfs root, systemd initramfs, +external kernel, `/nix/store` and `/lib/modules` via virtiofs) +and `backends.libvirt` (a qcow2 disk image — `/dev/vda` root, +grub, NixOS-built kernel). The imageless closure also activates +on real hardware with `switch-to-configuration` instead of QEMU +— the baremetal case. Opt-in modules — `user`, the mounts +modules (`mounts.shares`, `mounts.storage`), the profiles +(`profiles.devel`, `profiles.build-tools`, `profiles.monitoring`, +`profiles.controller`), and the per-suite modules under +`testSuites.*` — compose on top of a backend; the `controller` +profile turns a host into a control node that builds kernels and +provisions guests. + +**License**: copyleft-next-0.3.1 + +## Project Structure + +``` +nixos-flake/ +├── flake.nix Flake (nixosModules, overlays, templates, packages, checks) +├── flake.lock Pinned nixpkgs revision +├── docs/ +│ ├── usage.md Configurations, overlays, packages, updating +│ └── design-decisions.md Hardcoded choices and upstream references +├── modules/ +│ ├── backends/ +│ │ ├── imageless/ Imageless boot (tmpfs root, virtiofs, networkd), standalone +│ │ │ ├── default.nix Backend module +│ │ │ └── user.nix Imageless extraGroups defaults for the unprivileged account +│ │ └── libvirt/ Disk-image boot (grub on vda, DHCP, nix+flakes), standalone +│ │ ├── default.nix Backend module +│ │ └── user.nix Libvirt extraGroups defaults for the unprivileged account +│ ├── profiles/ +│ │ ├── build-tools.nix Build toolchain (Autotools plus common headers) +│ │ ├── controller.nix Control-node profile: kernel build, Ansible, libvirt (options.nixos-flake.controller) +│ │ ├── devel.nix Development profile (kernel testing tools) +│ │ └── monitoring.nix Test-run-bracketed monitor units (options.nixos-flake.monitoring) +│ ├── mounts/ +│ │ ├── shares.nix Opt-in virtiofs shares (options.nixos-flake.shares) +│ │ └── storage.nix Opt-in block-device mounts with optional mkfs (options.nixos-flake.storage) +│ ├── testSuites/ Per-suite modules (fstests, blktests, ltp, …) +│ ├── user.nix Opt-in unprivileged account with configurable name (options.nixos-flake.user) +│ └── user-options.nix Shared option schema for the unprivileged account +├── pkgs/ +│ ├── default.nix Custom packages via callPackage +│ ├── cpupower.nix Standalone cpupower from kernel source tree +│ ├── damo.nix DAMON user-space tool +│ ├── libbpf-tools.nix CO-RE BPF tracing tools from BCC +│ ├── nfstest.nix NFS test suite +│ ├── pynfs.nix Python NFSv4 conformance tests +│ └── xnvme.nix Cross-platform NVMe library and tools +├── overlays/ +│ ├── default.nix Composes per-package overlays + pkgs/ +│ ├── fio.nix fio with liburing + test suite + examples +│ └── xfstests.nix Bump xfstests to 2026.03.20 +├── templates/ +│ ├── imageless/flake.nix Imageless starter (default) +│ └── libvirt/flake.nix Libvirt disk-image starter +├── LICENSES/ +│ └── preferred/copyleft-next-0.3.1 +├── COPYING License overview and dual-licensing guidance +├── LICENSE Copyright and license reference +└── README.md +``` + +## Critical Rules + +### Never fabricate facts + +Every NixOS option must be verified against the NixOS options search +(search.nixos.org/options) or the nixpkgs source. + +### Long-form command options + +NEVER use short flags when a long-form alternative exists. +`--template` not `-t`, `--recursive` not `-r`, `--parents` not +`-p`. Exception: tools without long-form options (`ssh -p`). + +### Reference only upstream projects + +Comments and commit messages describe this flake and the upstream +projects it packages — the Linux kernel, QEMU, xfstests, SPDK, BCC, +and so on. Never name the downstream projects that combine this +flake, nor the pipeline that drives it. The flake stands on its own; +how a consumer wires it up belongs in that consumer's tree. + +## Rules + +### Imageless: external kernel, NixOS-built initramfs + +`boot.kernel.enable = false` tells NixOS not to build a kernel. +`boot.initrd.systemd.enable = true` tells NixOS to build a systemd +initramfs that mounts root (tmpfs), /nix/store, and /lib/modules +via virtiofs before switch-root into the system closure. + +### Imageless: tmpfs root + +`fileSystems."/" = lib.mkImageMediaOverride { fsType = "tmpfs"; }` +declares root as tmpfs. `mkImageMediaOverride` has higher priority +than default NixOS filesystem declarations. Changes are lost on +shutdown. + +### Imageless: minimal profile + +`imports = [ (modulesPath + "/profiles/minimal.nix") ]` reduces the +closure size by excluding unnecessary packages. Libvirt does not +import this profile; disk-image deployments expect a full system. + +### Libvirt: grub on /dev/vda, ext4 root + +`boot.loader.grub.device = "/dev/vda"` and +`fileSystems."/" = { device = "/dev/vda1"; fsType = "ext4"; }` +assume the standard virt-builder qcow2 layout. Consumers with +different disk bus or partition schemes override these. + +### Password auth + +`users.users.root.initialPassword = "root"` with +`users.mutableUsers = false` sets the root password on every boot +(tmpfs root resets it). The password is a serial-console +break-glass; SSH itself is key-only (each backend module sets +`PasswordAuthentication = false` inline to keep modules +standalone). + +### Overlays + +Each file in `overlays/` overrides one nixpkgs package using +`overrideAttrs`. The `overlays/default.nix` composes all per-package +overlays and merges custom packages from `pkgs/` into a single +overlay exported as `overlays.default`. + +### Custom packages (pkgs/) + +Packages not available in nixpkgs are defined in `pkgs/` using the +`callPackage` pattern from nix.dev. Each package is a file declaring +a function whose arguments are its dependencies. The overlay imports +`pkgs/default.nix` and merges them into the nixpkgs set. + +### Templates + +`templates/imageless/` (default) and `templates/libvirt/` are +copied by `nix flake init --template`. Each imports its matching +backend plus `user`, `devel`, and `overlays.default`. Users add +their own packages and NixOS options below. + +### Baremetal: activate, do not boot + +The imageless backend builds a `system.build.toplevel` closure. +On real hardware the closure is activated with +`<closure>/bin/switch-to-configuration switch` rather than booted +in QEMU. There is no baremetal module: the difference from the VM +case is how the closure is activated, not how it is built. A +baremetal host overrides the `fileSystems` and `boot` options the +imageless module declares for the virtiofs case. + +### Controller: gated profile, composes on a backend + +`profiles/controller.nix` is gated behind +`nixos-flake.controller.enable` and composes on top of a +disk-booted backend. It adds the kernel and Kconfig build +toolchain, Ansible and its Python runtime, git, the QEMU and +virtiofs tooling, and system libvirt with the QEMU/KVM stack +(`runAsRoot` off, virtiofsd as a vhost-user backend). + +### Per-backend unprivileged-account defaults + +`modules/user.nix` declares the unprivileged account and imports +the shared option schema from `modules/user-options.nix`. Each +backend ships its own `user.nix` (e.g. +`modules/backends/imageless/user.nix`) that sets +`nixos-flake.user.extraGroups` at `lib.mkDefault` priority — the +imageless backend defaults to `[ "wheel" "kvm" ]`, the libvirt +backend to `[ "wheel" "kvm" "libvirtd" ]`. Each backend's +`default.nix` imports its sibling `user.nix` so the default +applies whenever the backend is composed. The shared schema in +`user-options.nix` is what lets the backend set the option even +when `modules/user.nix` itself is not in the import set. + +## Build + +```shell +nix flake check # both backends +nix build .#checks.x86_64-linux.imageless # single backend +readlink --canonicalize result +``` + +The `result` symlink points to the system closure. The NixOS init +is at `<closure>/init`. The init= path changes on every rebuild. + +Before committing, run the verification steps in +[docs/verifying.md](docs/verifying.md): `nix fmt`, `nix flake check`, +the libvirt disk-image build, and the custom-package builds. + +## Git Commit Guidelines + +### One commit per change + +Atomic commits. Spell fixes go in separate commits from code changes. + +### Commit message format + +``` +subsystem: brief description in imperative mood (max 50 chars) + +Plain English explanation of the change, 1-3 short paragraphs. +NEVER use bullet points or itemized lists in commit messages. + +Generated-by: Claude AI +Signed-off-by: Your Name <[email protected]> +``` + +The subject line stays at or below 50 characters. + +### Use Signed-off-by and Generated-by tags + +Generated-by MUST be immediately followed by Signed-off-by with NO +empty lines between them. No Co-Authored-By trailer. + +### No shopping cart lists + +NEVER use bullet points or itemized lists in commit messages. Use +plain English paragraphs. + +### Subsystem prefix + +Prefix with the part of the flake the change touches: `flake:` for +flake.nix, `modules:` or the specific module name for a module, +`overlays:` for package overlays, `pkgs:` for custom packages, +`templates:` for template changes, `docs:` for documentation in +docs/, `README:` for README changes. Use `tree:` for the rare +change that genuinely spans the whole repository. + +## Related work + +- [run-kernel](https://github.com/metaspace/run-kernel). Rust init + NixOS boot via virtiofs. The direct inspiration for this project's boot model. +- [nixos-shell](https://github.com/Mic92/nixos-shell). Nix-based lightweight QEMU VMs with host mounts. +- [kernel-development-flake](https://github.com/jordanisaacs/kernel-development-flake). Nix flake for Linux kernel development with QEMU. diff --git a/COPYING b/COPYING new file mode 100644 index 00000000..f5479109 --- /dev/null +++ b/COPYING @@ -0,0 +1,22 @@ +This project is provided under: + + SPDX-License-Identifier: copyleft-next-0.3.1 + LICENSES/preferred/copyleft-next-0.3.1 + +In addition, other licenses may also apply. We embrace the same SPDX +practice as used in the Linux kernel, for those details refer to under +Linux: + + Documentation/process/license-rules.rst + +for more details. + +Note: Linux accepts copyleft-next licensed code, however, even though +copyleft-next is GPL v2 compatible we use a dual license tag on Linux +to err on the side of caution. And so, if you ever do wish to use code +from this project on Linux be sure to use this tag instead: + +// SPDX-License-Identifier: GPL-2.0-or-later OR copyleft-next-0.3.1 + +This project equally accepts GPL-2.0 code only as copyleft-next is GPL +v2 compatible. diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..38249038 --- /dev/null +++ b/LICENSE @@ -0,0 +1,5 @@ +Copyright (c) 2026 Daniel Gomez <[email protected]> + +nixos-flake is licensed under copyleft-next-0.3.1 + +Refer to LICENSES/preferred/copyleft-next-0.3.1 for license details. diff --git a/LICENSES/preferred/copyleft-next-0.3.1 b/LICENSES/preferred/copyleft-next-0.3.1 new file mode 100644 index 00000000..431ea9e9 --- /dev/null +++ b/LICENSES/preferred/copyleft-next-0.3.1 @@ -0,0 +1,239 @@ +Valid-License-Identifier: copyleft-next-0.3.1 +SPDX-URL: https://spdx.org/licenses/copyleft-next-0.3.1 +Usage-Guide: + This license can be used in code, it has been found to be GPLv2 compatible + by attorneys at Redhat and SUSE, however to err on the side of caution, + if used on Linux it's best to only use it together with a GPL2 compatible + license using "OR". You do not have to do this for this project, kdevops, + as this project is licensed under the copyleft-next-0.3.1 license. + To use the copyleft-next-0.3.1 license on Linux put the following SPDX + tag/value pair into a comment according to the placement guidelines in the + licensing rules documentation: + SPDX-License-Identifier: GPL-2.0 OR copyleft-next-0.3.1 + SPDX-License-Identifier: GPL-2.0-only OR copyleft-next 0.3.1 + SPDX-License-Identifier: GPL-2.0+ OR copyleft-next-0.3.1 + SPDX-License-Identifier: GPL-2.0-or-later OR copyleft-next-0.3.1 +License-Text: + +======================================================================= + + copyleft-next 0.3.1 ("this License") + Release date: 2016-04-29 + +1. License Grants; No Trademark License + + Subject to the terms of this License, I grant You: + + a) A non-exclusive, worldwide, perpetual, royalty-free, irrevocable + copyright license, to reproduce, Distribute, prepare derivative works + of, publicly perform and publicly display My Work. + + b) A non-exclusive, worldwide, perpetual, royalty-free, irrevocable + patent license under Licensed Patents to make, have made, use, sell, + offer for sale, and import Covered Works. + + This License does not grant any rights in My name, trademarks, service + marks, or logos. + +2. Distribution: General Conditions + + You may Distribute Covered Works, provided that You (i) inform + recipients how they can obtain a copy of this License; (ii) satisfy the + applicable conditions of sections 3 through 6; and (iii) preserve all + Legal Notices contained in My Work (to the extent they remain + pertinent). "Legal Notices" means copyright notices, license notices, + license texts, and author attributions, but does not include logos, + other graphical images, trademarks or trademark legends. + +3. Conditions for Distributing Derived Works; Outbound GPL Compatibility + + If You Distribute a Derived Work, You must license the entire Derived + Work as a whole under this License, with prominent notice of such + licensing. This condition may not be avoided through such means as + separate Distribution of portions of the Derived Work. + + If the Derived Work includes material licensed under the GPL, You may + instead license the Derived Work under the GPL. + +4. Condition Against Further Restrictions; Inbound License Compatibility + + When Distributing a Covered Work, You may not impose further + restrictions on the exercise of rights in the Covered Work granted under + this License. This condition is not excused merely because such + restrictions result from Your compliance with conditions or obligations + extrinsic to this License (such as a court order or an agreement with a + third party). + + However, You may Distribute a Covered Work incorporating material + governed by a license that is both OSI-Approved and FSF-Free as of the + release date of this License, provided that compliance with such + other license would not conflict with any conditions stated in other + sections of this License. + +5. Conditions for Distributing Object Code + + You may Distribute an Object Code form of a Covered Work, provided that + you accompany the Object Code with a URL through which the Corresponding + Source is made available, at no charge, by some standard or customary + means of providing network access to source code. + + If you Distribute the Object Code in a physical product or tangible + storage medium ("Product"), the Corresponding Source must be available + through such URL for two years from the date of Your most recent + Distribution of the Object Code in the Product. However, if the Product + itself contains or is accompanied by the Corresponding Source (made + available in a customarily accessible manner), You need not also comply + with the first paragraph of this section. + + Each direct and indirect recipient of the Covered Work from You is an + intended third-party beneficiary of this License solely as to this + section 5, with the right to enforce its terms. + +6. Symmetrical Licensing Condition for Upstream Contributions + + If You Distribute a work to Me specifically for inclusion in or + modification of a Covered Work (a "Patch"), and no explicit licensing + terms apply to the Patch, You license the Patch under this License, to + the extent of Your copyright in the Patch. This condition does not + negate the other conditions of this License, if applicable to the Patch. + +7. Nullification of Copyleft/Proprietary Dual Licensing + + If I offer to license, for a fee, a Covered Work under terms other than + a license that is OSI-Approved or FSF-Free as of the release date of this + License or a numbered version of copyleft-next released by the + Copyleft-Next Project, then the license I grant You under section 1 is no + longer subject to the conditions in sections 3 through 5. + +8. Copyleft Sunset + + The conditions in sections 3 through 5 no longer apply once fifteen + years have elapsed from the date of My first Distribution of My Work + under this License. + +9. Pass-Through + + When You Distribute a Covered Work, the recipient automatically receives + a license to My Work from Me, subject to the terms of this License. + +10. Termination + + Your license grants under section 1 are automatically terminated if You + + a) fail to comply with the conditions of this License, unless You cure + such noncompliance within thirty days after becoming aware of it, or + + b) initiate a patent infringement litigation claim (excluding + declaratory judgment actions, counterclaims, and cross-claims) + alleging that any part of My Work directly or indirectly infringes + any patent. + + Termination of Your license grants extends to all copies of Covered + Works You subsequently obtain. Termination does not terminate the + rights of those who have received copies or rights from You subject to + this License. + + To the extent permission to make copies of a Covered Work is necessary + merely for running it, such permission is not terminable. + +11. Later License Versions + + The Copyleft-Next Project may release new versions of copyleft-next, + designated by a distinguishing version number ("Later Versions"). + Unless I explicitly remove the option of Distributing Covered Works + under Later Versions, You may Distribute Covered Works under any Later + Version. + +** 12. No Warranty ** +** ** +** My Work is provided "as-is", without warranty. You bear the risk ** +** of using it. To the extent permitted by applicable law, each ** +** Distributor of My Work excludes the implied warranties of title, ** +** merchantability, fitness for a particular purpose and ** +** non-infringement. ** + +** 13. Limitation of Liability ** +** ** +** To the extent permitted by applicable law, in no event will any ** +** Distributor of My Work be liable to You for any damages ** +** whatsoever, whether direct, indirect, special, incidental, or ** +** consequential damages, whether arising under contract, tort ** +** (including negligence), or otherwise, even where the Distributor ** +** knew or should have known about the possibility of such damages. ** + +14. Severability + + The invalidity or unenforceability of any provision of this License + does not affect the validity or enforceability of the remainder of + this License. Such provision is to be reformed to the minimum extent + necessary to make it valid and enforceable. + +15. Definitions + + "Copyleft-Next Project" means the project that maintains the source + code repository at <https://github.com/copyleft-next/copyleft-next.git/> + as of the release date of this License. + + "Corresponding Source" of a Covered Work in Object Code form means (i) + the Source Code form of the Covered Work; (ii) all scripts, + instructions and similar information that are reasonably necessary for + a skilled developer to generate such Object Code from the Source Code + provided under (i); and (iii) a list clearly identifying all Separate + Works (other than those provided in compliance with (ii)) that were + specifically used in building and (if applicable) installing the + Covered Work (for example, a specified proprietary compiler including + its version number). Corresponding Source must be machine-readable. + + "Covered Work" means My Work or a Derived Work. + + "Derived Work" means a work of authorship that copies from, modifies, + adapts, is based on, is a derivative work of, transforms, translates or + contains all or part of My Work, such that copyright permission is + required. The following are not Derived Works: (i) Mere Aggregation; + (ii) a mere reproduction of My Work; and (iii) if My Work fails to + explicitly state an expectation otherwise, a work that merely makes + reference to My Work. + + "Distribute" means to distribute, transfer or make a copy available to + someone else, such that copyright permission is required. + + "Distributor" means Me and anyone else who Distributes a Covered Work. + + "FSF-Free" means classified as 'free' by the Free Software Foundation. + + "GPL" means a version of the GNU General Public License or the GNU + Affero General Public License. + + "I"/"Me"/"My" refers to the individual or legal entity that places My + Work under this License. "You"/"Your" refers to the individual or legal + entity exercising rights in My Work under this License. A legal entity + includes each entity that controls, is controlled by, or is under + common control with such legal entity. "Control" means (a) the power to + direct the actions of such legal entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent of the + outstanding shares or beneficial ownership of such legal entity. + + "Licensed Patents" means all patent claims licensable royalty-free by + Me, now or in the future, that are necessarily infringed by making, + using, or selling My Work, and excludes claims that would be infringed + only as a consequence of further modification of My Work. + + "Mere Aggregation" means an aggregation of a Covered Work with a + Separate Work. + + "My Work" means the particular work of authorship I license to You + under this License. + + "Object Code" means any form of a work that is not Source Code. + + "OSI-Approved" means approved as 'Open Source' by the Open Source + Initiative. + + "Separate Work" means a work that is separate from and independent of a + particular Covered Work and is not by its nature an extension or + enhancement of the Covered Work, and/or a runtime library, standard + library or similar component that is used to generate an Object Code + form of a Covered Work. + + "Source Code" means the preferred form of a work for making + modifications to it. diff --git a/README.md b/README.md new file mode 100644 index 00000000..b123be18 --- /dev/null +++ b/README.md @@ -0,0 +1,179 @@ +# nixos-flake + +NixOS modules, overlays, and templates for provisioning NixOS +systems. The same module set drives several modes: + +- **imageless guest**: a system closure that boots a VM directly — + tmpfs root, systemd initramfs, external kernel, `/nix/store` and + `/lib/modules` mounted via virtiofs from the host. No disk image + and nothing persists across boots. +- **libvirt guest**: a qcow2 disk image consumed by libvirt — + `/dev/vda` root, grub bootloader, NixOS-built kernel, DHCP from + libvirt's default network. +- **baremetal**: the imageless system activated on real hardware + with `switch-to-configuration` instead of booted in QEMU. +- **controller**: a host that builds kernels, runs Kconfig and + Ansible, and provisions the guests above. + +All modes compose with the opt-in modules — `user`, the mounts +modules (`shares`, `storage`), the profiles (`devel`, +`build-tools`, `monitoring`, `controller`), and the per-suite +modules under `testSuites.*` — so a system can be minimal or +carry the full kernel and storage testing toolchain. + +Beyond provisioning, the flake exposes development shells that bring +the same reproducible toolchain to any host via `nix develop`: +`build-kernel` and `build-qemu` for kernel and QEMU builds (the +package set the `controller` profile installs, defined once in +`lib/toolchain.nix`) and `systemd` for the host systemd control +toolkit. + +**License**: copyleft-next-0.3.1 + +## Prerequisites + +The [Nix package manager](https://nixos.org/download/) with flake +support enabled: + +```shell +mkdir --parents ~/.config/nix +echo 'experimental-features = nix-command flakes' >> ~/.config/nix/nix.conf +``` + +## Quick start + +Validate that both backends evaluate and build: + +```shell +nix flake check +``` + +Build a single backend closure: + +```shell +nix build .#checks.x86_64-linux.imageless # or .libvirt +readlink --canonicalize result +``` + +The `result` symlink points to the system closure. For imageless, +`result/boot.json` contains the `init` and `initrd` paths needed to +configure QEMU: + +```shell +cat result/boot.json +``` + +To create a downstream configuration, use a template: + +```shell +nix flake init --template "github:linux-kdevops/nixos-flake#imageless" +nix flake init --template "github:linux-kdevops/nixos-flake#libvirt" +``` + +See [docs/usage.md](docs/usage.md) for customizing packages and +NixOS options. + +## Development shells + +The same toolchain is available on any host without building a NixOS +system: + +```shell +nix develop .#build-kernel -c make ... # kernel build env (gcc) +nix develop .#build-kernel -c make LLVM=1 ... # clang +nix develop .#build-qemu -c make ... # QEMU build env +nix develop .#systemd -c systemctl --user list-units # host systemd control +``` + +See [docs/usage.md](docs/usage.md) for what each shell provides. + +## How it boots + +### Imageless + +This backend builds two artifacts: a NixOS system closure and a +systemd initramfs. Booting requires an external kernel and QEMU +with virtiofsd sharing the host's `/nix/store` and `/lib/modules` +into the guest. + +The external kernel must have the boot-critical virtio drivers +built-in (`CONFIG_VIRTIO_FS=y`, `CONFIG_VIRTIO_PCI=y`, +`CONFIG_TMPFS=y`). All other drivers can be kernel modules loaded +from `/lib/modules` after switch-root. + +QEMU needs two virtiofsd instances sharing host directories into +the guest with these tags: + +- `store`: the host's `/nix/store` (read-only) +- `modules`: the kernel build's `/lib/modules` directory + +The kernel command line: + +``` +root=tmpfs console=ttyS0,115200 console=hvc0 init=/nix/store/<hash>/init +``` + +systemd in the initramfs reads the NixOS-generated fstab, mounts +root (tmpfs), `/nix/store` (virtiofs tag `store`), and +`/lib/modules` (virtiofs tag `modules`), then switch-roots into +the system closure. The `init=` and `initrd` paths change on +every rebuild and are available in `result/boot.json`. + +### Libvirt + +This backend builds a full NixOS system closure including its own +kernel. The consumer supplies a qcow2 disk image containing the +closure; libvirt presents it to the guest as `/dev/vda`. Grub on +the MBR loads the kernel, initramfs mounts the ext4 root from +`/dev/vda1`, and systemd activation proceeds normally. Networking +comes up via DHCP from libvirt's default network (typically +`192.168.122.0/24`). + +## Baremetal + +The imageless backend builds a NixOS system closure. A VM is one +way to run it; a real machine is another. Activate the closure on +baremetal with the closure's own switch script — no QEMU, no VM: + +```shell +<closure>/bin/switch-to-configuration switch +``` + +The host then runs the same NixOS system the imageless backend +builds. A baremetal host supplies its own root filesystem and +kernel, so it overrides the `fileSystems` and `boot` entries the +imageless module declares for the virtiofs case. + +## Controller + +The `profiles.controller` module turns a NixOS host into a control node: +the toolchain to build a kernel and drive `make menuconfig`, +Ansible and its Python runtime, git, the QEMU and virtiofs +tooling, and system libvirt with the QEMU/KVM stack. Enable it on +top of a backend on a real machine: + +```nix +modules = [ + nixos-flake.nixosModules.backends.libvirt + nixos-flake.nixosModules.profiles.controller + { nixos-flake.controller.enable = true; } +]; +``` + +The same host can also import a test-suite module and run the +suite itself — the baremetal case above, driven from the +controller. + +## Documentation + +| Document | Content | +|---|---| +| [docs/usage.md](docs/usage.md) | Configurations, modules, overlays, packages, home overlay, block-device filesystems | +| [docs/design-decisions.md](docs/design-decisions.md) | Imageless and libvirt boot model, upstream references | +| [docs/verifying.md](docs/verifying.md) | Pre-commit checklist: format, flake check, image and package builds | + +## Related work + +- [run-kernel](https://github.com/metaspace/run-kernel). Rust init + NixOS boot via virtiofs. The direct inspiration for the imageless boot model. +- [nixos-shell](https://github.com/Mic92/nixos-shell). Nix-based lightweight QEMU VMs with host mounts. +- [kernel-development-flake](https://github.com/jordanisaacs/kernel-development-flake). Nix flake for Linux kernel development with QEMU. diff --git a/docs/design-decisions.md b/docs/design-decisions.md new file mode 100644 index 00000000..cf8beb3f --- /dev/null +++ b/docs/design-decisions.md @@ -0,0 +1,393 @@ +# Design decisions + +This project exposes two NixOS backend modules that produce +different artifacts for different consumers: `imageless` builds a +system closure that QEMU boots directly with `/nix/store` and +`/lib/modules` served over virtiofs (tmpfs root, systemd +initramfs, external kernel), and `libvirt` builds a qcow2 disk +image that libvirt consumes (qcow2 root, grub, NixOS-built kernel, +libvirt DHCP). Each section below explains the upstream mechanisms +the backend relies on and the design choices the module makes, +deferring to NixOS defaults where practical and using +`lib.mkDefault` so downstream consumers can override. + +# Imageless + +Boots from a NixOS system closure and systemd initramfs running +against the host's `/nix/store` shared read-only into the guest. +Root is tmpfs and does not persist across boots. + +## systemd initramfs + +NixOS builds a systemd-based initramfs where systemd runs as +PID 1. systemd's +[fstab-generator](https://github.com/systemd/systemd/blob/main/src/fstab-generator/fstab-generator.c) +reads the NixOS-generated initrd fstab and creates mount units +for `/nix/store` automatically. All upstream code, no custom init +binary. + +The NixOS module that implements `boot.initrd.systemd` is +[`nixos/modules/system/boot/systemd/initrd.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/system/boot/systemd/initrd.nix). +The initrd fstab generation and `SYSTEMD_SYSROOT_FSTAB` wiring is +in +[`nixos/modules/tasks/filesystems.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/tasks/filesystems.nix). + +The configuration enables the systemd initramfs and declares +virtiofs filesystems: + +```nix +boot.initrd.systemd.enable = true; +boot.initrd.supportedFilesystems = [ "virtiofs" ]; + +fileSystems."/nix/store" = { + device = "store"; + fsType = "virtiofs"; +}; +``` + +Kernel command line (standard parameters only): +``` +root=tmpfs console=ttyS0,115200 console=hvc0 init=/nix/store/<hash>/init +``` + +## How the systemd initramfs works + +The boot sequence uses standard systemd and NixOS mechanisms. Every +step is upstream code. + +### NixOS initrd fstab generation + +NixOS generates an initrd-specific fstab from `fileSystems` +declarations. The function `fsNeededForBoot` in +`nixos/lib/utils.nix` determines which filesystems go into the +initrd fstab. It returns true for any filesystem where either +`neededForBoot = true` or the mount point is in `pathsNeededForBoot`: + +```nix +pathsNeededForBoot = [ + "/" + "/nix" + "/nix/store" + "/var" + "/var/log" + "/var/lib" + "/var/lib/nixos" + "/etc" + "/usr" +]; +``` + +`/nix/store` is explicitly in this list. Any `fileSystems."/nix/store"` +declaration is automatically included in the initrd fstab without +requiring `neededForBoot = true`. + +The initrd fstab is written to a file and passed to systemd via the +`SYSTEMD_SYSROOT_FSTAB` environment variable. NixOS wires this up in +`nixos/modules/tasks/filesystems.nix` through +`boot.initrd.systemd.managerEnvironment` and the `initrd-parse-etc` +service environment. + +### systemd fstab-generator in the initrd + +systemd-fstab-generator runs in the initrd and reads the initrd +fstab (from `SYSTEMD_SYSROOT_FSTAB`). For each entry, it generates +a systemd mount unit. The mount points are prefixed with `/sysroot` +because the generator runs in the initrd context +(`src/fstab-generator/fstab-generator.c`, `prefix_sysroot` logic). + +For our configuration, the generator creates: +- `sysroot.mount`: tmpfs on `/sysroot` (from `root=tmpfs`) +- `sysroot-nix-store.mount`: virtiofs `store` on `/sysroot/nix/store` + +### root=tmpfs handling + +systemd-fstab-generator explicitly supports `root=tmpfs` as a +shortcut for a writable tmpfs root (see the `arg_root_what == "tmpfs"` +branch in `src/fstab-generator/fstab-generator.c`): + +```c +} else if (streq(arg_root_what, "tmpfs")) { + /* If root=tmpfs is specified, then take this as shortcut + for a writable tmpfs mount as root */ + what = strdup("rootfs"); + fstype = arg_root_fstype ?: "tmpfs"; +``` + +This creates a tmpfs mount at `/sysroot` with mode 0755. + +### switch-root sequence + +After all initrd mounts complete (`initrd-fs.target`), systemd +performs switch-root to `/sysroot`: + +1. systemd reaches `initrd.target` (all initrd services done) +2. `initrd-cleanup.service` runs +3. `initrd-switch-root.target` activates +4. `initrd-switch-root.service` calls `systemctl switch-root /sysroot` + +After switch-root, `/sysroot` becomes `/`. The virtiofs mount that +was at `/sysroot/nix/store` is now at `/nix/store`. The NixOS +stage-2 init at `/nix/store/<hash>/init` becomes accessible at its +expected path. + +### Kernel module matching + +When NixOS builds the initramfs, it can include kernel modules +from the NixOS kernel package. These modules must match the +running kernel version exactly. With an external custom kernel +(`boot.kernel.enable = false`), the versions will not match and +module loading in the initramfs will fail. + +The solution is to exclude all kernel modules from the initramfs +and provide them via virtiofs instead: + +```nix +boot.initrd.availableKernelModules = lib.mkForce []; +boot.initrd.kernelModules = lib.mkForce []; +``` + +This requires the external kernel to have the boot-critical +drivers built-in: `CONFIG_VIRTIO_FS=y` (mount /nix/store and +/lib/modules in the initramfs), `CONFIG_VIRTIO_PCI=y` (PCI +transport), and `CONFIG_TMPFS=y` (root filesystem). All other +drivers can be kernel modules (`=m`), loaded from `/lib/modules` +after switch-root. The `/lib/modules` directory is mounted via +virtiofs from the external kernel build's module install path. + +## Root filesystem: tmpfs + +```nix +fileSystems."/" = lib.mkImageMediaOverride { + fsType = "tmpfs"; + options = [ "mode=0755" ]; +}; +``` + +Root is tmpfs. Everything written to `/` is lost on shutdown. This +is the standard NixOS approach for ephemeral systems. The operating +system state comes from `/nix/store` (read-only, shared from host) +and `/etc` (generated by NixOS activation from the store). + +`lib.mkImageMediaOverride` sets the NixOS option priority to 60, +overriding the default root filesystem declaration from NixOS +modules. Without this, NixOS expects a persistent root device. +See: `lib/modules.nix` in nixpkgs (`mkImageMediaOverride`). + +`systemd.services.systemd-remount-fs.enable = false` is set because +there is nothing to remount. The root is already writable tmpfs. + +## /nix/store: virtiofs read-only + +```nix +fileSystems."/nix/store" = { + device = "store"; + fsType = "virtiofs"; +}; +``` + +The Nix store is immutable by design. Packages are content-addressed +and never modified in place. Read-only virtiofs mounting enforces +this at the mount level. The `device` field is the virtiofs tag +name that must match the tag configured in the virtiofsd instance +sharing the host's `/nix/store` into the guest. + +## External kernel + +```nix +boot.kernel.enable = false; +``` + +NixOS does not build a kernel. The kernel is built separately using +Kconfig fragments and installed to a destdir. This allows rapid +kernel development iteration without rebuilding the NixOS closure. + +The kernel command line is passed to QEMU via the `-append` flag +(or equivalent configuration), not by NixOS. The imageless module +does not set `boot.kernelParams` because no NixOS bootloader is +active to enforce it. The QEMU configuration must include +`root=tmpfs` and `init=<closure>/init` explicitly. + +## Minimal profile + +```nix +imports = [ (modulesPath + "/profiles/minimal.nix") ]; +``` + +The minimal profile disables documentation, fonts, and other +non-essential modules. This reduces the system closure size from +~500MB to ~200MB. The closure contains only systemd, SSH, network +configuration, and coreutils. + +## Password authentication + +```nix +users.mutableUsers = false; +users.users.root.initialPassword = "root"; +services.openssh.settings.PasswordAuthentication = lib.mkDefault false; +``` + +SSH itself is key-only: the imageless module sets +`PasswordAuthentication = lib.mkDefault false`, and consumers +inject authorized keys per node. Root keeps a known +`initialPassword` for serial-console break-glass only. Because +root is tmpfs, `/etc/shadow` is generated fresh on every boot from +the NixOS configuration; `mutableUsers = false` ensures the +password is always reset to the configured value. The weak +password is not reachable over SSH and the console socket is only +accessible from the host. + +## systemd-networkd + +```nix +networking.useNetworkd = true; +systemd.network.networks."80-ethernet" = { + matchConfig.Name = "en*"; + networkConfig.DHCP = "yes"; +}; +``` + +systemd-networkd is the standard network manager for systemd-based +systems. NetworkManager is heavier and designed for desktop use. +The network configuration matches all ethernet interfaces (virtio +NIC appears as `enp0s2` in QEMU with q35 machine type) and enables +DHCP. + +# Libvirt + +Boots from a qcow2 disk image that libvirt presents to the guest as +`/dev/vda`. The kernel and initramfs are part of the NixOS closure +installed on the disk. Grub on the MBR loads them; systemd +activation proceeds normally. + +## Grub on /dev/vda + +```nix +boot.loader.grub = { + enable = true; + device = "/dev/vda"; +}; +boot.loader.timeout = lib.mkDefault 1; +``` + +Libvirt's default virtio-blk configuration exposes the qcow2 disk +to the guest as `/dev/vda`. Grub installs into its MBR. The short +one-second timeout skips the interactive menu for test boots. +Consumers with a different disk bus (SCSI, SATA) set +`boot.loader.grub.device` to the matching path; consumers who +prefer the interactive menu override `boot.loader.timeout`. + +## ext4 on /dev/vda1 + +```nix +fileSystems."/" = { + device = "/dev/vda1"; + fsType = "ext4"; +}; +``` + +Assumes the standard virt-builder qcow2 layout (MBR partition +table, first partition formatted ext4). Consumers with a different +partition scheme override `fileSystems."/".device` and `fsType`. + +## Kernel from nixpkgs + +```nix +boot.kernelPackages = lib.mkDefault pkgs.linuxPackages_latest; +``` + +The libvirt backend does not disable `boot.kernel.enable`; NixOS +builds a kernel as part of the closure. `linuxPackages_latest` +tracks the most recent nixpkgs kernel. Consumers who want to pin +to a specific branch or use a custom kernel override +`boot.kernelPackages`. + +## Scripted DHCP + +```nix +networking.useDHCP = lib.mkDefault true; +``` + +Libvirt's default network assigns DHCP leases from +`192.168.122.0/24`. `networking.useDHCP` enables the NixOS +scripted-networking path, which brings up every interface with +DHCP. Consumers who prefer systemd-networkd or need static +addresses override `useDHCP` and declare their own configuration. +(Imageless uses `useNetworkd = true` instead because its minimal +profile excludes scripted-networking; libvirt carries the full +system, so the default scripted path works.) + +## Nix with flakes and weekly garbage collection + +```nix +nix.settings.experimental-features = [ "nix-command" "flakes" ]; +nix.gc = { + automatic = lib.mkDefault true; + dates = lib.mkDefault "weekly"; + options = lib.mkDefault "--delete-older-than 7d"; +}; +``` + +Flakes are required because consumers may re-evaluate nix +expressions inside the guest. Weekly garbage collection with a +seven-day retention keeps the qcow2 from growing unbounded +across long-running test iterations. + +## No nested libvirt + +```nix +virtualisation.libvirtd.enable = lib.mkDefault false; +``` + +Nested virtualization is off by default. Use cases that need +virt-within-virt (KVM nested, for instance) opt in from a per-node +override. + +## SSH posture matches imageless + +The libvirt module inherits the same key-only sshd posture +described under Imageless. It does not carry its own +`initialPassword`: disk persistence means any break-glass setup +survives, and consumers compose the user module (which does set +an initial root password) on top when they need it. A standalone +libvirt consumer with SSH keys injected does not get a weak +password baked in. + +# Baremetal + +The imageless backend's `system.build.toplevel` is a complete +NixOS system closure. Booting it in QEMU is one consumer; a real +machine is another. `<closure>/bin/switch-to-configuration switch` +activates the closure on the running host: it sets up `/etc`, +users, and systemd units and reloads the service manager. With no +bootloader enabled (`boot.loader.grub.enable = false`) the switch +skips the bootloader step. + +The imageless module declares `fileSystems` for the virtiofs case +(tmpfs root, `/nix/store` and `/lib/modules` over virtiofs). On +baremetal those declarations do not match the real hardware, so a +baremetal host overrides `fileSystems` and the `boot` options for +its own root device and kernel. No separate module is needed: the +difference between the VM and baremetal cases is how the closure +is activated, not how it is built. + +# Controller + +A controller is a host that provisions and drives the systems +above rather than being one of them. The `controller` module +(`modules/profiles/controller.nix`) is a profile, gated behind +`nixos-flake.controller.enable`, that adds three things to a host. + +The first is the toolchain to build a Linux kernel and drive its +Kconfig interface: the menuconfig frontend's ncurses dependency +plus the parser generators, libraries, and helpers a modern +kernel tree's `make` invokes. The second is the orchestration +layer: Ansible and the Python runtime its playbooks import, git, +and the QEMU image and virtiofs tooling. The third is system +libvirt with the QEMU/KVM stack, with `runAsRoot` off and +virtiofsd registered as a vhost-user backend so guests share the +host store without an elevated daemon. + +The module composes on top of a disk-booted backend — typically +libvirt — on a real machine. A controller that also imports a +test-suite module runs that suite itself, which is the baremetal +case. diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 00000000..8f4dfa1f --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,802 @@ +# Usage + +nixos-flake exports NixOS modules, overlays, and templates. +Downstream projects consume these as a flake input to build their +own NixOS systems — the imageless backend yields a system closure +consumed directly by QEMU + virtiofsd, and the libvirt backend +yields a qcow2 disk image consumed by libvirt. This document +covers building, creating configurations, and customizing +packages. + +## Validating the library + +`nix flake check` evaluates and builds both backends so you can +verify the modules still produce a valid system: + +```shell +nix flake check +``` + +To build a single backend directly: + +```shell +nix build .#checks.x86_64-linux.imageless +nix build .#checks.x86_64-linux.libvirt +readlink --canonicalize result +``` + +The `result` symlink points to the system closure in `/nix/store`. +The NixOS init is at `<closure>/init`. The `init=` path contains a +Nix store hash that changes on every rebuild. + +For kernel development, create a downstream configuration from a +template; see the next section. + +List all files installed by a package (equivalent to +`dpkg --listfiles` on Debian): + +```shell +find $(dirname $(readlink --canonicalize $(which fio)))/.. -type f | sort +``` + +`readlink --canonicalize` resolves the profile symlink to the +actual store path. Replace `fio` with any binary name. + +## Development shells + +The flake exposes development shells so the same reproducible +toolchain is available on any host, NixOS or not, without building a +NixOS system. They are the non-NixOS counterpart to the `controller` +profile: both draw from `lib/toolchain.nix`, so a NixOS control node +and a `nix develop` shell agree on versions. + +`nix develop .#build-kernel` enters the kernel build environment — +gcc, clang, rust and sparse, the LLVM binutils for `LLVM=1`, ccache, +b4, and the kernel `rust-src`. `nix develop .#build-qemu` adds QEMU's +own build inputs (meson/ninja/glib/...) for building QEMU from source. + +```shell +nix develop .#build-kernel -c make ... # gcc +nix develop .#build-kernel -c make LLVM=1 ... # clang +nix develop .#build-qemu -c make ... # QEMU +``` + +`nix develop .#systemd` provides the systemd client tools +(`systemctl`, `systemd-run`, `busctl`, `varlinkctl`, `loginctl`) for +driving a host's `systemd --user` manager over its D-Bus socket: + +```shell +nix develop .#systemd -c systemctl --user list-units +``` + +## Creating a configuration + +A configuration is a standalone flake project: its own directory +with a `flake.nix` that references nixos-flake as a flake input +and imports its modules and overlays. Each configuration has its own +`flake.lock` and `result` symlink. Multiple configurations can coexist +independently. + +The `configurations/` directory is gitignored for this purpose. +Create configurations there to keep them out of the tracked tree: + +```shell +mkdir --parents configurations/my-vm && cd configurations/my-vm +``` + +### Using the template + +The flake exports a +[template](https://nix.dev/manual/nix/stable/command-ref/new-cli/nix3-flake-init) +that scaffolds a configuration with the development profile, +overlays, and commented-out examples for building packages from +local source checkouts: + +```shell +nix flake init --template "path:$PWD/../.." +``` + +`nix flake init` copies `templates/imageless/flake.nix` into the +current directory. The copied file is independent of the template: +future changes to the upstream template do not update your copy. +The link between your configuration and nixos-flake is the +`inputs.nixos-flake.url` flake input, not the template itself. + +Edit `flake.nix` and set `nixos-flake.url` to the absolute path of +your nixos-flake checkout. The `path:` scheme does not expand `~`, +use the full path or `$HOME`: + +```shell +$EDITOR flake.nix +``` + +Nix flakes only evaluate files +[tracked by git](https://nix.dev/manual/nix/stable/command-ref/new-cli/nix3-flake#flake-references). +The `git add` is required before the first build. Subsequent edits +to `flake.nix` are picked up from the working tree without a new +commit: + +```shell +git init && git add flake.nix +nix build .#nixosConfigurations.vm.config.system.build.toplevel +readlink --canonicalize result +``` + +### Writing a configuration from scratch + +The template is optional. Any flake that declares nixos-flake as an +input and imports its modules works as a configuration: + +```nix +{ + inputs = { + nixos-flake.url = "path:/home/user/src/nixos-flake"; + nixpkgs.follows = "nixos-flake/nixpkgs"; + }; + + outputs = { self, nixpkgs, nixos-flake, ... }: { + nixosConfigurations.vm = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ + nixos-flake.nixosModules.backends.imageless + nixos-flake.nixosModules.profiles.devel + { + nixpkgs.overlays = [ nixos-flake.overlays.default ]; + environment.systemPackages = with nixpkgs.legacyPackages.x86_64-linux; [ + gdb + ]; + } + ]; + }; + }; +} +``` + +`nixpkgs.follows = "nixos-flake/nixpkgs"` ensures both flakes use +the same nixpkgs revision. Without it, the configuration would +pull a second copy of nixpkgs. + +### What the modules provide + +The `modules/` tree is grouped into nix-native buckets so the +import path matches the file path. Backends define the system +shape (one per system); profiles add capability on top; mounts +emit `fileSystems` entries from a small DSL; testSuites carry +per-suite package sets and service wiring. + +`nixosModules.backends.imageless` +(`modules/backends/imageless/default.nix`) declares the imageless +base NixOS system: + +- Root as tmpfs (`fileSystems."/" = lib.mkImageMediaOverride { fsType = "tmpfs"; }`) +- Key-only SSH; root carries a known initial password (`root`) + for serial-console break-glass only +- systemd-networkd with DHCP on `en*` interfaces +- No bootloader, no kernel (external, `boot.kernel.enable = false`) +- Firewall disabled, unnecessary services disabled + +It also auto-imports its sibling `user.nix`, which sets +`nixos-flake.user.extraGroups` to `[ "wheel" "kvm" ]` at +`lib.mkDefault` priority so the unprivileged account picks up +the right groups when `nixosModules.user` is composed. + +`nixosModules.backends.libvirt` +(`modules/backends/libvirt/default.nix`) is the disk-image +counterpart: grub on `/dev/vda`, ext4 root, DHCP through +scripted networking, nix with flakes, and weekly garbage +collection. Each backend module is standalone (no cross-import); +both set `system.stateVersion` and the sshd posture inline. +This backend's bundled `user.nix` defaults extraGroups to +`[ "wheel" "kvm" "libvirtd" ]` since a libvirt-backed host +frequently runs libvirtd locally (controller nodes compose the +controller profile on top of this backend). + +`nixosModules.user` (`modules/user.nix`) adds an opt-in +unprivileged account with passwordless sudo. The account name is +parametric through `options.nixos-flake.user.name` (default +`user`); consumers override it to their project name. The +`extraGroups` option defaults to `[ ]` here; each backend ships a +sibling user.nix that sets backend-appropriate defaults via +`lib.mkDefault`. The module also sets the serial-console +break-glass root password. + +`nixosModules.mounts.shares` (`modules/mounts/shares.nix`) turns +a set of virtiofs shares into `fileSystems` entries. Each share +is keyed by its mount point and declares the virtiofs `tag` +announced by the host and optional mount `options`. The module +does nothing else: overlays, XDG env vars, and directory creation +stay the consumer's responsibility. See "Host configuration via +virtiofs" for usage. + +`nixosModules.mounts.storage` (`modules/mounts/storage.nix`) +turns a set of block devices into `fileSystems` entries. Each +entry is keyed by mount point with fields `device`, `fsType`, +`options`, `mkfsArgs`, and `autoFormat`. Set `mkfsArgs` for a +pre-mount `mkfs` oneshot with custom arguments, or +`autoFormat = true` for NixOS's default first-mount formatting +(the two are mutually exclusive). See "Block device filesystems" +for usage. + +`nixosModules.profiles.devel` (`modules/profiles/devel.nix`) +adds kernel testing and storage tools, grouped by purpose: +filesystem and block layer tooling (xfstests, xfsprogs, +btrfs-progs, e2fsprogs, lvm2, parted), NVMe and SCSI (nvme-cli, +libnvme, sg3_utils, xnvme), I/O generation (fio with liburing, +stress-ng), NFS (nfs-utils, nfstest, pynfs), BPF and tracing +(bpftrace, bcc, libbpf-tools, blktrace, trace-cmd), monitoring +and profiling (perf, cpupower, damo, htop, iotop, numactl, +pagemon, powertop, sysstat), selftest and blktests/fstests +runtime dependencies (acl, attr, keyutils, libcap, libseccomp, +mdadm, quota, rpcbind), and general developer comfort (editors, +shells, git, jq, strace, kmod). The module also pulls in +`programs.bash.completion.enable` and the Python data-analysis +stack (matplotlib, numpy, pandas). See `modules/profiles/devel.nix` +for the authoritative list. + +## Updating nixpkgs + +`flake.lock` pins the exact nixpkgs revision used by every build. +Refresh it to pick up upstream package and module changes: + +```shell +nix flake update +git add flake.lock +nix flake check +``` + +`flake.lock` must be tracked by git for the updated revision to +be picked up on the next build. + +Downstream configurations that use `nixpkgs.follows` inherit +the nixpkgs pin from nixos-flake. After updating nixos-flake, run +`nix flake update` in each configuration to pick up the new pin. + +## Overlays + +Nix +[overlays](https://nixos.org/manual/nixpkgs/stable/#chap-overlays) +modify or extend the nixpkgs package set. An overlay is a function +`final: prev: { ... }` where `prev` is the package set from all +previous overlays and `final` is the fully evaluated set including +the current overlay. Overlays are applied in order, so each layer +can see and build on the changes from previous layers. + +[`overrideAttrs`](https://nixos.org/manual/nixpkgs/stable/#sec-pkg-overrideAttrs) +modifies a derivation by merging new attributes into the existing +ones. Attributes not specified in the override are retained from +the original. This means overrides compose: each `overrideAttrs` +call wraps the previous derivation, replacing only the attributes +it specifies. + +### What nixos-flake provides + +The `overlays/` directory customizes nixpkgs packages. Each file +overrides one package. The default overlay (`overlays.default`) +composes all per-package overlays and merges custom packages from +`pkgs/`. + +Currently included: + +- **fio**: enables liburing, installs t/io_uring exerciser, + NVMe test scripts, and example job files +- **xfstests**: bumps nixpkgs 2023.05.14 (broken with modern + GCC) to 2026.03.20 + +### Overriding a package from a configuration + +A configuration lists overlays in order. nixos-flake's overlay +goes first, then any user overlays that build on top of it. + +For example, xfstests goes through three layers: + +``` +nixpkgs base → xfstests 2023.05.14 (broken) +nixos-flake.overlays.default → xfstests 2026.03.20 (bumped) +user overlay → xfstests from local checkout +``` + +The user overlay's `prev.xfstests` is the xfstests from +nixos-flake's overlay (the bumped version). `overrideAttrs` +replaces only `src`, keeping everything else (build inputs, +install phase, patches) from the previous layer: + +```nix +nixpkgs.overlays = [ + nixos-flake.overlays.default + (final: prev: { + xfstests = prev.xfstests.overrideAttrs { + src = inputs.xfstests-src; + }; + }) +]; +``` + +The `xfstests-src` input is a local source checkout declared as +a flake input with `flake = false` (see the template for +commented-out examples). + +### Pinning a specific upstream commit + +To pin a package to a specific commit without a local checkout, +use a fetcher in the overlay. The fetcher depends on where the +canonical upstream repository is hosted. + +For projects hosted on kernel.org, use `fetchgit`: + +```nix +# xfstests: git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git +(final: prev: { + xfstests = prev.xfstests.overrideAttrs { + src = final.fetchgit { + url = "git://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git"; + rev = "<commit-hash>"; + hash = ""; + }; + }; +}) + +# fio: https://git.kernel.org/pub/scm/linux/kernel/git/axboe/fio +(final: prev: { + fio = prev.fio.overrideAttrs { + src = final.fetchgit { + url = "https://git.kernel.org/pub/scm/linux/kernel/git/axboe/fio.git"; + rev = "<commit-hash>"; + hash = ""; + }; + patches = []; + }; +}) +``` + +For projects hosted on GitHub, use `fetchFromGitHub`: + +```nix +# damo: https://github.com/damonitor/damo +(final: prev: { + damo = prev.damo.overrideAttrs { + src = final.fetchFromGitHub { + owner = "damonitor"; + repo = "damo"; + rev = "<commit-hash>"; + hash = ""; + }; + }; +}) + +# xnvme: https://github.com/xnvme/xnvme +(final: prev: { + xnvme = prev.xnvme.overrideAttrs { + src = final.fetchFromGitHub { + owner = "xnvme"; + repo = "xnvme"; + rev = "<commit-hash>"; + hash = ""; + }; + }; +}) +``` + +Nix will fail with a hash mismatch on the first build and print +the correct `hash` value to use. Clear `patches = [];` when the +upstream source already includes fixes that nixos-flake's overlay +backports. + +The user never needs to modify nixos-flake's overlays. All +customization happens in the configuration's own `flake.nix`. + +## Building from local source + +To build a package from a local source checkout (for development +or testing), declare the source directory as a flake input with +`flake = false` and reference it in an overlay. This lets you +rebuild the NixOS closure with your modified source without +publishing it upstream. + +In the `inputs` block of your configuration's `flake.nix`: + +```nix +kmod-src = { + url = "path:/home/user/src/kmod"; + flake = false; +}; +``` + +`flake = false` tells Nix to import the path as plain source +rather than expecting a `flake.nix`. Then in the `nixpkgs.overlays` +list, add an overlay that replaces the package source: + +```nix +(final: prev: { + kmod = prev.kmod.overrideAttrs { + src = inputs.kmod-src; + }; +}) +``` + +`prev.kmod` is kmod from nixpkgs (or from a previous overlay). +`overrideAttrs` replaces only `src`, keeping the build system, +dependencies, and install phase from the original derivation. +Changes to the local source directory are picked up on every +`nix build` without updating the flake lock. + +This pattern works for any nixpkgs package. The template includes +commented-out examples for fio and kmod. Clear `patches = [];` +when the local source already includes fixes that the nixos-flake +overlay backports. + +The following packages are supported by the devel module and can +be overridden this way: + +| Package | Input name | Overlay | +|---|---|---| +| cpupower | `kernel-src` | `prev.cpupower.overrideAttrs { src = inputs.kernel-src; }` | +| damo | `damo-src` | `prev.damo.overrideAttrs { src = inputs.damo-src; }` | +| fio | `fio-src` | `prev.fio.overrideAttrs { src = inputs.fio-src; patches = []; }` | +| kmod | `kmod-src` | `prev.kmod.overrideAttrs { src = inputs.kmod-src; }` | +| nfstest | `nfstest-src` | `prev.nfstest.overrideAttrs { src = inputs.nfstest-src; }` | +| pynfs | `pynfs-src` | `prev.pynfs.overrideAttrs { src = inputs.pynfs-src; }` | +| xfstests | `xfstests-src` | `prev.xfstests.overrideAttrs { src = inputs.xfstests-src; }` | +| xnvme | `xnvme-src` | `prev.xnvme.overrideAttrs { src = inputs.xnvme-src; }` | + +`cpupower` uses the kernel source tree (it builds from +`tools/power/cpupower/`). Point `kernel-src` to your kernel +checkout to match the running kernel version. + +## Custom packages + +The `pkgs/` directory holds packages not available in nixpkgs. +Each package is a file declaring a function, composed via +[callPackage](https://nix.dev/tutorials/callpackage). The overlay +imports `pkgs/default.nix` and merges them into the nixpkgs set. + +Currently included: + +- **damo**: DAMON user-space tool for data access monitoring +- **libbpf-tools**: standalone CO-RE BPF tracing tools (74 + binaries with `-libbpf` suffix, matching Debian convention) +- **nfstest**: NFS test suite (17 test scripts) +- **pynfs**: Python NFSv4 conformance test suite +- **xnvme**: cross-platform NVMe user space library and tools + +## Host configuration via virtiofs + +XDG-compliant tools (helix, neovim, git, etc.) read configuration +from `XDG_CONFIG_HOME` (per-user, defaults to `~/.config/`) and +fall back to `XDG_CONFIG_DIRS` (system-wide, colon-separated +list). On a tmpfs root `~/.config/` is always empty, so the +fallback is the only source of configuration. + +To share the host's `~/.config` into the guest, add a virtiofsd +instance with tag `xdg` sharing the host's `~/.config` +directory, then mount it in the configuration and set +`XDG_CONFIG_HOME` to point to it: + +```nix +fileSystems."/etc/xdg-host" = { + device = "xdg"; + fsType = "virtiofs"; +}; + +environment.variables.XDG_CONFIG_HOME = "/etc/xdg-host"; +environment.variables.XDG_CONFIG_DIRS = lib.mkForce "/etc/xdg-host:/etc/xdg"; +``` + +Or declare it through the shares module: + +```nix +imports = [ nixos-flake.nixosModules.mounts.shares ]; + +nixos-flake.shares."/etc/xdg-host" = { tag = "xdg"; }; + +environment.variables.XDG_CONFIG_HOME = "/etc/xdg-host"; +environment.variables.XDG_CONFIG_DIRS = lib.mkForce "/etc/xdg-host:/etc/xdg"; +``` + +The module replaces only the `fileSystems` entry. The XDG env vars +stay in the consumer's module because they are a policy choice tied +to one specific share's role, not a generic virtiofs concern. + +`XDG_CONFIG_HOME` is the primary config directory that all +XDG-compliant tools check first. Some tools (like helix) only +read from `XDG_CONFIG_HOME` and do not fall back to +`XDG_CONFIG_DIRS`, so setting both ensures all tools find the +host configs. On a tmpfs root `~/.config` is always empty, so +redirecting `XDG_CONFIG_HOME` loses nothing. + +The mount point is `/etc/xdg-host` rather than `/etc/xdg` because +NixOS generates its own files in `/etc/xdg`. The `XDG_CONFIG_DIRS` +override uses `lib.mkForce` because NixOS already sets this +variable in `shells-environment.nix`. Changes on the host are live +in the guest without rebuilding. + +This works for any user in the guest (root, test accounts) because +both variables are system-wide. The guest finds host configs at +`/etc/xdg-host/helix/`, `/etc/xdg-host/nvim/`, +`/etc/xdg-host/git/`, and so on. + +### Sharing a curated subset + +The host's `~/.config` may contain application state, browser +profiles, or credentials that should not be exposed to the guest. +To share only specific tool configs, create a dedicated directory +on the host and symlink the configs you want: + +```shell +mkdir --parents ~/.config/vm +ln --symbolic ~/.config/helix ~/.config/vm/helix +ln --symbolic ~/.config/nvim ~/.config/vm/nvim +ln --symbolic ~/.config/git ~/.config/vm/git +``` + +Share `~/.config/vm` with virtiofsd tag `xdg` instead of the +full `~/.config`. The NixOS configuration is the same — only the +virtiofsd source directory changes. + +### Dotfiles repo as a flake input + +A dotfiles repository can be declared as a flake input with +`flake = false` and its files referenced directly in the NixOS +configuration. This bakes the configs into the closure, so +changes require a rebuild: + +```nix +inputs.dotfiles = { + url = "path:/home/user/src/dotfiles"; + flake = false; +}; +``` + +Then in the module block: + +```nix +environment.etc."xdg-host/helix/config.toml".source = + "${inputs.dotfiles}/.config/helix/config.toml"; +environment.etc."xdg-host/nvim".source = + "${inputs.dotfiles}/.config/nvim"; +environment.variables.XDG_CONFIG_DIRS = lib.mkForce "/etc/xdg-host:/etc/xdg"; +``` + +This is declarative and version-controlled. The dotfiles input +is pinned in `flake.lock` like any other dependency. Run +`nix flake update` to pick up changes from the dotfiles repo. + +### Home-manager + +[Home-manager](https://github.com/nix-community/home-manager) is +a Nix tool for managing user configuration declaratively. Instead +of maintaining dotfiles as plain text, tool configs are expressed +as Nix options that home-manager evaluates into the correct files. + +Add it as a flake input in the configuration: + +```nix +inputs.home-manager = { + url = "github:nix-community/home-manager"; + inputs.nixpkgs.follows = "nixos-flake/nixpkgs"; +}; +``` + +Then import the home-manager NixOS module and declare user +configs: + +```nix +imports = [ inputs.home-manager.nixosModules.home-manager ]; + +home-manager.users.root = { + programs.helix = { + enable = true; + settings = { + theme = "onedark"; + editor.line-number = "relative"; + }; + }; + programs.git = { + enable = true; + userName = "Your Name"; + userEmail = "[email protected]"; + }; +}; +``` + +Home-manager generates the dotfiles from these declarations and +places them in the user's home directory. This is the fully +declarative NixOS-native approach but adds a dependency and +requires expressing configs in Nix rather than using existing +dotfiles directly. + +## Host home directory with ephemeral overlay + +The host's home directory can be shared into the guest via +virtiofs for access to scripts, source trees, and tools. Mounting +it read-only protects the host, and an overlayfs layer on top +provides ephemeral writable storage for programs that write to +the home directory (shell history, `.ssh/known_hosts`, editor +state). Writes go to the tmpfs upper layer and are lost on +shutdown. + +```nix +# Host home via virtiofs (read-only base). +fileSystems."/mnt/home" = { + device = "home"; + fsType = "virtiofs"; + options = [ "ro" ]; +}; + +# Overlay: host home (read-only) + tmpfs (writable, ephemeral). +fileSystems."/root" = { + device = "overlay"; + fsType = "overlay"; + options = [ + "lowerdir=/mnt/home" + "upperdir=/.root-overlay/upper" + "workdir=/.root-overlay/work" + ]; + depends = [ "/mnt/home" ]; +}; + +# Create overlay work directories before the mount. +systemd.services."prepare-root-overlay" = { + description = "Create overlay work directories for /root"; + wantedBy = [ "local-fs.target" ]; + before = [ "root.mount" ]; + unitConfig.DefaultDependencies = false; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${pkgs.coreutils}/bin/mkdir --parents /.root-overlay/upper /.root-overlay/work"; + }; +}; +``` + +The read-only base mount can also be declared through the shares +module, leaving the overlay and prepare-root-overlay service in the +consumer's module: + +```nix +imports = [ nixos-flake.nixosModules.mounts.shares ]; + +nixos-flake.shares."/mnt/home" = { tag = "home"; options = [ "ro" ]; }; + +# Overlay and prepare-root-overlay service as above. +``` + +The virtiofsd instance on the host shares the home directory with +tag `home`. The upper and work directories live on the root tmpfs +(`/`), so they are created fresh on every boot by the +`prepare-root-overlay` service. + +This requires `CONFIG_OVERLAY_FS=y` or `CONFIG_OVERLAY_FS=m` in +the guest kernel. If built as a module, it loads from +`/lib/modules` after switch-root. + +## Block device filesystems + +NixOS can format and mount block devices (NVMe, virtio-blk) +declared in the configuration. + +### Default formatting + +`autoFormat = true` runs `mkfs` with default options if the +device has no filesystem: + +```nix +fileSystems."/mnt/nvme0" = { + device = "/dev/nvme0n1"; + fsType = "xfs"; + autoFormat = true; +}; +``` + +### Custom block and sector size + +For non-default formatting options (block size, sector size, +inode size), use a systemd service that runs before the mount +unit: + +```nix +systemd.services."format-nvme0" = { + description = "Format /dev/nvme0n1 with XFS (16k block/sector)"; + wantedBy = [ "local-fs.target" ]; + before = [ "mnt-nvme0.mount" ]; + requires = [ "dev-nvme0n1.device" ]; + after = [ "dev-nvme0n1.device" ]; + unitConfig.DefaultDependencies = false; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${pkgs.xfsprogs}/bin/mkfs.xfs -b size=16k -s size=16k /dev/nvme0n1"; + }; +}; + +fileSystems."/mnt/nvme0" = { + device = "/dev/nvme0n1"; + fsType = "xfs"; +}; +``` + +NixOS generates the mount unit name from the path (`/mnt/nvme0` +becomes `mnt-nvme0.mount`). The ordering chain is: device appears, +format service runs mkfs, mount unit mounts the filesystem. + +### Using the storage module + +The storage module consolidates multiple extra drives into one +attrset keyed by mount point. Devices with `mkfsArgs` go through a +pre-mount format oneshot guarded by `blkid --probe`; devices with +`autoFormat = true` fall through to NixOS's own first-mount +format machinery; devices with neither are assumed pre-formatted: + +```nix +imports = [ nixos-flake.nixosModules.mounts.storage ]; + +nixos-flake.storage = { + "/mnt/nvme0" = { + device = "/dev/nvme0n1"; + fsType = "xfs"; + mkfsArgs = [ "-b" "size=16k" "-s" "size=16k" ]; + }; + "/mnt/nvme1" = { + device = "/dev/nvme1n1"; + fsType = "xfs"; + autoFormat = true; + }; + "/mnt/data" = { + device = "/dev/vdb"; + fsType = "ext4"; + }; +}; +``` + +`mkfsArgs` and `autoFormat` are mutually exclusive; an assertion +rejects configurations that set both. + +## Controller host + +A controller host provisions and drives guests rather than being +one. Compose the `controller` module on top of a disk-booted +backend and enable it: + +```nix +modules = [ + nixos-flake.nixosModules.backends.libvirt + nixos-flake.nixosModules.profiles.controller + nixos-flake.nixosModules.user + { nixos-flake.controller.enable = true; } +]; +``` + +This adds the kernel and Kconfig build toolchain, Ansible, git, +the QEMU and virtiofs tooling, and system libvirt with KVM. The +libvirt backend's bundled `user.nix` places the unprivileged +account in the `libvirtd` and `kvm` groups by default so it can +drive libvirt without sudo. + +To run a workload on the controller itself instead of in a guest +— the baremetal case — also import the matching `testSuites.*` +module. The imageless closure activates on real hardware with +`<closure>/bin/switch-to-configuration switch`; a baremetal host +overrides the imageless `fileSystems` and `boot` options for its +own root device and kernel. + +## Multiple configurations + +Each configuration in `configurations/` is fully independent. +Configurations do not share state, lock files, or build results. +Common patterns: + +``` +configurations/ +├── devel/ Development VM with all testing tools +│ ├── flake.nix +│ └── flake.lock +├── storage/ VM for storage subsystem testing only +│ ├── flake.nix +│ └── flake.lock +└── minimal/ Base VM without the development profile + ├── flake.nix + └── flake.lock +``` + +Each configuration can import different modules, apply different +overlays, and add different packages. All of them share the same +nixos-flake base through the `inputs.nixos-flake.url` flake input. diff --git a/docs/verifying.md b/docs/verifying.md new file mode 100644 index 00000000..012fa0f7 --- /dev/null +++ b/docs/verifying.md @@ -0,0 +1,72 @@ +# Verifying changes + +Run these steps before every commit. They are manual for now; whether +and how to wire them as git hooks is a later decision. Each step states +why it exists and what it does. + +## 1. Format — `nix fmt` + +Why: the whole history is `nixfmt-rfc-style`-clean; a commit that drifts +breaks that invariant. + +What: rewrites every `.nix` file in place to the canonical style. Run +`git diff` afterwards — a non-empty diff means your change introduced +unformatted code; stage the reformat. + +## 2. Evaluate and build the system closures — `nix flake check` + +Why: catches evaluation errors, broken module options, and malformed +flake outputs, and proves each backend still produces a valid NixOS +system. Also exercises every test-suite module composed on top of the +imageless backend, the user account, and monitoring — the realistic +shape a consumer deploys. + +What: evaluates every flake output and builds +`checks.<system>.{imageless,libvirt,controller,imageless-<suite>}` +— the three base closures and one closure per test-suite module. The +first run builds them all; later runs are cached. Add `--all-systems` +to also cover `aarch64-linux` (needs an aarch64 builder). + +## 3. Build the templates + +Why: step 2 builds the backends in isolation but not the templates the +downstream consumer actually copies in with `nix flake init`. The +templates have their own `flake.nix` with `inputs.nixos-flake.url`, +and a regression in their composition surfaces only on an actual build. +The libvirt template's qcow2 *disk image* is also assembled here — +`make-disk-image.nix` is not exercised by step 2's libvirt closure +check. + +What: builds the imageless template's `toplevel` closure and the +libvirt template's qcow2 against the local checkout, in both cases +overriding the placeholder `path:/path/to/nixos-flake` input with the +working tree. + +```shell +nix build "path:./templates/imageless#packages.x86_64-linux.toplevel" \ + --override-input nixos-flake "path:$PWD" + +nix build "path:./templates/libvirt#packages.x86_64-linux.image" \ + --override-input nixos-flake "path:$PWD" +``` + +## 4. Build the custom packages — when `pkgs/` or `overlays/` changed + +Why: `nix flake check` exercises the `checks` closures but does not build +the standalone `packages` outputs; overlay and package changes only +surface on an actual build. + +What: builds each custom package. + +```shell +nix build .#cpupower .#damo .#libbpf-tools .#nfstest .#pynfs .#xnvme +``` + +## 5. Review the commit message + +Why: the project enforces strict commit conventions (see CLAUDE.md). + +What: subject in imperative mood, at or below 50 characters; body in +plain-English paragraphs, never bullet lists; `Generated-by:` +immediately followed by `Signed-off-by:` with no blank line between +them. diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..09a27b21 --- /dev/null +++ b/flake.lock @@ -0,0 +1,27 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1780902259, + "narHash": "sha256-q8yYEC5f1mFlQO9RGna4LTc9QrcvWunX6FYp83munkQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "bd0ff2d3eac24699c3664d5966b9ef36f388e2ca", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-26.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..b90d06f0 --- /dev/null +++ b/flake.nix @@ -0,0 +1,239 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Library flake for provisioning NixOS systems: backend modules (imageless +# closure / libvirt disk image), profiles, overlays, packages and templates, +# plus build-kernel/build-qemu/systemd devShells that bring the same toolchain +# (lib/toolchain.nix) to any host via `nix develop`. +# +# nix flake check # validate backends + checks +# nix build .#checks.x86_64-linux.imageless # one backend closure +# nix develop .#build-kernel -c make ... # kernel build env +# nix flake init -t github:linux-kdevops/nixos-flake +{ + description = "NixOS modules, overlays, and templates for provisioning NixOS systems"; + + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-26.05"; + + outputs = + { self, nixpkgs }: + let + systems = [ + "x86_64-linux" + "aarch64-linux" + ]; + forAllSystems = + f: + nixpkgs.lib.genAttrs systems ( + system: + f ( + import nixpkgs { + inherit system; + overlays = [ self.overlays.default ]; + } + ) + ); + in + { + # The on-disk layout groups modules into nix-native buckets: + # + # backends/ complete system shape (imageless, libvirt) — + # import one and only one per nixosSystem + # profiles/ additive feature sets (build-tools, controller, + # devel, monitoring) on top of a backend + # mounts/ fileSystems-emitting DSLs (shares, storage) + # testSuites/ per-suite modules (fstests, blktests, …) + # + # The nixosModules attrset mirrors this layout so the import + # path matches the file path. + nixosModules = { + # The default aggregate matches the imageless template's + # starter composition (imageless backend plus the opt-in + # unprivileged account). Consumers that prefer the libvirt + # backend or want bare modules import them by name. + default.imports = [ + ./modules/backends/imageless + ./modules/user.nix + ]; + + backends = { + imageless = ./modules/backends/imageless; + libvirt = ./modules/backends/libvirt; + }; + + profiles = { + build-tools = ./modules/profiles/build-tools.nix; + controller = ./modules/profiles/controller.nix; + devel = ./modules/profiles/devel.nix; + monitoring = ./modules/profiles/monitoring.nix; + }; + + mounts = { + shares = ./modules/mounts/shares.nix; + storage = ./modules/mounts/storage.nix; + }; + + testSuites = { + blktests = ./modules/testSuites/blktests.nix; + fstests = ./modules/testSuites/fstests.nix; + gitr = ./modules/testSuites/gitr.nix; + ltp = ./modules/testSuites/ltp.nix; + mmtests = ./modules/testSuites/mmtests.nix; + pynfs = ./modules/testSuites/pynfs.nix; + selftests = ./modules/testSuites/selftests.nix; + sysbench = ./modules/testSuites/sysbench.nix; + }; + + user = ./modules/user.nix; + }; + + overlays = { + default = import ./overlays; + fio = import ./overlays/fio.nix; + xfstests = import ./overlays/xfstests.nix; + custom-pkgs = final: _: import ./pkgs final; + }; + + templates = { + default = self.templates.imageless; + imageless = { + path = ./templates/imageless; + description = "Imageless NixOS VM (tmpfs root, virtiofs /nix/store, external kernel)"; + }; + libvirt = { + path = ./templates/libvirt; + description = "Libvirt-managed disk-image NixOS VM"; + }; + }; + + # Expose the custom packages as direct flake outputs so they can be + # built without going through a NixOS configuration. + packages = forAllSystems (pkgs: { + inherit (pkgs) + cpupower + damo + libbpf-tools + nfstest + pynfs + xnvme + qemu + virtiofsd + socat + ; + }); + + # Reproducible build toolchains usable on any host (NixOS or not). Nix + # provides the environment; the build picks the compiler/flags inside it. + # nix develop .#build-kernel -c make ... + # nix develop .#build-qemu -c make ... + devShells = forAllSystems ( + pkgs: + let + tc = import ./lib/toolchain.nix { inherit pkgs; }; + kernelPackages = + tc.kernel + ++ tc.matrixExtras + ++ [ + pkgs.ccache + pkgs.b4 + ]; + # CONFIG_RUST builds core/alloc from source -> needs rust-src. + rustLibSrc = pkgs.rustPlatform.rustLibSrc; + in + { + # No qemu inputsFrom: its NIX_CFLAGS_COMPILE overflows the kernel + # host-tool argv (E2BIG on fixdep). + build-kernel = pkgs.mkShell { + packages = kernelPackages; + env.RUST_LIB_SRC = rustLibSrc; + }; + + # Adds qemu's build inputs via inputsFrom + the python deps qemu's + # offline configure venv (mkvenv) needs. + build-qemu = pkgs.mkShell { + inputsFrom = [ pkgs.qemu ]; + packages = kernelPackages ++ [ + (pkgs.python3.withPackages ( + ps: with ps; [ + setuptools + wheel + pip + ] + )) + ]; + env.RUST_LIB_SRC = rustLibSrc; + }; + + # systemd client tools to drive a host user manager over D-Bus. + systemd = pkgs.mkShell { + packages = [ pkgs.systemd ]; + }; + } + ); + + # Tree formatter for `nix fmt`. No overlay needed, so this uses + # legacyPackages rather than re-importing nixpkgs. + formatter = nixpkgs.lib.genAttrs systems (system: nixpkgs.legacyPackages.${system}.nixfmt); + + # Per-backend system closures exercised by nix flake check. + checks = nixpkgs.lib.genAttrs systems ( + system: + let + buildBackend = + module: + (nixpkgs.lib.nixosSystem { + inherit system; + modules = [ + module + { nixpkgs.overlays = [ self.overlays.default ]; } + ]; + }).config.system.build.toplevel; + + # Build each test-suite module composed on top of the + # imageless backend plus the unprivileged account and + # the monitoring profile. The composition is the + # realistic shape a consumer deploys (a guest that runs + # a test suite on an unprivileged account with monitor + # units available), so a test-suite module that conflicts + # with imageless, user, or monitoring defaults surfaces + # in nix flake check rather than at consumer bringup. + # Monitoring is composed alongside because the monitor + # template units (monitor-<name>@<run-id>.service) + # exercise systemd template-instantiation paths that + # standalone evaluation of the module does not. + testSuiteChecks = nixpkgs.lib.mapAttrs' ( + name: module: + nixpkgs.lib.nameValuePair "imageless-${name}" (buildBackend { + imports = [ + self.nixosModules.backends.imageless + self.nixosModules.user + self.nixosModules.profiles.monitoring + module + ]; + nixos-flake.monitoring.enable = true; + }) + ) self.nixosModules.testSuites; + in + { + imageless = buildBackend self.nixosModules.backends.imageless; + libvirt = buildBackend self.nixosModules.backends.libvirt; + # The controller profile composes on top of a disk-booted + # backend; build it against libvirt so the check exercises + # the realistic control-node configuration. + controller = buildBackend { + imports = [ + self.nixosModules.backends.libvirt + self.nixosModules.profiles.controller + ]; + nixos-flake.controller.enable = true; + }; + devel = buildBackend { + imports = [ + self.nixosModules.backends.imageless + self.nixosModules.profiles.devel + ]; + }; + } + // testSuiteChecks + ); + }; +} diff --git a/lib/toolchain.nix b/lib/toolchain.nix new file mode 100644 index 00000000..c9e456b8 --- /dev/null +++ b/lib/toolchain.nix @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Build toolchain package sets, shared by the controller NixOS module and the +# build devShells so a control node and an any-host shell agree on versions. +{ pkgs }: +{ + # Kernel + Kconfig toolchain a kernel tree's make / `make *config` needs + # beyond stdenv. + kernel = with pkgs; [ + ncurses + pkg-config + gcc + gnumake + bison + flex + bc + perl + openssl + elfutils + pahole + cpio + kmod + zstd + rsync + hostname + ]; + + # Controller orchestration / provisioning tools. + orchestration = with pkgs; [ + ansible + git + qemu-utils + virtiofsd + python3 + python3Packages.pyyaml + python3Packages.jinja2 + ]; + + # Multi-toolchain build matrix (gcc AND clang AND rust AND sparse), + # selected per build via make flags (LLVM=1, C=1) / CONFIG_RUST. + matrixExtras = with pkgs; [ + clang + lld + llvm # llvm-objcopy/ar/nm/... so an LLVM=1 build is self-contained + rustc + rust-bindgen + rustfmt + sparse + ]; +} diff --git a/modules/backends/imageless/default.nix b/modules/backends/imageless/default.nix new file mode 100644 index 00000000..708e412c --- /dev/null +++ b/modules/backends/imageless/default.nix @@ -0,0 +1,216 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Minimal NixOS configuration for imageless boot via virtiofs. +# +# Root is tmpfs. /nix/store and /lib/modules are mounted via virtiofs +# from the host. The kernel is built externally with all boot-critical +# drivers built-in (CONFIG_VIRTIO_FS=y, CONFIG_TMPFS=y). NixOS builds +# a systemd initramfs that mounts root (tmpfs), /nix/store, and +# /lib/modules before switch-root. +# +# Key-only SSH. Root password is serial-console break-glass only. +{ + config, + pkgs, + lib, + modulesPath, + ... +}: +{ + imports = [ + (modulesPath + "/profiles/minimal.nix") + # Imageless-specific defaults for the unprivileged account. + # The user module itself is opt-in: importing this backend + # alone does not create the account. Importing the user + # module alongside picks up these backend-shaped defaults. + ./user.nix + ]; + + # Built fresh, never upgraded in place, so this tracks the release each + # build is made from (the option's default). Pin a literal only if you + # persist a disk across nixpkgs upgrades: + # https://wiki.nixos.org/wiki/FAQ/When_do_I_update_stateVersion + system.stateVersion = lib.mkDefault config.system.nixos.release; + + services.openssh = { + enable = true; + settings = { + PermitRootLogin = lib.mkDefault "yes"; + PubkeyAuthentication = true; + PasswordAuthentication = lib.mkDefault false; + }; + }; + + # Root as tmpfs: ephemeral, no disk image. systemd in the initramfs + # creates this from root=tmpfs on the kernel command line. + fileSystems."/" = lib.mkImageMediaOverride { + fsType = "tmpfs"; + options = [ "mode=0755" ]; + }; + + # /nix/store via virtiofs (read-only, shared from host). + # Automatically included in initrd fstab (pathsNeededForBoot). + fileSystems."/nix/store" = { + device = "store"; + fsType = "virtiofs"; + }; + + # Kernel modules via virtiofs (from the external kernel build). + # neededForBoot puts it in the initrd fstab since /lib/modules + # is not in pathsNeededForBoot. + fileSystems."/lib/modules" = { + device = "modules"; + fsType = "virtiofs"; + neededForBoot = true; + }; + + # Disable systemd-remount-fs. Nothing to remount on tmpfs. + systemd.services.systemd-remount-fs.enable = lib.mkForce false; + + # No bootloader. Kernel is external, but NixOS builds the initramfs. + boot.loader.grub.enable = false; + boot.kernel.enable = false; + + # Workaround: boot.kernel.enable = false does not define + # system.build.kernel, but boot.initrd.systemd accesses + # kernel.config.isYes and kernel.config.isSet to decide whether + # to include kernel modules in the initramfs. + # + # The initrd module uses: isSet "MODULES" -> isYes "MODULES" + # (Nix implication). To exclude modules, isSet must return true + # and isYes must return false (true -> false = false). + # https://github.com/NixOS/nixpkgs/issues/467069 + system.build.kernel.config = { + isYes = _: false; + isSet = _: true; + }; + + # systemd initramfs: systemd runs as PID 1, reads the initrd fstab, + # mounts root + /nix/store + /lib/modules, then switch-roots. + boot.initrd.systemd.enable = true; + boot.initrd.supportedFilesystems = [ "virtiofs" ]; + + # Passwordless emergency shell in the initramfs. Without this, + # sulogin blocks on "root account is locked" when boot fails. + boot.initrd.systemd.emergencyAccess = true; + + # No kernel modules in the initramfs. The external kernel has all + # boot-critical drivers built-in (CONFIG_VIRTIO_FS=y, CONFIG_TMPFS=y, + # CONFIG_VIRTIO_PCI=y). Runtime modules come from /lib/modules via + # virtiofs after switch-root. + boot.initrd.availableKernelModules = lib.mkForce [ ]; + boot.initrd.kernelModules = lib.mkForce [ ]; + + # Restore the standard `boot.kernelModules → /etc/modules-load.d + # → systemd-modules-load.service` wiring that NixOS' + # nixos/modules/system/boot/kernel.nix gates off when + # boot.kernel.enable=false. + # + # The upstream `mkIf config.boot.kernel.enable` block in + # kernel.nix (around line 441) bundles eleven separate things + # under one gate. Five of them genuinely depend on NixOS + # having built the kernel itself — system.build.kernel, + # system.modulesTree, the system.systemBuilderCommands that + # symlink $out/kernel and $out/initrd, the kernel-side + # boot.kernelParams, hardware.firmware exposure. Two are + # default-module hints (loop, atkbd) that aren't useful in a + # VM. The remaining four are pure runtime mechanism and have + # no dependency on the kernel package whatsoever: + # + # environment.etc."modules-load.d/nixos.conf" + # systemd.services.systemd-modules-load.wantedBy + # systemd.services.systemd-modules-load.serviceConfig + # lib.kernelConfig (assertion helpers) + # + # The first three are needed for `boot.kernelModules` to + # actually load anything; the fourth is unused outside + # NixOS-built-kernel paths. Bundling all eleven under one gate + # was a reasonable simplification when "no kernel package = + # no modules" was the only no-NixOS-kernel use case the + # upstream module had to think about, but the assumption + # breaks on imageless guests: the consumer builds the kernel + # out-of-tree relative to nixpkgs, and the matching modules + # tree lives at /lib/modules served by virtiofs from the host. + # The modules exist, modprobe works, the only thing missing is + # the standard /etc/modules-load.d wiring that + # systemd-modules-load.service consumes at stage-2 boot. + # + # Lifting `boot.kernel.enable = true` to use the upstream path + # is wrong for two reasons. First, it would re-engage every + # piece of the gate, including the system.build.kernel and + # system.systemBuilderCommands branches that emit symlinks + # ($out/kernel, $out/initrd) pointing at the nixpkgs kernel — + # we'd ship two kernels in the closure, only one of which + # actually runs. Second, even if we patched around the + # symlinks, downstream tooling that reads system.build.kernel + # (image generators, future kexec/snapshot paths, debug + # attestation) would silently report the wrong-but-built + # kernel rather than the actual one. + # + # The minimal fix is to lift only the four pure-runtime pieces + # at this layer, where the unbundling decision happens. The + # mkIf guard makes this a no-op if a future kernel.nix lifts + # the gating itself, or if a downstream module ever sets + # boot.kernel.enable=true on top of imageless (the natural + # path for someone who wants both the nixpkgs and the external + # kernel available, even though that's not the common path). + # The config shape and SuccessExitStatus value mirror upstream + # so the runtime behavior is byte-identical to a + # kernel.enable=true closure for the modules-load surface. + environment.etc."modules-load.d/nixos.conf".source = lib.mkIf (!config.boot.kernel.enable) ( + pkgs.writeText "nixos.conf" '' + ${lib.concatStringsSep "\n" config.boot.kernelModules} + '' + ); + systemd.services.systemd-modules-load = lib.mkIf (!config.boot.kernel.enable) { + wantedBy = [ "multi-user.target" ]; + serviceConfig.SuccessExitStatus = "0 1"; + }; + + # Re-add the upstream-NixOS default kernelModules list that the + # same boot.kernel.enable gate skips. Both `loop` and `atkbd` are + # part of nixos/modules/system/boot/kernel.nix:455 — `loop` + # specifically because every fstests test that exercises + # snapshot/dm-flakey/loop-backed scratch (xfs/664, generic/097 + # and friends) calls `losetup` against a freshly-created file + # under TEST_DIR. Without loop loaded, losetup fails with "No + # such file or directory" on /dev/loopN and the test falls into + # a confusing tail of cascading "extra arguments to mkfs" noise + # rather than skipping cleanly. atkbd is harmless on a VM but + # we keep the upstream pair intact so the lift is a faithful + # mirror — if a future kernel.nix change adjusts the default + # list, the same default applies here. + boot.kernelModules = lib.mkIf (!config.boot.kernel.enable) [ + "loop" + "atkbd" + ]; + + # Serial getty on hvc0 for interactive login via the console socket. + # hvc0 is a virtio console, handled by systemd's serial-getty@ template + # (the plain getty@ template is VT-only, gated on /dev/tty0). + # Requires CONFIG_VIRTIO_CONSOLE=y (or =m) in the guest kernel. + systemd.services."serial-getty@hvc0" = { + enable = true; + wantedBy = [ "getty.target" ]; + }; + + # Serial-console break-glass. Tmpfs root resets it every boot. + users.mutableUsers = false; + users.users.root.initialPassword = "root"; + + # Networking: systemd-networkd with DHCP on all ethernet interfaces. + networking.useNetworkd = true; + networking.hostName = lib.mkDefault "nixos"; + networking.firewall.enable = false; + systemd.network.networks."80-ethernet" = { + matchConfig.Name = "en*"; + networkConfig.DHCP = "yes"; + }; + + # Disable unnecessary services. + systemd.oomd.enable = false; + nix.enable = false; + services.lvm.enable = false; + + environment.systemPackages = with pkgs; [ coreutils ]; +} diff --git a/modules/backends/imageless/user.nix b/modules/backends/imageless/user.nix new file mode 100644 index 00000000..3a88d3e7 --- /dev/null +++ b/modules/backends/imageless/user.nix @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Imageless-specific defaults for the unprivileged account. +# Auto-imported by the imageless backend module. An imageless +# guest does not run libvirt locally (it boots as a guest under +# a host's libvirtd, served via virtiofs), so the account only +# needs wheel and kvm by default. +# +# The shared option schema in ../../user-options.nix makes +# nixos-flake.user.extraGroups available even when modules/ +# user.nix is not in the import set (e.g. the standalone +# backend check builds the backend without the user module). +# +# mkDefault on the value so a consumer who wants a different +# group set can override at normal priority without lib.mkForce. +{ lib, ... }: +{ + imports = [ ../../user-options.nix ]; + + nixos-flake.user.extraGroups = lib.mkDefault [ + "wheel" + "kvm" + ]; +} diff --git a/modules/backends/libvirt/default.nix b/modules/backends/libvirt/default.nix new file mode 100644 index 00000000..b58fc98a --- /dev/null +++ b/modules/backends/libvirt/default.nix @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Disk-image NixOS boot for libvirt-managed VMs. Expects a qcow2 +# virtio-blk disk at /dev/vda. +# +# Per-node content (hostname, keys, virtiofs shares, mounts) is +# the consumer's responsibility and composes as additional +# modules. +{ + config, + pkgs, + lib, + modulesPath, + ... +}: +{ + + imports = [ + # qemu-guest wires up virtio drivers, the QEMU guest agent, and + # the small set of host-facing services that every NixOS VM + # under QEMU needs. Brings the hardware setup into the module + # so downstream consumers do not have to hand-write a matching + # hardware-configuration.nix. + (modulesPath + "/profiles/qemu-guest.nix") + # Libvirt-specific defaults for the unprivileged account. + # The user module itself is opt-in: importing this backend + # alone does not create the account. Importing the user + # module alongside picks up these backend-shaped defaults. + ./user.nix + ]; + + # Built fresh, never upgraded in place, so this tracks the release each + # build is made from (the option's default). Pin a literal only if you + # persist a disk across nixpkgs upgrades: + # https://wiki.nixos.org/wiki/FAQ/When_do_I_update_stateVersion + system.stateVersion = lib.mkDefault config.system.nixos.release; + + # Key-only SSH. Password below is serial-console break-glass. + services.openssh = { + enable = true; + settings = { + PermitRootLogin = lib.mkDefault "yes"; + PubkeyAuthentication = true; + PasswordAuthentication = lib.mkDefault false; + }; + }; + + boot.loader.grub = { + enable = true; + device = "/dev/vda"; + }; + boot.loader.timeout = lib.mkDefault 1; + boot.kernelPackages = lib.mkDefault pkgs.linuxPackages_latest; + + fileSystems."/" = { + device = "/dev/vda1"; + fsType = "ext4"; + }; + + # libvirt's default network hands out 192.168.122.0/24 via DHCP. + networking.useDHCP = lib.mkDefault true; + + virtualisation.libvirtd.enable = lib.mkDefault false; + + time.timeZone = lib.mkDefault "UTC"; + i18n.defaultLocale = lib.mkDefault "en_US.UTF-8"; + + nix.settings.experimental-features = [ + "nix-command" + "flakes" + ]; + nix.gc = { + automatic = lib.mkDefault true; + dates = lib.mkDefault "weekly"; + options = lib.mkDefault "--delete-older-than 7d"; + }; +} diff --git a/modules/backends/libvirt/user.nix b/modules/backends/libvirt/user.nix new file mode 100644 index 00000000..7164dc45 --- /dev/null +++ b/modules/backends/libvirt/user.nix @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Libvirt-specific defaults for the unprivileged account. +# Auto-imported by the libvirt backend module. A libvirt-backed +# system frequently runs libvirtd locally (controller nodes +# compose the controller profile on top of this backend), so the +# account gets libvirtd group membership by default. +# Membership in libvirtd is silently dropped on systems where +# the group does not exist, so the default is safe on a plain +# libvirt guest that never enables libvirtd. +# +# The shared option schema in ../../user-options.nix makes +# nixos-flake.user.extraGroups available even when modules/ +# user.nix is not in the import set (e.g. the standalone +# backend check builds the backend without the user module). +# +# mkDefault on the value so a consumer who wants a different +# group set can override at normal priority without lib.mkForce. +{ lib, ... }: +{ + imports = [ ../../user-options.nix ]; + + nixos-flake.user.extraGroups = lib.mkDefault [ + "wheel" + "kvm" + "libvirtd" + ]; +} diff --git a/modules/mounts/shares.nix b/modules/mounts/shares.nix new file mode 100644 index 00000000..a094c95d --- /dev/null +++ b/modules/mounts/shares.nix @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Opt-in virtiofs shares. Each share is a host-to-guest mount +# announced by the host's virtiofsd under a tag; the guest mounts +# it at a path of the consumer's choice. +# +# Shares are attrsOf submodule keyed by mount point, mirroring the +# fileSystems.<path> convention. The module emits one fileSystems +# entry per share and nothing else: no overlays, no env vars, no +# directory creation. Consumers compose those themselves when they +# need them. +{ config, lib, ... }: +let + cfg = config.nixos-flake.shares; +in +{ + options.nixos-flake.shares = lib.mkOption { + default = { }; + description = "Virtiofs shares to mount in the guest, keyed by mount point."; + type = lib.types.attrsOf ( + lib.types.submodule { + options = { + tag = lib.mkOption { + type = lib.types.str; + example = "home"; + description = "Virtiofs tag announced by the host's virtiofsd."; + }; + options = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ "ro" ]; + description = "Additional mount options passed to mount(8)."; + }; + }; + } + ); + }; + + config.fileSystems = lib.mapAttrs ( + mountPoint: share: + { + device = share.tag; + fsType = "virtiofs"; + } + // lib.optionalAttrs (share.options != [ ]) { + inherit (share) options; + } + ) cfg; +} diff --git a/modules/mounts/storage.nix b/modules/mounts/storage.nix new file mode 100644 index 00000000..a264ffc1 --- /dev/null +++ b/modules/mounts/storage.nix @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Opt-in block-device mounts with optional pre-mount mkfs. Each +# entry is keyed by its mount point and declares the source device, +# filesystem type, mount options, and either a list of mkfs +# arguments or an autoFormat flag for default first-mount +# formatting. The two format modes are mutually exclusive; an +# assertion rejects configurations that set both. +# +# When mkfsArgs is non-empty, a oneshot service runs +# mkfs --type <fsType> with those arguments before the mount unit, +# guarded by blkid so a device that already carries a filesystem is +# left untouched. When autoFormat is true, the fileSystems entry +# passes through to NixOS's own first-mount format machinery. +{ + config, + lib, + pkgs, + utils, + ... +}: +let + cfg = config.nixos-flake.storage; + + mkFormatUnit = mountPoint: drive: { + name = "format-${utils.escapeSystemdPath drive.device}"; + value = { + description = "Format ${drive.device} before ${mountPoint} mounts"; + wantedBy = [ "${utils.escapeSystemdPath mountPoint}.mount" ]; + before = [ "${utils.escapeSystemdPath mountPoint}.mount" ]; + requires = [ "${utils.escapeSystemdPath drive.device}.device" ]; + after = [ "${utils.escapeSystemdPath drive.device}.device" ]; + unitConfig.DefaultDependencies = false; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecCondition = pkgs.writeShellScript "blkid-empty-${utils.escapeSystemdPath drive.device}" '' + ! ${pkgs.util-linux}/bin/blkid --probe ${lib.escapeShellArg drive.device} + ''; + ExecStart = lib.escapeShellArgs ( + [ + "${pkgs.util-linux}/bin/mkfs" + "--type" + drive.fsType + ] + ++ drive.mkfsArgs + ++ [ drive.device ] + ); + }; + }; + }; +in +{ + options.nixos-flake.storage = lib.mkOption { + default = { }; + description = "Extra block-device mounts, keyed by mount point."; + type = lib.types.attrsOf ( + lib.types.submodule { + options = { + device = lib.mkOption { + type = lib.types.str; + example = "/dev/nvme0n1"; + description = "Source block device."; + }; + fsType = lib.mkOption { + type = lib.types.str; + example = "xfs"; + description = "Filesystem type passed to mount(8) and mkfs(8)."; + }; + options = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Additional mount options passed to mount(8)."; + }; + mkfsArgs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ + "-b" + "size=16k" + "-s" + "size=16k" + ]; + description = '' + Arguments passed to `mkfs --type <fsType>` before the + mount unit runs. The format service skips devices that + already carry a filesystem. Mutually exclusive with + {option}`autoFormat`. + ''; + }; + autoFormat = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Pass through to + {option}`fileSystems.<path>.autoFormat` so NixOS formats + the device with default mkfs arguments on first mount if + it has no filesystem. Mutually exclusive with + {option}`mkfsArgs`. + ''; + }; + }; + } + ); + }; + + config = { + assertions = lib.mapAttrsToList (mountPoint: drive: { + assertion = !(drive.autoFormat && drive.mkfsArgs != [ ]); + message = '' + nixos-flake.storage."${mountPoint}": autoFormat and mkfsArgs + are mutually exclusive. Pick one. + ''; + }) cfg; + + fileSystems = lib.mapAttrs ( + mountPoint: drive: + { + inherit (drive) device fsType autoFormat; + } + // lib.optionalAttrs (drive.options != [ ]) { + inherit (drive) options; + } + ) cfg; + + systemd.services = lib.listToAttrs ( + lib.mapAttrsToList mkFormatUnit (lib.filterAttrs (_: d: d.mkfsArgs != [ ]) cfg) + ); + }; +} diff --git a/modules/profiles/build-tools.nix b/modules/profiles/build-tools.nix new file mode 100644 index 00000000..d77e2f3b --- /dev/null +++ b/modules/profiles/build-tools.nix @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Common build toolchain: GNU Autotools plus a handful of headers +# and libraries that kernel-style compile jobs and several in-tree +# test frameworks routinely expect on a target host. +# +# Kept deliberately narrow. This module does not pick a C compiler +# (nixpkgs' stdenv already provides one) and does not pull in +# language-specific build systems; consumers that need more compose +# the additional packages themselves. +{ pkgs, ... }: +{ + environment.systemPackages = with pkgs; [ + autoconf + automake + libtool + pkg-config + flex + bison + bc + openssl + elfutils + ]; +} diff --git a/modules/profiles/controller.nix b/modules/profiles/controller.nix new file mode 100644 index 00000000..64a76eca --- /dev/null +++ b/modules/profiles/controller.nix @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Controller profile: turns a NixOS host into a control node that +# drives the provisioning and test execution itself — building +# kernels, running Kconfig and `make menuconfig`, executing +# Ansible, and provisioning libvirt/QEMU guests. +# +# Enable it on top of a backend on a real machine (typically the +# libvirt backend, which is disk-booted with its own kernel). The +# same host can also import a test-suite module and run the +# suite directly, which is the baremetal case. +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.nixos-flake.controller; + tc = import ../../lib/toolchain.nix { inherit pkgs; }; +in +{ + options.nixos-flake.controller.enable = lib.mkEnableOption "the control-node environment"; + + config = lib.mkIf cfg.enable { + # Kernel/Kconfig toolchain + orchestration tools, shared with the build + # devShells through lib/toolchain.nix. + environment.systemPackages = tc.kernel ++ tc.orchestration; + + # System libvirt with the QEMU/KVM stack so the controller can + # define, run, and tear down guest VMs. runAsRoot stays off and + # virtiofsd is registered as a vhost-user backend so guests can + # share the host store without the daemon running as root. + virtualisation.libvirtd = { + enable = true; + qemu = { + runAsRoot = false; + vhostUserPackages = [ pkgs.virtiofsd ]; + }; + }; + + # Flakes so the controller can build system closures and disk + # images locally. + nix.settings.experimental-features = [ + "nix-command" + "flakes" + ]; + }; +} diff --git a/modules/profiles/devel.nix b/modules/profiles/devel.nix new file mode 100644 index 00000000..d4a2eb67 --- /dev/null +++ b/modules/profiles/devel.nix @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Development profile: kernel testing and storage tools. +# +# Import this module on top of the base configuration to get a +# full kernel development environment with storage, NVMe, NFS, +# BPF, and filesystem testing tools. +# +# Usage in a flake: +# modules = [ +# nixos-flake.nixosModules.backends.imageless # or backends.libvirt +# nixos-flake.nixosModules.profiles.devel +# { nixpkgs.overlays = [ nixos-flake.overlays.default ]; } +# ]; +{ pkgs, ... }: +{ + # Wire up completion scripts for bash so tab completion works for + # the tools installed below. Just having bash-completion on the path + # does not enable it; the NixOS option sources the completion + # dispatcher from /etc/bashrc. + programs.bash.completion.enable = true; + + # SPDK/DPDK guest VFIO: QEMU's emulated IOMMU advertises no interrupt + # remapping, so allow the unsafe path (safe inside a guest). + boot.extraModprobeConfig = "options vfio_iommu_type1 allow_unsafe_interrupts=1"; + + environment.systemPackages = with pkgs; [ + # Storage and filesystem tools + btrfs-progs + e2fsprogs + f2fs-tools + lvm2 + libndctl + parted + xfsdump + xfsprogs + + # NVMe and SCSI + libnvme + nvme-cli + sg3_utils + spdk + xnvme + + # I/O performance + fio + libaio + liburing + stress-ng + + # NFS + nfstest + nfs-utils + pynfs + + # Network + iperf + + # BPF and tracing + bcc + blktrace + bpftools + bpftrace + libbpf-tools + trace-cmd + + # Monitoring + cpupower + btop + damo + dmidecode + gnuplot + htop + iotop + lsof + numactl + pagemon + perf + powertop + sysstat + + # Test suites and runtime dependencies + xfstests + + # fstests/blktests/selftests runtime dependencies + acl + attr + keyutils + libcap + libseccomp + mdadm + quota + rpcbind + + # General development + bc + ethtool + file + gawk + git + helix + iproute2 + jq + kmod + neovim + pciutils + screen + strace + tmux + usbutils + util-linux + vim + zellij + + # Python: interpreter plus data analysis libraries. The bare + # python3 interpreter is kept next to the libraries so downstream + # automation (Ansible, ad-hoc scripting) can find + # /run/current-system/sw/bin/python3 without having to pull in a + # test-suite module that happens to install it transitively. + python3 + python3Packages.matplotlib + python3Packages.numpy + python3Packages.pandas + ]; +} diff --git a/modules/profiles/monitoring.nix b/modules/profiles/monitoring.nix new file mode 100644 index 00000000..e8bc2f59 --- /dev/null +++ b/modules/profiles/monitoring.nix @@ -0,0 +1,464 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Test-run-bracketed system monitoring on the guest. +# +# Provides systemd-managed monitor daemons that an external +# orchestrator starts before a workload runs and stops after. +# Every monitor runs under a systemd template +# (monitor-<name>@<run-id>.service); the orchestrator instantiates +# each unit with the workload's run-id at start time, and systemd +# substitutes the run-id wherever the unit references %i. Output +# lands under nixos-flake.monitoring.outputDir, which is by default +# /var/lib/monitoring/runs/%i and which consumers typically back +# with a virtiofs share or a block-device mount so the data ends +# up on the host alongside the run's other artifacts. +# +# Each unit pins ReadWritePaths to the monitoring output directory +# so the kernel hard-confines writes to the current run-id's +# monitoring tree — defense in depth for the systemd-managed +# writers we author. +# +# Daemons: +# +# monitor-sysstat: long-lived sadc(1) recorder. Captures CPU +# (global + per-CPU + virtualization), memory, hugepages, swap, +# paging, IO (global + per-device), interrupts, network (DEV + +# IP/TCP/UDP/ICMP + IPv6), NFS, sockets, run queue + load, +# softnet, kernel tables, TTY, USB, filesystem util, power +# (CPU freq, temp, fans, voltage), pressure-stall (PSI). On +# stop, ExecStopPost converts the binary recording to JSON via +# sadf -j. +# +# monitor-cpu-governor: oneshot snapshot of every CPU's +# /sys/devices/system/cpu/cpuN/cpufreq/scaling_governor at +# start and end of the workload window. Useful to verify test +# conditions and detect governor changes mid-run. +# +# monitor-blkalgn: long-lived blkalgn-libbpf recorder. blkalgn +# traces block I/O issue events via eBPF and reports per-request +# granularity and alignment distribution against a configurable +# indirection unit (IU) size set, plus a per-IU write +# amplification factor. The service exec's +# blkalgn-libbpf --json <out>/blkalgn/blkalgn.json, which +# rewrites the JSON file on every main-loop iteration so the +# most recent dump survives even an ungraceful stop. Requires +# the closure to ship blkalgn-libbpf; the libbpf-tools +# derivation in pkgs/ builds the upstream BCC tag, which does +# not yet carry blkalgn — consumers that need it override the +# derivation src to a bcc fork that does. +# +# monitor-biolatency: long-lived biolatency-libbpf recorder. +# Prints log2 histograms of block I/O latency (issue to +# completion, optionally including OS queue time) at a +# configurable interval. biolatency does not write JSON +# itself, so the unit captures stdout to +# <out>/biolatency/biolatency.txt via systemd's +# StandardOutput=append: directive. Bucketing defaults to +# milliseconds; the per-disk and per-flag breakdowns are +# off by default to keep the captured text compact. +# +# This module declares the daemons but does not enable them by +# default — they are started and stopped by the orchestrator that +# brackets the workload, instantiated as +# monitor-<name>@<run-id>.service so the run-id is substituted +# everywhere %i appears in the unit. The module assumes the +# configured outputDir is writable at unit start time; consumers +# typically provision it via a virtiofs share, a block-device +# mount, or a tmpfiles rule. + +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.nixos-flake.monitoring; + + governorSnapshot = pkgs.writeShellScript "monitor-cpu-governor-snapshot" '' + set -eu + out_dir="$1" + phase="$2" # start or end + mkdir -p "$out_dir" + + # Emit a JSON object: {timestamp, governors:{cpuN:gov}} + { + printf '{\n' + printf ' "timestamp": "%s",\n' "$(${pkgs.coreutils}/bin/date -Iseconds)" + printf ' "governors": {\n' + first=1 + for cpu_path in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do + [ -e "$cpu_path" ] || continue + # Strip the /cpufreq/scaling_governor suffix, then take the basename. + cpu_dir=''${cpu_path%/cpufreq/scaling_governor} + cpu_name=''${cpu_dir##*/} + gov=$(${pkgs.coreutils}/bin/cat "$cpu_path") + if [ "$first" -eq 1 ]; then first=0; else printf ',\n'; fi + printf ' "%s": "%s"' "$cpu_name" "$gov" + done + printf '\n }\n}\n' + } > "$out_dir/$phase.json" + ''; + + sysstatJsonExport = pkgs.writeShellScript "monitor-sysstat-json-export" '' + set -eu + out_dir="$1" + if [ -f "$out_dir/sa-current" ]; then + # sadf -j defaults to CPU only; pass -A through to sar so the + # JSON output includes every activity sadc recorded (memory, + # disk IO, network, scheduler runqueue, paging, sockets, NFS, + # power/freq, PSI, etc.). + ${pkgs.sysstat}/bin/sadf -j "$out_dir/sa-current" -- -A \ + > "$out_dir/sa-current.json" || true + fi + ''; + + # Per-instance run-id-scoped output dir. systemd substitutes %i + # (the instance arg, e.g. r0001 for [email protected]) + # at unit-load time; the literal "%i" inside the configured + # outputDir flows through the nix build unchanged and only gets + # resolved when systemd instantiates the unit. + monitorOutDir = cfg.outputDir; + + # Factory for libbpf-tools-backed monitor services. The + # libbpf-tools binaries in pkgs/ install with a -libbpf suffix + # (so they don't collide with BCC's Python wrappers), are + # long-lived, and respond to SIGTERM by cleaning up and exiting; + # that envelope fits every tool in the collection, so the only + # per-tool inputs are the binary's basename and its argv. The + # factory returns an attrset suitable for a systemd template + # unit (declared as "monitor-<name>@") so the run-id is the + # instance argument every consumer passes in. + # + # captureStdout tells the unit to redirect the binary's stdout to + # ${monitorOutDir}/${name}/${name}.txt via systemd's + # StandardOutput=append:. Tools that write structured output to a + # path themselves (blkalgn -j) leave this off and pass the path in + # args; tools that print histograms to stdout (biolatency, + # biopattern, biostacks, ...) flip it on so systemd captures the + # text without a shell wrapper. + mkLibbpfMonitor = + { + name, + description, + args, + captureStdout ? false, + }: + { + inherit description; + documentation = [ "https://github.com/iovisor/bcc/tree/master/libbpf-tools" ]; + + serviceConfig = { + Type = "simple"; + ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p ${monitorOutDir}/${name}"; + ExecStart = lib.concatStringsSep " " ([ "${pkgs.libbpf-tools}/bin/${name}-libbpf" ] ++ args); + KillSignal = "SIGTERM"; + TimeoutStopSec = "30s"; + Restart = "no"; + # systemd specifiers (%i etc.) are expanded in path-bearing + # directives like ReadWritePaths, so the kernel-enforced write + # boundary tracks the per-instance output dir automatically. + ReadWritePaths = "${monitorOutDir}"; + } + // lib.optionalAttrs captureStdout { + StandardOutput = "append:${monitorOutDir}/${name}/${name}.txt"; + StandardError = "journal"; + }; + }; + + blkalgnArgs = lib.concatLists [ + (lib.optionals (cfg.blkalgn.disk != "") [ + "--disk" + cfg.blkalgn.disk + ]) + (lib.optionals (cfg.blkalgn.ops != "") [ + "--ops" + cfg.blkalgn.ops + ]) + [ + "--json" + "${monitorOutDir}/blkalgn/blkalgn.json" + ] + ]; + + # biolatency prints log2 histograms of block-I/O latency to stdout + # every <interval> seconds. Args order matches biolatency's argp + # parser: option flags first, then the trailing positional + # interval. Omitting `count` lets the daemon print histograms + # forever until SIGTERM, which is what test-run-bracketed monitor + # mode wants - the orchestrator stops the unit when the workload + # ends and the last histogram lands in the captured stdout. + # + # biolatency's argp registers the long name "disk" twice — once + # for -D (per-disk grouping, no arg) and once for -d (filter to + # one disk, takes a DISK arg). The `--disk` long form is + # therefore ambiguous; short forms are the only unambiguous + # spelling here. This is the documented exception to the + # long-form-flag preference: when the tool's CLI does not have + # an unambiguous long form, the short flag is the correct + # choice. + biolatencyArgs = lib.concatLists [ + (lib.optional cfg.biolatency.milliseconds "--milliseconds") + (lib.optional cfg.biolatency.queued "--queued") + (lib.optional cfg.biolatency.perDisk "-D") + (lib.optional cfg.biolatency.perFlag "-F") + (lib.optionals (cfg.biolatency.disk != "") [ + "-d" + cfg.biolatency.disk + ]) + [ + "--timestamp" + (toString cfg.biolatency.interval) + ] + ]; +in +{ + options.nixos-flake.monitoring = { + enable = lib.mkEnableOption "test-run-bracketed system monitoring"; + + outputDir = lib.mkOption { + type = lib.types.str; + default = "/var/lib/monitoring/runs/%i"; + example = "/mnt/results/%i/monitoring"; + description = '' + Per-instance directory under which monitor daemons write + their output. The literal %i is resolved by systemd to the + unit's instance argument (typically the run-id), so each + instantiation gets its own subtree. Consumers usually back + this path with a virtiofs share or a block-device mount so + the captured data lives outside the guest's ephemeral + root. + ''; + }; + + sysstat = { + enable = lib.mkOption { + type = lib.types.bool; + default = cfg.enable; + description = '' + Enable the monitor-sysstat@<run-id>.service template + unit. The orchestrator instantiates one unit per run-id. + ''; + }; + interval = lib.mkOption { + type = lib.types.ints.positive; + default = 5; + description = "sadc sample interval in seconds."; + }; + maxSamples = lib.mkOption { + type = lib.types.ints.positive; + default = 999999; + description = '' + Upper bound on samples sadc records before exiting on its + own. Large enough that the recorder runs for the full + workload duration; the orchestrator stops the service + when the workload ends. + ''; + }; + }; + + cpuGovernor = { + enable = lib.mkOption { + type = lib.types.bool; + default = cfg.enable; + description = '' + Enable the monitor-cpu-governor@<run-id>.service + template unit. The orchestrator instantiates one unit + per run-id. + ''; + }; + }; + + blkalgn = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Enable the monitor-blkalgn@<run-id>.service template + unit. Off by default even when monitoring.enable is on, + because blkalgn requires the closure to ship + blkalgn-libbpf — and the upstream BCC tag the + libbpf-tools derivation builds against does not yet + carry it. Consumers that need blkalgn override + libbpf-tools.src to a bcc fork that does. + ''; + }; + disk = lib.mkOption { + type = lib.types.str; + default = ""; + description = '' + Disk to trace (blkalgn --disk). Empty traces every + block device the kernel exposes. + ''; + }; + ops = lib.mkOption { + type = lib.types.str; + default = ""; + description = '' + Block operation type to trace (blkalgn --ops). Empty + traces every op type. Spelling is case-sensitive and + must match blkalgn's internal ops[] table verbatim. + + Data ops: Read, Write, Flush, Discard, + SecureErase, WriteSame, + WriteZeroes. + + Zoned-device ops: ZoneReset, ZoneResetAll, ZoneOpen, + ZoneClose, ZoneFinish. + + Driver/transport: SCSIIn, SCSIOut, DrvIn, DrvOut. + + For write-amplification analysis the meaningful filter + is "Write". + ''; + }; + }; + + biolatency = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Enable the monitor-biolatency@<run-id>.service template + unit. biolatency-libbpf prints log2 histograms of block + I/O latency to stdout at the configured interval; the + unit captures stdout to + <outputDir>/biolatency/biolatency.txt via systemd's + StandardOutput=append:. Off by default even when + monitoring.enable is on so it stays opt-in like + blkalgn. The guest kernel must enable eBPF tracepoint + support (CONFIG_BPF_EVENTS, CONFIG_DEBUG_INFO_BTF) for + the binary to attach. + ''; + }; + milliseconds = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Print histograms in milliseconds instead of microseconds + (biolatency --milliseconds). Default true because typical + fstests block I/O sits in the 10us-100ms range and the + millisecond bucketing keeps the bucket count manageable + for time-series visualisation. + ''; + }; + queued = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Include OS queued time in the I/O latency total + (biolatency --queued). Default false: the standard + measurement covers issue-to-completion only, which is + what the storage-tier hardware sees. + ''; + }; + perDisk = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Emit a histogram per disk device (biolatency -D). + Useful when several block devices participate in the + workload and per-device tail latency comparison + matters; off by default to keep the captured stdout + compact when there is only one device under test. + ''; + }; + perFlag = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Emit a histogram per set of I/O flags (biolatency -F). + Splits read / write / sync / discard / etc. into + separate histograms. + ''; + }; + disk = lib.mkOption { + type = lib.types.str; + default = ""; + description = '' + Restrict tracing to a single disk by name (biolatency + -d DISK). Empty traces every block device. Note this + is the lowercase short -d form, distinct from the + uppercase -D --disk-grouping flag exposed via perDisk. + ''; + }; + interval = lib.mkOption { + type = lib.types.ints.positive; + default = 30; + description = '' + Histogram print interval in seconds. biolatency's + positional [interval] argument; the trailing [count] + is intentionally omitted so the daemon runs until + SIGTERM rather than terminating after a fixed number + of intervals. Default 30 s gives enough samples for a + typical fstests run without flooding stdout. + ''; + }; + }; + }; + + config = lib.mkIf cfg.enable { + environment.systemPackages = with pkgs; [ + sysstat + # fastfetch is the host_info monitor's collector — invoked + # directly from the ansible host_info/{run,collect}.yml tasks + # with --format json and a fixed module list. The binary is + # ~1.7MB, depends only on libc + libyyjson; cheap addition + # for the host_info coverage even when the monitor is off. + fastfetch + ]; + + systemd.services."monitor-sysstat@" = lib.mkIf cfg.sysstat.enable { + description = "sysstat sadc recorder for run %i"; + documentation = [ + "man:sadc(8)" + "man:sadf(1)" + ]; + + serviceConfig = { + Type = "simple"; + # Each start wipes the previous binary recording so sa-current + # is always per-run-id. Per-run-id means a fresh sa-current + # by construction (each instance has a different %i and a + # different output dir), but the rm guard keeps the unit + # idempotent across same-instance restarts. + ExecStartPre = [ + "${pkgs.coreutils}/bin/mkdir -p ${monitorOutDir}/sysstat" + "${pkgs.coreutils}/bin/rm -f ${monitorOutDir}/sysstat/sa-current ${monitorOutDir}/sysstat/sa-current.json" + ]; + ExecStart = "${pkgs.sysstat}/lib/sa/sadc -S ALL ${toString cfg.sysstat.interval} ${toString cfg.sysstat.maxSamples} ${monitorOutDir}/sysstat/sa-current"; + ExecStopPost = "${sysstatJsonExport} ${monitorOutDir}/sysstat"; + + KillSignal = "SIGTERM"; + TimeoutStopSec = "30s"; + Restart = "no"; + ReadWritePaths = "${monitorOutDir}"; + }; + }; + + systemd.services."monitor-cpu-governor@" = lib.mkIf cfg.cpuGovernor.enable { + description = "CPU governor snapshot for run %i"; + + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${governorSnapshot} ${monitorOutDir}/cpu_governor start"; + ExecStop = "${governorSnapshot} ${monitorOutDir}/cpu_governor end"; + ReadWritePaths = "${monitorOutDir}"; + }; + }; + + systemd.services."monitor-blkalgn@" = lib.mkIf cfg.blkalgn.enable (mkLibbpfMonitor { + name = "blkalgn"; + description = "blkalgn libbpf-tools recorder for run %i"; + args = blkalgnArgs; + }); + + systemd.services."monitor-biolatency@" = lib.mkIf cfg.biolatency.enable (mkLibbpfMonitor { + name = "biolatency"; + description = "biolatency libbpf-tools recorder for run %i"; + args = biolatencyArgs; + captureStdout = true; + }); + }; +} diff --git a/modules/testSuites/blktests.nix b/modules/testSuites/blktests.nix new file mode 100644 index 00000000..966b03ed --- /dev/null +++ b/modules/testSuites/blktests.nix @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# blktests: block layer regression tests. +# +# Upstream: https://github.com/osandov/blktests +# +# Provides the block-device userland tools and SCSI target +# framework that blktests scripts invoke, plus the I/O generator +# and metrics collectors a few test groups rely on. +{ pkgs, ... }: +{ + environment.systemPackages = with pkgs; [ + # Block-device userland tools + nvme-cli + sg3_utils + multipath-tools + dmraid + lvm2 + mdadm + + # SCSI target framework used by iSCSI and FC test groups + # (nixpkgs ships the fork as targetcli-fb; the upstream name + # 'targetcli' has been retired from nixpkgs.) + targetcli-fb + + # I/O generator and stats used across several groups + fio + sysstat + ]; +} diff --git a/modules/testSuites/fstests.nix b/modules/testSuites/fstests.nix new file mode 100644 index 00000000..8a1371c0 --- /dev/null +++ b/modules/testSuites/fstests.nix @@ -0,0 +1,370 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# fstests (xfstests-dev): filesystem regression tests. +# +# Upstream: https://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git +# +# Provides the userland filesystem tools, load generators, and NFS +# and SMB clients that the local and network fstests variants +# invoke on the target host. Services are enabled with mkDefault so +# consumers can turn them off for test subsets that do not exercise +# them. +{ pkgs, lib, ... }: +{ + environment.systemPackages = with pkgs; [ + # Filesystem userland tools for the filesystems fstests covers + xfsprogs + xfsdump + btrfs-progs + e2fsprogs + f2fs-tools + + # Load generators and stressors fstests scripts invoke. + # (dbench used to be in nixpkgs; upstream removed it in late + # 2025 as unmaintained for 14 years and broken, so fstests + # subsets that need dbench currently fall back to the consumer's + # own checkout.) + fio + stress-ng + + # POSIX attribute, ACL, and quota userland + attr + acl + quota + + # NFS and SMB clients for the network filesystem fstests variants + nfs-utils + cifs-utils + + # The xfstests-dev test harness. The overlay in overlays/xfstests.nix + # bumps the nixpkgs version (2023.05.14 broken with modern GCC) to + # a current upstream snapshot. Installing it here puts `check` on + # the guest's PATH via /run/current-system/sw/bin so consumers + # that skip the in-guest clone-and-build path can drive the same + # binary the package builds from source. + xfstests + + # Runtime deps xfstests ./check (and the individual test scripts + # under tests/*) invoke. nixpkgs' xfstests-check wrapper embeds + # an explicit PATH with these store paths before exec'ing ./check; + # when consumers drive ./check directly we need the same tools + # reachable via /run/current-system/sw/bin. Keeping the list here + # alongside xfstests mirrors the wrapper's intent and keeps the + # dependency relationship explicit rather than relying on + # nixpkgs' propagatedBuildInputs (which only affect build-time + # closure). + perl + bc + keyutils + libcap + lvm2 + psmisc + which + util-linux + + # Userland utilities individual tests reach for. Each one + # corresponds to a _require_command guard in xfstests' + # common/* helpers; absent the binary, the affected tests + # skip with "X utility required, skipped this test". Keeping + # the list scoped to this module so consumers that opt out of + # fstests don't pay closure size for them. + duperemove + indent + acct + fsverity-utils + man-db + thin-provisioning-tools + + # checkbashisms flags /bin/sh portability slips against the + # xfsprogs source tree. xfstests' auto-bashisms tests skip + # with "checkbashisms utility required, skipped this test" + # absent the binary. nixpkgs splits this out from the full + # debian-devscripts suite as a standalone package, so pull + # that and skip the rest of the suite. + checkbashisms + + # python3 with dbus-python importable, so xfs/802 can talk to + # the xfs_scrub@ family of systemd units via D-Bus. The test's + # gate is literally `python3 -c 'import dbus'` (xfs/802:45) — + # absent the import succeeding, the test skips with + # "test requires python3-dbus". + # Build a per-closure python3 wrapper that has dbus-python on + # its PYTHONPATH and place that wrapper on PATH; xfsprogs' + # xfs_scrub_all.py uses the same `python3` from /run/current- + # system/sw/bin so it picks up the same import surface + # automatically. + (python3.withPackages (ps: [ ps.dbus-python ])) + + # Build toolchain that xfstests' ./check (and the common + # wrapper scripts that drive it) expect. The wrappers bail + # early when `which {gcc,make,git,automake}` comes up empty, + # and several tests rebuild helper binaries (e.g. src/*.c) on + # the fly via a plain `make` in the xfstests tree. The nixpkgs + # xfstests derivation ships its helpers pre-built, but the + # request-side checks still insist the toolchain is present so + # the test scripts can invoke make/gcc themselves when they + # regenerate fixtures. Ship gcc + gnumake + git + automake so + # both paths work. + gcc + gnumake + git + automake + ]; + + # xfstests convention: most tests run as an unprivileged user + # named `fsgqa` in group `fsgqa`, and a few tests (xfs/106 in + # particular) explicitly check that fsgqa's primary group is + # fsgqa. Test wrappers fail the whole run if the user or group + # is missing, and NixOS' default user database ships neither. + # Declare both here, scoped to the fstests module so consumers + # that opt out don't carry an extra user. + # The `sys` group is a second xfstests convention (some tests + # set group ownership to `sys`); nixpkgs' default `sys` is only + # created if a service declares it, so declare it explicitly. + # Explicit gids for the test-framework groups. Without an + # explicit `gid =`, NixOS' auto-allocator can land on a slot + # already reserved by the closure (e.g. gid 2 = kmem in + # nixos/modules/misc/ids.nix), which fails module evaluation + # with "Failed assertions: UIDs and GIDs must be unique!". + # NixOS' system gid reservations top out around 327; pinning + # to 500+ stays clear of every reserved slot in ids.nix and + # safely below the 1000 boundary that separates system from + # regular-user gids. xfstests checks group existence by name + # (not by numeric gid), so the specific values are arbitrary + # as long as they are stable and non-conflicting. + users.groups.fsgqa = { + gid = 500; + }; + users.groups.sys = { + gid = 501; + }; + users.groups.daemon = { + gid = 502; + }; + # fsgqa2 group is required by ~6 xfstests per section that gate + # on `_require_group fsgqa2` — generic/097, generic/132 and + # similar tests that swap effective gid to "the second user's + # primary group". The user already exists below. Conventional + # distro `useradd -U fsgqa2` creates the matching group + # implicitly; this module declares users individually so the + # group has to be declared too. Without this, the tests skip + # with "fsgqa2 group not defined." + users.groups.fsgqa2 = { + gid = 504; + }; + # Same convention applies to the high-uid 123456-fsgqa user + # used by generic/381 and a handful of xattr/quota tests: + # `_require_group 123456-fsgqa` checks the same-name group + # exists. Distro `useradd -U` would create it implicitly; + # declare it here for the closure path. + users.groups."123456-fsgqa" = { + gid = 505; + }; + + # Each test-framework user gets a real shell via + # `useDefaultShell = true`. xfstests' `_require_user` helper at + # common/rc:2861 runs `echo /bin/true | _su <user>`, which in + # turn invokes `su - <user> -c /bin/true`. NixOS' default for + # isSystemUser=true is shell=nologin, which makes su fail and + # the test skips with "<user> cannot execute commands." That + # pattern can account for dozens of unexpected skips per section + # even after the users themselves are declared. Conventional + # distro `useradd` leaves the shell at the system default + # (/bin/bash); this restores parity. + users.users.fsgqa = { + isSystemUser = true; + group = "fsgqa"; + useDefaultShell = true; + description = "xfstests unprivileged test user"; + }; + + # Additional accounts xfstests reaches for. fsgqa2 is the + # convention for the "second unprivileged user" that group + # ownership tests need (generic/596 and friends). 123456-fsgqa + # exercises the high-uid path in tools that don't truncate to + # 16 bits (generic/381 hardcodes the 123456 numeric uid in its + # check). daemon is the canonical "system service user" that + # quota and accounting tests target (generic/079 calls daemon + # explicitly). Declaring all three matches the conventional + # distro useradd loop so the test framework sees the user + # database it expects. + users.users.fsgqa2 = { + isSystemUser = true; + # fsgqa2's primary group must be fsgqa2 (not fsgqa). The + # xfstests pattern is "two unprivileged users with their own + # primary groups so tests that swap egid get a distinct + # ownership context"; mapping fsgqa2 onto fsgqa breaks that + # split and confuses tests like generic/097 that compare egid + # before/after a setegid(fsgqa2's gid) call. + group = "fsgqa2"; + useDefaultShell = true; + description = "xfstests secondary unprivileged test user"; + }; + users.users."123456-fsgqa" = { + isSystemUser = true; + uid = 123456; + # Primary group must match the user's own name so + # `_require_group 123456-fsgqa` resolves and the per-user + # gid the test sets up via setegid is distinct from + # fsgqa's gid. + group = "123456-fsgqa"; + useDefaultShell = true; + description = "xfstests numeric-uid test user (generic/381)"; + }; + users.users.daemon = { + isSystemUser = true; + group = "daemon"; + useDefaultShell = true; + description = "xfstests system-service test user"; + }; + + # `bin` is the traditional Unix system account for binary-owned + # files. Conventional distros add it via useradd, and a subset + # of xfstests (~28 tests, including some generic/* and xfs/* + # tests that chown to bin during setup) skip with "bin user not + # defined" when it is absent. NixOS does not ship a bin user by + # default, so declare it here. + # + # Note: do NOT pin uid/gid to 2. NixOS' default user database + # already declares the historical uid 2 / gid 2 slot for kmem + # (config.ids.uids.bin / config.ids.gids.bin map elsewhere on + # NixOS), so an explicit gid = 2 here triggers + # "Failed assertions: UIDs and GIDs must be unique!" at module + # evaluation time. But leaving gid unpinned is also wrong: + # NixOS' auto-allocator scans from a low base and can land on + # gid 2 itself, hitting the same assertion. Pin to a safe slot + # in the same 500+ band as the other test-framework groups + # (well clear of NixOS' system gid reservations that top out + # around 327, and below the 1000 boundary that separates system + # from regular-user gids). xfstests' bin-using tests check user + # existence (_require_user_exists), not the numeric uid, so + # the specific value is arbitrary as long as it is stable and + # non-conflicting. + users.groups.bin = { + gid = 503; + }; + users.users.bin = { + isSystemUser = true; + group = "bin"; + useDefaultShell = true; + description = "xfstests bin test user (system binaries owner)"; + }; + + services.nfs.server.enable = lib.mkDefault true; + services.rpcbind.enable = lib.mkDefault true; + + # The imageless backend imports profiles/minimal.nix, which sets + # documentation.enable = mkDefault false; that propagates into + # documentation.man.enable and skips aggregating each package's + # share/man/ into system-path. xfs/293, xfs/294 and similar + # tests gate on `man -P cat xfs_io` returning a manpage and fail + # outright (not notrun) when no manual entry resolves. Force the + # man infrastructure on so xfsprogs' man8 pages are reachable + # via the standard /run/current-system/sw/share/man/ hierarchy. + documentation.enable = lib.mkForce true; + documentation.man.enable = lib.mkForce true; + + # Expose xfsprogs' bundled systemd units ([email protected], + # xfs_scrub_all.service+.timer, [email protected], plus + # the matching _fail siblings, and [email protected] / + # xfs_healer_start.service / system-xfs_healer.slice when the + # source carries them) so xfstests can resolve them via + # `_require_systemd_unit_defined`. xfsprogs installs these to + # $out/lib/systemd/system/ — note the path is in the `out` + # output, not `bin`. NixOS only forwards units from packages + # explicitly listed in systemd.packages, and the path it walks + # is "<package>/lib/systemd/system/" against whichever output + # the user passes. `systemd.packages = [ pkgs.xfsprogs ]` + # resolves to the package's *default* output (bin), which has + # no lib/systemd/system/ — units silently never get forwarded + # and tests skip with `systemd unit "[email protected]" not + # found`. Pass pkgs.xfsprogs.out explicitly to point the + # forwarder at the output that actually carries the units. + systemd.packages = [ pkgs.xfsprogs.out ]; + + # Force-load the xfs module early so /sys/fs/xfs/debug/bug_on_assert + # exists by the time the systemd-tmpfiles rule below fires. The + # imageless module restores the standard + # `boot.kernelModules → /etc/modules-load.d/nixos.conf` wiring that + # NixOS' kernel.nix gates off when boot.kernel.enable=false, so the + # natural option Just Works here. + boot.kernelModules = [ "xfs" ]; + + # dm-flakey, dm-delay, dm-log-writes etc. (used by xfstests under + # generic/, and exercised heavily once CONFIG_DM_* are enabled) drive + # device-mapper through libdevmapper, which serializes operations via + # an SysV semaphore "udev cookie": dmsetup increments the semaphore + # before the ioctl and udevd is expected to decrement it once it has + # processed the resulting uevent. The decrement is performed by lvm2's + # /lib/udev/rules.d/{10-dm,13-dm-disk,95-dm-notify}.rules. Putting + # lvm2 on PATH (above) is not enough — the rules also need to land + # in the active udev ruleset, and on NixOS that is wired up by the + # upstream services.lvm module (which knows to grab lvm2's `out` + # output containing the rules and to ship its tmpfiles + systemd + # units). The imageless module disables services.lvm by default to + # keep the minimal closure small, so use mkForce here to override + # that default and opt the fstests test suite back in. Without this, + # dm-* tests like generic/034 hang indefinitely with dmsetup blocked + # in __do_semtimedop because the cookie semaphore is never released. + services.lvm.enable = lib.mkForce true; + + # xfstests ./check and the wrapper scripts that drive it all + # start with `#!/bin/bash`, and ld-version.sh uses + # `#!/usr/bin/awk -f`. NixOS only ships /bin/sh (pointing at + # bash) and /usr/bin/env out of the box, so the kernel's + # shebang resolver fails with ENOENT on those interpreters and + # the scripts refuse to exec with a misleading "No such file or + # directory" on the script itself. Patching every shebang in + # the xfstests tree is intractable, and redirecting through + # `env` in a wrapper only shifts the problem. Declare the two + # additional compat symlinks here so the unmodified upstream + # scripts run unmodified. Scoped to this module so consumers + # that opt out of the fstests test suite don't carry the FHS bits + # they don't need. + systemd.tmpfiles.rules = [ + "L+ /bin/bash - - - - ${pkgs.bash}/bin/bash" + "L+ /usr/bin/awk - - - - ${pkgs.gawk}/bin/awk" + + # Expose the xfstests tree at a stable, FHS-standard path so + # driver scripts and external tools can cd into it without + # resolving the nixpkgs store hash at runtime. Matches what + # distribution packages of xfstests conventionally install + # under /usr/lib/xfstests (Debian, Fedora, nixpkgs via the + # libexec→/usr/lib translation) and gives consumers a + # predictable path to use as CWD. + "L+ /usr/lib/xfstests - - - - ${pkgs.xfstests}/lib/xfstests" + + # Same FHS-symlink pattern for xfsprogs' libexec dir. + # xfstests' common/config probes + # if [ ! -x "$XFS_HEALER_PROG" ] && \ + # [ -e /usr/libexec/xfsprogs/xfs_healer ]; then ... + # to find xfs_healer when `type -P` returns empty. xfsprogs + # installs xfs_healer + xfs_healer_start to + # <xfsprogs.out>/libexec/xfsprogs/ — outside PATH, since the + # `bin` and `out` outputs are split. Without this symlink, + # the fallback branch never resolves and ~9 tests + # (xfs/657-664, 667) skip with "xfs_healer utility required". + "L+ /usr/libexec/xfsprogs - - - - ${pkgs.xfsprogs.out}/libexec/xfsprogs" + + # Override the runtime default of XFS bug_on_assert from 1 + # to 0 when the running kernel was built with + # CONFIG_XFS_ASSERT_FATAL=y. CONFIG_XFS_ASSERT_FATAL=y makes + # ASSERT() call BUG() — instantly panicing the guest the + # moment any XFS internal invariant is violated. That is the + # right default for a developer's working kernel (catch bugs + # immediately, see the stack), but on a test-framework guest + # it bails xfstests mid-run on the first assertion and every + # test that runs after the first ASSERT site is touched is + # lost. xfstests itself flags this with the + # `_require_no_xfs_bug_on_assert` helper (~2 .notruns per + # section). Override to 0 so ASSERT() downgrades to WARN(); + # the assertion still fires into dmesg and the test + # continues, which is the exact policy xfstests assumes for + # its bug_on_assert-gated tests. + # Persistence: w+ writes the value, follows symlinks, and + # tolerates the file being missing (xfs may be unloaded). + # The setting reapplies on every boot via systemd-tmpfiles, + # which on NixOS runs as part of the activation phase. + "w+ /sys/fs/xfs/debug/bug_on_assert - - - - 0" + ]; +} diff --git a/modules/testSuites/gitr.nix b/modules/testSuites/gitr.nix new file mode 100644 index 00000000..44443c53 --- /dev/null +++ b/modules/testSuites/gitr.nix @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# gitr: git regression tests, the in-tree test suite +# shipped with git.git under t/. +# +# Upstream: https://github.com/git/git +# +# Git's own test suite shells out to Subversion and Mercurial for +# its foreign-SCM suites and exercises perl integration heavily; +# the suite expects gitFull rather than the minimal git package +# so perl bindings and gitweb support are available. +{ pkgs, ... }: +{ + environment.systemPackages = with pkgs; [ + gitFull + + subversion + mercurial + + perl + ]; +} diff --git a/modules/testSuites/ltp.nix b/modules/testSuites/ltp.nix new file mode 100644 index 00000000..95ad95a5 --- /dev/null +++ b/modules/testSuites/ltp.nix @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# LTP: Linux Test Project test suites. +# +# Upstream: https://github.com/linux-test-project/ltp +# +# LTP is cloned and built from source by the consumer's own +# tooling. This module supplies only the build toolchain LTP's +# configure-and-make cycle expects and the runtime libraries the +# syscall, IPC, security, and filesystem suites link against. LTP +# itself is not in nixpkgs. +{ pkgs, ... }: +{ + environment.systemPackages = with pkgs; [ + # Build toolchain LTP's configure and make cycle invokes. + autoconf + automake + m4 + libtool + pkg-config + flex + bison + + # Runtime libraries LTP suites link against. libacl is provided + # by the acl package, libnuma by numactl, libssl by openssl. + acl + libcap + libaio + numactl + libsepol + libselinux + openssl + ]; +} diff --git a/modules/testSuites/mmtests.nix b/modules/testSuites/mmtests.nix new file mode 100644 index 00000000..4f9aef3e --- /dev/null +++ b/modules/testSuites/mmtests.nix @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# mmtests: memory management test and benchmark harness. +# +# Upstream: https://github.com/gormanm/mmtests +# +# mmtests is perl-based and consumes its own result sets through +# gnuplot. CPU frequency control, DMI inventory, and the standard +# system metric collectors are invoked by several benchmark +# harnesses in the suite. +{ pkgs, ... }: +{ + environment.systemPackages = with pkgs; [ + perl + gnuplot + + cpupower + dmidecode + + sysstat + iotop + powertop + ]; +} diff --git a/modules/testSuites/pynfs.nix b/modules/testSuites/pynfs.nix new file mode 100644 index 00000000..b7917ef2 --- /dev/null +++ b/modules/testSuites/pynfs.nix @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# pynfs: Python-based NFSv4 protocol conformance tests. +# +# Upstream: https://git.linux-nfs.org/?p=bfields/pynfs.git +# +# pynfs is cloned and driven by the consumer's own tooling. This +# module supplies only the Python runtime and ply parser-generator +# the harness imports, plus the NFS userland the scripts shell out +# to for server-side fixtures. A pre-built pynfs binary is +# available as a custom package in this flake (pkgs/pynfs.nix) but +# is not installed by default to avoid duplicating the consumer's +# own checkout. +{ pkgs, ... }: +{ + environment.systemPackages = with pkgs; [ + python3 + python3Packages.ply + + nfs-utils + ]; +} diff --git a/modules/testSuites/selftests.nix b/modules/testSuites/selftests.nix new file mode 100644 index 00000000..a4e1c27c --- /dev/null +++ b/modules/testSuites/selftests.nix @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# selftests: Linux kernel in-tree self-tests. +# +# Upstream: tools/testing/selftests in the Linux kernel tree. +# +# Installs the userland tools selftests invoke: perf and related +# tracing, CPU and memory placement, capability and seccomp +# userland, traffic-control helpers, and ethernet link controls. +# tc(8) is provided by iproute2 on NixOS, so no separate package +# is required for the kselftest net suites. +{ pkgs, ... }: +{ + environment.systemPackages = with pkgs; [ + perf-tools + + numactl + + libcap + libseccomp + keyutils + + iproute2 + ethtool + ]; +} diff --git a/modules/testSuites/sysbench.nix b/modules/testSuites/sysbench.nix new file mode 100644 index 00000000..3b989aac --- /dev/null +++ b/modules/testSuites/sysbench.nix @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# sysbench: scriptable benchmarking tool, commonly used +# for MySQL and PostgreSQL workload measurement. +# +# Upstream: https://github.com/akopytov/sysbench +# +# Enables MariaDB (NixOS' mysql service uses it) and PostgreSQL +# with lib.mkDefault so consumers can disable one or both when +# a run only targets a specific backend. The MySQL service +# package is also set via mkDefault so downstreams that prefer +# the upstream Oracle MySQL can substitute without mkForce. +{ pkgs, lib, ... }: +{ + environment.systemPackages = with pkgs; [ + sysbench + mariadb + postgresql + ]; + + services.mysql = { + enable = lib.mkDefault true; + package = lib.mkDefault pkgs.mariadb; + }; + + services.postgresql.enable = lib.mkDefault true; +} diff --git a/modules/user-options.nix b/modules/user-options.nix new file mode 100644 index 00000000..574807cf --- /dev/null +++ b/modules/user-options.nix @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Shared option schema for the unprivileged account. +# Imported by modules/user.nix (which creates the account) and +# by each backend's user.nix (which sets backend-specific +# defaults). NixOS deduplicates same-path imports, so the +# option is declared exactly once at evaluation time regardless +# of how many modules pull this in. +{ lib, ... }: +{ + options.nixos-flake.user = { + name = lib.mkOption { + type = lib.types.str; + default = "user"; + description = "Unprivileged account name."; + }; + + extraGroups = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + example = [ + "wheel" + "kvm" + ]; + description = '' + Supplementary groups for the unprivileged account. The + option default is empty; backends set per-backend + defaults at mkDefault priority via their bundled + user.nix file (modules/backends/<backend>/user.nix). + Consumers that want to add groups without replacing + the backend's default set them via + users.users.<name>.extraGroups, which NixOS list- + merges with the value this module assigns from + cfg.extraGroups. + ''; + }; + }; +} diff --git a/modules/user.nix b/modules/user.nix new file mode 100644 index 00000000..329b98e7 --- /dev/null +++ b/modules/user.nix @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Opt-in unprivileged account with passwordless sudo. The +# option schema lives in modules/user-options.nix, shared with +# the per-backend user.nix files so the backends can set +# extraGroups defaults without forcing user.nix to be imported. +# This module imports the same schema and adds the config block +# that actually creates the account. +# +# Also sets a break-glass root password ("root") for serial- +# console recovery when SSH is unreachable. Safe because the +# backend modules disable password authentication at the sshd +# layer. +{ config, lib, ... }: +let + cfg = config.nixos-flake.user; +in +{ + imports = [ ./user-options.nix ]; + + config = { + users.users.${cfg.name} = { + isNormalUser = true; + extraGroups = cfg.extraGroups; + }; + + # Unattended tests would deadlock on a sudo prompt. + security.sudo.wheelNeedsPassword = false; + + # Serial-console break-glass. mkDefault so a consumer who + # wants mutable users or a different break-glass password + # can override without lib.mkForce; the imageless backend + # hardcodes the same values at normal priority and wins on + # imageless+user. + users.mutableUsers = lib.mkDefault false; + users.users.root.initialPassword = lib.mkDefault "root"; + }; +} diff --git a/overlays/default.nix b/overlays/default.nix new file mode 100644 index 00000000..f380d5d0 --- /dev/null +++ b/overlays/default.nix @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Package overlays and custom packages. Each file in this directory +# modifies one nixpkgs package. Custom packages not in nixpkgs are +# defined in pkgs/ using callPackage and merged here. +# +# Usage in a NixOS module: +# { nixpkgs.overlays = [ (import ./overlays) ]; } +# +# Reference: https://nixos.org/manual/nixpkgs/stable/#chap-overlays +final: prev: +prev.lib.composeManyExtensions [ + (import ./fio.nix) + (import ./xfstests.nix) + (import ./spdk.nix) + (overlayFinal: _: import ../pkgs overlayFinal) +] final prev diff --git a/overlays/fio.nix b/overlays/fio.nix new file mode 100644 index 00000000..819cd83a --- /dev/null +++ b/overlays/fio.nix @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Override fio to enable io_uring support via liburing and install +# the storage stack test suite, io_uring exerciser, and example +# job files that nixpkgs omits. +# +# nixpkgs fio (pkgs/by-name/fi/fio/package.nix) builds with libaio +# and libnbd but does not enable liburing or install t/io_uring, +# the Python NVMe test scripts, or the example .fio files. +final: prev: { + fio = prev.fio.overrideAttrs (prevAttrs: { + buildInputs = (prevAttrs.buildInputs or [ ]) ++ [ final.liburing ]; + + postInstall = (prevAttrs.postInstall or "") + '' + # t/io_uring: standalone io_uring and NVMe passthrough exerciser. + # Not a fio self-test. Uses raw io_uring syscalls to benchmark + # the kernel block layer. Fail the build if it is missing, since + # liburing is now a declared buildInput. + test -x t/io_uring + install --mode=755 t/io_uring $out/bin/fio-io-uring + + # Storage stack test suite. Python scripts that validate kernel + # NVMe features (FDP, DIF/DIX, streams, multi-range TRIM, ZBD) + # using fio as the I/O generator. + install --mode=755 --directory $out/share/fio/t + install --mode=755 --directory $out/share/fio/t/zbd + install --mode=644 t/*.py $out/share/fio/t/ + install --mode=755 t/one-core-peak.sh $out/share/fio/t/ + install --mode=755 t/zbd/* $out/share/fio/t/zbd/ + + # Example job files. + install --mode=755 --directory $out/share/doc/fio/examples + install --mode=644 examples/*.fio $out/share/doc/fio/examples/ + ''; + }); +} diff --git a/overlays/spdk.nix b/overlays/spdk.nix new file mode 100644 index 00000000..a06b6fe4 --- /dev/null +++ b/overlays/spdk.nix @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Install the SPDK NVMe example exercisers nixpkgs builds but doesn't ship: +# cmb_copy (Controller Memory Buffer) and pmr_persistence (Persistent Memory +# Region), for testing emulated NVMe CMB/PMR from SPDK's userspace driver. +final: prev: { + spdk = prev.spdk.overrideAttrs (prevAttrs: { + # Prepend so the package's own postInstall rpath cleanup covers these too. + postInstall = '' + for ex in cmb_copy pmr_persistence; do + test -x build/examples/$ex + install --mode=755 build/examples/$ex $out/bin/spdk_nvme_$ex + done + '' + + (prevAttrs.postInstall or ""); + }); +} diff --git a/overlays/xfstests.nix b/overlays/xfstests.nix new file mode 100644 index 00000000..5e3e007e --- /dev/null +++ b/overlays/xfstests.nix @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Override xfstests to a current version. The nixpkgs version +# (2023.05.14) fails to build with modern GCC due to +# -Werror=implicit-function-declaration in ltp/fsx.c. +# +# The nixpkgs recipe handles all the packaging quirks (path +# patching, wrapper script, libtool workaround). We only need +# to bump the version and hash. +final: prev: +let + version = "2026.03.20"; +in +{ + xfstests = prev.xfstests.overrideAttrs (prevAttrs: { + inherit version; + src = prev.fetchzip { + url = "https://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git/snapshot/xfstests-dev-v${version}.tar.gz"; + hash = "sha256-f82TsOEikGo9uQyM0hQmgy6A6R/2U4Ot/eINq34kohM="; + }; + # Newer xfstests needs pkg-config for autoconf macros. + nativeBuildInputs = (prevAttrs.nativeBuildInputs or [ ]) ++ [ prev.pkg-config ]; + # gdbm.dev provides gdbm-ndbm.h which xfstests' configure + # probes for to enable HAVE_DB; without it src/dbtest is not + # built and tests like generic/010 are skipped with the + # "src/dbtest not built" .notrun reason. + # + # liburing unlocks the io_uring probe in src/feature.c. + # Without HAVE_LIBURING defined at compile time, + # check_uring_support() returns 1 unconditionally and every + # _require_io_uring callsite skips with + # "kernel does not support IO_URING" — even when + # CONFIG_IO_URING=y and the syscall is fully functional. + # The nixpkgs xfstests recipe omits liburing from + # buildInputs, so xfstests winds up never probing the kernel + # regardless of the runtime configuration. Adding it here + # lets autoconf detect liburing.h, sets HAVE_LIBURING, and + # check_uring_support() actually calls io_uring_queue_init + # against the running kernel. + buildInputs = (prevAttrs.buildInputs or [ ]) ++ [ + prev.gdbm + prev.liburing + ]; + }); +} diff --git a/pkgs/cpupower.nix b/pkgs/cpupower.nix new file mode 100644 index 00000000..33d45bf6 --- /dev/null +++ b/pkgs/cpupower.nix @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Standalone cpupower built from a kernel source tree. +# +# nixpkgs cpupower is tied to linuxPackages and requires a +# NixOS-built kernel. This package builds from any kernel source, +# defaulting to linux_latest when no override is provided. +{ + lib, + stdenv, + buildPackages, + linux_latest, + pciutils, + gettext, + which, +}: + +stdenv.mkDerivation { + pname = "cpupower"; + inherit (linux_latest) version src; + + nativeBuildInputs = [ + gettext + which + ]; + buildInputs = [ pciutils ]; + + postPatch = '' + cd tools/power/cpupower + substituteInPlace Makefile \ + --replace-fail /bin/true ${buildPackages.coreutils}/bin/true \ + --replace-fail /usr/bin/install ${buildPackages.coreutils}/bin/install + ''; + + makeFlags = [ + "CROSS=${stdenv.cc.targetPrefix}" + "CC=${stdenv.cc.targetPrefix}cc" + "LD=${stdenv.cc.targetPrefix}cc" + ]; + + installFlags = lib.mapAttrsToList (n: v: "${n}dir=${placeholder "out"}/${v}") { + bin = "bin"; + sbin = "sbin"; + man = "share/man"; + include = "include"; + lib = "lib"; + libexec = "libexec"; + locale = "share/locale"; + doc = "share/doc/cpupower"; + conf = "etc"; + bash_completion_ = "share/bash-completion/completions"; + unit = "lib/systemd/system"; + }; + + enableParallelBuilding = true; + + meta = { + description = "Tool to examine and tune power saving features"; + homepage = "https://www.kernel.org/"; + license = lib.licenses.gpl2Only; + mainProgram = "cpupower"; + platforms = lib.platforms.linux; + maintainers = [ ]; + }; +} diff --git a/pkgs/damo.nix b/pkgs/damo.nix new file mode 100644 index 00000000..b618a9b0 --- /dev/null +++ b/pkgs/damo.nix @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# DAMO: DAMON (Data Access MONitor) user-space tool. +# +# damo's upstream source layout is flat (src/*.py, no __init__.py). +# The packaging/ directory provides setup.py with console_scripts +# and package_dir={"": "src"}, but expects a src/damo/ package +# directory. The postPatch phase performs the same reorganization +# that Debian and packaging/build.sh do: +# +# 1. Copy packaging/pyproject.toml and packaging/setup.py to root +# 2. Create src/damo/ and move all .py files into it +# 3. Create src/damo/__init__.py +# +# Source: https://github.com/damonitor/damo +# Debian: https://packages.debian.org/trixie/damo +# Fedora: https://src.fedoraproject.org/rpms/python-damo +{ + lib, + python3Packages, + fetchFromGitHub, +}: + +python3Packages.buildPythonApplication rec { + pname = "damo"; + version = "3.2.0"; + pyproject = true; + + src = fetchFromGitHub { + owner = "damonitor"; + repo = "damo"; + rev = "v${version}"; + hash = "sha256-DflgnU7/tepLwnDOOVahqsPXMSu0GJ/OWYnVd5qSCQA="; + }; + + # Reorganize flat src/*.py into a proper Python package at + # src/damo/ so setuptools.find_packages(where="src") finds it. + # This matches Debian's debian/rules execute_before_dh_auto_configure + # and upstream's packaging/build.sh. + postPatch = '' + cp packaging/pyproject.toml pyproject.toml + cp packaging/setup.py setup.py + mkdir --parents src/damo + cp src/*.py src/damo/ + touch src/damo/__init__.py + ''; + + build-system = [ python3Packages.setuptools ]; + + # Pure Python, no runtime dependencies beyond the standard library. + + # Tests require a running kernel with CONFIG_DAMON enabled. + doCheck = false; + + meta = { + description = "DAMON user-space tool for data access monitoring"; + homepage = "https://github.com/damonitor/damo"; + license = lib.licenses.gpl2Only; + platforms = lib.platforms.linux; + mainProgram = "damo"; + maintainers = [ ]; + }; +} diff --git a/pkgs/default.nix b/pkgs/default.nix new file mode 100644 index 00000000..38c3d91a --- /dev/null +++ b/pkgs/default.nix @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Custom packages not available in nixpkgs. +# +# Each package is a file declaring a function whose arguments are its +# dependencies. Use callPackage to compose them: +# +# xnvme = pkgs.callPackage ./xnvme.nix { }; +# +# The overlay at overlays/default.nix imports this file and merges +# the packages into the nixpkgs set. +# +# Reference: https://nix.dev/tutorials/callpackage +pkgs: { + cpupower = pkgs.callPackage ./cpupower.nix { }; + damo = pkgs.callPackage ./damo.nix { }; + libbpf-tools = pkgs.callPackage ./libbpf-tools.nix { }; + nfstest = pkgs.callPackage ./nfstest.nix { }; + pynfs = pkgs.callPackage ./pynfs.nix { }; + xnvme = pkgs.callPackage ./xnvme.nix { }; +} diff --git a/pkgs/libbpf-tools.nix b/pkgs/libbpf-tools.nix new file mode 100644 index 00000000..2915782e --- /dev/null +++ b/pkgs/libbpf-tools.nix @@ -0,0 +1,186 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# libbpf-tools: standalone CO-RE BPF tracing tools from the BCC project. +# +# These are lightweight alternatives to the Python-based BCC tools. +# Each tool is a statically-linked binary (against vendored libbpf) +# that embeds its BPF program via a skeleton header. No BCC runtime, +# no Python, no LLVM needed at runtime. +# +# The build requires the full BCC source tree with submodules because +# libbpf-tools/ references ../src/cc/libbpf/src (vendored libbpf) and +# ./bpftool/src (vendored bpftool) via relative paths. +# +# Source: https://github.com/iovisor/bcc/tree/master/libbpf-tools +# Debian: https://packages.debian.org/trixie/libbpf-tools +{ + lib, + stdenv, + fetchFromGitHub, + cargo, + rustc, + rustPlatform, + clang, + llvmPackages, + elfutils, + zlib, + openssl, + json_c, + pkg-config, + gnumake, +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "libbpf-tools"; + version = "0.36.1"; + + src = fetchFromGitHub { + owner = "iovisor"; + repo = "bcc"; + tag = "v${finalAttrs.version}"; + hash = "sha256-yfIyV+NKaSYMNjqFpxPwCgetPacC8OrBBGxRG3P2z0o="; + fetchSubmodules = true; + }; + + # Let stdenv auto-detect the unpacked top-level dir (its name varies + # by src — fetchFromGitHub gives "source", a flake-input override + # gives "${shortRev}-source") and just navigate into the + # libbpf-tools subdir from there. Avoids hardcoding src.name, which + # is not present on every src form (notably flake-input overrides + # that point src at a bcc fork). + postUnpack = "sourceRoot=$sourceRoot/libbpf-tools"; + + # USE_BLAZESYM=1 pulls in the blazesym Rust crate that ships as a + # bcc submodule under libbpf-tools/blazesym. blkalgn (and a few + # other tools — futexctn, memleak, opensnoop) #include + # blazesym.h unconditionally and link against libblazesym_c.a, + # which the Makefile builds via `cargo build --release` inside + # libbpf-tools/blazesym. The cargo invocation runs in the nix + # sandbox with no network, so its dependencies must be vendored + # ahead of time. + # + # importCargoLock + finalAttrs.src reads the lockfile that + # determines the dependency closure from whatever src is + # currently in play. When the per-VM flake overlay overrides src + # (via overrideAttrs pointing at a bcc fork), finalAttrs.src + # updates and cargoDeps re-derives from that fork's pinned + # blazesym submodule SHA — no separate cargoHash to chase when + # the bcc tag (or fork branch) bumps the blazesym submodule. + # + # outputHashes pins blazesym's one git-only dependency (vmlinux, + # not on crates.io). The pinned rev is stable across recent BCC + # tags and the forks that carry blkalgn, so a single hash covers + # both src cases. Update this hash if a future blazesym bump + # moves to a different vmlinux.h rev — nix prints the expected + # hash on mismatch. + # + # cargoRoot is relative to the source root the build cd's into + # (sourceRoot above), which is .../libbpf-tools, so cargoRoot is + # just "blazesym". The lockFile path used by importCargoLock is + # rooted at finalAttrs.src and walks the fully-laid-out source + # tree, so it carries the full prefix. + cargoRoot = "blazesym"; + cargoDeps = rustPlatform.importCargoLock { + lockFile = "${finalAttrs.src}/libbpf-tools/blazesym/Cargo.lock"; + outputHashes = { + "vmlinux-0.0.0" = "sha256-a2q2AuTpqCU7gD0oZmjA+UbGwh4kVazaK6xKgK2L/Nk="; + }; + }; + + nativeBuildInputs = [ + # clang compiles the BPF programs with -target bpf. The BPF target + # is freestanding, so any clang wrapper works; the plain wrapper + # keeps the build closure small. The host-side loader uses + # stdenv.cc via $(CC), not this clang. + llvmPackages.clang + llvmPackages.llvm # llvm-strip + pkg-config + gnumake + # Rust toolchain is needed only because libbpf-tools/Makefile + # does `cd blazesym && cargo build` to produce libblazesym_c.a. + # cargoSetupHook reads cargoDeps and writes the .cargo/config.toml + # that points cargo at the vendored crate copies. + cargo + rustc + rustPlatform.cargoSetupHook + ]; + + buildInputs = [ + elfutils # libelf (dynamic) + zlib # libz (dynamic) + openssl # for bpftool build + json_c # blkalgn (lbs branch) writes JSON via json-c + ]; + + # Nix's hardening flags (-fzero-call-used-regs, -fstack-protector) + # are not valid for the BPF target. Disable them since the BPF + # programs are compiled with clang --target=bpf. + hardeningDisable = [ + "zerocallusedregs" + "stackprotector" + ]; + + makeFlags = [ + "prefix=${placeholder "out"}" + "USE_BLAZESYM=1" + # The bcc Makefile detects cargo via `CARGO ?= $(shell which cargo)`. + # The nix build sandbox has cargo on PATH (nativeBuildInputs above) + # but ships no `which` binary, so the shell call returns empty and + # the blazesym build recipe collapses to `cd ... && build ...`, + # failing with "build: command not found". Pin CARGO=cargo on the + # make line; ?= treats it as already-set and skips the shell call. + "CARGO=cargo" + ]; + + enableParallelBuilding = true; + + installPhase = '' + runHook preInstall + + make install prefix=$out + + # Add -libbpf suffix to avoid collisions with BCC's Python + # wrappers that share the same names (biolatency, execsnoop, + # etc.). Follows the Debian libbpf-tools package convention + # (e.g., /usr/sbin/biolatency-libbpf). + for bin in $out/bin/*; do + name=$(basename "$bin") + # Skip if already suffixed (symlink aliases like btrfsdist). + if [[ "$name" != *-libbpf ]]; then + mv "$bin" "''${bin}-libbpf" + fi + done + + # Recreate filesystem aliases with the suffix. + for link in $out/bin/*-libbpf; do + if [ -L "$link" ]; then + target=$(readlink "$link") + ln --symbolic --force "$(basename "$target")-libbpf" "$link" + fi + done + + # Install man pages from the BCC man/man8/ directory for tools + # that have matching names. + mkdir --parents $out/share/man/man8 + for bin in $out/bin/*-libbpf; do + tool=$(basename "$bin" -libbpf) + manpage="../man/man8/''${tool}.8" + if [ -f "$manpage" ]; then + install --mode=644 "$manpage" $out/share/man/man8/ + fi + done + + runHook postInstall + ''; + + meta = { + description = "Standalone libbpf-based BPF tracing tools from BCC"; + homepage = "https://github.com/iovisor/bcc/tree/master/libbpf-tools"; + license = with lib.licenses; [ + lgpl21Only + bsd2 + ]; + platforms = lib.platforms.linux; + maintainers = [ ]; + }; +}) diff --git a/pkgs/nfstest.nix b/pkgs/nfstest.nix new file mode 100644 index 00000000..97f19c8c --- /dev/null +++ b/pkgs/nfstest.nix @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# NFStest: NFS test suite covering space reservation, direct I/O, +# interoperability, file locking, POSIX compliance, sparse files, +# and server-side copy. +# +# The upstream setup.py tries to generate manpages during build by +# running installed scripts, which fails before install. Patch it +# out so the build completes without running uninstalled scripts. +# +# Source: git://linux-nfs.org/projects/mora/nfstest +{ + lib, + python3Packages, + fetchurl, + nfs-utils, + tcpdump, +}: + +python3Packages.buildPythonApplication rec { + pname = "nfstest"; + version = "3.2"; + pyproject = false; + + src = fetchurl { + url = "https://www.linux-nfs.org/~mora/nfstest/releases/NFStest-${version}.tar.gz"; + hash = "sha256-TUxcWttygx9Xhyde8jxUFP9cV84SDK+xzVPeB2ZXRcE="; + }; + + postPatch = '' + # Disable manpage generation during build. The setup.py build + # command runs installed scripts with --version/--help which + # fails before install. + substituteInPlace setup.py \ + --replace-fail 'create_manpage.run()' 'pass' + + # Replace distutils with setuptools (distutils removed in 3.12). + substituteInPlace setup.py \ + --replace-fail 'from distutils.core import setup' \ + 'from setuptools import setup' \ + --replace-fail 'from distutils.command.build import build' \ + 'from setuptools.command.build import build' + ''; + + # pyproject = false, so the legacy setup.py path is used; pip + # drives the install in installPhase below. build-system still + # seeds setuptools into the build environment even on the legacy + # path, so pip can import setuptools.build_meta. + nativeBuildInputs = [ python3Packages.pip ]; + build-system = [ python3Packages.setuptools ]; + + buildPhase = '' + runHook preBuild + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + pip install --prefix=$out --no-deps --no-build-isolation . + patchShebangs $out/bin + runHook postInstall + ''; + + # Pure Python, no compiled extensions. + doCheck = false; + + # nfs-utils (mount.nfs) and tcpdump (packet capture) are runtime + # CLI tools, not Python dependencies — put them on the wrapped + # scripts' PATH rather than in the propagated closure. + makeWrapperArgs = [ + "--prefix" + "PATH" + ":" + (lib.makeBinPath [ + nfs-utils + tcpdump + ]) + ]; + + meta = { + description = "NFS test suite for Linux"; + homepage = "https://www.linux-nfs.org/~mora/nfstest/"; + license = lib.licenses.gpl2Only; + platforms = lib.platforms.linux; + mainProgram = "nfstest_posix"; + maintainers = [ ]; + }; +} diff --git a/pkgs/pynfs.nix b/pkgs/pynfs.nix new file mode 100644 index 00000000..7ae32b16 --- /dev/null +++ b/pkgs/pynfs.nix @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# pynfs: Python NFSv4 conformance test suite covering NFSv4.0, +# NFSv4.1, and pNFS block layouts with Kerberos bindings. +# +# The top-level setup.py delegates to 4 sub-packages (xdr, rpc, +# nfs4.1, nfs4.0). pip cannot handle this; we use setup.py directly. +# The build includes SWIG-generated Kerberos bindings (compiled C). +# +# No formal releases exist. We pin to a commit. +# +# Source: https://github.com/ffilz/pynfs +{ + lib, + python3Packages, + fetchFromGitHub, + swig, + krb5, +}: + +python3Packages.buildPythonApplication { + pname = "pynfs"; + version = "unstable-2025-03-05"; + pyproject = false; + + src = fetchFromGitHub { + owner = "ffilz"; + repo = "pynfs"; + rev = "d3a1610815117cb6bdf6567e575baedb0d88095e"; + hash = "sha256-grigxIDAQG4hnwA/YmsHNP3aKX61uw76I/ClgT0+u8Q="; + }; + + nativeBuildInputs = [ + swig + krb5.dev + python3Packages.setuptools + ]; + + buildInputs = [ krb5 ]; + + dependencies = with python3Packages; [ + gssapi + ply + ]; + + # Upstream uses distutils which was removed in Python 3.12. + # Patch to use setuptools instead. + postPatch = '' + substituteInPlace setup.py \ + --replace-fail 'from distutils.core import setup' \ + 'from setuptools import setup' + for f in xdr/setup.py rpc/setup.py nfs4.1/setup.py nfs4.0/setup.py; do + if [ -f "$f" ]; then + substituteInPlace "$f" \ + --replace-fail 'from distutils.core import setup' \ + 'from setuptools import setup' + fi + done + ''; + + # The delegating setup.py handles all 4 sub-packages. + buildPhase = '' + runHook preBuild + python setup.py build + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + python setup.py install --prefix=$out --optimize=1 + + # The delegating setup.py installs libraries but not scripts. + # Install the test runners from the sub-packages manually. + mkdir --parents $out/bin + install --mode=755 nfs4.0/testserver.py $out/bin/nfs4-testserver + install --mode=755 nfs4.0/showresults.py $out/bin/nfs4-showresults + if [ -f nfs4.1/testserver.py ]; then + install --mode=755 nfs4.1/testserver.py $out/bin/nfs41-testserver + fi + + # Patch shebangs. + patchShebangs $out/bin + runHook postInstall + ''; + + doCheck = false; + + meta = { + description = "Python NFSv4 conformance test suite"; + homepage = "https://github.com/ffilz/pynfs"; + license = lib.licenses.gpl2Only; + platforms = lib.platforms.linux; + mainProgram = "nfs4-testserver"; + maintainers = [ ]; + }; +} diff --git a/pkgs/xnvme.nix b/pkgs/xnvme.nix new file mode 100644 index 00000000..66dc4e1a --- /dev/null +++ b/pkgs/xnvme.nix @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# xNVMe: cross-platform NVMe user space library and tools. +# +# Built with libaio and liburing backends. SPDK, libvfn, and ISA-L +# are disabled (not needed for kernel development). +# +# Source: https://github.com/xnvme/xnvme +{ + lib, + stdenv, + fetchurl, + meson, + ninja, + pkg-config, + libaio, + liburing, +}: + +stdenv.mkDerivation (finalAttrs: { + pname = "xnvme"; + version = "0.7.5"; + + src = fetchurl { + url = "https://github.com/xnvme/xnvme/releases/download/v${finalAttrs.version}/xnvme-${finalAttrs.version}.tar.gz"; + hash = "sha256-2VGjEV9oaVb1xrg5s5D/tQu0Q/dxH+NfV6B/TQZnJ38="; + }; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + libaio + liburing + ]; + + mesonFlags = [ + "-Dwith-spdk=disabled" + "-Dwith-libvfn=disabled" + "-Dwith-isal=disabled" + "-Dexamples=false" + "-Dtests=false" + "-Dbuild_subprojects=false" + ]; + + meta = { + description = "Cross-platform NVMe user space library and tools"; + homepage = "https://xnvme.io"; + license = lib.licenses.bsd3; + mainProgram = "xnvme"; + platforms = lib.platforms.linux; + maintainers = [ ]; + }; +}) diff --git a/templates/imageless/default.nix b/templates/imageless/default.nix new file mode 100644 index 00000000..969e5dad --- /dev/null +++ b/templates/imageless/default.nix @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Per-VM overrides for the imageless template. +# +# The flake passes nixos-flake and the flake inputs via specialArgs +# so this module can pull in test-suite modules (LTP, fstests, and so +# on) and reference local source checkouts without having to edit +# flake.nix again. +{ + config, + lib, + pkgs, + nixos-flake, + inputs, + ... +}: +{ + imports = [ + # Development tools (editors, tracing, kernel dev comfort). Drop + # this import for a minimal VM that only runs a specific test suite. + nixos-flake.nixosModules.profiles.devel + + # Pick the test-suite modules that match what you intend to + # run, for example: + # nixos-flake.nixosModules.profiles.build-tools + # nixos-flake.nixosModules.testSuites.fstests + # nixos-flake.nixosModules.testSuites.blktests + ]; + + networking.hostName = "vm"; + + # Build a package from a local source checkout. The matching + # fio-src input must be declared in flake.nix. + # nixpkgs.overlays = [ + # (final: prev: { fio = prev.fio.overrideAttrs { src = inputs.fio-src; patches = []; }; }) + # ]; + + # SSH keys for the root and unprivileged accounts. + # users.users.root.openssh.authorizedKeys.keys = [ "ssh-ed25519 ..." ]; + # users.users.${config.nixos-flake.user.name}.openssh.authorizedKeys.keys = [ "ssh-ed25519 ..." ]; +} diff --git a/templates/imageless/flake.nix b/templates/imageless/flake.nix new file mode 100644 index 00000000..a6923eb1 --- /dev/null +++ b/templates/imageless/flake.nix @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Imageless NixOS VM starter. +# +# Create with: +# nix flake init --template "github:linux-kdevops/nixos-flake" +# nix build .#nixosConfigurations.vm.config.system.build.toplevel +# +# Per-VM configuration lives in ./default.nix. This flake composes +# the base modules, applies the overlay, and passes nixos-flake and +# the flake inputs through specialArgs so default.nix can import +# additional modules (for example nixos-flake.nixosModules.testSuites.*) +# and reference local source inputs without re-declaring them. +{ + inputs = { + # Local checkout preferred: consumers pin to a specific + # revision via a subtree or vendored copy and should not + # track upstream HEAD. For upstream, use: + # nixos-flake.url = "github:linux-kdevops/nixos-flake"; + nixos-flake.url = "path:/path/to/nixos-flake"; + nixpkgs.follows = "nixos-flake/nixpkgs"; + + # Local source checkouts (uncomment to use). Multi-attribute + # non-flake inputs decompose into the type/path or type/url + # attribute-set form so each piece (type, source, ref, + # submodules) reads on its own line: + # + # fio-src = { + # type = "path"; + # path = "/home/user/src/fio"; + # flake = false; + # }; + # kmod-src = { + # type = "git"; + # url = "https://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git"; + # ref = "master"; + # flake = false; + # }; + }; + + outputs = + { + self, + nixpkgs, + nixos-flake, + ... + }@inputs: + let + system = "x86_64-linux"; + in + { + nixosConfigurations.vm = nixpkgs.lib.nixosSystem { + inherit system; + specialArgs = { inherit inputs nixos-flake; }; + modules = [ + nixos-flake.nixosModules.backends.imageless + nixos-flake.nixosModules.user + { nixpkgs.overlays = [ nixos-flake.overlays.default ]; } + ./default.nix + ]; + }; + + # System closure for the imageless VM. The output directory + # exposes `kernel` and `initrd` symlinks consumers hand to + # QEMU via -kernel and -initrd; the closure itself is what + # virtiofsd serves to the guest as /nix/store. + # Run: nix build .#toplevel (or .#packages.<system>.toplevel) + packages.${system}.toplevel = self.nixosConfigurations.vm.config.system.build.toplevel; + }; +} diff --git a/templates/libvirt/default.nix b/templates/libvirt/default.nix new file mode 100644 index 00000000..a6ea619f --- /dev/null +++ b/templates/libvirt/default.nix @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Per-VM overrides for the libvirt template. +# +# The flake passes nixos-flake and the flake inputs via specialArgs +# so this module can pull in test-suite modules (LTP, fstests, and so +# on) and reference local source checkouts without having to edit +# flake.nix again. +{ + config, + lib, + pkgs, + nixos-flake, + inputs, + ... +}: +{ + imports = [ + # Development tools (editors, tracing, kernel dev comfort). Drop + # this import for a minimal VM that only runs a specific test suite. + nixos-flake.nixosModules.profiles.devel + + # Pick the test-suite modules that match what you intend to + # run, for example: + # nixos-flake.nixosModules.profiles.build-tools + # nixos-flake.nixosModules.testSuites.fstests + # nixos-flake.nixosModules.testSuites.blktests + ]; + + networking.hostName = "vm"; + + # Build a package from a local source checkout. The matching + # fio-src input must be declared in flake.nix. + # nixpkgs.overlays = [ + # (final: prev: { fio = prev.fio.overrideAttrs { src = inputs.fio-src; patches = []; }; }) + # ]; + + # SSH keys for the root and unprivileged accounts. + # users.users.root.openssh.authorizedKeys.keys = [ "ssh-ed25519 ..." ]; + # users.users.${config.nixos-flake.user.name}.openssh.authorizedKeys.keys = [ "ssh-ed25519 ..." ]; +} diff --git a/templates/libvirt/flake.nix b/templates/libvirt/flake.nix new file mode 100644 index 00000000..547dc71d --- /dev/null +++ b/templates/libvirt/flake.nix @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: copyleft-next-0.3.1 +# +# Libvirt disk-image NixOS VM starter. +# +# Create with: +# nix flake init --template "github:linux-kdevops/nixos-flake#libvirt" +# nix build .#nixosConfigurations.vm.config.system.build.toplevel +# +# Per-VM configuration lives in ./default.nix. This flake composes +# the base modules, applies the overlay, and passes nixos-flake and +# the flake inputs through specialArgs so default.nix can import +# additional modules (for example nixos-flake.nixosModules.testSuites.*) +# and reference local source inputs without re-declaring them. +{ + inputs = { + # Local checkout preferred: consumers pin to a specific + # revision via a subtree or vendored copy and should not + # track upstream HEAD. For upstream, use: + # nixos-flake.url = "github:linux-kdevops/nixos-flake"; + nixos-flake.url = "path:/path/to/nixos-flake"; + nixpkgs.follows = "nixos-flake/nixpkgs"; + + # Local source checkouts (uncomment to use). Multi-attribute + # non-flake inputs decompose into the type/path or type/url + # attribute-set form so each piece (type, source, ref, + # submodules) reads on its own line: + # + # fio-src = { + # type = "path"; + # path = "/home/user/src/fio"; + # flake = false; + # }; + # kmod-src = { + # type = "git"; + # url = "https://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git"; + # ref = "master"; + # flake = false; + # }; + }; + + outputs = + { + self, + nixpkgs, + nixos-flake, + ... + }@inputs: + let + system = "x86_64-linux"; + pkgs = import nixpkgs { inherit system; }; + in + { + nixosConfigurations.vm = nixpkgs.lib.nixosSystem { + inherit system; + specialArgs = { inherit inputs nixos-flake; }; + modules = [ + nixos-flake.nixosModules.backends.libvirt + nixos-flake.nixosModules.user + { nixpkgs.overlays = [ nixos-flake.overlays.default ]; } + ./default.nix + ]; + }; + + # qcow2 disk image built from the nixosConfiguration above. + # Run: nix build .#image (or .#packages.<system>.image) + # The output is the path to the qcow2 file; libvirt consumes it + # as <disk type='file' device='disk'>/<source file='...'/>. + packages.${system}.image = import "${nixpkgs}/nixos/lib/make-disk-image.nix" { + inherit pkgs; + inherit (pkgs) lib; + config = self.nixosConfigurations.vm.config; + diskSize = 20480; + format = "qcow2"; + partitionTableType = "legacy"; + installBootLoader = true; + }; + }; +} -- 2.54.0