git: 5e1fdbdf938b - stable/15 - amd64: FRED support

The branch stable/15 has been updated by kib:

URL: https://cgit.FreeBSD.org/src/commit/?id=5e1fdbdf938bf452912312ff61a31d75d9ce55d2

commit 5e1fdbdf938bf452912312ff61a31d75d9ce55d2
Author:     Konstantin Belousov <[email protected]>
AuthorDate: 2026-02-13 11:24:40 +0000
Commit:     Konstantin Belousov <[email protected]>
CommitDate: 2026-08-07 10:27:33 +0000

    amd64: FRED support
    
    (cherry picked from commit 6e93f5e4d6932c423b89dff8fc08d86f8bdeb7b9)
---
 sys/amd64/amd64/cpu_switch.S         |  73 ++++-
 sys/amd64/amd64/db_trace.c           |   5 +-
 sys/amd64/amd64/exception.S          | 282 ++++++++++++++++++
 sys/amd64/amd64/genassym.c           |  15 +
 sys/amd64/amd64/initcpu.c            |  43 +++
 sys/amd64/amd64/machdep.c            | 104 +++++--
 sys/amd64/amd64/mp_machdep.c         |  82 ++++--
 sys/amd64/amd64/trap.c               | 554 +++++++++++++++++++++++++++++++++++
 sys/amd64/amd64/vm_machdep.c         |  20 +-
 sys/amd64/include/md_var.h           |   2 +
 sys/amd64/include/pcb.h              |   8 +-
 sys/amd64/vmm/intel/vmx.c            |   2 +-
 sys/amd64/vmm/vmm.c                  |   2 +-
 sys/amd64/vmm/vmm_lapic.c            |   5 +
 sys/amd64/vmm/vmm_lapic.h            |   2 +
 sys/dev/hyperv/vmbus/x86/vmbus_x86.c |   3 +-
 sys/i386/i386/machdep.c              |   2 +
 sys/kern/kern_thread.c               |   2 +-
 sys/x86/include/apicvar.h            |   3 +-
 sys/x86/include/frame.h              |   1 -
 sys/x86/include/x86_var.h            |   3 +
 sys/x86/x86/local_apic.c             |  38 +--
 22 files changed, 1163 insertions(+), 88 deletions(-)

diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S
index 17ff8005e3cd..a9a8ad3da872 100644
--- a/sys/amd64/amd64/cpu_switch.S
+++ b/sys/amd64/amd64/cpu_switch.S
@@ -187,14 +187,22 @@ do_kthread:
 	cmpq	%rax,%rdx
 	jne	do_tss
 done_tss:
+	movq	%r8,PCPU(CURPCB)
+	cmpb	$0,fred
+	jne	1f
 	movq	TD_MD_STACK_BASE(%r12),%r9
 	movq	%r9,PCPU(RSP0)
-	movq	%r8,PCPU(CURPCB)
 	movq	PCPU(PTI_RSP0),%rax
 	cmpq	$~0,PCPU(UCR3)
 	cmove	%r9,%rax
 	movq	%rax,TSS_RSP0(%rdx)
-	movq	%r12,PCPU(CURTHREAD)		/* into next thread */
+	jmp	2f
+1:	movq	TD_MD_STACK_BASE(%r12),%rdx
+	movl	%edx,%eax
+	shrq	$32,%rdx
+	movl	$MSR_FRED_RSP0,%ecx
+	wrmsr
+2:	movq	%r12,PCPU(CURTHREAD)		/* into next thread */
 
 	/* Test if debug registers should be restored. */
 	testl	$PCB_DBREGS,PCB_FLAGS(%r8)
@@ -365,8 +373,36 @@ ENTRY(savectx)
 	movl	%eax,PCB_SFMASK(%rdi)
 	movl	%edx,PCB_SFMASK+4(%rdi)
 
-	sgdt	PCB_GDT(%rdi)
-	sidt	PCB_IDT(%rdi)
+	cmpb	$0, fred
+	je	1f
+	movl	$MSR_FRED_RSP0,%ecx
+	rdmsr
+	movl	%eax,PCB_FRED_RSP0(%rdi)
+	movl	%edx,PCB_FRED_RSP0+4(%rdi)
+	movl	$MSR_FRED_RSP1,%ecx
+	rdmsr
+	movl	%eax,PCB_FRED_RSP1(%rdi)
+	movl	%edx,PCB_FRED_RSP1+4(%rdi)
+	movl	$MSR_FRED_RSP2,%ecx
+	rdmsr
+	movl	%eax,PCB_FRED_RSP2(%rdi)
+	movl	%edx,PCB_FRED_RSP2+4(%rdi)
+	movl	$MSR_FRED_RSP3,%ecx
+	rdmsr
+	movl	%eax,PCB_FRED_RSP3(%rdi)
+	movl	%edx,PCB_FRED_RSP3+4(%rdi)
+	movl	$MSR_FRED_STKLVLS,%ecx
+	rdmsr
+	movl	%eax,PCB_FRED_STKLVLS(%rdi)
+	movl	%edx,PCB_FRED_STKLVLS+4(%rdi)
+	movl	$MSR_FRED_CONFIG,%ecx
+	rdmsr
+	movl	%eax,PCB_FRED_CONFIG(%rdi)
+	movl	%edx,PCB_FRED_CONFIG+4(%rdi)
+	jmp	2f
+
+1:	sidt	PCB_IDT(%rdi)
+2:	sgdt	PCB_GDT(%rdi)
 	sldt	PCB_LDT(%rdi)
 	str	PCB_TR(%rdi)
 
@@ -442,8 +478,35 @@ ENTRY(resumectx)
 	movq	%rax,%cr3
 
 	/* Restore descriptor tables. */
+	cmpb	$0,fred
+	jne	1f
 	lidt	PCB_IDT(%rdi)
-	lldt	PCB_LDT(%rdi)
+	jmp	2f
+1:	movl	$MSR_FRED_RSP0,%ecx
+	movl	PCB_FRED_RSP0(%rdi),%eax
+	movl	4 + PCB_FRED_RSP0(%rdi),%edx
+	wrmsr
+	movl	$MSR_FRED_RSP1,%ecx
+	movl	PCB_FRED_RSP1(%rdi),%eax
+	movl	4 + PCB_FRED_RSP1(%rdi),%edx
+	wrmsr
+	movl	$MSR_FRED_RSP2,%ecx
+	movl	PCB_FRED_RSP2(%rdi),%eax
+	movl	4 + PCB_FRED_RSP2(%rdi),%edx
+	wrmsr
+	movl	$MSR_FRED_RSP3,%ecx
+	movl	PCB_FRED_RSP3(%rdi),%eax
+	movl	4 + PCB_FRED_RSP3(%rdi),%edx
+	wrmsr
+	movl	$MSR_FRED_STKLVLS,%ecx
+	movl	PCB_FRED_STKLVLS(%rdi),%eax
+	movl	4 + PCB_FRED_STKLVLS(%rdi),%edx
+	wrmsr
+	movl	$MSR_FRED_CONFIG,%ecx
+	movl	PCB_FRED_CONFIG(%rdi),%eax
+	movl	4 + PCB_FRED_CONFIG(%rdi),%edx
+	wrmsr
+2:	lldt	PCB_LDT(%rdi)
 
 #define	SDT_SYSTSS	9
 #define	SDT_SYSBSY	11
diff --git a/sys/amd64/amd64/db_trace.c b/sys/amd64/amd64/db_trace.c
index 5f751906b7e6..e14cd55978de 100644
--- a/sys/amd64/amd64/db_trace.c
+++ b/sys/amd64/amd64/db_trace.c
@@ -167,7 +167,10 @@ db_nextframe(db_addr_t *fp, db_addr_t *ip, struct thread *td)
 		    strcmp(name, "fork_trampoline") == 0 ||
 		    strcmp(name, "mchk_calltrap") == 0 ||
 		    strcmp(name, "nmi_calltrap") == 0 ||
-		    strcmp(name, "Xdblfault") == 0)
+		    strcmp(name, "Xdblfault") == 0 ||
+		    strcmp(name, "fred_events_handlers") == 0 ||
+		    strcmp(name, "user_fred_syscall") == 0 ||
+		    strcmp(name, "user_fred_non_syscall") == 0)
 			frame_type = TRAP;
 		else if (strncmp(name, "Xatpic_intr", 11) == 0 ||
 		    strncmp(name, "Xapic_isr", 9) == 0 ||
diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S
index afdcdeedf0f5..8eb5c185974d 100644
--- a/sys/amd64/amd64/exception.S
+++ b/sys/amd64/amd64/exception.S
@@ -1063,6 +1063,8 @@ ENTRY(fork_trampoline)
 	movq	%rbx,%rsi		/* arg1 */
 	movq	%rsp,%rdx		/* trapframe pointer */
 	call	fork_exit
+	cmpb	$0, fred
+	jne	fred_checkast
 	jmp	doreti			/* Handle any ASTs */
 
 /*
@@ -1401,6 +1403,286 @@ gsbase_load_fault:
 	movq	$0,PCB_GSBASE(%r8)
 	jmp	doreti
 
+/*
+ * FRED guarantees that on entry, %rflags is set to '2', which in
+ * particular means that interrupts are disabled, and DF/AC are clear.
+ * Interrupts are re-enabled by the C handler, if appropriate.
+ */
+	ALIGN_TEXT
+user_fred_syscall:
+	movq	%rdx,TF_RDX(%rsp)
+	movq	%rcx,TF_RCX(%rsp)
+	SAVE_SEGS
+	movq	PCPU(CURPCB),%r11
+	andl	$~PCB_FULL_IRET,PCB_FLAGS(%r11)
+	sti
+	movw	$KUDSEL,TF_SS(%rsp)
+	movw	$KUCSEL,TF_CS(%rsp)
+	movq	%r8,TF_R8(%rsp)
+	movq	%r9,TF_R9(%rsp)
+	movq	%rbx,TF_RBX(%rsp)
+	movq	%rbp,TF_RBP(%rsp)
+	movq	%r12,TF_R12(%rsp)
+	movq	%r13,TF_R13(%rsp)
+	movq	%r14,TF_R14(%rsp)
+	movq	%r15,TF_R15(%rsp)
+	movl	$TF_HASSEGS,TF_FLAGS(%rsp)
+	andl	$TF_FRED_EVINFO2_INSTLENMASK,%eax
+	shrl	$TF_FRED_EVINFO2_INSTLENSHIFT,%eax
+	movq	%rax,TF_ERR(%rsp)
+	movq	PCPU(CURTHREAD),%rdi
+	movl	TF_RFLAGS(%rsp),%esi
+	andl	$PSL_T,%esi
+	call	amd64_syscall
+	movq	PCPU(CURPCB),%rax
+	cli
+	testl	$PCB_FULL_IRET,PCB_FLAGS(%rax)
+	jnz	fred_checkast
+	movq	PCPU(CURTHREAD),%rax
+	cmpl	$0,TD_AST(%rax)
+	jnz	fred_checkast
+	movq	TF_RDI(%rsp),%rdi
+	movq	TF_RSI(%rsp),%rsi
+	movq	TF_RDX(%rsp),%rdx
+	movq	TF_RAX(%rsp),%rax
+	xorl	%r8d,%r8d
+	xorl	%r9d,%r9d
+	xorl	%r10d,%r10d
+	xorl	%r11d,%r11d
+	addq	$TF_ERR,%rsp
+	.globl	fred_syscall_eretu
+fred_syscall_eretu:
+	eretu
+
+	ALIGN_TEXT
+user_fred_non_syscall:
+	movq	%rdx,TF_RDX(%rsp)
+	movq	%rcx,TF_RCX(%rsp)
+	movq	%r8,TF_R8(%rsp)
+	movq	%r9,TF_R9(%rsp)
+	movq	%rbx,TF_RBX(%rsp)
+	movq	%rbp,TF_RBP(%rsp)
+	movq	%r10,TF_R10(%rsp)
+	movq	%r11,TF_R11(%rsp)
+	movq	%r12,TF_R12(%rsp)
+	movq	%r13,TF_R13(%rsp)
+	movq	%r14,TF_R14(%rsp)
+	movq	%r15,TF_R15(%rsp)
+	SAVE_SEGS
+	movq	PCPU(CURTHREAD),%r9
+	movq	TD_PCB(%r9),%r8
+	rdfsbase %rax
+	movq	%rax,PCB_FSBASE(%r8)
+	rdgsbase %rax
+	movq	%rax,PCB_GSBASE(%r8)
+	movl	$TF_HASSEGS,TF_FLAGS(%rsp)
+	KMSAN_ENTER
+	movq	%rsp, %rdi
+	call	trap_fred_u
+	KMSAN_LEAVE
+fred_checkast:
+	cli
+	movq	PCPU(CURTHREAD),%rax
+	movq	TD_PCB(%rax),%r8
+	cmpl	$0,TD_AST(%rax)
+	je	fred_gouser
+	sti
+	movq	%rsp,%rdi	/* pass a pointer to the trapframe */
+	call	ast
+	jmp	fred_checkast
+fred_gouser:
+	testl	$PCB_FULL_IRET,PCB_FLAGS(%r8)
+	jz	fred_u_regs
+	andl	$~PCB_FULL_IRET,PCB_FLAGS(%r8)
+	testl	$TF_HASSEGS,TF_FLAGS(%rsp)
+	jne	fred_do_segs
+	movw	$KUDSEL,%ax
+	movw	%ax,TF_DS(%rsp)
+	movw	%ax,TF_ES(%rsp)
+	movw	$KUF32SEL,TF_FS(%rsp)
+	movw	$KUG32SEL,TF_GS(%rsp)
+fred_do_segs:
+	/* Restore %fs and fsbase */
+	movw	TF_FS(%rsp),%ax
+	.globl	fred_ld_fs
+fred_ld_fs:
+	movw	%ax,%fs
+	testl	$PCB_32BIT,PCB_FLAGS(%r8)
+	jnz	after_fred_fs
+	movq	PCB_FSBASE(%r8),%rax
+	.globl	fred_ld_fsbase
+fred_ld_fsbase:
+	wrfsbase %rax
+after_fred_fs:
+	/* Restore %gs and kgsbase */
+	movw	TF_GS(%rsp),%ax
+	.globl	fred_lkgs
+fred_lkgs:
+	lkgs	%ax
+	testl	$PCB_32BIT,PCB_FLAGS(%r8)
+	jnz	after_fred_gs
+	movl	PCB_GSBASE(%r8),%eax
+	movl	PCB_GSBASE+4(%r8),%edx
+	movl	$MSR_KGSBASE,%ecx
+	.globl	fred_ld_kgsbase
+fred_ld_kgsbase:
+	wrmsr
+after_fred_gs:
+	.globl	fred_ld_es
+fred_ld_es:
+	movw	TF_ES(%rsp),%es
+	.globl	fred_ld_ds
+fred_ld_ds:
+	movw	TF_DS(%rsp),%ds
+fred_u_regs:
+	RESTORE_REGS
+	addq	$TF_ERR,%rsp
+	.globl	fred_eretu
+fred_eretu:
+	eretu
+
+	.globl	fred_ld_fs_fault
+fred_ld_fs_fault:
+	movl	$(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+	movq	%rsp,%rdi
+	call	trap_fred_u
+	movw	$KUF32SEL,TF_FS(%rsp)
+	jmp	fred_checkast
+
+	.globl	fred_ld_fsbase_fault
+fred_ld_fsbase_fault:
+	movl	$(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+	movq	%rsp,%rdi
+	call	trap_fred_u
+	movq	PCPU(CURTHREAD),%r9
+	movq	TD_PCB(%r9),%r8
+	movq	$0,PCB_FSBASE(%r8)
+	jmp	fred_checkast
+
+	.globl	fred_lkgs_fault
+fred_lkgs_fault:
+	movl	$(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+	movq	%rsp,%rdi
+	call	trap_fred_u
+	movw	$KUG32SEL,TF_GS(%rsp)
+	jmp	fred_checkast
+
+	.globl	fred_ld_kgsbase_fault
+fred_ld_kgsbase_fault:
+	movl	$(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+	movq	%rsp,%rdi
+	call	trap_fred_u
+	movq	PCPU(CURTHREAD),%r9
+	movq	TD_PCB(%r9),%r8
+	movq	$0,PCB_GSBASE(%r8)
+	jmp	fred_checkast
+
+	.globl	fred_ld_es_fault
+fred_ld_es_fault:
+	movl	$(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+	movq	%rsp,%rdi
+	call	trap_fred_u
+	movw	$KUDSEL,TF_ES(%rsp)
+	jmp	fred_checkast
+
+	.globl	fred_ld_ds_fault
+fred_ld_ds_fault:
+	movl	$(TF_FRED_EVINFO2_TYPE_EXC | IDT_NP), TF_FRED_EVINFO2(%rsp)
+	movq	%rsp,%rdi
+	call	trap_fred_u
+	movw	$KUDSEL,TF_DS(%rsp)
+	jmp	fred_checkast
+
+	.globl	fred_eretu_fault
+fred_eretu_fault:
+	subq	$TF_ERR,%rsp
+	movq	%rdi,TF_RDI(%rsp)
+	movq	%rdx,TF_RDX(%rsp)
+	movq	%rcx,TF_RCX(%rsp)
+	movq	%rsi,TF_RSI(%rsp)
+	movq	%r8,TF_R8(%rsp)
+	movq	%r9,TF_R9(%rsp)
+	movq	%rbx,TF_RBX(%rsp)
+	movq	%rbp,TF_RBP(%rsp)
+	movq	%r10,TF_R10(%rsp)
+	movq	%r11,TF_R11(%rsp)
+	movq	%r12,TF_R12(%rsp)
+	movq	%r13,TF_R13(%rsp)
+	movq	%r14,TF_R14(%rsp)
+	movq	%r15,TF_R15(%rsp)
+	SAVE_SEGS
+	movl	$(TF_FRED_EVINFO2_TYPE_EXC | IDT_GP), TF_FRED_EVINFO2(%rsp)
+	movq	%rsp, %rdi
+	KMSAN_ENTER
+	call	trap_fred_u
+	KMSAN_LEAVE
+	jmp	fred_checkast
+
+/*
+ * Content for the FRED entries page.
+*/
+
+	.p2align 12,0x90
+	.globl	fred_events_handlers, fred_events_handlers_end
+fred_events_handlers:
+	subq	$TF_ERR,%rsp
+	movq	%rax,TF_RAX(%rsp)
+	movq	%rdi,TF_RDI(%rsp)
+	movq	%rsi,TF_RSI(%rsp)
+	movl	TF_FRED_EVINFO2(%rsp),%edi
+	movl	%edi,%eax
+	andl	$TF_FRED_EVINFO2_TYPEMASK,%edi
+	cmpl	$TF_FRED_EVINFO2_TYPE_SYSCALL,%edi
+	je	user_fred_syscall
+	jmp	user_fred_non_syscall
+
+	. = fred_events_handlers + 256
+	subq	$TF_ERR,%rsp
+	movq	%rax,TF_RAX(%rsp)
+	movq	%rdi,TF_RDI(%rsp)
+	movq	%rdx,TF_RDX(%rsp)
+	movq	%rcx,TF_RCX(%rsp)
+	movq	%rsi,TF_RSI(%rsp)
+	movq	%r8,TF_R8(%rsp)
+	movq	%r9,TF_R9(%rsp)
+	movq	%rbx,TF_RBX(%rsp)
+	movq	%rbp,TF_RBP(%rsp)
+	movq	%r10,TF_R10(%rsp)
+	movq	%r11,TF_R11(%rsp)
+	movq	%r12,TF_R12(%rsp)
+	movq	%r13,TF_R13(%rsp)
+	movq	%r14,TF_R14(%rsp)
+	movq	%r15,TF_R15(%rsp)
+	movq	%rsp, %rdi
+	KMSAN_ENTER
+	call	trap_fred_k
+	KMSAN_LEAVE
+	RESTORE_REGS
+	addq	$TF_ERR,%rsp
+	erets
+fred_events_handlers_end:
+
+	.globl	fred_out_of_nmi
+fred_out_of_nmi:
+	pushq	%rbp
+	movq	%rsp,%rbp
+	xorl	%eax,%eax
+	pushq	$0
+	pushq	$0
+	movw	%ss,%ax
+	orl	$(1<<18),%eax	/* NMI block release for ERETS */
+	pushq	%rax
+	pushq	%rbp
+	pushfq
+	xorl	%eax,%eax
+	movw	%cs,%ax
+	pushq	%rax
+	pushq	$1f
+	pushq	$0
+	erets
+1:	popq	%rbp
+	retq
+
 #ifdef HWPMC_HOOKS
 	ENTRY(end_exceptions)
 #endif
diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
index b4cd7d64ddea..96663a9a2b55 100644
--- a/sys/amd64/amd64/genassym.c
+++ b/sys/amd64/amd64/genassym.c
@@ -144,6 +144,12 @@ ASSYM(PCB_STAR, offsetof(struct pcb, pcb_star));
 ASSYM(PCB_LSTAR, offsetof(struct pcb, pcb_lstar));
 ASSYM(PCB_CSTAR, offsetof(struct pcb, pcb_cstar));
 ASSYM(PCB_SFMASK, offsetof(struct pcb, pcb_sfmask));
+ASSYM(PCB_FRED_RSP0, offsetof(struct pcb, pcb_fred_rsp0));
+ASSYM(PCB_FRED_RSP1, offsetof(struct pcb, pcb_fred_rsp1));
+ASSYM(PCB_FRED_RSP2, offsetof(struct pcb, pcb_fred_rsp2));
+ASSYM(PCB_FRED_RSP3, offsetof(struct pcb, pcb_fred_rsp3));
+ASSYM(PCB_FRED_STKLVLS, offsetof(struct pcb, pcb_fred_stklvls));
+ASSYM(PCB_FRED_CONFIG, offsetof(struct pcb, pcb_fred_config));
 ASSYM(PCB_SIZE, sizeof(struct pcb));
 ASSYM(PCB_FULL_IRET, PCB_FULL_IRET);
 ASSYM(PCB_DBREGS, PCB_DBREGS);
@@ -178,9 +184,15 @@ ASSYM(TF_DS, offsetof(struct trapframe, tf_ds));
 ASSYM(TF_ES, offsetof(struct trapframe, tf_es));
 ASSYM(TF_FS, offsetof(struct trapframe, tf_fs));
 ASSYM(TF_GS, offsetof(struct trapframe, tf_gs));
+ASSYM(TF_FRED_EVINFO2, offsetof(struct trapframe, tf_fred_evinfo2));
 ASSYM(TF_FLAGS, offsetof(struct trapframe, tf_flags));
 ASSYM(TF_SIZE, sizeof(struct trapframe));
 ASSYM(TF_HASSEGS, TF_HASSEGS);
+ASSYM(TF_FRED_EVINFO2_TYPE_EXC, TF_FRED_EVINFO2_TYPE_EXC);
+ASSYM(TF_FRED_EVINFO2_TYPE_SYSCALL, TF_FRED_EVINFO2_TYPE_SYSCALL);
+ASSYM(TF_FRED_EVINFO2_TYPEMASK, TF_FRED_EVINFO2_TYPEMASK);
+ASSYM(TF_FRED_EVINFO2_INSTLENMASK, TF_FRED_EVINFO2_INSTLENMASK);
+ASSYM(TF_FRED_EVINFO2_INSTLENSHIFT, TF_FRED_EVINFO2_INSTLENSHIFT);
 
 ASSYM(PTI_RDX, offsetof(struct pti_frame, pti_rdx));
 ASSYM(PTI_RAX, offsetof(struct pti_frame, pti_rax));
@@ -270,6 +282,9 @@ ASSYM(TSSSEL, GSEL(GPROC0_SEL, SEL_KPL));
 ASSYM(LDTSEL, GSEL(GUSERLDT_SEL, SEL_KPL));
 ASSYM(SEL_RPL_MASK, SEL_RPL_MASK);
 
+ASSYM(IDT_NP, IDT_NP);
+ASSYM(IDT_GP, IDT_GP);
+
 ASSYM(__FreeBSD_version, __FreeBSD_version);
 
 #ifdef	HWPMC_HOOKS
diff --git a/sys/amd64/amd64/initcpu.c b/sys/amd64/amd64/initcpu.c
index 7161cf631508..5b615be4b8dc 100644
--- a/sys/amd64/amd64/initcpu.c
+++ b/sys/amd64/amd64/initcpu.c
@@ -397,3 +397,46 @@ initializecpucache(void)
 		cpu_stdext_feature &= ~CPUID_STDEXT_CLFLUSHOPT;
 	}
 }
+
+extern const char fred_events_handlers[], fred_events_handlers_end[];
+
+void
+amd64_cpu_init_fred(void)
+{
+	MPASS((((uintptr_t)fred_events_handlers) & PAGE_MASK) == 0);
+	/*
+	 * 128b = 0x2 * 64b redzone
+	 * SLC 0 for EXTINT
+	 * SLC 0 current
+	 */
+	wrmsr(MSR_FRED_CONFIG, (uintptr_t)fred_events_handlers |
+	    (0x2 << IA32_FRED_CONFIG_REDZONESZ_SHIFT) |
+	    (0 << IA32_FRED_CONFIG_EXTINT_SLC_SHIFT) | 0);
+
+	/*
+	 * With FRED, there is no need for workarounds requiring
+	 * dedicated stack for #DB.  Also, there is no need to have
+	 * separate stacks for #NM and #MC, because CSL rules ensure
+	 * that interrupted frame is safe even if NMI was interrupted
+	 * by machine check or vice versa.
+	 *
+	 * Still, we need to provide a stack for events that can occur
+	 * with the interrupts disabled, to avoid caring about the
+	 * ordering of the context update in cpu_switch.S.  Use CSL 1
+	 * for #NM and #MC.
+	 *
+	 * Since double fault handler is used to handle stack
+	 * overflow, it gets the dedicated stack, of the highest used
+	 * CSL 2.
+	 *
+	 * CSL 3 is unused, and we have some space to re-arrange this
+	 * if a need in the future appears.
+	 */
+	wrmsr(MSR_FRED_STKLVLS, (1ul << (IDT_NM * 2)) |
+	    (1ul << (IDT_MC * 2)) | (2ul << (IDT_DF * 2)));
+	wrmsr(MSR_FRED_RSP3, 0);
+	load_cr4(rcr4() | CR4_FRED);
+	if (IS_BSP() && bootverbose)
+		printf("FRED enabled\n");
+
+}
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index 0c12007e1f74..266ca0a6b7a3 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -207,10 +207,14 @@ long realmem = 0;
 int late_console = 1;
 int lass_enabled = 0;
 
+int __read_frequently fred = 0;
+SYSCTL_INT(_hw, OID_AUTO, fred, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
+    &fred, 0,
+    "FRED is used");
+
 struct kva_md_info kmi;
 
 struct region_descriptor r_idt;
-
 struct pcpu *__pcpu;
 struct pcpu temp_bsp_pcpu;
 
@@ -348,9 +352,9 @@ cpu_setregs(void)
 static struct gate_descriptor idt0[NIDT];
 struct gate_descriptor *idt = &idt0[0];	/* interrupt descriptor table */
 
-static char dblfault_stack[DBLFAULT_STACK_SIZE] __aligned(16);
+static char dblfault_stack[DBLFAULT_STACK_SIZE] __aligned(64);
 static char mce0_stack[MCE_STACK_SIZE] __aligned(16);
-static char nmi0_stack[NMI_STACK_SIZE] __aligned(16);
+static char nmi0_stack[NMI_STACK_SIZE] __aligned(64);
 static char dbg0_stack[DBG_STACK_SIZE] __aligned(16);
 CTASSERT(sizeof(struct nmi_pcpu) == 16);
 
@@ -486,6 +490,9 @@ setidt(int idx, inthand_t *func, int typ, int dpl, int ist)
 {
 	struct gate_descriptor *ip;
 
+	if (fred)
+		return;
+
 	ip = idt + idx;
 	ip->gd_looffset = (uintptr_t)func;
 	ip->gd_selector = GSEL(GCODE_SEL, SEL_KPL);
@@ -1187,9 +1194,11 @@ amd64_conf_fast_syscall(void)
 
 	msr = rdmsr(MSR_EFER) | EFER_SCE;
 	wrmsr(MSR_EFER, msr);
-	wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) :
-	    (u_int64_t)IDTVEC(fast_syscall));
-	wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
+	if (!fred) {
+		wrmsr(MSR_LSTAR, pti ? (u_int64_t)IDTVEC(fast_syscall_pti) :
+		    (u_int64_t)IDTVEC(fast_syscall));
+		wrmsr(MSR_CSTAR, (u_int64_t)IDTVEC(fast_syscall32));
+	}
 	msr = ((u_int64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
 	    ((u_int64_t)GSEL(GUCODE32_SEL, SEL_UPL) << 48);
 	wrmsr(MSR_STAR, msr);
@@ -1231,33 +1240,49 @@ amd64_bsp_ist_init(struct pcpu *pc)
 
 	tssp = &pc->pc_common_tss;
 
-	/* doublefault stack space, runs on ist1 */
-	np = ((struct nmi_pcpu *)&dblfault_stack[sizeof(dblfault_stack)]) - 1;
-	np->np_pcpu = (register_t)pc;
-	tssp->tss_ist1 = (long)np;
+	/* Doublefault stack space, runs on ist1 for IDT. */
+	if (fred) {
+		wrmsr(MSR_FRED_RSP2, (uint64_t)&dblfault_stack[
+		    sizeof(dblfault_stack)]);
+	} else {
+		np = ((struct nmi_pcpu *)&dblfault_stack[sizeof(
+		    dblfault_stack)]) - 1;
+		np->np_pcpu = (register_t)pc;
+		tssp->tss_ist1 = (long)np;
+	}
 
 	/*
-	 * NMI stack, runs on ist2.  The pcpu pointer is stored just
-	 * above the start of the ist2 stack.
+	 * NMI stack.
 	 */
-	np = ((struct nmi_pcpu *)&nmi0_stack[sizeof(nmi0_stack)]) - 1;
-	np->np_pcpu = (register_t)pc;
-	tssp->tss_ist2 = (long)np;
+	if (fred) {
+		wrmsr(MSR_FRED_RSP1, (uint64_t)&nmi0_stack[
+		    sizeof(nmi0_stack)]);
+	} else {
+		/*
+		 * Runs on ist2 for IDT.  The pcpu pointer is stored
+		 * just above the start of the ist2 stack.
+		 */
+		np = ((struct nmi_pcpu *)&nmi0_stack[sizeof(nmi0_stack)]) - 1;
+		np->np_pcpu = (register_t)pc;
+		tssp->tss_ist2 = (long)np;
+	}
 
-	/*
-	 * MC# stack, runs on ist3.  The pcpu pointer is stored just
-	 * above the start of the ist3 stack.
-	 */
-	np = ((struct nmi_pcpu *)&mce0_stack[sizeof(mce0_stack)]) - 1;
-	np->np_pcpu = (register_t)pc;
-	tssp->tss_ist3 = (long)np;
+	if (!fred) {
+		/*
+		 * MC# stack for IDT, runs on ist3.  The pcpu pointer
+		 * is stored just above the start of the ist3 stack.
+		 */
+		np = ((struct nmi_pcpu *)&mce0_stack[sizeof(mce0_stack)]) - 1;
+		np->np_pcpu = (register_t)pc;
+		tssp->tss_ist3 = (long)np;
 
-	/*
-	 * DB# stack, runs on ist4.
-	 */
-	np = ((struct nmi_pcpu *)&dbg0_stack[sizeof(dbg0_stack)]) - 1;
-	np->np_pcpu = (register_t)pc;
-	tssp->tss_ist4 = (long)np;
+		/*
+		 * DB# stack for IDT, runs on ist4.
+		 */
+		np = ((struct nmi_pcpu *)&dbg0_stack[sizeof(dbg0_stack)]) - 1;
+		np->np_pcpu = (register_t)pc;
+		tssp->tss_ist4 = (long)np;
+	}
 }
 
 /*
@@ -1360,6 +1385,13 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 		TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave);
 	}
 
+	if ((cpu_stdext_feature4 & (CPUID_STDEXT4_FRED | CPUID_STDEXT4_LKGS)) ==
+	    (CPUID_STDEXT4_FRED | CPUID_STDEXT4_LKGS) &&
+	    (cpu_stdext_feature & CPUID_STDEXT_FSGSBASE) != 0 && !pti) {
+		fred = 1;
+		TUNABLE_INT_FETCH("hw.fred", &fred);
+	}
+
 	sched_instance_select();
 
 	link_elf_ireloc();
@@ -1473,9 +1505,11 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	setidt(IDT_EVTCHN, pti ? &IDTVEC(xen_intr_upcall_pti) :
 	    &IDTVEC(xen_intr_upcall), SDT_SYSIGT, SEL_KPL, 0);
 #endif
-	r_idt.rd_limit = sizeof(idt0) - 1;
-	r_idt.rd_base = (long) idt;
-	lidt(&r_idt);
+	if (!fred) {
+		r_idt.rd_limit = sizeof(idt0) - 1;
+		r_idt.rd_base = (long) idt;
+		lidt(&r_idt);
+	}
 
 	TUNABLE_INT_FETCH("hw.ibrs_disable", &hw_ibrs_disable);
 	TUNABLE_INT_FETCH("machdep.mitigations.ibrs.disable", &hw_ibrs_disable);
@@ -1565,6 +1599,9 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
 	if (getenv_is_true("debug.dump_modinfo_at_boot"))
 		preload_dump();
 
+	if (fred)
+		amd64_cpu_init_fred();
+
 #ifdef DEV_ISA
 #ifdef DEV_ATPIC
 	elcr_probe();
@@ -1829,6 +1866,11 @@ clear_pcb_flags(struct pcb *pcb, const u_int flags)
 
 extern const char wrmsr_early_safe_gp_handler[];
 
+/*
+ * What about FRED?  wrmsr_early_safe_start() is used before we
+ * switched CPU to the FRED mode.  We use IDT to catch #GP from MSR
+ * write even if BSP is switched to the FRED mode later.
+ */
 void
 wrmsr_early_safe_start(void)
 {
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 1de6fe9227c7..96e91b59834e 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -227,25 +227,51 @@ init_secondary(void)
 	    IOPERM_BITMAP_SIZE;
 	pc->pc_common_tss.tss_rsp0 = 0;
 
-	/* The doublefault stack runs on IST1. */
-	np = ((struct nmi_pcpu *)&doublefault_stack[DBLFAULT_STACK_SIZE]) - 1;
-	np->np_pcpu = (register_t)pc;
-	pc->pc_common_tss.tss_ist1 = (long)np;
-
-	/* The NMI stack runs on IST2. */
-	np = ((struct nmi_pcpu *)&nmi_stack[NMI_STACK_SIZE]) - 1;
-	np->np_pcpu = (register_t)pc;
-	pc->pc_common_tss.tss_ist2 = (long)np;
-
-	/* The MC# stack runs on IST3. */
-	np = ((struct nmi_pcpu *)&mce_stack[MCE_STACK_SIZE]) - 1;
-	np->np_pcpu = (register_t)pc;
-	pc->pc_common_tss.tss_ist3 = (long)np;
-
-	/* The DB# stack runs on IST4. */
-	np = ((struct nmi_pcpu *)&dbg_stack[DBG_STACK_SIZE]) - 1;
-	np->np_pcpu = (register_t)pc;
-	pc->pc_common_tss.tss_ist4 = (long)np;
+	/*
+	 * The doublefault stack.
+	 * Runs on IST1 for IDT.
+	 * Uses CSL 2 for FRED.
+	 */
+	if (fred) {
+		wrmsr(MSR_FRED_RSP2, (uint64_t)&doublefault_stack[
+		    DBLFAULT_STACK_SIZE]);
+	} else {
+		np = ((struct nmi_pcpu *)&doublefault_stack[
+		    DBLFAULT_STACK_SIZE]) - 1;
+		np->np_pcpu = (register_t)pc;
+		pc->pc_common_tss.tss_ist1 = (long)np;
+	}
+
+	/*
+	 * The NMI stack.
+	 * Runs on IST2 for IDT.
+	 * Uses CSL 1 for FRED.
+	 */
+	if (fred) {
+		wrmsr(MSR_FRED_RSP1, (uint64_t)&nmi_stack[NMI_STACK_SIZE]);
+	} else {
+		np = ((struct nmi_pcpu *)&nmi_stack[NMI_STACK_SIZE]) - 1;
+		np->np_pcpu = (register_t)pc;
+		pc->pc_common_tss.tss_ist2 = (long)np;
+	}
+
+	/*
+	 * The MC# stack.
+	 * Runs on IST3 for IDT.
+	 * Shares CSL 1 with NMI for FRED.
+	 */
+	if (!fred) {
+		np = ((struct nmi_pcpu *)&mce_stack[MCE_STACK_SIZE]) - 1;
+		np->np_pcpu = (register_t)pc;
+		pc->pc_common_tss.tss_ist3 = (long)np;
+	}
+
+	if (!fred) {
+		/* The DB# stack, used for for IDT, runs on IST4. */
+		np = ((struct nmi_pcpu *)&dbg_stack[DBG_STACK_SIZE]) - 1;
+		np->np_pcpu = (register_t)pc;
+		pc->pc_common_tss.tss_ist4 = (long)np;
+	}
 
 	/* Prepare private GDT */
 	gdt_segs[GPROC0_SEL].ssd_base = (long)&pc->pc_common_tss;
@@ -265,7 +291,10 @@ init_secondary(void)
 	wrmsr(MSR_KGSBASE, 0);		/* User value */
 	fix_cpuid();
 
-	lidt(&r_idt);
+	if (fred)
+		amd64_cpu_init_fred();
+	else
+		lidt(&r_idt);
 
 	gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
 	ltr(gsel_tss);
@@ -417,12 +446,17 @@ start_all_aps(void)
 		    M_WAITOK | M_ZERO);
 		doublefault_stack = kmem_malloc(DBLFAULT_STACK_SIZE,
 		    M_WAITOK | M_ZERO);
-		mce_stack = kmem_malloc(MCE_STACK_SIZE,
-		    M_WAITOK | M_ZERO);
+		if (!fred) {
+			mce_stack = kmem_malloc(MCE_STACK_SIZE,
+			    M_WAITOK | M_ZERO);
+		}
 		nmi_stack = kmem_malloc_domainset(
 		    DOMAINSET_PREF(domain), NMI_STACK_SIZE, M_WAITOK | M_ZERO);
-		dbg_stack = kmem_malloc_domainset(
-		    DOMAINSET_PREF(domain), DBG_STACK_SIZE, M_WAITOK | M_ZERO);
+		if (!fred) {
+			dbg_stack = kmem_malloc_domainset(
+			    DOMAINSET_PREF(domain), DBG_STACK_SIZE,
+			    M_WAITOK | M_ZERO);
+		}
 		dpcpu = kmem_malloc_domainset(DOMAINSET_PREF(domain),
 		    DPCPU_SIZE, M_WAITOK | M_ZERO);
 
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
index df9cc44bcbb2..38b7317182a8 100644
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -41,6 +41,7 @@
  * AMD64 Trap and System call handling
  */
 
+#include "opt_atpic.h"
 #include "opt_clock.h"
 #include "opt_cpu.h"
 #include "opt_hwpmc_hooks.h"
@@ -61,6 +62,7 @@
 #include <sys/mutex.h>
 #include <sys/resourcevar.h>
 #include <sys/signalvar.h>
+#include <sys/smp.h>
 #include <sys/syscall.h>
 #include <sys/sysctl.h>
 #include <sys/sysent.h>
@@ -83,6 +85,8 @@ PMC_SOFT_DEFINE( , , page_fault, write);
 
 #include <machine/cpu.h>
 #include <machine/intr_machdep.h>
+#include <x86/apicreg.h>
+#include <x86/apicvar.h>
 #include <x86/mca.h>
 #include <machine/md_var.h>
 #include <machine/pcb.h>
@@ -90,6 +94,7 @@ PMC_SOFT_DEFINE( , , page_fault, write);
 #include <machine/stack.h>
 #include <machine/trap.h>
 #include <machine/tss.h>
+#include <x86/isa/icu.h>
 
 #ifdef KDTRACE_HOOKS
 #include <sys/dtrace_bsd.h>
@@ -995,6 +1000,14 @@ trap_diag(struct trapframe *frame, vm_offset_t eva, const char *type_str)
 	    frame->tf_r11, frame->tf_r12);
 	printf("r13: %016lx r14: %016lx r15: %016lx\n", frame->tf_r13,
 	    frame->tf_r14, frame->tf_r15);
+	if (fred) {
+		struct trapframe_fred *f;
+
+		f = __containerof(frame, struct trapframe_fred, tf_idt);
+		printf("evdata %016lx evinfo1 %04x evinfo2 %08x\n",
+		    f->tf_fred_evdata, frame->tf_fred_evinfo1,
+		    frame->tf_fred_evinfo2);
+	}
 }
 
 static void
@@ -1069,6 +1082,14 @@ dblfault_handler(struct trapframe *frame)
 	    frame->tf_cs, frame->tf_ss, frame->tf_ds, frame->tf_es,
 	    frame->tf_fs, frame->tf_gs,
 	    rdmsr(MSR_FSBASE), rdmsr(MSR_GSBASE), rdmsr(MSR_KGSBASE));
+	if (fred) {
+		struct trapframe_fred *f;
+
+		f = __containerof(frame, struct trapframe_fred, tf_idt);
+		printf("evdata %016lx evinfo1 %04x evinfo2 %08x\n",
+		    f->tf_fred_evdata, frame->tf_fred_evinfo1,
+		    frame->tf_fred_evinfo2);
+	}
 	/* Print these separately in case pcpu accesses trap. */
 	printf("cpuid = %d; apic id = %02x\n", PCPU_GET(cpuid),
 	    PCPU_GET(apic_id));
@@ -1295,3 +1316,536 @@ amd64_syscall(struct thread *td, int traced)
 
 	amd64_syscall_ret_flush_l1d_check_inline(td->td_errno);
 }
+
+static void
+trap_fred_extint(struct trapframe_fred *frame, u_int vec)
+{
+	switch (vec) {
+	case IPI_RENDEZVOUS:
+#ifdef COUNT_IPIS
+		ipi_rendezvous_counts[PCPU_GET(cpuid)]++;
+#endif
+		smp_rendezvous_action();
+		lapic_eoi();
+		break;
+	case IPI_INVLOP:
+		invlop_handler();
+		lapic_eoi();
+		break;
+	case IPI_BITMAP_VECTOR:
+		lapic_eoi();
+		ipi_bitmap_handler(&frame->tf_idt);
+		break;
+	case IPI_STOP:
+		lapic_eoi();
+		cpustop_handler();
+		break;
+	case IPI_SUSPEND:
+		cpususpend_handler();
+		lapic_eoi();
+		break;
+	case IPI_SWI:
+		lapic_eoi();
+		ipi_swi_handler(&frame->tf_idt);
+		break;
+	case APIC_SPURIOUS_INT:
+		break;
+	case APIC_TIMER_INT:
+		lapic_handle_timer(&frame->tf_idt);
+		break;
+	case APIC_CMC_INT:
+		lapic_handle_cmc();
+		break;
+	case APIC_ERROR_INT:
+		lapic_handle_error();
+		break;
+	case APIC_THERMAL_INT:
+		lapic_handle_thermal();
+		break;
+#ifdef XENHVM
+	case IDT_EVTCHN:
+		xen_arch_intr_handle_upcall(&frame->tf_idt);
+		break;
+#endif
+	default:
+		if (vec >= IPI_DYN_FIRST && vec <= IPI_DYN_LAST) {
+			void (*fi)(struct trapframe *);
+
+			fi = fred_ipi_handlers[vec - IPI_DYN_FIRST];
+			if (fi != NULL) {
+				fi(&frame->tf_idt);
+				lapic_eoi();
+			} else {
+				panic("DYN IPI %d without FRED handler", vec);
+			}
+			break;
+		}
+#ifdef DEV_ISA
+#ifdef DEV_ATPIC
+		if (vec >= IDT_IO_INTS && vec < IDT_IO_INTS + NUM_ISA_IRQS) {
+			atpic_handle_intr(vec - IDT_IO_INTS, &frame->tf_idt);
+			break;
+		}
+#endif
+#endif
+		/* apic ioint */
+		lapic_handle_intr(vec, &frame->tf_idt);
*** 749 LINES SKIPPED ***