[PATCH 2/2] bootlinux: handle 9P remounting when tree configuration changes

From: Daniel Gomez <[email protected]>

When kernel tree configuration changes to a different mount point, the
9P filesystem cannot be reconfigured with simple remounting because the
QEMU virtio-9p device retains the original command line fsdev path. This
causes guests to see stale cached data from the previous tree location.

The solution requires a full VM power cycle with libvirt domain undefine
and redefine. The undefine step removes the cached VM definition, and
redefine forces libvirt to regenerate the QEMU command line from the
updated XML file containing the new 9P host path.

Detect when the 9P mount tag is mounted at a different path than the
current target_linux_dir_path. When detected, remove the old fstab entry
to prevent boot failures, then execute the power cycle. After restart,
mount at the new location and update fstab.

The power cycle implementation in
playbooks/roles/bootlinux/tasks/power-cycle-vm.yml uses graceful
shutdown via community.libvirt.virt module, waiting up to 120 seconds
for the VM to reach shut off state. A 3-second pause after port closure
allows libvirt domain state to fully transition before verification.
If graceful shutdown fails, pause the playbook and prompt the user
to manually shut down the VM, providing options to investigate via
console, retry shutdown, or force destroy. Use wait_for_connection
after VM restart to properly test SSH connectivity rather than just
port availability.

This enables seamless switching between kernel trees like
BOOTLINUX_TREE_LINUS and BOOTLINUX_CUSTOM without manual VM recreation.

Generated-by: Claude AI
Signed-off-by: Daniel Gomez <[email protected]>
---
 playbooks/roles/bootlinux/tasks/main.yml           |  38 ++++++
 playbooks/roles/bootlinux/tasks/power-cycle-vm.yml | 150 +++++++++++++++++++++
 2 files changed, 188 insertions(+)

diff --git a/playbooks/roles/bootlinux/tasks/main.yml b/playbooks/roles/bootlinux/tasks/main.yml
index da42612e..98b3eb33 100644
--- a/playbooks/roles/bootlinux/tasks/main.yml
+++ b/playbooks/roles/bootlinux/tasks/main.yml
@@ -195,6 +195,44 @@
   when:
     - not workflow_linux_packaged|bool
 
+- name: Check if 9p mount tag is already mounted elsewhere
+  become: true
+  become_flags: "su - -c"
+  become_method: sudo
+  ansible.builtin.shell: "mount | grep '{{ bootlinux_9p_mount_tag }}' | awk '{print $3}'"
+  register: existing_9p_mount
+  changed_when: false
+  failed_when: false
+  tags: ["data_partition", "9p_mount"]
+  when:
+    - bootlinux_9p|bool
+
+- name: Remove old 9P mount from fstab before power cycle
+  become: true
+  become_flags: "su - -c"
+  become_method: sudo
+  ansible.posix.mount:
+    name: "{{ existing_9p_mount.stdout }}"
+    src: "{{ bootlinux_9p_mount_tag }}"
+    fstype: "9p"
+    state: "absent"
+  when:
+    - bootlinux_9p|bool
+    - existing_9p_mount.stdout is defined
+    - existing_9p_mount.stdout != ""
+    - existing_9p_mount.stdout != target_linux_dir_path
+  tags: ["data_partition", "9p_mount"]
+
+- name: Power cycle VM if 9p mounted at different location (clears cache)
+  ansible.builtin.include_tasks:
+    file: "{{ role_path }}/tasks/power-cycle-vm.yml"
+  when:
+    - bootlinux_9p|bool
+    - existing_9p_mount.stdout is defined
+    - existing_9p_mount.stdout != ""
+    - existing_9p_mount.stdout != target_linux_dir_path
+  tags: ["data_partition", "9p_mount"]
+
 - name: Mount bootlinux 9p on each target node
   become: true
   become_flags: "su - -c"
diff --git a/playbooks/roles/bootlinux/tasks/power-cycle-vm.yml b/playbooks/roles/bootlinux/tasks/power-cycle-vm.yml
new file mode 100644
index 00000000..5e1ea0cc
--- /dev/null
+++ b/playbooks/roles/bootlinux/tasks/power-cycle-vm.yml
@@ -0,0 +1,150 @@
+---
+# Power cycle VM to clear 9P cache and reload virtio device configuration
+# This is necessary when the 9P host path changes because:
+# - Simple remount doesn't clear virtio device state
+# - Restart doesn't properly clear 9P caches
+# - Full shutdown + start is required
+
+- name: Shutdown VM to clear 9P cache
+  community.libvirt.virt:
+    name: "{{ ansible_hostname }}"
+    state: shutdown
+    uri: "{{ libvirt_uri }}"
+  delegate_to: localhost
+  become: true
+  become_method: ansible.builtin.sudo
+  become_flags: "su - -c"
+  tags: ["data_partition", "9p_mount", "vm_power_cycle"]
+
+- name: Wait for SSH to become unreachable (VM shutting down)
+  ansible.builtin.wait_for:
+    host: "{{ inventory_hostname }}"
+    port: 22
+    state: stopped
+    timeout: 120
+    delay: 2
+  delegate_to: localhost
+  register: shutdown_wait_result
+  ignore_errors: true
+  tags: ["data_partition", "9p_mount", "vm_power_cycle"]
+
+- name: Give VM a moment to fully transition to shut off state
+  ansible.builtin.pause:
+    seconds: 3
+  delegate_to: localhost
+  tags: ["data_partition", "9p_mount", "vm_power_cycle"]
+
+- name: Check if VM actually shut down
+  community.libvirt.virt:
+    name: "{{ ansible_hostname }}"
+    command: status
+    uri: "{{ libvirt_uri }}"
+  register: vm_state_after_shutdown
+  delegate_to: localhost
+  become: true
+  become_method: ansible.builtin.sudo
+  become_flags: "su - -c"
+  tags: ["data_partition", "9p_mount", "vm_power_cycle"]
+
+- name: Debug VM state after shutdown check
+  delegate_to: localhost
+  ansible.builtin.debug:
+    msg: "VM {{ ansible_hostname }} state after shutdown: {{ vm_state_after_shutdown.status }}"
+  tags: ["data_partition", "9p_mount", "vm_power_cycle"]
+
+- name: Notify user if graceful shutdown failed
+  delegate_to: localhost
+  ansible.builtin.pause:
+    prompt: |
+
+      ============================================================
+      WARNING: VM {{ ansible_hostname }} did not shut down gracefully
+      ============================================================
+
+      Current VM state: {{ vm_state_after_shutdown.status }}
+
+      The VM needs to be shut down to reload the 9P filesystem configuration.
+      Graceful shutdown (virsh shutdown) timed out after 120 seconds.
+
+      Please manually shut down the VM using one of these methods:
+
+      1. PREFERRED - Investigate why shutdown failed:
+         sudo virsh console {{ ansible_hostname }}
+         (Check if system is hung, then try clean shutdown from inside VM)
+
+      2. Force shutdown (may cause data loss):
+         sudo virsh destroy {{ ansible_hostname }}
+
+      3. Try graceful shutdown again:
+         sudo virsh shutdown {{ ansible_hostname }}
+         (Wait and check: sudo virsh domstate {{ ansible_hostname }})
+
+      After the VM shows "shut off" state, press ENTER to continue.
+      The playbook will then redefine and restart the VM with the new 9P path.
+
+      To check VM state: sudo virsh domstate {{ ansible_hostname }}
+
+      ============================================================
+  when:
+    - vm_state_after_shutdown.status != "shutdown"
+  tags: ["data_partition", "9p_mount", "vm_power_cycle"]
+
+- name: Verify VM is shut off before continuing
+  community.libvirt.virt:
+    name: "{{ ansible_hostname }}"
+    command: status
+    uri: "{{ libvirt_uri }}"
+  register: vm_state_verified
+  failed_when: vm_state_verified.status != "shutdown"
+  delegate_to: localhost
+  become: true
+  become_method: ansible.builtin.sudo
+  become_flags: "su - -c"
+  tags: ["data_partition", "9p_mount", "vm_power_cycle"]
+
+- name: Undefine VM to clear cached device configuration
+  community.libvirt.virt:
+    name: "{{ ansible_hostname }}"
+    command: undefine
+    flags: "--nvram"
+    uri: "{{ libvirt_uri }}"
+  delegate_to: localhost
+  become: true
+  become_method: ansible.builtin.sudo
+  become_flags: "su - -c"
+  failed_when: false
+  tags: ["data_partition", "9p_mount", "vm_power_cycle"]
+
+- name: Redefine VM with updated 9P host path from XML
+  community.libvirt.virt:
+    command: define
+    xml: "{{ lookup('file', topdir_path + '/guestfs/' + ansible_hostname + '/' + ansible_hostname + '.xml') }}"
+    uri: "{{ libvirt_uri }}"
+  delegate_to: localhost
+  become: true
+  become_method: ansible.builtin.sudo
+  become_flags: "su - -c"
+  failed_when: false
+  tags: ["data_partition", "9p_mount", "vm_power_cycle"]
+
+- name: Start VM with fresh 9P device configuration
+  community.libvirt.virt:
+    name: "{{ ansible_hostname }}"
+    state: running
+    uri: "{{ libvirt_uri }}"
+  delegate_to: localhost
+  become: true
+  become_method: ansible.builtin.sudo
+  become_flags: "su - -c"
+  failed_when: false
+  tags: ["data_partition", "9p_mount", "vm_power_cycle"]
+
+- name: Wait for VM to become reachable after restart
+  ansible.builtin.wait_for_connection:
+    timeout: 120
+    delay: 5
+  tags: ["data_partition", "9p_mount", "vm_power_cycle"]
+
+- name: Gather facts after VM restart
+  ansible.builtin.setup:
+  tags: ["data_partition", "9p_mount", "vm_power_cycle"]

-- 
2.52.0