From e766c8466d79c6b109c9d01ef42470f4282da30c Mon Sep 17 00:00:00 2001 From: Joshua Bell Date: Mon, 26 Jan 2026 08:52:13 -0600 Subject: [PATCH] Increase VM defaults, restart VM on new workspace, make rebuild writable --- .gitignore | 1 - README.md | 13 +- bin/qvm-rebuild | 4 + bin/qvm-run | 24 ++- bin/qvm-start | 4 +- flake/default-vm/flake.nix | 4 +- qemu_dev_vm.md | 349 ------------------------------------- 7 files changed, 35 insertions(+), 364 deletions(-) delete mode 100644 .gitignore delete mode 100644 qemu_dev_vm.md diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 8fa639c..0000000 --- a/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.sisyphus diff --git a/README.md b/README.md index 7100a81..7eaf5a2 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Running AI coding agents in isolation presents a security challenge. Containers **Why QVM?** - **VM isolation over container isolation** - Hypervisor boundary is fundamentally stronger than kernel namespaces -- **One master image, shared caches** - Single ~7GB base image instead of per-project images +- **One master image, shared caches** - Single ~#GB base image instead of per-project images - **Transparent workspace mounting** - Current directory automatically available in VM - **Persistent state** - VM overlay preserves installed tools and configuration - **Shared build caches** - Cargo, pnpm, and sccache caches shared across all projects @@ -24,7 +24,7 @@ Add QVM to your NixOS configuration: ```nix { - inputs.qvm.url = "github:yourusername/qvm"; + inputs.qvm.url = "git+https://git.joshuabell.xyz/ringofstorms/qvm"; environment.systemPackages = [ inputs.qvm.packages.${system}.default @@ -37,7 +37,7 @@ Add QVM to your NixOS configuration: Run without installation: ```bash -nix run github:yourusername/qvm -- start +nix run git+https://git.joshuabell.xyz/ringofstorms/nvim -- start ``` ### Development shell @@ -45,7 +45,7 @@ nix run github:yourusername/qvm -- start For local development: ```bash -git clone https://github.com/yourusername/qvm +git clone https://git.joshuabell.xyz/ringofstorms/qvm.git cd qvm nix develop ``` @@ -358,11 +358,6 @@ qvm start ### Resource Allocation -Default resources: -- **Memory:** 8GB -- **CPUs:** 4 cores -- **Disk:** 20GB - To customize, set environment variables before `qvm start`: ```bash diff --git a/bin/qvm-rebuild b/bin/qvm-rebuild index 8ff9ec3..739eccc 100755 --- a/bin/qvm-rebuild +++ b/bin/qvm-rebuild @@ -79,7 +79,11 @@ build_base_image() { # Copy the qcow2 to base image location log_info "Copying image to: $QVM_BASE_IMAGE" + # Remove existing image first (may be read-only from Nix store copy) + rm -f "$QVM_BASE_IMAGE" cp -L "$qcow2_path" "$QVM_BASE_IMAGE" + # Ensure the new image is writable for future rebuilds + chmod 644 "$QVM_BASE_IMAGE" # Remove the result symlink rm -f "$build_result" diff --git a/bin/qvm-run b/bin/qvm-run index 1beead7..b16cb31 100755 --- a/bin/qvm-run +++ b/bin/qvm-run @@ -183,7 +183,29 @@ main() { newly_added=1 fi - # Ensure VM is running + # If this is a newly registered workspace, restart VM to mount it + if [[ "$newly_added" -eq 1 ]] && is_vm_running; then + log_info "New workspace registered. Restarting VM to mount it..." + + # Path to qvm-stop and qvm-start scripts + local script_dir="${QVM_LIB_DIR}/../bin" + local qvm_stop="$script_dir/qvm-stop" + local qvm_start="$script_dir/qvm-start" + + # Stop the VM + if ! "$qvm_stop"; then + die "Failed to stop VM" + fi + + # Start the VM with new workspace mount + if ! "$qvm_start"; then + die "Failed to start VM" + fi + + log_info "VM restarted with new workspace mounted" + fi + + # Ensure VM is running (if it wasn't running before) if ! is_vm_running; then log_info "VM not running, starting..." diff --git a/bin/qvm-start b/bin/qvm-start index 5dc5620..382d050 100755 --- a/bin/qvm-start +++ b/bin/qvm-start @@ -153,8 +153,8 @@ main() { log_info "Using SSH port: $ssh_port" # Get memory and CPU settings from environment or use defaults - local memory="${QVM_MEMORY:-8G}" - local cpus="${QVM_CPUS:-4}" + local memory="${QVM_MEMORY:-40G}" + local cpus="${QVM_CPUS:-30}" log_info "VM resources: ${memory} memory, ${cpus} CPUs" # Build QEMU command diff --git a/flake/default-vm/flake.nix b/flake/default-vm/flake.nix index bf437f0..0d3dcb5 100644 --- a/flake/default-vm/flake.nix +++ b/flake/default-vm/flake.nix @@ -209,8 +209,8 @@ ╚════════════════════════════════════════╝ ''; - # 20GB disk size - virtualisation.diskSize = 20 * 1024; + # 35GB disk size + virtualisation.diskSize = 40 * 1024; system.stateVersion = stateVersion; }; diff --git a/qemu_dev_vm.md b/qemu_dev_vm.md deleted file mode 100644 index a86b455..0000000 --- a/qemu_dev_vm.md +++ /dev/null @@ -1,349 +0,0 @@ -# QVM - Lightweight QEMU Development VM Wrapper - -A standalone CLI tool for running commands in an isolated NixOS VM with persistent state and shared caches. - ---- - -## Motivation - -Complex per-project VM systems create too many qcow2 images (~7GB each). QVM provides a simpler approach: -- **One master image** shared across all projects -- **Persistent overlay** for VM state -- **Shared caches** for cargo, pnpm, etc. -- **Mount any directory** as workspace - -Primary use case: Running AI coding agents (opencode, etc.) in isolation to prevent host filesystem access while maintaining build cache performance. - ---- - -## Security Model - -**Full VM isolation** (not containers): -- Container escapes via kernel exploits are a known attack surface -- VM escapes are rare - hypervisor boundary is fundamentally stronger -- For long unattended AI sessions, VM isolation is the safer choice - -**9p mount restrictions**: -- Only explicitly mounted directories are accessible -- Uses `security_model=mapped-xattr` (no passthrough) -- Host filesystem outside mounts is invisible to VM - ---- - -## Architecture - -``` -HOST VM -──── ── - -~/.local/share/qvm/ - └── base.qcow2 (read-only base image, ~7GB) - -~/.local/state/qvm/ - ├── overlay.qcow2 (persistent VM state, CoW) - ├── vm.pid (QEMU process ID) - ├── ssh.port (forwarded SSH port) - ├── serial.log (console output) - └── workspaces.json (mounted workspace registry) - -~/.cache/qvm/ - ├── cargo-home/ ──9p──▶ /cache/cargo/ - ├── cargo-target/ ──9p──▶ /cache/target/ - ├── pnpm-store/ ──9p──▶ /cache/pnpm/ - └── sccache/ ──9p──▶ /cache/sccache/ - -$(pwd) ──9p──▶ /workspace/project-{hash}/ - -~/.config/qvm/ - └── flake/ (user's NixOS flake definition) - ├── flake.nix - └── flake.lock -``` - -### Multiple Workspaces - -When `qvm run` is called from different directories, each gets mounted simultaneously: -- `/workspace/abc123/` ← `/home/josh/projects/foo` -- `/workspace/def456/` ← `/home/josh/projects/bar` - -The hash is derived from the absolute path. Commands run with CWD set to their workspace. - -### Shared Caches - -Caches are mounted from host, so: -- Cargo dependencies shared across all projects -- pnpm store shared (content-addressable) -- sccache compilation cache shared -- Each project still uses its own Cargo.lock/package.json - different versions coexist - ---- - -## CLI Interface - -```bash -# Run a command in VM (mounts $PWD as workspace, CDs into it) -qvm run opencode -qvm run "cargo build --release" -qvm run bash # interactive shell - -# VM lifecycle -qvm start # start VM daemon if not running -qvm stop # graceful shutdown -qvm status # show VM state, SSH port, mounted workspaces - -# Maintenance -qvm rebuild # rebuild base image from flake -qvm reset # wipe overlay, start fresh (keeps base image) - -# Direct access -qvm ssh # SSH into VM -qvm ssh -c "command" # run command via SSH -``` - -### Behavior Details - -**`qvm run `**: -1. If VM not running, start it (blocking until SSH ready) -2. Mount $PWD into VM if not already mounted (via 9p hotplug or pre-mount) -3. SSH in and execute: `cd /workspace/{hash} && ` -4. Stream stdout/stderr to terminal -5. Exit with command's exit code -6. VM stays running for next command - -**`qvm start`**: -1. Check if VM already running (via pid file) -2. If base.qcow2 missing, run `qvm rebuild` first -3. Create overlay.qcow2 if missing (backed by base.qcow2) -4. Launch QEMU with KVM, virtio, 9p mounts -5. Wait for SSH to become available -6. Print SSH port - -**`qvm stop`**: -1. Send ACPI shutdown to VM -2. Wait for graceful shutdown (timeout 30s) -3. If timeout, SIGKILL QEMU -4. Clean up pid file - -**`qvm rebuild`**: -1. Run `nix build` on ~/.config/qvm/flake -2. Copy result to ~/.local/share/qvm/base.qcow2 -3. If VM running, warn user to restart for changes - -**`qvm reset`**: -1. Stop VM if running -2. Delete overlay.qcow2 -3. Delete workspaces.json -4. Next start creates fresh overlay - ---- - -## Default NixOS Flake - -The tool ships with a default flake template. User can customize at `~/.config/qvm/flake/`. - -### Included by Default - -```nix -{ - # Base system - boot.kernelPackages = pkgs.linuxPackages_latest; - - # Shell - programs.zsh.enable = true; - users.users.dev = { - isNormalUser = true; - shell = pkgs.zsh; - extraGroups = [ "wheel" ]; - }; - - # Essential tools - environment.systemPackages = with pkgs; [ - git - vim - tmux - htop - curl - wget - jq - ripgrep - fd - - # Language tooling (user can remove/customize) - rustup - nodejs_22 - pnpm - python3 - go - ]; - - # SSH server - services.openssh = { - enable = true; - settings.PasswordAuthentication = false; - }; - - # 9p mounts (populated at runtime) - fileSystems."/cache/cargo" = { device = "cargo"; fsType = "9p"; options = [...]; }; - fileSystems."/cache/target" = { device = "target"; fsType = "9p"; options = [...]; }; - fileSystems."/cache/pnpm" = { device = "pnpm"; fsType = "9p"; options = [...]; }; - fileSystems."/cache/sccache" = { device = "sccache"; fsType = "9p"; options = [...]; }; - - # Environment - environment.variables = { - CARGO_HOME = "/cache/cargo"; - CARGO_TARGET_DIR = "/cache/target"; - PNPM_HOME = "/cache/pnpm"; - SCCACHE_DIR = "/cache/sccache"; - }; - - # Disk size - virtualisation.diskSize = 20480; # 20GB -} -``` - -### User Customization - -Users edit `~/.config/qvm/flake/flake.nix` to: -- Add/remove packages -- Change shell configuration -- Add custom NixOS modules -- Pin nixpkgs version -- Include their dotfiles - -After editing: `qvm rebuild` - ---- - -## QEMU Configuration - -```bash -qemu-system-x86_64 \ - -enable-kvm \ - -cpu host \ - -m "${MEMORY:-8G}" \ - -smp "${CPUS:-4}" \ - \ - # Disk (overlay backed by base) - -drive file=overlay.qcow2,format=qcow2,if=virtio \ - \ - # Network (user mode with SSH forward) - -netdev user,id=net0,hostfwd=tcp::${SSH_PORT}-:22 \ - -device virtio-net-pci,netdev=net0 \ - \ - # 9p shares for caches - -virtfs local,path=${CACHE_DIR}/cargo-home,mount_tag=cargo,security_model=mapped-xattr \ - -virtfs local,path=${CACHE_DIR}/cargo-target,mount_tag=target,security_model=mapped-xattr \ - -virtfs local,path=${CACHE_DIR}/pnpm-store,mount_tag=pnpm,security_model=mapped-xattr \ - -virtfs local,path=${CACHE_DIR}/sccache,mount_tag=sccache,security_model=mapped-xattr \ - \ - # 9p shares for workspaces (added dynamically or pre-mounted) - -virtfs local,path=/path/to/project,mount_tag=ws_abc123,security_model=mapped-xattr \ - \ - # Console - -serial file:serial.log \ - -monitor none \ - -nographic \ - \ - # Daemonize - -daemonize \ - -pidfile vm.pid -``` - -### Resource Allocation - -Default: 50% of RAM, 90% of CPUs (can be configured via env vars or config file later) - ---- - -## Implementation Plan - -### Phase 1: Core Scripts - -1. **`qvm` main script** - dispatcher for subcommands -2. **`qvm-start`** - launch QEMU with all mounts -3. **`qvm-stop`** - graceful shutdown -4. **`qvm-run`** - mount workspace + execute command via SSH -5. **`qvm-ssh`** - direct SSH access -6. **`qvm-status`** - show state - -### Phase 2: Image Management - -1. **`qvm-rebuild`** - build image from flake -2. **`qvm-reset`** - wipe overlay -3. **Default flake template** - copy to ~/.config/qvm/flake on first run - -### Phase 3: Polish - -1. **First-run experience** - auto-create dirs, copy default flake, build image -2. **Error handling** - clear messages for common failures -3. **README** - usage docs - ---- - -## File Structure (New Repo) - -``` -qvm/ -├── bin/ -│ ├── qvm # Main dispatcher -│ ├── qvm-start -│ ├── qvm-stop -│ ├── qvm-run -│ ├── qvm-ssh -│ ├── qvm-status -│ ├── qvm-rebuild -│ └── qvm-reset -├── lib/ -│ └── common.sh # Shared functions -├── flake/ -│ ├── flake.nix # Default NixOS flake template -│ └── flake.lock -├── flake.nix # Nix flake for installing qvm itself -├── README.md -└── LICENSE -``` - ---- - -## Edge Cases & Decisions - -| Scenario | Behavior | -|----------|----------| -| `qvm run` with no args | Error: "Usage: qvm run " | -| `qvm run` while image missing | Auto-trigger `qvm rebuild` first | -| `qvm run` from same dir twice | Reuse existing mount, just run command | -| `qvm stop` when not running | No-op, exit 0 | -| `qvm rebuild` while VM running | Warn but proceed; user must restart VM | -| `qvm reset` while VM running | Stop VM first, then reset | -| SSH not ready after 60s | Error with troubleshooting hints | -| QEMU crashes | Detect via pid, clean up state files | - ---- - -## Not In Scope (Explicit Exclusions) - -- **Multi-VM**: Only one VM at a time -- **Per-project configs**: Single global config (use qai for project-specific VMs) -- **Windows/macOS**: Linux + KVM only -- **GUI/Desktop**: Headless only -- **Snapshots**: Just overlay reset, no checkpoint/restore -- **Resource limits**: Trust the VM, no cgroups on host - ---- - -## Dependencies - -- `qemu` (with KVM support) -- `nix` (for building images) -- `openssh` (ssh client) -- `jq` (for workspace registry) - ---- - -## Future Considerations (Not v1) - -- Config file for memory/CPU/ports -- Tab completion for zsh/bash -- Systemd user service for auto-start -- Health checks and auto-restart -- Workspace unmount command