diff --git a/.air.darwin.toml b/.air.darwin.toml new file mode 100644 index 00000000..ded73f54 --- /dev/null +++ b/.air.darwin.toml @@ -0,0 +1,48 @@ +root = "." +testdata_dir = "testdata" +tmp_dir = "tmp" + +[build] + args_bin = [] + bin = "./tmp/main" + # Build for macOS with vz support, then sign with entitlements + # Also builds and signs vz-shim (subprocess that hosts vz VMs) + cmd = "make build-embedded && go build -o ./tmp/vz-shim ./cmd/vz-shim && codesign --sign - --entitlements vz.entitlements --force ./tmp/vz-shim && go build -tags containers_image_openpgp -o ./tmp/main ./cmd/api && codesign --sign - --entitlements vz.entitlements --force ./tmp/main" + delay = 1000 + exclude_dir = ["assets", "tmp", "vendor", "testdata", "bin", "scripts", "data", "kernel"] + exclude_file = [] + exclude_regex = ["_test.go"] + exclude_unchanged = false + follow_symlink = false + # No sudo needed on macOS - vz doesn't require root + full_bin = "./tmp/main" + include_dir = [] + include_ext = ["go", "tpl", "tmpl", "html", "yaml"] + include_file = [] + log = "build-errors.log" + poll = false + poll_interval = 0 + post_cmd = [] + kill_delay = '1s' + rerun = false + rerun_delay = 500 + send_interrupt = true + stop_on_error = false + +[color] + app = "" + build = "yellow" + main = "magenta" + runner = "green" + watcher = "cyan" + +[log] + main_only = false + time = false + +[misc] + clean_on_exit = false + +[screen] + clear_on_rebuild = false + keep_scroll = true diff --git a/.env.darwin.example b/.env.darwin.example new file mode 100644 index 00000000..f714f06e --- /dev/null +++ b/.env.darwin.example @@ -0,0 +1,122 @@ +# ============================================================================= +# macOS (Darwin) Configuration for Hypeman +# ============================================================================= +# Copy this file to .env and customize for your environment. +# +# Key differences from Linux (.env.example): +# - DEFAULT_HYPERVISOR: Use "vz" (Virtualization.framework) instead of cloud-hypervisor/qemu +# - DATA_DIR: Uses macOS conventions (~/Library/Application Support) +# - Network settings: BRIDGE_NAME, SUBNET_CIDR, etc. are IGNORED (vz uses NAT) +# - Rate limiting: Not supported on macOS (no tc/HTB equivalent) +# - GPU passthrough: Not supported on macOS +# ============================================================================= + +# Required +JWT_SECRET=dev-secret-change-me + +# Data directory - use macOS conventions +# Note: ~ expands to $HOME at runtime +DATA_DIR=~/Library/Application Support/hypeman + +# Server configuration +PORT=8080 + +# Logging +LOG_LEVEL=debug + +# ============================================================================= +# Hypervisor Configuration (IMPORTANT FOR MACOS) +# ============================================================================= +# On macOS, use "vz" (Virtualization.framework) +# - "cloud-hypervisor" and "qemu" are NOT supported on macOS +DEFAULT_HYPERVISOR=vz + +# ============================================================================= +# Network Configuration (DIFFERENT ON MACOS) +# ============================================================================= +# On macOS with vz, network is handled automatically via NAT: +# - VMs get IP addresses from 192.168.64.0/24 via DHCP +# - No TAP devices, bridges, or iptables needed +# - The following settings are IGNORED on macOS: +# BRIDGE_NAME, SUBNET_CIDR, SUBNET_GATEWAY, UPLINK_INTERFACE + +# DNS Server for VMs (used by guest for resolution) +DNS_SERVER=8.8.8.8 + +# ============================================================================= +# Caddy / Ingress Configuration +# ============================================================================= +CADDY_LISTEN_ADDRESS=0.0.0.0 +CADDY_ADMIN_ADDRESS=127.0.0.1 +CADDY_ADMIN_PORT=2019 +# Note: 5353 is used by mDNSResponder (Bonjour) on macOS, using 5354 instead +INTERNAL_DNS_PORT=5354 +CADDY_STOP_ON_SHUTDOWN=false + +# ============================================================================= +# Build System Configuration +# ============================================================================= +# For builds on macOS with vz, the registry URL needs to be accessible from +# NAT VMs. Since vz uses 192.168.64.0/24 for NAT, the host is at 192.168.64.1. +# +# IMPORTANT: "host.docker.internal" does NOT work in vz VMs - that's a Docker +# Desktop-specific hostname. Use the NAT gateway IP instead. +# +# Registry URL (the host's hypeman API, accessible from VMs) +REGISTRY_URL=192.168.64.1:8080 +# Use HTTP (not HTTPS) since hypeman's internal registry uses plaintext +REGISTRY_INSECURE=true + +BUILDER_IMAGE=hypeman/builder:latest +MAX_CONCURRENT_SOURCE_BUILDS=2 +BUILD_TIMEOUT=600 + +# ============================================================================= +# Resource Limits (same as Linux) +# ============================================================================= +# Per-instance limits +MAX_VCPUS_PER_INSTANCE=4 +MAX_MEMORY_PER_INSTANCE=8GB + +# Aggregate limits (0 or empty = unlimited) +# MAX_TOTAL_VOLUME_STORAGE= + +# ============================================================================= +# OpenTelemetry (optional, same as Linux) +# ============================================================================= +# OTEL_ENABLED=false +# OTEL_ENDPOINT=127.0.0.1:4317 +# OTEL_SERVICE_NAME=hypeman +# OTEL_INSECURE=true +# ENV=dev + +# ============================================================================= +# TLS / ACME Configuration (same as Linux) +# ============================================================================= +# ACME_EMAIL=admin@example.com +# ACME_DNS_PROVIDER=cloudflare +# TLS_ALLOWED_DOMAINS=*.example.com +# CLOUDFLARE_API_TOKEN= + +# ============================================================================= +# macOS Limitations +# ============================================================================= +# The following features are NOT AVAILABLE on macOS: +# +# 1. GPU Passthrough (VFIO, mdev) +# - GPU_PROFILE_CACHE_TTL is ignored +# - Device registration/binding will fail +# +# 2. Network Rate Limiting +# - UPLOAD_BURST_MULTIPLIER, DOWNLOAD_BURST_MULTIPLIER are ignored +# - No tc/HTB equivalent on macOS +# +# 3. CPU/Memory Hotplug +# - Resize operations not supported +# +# 4. Disk I/O Limiting +# - DISK_IO_LIMIT, OVERSUB_DISK_IO are ignored +# +# 5. Snapshots (requires macOS 14+ on Apple Silicon) +# - SaveMachineStateToPath/RestoreMachineStateFromURL require macOS 14+ +# - Only supported on ARM64 (Apple Silicon) Macs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ee4f8a75..9ff9862d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,3 +55,64 @@ jobs: TLS_TEST_DOMAIN: "test.hypeman-development.com" TLS_ALLOWED_DOMAINS: '*.hypeman-development.com' run: make test + + test-darwin: + runs-on: [self-hosted, macos, arm64] + concurrency: + group: macos-ci-test-${{ github.ref }} + cancel-in-progress: true + env: + DATA_DIR: /tmp/hypeman-ci-${{ github.run_id }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v6 + with: + go-version: '1.25' + cache: false + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Install dependencies + run: | + brew list e2fsprogs &>/dev/null || brew install e2fsprogs + brew list erofs-utils &>/dev/null || brew install erofs-utils + go mod download + - name: Create run-scoped data directory + run: mkdir -p "$DATA_DIR" + - name: Generate OpenAPI code + run: make oapi-generate + - name: Build + run: make build + - name: Run tests + env: + DEFAULT_HYPERVISOR: vz + JWT_SECRET: ci-test-secret + run: make test + - name: Cleanup + if: always() + run: | + pkill -f "vz-shim.*$DATA_DIR" || true + rm -rf "$DATA_DIR" + make clean + + e2e-install: + runs-on: [self-hosted, macos, arm64] + needs: test-darwin + concurrency: + group: macos-ci-e2e-${{ github.ref }} + cancel-in-progress: true + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v6 + with: + go-version: '1.25' + cache: false + - name: Install dependencies + run: brew list caddy &>/dev/null || brew install caddy + - name: Run E2E install test + run: bash scripts/e2e-install-test.sh + - name: Cleanup on failure + if: failure() + run: bash scripts/uninstall.sh || true diff --git a/.gitignore b/.gitignore index b2b815d4..8de172d4 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ cloud-hypervisor/** lib/system/exec_agent/exec-agent lib/system/guest_agent/guest-agent lib/system/init/init +lib/hypervisor/vz/vz-shim/vz-shim # Envoy binaries lib/ingress/binaries/** @@ -29,3 +30,9 @@ dist/** # UTM VM - downloaded ISO files scripts/utm/images/ + +# IDE and editor +.cursor/ + +# Build artifacts +api diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 85a14f8b..857cb954 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -4,7 +4,17 @@ This document covers development setup, configuration, and contributing to Hypem ## Prerequisites -> **macOS Users:** Hypeman requires KVM, which is only available on Linux. See [scripts/utm/README.md](scripts/utm/README.md) for instructions on setting up a Linux VM with nested virtualization on Apple Silicon Macs. +### Linux (Default) + +**Go 1.25.4+**, **KVM**, **erofs-utils**, **dnsmasq** + +### macOS (Experimental) + +See [macOS Development](#macos-development) below for native macOS development using Virtualization.framework. + +--- + +**Linux Prerequisites:** **Go 1.25.4+**, **KVM**, **erofs-utils**, **dnsmasq** @@ -111,6 +121,7 @@ Hypeman can be configured using the following environment variables: | `DNS_PROPAGATION_TIMEOUT` | Max time to wait for DNS propagation (e.g., `2m`) | _(empty)_ | | `DNS_RESOLVERS` | Comma-separated DNS resolvers for propagation checking | _(empty)_ | | `CLOUDFLARE_API_TOKEN` | Cloudflare API token (when using `cloudflare` provider) | _(empty)_ | +| `DOCKER_SOCKET` | Path to Docker socket (for builder image builds) | `/var/run/docker.sock` | **Important: Subnet Configuration** @@ -244,6 +255,49 @@ make dev The server will start on port 8080 (configurable via `PORT` environment variable). +### Setting Up the Builder Image (for Dockerfile builds) + +The builder image is required for `hypeman build` to work. There are two modes: + +**Automatic mode (default):** When `BUILDER_IMAGE` is unset or empty, the server will automatically build and push the builder image on startup using Docker. This is the easiest way to get started — just ensure Docker is available and run `make dev`. If a build is requested while the builder image is still being prepared, the server returns a clear error asking you to retry shortly. + +On macOS with Colima, set the Docker socket path: +```bash +DOCKER_SOCKET=$HOME/.colima/default/docker.sock +``` + +**Manual mode:** When `BUILDER_IMAGE` is explicitly set, the server assumes you manage your own image. Follow these steps: + +1. **Build the builder image** (requires Docker): + ```bash + docker build -t hypeman/builder:latest -f lib/builds/images/generic/Dockerfile . + ``` + +2. **Start the Hypeman server** (if not already running): + ```bash + make dev + ``` + +3. **Push to Hypeman's internal registry**: + ```bash + # Generate a token with registry push permissions + export JWT_SECRET="dev-secret-for-local-testing" + export HYPEMAN_API_KEY=$(go run ./cmd/gen-jwt -registry-push "hypeman/builder") + export HYPEMAN_BASE_URL="http://localhost:8080" + + # Push using hypeman-cli + hypeman push hypeman/builder:latest + ``` + +4. **Configure the builder image** in `.env`: + ```bash + BUILDER_IMAGE=localhost:8080/hypeman/builder:latest + ``` + +5. **Restart the server** to pick up the new config. + +Now `hypeman build ` will work for Dockerfile-based builds. + ### Local OpenTelemetry (optional) To collect traces and metrics locally, run the Grafana LGTM stack (Loki, Grafana, Tempo, Mimir): @@ -314,3 +368,176 @@ Or generate everything at once: ```bash make generate-all ``` + +## macOS Development + +Hypeman supports native macOS development using Apple's Virtualization.framework (via the `vz` hypervisor). + +### Requirements + +- **macOS 11.0+** (Big Sur or later) +- **Apple Silicon** (M1/M2/M3) recommended +- **macOS 14.0+** (Sonoma) required for snapshot/restore (ARM64 only) +- **Go 1.25.4+** +- **Caddy** (for ingress): `brew install caddy` +- **e2fsprogs** (for ext4 disk images): `brew install e2fsprogs` + +### Quick Start + +```bash +# 1. Install dependencies +brew install caddy e2fsprogs + +# 2. Add e2fsprogs to PATH (it's keg-only) +export PATH="/opt/homebrew/opt/e2fsprogs/bin:/opt/homebrew/opt/e2fsprogs/sbin:$PATH" +# Add to ~/.zshrc for persistence + +# 3. Configure environment +cp .env.darwin.example .env +# Edit .env as needed (defaults work for local development) + +# 4. Create data directory +mkdir -p ~/Library/Application\ Support/hypeman + +# 5. Run in development mode (auto-detects macOS, builds, signs, and runs with hot reload) +make dev +``` + +The `make dev` command automatically detects macOS and: +- Builds with vz support +- Signs with required entitlements +- Runs with hot reload (no sudo required) + +### Alternative Commands + +```bash +# Build and sign only (no hot reload) +make sign-darwin + +# Verify entitlements are correct +make verify-entitlements + +# Run manually after signing +./bin/hypeman +``` + +### Key Differences from Linux Development + +| Aspect | Linux | macOS | +|--------|-------|-------| +| Hypervisor | Cloud Hypervisor, QEMU | vz (Virtualization.framework) | +| Binary signing | Not required | Automatic via `make dev` or `make sign-darwin` | +| Networking | TAP + bridge + iptables | Automatic NAT (no setup needed) | +| Root/sudo | Required for networking | Not required | +| Caddy | Embedded binary | Install via `brew install caddy` | +| DNS port | 5353 | 5354 (avoids mDNSResponder conflict) | + +### macOS-Specific Configuration + +The following environment variables work differently on macOS (see `.env.darwin.example`): + +| Variable | Linux | macOS | +|----------|-------|-------| +| `DEFAULT_HYPERVISOR` | `cloud-hypervisor` | `vz` | +| `DATA_DIR` | `/var/lib/hypeman` | `~/Library/Application Support/hypeman` | +| `INTERNAL_DNS_PORT` | `5353` | `5354` (5353 is used by mDNSResponder) | +| `BRIDGE_NAME` | Used | Ignored (NAT) | +| `SUBNET_CIDR` | Used | Ignored (NAT) | +| `UPLINK_INTERFACE` | Used | Ignored (NAT) | +| Network rate limiting | Supported | Not supported | +| GPU passthrough | Supported (VFIO) | Not supported | + +### Code Organization + +Platform-specific code uses Go build tags: + +``` +lib/network/ +├── bridge_linux.go # Linux networking (TAP, bridges, iptables) +├── bridge_darwin.go # macOS stubs (uses NAT) +└── ip.go # Shared utilities + +lib/devices/ +├── discovery_linux.go # Linux PCI device discovery +├── discovery_darwin.go # macOS stubs (no passthrough) +├── mdev_linux.go # Linux vGPU (mdev) +├── mdev_darwin.go # macOS stubs +├── vfio_linux.go # Linux VFIO binding +├── vfio_darwin.go # macOS stubs +└── types.go # Shared types + +lib/hypervisor/ +├── cloudhypervisor/ # Cloud Hypervisor (Linux) +├── qemu/ # QEMU (Linux, vsock_linux.go) +└── vz/ # Virtualization.framework (macOS only) + ├── starter.go # VMStarter implementation + ├── hypervisor.go # Hypervisor interface + └── vsock.go # VsockDialer via VirtioSocketDevice +``` + +### Testing on macOS + +```bash +# Verify vz package compiles correctly +make test-vz-compile + +# Run unit tests (Linux-specific tests like networking will be skipped) +go test ./lib/hypervisor/vz/... +go test ./lib/resources/... +go test ./lib/images/... +``` + +Note: Full integration tests require Linux. On macOS, focus on unit tests and manual API testing. + +### Known Limitations + +1. **Disk Format**: vz only supports raw disk images (not qcow2). Convert images: + ```bash + qemu-img convert -f qcow2 -O raw disk.qcow2 disk.raw + ``` + +2. **Snapshots**: Only available on macOS 14+ (Sonoma) on Apple Silicon: + ```go + // Check support at runtime + valid, err := vmConfig.ValidateSaveRestoreSupport() + ``` + +3. **Network Ingress**: VMs get DHCP addresses from macOS NAT. To access a VM's services: + - Query the VM's IP via guest agent + - Use vsock for internal communication (no NAT traversal needed) + +4. **In-Process VMM**: Unlike CH/QEMU which run as separate processes, vz VMs run in the hypeman process. If hypeman crashes, all VMs stop. + +### Troubleshooting + +**"binary needs to be signed with entitlements"** +```bash +make sign-darwin +# Or just use: make dev (handles signing automatically) +``` + +**"caddy binary is not embedded on macOS"** +```bash +brew install caddy +``` + +**"address already in use" on port 5353** +- Port 5353 is used by mDNSResponder (Bonjour) on macOS +- Use port 5354 instead: `INTERNAL_DNS_PORT=5354` in `.env` +- The `.env.darwin.example` already has this configured correctly + +**"Virtualization.framework is not available"** +- Ensure you're on macOS 11.0+ +- Check if virtualization is enabled in Recovery Mode settings + +**"snapshot not supported"** +- Requires macOS 14.0+ on Apple Silicon +- Check: `sw_vers` and `uname -m` (should be arm64) + +**VM fails to start** +- Check serial log: `$DATA_DIR/instances//serial.log` +- Ensure kernel and initrd paths are correct in config + +**IOMMU/VFIO warnings at startup** +- These are expected on macOS and can be ignored +- GPU passthrough is not supported on macOS diff --git a/Makefile b/Makefile index 88eab9c9..8905538d 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SHELL := /bin/bash -.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build test install-tools gen-jwt download-ch-binaries download-ch-spec ensure-ch-binaries build-caddy-binaries build-caddy ensure-caddy-binaries release-prep clean build-embedded +.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build build-linux test test-linux test-darwin install-tools gen-jwt download-ch-binaries download-ch-spec ensure-ch-binaries build-caddy-binaries build-caddy ensure-caddy-binaries release-prep clean build-embedded # Directory where local binaries will be installed BIN_DIR ?= $(CURDIR)/bin @@ -174,33 +174,59 @@ ensure-caddy-binaries: fi # Build guest-agent (guest binary) into its own directory for embedding +# Cross-compile for Linux since it runs inside the VM lib/system/guest_agent/guest-agent: lib/system/guest_agent/*.go - @echo "Building guest-agent..." - cd lib/system/guest_agent && CGO_ENABLED=0 go build -ldflags="-s -w" -o guest-agent . + @echo "Building guest-agent for Linux..." + cd lib/system/guest_agent && CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o guest-agent . # Build init binary (runs as PID 1 in guest VM) for embedding +# Cross-compile for Linux since it runs inside the VM lib/system/init/init: lib/system/init/*.go - @echo "Building init binary..." - cd lib/system/init && CGO_ENABLED=0 go build -ldflags="-s -w" -o init . + @echo "Building init binary for Linux..." + cd lib/system/init && CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o init . build-embedded: lib/system/guest_agent/guest-agent lib/system/init/init # Build the binary -build: ensure-ch-binaries ensure-caddy-binaries build-embedded | $(BIN_DIR) +build: +ifeq ($(shell uname -s),Darwin) + $(MAKE) build-darwin +else + $(MAKE) build-linux +endif + +build-linux: ensure-ch-binaries ensure-caddy-binaries build-embedded | $(BIN_DIR) go build -tags containers_image_openpgp -o $(BIN_DIR)/hypeman ./cmd/api # Build all binaries build-all: build # Run in development mode with hot reload -dev: ensure-ch-binaries ensure-caddy-binaries build-embedded $(AIR) +# On macOS, redirects to dev-darwin which uses vz instead of cloud-hypervisor +dev: + @if [ "$$(uname)" = "Darwin" ]; then \ + $(MAKE) dev-darwin; \ + else \ + $(MAKE) dev-linux; \ + fi + +# Linux development mode with hot reload +dev-linux: ensure-ch-binaries ensure-caddy-binaries build-embedded $(AIR) @rm -f ./tmp/main $(AIR) -c .air.toml -# Run tests (as root for network capabilities, enables caching and parallelism) +# Run tests # Usage: make test - runs all tests # make test TEST=TestCreateInstanceWithNetwork - runs specific test -test: ensure-ch-binaries ensure-caddy-binaries build-embedded +test: +ifeq ($(shell uname -s),Darwin) + $(MAKE) test-darwin +else + $(MAKE) test-linux +endif + +# Linux tests (as root for network capabilities) +test-linux: ensure-ch-binaries ensure-caddy-binaries build-embedded @VERBOSE_FLAG=""; \ if [ -n "$(VERBOSE)" ]; then VERBOSE_FLAG="-v"; fi; \ if [ -n "$(TEST)" ]; then \ @@ -210,6 +236,24 @@ test: ensure-ch-binaries ensure-caddy-binaries build-embedded sudo env "PATH=$$PATH" "DOCKER_CONFIG=$${DOCKER_CONFIG:-$$HOME/.docker}" go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=180s ./...; \ fi +# macOS tests (no sudo needed, adds e2fsprogs to PATH) +# Uses 'go list' to discover compilable packages, then filters out packages +# whose test files reference Linux-only symbols (network, devices, system/init). +DARWIN_EXCLUDE_PKGS := /lib/network|/lib/devices|/lib/system/init +test-darwin: build-embedded sign-vz-shim + @VERBOSE_FLAG=""; \ + if [ -n "$(VERBOSE)" ]; then VERBOSE_FLAG="-v"; fi; \ + PKGS=$$(PATH="/opt/homebrew/opt/e2fsprogs/sbin:$(PATH)" \ + go list -tags containers_image_openpgp ./... 2>/dev/null | grep -Ev '$(DARWIN_EXCLUDE_PKGS)'); \ + if [ -n "$(TEST)" ]; then \ + echo "Running specific test: $(TEST)"; \ + PATH="/opt/homebrew/opt/e2fsprogs/sbin:$(PATH)" \ + go test -tags containers_image_openpgp -run=$(TEST) $$VERBOSE_FLAG -timeout=180s $$PKGS; \ + else \ + PATH="/opt/homebrew/opt/e2fsprogs/sbin:$(PATH)" \ + go test -tags containers_image_openpgp $$VERBOSE_FLAG -timeout=180s $$PKGS; \ + fi + # Generate JWT token for testing # Usage: make gen-jwt [USER_ID=test-user] gen-jwt: $(GODOTENV) @@ -233,8 +277,102 @@ clean: rm -rf lib/ingress/binaries/ rm -f lib/system/guest_agent/guest-agent rm -f lib/system/init/init + rm -f lib/hypervisor/vz/vz-shim/vz-shim # Prepare for release build (called by GoReleaser) # Downloads all embedded binaries and builds embedded components release-prep: download-ch-binaries build-caddy-binaries build-embedded go mod tidy + +# ============================================================================= +# macOS (vz/Virtualization.framework) targets +# ============================================================================= + +# Entitlements file for macOS codesigning +ENTITLEMENTS_FILE ?= vz.entitlements + +# Build vz-shim (subprocess that hosts vz VMs) +# Also copies to embed directory so it gets embedded in the hypeman binary +.PHONY: build-vz-shim +build-vz-shim: | $(BIN_DIR) + @echo "Building vz-shim for macOS..." + go build -o $(BIN_DIR)/vz-shim ./cmd/vz-shim + cp $(BIN_DIR)/vz-shim lib/hypervisor/vz/vz-shim/vz-shim + @echo "Build complete: $(BIN_DIR)/vz-shim" + +# Sign vz-shim with entitlements +.PHONY: sign-vz-shim +sign-vz-shim: build-vz-shim + @echo "Signing $(BIN_DIR)/vz-shim with entitlements..." + codesign --sign - --entitlements $(ENTITLEMENTS_FILE) --force $(BIN_DIR)/vz-shim + @echo "Signed: $(BIN_DIR)/vz-shim" + +# Build for macOS with vz support +# Note: This builds without embedded CH/Caddy binaries since vz doesn't need them +# Guest-agent and init are cross-compiled for Linux (they run inside the VM) +.PHONY: build-darwin +build-darwin: build-embedded build-vz-shim | $(BIN_DIR) + @echo "Building hypeman for macOS with vz support..." + go build -tags containers_image_openpgp -o $(BIN_DIR)/hypeman ./cmd/api + @echo "Build complete: $(BIN_DIR)/hypeman" + +# Sign the binary with entitlements (required for Virtualization.framework) +# Usage: make sign-darwin +.PHONY: sign-darwin +sign-darwin: build-darwin sign-vz-shim + @echo "Signing $(BIN_DIR)/hypeman with entitlements..." + codesign --sign - --entitlements $(ENTITLEMENTS_FILE) --force $(BIN_DIR)/hypeman + @echo "Verifying signature..." + codesign --display --entitlements - $(BIN_DIR)/hypeman + +# Sign with a specific identity (for distribution) +# Usage: make sign-darwin-identity IDENTITY="Developer ID Application: Your Name" +.PHONY: sign-darwin-identity +sign-darwin-identity: build-darwin + @if [ -z "$(IDENTITY)" ]; then \ + echo "Error: IDENTITY not set. Usage: make sign-darwin-identity IDENTITY='Developer ID Application: ...'"; \ + exit 1; \ + fi + @echo "Signing $(BIN_DIR)/hypeman with identity: $(IDENTITY)" + codesign --sign "$(IDENTITY)" --entitlements $(ENTITLEMENTS_FILE) --force --options runtime $(BIN_DIR)/hypeman + @echo "Verifying signature..." + codesign --verify --verbose $(BIN_DIR)/hypeman + +# Run on macOS with vz support (development mode) +# Automatically signs the binary before running +.PHONY: dev-darwin +# macOS development mode with hot reload (uses vz, no sudo needed) +dev-darwin: build-embedded $(AIR) + @rm -f ./tmp/main + PATH="/opt/homebrew/opt/e2fsprogs/sbin:$(PATH)" $(AIR) -c .air.darwin.toml + +# Run without hot reload (for testing) +run: + @if [ "$$(uname)" = "Darwin" ]; then \ + $(MAKE) run-darwin; \ + else \ + $(MAKE) run-linux; \ + fi + +run-linux: ensure-ch-binaries ensure-caddy-binaries build-embedded build + ./bin/hypeman + +run-darwin: sign-darwin + PATH="/opt/homebrew/opt/e2fsprogs/sbin:$(PATH)" ./bin/hypeman + +# Quick test of vz package compilation +.PHONY: test-vz-compile +test-vz-compile: + @echo "Testing vz package compilation..." + go build ./lib/hypervisor/vz/... + @echo "vz package compiles successfully" + +# Verify entitlements on a signed binary +.PHONY: verify-entitlements +verify-entitlements: + @if [ ! -f $(BIN_DIR)/hypeman ]; then \ + echo "Error: $(BIN_DIR)/hypeman not found. Run 'make sign-darwin' first."; \ + exit 1; \ + fi + @echo "Entitlements on $(BIN_DIR)/hypeman:" + codesign --display --entitlements - $(BIN_DIR)/hypeman diff --git a/README.md b/README.md index 69a5f18f..324102fe 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,13 @@ ## Requirements -Hypeman server runs on **Linux** with **KVM** virtualization support. The CLI can run locally on the server or connect remotely from any machine. +### Linux (Production) +Hypeman server runs on **Linux** with **KVM** virtualization support. Supports Cloud Hypervisor and QEMU as hypervisors. + +### macOS (Experimental) +Hypeman also supports **macOS** (11.0+) using Apple's **Virtualization.framework** via the `vz` hypervisor. See [macOS Support](#macos-support) below. + +The CLI can run locally on the server or connect remotely from any machine. ## Quick Start @@ -153,6 +159,59 @@ hypeman logs --source hypeman my-app For all available commands, run `hypeman --help`. +## macOS Support + +Hypeman supports macOS using Apple's Virtualization.framework through the `vz` hypervisor. This provides native virtualization on Apple Silicon Macs (Intel Macs are not supported). + +### Requirements + +- macOS 11.0+ (macOS 14.0+ required for snapshot/restore on ARM64) +- Apple Silicon (M1/M2/M3) recommended +- Caddy: `brew install caddy` +- e2fsprogs: `brew install e2fsprogs` (for ext4 disk images) + +### Quick Start (macOS) + +```bash +# Install dependencies +brew install caddy e2fsprogs + +# Add e2fsprogs to PATH (it's keg-only) +export PATH="/opt/homebrew/opt/e2fsprogs/bin:/opt/homebrew/opt/e2fsprogs/sbin:$PATH" + +# Configure environment +cp .env.darwin.example .env + +# Create data directory +mkdir -p ~/Library/Application\ Support/hypeman + +# Run with hot reload (auto-detects macOS, builds, signs, and runs) +make dev +``` + +The `make dev` command automatically detects macOS and handles building with vz support and signing with required entitlements. + +### Key Differences from Linux + +| Feature | Linux | macOS | +|---------|-------|-------| +| Hypervisors | Cloud Hypervisor, QEMU | vz (Virtualization.framework) | +| Networking | TAP devices, bridges, iptables | NAT (built-in, automatic) | +| Rate Limiting | HTB/tc | Not supported | +| GPU Passthrough | VFIO | Not supported | +| Disk Format | qcow2, raw | raw only | +| Snapshots | Always available | macOS 14+ ARM64 only | + +### Limitations + +- **Networking**: macOS uses NAT networking automatically. No manual bridge/TAP configuration needed, but ingress requires discovering the VM's NAT IP. +- **Rate Limiting**: Network and disk I/O rate limiting is not available on macOS. +- **GPU**: PCI device passthrough is not supported on macOS. +- **Disk Images**: qcow2 format is not directly supported; use raw disk images. +- **Snapshots**: Requires macOS 14.0+ on Apple Silicon (ARM64). + +For detailed development setup, see [DEVELOPMENT.md](DEVELOPMENT.md). + ## Development See [DEVELOPMENT.md](DEVELOPMENT.md) for build instructions, configuration options, and contributing guidelines. diff --git a/cmd/api/api/cp.go b/cmd/api/api/cp.go index 3b060d39..6bae53ed 100644 --- a/cmd/api/api/cp.go +++ b/cmd/api/api/cp.go @@ -11,7 +11,6 @@ import ( "github.com/gorilla/websocket" "github.com/kernel/hypeman/lib/guest" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/logger" mw "github.com/kernel/hypeman/lib/middleware" @@ -219,10 +218,9 @@ func (s *ApiService) CpHandler(w http.ResponseWriter, r *http.Request) { // handleCopyTo handles copying files from client to guest // Returns the number of bytes transferred and any error. func (s *ApiService) handleCopyTo(ctx context.Context, ws *websocket.Conn, inst *instances.Instance, req CpRequest) (int64, error) { - // Create vsock dialer for this hypervisor type - dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID) + dialer, err := s.InstanceManager.GetVsockDialer(ctx, inst.Id) if err != nil { - return 0, fmt.Errorf("create vsock dialer: %w", err) + return 0, fmt.Errorf("get vsock dialer: %w", err) } grpcConn, err := guest.GetOrCreateConn(ctx, dialer) @@ -329,10 +327,9 @@ func (s *ApiService) handleCopyTo(ctx context.Context, ws *websocket.Conn, inst // handleCopyFrom handles copying files from guest to client // Returns the number of bytes transferred and any error. func (s *ApiService) handleCopyFrom(ctx context.Context, ws *websocket.Conn, inst *instances.Instance, req CpRequest) (int64, error) { - // Create vsock dialer for this hypervisor type - dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID) + dialer, err := s.InstanceManager.GetVsockDialer(ctx, inst.Id) if err != nil { - return 0, fmt.Errorf("create vsock dialer: %w", err) + return 0, fmt.Errorf("get vsock dialer: %w", err) } grpcConn, err := guest.GetOrCreateConn(ctx, dialer) diff --git a/cmd/api/api/cp_test.go b/cmd/api/api/cp_test.go index 62e62ee2..22e6f515 100644 --- a/cmd/api/api/cp_test.go +++ b/cmd/api/api/cp_test.go @@ -19,7 +19,7 @@ import ( func TestCpToAndFromInstance(t *testing.T) { // Require KVM access for VM creation if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { - t.Fatal("/dev/kvm not available - ensure KVM is enabled and user is in 'kvm' group (sudo usermod -aG kvm $USER)") + t.Skip("/dev/kvm not available, skipping on this platform") } if testing.Short() { @@ -158,7 +158,7 @@ func TestCpToAndFromInstance(t *testing.T) { func TestCpDirectoryToInstance(t *testing.T) { // Require KVM access for VM creation if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { - t.Fatal("/dev/kvm not available - ensure KVM is enabled and user is in 'kvm' group (sudo usermod -aG kvm $USER)") + t.Skip("/dev/kvm not available, skipping on this platform") } if testing.Short() { diff --git a/cmd/api/api/exec.go b/cmd/api/api/exec.go index b9f5f3b3..b1e13c2c 100644 --- a/cmd/api/api/exec.go +++ b/cmd/api/api/exec.go @@ -12,7 +12,6 @@ import ( "github.com/gorilla/websocket" "github.com/kernel/hypeman/lib/guest" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/logger" mw "github.com/kernel/hypeman/lib/middleware" @@ -132,10 +131,9 @@ func (s *ApiService) ExecHandler(w http.ResponseWriter, r *http.Request) { // Create WebSocket read/writer wrapper that handles resize messages wsConn := &wsReadWriter{ws: ws, ctx: ctx, resizeChan: resizeChan} - // Create vsock dialer for this hypervisor type - dialer, err := hypervisor.NewVsockDialer(hypervisor.Type(inst.HypervisorType), inst.VsockSocket, inst.VsockCID) + dialer, err := s.InstanceManager.GetVsockDialer(ctx, inst.Id) if err != nil { - log.ErrorContext(ctx, "failed to create vsock dialer", "error", err) + log.ErrorContext(ctx, "failed to get vsock dialer", "error", err) ws.WriteMessage(websocket.BinaryMessage, []byte(fmt.Sprintf("Error: %v\r\n", err))) ws.WriteMessage(websocket.TextMessage, []byte(`{"exitCode":127}`)) return diff --git a/cmd/api/api/exec_test.go b/cmd/api/api/exec_test.go index be21bc5c..a08cb8b2 100644 --- a/cmd/api/api/exec_test.go +++ b/cmd/api/api/exec_test.go @@ -20,7 +20,7 @@ import ( func TestExecInstanceNonTTY(t *testing.T) { // Require KVM access for VM creation if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { - t.Fatal("/dev/kvm not available - ensure KVM is enabled and user is in 'kvm' group (sudo usermod -aG kvm $USER)") + t.Skip("/dev/kvm not available, skipping on this platform") } if testing.Short() { @@ -160,7 +160,7 @@ func TestExecInstanceNonTTY(t *testing.T) { func TestExecWithDebianMinimal(t *testing.T) { // Require KVM access for VM creation if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { - t.Fatal("/dev/kvm not available - ensure KVM is enabled and user is in 'kvm' group (sudo usermod -aG kvm $USER)") + t.Skip("/dev/kvm not available, skipping on this platform") } if testing.Short() { diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go index 86d1ff9e..5026646a 100644 --- a/cmd/api/api/images_test.go +++ b/cmd/api/api/images_test.go @@ -225,12 +225,13 @@ func TestCreateImage_Idempotent(t *testing.T) { t.Fatal("Build failed - this is the root cause of test failures") } - // Status can be "pending" (still processing) or "ready" (already completed in fast CI) + // Status can be "pending" (still queued), "pulling" (pull started), or "ready" (completed) // The key idempotency invariant is that the digest is the same (verified above) require.Contains(t, []oapi.ImageStatus{ oapi.ImageStatus(images.StatusPending), + oapi.ImageStatus(images.StatusPulling), oapi.ImageStatus(images.StatusReady), - }, img2.Status, "status should be pending or ready") + }, img2.Status, "status should be pending, pulling, or ready") // If still pending, should have queue position if img2.Status == oapi.ImageStatus(images.StatusPending) { diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index 62621dec..5907af10 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -592,13 +592,12 @@ func (s *ApiService) StatInstancePath(ctx context.Context, request oapi.StatInst }, nil } - // Create vsock dialer for this hypervisor type - dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID) + dialer, err := s.InstanceManager.GetVsockDialer(ctx, inst.Id) if err != nil { - log.ErrorContext(ctx, "failed to create vsock dialer", "error", err) + log.ErrorContext(ctx, "failed to get vsock dialer", "error", err) return oapi.StatInstancePath500JSONResponse{ Code: "internal_error", - Message: "failed to create vsock dialer", + Message: "failed to get vsock dialer", }, nil } diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 005241a4..81af2bd5 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -35,7 +35,7 @@ func TestGetInstance_NotFound(t *testing.T) { func TestCreateInstance_ParsesHumanReadableSizes(t *testing.T) { // Require KVM access for VM creation if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { - t.Fatal("/dev/kvm not available - ensure KVM is enabled and user is in 'kvm' group (sudo usermod -aG kvm $USER)") + t.Skip("/dev/kvm not available, skipping on this platform") } svc := newTestService(t) diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 79dcd0e5..12b22318 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -115,6 +115,7 @@ type Config struct { RegistryCACertFile string // Path to CA certificate file for registry TLS verification BuildTimeout int // Default build timeout in seconds BuildSecretsDir string // Directory containing build secrets (optional) + DockerSocket string // Path to Docker socket (for building builder image) // Hypervisor configuration DefaultHypervisor string // Default hypervisor type: "cloud-hypervisor" or "qemu" @@ -213,6 +214,7 @@ func Load() *Config { RegistryCACertFile: getEnv("REGISTRY_CA_CERT_FILE", ""), // Path to CA cert for registry TLS BuildTimeout: getEnvInt("BUILD_TIMEOUT", 600), BuildSecretsDir: getEnv("BUILD_SECRETS_DIR", ""), // Optional: path to directory with build secrets + DockerSocket: getEnv("DOCKER_SOCKET", "/var/run/docker.sock"), // Hypervisor configuration DefaultHypervisor: getEnv("DEFAULT_HYPERVISOR", "cloud-hypervisor"), diff --git a/cmd/api/hypervisor_check_darwin.go b/cmd/api/hypervisor_check_darwin.go new file mode 100644 index 00000000..51950cc2 --- /dev/null +++ b/cmd/api/hypervisor_check_darwin.go @@ -0,0 +1,31 @@ +//go:build darwin + +package main + +import ( + "fmt" + "runtime" + + "github.com/Code-Hex/vz/v3" +) + +// checkHypervisorAccess verifies Virtualization.framework is available on macOS +func checkHypervisorAccess() error { + if runtime.GOARCH != "arm64" { + return fmt.Errorf("Virtualization.framework on macOS requires Apple Silicon (arm64), got %s", runtime.GOARCH) + } + + // Validate virtualization is usable by attempting to get max CPU count + // This will fail if entitlements are missing or virtualization is not available + maxCPU := vz.VirtualMachineConfigurationMaximumAllowedCPUCount() + if maxCPU < 1 { + return fmt.Errorf("Virtualization.framework reports 0 max CPUs - check entitlements") + } + + return nil +} + +// hypervisorAccessCheckName returns the name of the hypervisor access check for logging +func hypervisorAccessCheckName() string { + return "Virtualization.framework" +} diff --git a/cmd/api/hypervisor_check_linux.go b/cmd/api/hypervisor_check_linux.go new file mode 100644 index 00000000..042e70ca --- /dev/null +++ b/cmd/api/hypervisor_check_linux.go @@ -0,0 +1,29 @@ +//go:build linux + +package main + +import ( + "fmt" + "os" +) + +// checkHypervisorAccess verifies KVM is available and the user has permission to use it +func checkHypervisorAccess() error { + f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("/dev/kvm not found - KVM not enabled or not supported") + } + if os.IsPermission(err) { + return fmt.Errorf("permission denied accessing /dev/kvm - user not in 'kvm' group") + } + return fmt.Errorf("cannot access /dev/kvm: %w", err) + } + f.Close() + return nil +} + +// hypervisorAccessCheckName returns the name of the hypervisor access check for logging +func hypervisorAccessCheckName() string { + return "KVM" +} diff --git a/cmd/api/main.go b/cmd/api/main.go index 7f5e4265..561a9f3c 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -130,11 +130,11 @@ func run() error { logger.Warn("JWT_SECRET not configured - API authentication will fail") } - // Verify KVM access (required for VM creation) - if err := checkKVMAccess(); err != nil { - return fmt.Errorf("KVM access check failed: %w\n\nEnsure:\n 1. KVM is enabled (check /dev/kvm exists)\n 2. User is in 'kvm' group: sudo usermod -aG kvm $USER\n 3. Log out and back in, or use: newgrp kvm", err) + // Verify hypervisor access (KVM on Linux, Virtualization.framework on macOS) + if err := checkHypervisorAccess(); err != nil { + return fmt.Errorf("hypervisor access check failed: %w", err) } - logger.Info("KVM access verified") + logger.Info("Hypervisor access verified", "type", hypervisorAccessCheckName()) // Check if QEMU is available (optional - only warn if not present) if _, err := (&qemu.Starter{}).GetBinaryPath(nil, ""); err != nil { @@ -465,18 +465,3 @@ func run() error { return err } -// checkKVMAccess verifies KVM is available and the user has permission to use it -func checkKVMAccess() error { - f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0) - if err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("/dev/kvm not found - KVM not enabled or not supported") - } - if os.IsPermission(err) { - return fmt.Errorf("permission denied accessing /dev/kvm - user not in 'kvm' group") - } - return fmt.Errorf("cannot access /dev/kvm: %w", err) - } - f.Close() - return nil -} diff --git a/cmd/gen-jwt/main.go b/cmd/gen-jwt/main.go index a14cd409..7fa2c03f 100644 --- a/cmd/gen-jwt/main.go +++ b/cmd/gen-jwt/main.go @@ -16,6 +16,7 @@ func main() { os.Exit(1) } userID := flag.String("user-id", "test-user", "User ID to include in the JWT token") + registryPush := flag.String("registry-push", "", "Repository to grant push access to (e.g., hypeman/builder)") flag.Parse() claims := jwt.MapClaims{ @@ -23,6 +24,14 @@ func main() { "iat": time.Now().Unix(), "exp": time.Now().Add(24 * time.Hour).Unix(), } + + // Add registry push permissions if requested + if *registryPush != "" { + claims["repo_access"] = []map[string]string{ + {"repo": *registryPush, "scope": "push"}, + } + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) tokenString, err := token.SignedString([]byte(jwtSecret)) if err != nil { diff --git a/cmd/vz-shim/main.go b/cmd/vz-shim/main.go new file mode 100644 index 00000000..f7ce1907 --- /dev/null +++ b/cmd/vz-shim/main.go @@ -0,0 +1,163 @@ +//go:build darwin + +// Package main implements hypeman-vz-shim, a subprocess that hosts vz VMs. +// This allows VMs to survive hypeman restarts by running in a separate process. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/Code-Hex/vz/v3" + "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" +) + +func main() { + configJSON := flag.String("config", "", "VM configuration as JSON") + flag.Parse() + + if *configJSON == "" { + fmt.Fprintln(os.Stderr, "error: -config is required") + os.Exit(1) + } + + var config shimconfig.ShimConfig + if err := json.Unmarshal([]byte(*configJSON), &config); err != nil { + fmt.Fprintf(os.Stderr, "error: invalid config JSON: %v\n", err) + os.Exit(1) + } + + // Setup logging to file + if err := setupLogging(config.LogPath); err != nil { + fmt.Fprintf(os.Stderr, "error: setup logging: %v\n", err) + os.Exit(1) + } + + slog.Info("vz-shim starting", "control_socket", config.ControlSocket, "vsock_socket", config.VsockSocket) + + // Create the VM + vm, vmConfig, err := createVM(config) + if err != nil { + slog.Error("failed to create VM", "error", err) + os.Exit(1) + } + + if err := vm.Start(); err != nil { + slog.Error("failed to start VM", "error", err) + os.Exit(1) + } + slog.Info("VM started", "vcpus", config.VCPUs, "memory_mb", config.MemoryBytes/1024/1024) + + // Create the shim server + server := NewShimServer(vm, vmConfig) + + // Start control socket listener (remove stale socket from previous run) + os.Remove(config.ControlSocket) + controlListener, err := net.Listen("unix", config.ControlSocket) + if err != nil { + slog.Error("failed to listen on control socket", "error", err, "path", config.ControlSocket) + os.Exit(1) + } + defer controlListener.Close() + + // Start vsock proxy listener (remove stale socket from previous run) + os.Remove(config.VsockSocket) + vsockListener, err := net.Listen("unix", config.VsockSocket) + if err != nil { + slog.Error("failed to listen on vsock socket", "error", err, "path", config.VsockSocket) + os.Exit(1) + } + defer vsockListener.Close() + + // Start HTTP server for control API + httpServer := &http.Server{Handler: server.Handler()} + go func() { + slog.Info("control API listening", "socket", config.ControlSocket) + if err := httpServer.Serve(controlListener); err != nil && err != http.ErrServerClosed { + slog.Error("control API server error", "error", err) + } + }() + + // Start vsock proxy + go func() { + slog.Info("vsock proxy listening", "socket", config.VsockSocket) + server.ServeVsock(vsockListener) + }() + + // Wait for shutdown signal or VM stop + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT) + + // Monitor VM state + stateChanged := vm.StateChangedNotify() + go func() { + for { + select { + case <-ctx.Done(): + return + case newState := <-stateChanged: + slog.Info("VM state changed", "state", newState) + if newState == vz.VirtualMachineStateStopped || newState == vz.VirtualMachineStateError { + slog.Info("VM stopped, shutting down shim") + cancel() + return + } + } + } + }() + + select { + case sig := <-sigChan: + slog.Info("received signal, shutting down", "signal", sig) + case <-ctx.Done(): + slog.Info("context cancelled, shutting down") + } + + // Graceful shutdown + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer shutdownCancel() + + httpServer.Shutdown(shutdownCtx) + + if vm.State() == vz.VirtualMachineStateRunning { + slog.Info("stopping VM") + if vm.CanStop() { + vm.Stop() + } + } + + slog.Info("vz-shim shutdown complete") +} + +func setupLogging(logPath string) error { + if logPath == "" { + // Log to stderr if no path specified + slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}))) + return nil + } + + if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { + return fmt.Errorf("create log directory: %w", err) + } + + file, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + return fmt.Errorf("open log file: %w", err) + } + + slog.SetDefault(slog.New(slog.NewJSONHandler(file, &slog.HandlerOptions{Level: slog.LevelDebug}))) + return nil +} diff --git a/cmd/vz-shim/server.go b/cmd/vz-shim/server.go new file mode 100644 index 00000000..029827e7 --- /dev/null +++ b/cmd/vz-shim/server.go @@ -0,0 +1,260 @@ +//go:build darwin + +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "sync" + + "github.com/Code-Hex/vz/v3" +) + +// ShimServer handles control API and vsock proxy for a vz VM. +type ShimServer struct { + vm *vz.VirtualMachine + vmConfig *vz.VirtualMachineConfiguration + mu sync.RWMutex +} + +// NewShimServer creates a new shim server. +func NewShimServer(vm *vz.VirtualMachine, vmConfig *vz.VirtualMachineConfiguration) *ShimServer { + return &ShimServer{ + vm: vm, + vmConfig: vmConfig, + } +} + +// VMInfoResponse matches the cloud-hypervisor VmInfo structure. +type VMInfoResponse struct { + State string `json:"state"` +} + +// Handler returns the HTTP handler for the control API. +func (s *ShimServer) Handler() http.Handler { + mux := http.NewServeMux() + + // Match cloud-hypervisor API patterns + mux.HandleFunc("GET /api/v1/vm.info", s.handleVMInfo) + mux.HandleFunc("PUT /api/v1/vm.pause", s.handlePause) + mux.HandleFunc("PUT /api/v1/vm.resume", s.handleResume) + mux.HandleFunc("PUT /api/v1/vm.shutdown", s.handleShutdown) + mux.HandleFunc("PUT /api/v1/vm.power-button", s.handlePowerButton) + mux.HandleFunc("GET /api/v1/vmm.ping", s.handlePing) + mux.HandleFunc("PUT /api/v1/vmm.shutdown", s.handleVMMShutdown) + + return mux +} + +func (s *ShimServer) handleVMInfo(w http.ResponseWriter, r *http.Request) { + s.mu.RLock() + defer s.mu.RUnlock() + + state := vzStateToString(s.vm.State()) + resp := VMInfoResponse{State: state} + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) +} + +func (s *ShimServer) handlePause(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + if !s.vm.CanPause() { + http.Error(w, "cannot pause VM", http.StatusBadRequest) + return + } + + if err := s.vm.Pause(); err != nil { + slog.Error("failed to pause VM", "error", err) + http.Error(w, fmt.Sprintf("pause failed: %v", err), http.StatusInternalServerError) + return + } + + slog.Info("VM paused") + w.WriteHeader(http.StatusNoContent) +} + +func (s *ShimServer) handleResume(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + if !s.vm.CanResume() { + http.Error(w, "cannot resume VM", http.StatusBadRequest) + return + } + + if err := s.vm.Resume(); err != nil { + slog.Error("failed to resume VM", "error", err) + http.Error(w, fmt.Sprintf("resume failed: %v", err), http.StatusInternalServerError) + return + } + + slog.Info("VM resumed") + w.WriteHeader(http.StatusNoContent) +} + +func (s *ShimServer) handleShutdown(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + // Request graceful shutdown via guest + success, err := s.vm.RequestStop() + if err != nil || !success { + slog.Warn("RequestStop failed, trying Stop", "error", err) + if s.vm.CanStop() { + if err := s.vm.Stop(); err != nil { + slog.Error("failed to stop VM", "error", err) + http.Error(w, fmt.Sprintf("shutdown failed: %v", err), http.StatusInternalServerError) + return + } + } + } + + slog.Info("VM shutdown requested") + w.WriteHeader(http.StatusNoContent) +} + +func (s *ShimServer) handlePowerButton(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + // RequestStop sends an ACPI power button event + success, err := s.vm.RequestStop() + if err != nil || !success { + slog.Error("failed to send power button", "error", err, "success", success) + http.Error(w, fmt.Sprintf("power button failed: %v", err), http.StatusInternalServerError) + return + } + + slog.Info("power button sent") + w.WriteHeader(http.StatusNoContent) +} + +func (s *ShimServer) handlePing(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) +} + +func (s *ShimServer) handleVMMShutdown(w http.ResponseWriter, r *http.Request) { + slog.Info("VMM shutdown requested") + w.WriteHeader(http.StatusNoContent) + + // Stop the VM and exit + go func() { + s.mu.Lock() + defer s.mu.Unlock() + + if s.vm.CanStop() { + s.vm.Stop() + } + // Process will exit when VM stops (monitored in main) + }() +} + +func vzStateToString(state vz.VirtualMachineState) string { + switch state { + case vz.VirtualMachineStateStopped: + return "Shutdown" + case vz.VirtualMachineStateRunning: + return "Running" + case vz.VirtualMachineStatePaused: + return "Paused" + case vz.VirtualMachineStateError: + return "Error" + case vz.VirtualMachineStateStarting: + return "Starting" + case vz.VirtualMachineStatePausing: + return "Pausing" + case vz.VirtualMachineStateResuming: + return "Resuming" + case vz.VirtualMachineStateStopping: + return "Stopping" + default: + return "Unknown" + } +} + +// ServeVsock handles vsock proxy connections using the Cloud Hypervisor protocol. +// Protocol: Client sends "CONNECT {port}\n", server responds "OK {port}\n", then proxies. +func (s *ShimServer) ServeVsock(listener net.Listener) { + for { + conn, err := listener.Accept() + if err != nil { + slog.Debug("vsock listener closed", "error", err) + return + } + go s.handleVsockConnection(conn) + } +} + +func (s *ShimServer) handleVsockConnection(conn net.Conn) { + defer conn.Close() + + // Read the CONNECT command + reader := bufio.NewReader(conn) + cmd, err := reader.ReadString('\n') + if err != nil { + slog.Error("failed to read vsock handshake", "error", err) + return + } + + // Parse "CONNECT {port}\n" + var port uint32 + if _, err := fmt.Sscanf(cmd, "CONNECT %d\n", &port); err != nil { + slog.Error("invalid vsock handshake", "cmd", cmd, "error", err) + conn.Write([]byte(fmt.Sprintf("ERR invalid command: %s", cmd))) + return + } + + slog.Debug("vsock connect request", "port", port) + + // Get vsock device and connect to guest + s.mu.RLock() + socketDevices := s.vm.SocketDevices() + s.mu.RUnlock() + + if len(socketDevices) == 0 { + slog.Error("no vsock device configured") + conn.Write([]byte("ERR no vsock device\n")) + return + } + + guestConn, err := socketDevices[0].Connect(port) + if err != nil { + slog.Error("failed to connect to guest vsock", "port", port, "error", err) + conn.Write([]byte(fmt.Sprintf("ERR connect failed: %v\n", err))) + return + } + defer guestConn.Close() + + // Send OK response (matching CH protocol) + if _, err := conn.Write([]byte(fmt.Sprintf("OK %d\n", port))); err != nil { + slog.Error("failed to send OK response", "error", err) + return + } + + slog.Debug("vsock connection established", "port", port) + + // Proxy data bidirectionally + done := make(chan struct{}, 2) + + go func() { + io.Copy(guestConn, conn) + done <- struct{}{} + }() + + go func() { + io.Copy(conn, guestConn) + done <- struct{}{} + }() + + // Wait for one direction to close + <-done +} diff --git a/cmd/vz-shim/vm.go b/cmd/vz-shim/vm.go new file mode 100644 index 00000000..2c6806ac --- /dev/null +++ b/cmd/vz-shim/vm.go @@ -0,0 +1,276 @@ +//go:build darwin + +package main + +import ( + "fmt" + "log/slog" + "net" + "os" + "runtime" + "strings" + + "github.com/Code-Hex/vz/v3" + "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" +) + +// createVM creates and configures a vz.VirtualMachine from ShimConfig. +func createVM(config shimconfig.ShimConfig) (*vz.VirtualMachine, *vz.VirtualMachineConfiguration, error) { + // Prepare kernel command line (vz uses hvc0 for serial console) + kernelArgs := config.KernelArgs + if kernelArgs == "" { + kernelArgs = "console=hvc0 root=/dev/vda" + } else { + kernelArgs = strings.ReplaceAll(kernelArgs, "console=ttyS0", "console=hvc0") + } + + bootLoader, err := vz.NewLinuxBootLoader( + config.KernelPath, + vz.WithCommandLine(kernelArgs), + vz.WithInitrd(config.InitrdPath), + ) + if err != nil { + return nil, nil, fmt.Errorf("create boot loader: %w", err) + } + + vcpus := computeCPUCount(config.VCPUs) + memoryBytes := computeMemorySize(uint64(config.MemoryBytes)) + + slog.Debug("VM config", "vcpus", vcpus, "memory_bytes", memoryBytes, "kernel", config.KernelPath, "initrd", config.InitrdPath) + + vmConfig, err := vz.NewVirtualMachineConfiguration(bootLoader, vcpus, memoryBytes) + if err != nil { + return nil, nil, fmt.Errorf("create vm configuration: %w", err) + } + + if err := configureSerialConsole(vmConfig, config.SerialLogPath); err != nil { + return nil, nil, fmt.Errorf("configure serial: %w", err) + } + + if err := configureNetwork(vmConfig, config.Networks); err != nil { + return nil, nil, fmt.Errorf("configure network: %w", err) + } + + entropyConfig, err := vz.NewVirtioEntropyDeviceConfiguration() + if err != nil { + return nil, nil, fmt.Errorf("create entropy device: %w", err) + } + vmConfig.SetEntropyDevicesVirtualMachineConfiguration([]*vz.VirtioEntropyDeviceConfiguration{entropyConfig}) + + if err := configureStorage(vmConfig, config.Disks); err != nil { + return nil, nil, fmt.Errorf("configure storage: %w", err) + } + + vsockConfig, err := vz.NewVirtioSocketDeviceConfiguration() + if err != nil { + return nil, nil, fmt.Errorf("create vsock device: %w", err) + } + vmConfig.SetSocketDevicesVirtualMachineConfiguration([]vz.SocketDeviceConfiguration{vsockConfig}) + + if balloonConfig, err := vz.NewVirtioTraditionalMemoryBalloonDeviceConfiguration(); err == nil { + vmConfig.SetMemoryBalloonDevicesVirtualMachineConfiguration([]vz.MemoryBalloonDeviceConfiguration{balloonConfig}) + } + + if validated, err := vmConfig.Validate(); !validated || err != nil { + return nil, nil, fmt.Errorf("invalid vm configuration: %w", err) + } + + vm, err := vz.NewVirtualMachine(vmConfig) + if err != nil { + return nil, nil, fmt.Errorf("create virtual machine: %w", err) + } + + return vm, vmConfig, nil +} + +func configureSerialConsole(vmConfig *vz.VirtualMachineConfiguration, logPath string) error { + var serialAttachment *vz.FileHandleSerialPortAttachment + + nullRead, err := os.OpenFile("/dev/null", os.O_RDONLY, 0) + if err != nil { + return fmt.Errorf("open /dev/null for reading: %w", err) + } + + if logPath != "" { + file, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + nullRead.Close() + return fmt.Errorf("open serial log file: %w", err) + } + serialAttachment, err = vz.NewFileHandleSerialPortAttachment(nullRead, file) + if err != nil { + nullRead.Close() + file.Close() + return fmt.Errorf("create serial attachment: %w", err) + } + } else { + nullWrite, err := os.OpenFile("/dev/null", os.O_WRONLY, 0) + if err != nil { + nullRead.Close() + return fmt.Errorf("open /dev/null for writing: %w", err) + } + serialAttachment, err = vz.NewFileHandleSerialPortAttachment(nullRead, nullWrite) + if err != nil { + nullRead.Close() + nullWrite.Close() + return fmt.Errorf("create serial attachment: %w", err) + } + } + + consoleConfig, err := vz.NewVirtioConsoleDeviceSerialPortConfiguration(serialAttachment) + if err != nil { + return fmt.Errorf("create console config: %w", err) + } + vmConfig.SetSerialPortsVirtualMachineConfiguration([]*vz.VirtioConsoleDeviceSerialPortConfiguration{ + consoleConfig, + }) + + return nil +} + +func configureNetwork(vmConfig *vz.VirtualMachineConfiguration, networks []shimconfig.NetworkConfig) error { + var devices []*vz.VirtioNetworkDeviceConfiguration + if len(networks) == 0 { + dev, err := createNATNetworkDevice("") + if err != nil { + return err + } + devices = append(devices, dev) + } else { + for _, netConfig := range networks { + dev, err := createNATNetworkDevice(netConfig.MAC) + if err != nil { + return err + } + devices = append(devices, dev) + } + } + vmConfig.SetNetworkDevicesVirtualMachineConfiguration(devices) + return nil +} + +func createNATNetworkDevice(macAddr string) (*vz.VirtioNetworkDeviceConfiguration, error) { + natAttachment, err := vz.NewNATNetworkDeviceAttachment() + if err != nil { + return nil, fmt.Errorf("create NAT attachment: %w", err) + } + + networkConfig, err := vz.NewVirtioNetworkDeviceConfiguration(natAttachment) + if err != nil { + return nil, fmt.Errorf("create network config: %w", err) + } + + mac, err := assignMACAddress(macAddr) + if err != nil { + return nil, err + } + networkConfig.SetMACAddress(mac) + + return networkConfig, nil +} + +func assignMACAddress(macAddr string) (*vz.MACAddress, error) { + if macAddr == "" { + mac, err := vz.NewRandomLocallyAdministeredMACAddress() + if err != nil { + return nil, fmt.Errorf("generate MAC address: %w", err) + } + slog.Info("generated random MAC address", "mac", mac.String()) + return mac, nil + } + + hwAddr, err := net.ParseMAC(macAddr) + if err != nil { + slog.Warn("failed to parse MAC address, generating random", "mac", macAddr, "error", err) + mac, err := vz.NewRandomLocallyAdministeredMACAddress() + if err != nil { + return nil, fmt.Errorf("generate MAC address: %w", err) + } + return mac, nil + } + + mac, err := vz.NewMACAddress(hwAddr) + if err != nil { + slog.Warn("failed to create MAC from parsed address, generating random", "mac", macAddr, "error", err) + mac, err := vz.NewRandomLocallyAdministeredMACAddress() + if err != nil { + return nil, fmt.Errorf("generate MAC address: %w", err) + } + return mac, nil + } + + slog.Info("using specified MAC address", "mac", macAddr) + return mac, nil +} + +func configureStorage(vmConfig *vz.VirtualMachineConfiguration, disks []shimconfig.DiskConfig) error { + var storageDevices []vz.StorageDeviceConfiguration + + for _, disk := range disks { + if _, err := os.Stat(disk.Path); os.IsNotExist(err) { + return fmt.Errorf("disk image not found: %s", disk.Path) + } + + if strings.HasSuffix(disk.Path, ".qcow2") { + return fmt.Errorf("qcow2 not supported by vz, use raw format: %s", disk.Path) + } + + attachment, err := vz.NewDiskImageStorageDeviceAttachment(disk.Path, disk.Readonly) + if err != nil { + return fmt.Errorf("create disk attachment for %s: %w", disk.Path, err) + } + + blockConfig, err := vz.NewVirtioBlockDeviceConfiguration(attachment) + if err != nil { + return fmt.Errorf("create block device config: %w", err) + } + + storageDevices = append(storageDevices, blockConfig) + } + + if len(storageDevices) > 0 { + vmConfig.SetStorageDevicesVirtualMachineConfiguration(storageDevices) + } + + return nil +} + +func computeCPUCount(requested int) uint { + virtualCPUCount := uint(requested) + if virtualCPUCount == 0 { + virtualCPUCount = uint(runtime.NumCPU() - 1) + if virtualCPUCount < 1 { + virtualCPUCount = 1 + } + } + + maxAllowed := vz.VirtualMachineConfigurationMaximumAllowedCPUCount() + minAllowed := vz.VirtualMachineConfigurationMinimumAllowedCPUCount() + + if virtualCPUCount > maxAllowed { + virtualCPUCount = maxAllowed + } + if virtualCPUCount < minAllowed { + virtualCPUCount = minAllowed + } + + return virtualCPUCount +} + +func computeMemorySize(requested uint64) uint64 { + if requested == 0 { + requested = 2 * 1024 * 1024 * 1024 // 2GB default + } + + maxAllowed := vz.VirtualMachineConfigurationMaximumAllowedMemorySize() + minAllowed := vz.VirtualMachineConfigurationMinimumAllowedMemorySize() + + if requested > maxAllowed { + requested = maxAllowed + } + if requested < minAllowed { + requested = minAllowed + } + + return requested +} diff --git a/go.mod b/go.mod index 16a40d39..5102f359 100644 --- a/go.mod +++ b/go.mod @@ -56,6 +56,8 @@ require ( require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/Code-Hex/go-infinity-channel v1.0.0 // indirect + github.com/Code-Hex/vz/v3 v3.7.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/apex/log v1.9.0 // indirect diff --git a/go.sum b/go.sum index 6fd5278f..1b933d0b 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,10 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8af github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Code-Hex/go-infinity-channel v1.0.0 h1:M8BWlfDOxq9or9yvF9+YkceoTkDI1pFAqvnP87Zh0Nw= +github.com/Code-Hex/go-infinity-channel v1.0.0/go.mod h1:5yUVg/Fqao9dAjcpzoQ33WwfdMWmISOrQloDRn3bsvY= +github.com/Code-Hex/vz/v3 v3.7.1 h1:EN1yNiyrbPq+dl388nne2NySo8I94EnPppvqypA65XM= +github.com/Code-Hex/vz/v3 v3.7.1/go.mod h1:1LsW0jqW0r0cQ+IeR4hHbjdqOtSidNCVMWhStMHGho8= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= diff --git a/lib/builds/builder_agent/main.go b/lib/builds/builder_agent/main.go index beb5b182..3212fee6 100644 --- a/lib/builds/builder_agent/main.go +++ b/lib/builds/builder_agent/main.go @@ -103,8 +103,63 @@ var ( // Encoder lock protects concurrent access to json.Encoder // (the goroutine sending build_result and the main loop handling get_status) encoderLock sync.Mutex + + // Log streaming channel - logs are sent here and forwarded to host via vsock + logChan = make(chan string, 1000) + logChanOnce sync.Once ) +// streamingLogWriter writes log lines to a channel for streaming to the host. +// It also writes to a buffer to include all logs in the final result. +type streamingLogWriter struct { + buffer *bytes.Buffer + mu sync.Mutex + closed bool + closedMu sync.RWMutex +} + +func newStreamingLogWriter() *streamingLogWriter { + return &streamingLogWriter{ + buffer: &bytes.Buffer{}, + } +} + +func (w *streamingLogWriter) Write(p []byte) (n int, err error) { + w.mu.Lock() + w.buffer.Write(p) + w.mu.Unlock() + + // Hold RLock through the send to prevent markClosed()+close(logChan) + // from racing between the check and the channel send. + w.closedMu.RLock() + if !w.closed { + line := string(p) + select { + case logChan <- line: + default: + // Channel full, drop the log line for streaming but it's still in buffer + } + } + w.closedMu.RUnlock() + + // Also write to stdout for local debugging + os.Stdout.Write(p) + + return len(p), nil +} + +func (w *streamingLogWriter) markClosed() { + w.closedMu.Lock() + w.closed = true + w.closedMu.Unlock() +} + +func (w *streamingLogWriter) String() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.buffer.String() +} + func main() { log.Println("=== Builder Agent Starting ===") @@ -212,6 +267,19 @@ func handleHostConnection(conn net.Conn) { close(secretsReady) }) + // Start streaming logs to host + go func() { + for logLine := range logChan { + encoderLock.Lock() + err := encoder.Encode(VsockMessage{Type: "log", Log: logLine}) + encoderLock.Unlock() + if err != nil { + // Connection closed, stop streaming + return + } + } + }() + // Wait for build to complete and send result to host go func() { <-buildDone @@ -341,12 +409,17 @@ func handleSecretsRequest(encoder *json.Encoder, decoder *json.Decoder) error { // runBuildProcess runs the actual build and stores the result func runBuildProcess() { start := time.Now() - var logs bytes.Buffer - logWriter := io.MultiWriter(os.Stdout, &logs) + logWriter := newStreamingLogWriter() log.SetOutput(logWriter) defer func() { + // Mark writer as closed first to prevent writes to closed channel + logWriter.markClosed() + // Close log channel so streaming goroutine terminates + logChanOnce.Do(func() { + close(logChan) + }) close(buildDone) }() @@ -356,7 +429,7 @@ func runBuildProcess() { setResult(BuildResult{ Success: false, Error: fmt.Sprintf("load config: %v", err), - Logs: logs.String(), + Logs: logWriter.String(), DurationMS: time.Since(start).Milliseconds(), }) return @@ -373,7 +446,7 @@ func runBuildProcess() { setResult(BuildResult{ Success: false, Error: fmt.Sprintf("setup registry auth: %v", err), - Logs: logs.String(), + Logs: logWriter.String(), DurationMS: time.Since(start).Milliseconds(), }) return @@ -403,7 +476,7 @@ func runBuildProcess() { setResult(BuildResult{ Success: false, Error: "build timeout while waiting for secrets", - Logs: logs.String(), + Logs: logWriter.String(), DurationMS: time.Since(start).Milliseconds(), }) return @@ -418,7 +491,7 @@ func runBuildProcess() { setResult(BuildResult{ Success: false, Error: "Dockerfile required: provide dockerfile parameter or include Dockerfile in source tarball", - Logs: logs.String(), + Logs: logWriter.String(), DurationMS: time.Since(start).Milliseconds(), }) return @@ -428,7 +501,7 @@ func runBuildProcess() { setResult(BuildResult{ Success: false, Error: fmt.Sprintf("write dockerfile: %v", err), - Logs: logs.String(), + Logs: logWriter.String(), DurationMS: time.Since(start).Milliseconds(), }) return @@ -443,8 +516,8 @@ func runBuildProcess() { // Run the build log.Println("=== Starting Build ===") - digest, buildLogs, err := runBuild(ctx, config, logWriter) - logs.WriteString(buildLogs) + digest, _, err := runBuild(ctx, config, logWriter) + // Note: buildLogs is already written to logWriter via io.MultiWriter in runBuild duration := time.Since(start).Milliseconds() @@ -452,7 +525,7 @@ func runBuildProcess() { setResult(BuildResult{ Success: false, Error: err.Error(), - Logs: logs.String(), + Logs: logWriter.String(), Provenance: provenance, DurationMS: duration, }) @@ -466,7 +539,7 @@ func runBuildProcess() { setResult(BuildResult{ Success: true, ImageDigest: digest, - Logs: logs.String(), + Logs: logWriter.String(), Provenance: provenance, DurationMS: duration, }) diff --git a/lib/builds/manager.go b/lib/builds/manager.go index 3a612baa..b4eb6097 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -12,6 +12,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" "github.com/nrednav/cuid2" @@ -77,6 +78,9 @@ type Config struct { // RegistrySecret is the secret used to sign registry access tokens // This should be the same secret used by the registry middleware RegistrySecret string + + // DockerSocket is the path to the Docker socket for building the builder image + DockerSocket string } // DefaultConfig returns the default build manager configuration @@ -113,6 +117,7 @@ type manager struct { logger *slog.Logger metrics *Metrics createMu sync.Mutex + builderReady atomic.Bool // Status subscription system for SSE streaming statusSubscribers map[string][]chan BuildEvent @@ -164,13 +169,72 @@ func NewManager( // Start starts the build manager's background services func (m *manager) Start(ctx context.Context) error { - // Note: We no longer use a global vsock listener. - // Instead, we connect TO each builder VM's vsock socket directly. - // This follows the Cloud Hypervisor vsock pattern where host initiates connections. + go m.ensureBuilderImage(ctx) m.logger.Info("build manager started") return nil } +// ensureBuilderImage ensures the builder image is available in the registry. +// If BUILDER_IMAGE is unset/empty, it builds from the embedded Dockerfile. +// If BUILDER_IMAGE is set, it checks if the image exists. +// This runs in a background goroutine during startup. +func (m *manager) ensureBuilderImage(ctx context.Context) { + defer m.builderReady.Store(true) + + builderImage := m.config.BuilderImage + if builderImage == "" { + builderImage = "hypeman/builder:latest" + } + + // Check if image already exists in the registry + registryHost := stripRegistryScheme(m.config.RegistryURL) + imageRef := fmt.Sprintf("%s/%s", registryHost, builderImage) + if _, err := m.imageManager.GetImage(ctx, imageRef); err == nil { + m.logger.Info("builder image already available", "image", imageRef) + return + } + + // Try to build the image using Docker + dockerSocket := m.config.DockerSocket + if dockerSocket == "" { + dockerSocket = "/var/run/docker.sock" + } + + // Check if Docker socket exists + if _, err := os.Stat(dockerSocket); err != nil { + m.logger.Warn("Docker socket not found, skipping builder image build", + "socket", dockerSocket, + "error", err) + return + } + + m.logger.Info("building builder image", "image", builderImage) + + // Find the Dockerfile - look relative to the binary or in common locations + dockerfilePath := "lib/builds/images/generic/Dockerfile" + if _, err := os.Stat(dockerfilePath); err != nil { + // Try relative to executable + if execPath, err := os.Executable(); err == nil { + altPath := filepath.Join(filepath.Dir(execPath), "..", dockerfilePath) + if _, err := os.Stat(altPath); err == nil { + dockerfilePath = altPath + } + } + } + + cmd := exec.CommandContext(ctx, "docker", "build", "-t", builderImage, "-f", dockerfilePath, ".") + cmd.Env = append(os.Environ(), fmt.Sprintf("DOCKER_HOST=unix://%s", dockerSocket)) + output, err := cmd.CombinedOutput() + if err != nil { + m.logger.Warn("failed to build builder image", + "error", err, + "output", string(output)) + return + } + + m.logger.Info("builder image built successfully", "image", builderImage) +} + // CreateBuild starts a new build job func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourceData []byte) (*Build, error) { m.logger.Info("creating build") @@ -331,12 +395,9 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques return } - // Save build logs (regardless of success/failure) - if result.Logs != "" { - if err := appendLog(m.paths, id, []byte(result.Logs)); err != nil { - m.logger.Warn("failed to save build logs", "id", id, "error", err) - } - } + // Note: Logs are now streamed via vsock "log" messages and written incrementally + // in waitForResult, so we no longer need to save them here. + // The result.Logs field is kept for backward compatibility but is redundant. if !result.Success { m.logger.Error("build failed", "id", id, "error", result.Error, "duration", duration) @@ -387,6 +448,10 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques // executeBuild runs the build in a builder VM func (m *manager) executeBuild(ctx context.Context, id string, req CreateBuildRequest, policy *BuildPolicy) (*BuildResult, error) { + if !m.builderReady.Load() { + return nil, fmt.Errorf("builder image is being prepared, please retry shortly") + } + // Create a volume with the source data sourceVolID := fmt.Sprintf("build-source-%s", id) sourcePath := m.paths.BuildSourceDir(id) + "/source.tar.gz" @@ -480,7 +545,7 @@ func (m *manager) executeBuild(ctx context.Context, id string, req CreateBuildRe // Wait for build result via vsock // The builder agent will send the result when complete - result, err := m.waitForResult(ctx, inst) + result, err := m.waitForResult(ctx, id, inst) if err != nil { return nil, fmt.Errorf("wait for result: %w", err) } @@ -489,7 +554,7 @@ func (m *manager) executeBuild(ctx context.Context, id string, req CreateBuildRe } // waitForResult waits for the build result from the builder agent via vsock -func (m *manager) waitForResult(ctx context.Context, inst *instances.Instance) (*BuildResult, error) { +func (m *manager) waitForResult(ctx context.Context, buildID string, inst *instances.Instance) (*BuildResult, error) { // Wait a bit for the VM to start and the builder agent to listen on vsock time.Sleep(3 * time.Second) @@ -504,9 +569,14 @@ func (m *manager) waitForResult(ctx context.Context, inst *instances.Instance) ( default: } - conn, err = m.dialBuilderVsock(inst.VsockSocket) - if err == nil { - break + dialer, dialerErr := m.instanceManager.GetVsockDialer(ctx, inst.Id) + if dialerErr == nil { + conn, err = dialer.DialVsock(ctx, BuildAgentVsockPort) + if err == nil { + break + } + } else { + err = dialerErr } m.logger.Debug("waiting for builder agent", "attempt", attempt+1, "error", err) @@ -590,6 +660,14 @@ func (m *manager) waitForResult(ctx context.Context, inst *instances.Instance) ( } m.logger.Info("sent secrets to agent", "count", len(secrets), "instance", inst.Id) + case "log": + // Stream log line to build log file immediately + if dr.response.Log != "" { + if err := appendLog(m.paths, buildID, []byte(dr.response.Log)); err != nil { + m.logger.Error("failed to append streamed log", "error", err, "build_id", buildID) + } + } + case "build_result": // Build completed if dr.response.Result == nil { @@ -603,62 +681,6 @@ func (m *manager) waitForResult(ctx context.Context, inst *instances.Instance) ( } } -// dialBuilderVsock connects to a builder VM's vsock socket using Cloud Hypervisor's handshake -func (m *manager) dialBuilderVsock(vsockSocketPath string) (net.Conn, error) { - // Connect to the Cloud Hypervisor vsock Unix socket - conn, err := net.DialTimeout("unix", vsockSocketPath, 5*time.Second) - if err != nil { - return nil, fmt.Errorf("dial vsock socket %s: %w", vsockSocketPath, err) - } - - // Set deadline for handshake - if err := conn.SetDeadline(time.Now().Add(5 * time.Second)); err != nil { - conn.Close() - return nil, fmt.Errorf("set handshake deadline: %w", err) - } - - // Perform Cloud Hypervisor vsock handshake - // Format: "CONNECT \n" -> "OK \n" - handshakeCmd := fmt.Sprintf("CONNECT %d\n", BuildAgentVsockPort) - if _, err := conn.Write([]byte(handshakeCmd)); err != nil { - conn.Close() - return nil, fmt.Errorf("send vsock handshake: %w", err) - } - - // Read handshake response - reader := bufio.NewReader(conn) - response, err := reader.ReadString('\n') - if err != nil { - conn.Close() - return nil, fmt.Errorf("read vsock handshake response: %w", err) - } - - // Clear deadline after successful handshake - if err := conn.SetDeadline(time.Time{}); err != nil { - conn.Close() - return nil, fmt.Errorf("clear deadline: %w", err) - } - - response = strings.TrimSpace(response) - if !strings.HasPrefix(response, "OK ") { - conn.Close() - return nil, fmt.Errorf("vsock handshake failed: %s", response) - } - - return &bufferedConn{Conn: conn, reader: reader}, nil -} - -// bufferedConn wraps a net.Conn with a bufio.Reader to ensure any buffered -// data from the handshake is properly drained before reading from the connection -type bufferedConn struct { - net.Conn - reader *bufio.Reader -} - -func (c *bufferedConn) Read(p []byte) (int, error) { - return c.reader.Read(p) -} - // updateStatus updates the build status func (m *manager) updateStatus(id string, status string, err error) { meta, readErr := readMetadata(m.paths, id) diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index 5a9e82cc..fdbf60a1 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/paths" @@ -130,6 +131,10 @@ func (m *mockInstanceManager) SetResourceValidator(v instances.ResourceValidator // no-op for mock } +func (m *mockInstanceManager) GetVsockDialer(ctx context.Context, instanceID string) (hypervisor.VsockDialer, error) { + return nil, nil +} + // mockVolumeManager implements volumes.Manager for testing type mockVolumeManager struct { volumes map[string]*volumes.Volume @@ -350,6 +355,7 @@ func setupTestManagerWithImageMgr(t *testing.T) (*manager, *mockInstanceManager, logger: logger, statusSubscribers: make(map[string][]chan BuildEvent), } + mgr.builderReady.Store(true) return mgr, instanceMgr, volumeMgr, imageMgr, tempDir } @@ -881,7 +887,7 @@ func TestStreamBuildEvents_WithStatusUpdate(t *testing.T) { // Read events until we see the initial log var foundInitialLog bool - timeout := time.After(2 * time.Second) + timeout := time.After(10 * time.Second) eventLoop: for !foundInitialLog { select { @@ -901,7 +907,7 @@ eventLoop: // Should receive "ready" status event and channel should close var readyReceived bool - timeout = time.After(2 * time.Second) + timeout = time.After(10 * time.Second) for !readyReceived { select { case event, ok := <-eventChan: @@ -942,7 +948,7 @@ func TestStreamBuildEvents_ContextCancellation(t *testing.T) { // Read events until we see the log line var foundLogLine bool - timeout := time.After(2 * time.Second) + timeout := time.After(10 * time.Second) eventLoop: for !foundLogLine { select { @@ -961,7 +967,7 @@ eventLoop: cancel() // Channel should close - timeout = time.After(2 * time.Second) + timeout = time.After(10 * time.Second) for { select { case _, ok := <-eventChan: diff --git a/lib/devices/discovery_darwin.go b/lib/devices/discovery_darwin.go new file mode 100644 index 00000000..219ba963 --- /dev/null +++ b/lib/devices/discovery_darwin.go @@ -0,0 +1,52 @@ +//go:build darwin + +package devices + +import ( + "fmt" +) + +// ErrNotSupportedOnMacOS is returned for operations not supported on macOS +var ErrNotSupportedOnMacOS = fmt.Errorf("PCI device passthrough is not supported on macOS") + +// ValidatePCIAddress validates that a string is a valid PCI address format. +// On macOS, this always returns false as PCI passthrough is not supported. +func ValidatePCIAddress(addr string) bool { + return false +} + +// DiscoverAvailableDevices returns an empty list on macOS. +// PCI device passthrough is not supported on macOS. +func DiscoverAvailableDevices() ([]AvailableDevice, error) { + return []AvailableDevice{}, nil +} + +// GetDeviceInfo returns an error on macOS as PCI passthrough is not supported. +func GetDeviceInfo(pciAddress string) (*AvailableDevice, error) { + return nil, ErrNotSupportedOnMacOS +} + +// GetIOMMUGroupDevices returns an error on macOS as IOMMU is not available. +func GetIOMMUGroupDevices(iommuGroup int) ([]string, error) { + return nil, ErrNotSupportedOnMacOS +} + +// DetermineDeviceType returns DeviceTypeGeneric on macOS. +func DetermineDeviceType(device *AvailableDevice) DeviceType { + return DeviceTypeGeneric +} + +// readSysfsFile is not available on macOS. +func readSysfsFile(path string) (string, error) { + return "", ErrNotSupportedOnMacOS +} + +// readIOMMUGroup is not available on macOS. +func readIOMMUGroup(pciAddress string) (int, error) { + return -1, ErrNotSupportedOnMacOS +} + +// readCurrentDriver is not available on macOS. +func readCurrentDriver(pciAddress string) *string { + return nil +} diff --git a/lib/devices/discovery.go b/lib/devices/discovery_linux.go similarity index 99% rename from lib/devices/discovery.go rename to lib/devices/discovery_linux.go index b04213c0..33798292 100644 --- a/lib/devices/discovery.go +++ b/lib/devices/discovery_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package devices import ( diff --git a/lib/devices/manager.go b/lib/devices/manager.go index d93a7572..6c0d84b6 100644 --- a/lib/devices/manager.go +++ b/lib/devices/manager.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "runtime" "strings" "sync" "time" @@ -552,6 +553,11 @@ func (m *manager) ReconcileDevices(ctx context.Context) error { func (m *manager) validatePrerequisites(ctx context.Context) { log := logger.FromContext(ctx) + // Skip GPU passthrough checks on macOS - not supported + if runtime.GOOS == "darwin" { + return + } + // Check IOMMU availability iommuGroupsDir := "/sys/kernel/iommu_groups" entries, err := os.ReadDir(iommuGroupsDir) diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go new file mode 100644 index 00000000..dacca12f --- /dev/null +++ b/lib/devices/mdev_darwin.go @@ -0,0 +1,57 @@ +//go:build darwin + +package devices + +import ( + "context" + "fmt" +) + +// ErrVGPUNotSupportedOnMacOS is returned for vGPU operations on macOS +var ErrVGPUNotSupportedOnMacOS = fmt.Errorf("vGPU (mdev) is not supported on macOS") + +// SetGPUProfileCacheTTL is a no-op on macOS. +func SetGPUProfileCacheTTL(ttl string) { + // No-op on macOS +} + +// DiscoverVFs returns an empty list on macOS. +// SR-IOV Virtual Functions are not available on macOS. +func DiscoverVFs() ([]VirtualFunction, error) { + return []VirtualFunction{}, nil +} + +// ListGPUProfiles returns an empty list on macOS. +func ListGPUProfiles() ([]GPUProfile, error) { + return []GPUProfile{}, nil +} + +// ListGPUProfilesWithVFs returns an empty list on macOS. +func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { + return []GPUProfile{}, nil +} + +// ListMdevDevices returns an empty list on macOS. +func ListMdevDevices() ([]MdevDevice, error) { + return []MdevDevice{}, nil +} + +// CreateMdev returns an error on macOS as mdev is not supported. +func CreateMdev(ctx context.Context, profileName, instanceID string) (*MdevDevice, error) { + return nil, ErrVGPUNotSupportedOnMacOS +} + +// DestroyMdev is a no-op on macOS. +func DestroyMdev(ctx context.Context, mdevUUID string) error { + return nil +} + +// IsMdevInUse returns false on macOS. +func IsMdevInUse(mdevUUID string) bool { + return false +} + +// ReconcileMdevs is a no-op on macOS. +func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error { + return nil +} diff --git a/lib/devices/mdev.go b/lib/devices/mdev_linux.go similarity index 98% rename from lib/devices/mdev.go rename to lib/devices/mdev_linux.go index de648e05..2e5bab44 100644 --- a/lib/devices/mdev.go +++ b/lib/devices/mdev_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package devices import ( @@ -604,13 +606,6 @@ func IsMdevInUse(mdevUUID string) bool { return err == nil // Has a driver = in use } -// MdevReconcileInfo contains information needed to reconcile mdevs for an instance -type MdevReconcileInfo struct { - InstanceID string - MdevUUID string - IsRunning bool // true if instance's VMM is running or state is unknown -} - // ReconcileMdevs destroys orphaned mdevs that belong to hypeman but are no longer in use. // This is called on server startup to clean up stale mdevs from previous runs. // diff --git a/lib/devices/types.go b/lib/devices/types.go index bd66fa86..d436ca1d 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -94,3 +94,10 @@ type PassthroughDevice struct { Name string `json:"name"` // GPU name, e.g., "NVIDIA L40S" Available bool `json:"available"` // true if not attached to an instance } + +// MdevReconcileInfo contains information needed to reconcile mdevs for an instance +type MdevReconcileInfo struct { + InstanceID string + MdevUUID string + IsRunning bool // true if instance's VMM is running or state is unknown +} diff --git a/lib/devices/vfio_darwin.go b/lib/devices/vfio_darwin.go new file mode 100644 index 00000000..ae47cbcd --- /dev/null +++ b/lib/devices/vfio_darwin.go @@ -0,0 +1,74 @@ +//go:build darwin + +package devices + +import ( + "fmt" +) + +// ErrVFIONotSupportedOnMacOS is returned for VFIO operations on macOS +var ErrVFIONotSupportedOnMacOS = fmt.Errorf("VFIO device passthrough is not supported on macOS") + +// VFIOBinder handles binding and unbinding devices to/from VFIO. +// On macOS, this is a stub that returns errors for all operations. +type VFIOBinder struct{} + +// NewVFIOBinder creates a new VFIOBinder +func NewVFIOBinder() *VFIOBinder { + return &VFIOBinder{} +} + +// IsVFIOAvailable returns false on macOS as VFIO is not available. +func (v *VFIOBinder) IsVFIOAvailable() bool { + return false +} + +// IsDeviceBoundToVFIO returns false on macOS. +func (v *VFIOBinder) IsDeviceBoundToVFIO(pciAddress string) bool { + return false +} + +// BindToVFIO returns an error on macOS as VFIO is not supported. +func (v *VFIOBinder) BindToVFIO(pciAddress string) error { + return ErrVFIONotSupportedOnMacOS +} + +// UnbindFromVFIO returns an error on macOS as VFIO is not supported. +func (v *VFIOBinder) UnbindFromVFIO(pciAddress string) error { + return ErrVFIONotSupportedOnMacOS +} + +// GetVFIOGroupPath returns an error on macOS as VFIO is not supported. +func (v *VFIOBinder) GetVFIOGroupPath(pciAddress string) (string, error) { + return "", ErrVFIONotSupportedOnMacOS +} + +// CheckIOMMUGroupSafe returns an error on macOS as IOMMU is not available. +func (v *VFIOBinder) CheckIOMMUGroupSafe(pciAddress string, allowedDevices []string) error { + return ErrVFIONotSupportedOnMacOS +} + +// GetDeviceSysfsPath returns an empty string on macOS. +func GetDeviceSysfsPath(pciAddress string) string { + return "" +} + +// unbindFromDriver is not available on macOS. +func (v *VFIOBinder) unbindFromDriver(pciAddress, driver string) error { + return ErrVFIONotSupportedOnMacOS +} + +// setDriverOverride is not available on macOS. +func (v *VFIOBinder) setDriverOverride(pciAddress, driver string) error { + return ErrVFIONotSupportedOnMacOS +} + +// triggerDriverProbe is not available on macOS. +func (v *VFIOBinder) triggerDriverProbe(pciAddress string) error { + return ErrVFIONotSupportedOnMacOS +} + +// startNvidiaPersistenced is not available on macOS. +func (v *VFIOBinder) startNvidiaPersistenced() error { + return nil // No-op, not an error +} diff --git a/lib/devices/vfio.go b/lib/devices/vfio_linux.go similarity index 99% rename from lib/devices/vfio.go rename to lib/devices/vfio_linux.go index 38606f5b..65be8104 100644 --- a/lib/devices/vfio.go +++ b/lib/devices/vfio_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package devices import ( diff --git a/lib/hypervisor/README.md b/lib/hypervisor/README.md index 2bab53d9..3eafd673 100644 --- a/lib/hypervisor/README.md +++ b/lib/hypervisor/README.md @@ -4,20 +4,29 @@ Provides a common interface for VM management across different hypervisors. ## Purpose -Hypeman originally supported only Cloud Hypervisor. This abstraction layer allows supporting multiple hypervisors (e.g., QEMU) through a unified interface, enabling: +Hypeman originally supported only Cloud Hypervisor. This abstraction layer allows supporting multiple hypervisors through a unified interface, enabling: - **Hypervisor choice per instance** - Different instances can use different hypervisors +- **Platform support** - Linux uses Cloud Hypervisor/QEMU, macOS uses Virtualization.framework - **Feature parity where possible** - Common operations work the same way - **Graceful degradation** - Features unsupported by a hypervisor can be detected and handled +## Implementations + +| Hypervisor | Platform | Process Model | Control Interface | +|------------|----------|---------------|-------------------| +| Cloud Hypervisor | Linux | External process | HTTP API over Unix socket | +| QEMU | Linux | External process | QMP over Unix socket | +| vz | macOS | In-process | Direct API calls | + ## How It Works The abstraction defines two key interfaces: 1. **Hypervisor** - VM lifecycle operations (create, boot, pause, resume, snapshot, restore, shutdown) -2. **ProcessManager** - Hypervisor process lifecycle (start binary, get binary path) +2. **VMStarter** - VM startup and configuration (start binary, get binary path) -Each hypervisor implementation translates the generic configuration and operations to its native format. For example, Cloud Hypervisor uses an HTTP API over a Unix socket, while QEMU would use QMP. +Each implementation translates generic configuration to its native format. Cloud Hypervisor and QEMU run as external processes with socket-based control. The vz implementation runs VMs in-process using Apple's Virtualization.framework. Before using optional features, callers check capabilities: @@ -27,6 +36,19 @@ if hv.Capabilities().SupportsSnapshot { } ``` +## Platform Differences + +### Linux (Cloud Hypervisor, QEMU) +- VMs run as separate processes with PIDs +- State persists across hypeman restarts (reconnect via socket) +- TAP devices and Linux bridges for networking + +### macOS (vz) +- VMs run in-process (no separate PID) +- VMs stop if hypeman stops (cannot reconnect) +- NAT networking via Virtualization.framework +- Requires code signing with virtualization entitlement + ## Hypervisor Switching Instances store their hypervisor type in metadata. An instance can switch hypervisors only when stopped (no running VM, no snapshot), since: diff --git a/lib/hypervisor/cloudhypervisor/process.go b/lib/hypervisor/cloudhypervisor/process.go index b81b72d4..c30b6c3d 100644 --- a/lib/hypervisor/cloudhypervisor/process.go +++ b/lib/hypervisor/cloudhypervisor/process.go @@ -15,6 +15,9 @@ import ( func init() { hypervisor.RegisterSocketName(hypervisor.TypeCloudHypervisor, "ch.sock") + hypervisor.RegisterClientFactory(hypervisor.TypeCloudHypervisor, func(socketPath string) (hypervisor.Hypervisor, error) { + return New(socketPath) + }) } // Starter implements hypervisor.VMStarter for Cloud Hypervisor. diff --git a/lib/hypervisor/hypervisor.go b/lib/hypervisor/hypervisor.go index 197a6ac7..b4287a79 100644 --- a/lib/hypervisor/hypervisor.go +++ b/lib/hypervisor/hypervisor.go @@ -5,6 +5,7 @@ package hypervisor import ( "context" + "errors" "fmt" "net" "time" @@ -12,6 +13,16 @@ import ( "github.com/kernel/hypeman/lib/paths" ) +// Common errors +var ( + // ErrHypervisorNotRunning is returned when trying to connect to a hypervisor + // that is not currently running or cannot be reconnected to. + ErrHypervisorNotRunning = errors.New("hypervisor is not running") + + // ErrNotSupported is returned when an operation is not supported by the hypervisor. + ErrNotSupported = errors.New("operation not supported by this hypervisor") +) + // Type identifies the hypervisor implementation type Type string @@ -20,6 +31,8 @@ const ( TypeCloudHypervisor Type = "cloud-hypervisor" // TypeQEMU is the QEMU VMM TypeQEMU Type = "qemu" + // TypeVZ is the Virtualization.framework VMM (macOS only) + TypeVZ Type = "vz" ) // socketNames maps hypervisor types to their socket filenames. @@ -164,3 +177,23 @@ func NewVsockDialer(hvType Type, vsockSocket string, vsockCID int64) (VsockDiale } return factory(vsockSocket, vsockCID), nil } + +// ClientFactory creates Hypervisor client instances for a hypervisor type. +type ClientFactory func(socketPath string) (Hypervisor, error) + +// clientFactories maps hypervisor types to their client factories. +var clientFactories = make(map[Type]ClientFactory) + +// RegisterClientFactory registers a Hypervisor client factory. +func RegisterClientFactory(t Type, factory ClientFactory) { + clientFactories[t] = factory +} + +// NewClient creates a Hypervisor client for the given type and socket. +func NewClient(hvType Type, socketPath string) (Hypervisor, error) { + factory, ok := clientFactories[hvType] + if !ok { + return nil, fmt.Errorf("no client factory registered for hypervisor type: %s", hvType) + } + return factory(socketPath) +} diff --git a/lib/hypervisor/qemu/process.go b/lib/hypervisor/qemu/process.go index 459d94eb..e2e1d098 100644 --- a/lib/hypervisor/qemu/process.go +++ b/lib/hypervisor/qemu/process.go @@ -37,6 +37,9 @@ const ( func init() { hypervisor.RegisterSocketName(hypervisor.TypeQEMU, "qemu.sock") + hypervisor.RegisterClientFactory(hypervisor.TypeQEMU, func(socketPath string) (hypervisor.Hypervisor, error) { + return New(socketPath) + }) } // Starter implements hypervisor.VMStarter for QEMU. diff --git a/lib/hypervisor/qemu/vsock.go b/lib/hypervisor/qemu/vsock.go index 50c0791f..88be6cc5 100644 --- a/lib/hypervisor/qemu/vsock.go +++ b/lib/hypervisor/qemu/vsock.go @@ -1,3 +1,5 @@ +//go:build linux + package qemu import ( diff --git a/lib/hypervisor/vz/client.go b/lib/hypervisor/vz/client.go new file mode 100644 index 00000000..50abbbe3 --- /dev/null +++ b/lib/hypervisor/vz/client.go @@ -0,0 +1,172 @@ +//go:build darwin + +package vz + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +// Client implements hypervisor.Hypervisor via HTTP to the vz-shim process. +type Client struct { + socketPath string + httpClient *http.Client +} + +// NewClient creates a new vz shim client. +func NewClient(socketPath string) (*Client, error) { + transport := &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial("unix", socketPath) + }, + } + httpClient := &http.Client{ + Transport: transport, + Timeout: 30 * time.Second, + } + + // Verify connectivity with a short timeout + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://vz-shim/api/v1/vmm.ping", nil) + if err != nil { + return nil, fmt.Errorf("ping shim: %w", err) + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("ping shim: %w", err) + } + resp.Body.Close() + + return &Client{ + socketPath: socketPath, + httpClient: httpClient, + }, nil +} + +var _ hypervisor.Hypervisor = (*Client)(nil) + +// vmInfoResponse matches the shim's VMInfoResponse structure. +type vmInfoResponse struct { + State string `json:"state"` +} + +func (c *Client) Capabilities() hypervisor.Capabilities { + return hypervisor.Capabilities{ + SupportsSnapshot: false, + SupportsHotplugMemory: false, + SupportsPause: true, + SupportsVsock: true, + SupportsGPUPassthrough: false, + SupportsDiskIOLimit: false, + } +} + +// doPut sends a PUT request to the shim and checks for success. +func (c *Client) doPut(ctx context.Context, path string, body io.Reader) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://vz-shim"+path, body) + if err != nil { + return err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("%s failed with status %d: %s", path, resp.StatusCode, string(bodyBytes)) + } + return nil +} + +// doGet sends a GET request to the shim and returns the response body. +func (c *Client) doGet(ctx context.Context, path string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://vz-shim"+path, nil) + if err != nil { + return nil, err + } + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return io.ReadAll(resp.Body) +} + +func (c *Client) DeleteVM(ctx context.Context) error { + return c.doPut(ctx, "/api/v1/vm.shutdown", nil) +} + +func (c *Client) Shutdown(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, "http://vz-shim/api/v1/vmm.shutdown", nil) + if err != nil { + return err + } + resp, err := c.httpClient.Do(req) + if err != nil { + // Connection reset is expected when shim exits + return nil + } + defer resp.Body.Close() + return nil +} + +func (c *Client) GetVMInfo(ctx context.Context) (*hypervisor.VMInfo, error) { + body, err := c.doGet(ctx, "/api/v1/vm.info") + if err != nil { + return nil, fmt.Errorf("get vm info: %w", err) + } + + var info vmInfoResponse + if err := json.Unmarshal(body, &info); err != nil { + return nil, fmt.Errorf("decode vm info: %w", err) + } + + var state hypervisor.VMState + switch info.State { + case "Running": + state = hypervisor.StateRunning + case "Paused": + state = hypervisor.StatePaused + case "Starting": + state = hypervisor.StateCreated + case "Shutdown", "Stopped", "Error": + state = hypervisor.StateShutdown + default: + state = hypervisor.StateShutdown + } + + return &hypervisor.VMInfo{State: state}, nil +} + +func (c *Client) Pause(ctx context.Context) error { + return c.doPut(ctx, "/api/v1/vm.pause", nil) +} + +func (c *Client) Resume(ctx context.Context) error { + return c.doPut(ctx, "/api/v1/vm.resume", nil) +} + +func (c *Client) Snapshot(ctx context.Context, destPath string) error { + return hypervisor.ErrNotSupported +} + +func (c *Client) ResizeMemory(ctx context.Context, bytes int64) error { + return hypervisor.ErrNotSupported +} + +func (c *Client) ResizeMemoryAndWait(ctx context.Context, bytes int64, timeout time.Duration) error { + return hypervisor.ErrNotSupported +} diff --git a/lib/hypervisor/vz/shimconfig/config.go b/lib/hypervisor/vz/shimconfig/config.go new file mode 100644 index 00000000..23056267 --- /dev/null +++ b/lib/hypervisor/vz/shimconfig/config.go @@ -0,0 +1,44 @@ +//go:build darwin + +// Package shimconfig defines the configuration types shared between +// the hypeman API server and the vz-shim subprocess. +package shimconfig + +// ShimConfig is the configuration passed from hypeman to the shim. +type ShimConfig struct { + // Compute resources + VCPUs int `json:"vcpus"` + MemoryBytes int64 `json:"memory_bytes"` + + // Storage + Disks []DiskConfig `json:"disks"` + + // Network + Networks []NetworkConfig `json:"networks"` + + // Console + SerialLogPath string `json:"serial_log_path"` + + // Boot configuration + KernelPath string `json:"kernel_path"` + InitrdPath string `json:"initrd_path"` + KernelArgs string `json:"kernel_args"` + + // Socket paths (where shim should listen) + ControlSocket string `json:"control_socket"` + VsockSocket string `json:"vsock_socket"` + + // Logging + LogPath string `json:"log_path"` +} + +// DiskConfig represents a disk attached to the VM. +type DiskConfig struct { + Path string `json:"path"` + Readonly bool `json:"readonly"` +} + +// NetworkConfig represents a network interface. +type NetworkConfig struct { + MAC string `json:"mac"` +} diff --git a/lib/hypervisor/vz/starter.go b/lib/hypervisor/vz/starter.go new file mode 100644 index 00000000..15ea9847 --- /dev/null +++ b/lib/hypervisor/vz/starter.go @@ -0,0 +1,202 @@ +//go:build darwin + +// Package vz implements the hypervisor.Hypervisor interface for +// Apple's Virtualization.framework on macOS via the vz-shim subprocess. +package vz + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "sync" + "syscall" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig" + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/paths" +) + +func init() { + hypervisor.RegisterSocketName(hypervisor.TypeVZ, "vz.sock") + hypervisor.RegisterVsockDialerFactory(hypervisor.TypeVZ, NewVsockDialer) + hypervisor.RegisterClientFactory(hypervisor.TypeVZ, func(socketPath string) (hypervisor.Hypervisor, error) { + return NewClient(socketPath) + }) +} + +var ( + shimOnce sync.Once + shimPath string + shimErr error +) + +// extractShim extracts the embedded vz-shim binary to a temp file and codesigns it. +func extractShim() (string, error) { + shimOnce.Do(func() { + f, err := os.CreateTemp("", "vz-shim-*") + if err != nil { + shimErr = fmt.Errorf("create temp file: %w", err) + return + } + defer f.Close() + + if _, err := f.Write(vzShimBinary); err != nil { + os.Remove(f.Name()) + shimErr = fmt.Errorf("write vz-shim binary: %w", err) + return + } + + if err := f.Chmod(0755); err != nil { + os.Remove(f.Name()) + shimErr = fmt.Errorf("chmod vz-shim binary: %w", err) + return + } + + // Codesign with entitlements for Virtualization.framework + cmd := exec.Command("codesign", "--sign", "-", "--entitlements", entitlementsPath(), "--force", f.Name()) + if out, err := cmd.CombinedOutput(); err != nil { + os.Remove(f.Name()) + shimErr = fmt.Errorf("codesign vz-shim: %s: %w", string(out), err) + return + } + + shimPath = f.Name() + }) + return shimPath, shimErr +} + +// entitlementsPath returns the path to the vz.entitlements file. +func entitlementsPath() string { + exe, err := os.Executable() + if err != nil { + return "vz.entitlements" + } + return filepath.Join(filepath.Dir(exe), "vz.entitlements") +} + +// Starter implements hypervisor.VMStarter for Virtualization.framework. +type Starter struct{} + +// NewStarter creates a new vz starter. +func NewStarter() *Starter { + return &Starter{} +} + +var _ hypervisor.VMStarter = (*Starter)(nil) + +func (s *Starter) SocketName() string { + return "vz.sock" +} + +// GetBinaryPath extracts the embedded vz-shim and returns its path. +func (s *Starter) GetBinaryPath(p *paths.Paths, version string) (string, error) { + return extractShim() +} + +// GetVersion returns "vz-shim". +func (s *Starter) GetVersion(p *paths.Paths) (string, error) { + return "vz-shim", nil +} + +// StartVM spawns a vz-shim subprocess to host the VM. +func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, socketPath string, config hypervisor.VMConfig) (int, hypervisor.Hypervisor, error) { + log := logger.FromContext(ctx) + + instanceDir := filepath.Dir(socketPath) + controlSocket := socketPath + vsockSocket := filepath.Join(instanceDir, "vz.vsock") + logPath := filepath.Join(instanceDir, "logs", "vz-shim.log") + + shimConfig := shimconfig.ShimConfig{ + VCPUs: config.VCPUs, + MemoryBytes: config.MemoryBytes, + SerialLogPath: config.SerialLogPath, + KernelPath: config.KernelPath, + InitrdPath: config.InitrdPath, + KernelArgs: config.KernelArgs, + ControlSocket: controlSocket, + VsockSocket: vsockSocket, + LogPath: logPath, + } + + for _, disk := range config.Disks { + shimConfig.Disks = append(shimConfig.Disks, shimconfig.DiskConfig{ + Path: disk.Path, + Readonly: disk.Readonly, + }) + } + + for _, net := range config.Networks { + shimConfig.Networks = append(shimConfig.Networks, shimconfig.NetworkConfig{ + MAC: net.MAC, + }) + } + + configJSON, err := json.Marshal(shimConfig) + if err != nil { + return 0, nil, fmt.Errorf("marshal shim config: %w", err) + } + + log.DebugContext(ctx, "spawning vz-shim", "config", string(configJSON)) + + shimBinary, err := s.GetBinaryPath(p, version) + if err != nil { + return 0, nil, fmt.Errorf("get vz-shim binary: %w", err) + } + + cmd := exec.Command(shimBinary, "-config", string(configJSON)) + cmd.Stdout = nil + cmd.Stderr = nil + cmd.Stdin = nil + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + } + + if err := cmd.Start(); err != nil { + return 0, nil, fmt.Errorf("start vz-shim: %w", err) + } + + pid := cmd.Process.Pid + log.InfoContext(ctx, "vz-shim started", "pid", pid, "control_socket", controlSocket) + + client, err := s.waitForShim(ctx, controlSocket, 30*time.Second) + if err != nil { + cmd.Process.Kill() + return 0, nil, fmt.Errorf("connect to vz-shim: %w", err) + } + + cmd.Process.Release() + + return pid, client, nil +} + +// RestoreVM is not supported by vz (Virtualization.framework cannot restore Linux guests). +func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, socketPath string, snapshotPath string) (int, hypervisor.Hypervisor, error) { + return 0, nil, hypervisor.ErrNotSupported +} + +func (s *Starter) waitForShim(ctx context.Context, socketPath string, timeout time.Duration) (*Client, error) { + deadline := time.Now().Add(timeout) + + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + client, err := NewClient(socketPath) + if err == nil { + return client, nil + } + + time.Sleep(100 * time.Millisecond) + } + + return nil, fmt.Errorf("timeout waiting for shim socket: %s", socketPath) +} diff --git a/lib/hypervisor/vz/vsock.go b/lib/hypervisor/vz/vsock.go new file mode 100644 index 00000000..2243ab7d --- /dev/null +++ b/lib/hypervisor/vz/vsock.go @@ -0,0 +1,111 @@ +//go:build darwin + +package vz + +import ( + "bufio" + "context" + "fmt" + "log/slog" + "net" + "strings" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +const ( + vsockDialTimeout = 5 * time.Second + vsockHandshakeTimeout = 5 * time.Second +) + +// VsockDialer implements hypervisor.VsockDialer for vz via the shim's Unix socket proxy. +// Uses the same protocol as Cloud Hypervisor: CONNECT {port}\n -> OK {port}\n +type VsockDialer struct { + socketPath string // path to vz.vsock Unix socket +} + +// NewVsockDialer creates a new VsockDialer for vz. +// vsockSocket is the path to the vz.vsock Unix socket proxy. +// vsockCID is unused because the vz proxy is per-VM (unlike QEMU which uses kernel AF_VSOCK with CID routing). +func NewVsockDialer(vsockSocket string, vsockCID int64) hypervisor.VsockDialer { + return &VsockDialer{ + socketPath: vsockSocket, + } +} + +// Key returns a unique identifier for this dialer, used for connection pooling. +func (d *VsockDialer) Key() string { + return "vz:" + d.socketPath +} + +// DialVsock connects to the guest on the specified port via the shim's vsock proxy. +func (d *VsockDialer) DialVsock(ctx context.Context, port int) (net.Conn, error) { + slog.DebugContext(ctx, "connecting to vsock via shim proxy", "socket", d.socketPath, "port", port) + + // Use dial timeout, respecting context deadline if shorter + dialTimeout := vsockDialTimeout + if deadline, ok := ctx.Deadline(); ok { + if remaining := time.Until(deadline); remaining < dialTimeout { + dialTimeout = remaining + } + } + + // Connect to the shim's vsock proxy Unix socket + dialer := net.Dialer{Timeout: dialTimeout} + conn, err := dialer.DialContext(ctx, "unix", d.socketPath) + if err != nil { + return nil, fmt.Errorf("dial vsock proxy socket %s: %w", d.socketPath, err) + } + + slog.DebugContext(ctx, "connected to vsock proxy, performing handshake", "port", port) + + // Set deadline for handshake + if err := conn.SetDeadline(time.Now().Add(vsockHandshakeTimeout)); err != nil { + conn.Close() + return nil, fmt.Errorf("set handshake deadline: %w", err) + } + + // Perform handshake (same protocol as Cloud Hypervisor) + handshakeCmd := fmt.Sprintf("CONNECT %d\n", port) + if _, err := conn.Write([]byte(handshakeCmd)); err != nil { + conn.Close() + return nil, fmt.Errorf("send vsock handshake: %w", err) + } + + // Read handshake response + reader := bufio.NewReader(conn) + response, err := reader.ReadString('\n') + if err != nil { + conn.Close() + return nil, fmt.Errorf("read vsock handshake response (is guest-agent running?): %w", err) + } + + // Clear deadline after successful handshake + if err := conn.SetDeadline(time.Time{}); err != nil { + conn.Close() + return nil, fmt.Errorf("clear deadline: %w", err) + } + + response = strings.TrimSpace(response) + if !strings.HasPrefix(response, "OK ") { + conn.Close() + return nil, fmt.Errorf("vsock handshake failed: %s", response) + } + + slog.DebugContext(ctx, "vsock handshake successful", "response", response) + + // Return wrapped connection that uses the bufio.Reader + return &bufferedConn{Conn: conn, reader: reader}, nil +} + +// bufferedConn wraps a net.Conn with a bufio.Reader to ensure any buffered +// data from the handshake is properly drained before reading from the connection. +type bufferedConn struct { + net.Conn + reader *bufio.Reader +} + +func (c *bufferedConn) Read(p []byte) (int, error) { + return c.reader.Read(p) +} diff --git a/lib/hypervisor/vz/vz-shim/.gitkeep b/lib/hypervisor/vz/vz-shim/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/lib/hypervisor/vz/vz_shim_binary.go b/lib/hypervisor/vz/vz_shim_binary.go new file mode 100644 index 00000000..fd278761 --- /dev/null +++ b/lib/hypervisor/vz/vz_shim_binary.go @@ -0,0 +1,11 @@ +//go:build darwin + +package vz + +import _ "embed" + +// vzShimBinary contains the embedded vz-shim binary. +// Built by the Makefile before the main binary is compiled. +// +//go:embed vz-shim/vz-shim +var vzShimBinary []byte diff --git a/lib/images/disk.go b/lib/images/disk.go index 53378b49..c76660d6 100644 --- a/lib/images/disk.go +++ b/lib/images/disk.go @@ -108,6 +108,17 @@ func convertToCpio(rootfsDir, outputPath string) (int64, error) { return stat.Size(), nil } +// sectorSize is the block size for disk images (required by Virtualization.framework) +const sectorSize = 4096 + +// alignToSector rounds size up to the nearest sector boundary +func alignToSector(size int64) int64 { + if size%sectorSize == 0 { + return size + } + return ((size / sectorSize) + 1) * sectorSize +} + // convertToExt4 converts a rootfs directory to an ext4 disk image using mkfs.ext4 func convertToExt4(rootfsDir, diskPath string) (int64, error) { // Calculate size of rootfs directory @@ -125,6 +136,9 @@ func convertToExt4(rootfsDir, diskPath string) (int64, error) { diskSizeBytes = minSize } + // Align to sector boundary (required by macOS Virtualization.framework) + diskSizeBytes = alignToSector(diskSizeBytes) + // Ensure parent directory exists if err := os.MkdirAll(filepath.Dir(diskPath), 0755); err != nil { return 0, fmt.Errorf("create disk parent dir: %w", err) @@ -142,7 +156,7 @@ func convertToExt4(rootfsDir, diskPath string) (int64, error) { f.Close() // Format as ext4 with rootfs contents using mkfs.ext4 - // -b 4096: 4KB blocks (standard, matches VM page size) + // -b 4096: 4KB blocks (standard, matches VM page size and sector alignment) // -O ^has_journal: Disable journal (not needed for read-only VM mounts) // -d: Copy directory contents into filesystem // -F: Force creation (file not block device) @@ -152,12 +166,21 @@ func convertToExt4(rootfsDir, diskPath string) (int64, error) { return 0, fmt.Errorf("mkfs.ext4 failed: %w, output: %s", err, output) } - // Get actual disk size + // Verify final size is sector-aligned (mkfs.ext4 should preserve our truncated size) stat, err := os.Stat(diskPath) if err != nil { return 0, fmt.Errorf("stat disk: %w", err) } + // Re-align if mkfs.ext4 changed the size (shouldn't happen with -F on a regular file) + if stat.Size()%sectorSize != 0 { + alignedSize := alignToSector(stat.Size()) + if err := os.Truncate(diskPath, alignedSize); err != nil { + return 0, fmt.Errorf("align disk to sector boundary: %w", err) + } + return alignedSize, nil + } + return stat.Size(), nil } @@ -204,6 +227,9 @@ func dirSize(path string) (int64, error) { // CreateEmptyExt4Disk creates a sparse disk file and formats it as ext4. // Used for volumes and instance overlays that need empty writable filesystems. func CreateEmptyExt4Disk(diskPath string, sizeBytes int64) error { + // Align to sector boundary (required by macOS Virtualization.framework) + sizeBytes = alignToSector(sizeBytes) + // Ensure parent directory exists if err := os.MkdirAll(filepath.Dir(diskPath), 0755); err != nil { return fmt.Errorf("create disk parent dir: %w", err) @@ -221,8 +247,8 @@ func CreateEmptyExt4Disk(diskPath string, sizeBytes int64) error { return fmt.Errorf("truncate disk file: %w", err) } - // Format as ext4 - cmd := exec.Command("mkfs.ext4", "-F", diskPath) + // Format as ext4 with 4KB blocks (matches sector alignment) + cmd := exec.Command("mkfs.ext4", "-b", "4096", "-F", diskPath) output, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("mkfs.ext4 failed: %w, output: %s", err, output) diff --git a/lib/images/manager.go b/lib/images/manager.go index a7e5d965..c423bf34 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -113,18 +113,26 @@ func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Ima // Check if we already have this digest (deduplication) if meta, err := readMetadata(m.paths, ref.Repository(), ref.DigestHex()); err == nil { - // We have this digest already - if meta.Status == StatusReady && ref.Tag() != "" { - // Update tag symlink to point to current digest - // (handles case where tag moved to new digest) - createTagSymlink(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()) - } - img := meta.toImage() - // Add queue position if pending - if meta.Status == StatusPending { - img.QueuePosition = m.queue.GetPosition(meta.Digest) + // Don't cache failed builds - allow retry + if meta.Status == StatusFailed { + // Clean up the failed build directory so we can retry + digestDir := filepath.Join(m.paths.ImagesDir(), ref.Repository(), ref.DigestHex()) + os.RemoveAll(digestDir) + // Fall through to re-queue the build + } else { + // We have this digest already (ready, pending, pulling, or converting) + if meta.Status == StatusReady && ref.Tag() != "" { + // Update tag symlink to point to current digest + // (handles case where tag moved to new digest) + createTagSymlink(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()) + } + img := meta.toImage() + // Add queue position if pending + if meta.Status == StatusPending { + img.QueuePosition = m.queue.GetPosition(meta.Digest) + } + return img, nil } - return img, nil } // Don't have this digest yet, queue the build @@ -156,15 +164,22 @@ func (m *manager) ImportLocalImage(ctx context.Context, repo, reference, digest // Check if we already have this digest (deduplication) if meta, err := readMetadata(m.paths, ref.Repository(), ref.DigestHex()); err == nil { - // We have this digest already - if meta.Status == StatusReady && ref.Tag() != "" { - createTagSymlink(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()) - } - img := meta.toImage() - if meta.Status == StatusPending { - img.QueuePosition = m.queue.GetPosition(meta.Digest) + // Don't cache failed builds - allow retry + if meta.Status == StatusFailed { + digestDir := filepath.Join(m.paths.ImagesDir(), ref.Repository(), ref.DigestHex()) + os.RemoveAll(digestDir) + // Fall through to re-queue the build + } else { + // We have this digest already + if meta.Status == StatusReady && ref.Tag() != "" { + createTagSymlink(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex()) + } + img := meta.toImage() + if meta.Status == StatusPending { + img.QueuePosition = m.queue.GetPosition(meta.Digest) + } + return img, nil } - return img, nil } // Don't have this digest yet, queue the build diff --git a/lib/images/oci.go b/lib/images/oci.go index 31962d88..1d07758d 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -64,11 +64,13 @@ func newOCIClient(cacheDir string) (*ociClient, error) { return &ociClient{cacheDir: cacheDir}, nil } -// currentPlatform returns the platform for the current host -func currentPlatform() gcr.Platform { +// vmPlatform returns the target platform for VM images. +// Always returns Linux since hypeman VMs are always Linux guests, +// regardless of the host OS (Linux or macOS). +func vmPlatform() gcr.Platform { return gcr.Platform{ Architecture: runtime.GOARCH, - OS: runtime.GOOS, + OS: "linux", } } @@ -77,6 +79,12 @@ func currentPlatform() gcr.Platform { // For multi-arch images, it returns the platform-specific manifest digest // (matching the current host platform) rather than the manifest index digest. func (c *ociClient) inspectManifest(ctx context.Context, imageRef string) (string, error) { + return c.inspectManifestWithPlatform(ctx, imageRef, vmPlatform()) +} + +// inspectManifestWithPlatform synchronously inspects a remote image to get its digest +// for a specific platform. +func (c *ociClient) inspectManifestWithPlatform(ctx context.Context, imageRef string, platform gcr.Platform) (string, error) { ref, err := name.ParseReference(imageRef) if err != nil { return "", fmt.Errorf("parse image reference: %w", err) @@ -89,7 +97,7 @@ func (c *ociClient) inspectManifest(ctx context.Context, imageRef string) (strin img, err := remote.Image(ref, remote.WithContext(ctx), remote.WithAuthFromKeychain(authn.DefaultKeychain), - remote.WithPlatform(currentPlatform())) + remote.WithPlatform(platform)) if err != nil { return "", fmt.Errorf("fetch manifest: %w", wrapRegistryError(err)) } @@ -109,6 +117,10 @@ type pullResult struct { } func (c *ociClient) pullAndExport(ctx context.Context, imageRef, digest, exportDir string) (*pullResult, error) { + return c.pullAndExportWithPlatform(ctx, imageRef, digest, exportDir, vmPlatform()) +} + +func (c *ociClient) pullAndExportWithPlatform(ctx context.Context, imageRef, digest, exportDir string, platform gcr.Platform) (*pullResult, error) { // Use a shared OCI layout for all images to enable automatic layer caching // The cacheDir itself is the OCI layout root with shared blobs/sha256/ directory // The digest is ALWAYS known at this point (from inspectManifest or digest reference) @@ -117,7 +129,7 @@ func (c *ociClient) pullAndExport(ctx context.Context, imageRef, digest, exportD // Check if this digest is already cached if !c.existsInLayout(layoutTag) { // Not cached, pull it using digest-based tag - if err := c.pullToOCILayout(ctx, imageRef, layoutTag); err != nil { + if err := c.pullToOCILayoutWithPlatform(ctx, imageRef, layoutTag, platform); err != nil { return nil, fmt.Errorf("pull to oci layout: %w", err) } } @@ -141,6 +153,10 @@ func (c *ociClient) pullAndExport(ctx context.Context, imageRef, digest, exportD } func (c *ociClient) pullToOCILayout(ctx context.Context, imageRef, layoutTag string) error { + return c.pullToOCILayoutWithPlatform(ctx, imageRef, layoutTag, vmPlatform()) +} + +func (c *ociClient) pullToOCILayoutWithPlatform(ctx context.Context, imageRef, layoutTag string, platform gcr.Platform) error { ref, err := name.ParseReference(imageRef) if err != nil { return fmt.Errorf("parse image reference: %w", err) @@ -152,7 +168,7 @@ func (c *ociClient) pullToOCILayout(ctx context.Context, imageRef, layoutTag str img, err := remote.Image(ref, remote.WithContext(ctx), remote.WithAuthFromKeychain(authn.DefaultKeychain), - remote.WithPlatform(currentPlatform())) + remote.WithPlatform(platform)) if err != nil { // Rate limits fail here immediately (429 is not retried by default) return fmt.Errorf("fetch image manifest: %w", wrapRegistryError(err)) diff --git a/lib/images/oci_public.go b/lib/images/oci_public.go index 5d20835e..66643b97 100644 --- a/lib/images/oci_public.go +++ b/lib/images/oci_public.go @@ -20,11 +20,18 @@ func NewOCIClient(cacheDir string) (*OCIClient, error) { } // InspectManifest inspects a remote image to get its digest (public for system manager) +// Always targets Linux platform since hypeman VMs are Linux guests. func (c *OCIClient) InspectManifest(ctx context.Context, imageRef string) (string, error) { return c.client.inspectManifest(ctx, imageRef) } +// InspectManifestForLinux is an alias for InspectManifest (all images target Linux) +func (c *OCIClient) InspectManifestForLinux(ctx context.Context, imageRef string) (string, error) { + return c.InspectManifest(ctx, imageRef) +} + // PullAndUnpack pulls an OCI image and unpacks it to a directory (public for system manager) +// Always targets Linux platform since hypeman VMs are Linux guests. func (c *OCIClient) PullAndUnpack(ctx context.Context, imageRef, digest, exportDir string) error { _, err := c.client.pullAndExport(ctx, imageRef, digest, exportDir) if err != nil { @@ -33,3 +40,7 @@ func (c *OCIClient) PullAndUnpack(ctx context.Context, imageRef, digest, exportD return nil } +// PullAndUnpackForLinux is an alias for PullAndUnpack (all images target Linux) +func (c *OCIClient) PullAndUnpackForLinux(ctx context.Context, imageRef, digest, exportDir string) error { + return c.PullAndUnpack(ctx, imageRef, digest, exportDir) +} diff --git a/lib/ingress/binaries_amd64.go b/lib/ingress/binaries_amd64.go index 309da631..551e12fb 100644 --- a/lib/ingress/binaries_amd64.go +++ b/lib/ingress/binaries_amd64.go @@ -1,4 +1,4 @@ -//go:build amd64 +//go:build amd64 && linux package ingress diff --git a/lib/ingress/binaries_arm64.go b/lib/ingress/binaries_arm64.go index 8fb413ce..995578a8 100644 --- a/lib/ingress/binaries_arm64.go +++ b/lib/ingress/binaries_arm64.go @@ -1,4 +1,4 @@ -//go:build arm64 +//go:build arm64 && linux package ingress diff --git a/lib/ingress/binaries_darwin.go b/lib/ingress/binaries_darwin.go new file mode 100644 index 00000000..1a2ba408 --- /dev/null +++ b/lib/ingress/binaries_darwin.go @@ -0,0 +1,33 @@ +//go:build darwin + +package ingress + +import ( + "fmt" + "os/exec" + + "github.com/kernel/hypeman/lib/paths" +) + +// CaddyVersion is the version of Caddy to use. +const CaddyVersion = "v2.10.2" + +// ErrCaddyNotEmbedded indicates Caddy is not embedded on macOS. +// Users should install Caddy via Homebrew or download from caddyserver.com. +var ErrCaddyNotEmbedded = fmt.Errorf("caddy binary is not embedded on macOS; install via: brew install caddy") + +// ExtractCaddyBinary on macOS attempts to find Caddy in PATH. +// Unlike Linux, we don't embed the binary on macOS. +func ExtractCaddyBinary(p *paths.Paths) (string, error) { + // Try to find caddy in PATH + path, err := exec.LookPath("caddy") + if err != nil { + return "", ErrCaddyNotEmbedded + } + return path, nil +} + +// GetCaddyBinaryPath returns path to Caddy, looking in PATH on macOS. +func GetCaddyBinaryPath(p *paths.Paths) (string, error) { + return ExtractCaddyBinary(p) +} diff --git a/lib/ingress/binaries.go b/lib/ingress/binaries_linux.go similarity index 99% rename from lib/ingress/binaries.go rename to lib/ingress/binaries_linux.go index 79143506..2b2a6a87 100644 --- a/lib/ingress/binaries.go +++ b/lib/ingress/binaries_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package ingress import ( diff --git a/lib/instances/README.md b/lib/instances/README.md index a2d42172..51a245ef 100644 --- a/lib/instances/README.md +++ b/lib/instances/README.md @@ -1,12 +1,12 @@ # Instance Manager -Manages VM instance lifecycle using Cloud Hypervisor. +Manages VM instance lifecycle across multiple hypervisors (Cloud Hypervisor, QEMU on Linux; vz on macOS). ## Design Decisions ### Why State Machine? (state.go) -**What:** Single-hop state transitions matching Cloud Hypervisor's actual states +**What:** Single-hop state transitions matching hypervisor states **Why:** - Validates transitions before execution (prevents invalid operations) @@ -132,6 +132,6 @@ TestStorageOperations - metadata persistence, directory cleanup - `lib/images` - Image manager for OCI image validation - `lib/system` - System manager for kernel/initrd files -- `lib/vmm` - Cloud Hypervisor client for VM operations -- System tools: `mkfs.erofs`, `cpio`, `gzip` +- `lib/hypervisor` - Hypervisor abstraction for VM operations +- System tools: `mkfs.erofs`, `cpio`, `gzip` (Linux); `mkfs.ext4` (macOS) diff --git a/lib/instances/create.go b/lib/instances/create.go index a28d0e26..75f4e1e4 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -113,6 +113,11 @@ func (m *manager) createInstance( vsockSocket := m.paths.InstanceVsockSocket(id) log.DebugContext(ctx, "generated vsock config", "instance_id", id, "cid", vsockCID) + // Override vsock socket path for vz (uses Virtio socket, not vhost-user) + if req.Hypervisor == hypervisor.TypeVZ || (req.Hypervisor == "" && m.defaultHypervisor == hypervisor.TypeVZ) { + vsockSocket = filepath.Join(m.paths.InstanceDir(id), "vz.vsock") + } + // 5. Check instance doesn't already exist if _, err := m.loadMetadata(id); err == nil { return nil, ErrAlreadyExists @@ -706,10 +711,19 @@ func (m *manager) buildHypervisorConfig(ctx context.Context, inst *Instance, ima PCIDevices: pciDevices, KernelPath: kernelPath, InitrdPath: initrdPath, - KernelArgs: "console=ttyS0", + KernelArgs: m.kernelArgs(inst.HypervisorType), }, nil } +// kernelArgs returns the kernel command line arguments for the given hypervisor type. +// vz uses hvc0 (virtio console), all others use ttyS0 (serial port). +func (m *manager) kernelArgs(hvType hypervisor.Type) string { + if hvType == hypervisor.TypeVZ { + return "console=hvc0" + } + return "console=ttyS0" +} + func ptr[T any](v T) *T { return &v } diff --git a/lib/instances/exec_test.go b/lib/instances/exec_test.go index 64fd1ae8..94f54ef2 100644 --- a/lib/instances/exec_test.go +++ b/lib/instances/exec_test.go @@ -36,7 +36,7 @@ func waitForExecAgent(ctx context.Context, mgr *manager, instanceID string, time // This validates that the exec infrastructure handles concurrent access correctly. func TestExecConcurrent(t *testing.T) { if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { - t.Fatal("/dev/kvm not available") + t.Skip("/dev/kvm not available, skipping on this platform") } if testing.Short() { diff --git a/lib/instances/hypervisor_darwin.go b/lib/instances/hypervisor_darwin.go new file mode 100644 index 00000000..183a928e --- /dev/null +++ b/lib/instances/hypervisor_darwin.go @@ -0,0 +1,16 @@ +//go:build darwin + +package instances + +import ( + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/cloudhypervisor" + "github.com/kernel/hypeman/lib/hypervisor/qemu" + "github.com/kernel/hypeman/lib/hypervisor/vz" +) + +func init() { + platformStarters[hypervisor.TypeCloudHypervisor] = cloudhypervisor.NewStarter() + platformStarters[hypervisor.TypeQEMU] = qemu.NewStarter() + platformStarters[hypervisor.TypeVZ] = vz.NewStarter() +} diff --git a/lib/instances/hypervisor_linux.go b/lib/instances/hypervisor_linux.go new file mode 100644 index 00000000..f6abe18c --- /dev/null +++ b/lib/instances/hypervisor_linux.go @@ -0,0 +1,14 @@ +//go:build linux + +package instances + +import ( + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/cloudhypervisor" + "github.com/kernel/hypeman/lib/hypervisor/qemu" +) + +func init() { + platformStarters[hypervisor.TypeCloudHypervisor] = cloudhypervisor.NewStarter() + platformStarters[hypervisor.TypeQEMU] = qemu.NewStarter() +} diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 8411d193..f1551045 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -7,8 +7,6 @@ import ( "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" - "github.com/kernel/hypeman/lib/hypervisor/cloudhypervisor" - "github.com/kernel/hypeman/lib/hypervisor/qemu" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/paths" @@ -44,6 +42,8 @@ type Manager interface { // SetResourceValidator sets the validator for aggregate resource limit checking. // Called after initialization to avoid circular dependencies. SetResourceValidator(v ResourceValidator) + // GetVsockDialer returns a VsockDialer for the specified instance. + GetVsockDialer(ctx context.Context, instanceID string) (hypervisor.VsockDialer, error) } // ResourceLimits contains configurable resource limits for instances @@ -79,6 +79,9 @@ type manager struct { defaultHypervisor hypervisor.Type // Default hypervisor type when not specified in request } +// platformStarters is populated by platform-specific init functions. +var platformStarters = make(map[hypervisor.Type]hypervisor.VMStarter) + // NewManager creates a new instances manager. // If meter is nil, metrics are disabled. // defaultHypervisor specifies which hypervisor to use when not specified in requests. @@ -88,20 +91,23 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste defaultHypervisor = hypervisor.TypeCloudHypervisor } + // Initialize VM starters from platform-specific init functions + vmStarters := make(map[hypervisor.Type]hypervisor.VMStarter, len(platformStarters)) + for hvType, starter := range platformStarters { + vmStarters[hvType] = starter + } + m := &manager{ - paths: p, - imageManager: imageManager, - systemManager: systemManager, - networkManager: networkManager, - deviceManager: deviceManager, - volumeManager: volumeManager, - limits: limits, - instanceLocks: sync.Map{}, - hostTopology: detectHostTopology(), // Detect and cache host topology - vmStarters: map[hypervisor.Type]hypervisor.VMStarter{ - hypervisor.TypeCloudHypervisor: cloudhypervisor.NewStarter(), - hypervisor.TypeQEMU: qemu.NewStarter(), - }, + paths: p, + imageManager: imageManager, + systemManager: systemManager, + networkManager: networkManager, + deviceManager: deviceManager, + volumeManager: volumeManager, + limits: limits, + instanceLocks: sync.Map{}, + hostTopology: detectHostTopology(), // Detect and cache host topology + vmStarters: vmStarters, defaultHypervisor: defaultHypervisor, } @@ -125,14 +131,7 @@ func (m *manager) SetResourceValidator(v ResourceValidator) { // getHypervisor creates a hypervisor client for the given socket and type. // Used for connecting to already-running VMs (e.g., for state queries). func (m *manager) getHypervisor(socketPath string, hvType hypervisor.Type) (hypervisor.Hypervisor, error) { - switch hvType { - case hypervisor.TypeCloudHypervisor: - return cloudhypervisor.New(socketPath) - case hypervisor.TypeQEMU: - return qemu.New(socketPath) - default: - return nil, fmt.Errorf("unsupported hypervisor type: %s", hvType) - } + return hypervisor.NewClient(hvType, socketPath) } // getVMStarter returns the VM starter for the given hypervisor type. diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index 4bfb9b4f..7120903d 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -184,7 +184,7 @@ func cleanupOrphanedProcesses(t *testing.T, mgr *manager) { func TestBasicEndToEnd(t *testing.T) { // Require KVM access (don't skip, fail informatively) if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { - t.Fatal("/dev/kvm not available - ensure KVM is enabled and user is in 'kvm' group (sudo usermod -aG kvm $USER)") + t.Skip("/dev/kvm not available, skipping on this platform") } manager, tmpDir := setupTestManager(t) // Automatically registers cleanup @@ -1007,7 +1007,7 @@ func TestStorageOperations(t *testing.T) { func TestStandbyAndRestore(t *testing.T) { // Require KVM access (don't skip, fail informatively) if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { - t.Fatal("/dev/kvm not available - ensure KVM is enabled and user is in 'kvm' group (sudo usermod -aG kvm $USER)") + t.Skip("/dev/kvm not available, skipping on this platform") } manager, tmpDir := setupTestManager(t) // Automatically registers cleanup diff --git a/lib/instances/network_test.go b/lib/instances/network_test.go index 70181cef..70ac861c 100644 --- a/lib/instances/network_test.go +++ b/lib/instances/network_test.go @@ -254,6 +254,6 @@ func execCommand(ctx context.Context, inst *Instance, command ...string) (string // requireKVMAccess checks for KVM availability func requireKVMAccess(t *testing.T) { if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { - t.Fatal("/dev/kvm not available - ensure KVM is enabled and user is in 'kvm' group") + t.Skip("/dev/kvm not available, skipping on this platform") } } diff --git a/lib/instances/qemu_test.go b/lib/instances/qemu_test.go index 4f34384d..98d0095e 100644 --- a/lib/instances/qemu_test.go +++ b/lib/instances/qemu_test.go @@ -171,7 +171,7 @@ func (r *qemuInstanceResolver) ResolveInstance(ctx context.Context, nameOrID str func TestQEMUBasicEndToEnd(t *testing.T) { // Require KVM access if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { - t.Fatal("/dev/kvm not available - ensure KVM is enabled and user is in 'kvm' group (sudo usermod -aG kvm $USER)") + t.Skip("/dev/kvm not available, skipping on this platform") } // Require QEMU to be installed @@ -727,7 +727,7 @@ func TestQEMUEntrypointEnvVars(t *testing.T) { func TestQEMUStandbyAndRestore(t *testing.T) { // Require KVM access if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { - t.Fatal("/dev/kvm not available - ensure KVM is enabled and user is in 'kvm' group (sudo usermod -aG kvm $USER)") + t.Skip("/dev/kvm not available, skipping on this platform") } // Require QEMU to be installed diff --git a/lib/instances/volumes_test.go b/lib/instances/volumes_test.go index 2dc48143..d1614f8d 100644 --- a/lib/instances/volumes_test.go +++ b/lib/instances/volumes_test.go @@ -42,7 +42,7 @@ func execWithRetry(ctx context.Context, inst *Instance, command []string) (strin func TestVolumeMultiAttachReadOnly(t *testing.T) { // Require KVM if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { - t.Fatal("/dev/kvm not available - ensure KVM is enabled and user is in 'kvm' group") + t.Skip("/dev/kvm not available, skipping on this platform") } if testing.Short() { @@ -334,7 +334,7 @@ func createTestTarGz(t *testing.T, files map[string][]byte) *bytes.Buffer { func TestVolumeFromArchive(t *testing.T) { // Require KVM if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { - t.Fatal("/dev/kvm not available - ensure KVM is enabled and user is in 'kvm' group") + t.Skip("/dev/kvm not available, skipping on this platform") } if testing.Short() { diff --git a/lib/instances/vsock.go b/lib/instances/vsock.go new file mode 100644 index 00000000..415dcc29 --- /dev/null +++ b/lib/instances/vsock.go @@ -0,0 +1,17 @@ +package instances + +import ( + "context" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +// GetVsockDialer returns a VsockDialer for the specified instance. +func (m *manager) GetVsockDialer(ctx context.Context, instanceID string) (hypervisor.VsockDialer, error) { + inst, err := m.GetInstance(ctx, instanceID) + if err != nil { + return nil, err + } + + return hypervisor.NewVsockDialer(hypervisor.Type(inst.HypervisorType), inst.VsockSocket, inst.VsockCID) +} diff --git a/lib/middleware/oapi_auth.go b/lib/middleware/oapi_auth.go index 6f8a8254..f60b25a1 100644 --- a/lib/middleware/oapi_auth.go +++ b/lib/middleware/oapi_auth.go @@ -209,8 +209,6 @@ func isTokenEndpoint(path string) bool { } -// extractRepoFromPath extracts the repository name from a registry path. -// e.g., "/v2/builds/abc123/manifests/latest" -> "builds/abc123" // extractRepoFromPath extracts the repository name from a registry path. // Uses the docker/distribution router which properly handles repository names // that can contain slashes (e.g., "builds/abc123" from "/v2/builds/abc123/manifests/latest"). diff --git a/lib/network/README.md b/lib/network/README.md index 1e771532..c54e66a8 100644 --- a/lib/network/README.md +++ b/lib/network/README.md @@ -1,6 +1,21 @@ # Network Manager -Manages the default virtual network for instances using a Linux bridge and TAP devices. +Manages the default virtual network for instances. + +## Platform Support + +| Platform | Network Model | Implementation | +|----------|---------------|----------------| +| Linux | Bridge + TAP | Linux bridge with TAP devices per VM, iptables NAT | +| macOS | NAT | Virtualization.framework built-in NAT (192.168.64.0/24) | + +On macOS, the network manager skips bridge/TAP creation since vz provides NAT networking automatically. + +--- + +## Linux Networking + +On Linux, hypeman manages a virtual network using a Linux bridge and TAP devices. ## How Linux VM Networking Works diff --git a/lib/network/bridge_darwin.go b/lib/network/bridge_darwin.go new file mode 100644 index 00000000..6eec3940 --- /dev/null +++ b/lib/network/bridge_darwin.go @@ -0,0 +1,68 @@ +//go:build darwin + +package network + +import ( + "context" + + "github.com/kernel/hypeman/lib/logger" +) + +// checkSubnetConflicts is a no-op on macOS as we use NAT networking. +func (m *manager) checkSubnetConflicts(ctx context.Context, subnet string) error { + // NAT networking doesn't conflict with host routes + return nil +} + +// createBridge is a no-op on macOS as we use NAT networking. +// Virtualization.framework provides built-in NAT with NATNetworkDeviceAttachment. +func (m *manager) createBridge(ctx context.Context, name, gateway, subnet string) error { + log := logger.FromContext(ctx) + log.InfoContext(ctx, "macOS: skipping bridge creation (using NAT networking)") + return nil +} + +// setupIPTablesRules is a no-op on macOS as we use NAT networking. +func (m *manager) setupIPTablesRules(ctx context.Context, subnet, bridgeName string) error { + return nil +} + +// setupBridgeHTB is a no-op on macOS as we use NAT networking. +// macOS doesn't use traffic control qdiscs. +func (m *manager) setupBridgeHTB(ctx context.Context, bridgeName string, capacityBps int64) error { + return nil +} + +// createTAPDevice is a no-op on macOS as we use NAT networking. +// Virtualization.framework creates virtual network interfaces internally. +func (m *manager) createTAPDevice(tapName, bridgeName string, isolated bool, downloadBps, uploadBps, uploadCeilBps int64) error { + // On macOS with vz, network devices are created by the VMM itself + return nil +} + +// deleteTAPDevice is a no-op on macOS as we use NAT networking. +func (m *manager) deleteTAPDevice(tapName string) error { + return nil +} + +// queryNetworkState returns a stub network state for macOS. +// On macOS, we use NAT which doesn't have a physical bridge. +func (m *manager) queryNetworkState(bridgeName string) (*Network, error) { + // Return a virtual network representing macOS NAT + // The actual IP will be assigned by Virtualization.framework's DHCP + return &Network{ + Bridge: "nat", + Gateway: "192.168.64.1", // Default macOS vz NAT gateway + Subnet: "192.168.64.0/24", + }, nil +} + +// CleanupOrphanedTAPs is a no-op on macOS as we don't create TAP devices. +func (m *manager) CleanupOrphanedTAPs(ctx context.Context, runningInstanceIDs []string) int { + return 0 +} + +// CleanupOrphanedClasses is a no-op on macOS as we don't use traffic control. +func (m *manager) CleanupOrphanedClasses(ctx context.Context) int { + return 0 +} diff --git a/lib/network/bridge.go b/lib/network/bridge_linux.go similarity index 98% rename from lib/network/bridge.go rename to lib/network/bridge_linux.go index a979c111..952d7dbb 100644 --- a/lib/network/bridge.go +++ b/lib/network/bridge_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package network import ( @@ -15,22 +17,6 @@ import ( "golang.org/x/sys/unix" ) -// DeriveGateway returns the first usable IP in a subnet (used as gateway). -// e.g., 10.100.0.0/16 -> 10.100.0.1 -func DeriveGateway(cidr string) (string, error) { - _, ipNet, err := net.ParseCIDR(cidr) - if err != nil { - return "", fmt.Errorf("parse CIDR: %w", err) - } - - // Gateway is network address + 1 - gateway := make(net.IP, len(ipNet.IP)) - copy(gateway, ipNet.IP) - gateway[len(gateway)-1]++ // Increment last octet - - return gateway.String(), nil -} - // checkSubnetConflicts checks if the configured subnet conflicts with existing routes. // Returns an error if a conflict is detected, with guidance on how to resolve it. func (m *manager) checkSubnetConflicts(ctx context.Context, subnet string) error { diff --git a/lib/network/ip.go b/lib/network/ip.go new file mode 100644 index 00000000..555ad579 --- /dev/null +++ b/lib/network/ip.go @@ -0,0 +1,22 @@ +package network + +import ( + "fmt" + "net" +) + +// DeriveGateway returns the first usable IP in a subnet (used as gateway). +// e.g., 10.100.0.0/16 -> 10.100.0.1 +func DeriveGateway(cidr string) (string, error) { + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + return "", fmt.Errorf("parse CIDR: %w", err) + } + + // Gateway is network address + 1 + gateway := make(net.IP, len(ipNet.IP)) + copy(gateway, ipNet.IP) + gateway[len(gateway)-1]++ // Increment last octet + + return gateway.String(), nil +} diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 935d2d99..de752467 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -50,6 +50,7 @@ const ( const ( CreateInstanceRequestHypervisorCloudHypervisor CreateInstanceRequestHypervisor = "cloud-hypervisor" CreateInstanceRequestHypervisorQemu CreateInstanceRequestHypervisor = "qemu" + CreateInstanceRequestHypervisorVz CreateInstanceRequestHypervisor = "vz" ) // Defines values for DeviceType. @@ -82,6 +83,7 @@ const ( const ( InstanceHypervisorCloudHypervisor InstanceHypervisor = "cloud-hypervisor" InstanceHypervisorQemu InstanceHypervisor = "qemu" + InstanceHypervisorVz InstanceHypervisor = "vz" ) // Defines values for InstanceState. @@ -10622,140 +10624,140 @@ var swaggerSpec = []string{ "vXguLNOUMywIiO4IcYZOLq4QjmMeWmNoojWsCZ1mgkT9mg0OvfuwhbD5F8jhJ2xOBWeJ1oXmWFBNPBXP", "wsfgxcvTJ6MnL66DI72TURZaM/3i5avXwVGwNxgMAp+o0zuxBhmfXVydwIp1+xlXaZxNR5J+IBWfWLD3", "7HFQn/hxvl6UkIQLo4/aPlBnVmUHRlyjmN4QNNT9mU3beVZn1Lsw1BLQZouUiDmVPjvzl/yd3u9MkjJt", - "GmKoooQkYk5Evtew+f2SrA9jnkW90pDd4B1JAK2LiXoa+W29VlJgDXvHcUoZaeTv3R+FJ99ycRNzHPV2", - "vjJLZkTpvpeX+MK8qG6mRQCS73/QXdLzWXRLIzUbRfyW6Sl7eI99g/LGOQN6r1eC49//9e/r80IB2Xk2", - "Ti032tl98IXcqMZ/dNde4yJfSJb6l3GV+hdxff77v/7tVvJ9F0GYxs+ownSMvV5dyj9mRM2IKEklt8H6", - "J6MdwufI4Utp+IoDoOy1X2KcfE5EjBceRrgz8HDCfwiqgL7sd0hLNKQ/XsMGdW9OeC0zwoGfE3om5ZnT", - "Y03fli+3mUk+kZ3dc/u425Y3yxuajqZa2Rjhae7AWHWecnlDUwRf9OALs41xbIg3ynTPaMy56g/ZP2aE", - "Idg72GDynoTAp7SFho4vziS6pXEM5g4wgmXeP2SvS6zANJdK/1dkrIvGmUKCJFwRbWsmum89SAZzgcZj", - "gjKG3YFNf8jKULELrOOVBcsNEYzEoxnBERGyJWTMR8h+1AgcWOoES0WE4dBZWoXX6d/PL1HndMFwQkP0", - "d9PrOY+ymKDLLNU0vFWFXnfIUkHmhIGiqxUGasflE8Qz1eOTnhKEuCkm0FluMNrThPmziyt7HiW3+kP2", - "imjAEhZpe5ML5KSERGqGFYo4+7OmWBJVuy2PXwO6n5a7wTxMsyqUd+sQfgGnQHo9cypUhmPNsioal/dQ", - "yBw3ejRUc5pZ1pQtK8oRDquqN7+tpWB6hrPHZb3ZbxwYhaPZOFhz9OrzsecOhzCTiiclTzvq1HwJtOp1", - "qDKPOY97EVYYVIOW+ouZ7vKpVbIwXZlNaeKSo+nY46DSzJAyNKVTPF6oqq69M1jeej+gXf8+UDed6Br0", - "INFI8dVnWnSCXNs2Lmw4/x0pPppPqKfnXGgWzhMqUVg7PrZIq7vopSG15NtFtzOqxaxEDghAwdfnZRuw", - "P2Q9YDlH6DQfIO8271JzVnCUQRcdLkqToODzROPFFsLo+ryPXuez/bNEDCs6J+6Ie4YlGhPCUAbqGYlg", - "fGCn5QlkUvMwquqfW15lTsO3wNTl9l0faVsiwZbva/ROsKIh+NnGtLYeON8wG6VH0gyAlaVOKymx6iTw", - "FZlSqUTtHBB1Xj092dvbe1TXF3Yf9AY7vZ0Hr3cGRwP9/3+2PzL8+gf+vr6Oq/zCei7LHOXk6ux01yon", - "1XHUh3386PD9e6weHdBb+ehDMhbT3/bwnYQE+NnTaeFyRZ1MEtFzrE9jlc/RWvJnNjhSP9s/ulE0gjuR", - "WSV+zOpe65bfIn7Bd4pmz3A2jzCoM8G153ClxS2tR/+q9YMC80u+AevuDqnXsX9K5c1jQfCNtio98lWL", - "Zzkycsfv68q0HTVeIPJeq2ckQoJzNZHGX1BVU3b2H+4f7h3sHw4GnmP7ZSTmIR2FWqq0msDLkzMU4wUR", - "CL5BHTD0IjSO+biKvA/2Dg4fDh7t7LadhzGT2sEh16LcV6hjIfIXFwLm3lQmtbv78GBvb29wcLC732pW", - "VsFrNSmnDFZUh4d7D/d3Dnf3W0HBZ3Y+cWEU9WPhyIOkx2kaU2Nk92RKQjqhIYJADKQ/QJ0ExBLJLb4q", - "TY5xNBJWDfTKA4Vp7AFDyetnBrMtTdRNksWKpjEx72BDWmm6sPJT6MnnIaaMETHKo0w26MkGn6z1jLm1", - "5E1QJYioArpzKkGzKBQiSuLoyFDoWj4Hu1lM7E0THtg1tMSG5/yWiF5M5iQuI4ERR3qyCRcE5XhiNq2y", - "KsrmOKbRiLI086JEIyifZgL0S9MpwmOeKWOqw4aVB4EjM7ARJppdtzuxLXzUS0NrO3NDx18q+ITGnmWA", - "0WrfWpHuXGLP9weXvZ3/A36wlyxeGD5AmTF0Ex6Rfi1OEdq3Xt5F05zyIFFUnt3SmnLXhMc9mlu7DiLW", - "6A4xQ2OCrJg0Tl1wmxSDFAz+kY9hTgROyDibTIgYJR5L66l+j0wD44OiDJ0/rjJNzZzbqlsXlc0BfWuC", - "Qxvj1w76HkuutoxuCZpv/Nv1ipiwhqYoAr1VwraxgQR99CIPy0XPLq4kKtxJHhOv5YHdxWwhtXFiejRB", - "QZSVLTNAztZs+KL40NqwHmaceBmQIwTUmU/TDMjw8lXv7OX1dhKRebcyJ3ABzXhM9Ly3SrrV3MUSFKeL", - "lSOXeZOKbBBDtiWgEqxyCm4NpBK9eqCjuMLxSMZceWbzWr9E8BJ1rp+aM2Q9gy5KK1upfy9BoYLfB16K", - "0RypadhLGLBua1cIfK3bIzFiq7y8yqA+UvmF4NgE8VfxuQhLcxvPb6obzW/WUq/txDfumTt1q0nOxGO7", - "nJyfGsss5ExhyohACVHYXhkonWxDgEXQDXpaGYgwScAnOvmv1WfdDb6bHF1WWf8nSxHA38Tyb4hy00wu", - "npMIJZjRCZHKRrlVRpYzvPvg4MjE10Zksv/goN/v+094lFiknPrCG5/k79ptxbY5H+0Vffbl7Mv24Ruc", - "4bdZy8fg4vj1L8FRsJ1JsR3zEMfbckzZUenv/M/iBTyYP8eUec/+W4Vk08lSKHZle1Mts8zvR3oljIQ5", - "QnLQEtf6Jv2S/IVGzZh+IBHyRkQpPEVa/waM+7LQpy8IYi5u0qhS8HL5mKBFIDP9sNrcdooRtLFjZkzR", - "uIjxXja0PytKX64MelwKeEwJy8Mc49g8hZzNNVX4Yh4rDNy9W9qMWy5uKJuOIurBzn+YlyiigoQKQkrW", - "01CwjdN0PSr6lb+cp7WN37bRWx7p8t05+ec4XKujv5z+7d3/lRcPf9t59/z6+r/nz/52+oL+93V88fKL", - "Qk5WB+591+i7lWdq4GWsRN21RY9zrEKP4jPjUjVAzb5BiqNEf9xHJ2CgHQ1ZDz2niggcH6FhgFPat8Ds", - "hzwZBqhD3uNQma8QZ0h3ZY+Ot/THFybsRn/80dmAn+p9RPaMWFgg5+EcMhtHPMGUbQ3ZkNm+kFuIhEMb", - "/RShEKcqE0TviNY14wUaCxwWZ8PF4F30Eafpp60hA0uUvFdCryDFQuVRvm4E2Gg7K3MoZJuTCM1xnBFp", - "Ldkhy+UHmOa6E4XFlKh+7kIER03tYKYBKF4zg4tqbMPhoOvZR6Tb6Y2MqVSEodwrQSUgL+q4IJXDQYX8", - "DweH688fcxxagX6A3cv3ah1StqAPg8AwtGHGo5lS6frwBeA3hkbQL69fX2gw6H8vkeuogEW+xcYYw2ka", - "UyLNqZqKQSexcUFbge/kzOxuywW9No31Z3GLMIwnMDB6/fwSKSISygz/7oQanBMa6vXB+Q6VMtOoSDE6", - "Pjl/stVvcTEYYJvPf8U+vs5XWDtGcM6tZQsTviic5hq+XXR22tXqlKXQQtGCc9OnXKDYMJiCro/QlSTV", - "KAbYKnPEY3YyXhQeMsPVh8GW6zGtc4oj9CrX73A+lfz2QYEMrsuCLqFbG9hiDnWXeu9W5wrH1dZ+sawN", - "jnCxQtbpDaK4mRWsJn8PxIHmOav7Hjej7bLTUg/mR41i77+5BrK3qS25aSR3NSitFISYB3N/3yjsz4mp", - "djv07OIKIpexHEmGUznjqjk4AyPXBpH3VCq5HMfWKpxgOYa7Kp5MdPaKwMCvGY0tMsYgMqK+jK8eZ/09", - "Yw1+vBjvlVHZXxpabRW0bxRZ3cgQfFHJVd5gfv66MdLfZDqVaGcfMyjLMRcI9tkBzt2AeoJgjqWkU0Yi", - "dHZR3PorHB6u+9qaHu32dw4O+zuDQX9n0Mb9k+BwxdjnxyftBx/sGoP4CI+PwuiITL7A/WQR2ygcOL7F", - "C4mGTiUcBkYHLSmfJbK1amOro73lOPLPCxuvC8F1geGbBIK3i/BecR3/snoRv7Ve8eCfX3Rnn7QVw5fQ", - "2H012sQxSlDIszhif1ZorCnPmAIkshaLJKrIcQDEesVuGL9l1aUb/5im33cZEQt0fX5e8aYKMrHXvVss", - "nKdp4z7wdKNt2F2j3q2dTSnY+i4CrOucsCSBvno4ddn14+I6DNa1cAEV6p/3mJQyA2699yvWVDPeIzIf", - "ZZlP0dGvXITm1dXZaWXDMT7YORwcPuodjncOevvRYKeHd/YOersP8GCyFz7ca0iM0j5M4vMjH6oU2hwR", - "DYAHR5gJYo+ONA3loQvjTKH8kpomzhOtMaKSHmrif8E2fWVUUt0DSNdQv4kXuaq68uMLrAnVfZvCX6u/", - "uJxlSqtB8I2cZQrpv2DKeglW1V/dhaH5I/SCwzd2pl0tKGs2g2mOWTReLDev2xcdGwEiiFRckAgGswzs", - "CD3NmVbO9iyb60hiHw0vtZFSEAW2ZQxqq97b3Qq6gYV60A0MCINu4CCjH80K4QkmH3QDOxFvkGUZb3yO", - "YoJj4GFFEEamaEw/GJLTU6dS0dCYWBh2s4ns7E02Eo2MCG06yjEn+1bM5h85qr4+Rx24d/AXZC0w/ddW", - "fuxTJqH93Uf7jw4e7j46aBW1WExwPTc+gbiT5cmtZc1hmo1cgqiGpZ9cXIHw0YJNZokJk7RrL2w3zThC", - "re1RhoqMU8Xgj/qPysGaEc/GccnTYKO1ISKwTXqwhnOOdzSe08mEvfsQ3uz+Jmiy8/5A7o69xlE+kF+T", - "PCt7x5bMLjLumavG/ng6QCghG0NOXxEJK0CXRCHAn55mWFqi5uEiFuVcYKqFuBex9vf29g4fPththVd2", - "diXCGYH9tzzLczuDEolBS9R5dXmJtksIZ/p0MXSpIFIvztyi8NIZGmaDwR5Bg0p4nbY99nxY0qCwFFhj", - "+54njSC/thqLXZQFOkS95NrMEpV7ob23N3i4/+DwQTsythbPSLxfzWFsO3taLEhI6Lyy8x3wqL4+vkC6", - "dzHBYVXD39nd239w8PBwo1mpjWalBGYyoUptNLHDhwcP9vd2d9rFTvu8pvZWQIVgq7zLQ3QepPDshgcU", - "y6y32yQtfFricqjdyui+IlywHhu2STBocROMSuiVluIQUUcrUWWFtHSbaauNn8HPIvU4TWkntbrYNk5z", - "dVjmBVazMzbhy27xTQw+G+ziDiFSrfhISMgVEUZJ5HhXbvlZXQrCZ2JJUJQRCzmjGwlsAY7N0UCK1QyU", - "VfiQsmk1cHhpwDZmmJnD6nt/MK5t2MZjJP0BGq9FBrAyDl2JcBGq0co7TeXIb1UsdyzINIuxQPVY5BVT", - "loskpuymTe9ykYx5TEOkP6ib8xMex/x2pF/Jv8JatlqtTn8wKk4la+a5mZw9kzYbUhu3WMJf9Sq3alEu", - "IPm3zffbkFe4jQPOG637VBtvJlz3itH3JUSvXqLZ3x00BTU1dFoJZ1oO9d6Ut1uU9VG8i8I+zpNNeI7E", - "zGlNzYKt6sGV9fpWC6daq0K4ljUB1HE+PXdJqQrX0mWhVoK43cFa3XvtZrMtSVgdff/wwcODlre1vkjV", - "XpF59QsU63myQqFu2KnzNlrb4YPDR4/29h882t1IP3IHHQ3703TYUd6fWk6Zms72YAD/22hS5qjDP6WG", - "447qhCr5YT57Qp9WkG5xgaLB6l6V9bzYSWfmVxXwdiruCm3puKJyldKEdchkQsBxNDJw6xWTqQX0tJpD", - "iFMcUrXwWID4FmIcUN6kdhGgRe+1yXpAavtGeKKIgNMImY2Lm3AdNzj6T2PZ1XDhsPWlT5mNm6zIl/VR", - "jQ1pgoKimoeihYPAYITvBPw2Bya6xbLi1dfPoSJRt5QGrn78Y1q0z3LrcD1PdFscbPsus/iT2pa3v7ad", - "JaujoiTXIb5KhDaToNYIIOKojYPdI5E9N2TC9REUNf5gBeDnfTUal69jr7zvXrm7XUjdzcdtl3hv+Tsj", - "wTYfr3SCv8mH9ZupgI92DhbkRd/dCkr4sMmcrzSlPUlcOZDaxVVqEqzb20mo1Bh1SJKqhYvAd5bp1mbn", - "Pcd5h15k/MoxU4NHXyNq+2plmPb/kEQ65SM2N8jaw7WlPW2MjfSrq6f18BVjE9pEAtVwi9r1aKlW1BFY", - "VbPGFI8Bg8/GJU+z+kWqDerUNJn4BeW4AgGuUM06y3WlP620stJMmvfGnK9+YVEfKl01n88EmTW/1gf6", - "mjMqbQD36pkmzD1VQcGeswAygNUgyE30ZT/A6rCPc/w+HwGsZSxRLTefWUcpz+2zx3D3/JXLOEAnrguY", - "Rj3L4uMvq3bksGp5M1aVP3In+F7Cs/xnBUdroq0achZjdFdXWNKsi4SZoGpxqQWCDU4jWBBxnBk0BEkB", - "i4Cfi8Eh2P3TJzBTJx5t9RlhRNAQHV+cAZYkmOGp3rLrcxTTCQkXYUxsrPLS2S5ctX95ctYzlyzyZHhQ", - "vEABQFwWquOLM0iAY8sGBIP+bh9y/fKUMJzS4CjY6+9Aih8NBljiNtxhg0friNJ0CJLsLLIS97FpokEr", - "U86kAc7uYFArQ4GLJCPbv0njYTHitbVSaOr8LMdbLIXgOk3ATv9TN9gf7Gw0n7V5QXzDXjGcqRkX9AOB", - "aT7YEAifNegZM1a1y0RMbMMCZ4OjX6vY+uubT2+6gcySBGsV0YCrgFXKZZMKQyTCiJFbe7nxNz7uo0tj", - "k0CSkKKCmnEZkEizJIwUFv3pB4RFOKNzMmSWE5scL1jATY4EaQ5s4uiraGaGNrtvSJhI9ZhHixp08+62", - "dXegjVQBvHF9jjxhYdpQqMPHHU1eJBlyb0IowjBTRZodkxDphsAh5oS+98bCQ3yv39t9mr9zFV2qvF2r", - "u5SFcRYVArBaScN7x9pUhLApnm6IR194Bi3s/Muh0E7SMB4RE9aaLtSMM/OcjTOmMvM8FvxWEqHlkb2S", - "YcGizea8EpfJn0cTuBZhLnHqMbfNFLc/3pDFp/6QHUeJu3Rr07jiWHKb+8oEKFCJ8mTCQ+bVoOUI635G", - "Y1dSrKaoEuhqGGhROQz081RgrZJlcoZwCAEJ+scycDoGm7kAcbdVn2uIGUp5msVaeYDtMcmxKn3A7TYc", - "x0gB/rhvtRAFmDSsR5JQEJ+t9LfLly8Q8E8orQLNiuhyWANlWvrlSWL1gP0he4LDGTKCEZInDgMaDYOi", - "hMYWCLFMEiObej2QrH+F2kJmmC6N/trv666M0D5Cv340vRxprEmTkeI3hA2DT11UejGlapaN83dvGhbc", - "4Ku5rKA86hiGtOXuA+sVlnizYWaYRYhbBhAvEEYFrZVNsjFlWCya6tHwTDXHu5jr0rZZcZfvYDDYWn+e", - "YZfqUVcqDTWmflqSzrtfTTBZobwsmEq157QYYPYufGTE8R1Ixsc4cle0fqoAa1QAa7uUhDt8bxXA7Y80", - "+mTQNyYmvrImoaFEkZPQKRY4IQqSVP/qx3kILaX6b3f6CL4GY8lXkbdbAk9doX+zhNj7jbWf8ipKgAv7", - "d4B/MG6RoQzGfXRX4+LY5MfN61HeK3SEzXKI2PVbH8+I+hEwbnBXrNQlUvyO+Htf8OcZsSpSAbQaN9uG", - "zPRl07Z+BUIQnEjbi2msbZlLmFPvkjCFoOqg7Nt/nZoN0eVvYz59e4QMCGNbc1Ha1Hi5D1gLRQtL+Mhk", - "Dsm/swl1whlmUyJRx8jP3//1b1c37vd//dvWjfv9X/8Gct+2VVChu7zi4dsj9HdC0h6O6Zy4xUDEJJkT", - "sUB7A1uLA1550vPIIRuyV0Rlgsk83kivC2BiOgSVncF6KMuIRBJACImzJzYQxriYPCaeo2UDyjul6O6S", - "pWtXUFqAlooOB+BkkzKqKI4Rz5TJcQnzgEs5xUTMmoPy4HVv2ZL/dD1/UeS9MtjbMxPckMGYiqEeujNF", - "NE2fqHN5+WSrj0DdN1gBwU5gNxTdWEug/5MnredJhqNUGQpA2fCmUmbGRl/bqW1zF862pqyNzd42ASnm", - "ibZd3WJ+qt0tPG9+uDkvnM8VduoyiTf7wj5/vb6Coq1syq+3zw73lmFu0+QXIPse1iTq2AzHeSKTSi7+", - "74X0d8KASyUcci6MuEmfcmcWzglnk5iGCvXcXGyRydzqqSLIfWEHr+ysEXbrqkfol0XFdiXgrFFo5LFn", - "dyk9aoNuIkaKWwQFrv2UJOtQ55TKkOtvS9jSC3EKgLRALOi0jEXrfDun8HsuclYq5nnZV0eQd+flsUNn", - "rC4b7oApntYY4ndkhLU0H6V7N/cJm6/yXXRVU1Y4gX4s1BzcnRZ01w4hH5rfJ49QVAOb5oKzPLF4E3rZ", - "1OPfcKPtCJ6FXxLhqNpM1KSXKJZlPkXhjIQ3ZkG2uM8qjeDM1f/59nqAyZ++gfS30/8p7lsYjgWsVhmL", - "ZzbnyLezFWGEjUzFr3f8aBHMA2SI0hg7R6pJ54HlgoVbf6gTyDuRDPViPPeIki6yOHaO+DkRqsgiX+an", - "2x+1ftBCT3bUtlIXuXr1vEdYyCEmx4CuUSFxSaO/rrZsNsws5SeatLGvAFQOMZqV0S/YfxM6hfJsjn/a", - "fWrzOf5p96nJ6PinvWOT03HrmyHL4K5Y811rr/cY+bTySqtAA9ZkUjuv0/byVnei8Nkc+puofPkEf2p9", - "bbS+MrhWKn55OYNvqPrZLPHf55wgRzYftOGViz/7g6l8d+t6shhZKvxX8cXbxCZcFJnZbdmw+xcgR3OM", - "K/Pflj7UgiBXagcOdc9OuzbpvkmVnweI35FH1c3jzrVEO+7du1OPkzGdZjyT5YB2qLFAZFGQtsKA75v+", - "WojnRg32B8bSwV2KjjtXUH/i/TdSnesbapi3rWG7Rnl2re5GeS6Oatprz26GP7XnVtpzCVyrtec8j+u3", - "VJ/NIN9Nf3b45gO4vcL8U4O+Cw1aZpMJDSlhqshBtBTVYlOY3cN7Jcw64Uun0RUm3FqDLpIrr1ZOLPJ+", - "j0iEfPC7V5xdorP7GR/LTUR85FTVQhg266o/Gj4M7pY5372Oep9R7Fm5np1fGzSXQ2I+XX81JO/J3YPw", - "3A0ZMlf87q1h6m9RjqhIcSRJTEKFbmc0nME9Ef0b9G+ukeA0fZtfDN06Qs8g/rR8VRUG70giKI4hYzqP", - "TbL/t/MkeXu0nDPi+vwcPjJXREx2iLdHyOWJyGlM6lblex96FTGWCr2wt1k6esMFj2OTnfmthmdpfVv2", - "Rkhxh3bIfLdDGLm1HdIJelu6KPK24aaIQ8Lnepe+E+V3m5Pjm7UojgQAztxZJyxquCWioea/I7Iz8KY+", - "anlfxUzjG19XWZrMcz7N8wtUUBmnaVv0tdMELJ4nyQocRp1SQQCpIp6pv0gVEWHq1VrsbkJu1MGh+UPh", - "G1NdtVJezpSg8IHK3r32giowNaRd5Qrz1zxJAlPrLsG+ShRffu+n3uGywah3pnS556fM2OTaTpXZl+7t", - "1CSHLYEC2Ua81uUr0+APr7m4WjHfGQ2/g6VXzIJCCRkWjRewt0URnvt1aQE2slgZyDu7Li+NuHeNNGJr", - "9/zhaaTAjz84lYRcQMFv6Qrw3Z/ospLFUSL3DlT8KippdZ3Ve31+vtVENKZmdCPJiJ/msA30/MPLFCiC", - "dv+oxdT/xPkCVjkLNUGoRhvd2ayVAoljnunel9KnQmEQuZCKJMZgn2Qx3LyDsHqbwACXC590EVUS0nB3", - "wWVVKnoxZGMy0fIwJUKPrT+H9GyF7eEzay8Vzsn3wtDgj2HXQkZVMOWwaoJarbpImrpkqj7bKc//+tlT", - "egqGarXwikSdmN6YaoJoLlGsH7ZWWrqmKsvXTs/w+ZSV1x3yXbs1OJsj8x+Bw53V2Jqrq3nv2NozUiYW", - "x39go/1sTa7la2LDwpQOdqUClf0hOydK6DZYEBTyOIZ6BEZ/304FD7ehaF6Y0shUz4PJAcNrfp3AiCcX", - "V9DOpIDvDpn+Y7lsW32irvrb2fbLNb4/U7Dzf7CeYxa4iiz8G/7TrbP5UUAjDckGEuXpKk2cpz8VcVuH", - "96fZei/NVjiLzVfTmQocglIsbaVlv4lqy5NtfzQPZ+tO9BUOZ9euWsSPoe3a5PLrhnELvBdEadcUEZMW", - "4O5pkuf5/+/p1S8NOLcEUGLKsQl+KWDqivzRsPvrx8mV4bhRlNyd0pZLufHD0NZdSz47BxeoVobHfSFz", - "g2luJZAAvex9EuUCZyttM1d/Cqrt5aqlq7vWLZf/Mxk+cx9SUTcmrzTWH7K8tJrLMKqtq64zrVBE5Y3p", - "wVpPfeSvgGfsPFsGb8gURyGOQ5N3Pi8FZ8o3ygbr61WpPOI3o7diEM9G5zXwZF6y7D6ZHH6cgN0r10QD", - "jLPq1Mr49Gvb5i6i060w2yA23a3gZ2R6i8j0ErDaVGAxBe0st7KVyPLyGVANqt9QSCVXSr5dXPtnyOuv", - "hx4OTxul9c+I9jtTCIoroWen9z+MvUxzFR69ra2Cni1vVHYNraJgC6JUkJ6r/xIZgFl4GFujXj2pP2Sv", - "Z8T9hagLpSSRraAfLxBlUPDGFcH7s0SCc1VU2G+usmRI5KngybFdzRrjpXU5SN9BzMb5KrqeEng0yZK8", - "WPyzx1D+WpjIPjTBNIa4UgdS8j4kJJKAk1v1MpPeUL+8nuTaWa6I0cwLSYWZVDxxe392ijo4U7w3JUzv", - "RVGzKRV8TqN6zeBKvU7fbMFC/ApG2vQDTaukt7bezTLhVfEW5UWqbMGdAj/d7gQ/xUQ9w7DebW3kOSAq", - "zlGMxZRs/RQl91mUlL1JTm5UJEq7C1HtHEwt/T7f4jJU7ny826tQ1z+OT6SUkfUeJgyY50Zf0x2sHwsF", - "B3cnH+767tX1PfahPyPOwC3du4IOdI8+hHnOQxyjiMxJzFMoRW3aBt0gE7EtrHu0vR3rdjMu1dHh4HAQ", - "fHrz6f8HAAD//wgwYbc14gAA", + "GmKoooQkYk5Evtew+f2SrA9jnkW90pDd4B1JtJibf9C4XczW09Jv8LUSBWt4PI5Tykgjk+/+KIz5loub", + "mOOot/OV+TIjSve9vMQX5kV1Ry0WkBwJgu6Sss+iWxqp2Sjit0xP2cOA7BuUN8650Hu9Ehz//q9/X58X", + "WsjOs3FqWdLO7oMvZEk1JqS79loY+UKy1L+Mq9S/iOvz3//1b7eS77sIwjR+RhXOY4z26lL+MSNqRkRJ", + "NLkN1j8ZFRE+Rw5fSsNXvABl1/0S9+RzImK88HDDnYGHHf5DUAX0Zb9DWqwh/fEaXqh7cxJsmRsO/OzQ", + "MynPnB5r+rbMuc1M8ons7J7bx922DFre0HQ01RrHCE9zL8aqQ5XLG5oi+KIHX5htjGNDvFGme0ZjzlV/", + "yP4xIwzB3sEGk/ckBD6lzTR0fHEm0S2NY7B5gBEsC4Ahe11iBaa5VPq/ImNdNM4UEiThimiDM9F960Ey", + "mAs0HhOUMexObfpDVoaKXWAdryxYbohgJB7NCI6IkC0hYz5C9qNG4MBSJ1gqIgyHztIqvE7/fn6JOqcL", + "hhMaor+bXs95lMUEXWappuGtKvS6Q5YKMicMtF2tNVA7Lp8gnqken/SUIMRNMYHOcqvRHinMn11c2UMp", + "udUfsldEA5awSBudXCAnJSRSM6xQxNmfNcWSqNptefwa0P203A3mYZpVobxbh/ALOArS65lToTIca5ZV", + "Ubu8J0PmzNGjppojzbK6bFlRjnBYVV36bc0F0zMcQC4rz34LwSgczRbCmvNXn6M99zqEmVQ8KbnbUafm", + "UKBV10OVecx53IuwwqAatNRfzHSXj66ShenKbEoTlxxNxx4vlWaGlKEpneLxQlUV7p3B8tb7Ae3694G6", + "6VjXoAeJRoqvPtiiE+TatvFjwyHwSPHRfEI9PedCs/CgUInC2hmyRVrdRS8NqSXfLrqdUS1mJXJAAAq+", + "Pi8bgv0h6wHLOUKn+QB5t3mXmrOCtwy66HBRmgQFxycaL7YQRtfnffQ6n+2fJWJY0Tlx59wzLNGYEIYy", + "UM9IBOMDOy1PIJOah1FV/9zyKnMkvgX2Lrfv+kgbFAm2fF+jd4IVDcHZNqa19cAhh9koPZJmAKwsdVpJ", + "iVXHga/IlEolaoeBqPPq6cne3t6jur6w+6A32OntPHi9Mzga6P//s/254dc/9ff1dVzlF9Z9WeYoJ1dn", + "p7tWOamOoz7s40eH799j9eiA3spHH5KxmP62h+8kLsDPnk4LvyvqZJKInmN9Gqt83taSU7PBm/rZTtKN", + "QhLcscwq8WNW91q3/BZBDL6jNHuQs3mYQZ0Jrj2MKy1uaT36V60fFJhfchBYn3dIvd79UypvHguCb7RV", + "6ZGvWjzLkZE7fodXpu2o8QKR91o9IxESnKuJNP6Cqpqys/9w/3DvYP9wMPCc3S8jMQ/pKNRSpdUEXp6c", + "oRgviEDwDeqAoRehcczHVeR9sHdw+HDwaGe37TyMmdQODrkW5b5CHQuRv7g4MPemMqnd3YcHe3t7g4OD", + "3f1Ws7IKXqtJOWWwojo83Hu4v3O4u98KCj6z84mLpaifDUceJD1O05gaI7snUxLSCQ0RRGMg/QHqJCCW", + "SG7xVWlyjKORsGqgVx4oTGMPGEquPzOYbWlCb5IsVjSNiXkHG9JK04WVn0JPPjcxZYyIUR5qskFPNgJl", + "rWfMrSVvgiqRRBXQnVMJmkWhEFESR0eGQtfyOdjNYmJvmvDArqElNjznt0T0YjIncRkJjDjSk024ICjH", + "E7NplVVRNscxjUaUpZkXJRpB+TQToF+aThEe80wZUx02rDwInJuBjTDR7LrdsW3hqF4aWtuZGzr+UsEn", + "NPYsA4xW+9aKdOcSe74/uOzt/B/wg71k8cLwAcqMoZvwiPRrwYrQvvXyLprmlEeKovLsltaUuyY87tHc", + "2nUQsUZ3iBkaE2TFpHHqgtukGKRg8I98DHMicELG2WRCxCjxWFpP9XtkGhgfFGXo/HGVaWrm3Fbduqhs", + "DuhbExzaQL920PdYcrVldEvQfOPfrlfExDY0hRLorRK2jY0m6KMXeWwuenZxJVHhTvKYeC1P7S5mC6mN", + "E9OjiQyirGyZAXK2ZsMXxYfWhvUw48TLgBwhoM58mmZAhpevemcvr7eTiMy7lTmBC2jGY6LnvVXSreYu", + "oKA4YqwcucybVGSDGLItAZVglVNwayCV6NUDHcUVjkcy5sozm9f6JYKXqHP91Bwk6xl0UVrZSv17CQoV", + "/D7wUozmSE3DXsKAdVu7QuBr3R6JEVvl5VUG9ZHKLwTHJpK/is9FbJrbeH5T3Wh+s5Z6bSe+cc/cqVtN", + "ciYe2+Xk/NRYZiFnClNGBEqIwvbeQOl4G6Isgm7Q08pAhEkCPtHJf60+8G7w3eTossr6P1kKA/4mln9D", + "qJtmcvGcRCjBjE6IVDbUrTKynOHdBwdHJsg2IpP9Bwf9ft9/wqPEIuXUF+P4JH/Xbiu2zflor+izL2df", + "tg/f4CC/zVo+BhfHr38JjoLtTIrtmIc43pZjyo5Kf+d/Fi/gwfw5pswbANAqLptOluKxK9ubapllfj/S", + "K2EkzBGSg5a41jfpl+QvNGrG9AOJkDcsSuEp0vo3YNyXxT99QSRzcZ1GlSKYy8cELaKZ6YfV5rZTjKCN", + "HTNjisZFoPeyof1ZofpyZeTjUtRjSlge6xjH5inkbK6pwhf4WGHg7t3SZtxycUPZdBRRD3b+w7xEERUk", + "VBBXsp6Ggm2cputR0a/85TytbRC3DeHySJfvzsk/x+FaHf3l9G/v/q+8ePjbzrvn19f/PX/2t9MX9L+v", + "44uXXxRysjp677uG4K08UwMvYyX0ri16nGMVehSfGZeqAWr2DVIcJfrjPjoBA+1oyHroOVVE4PgIDQOc", + "0r4FZj/kyTBAHfIeh8p8hThDuit7dLylP74wYTf644/OBvxU7yOyZ8TCAjkP55DZOOIJpmxryIbM9oXc", + "QiQc2uinCIU4VZkgeke0rhkv0FjgsDgbLgbvoo84TT9tDRlYouS9EnoFKRYqD/V1I8BG21mZQyHbnERo", + "juOMSGvJDlkuP8A0150oLKZE9XMXIjhqagczDUDxmhlcVGMbDgddzz4i3U5vZEylIgzlXgkqAXlRxwWp", + "HA4q5H84OFx//pjj0Ar0A+xevlzrkLIFfRgEhqENMx7NlErXhy8AvzE0gn55/fpCg0H/e4lcRwUs8i02", + "xhhO05gSaU7VVAw6iY0L2gp8J2dmd1su6LVprD+LW4RhPIGB0evnl0gRkVBm+Hcn1OCc0FCvD853qJSZ", + "RkWK0fHJ+ZOtfovbwQDbfP4r9vF1vsLaMYJzbi1bmPBF4TTX8O2is9OuVqcshRaKFpybPuUCxYbBFHR9", + "hK4kqUYxwFaZIx6zk/Gi8JAZrj4MtlyPaZ1THKFXuX6H86nkVxAKZHBdFnQJ3drAFnOou9R7tzpXOK62", + "9otlbXCEixWyTm8Qxc2sYDX5eyAONM9Z3fe4GW2XnZZ6MD9qFHv/zTWQvU1tyU3DuatBaaUgxDyi+/uG", + "Yn9OYLXboWcXVxC+jOVIMpzKGVfNwRkYuTaIvKdSyeU4tlbhBMuB3FXxZEK0VwQGfs2QbJExBpER9WV8", + "m2Dr7xlw8OMFeq8Mzf7S+GqrpX2j8OpGruALTa4yCPPz1w2U/ibTqYQ8+zhCWZi5aLDPjnLuBtQTCXMs", + "JZ0yEqGzi+L+X+H1cN3X1vRot79zcNjfGQz6O4M2PqAEhyvGPj8+aT/4YNdYxUd4fBRGR2TyBT4oi9hG", + "68DxLV5INHR64TAwimhJAy2RrdUdW53vLQeTf17seF0SrosO3yQavF2Y94qL+ZfVK/mtlYsH//yi2/uk", + "rSy+hMbuq9Em3lGCQp7FEfuzQmNNecYeIJE1WyRRRbYDINYrdsP4Lasu3TjJNP2+y4hYoOvz84pLVZCJ", + "vfjdYuE8TRv3gacbbcPuGh1v7WxKEdd3EWVd54QlCfTVY6rL/h8X3GGwroUfqNABvWellBlw671fsaaa", + "BR+R+SjLfIqOfuXCNK+uzk4rG47xwc7h4PBR73C8c9DbjwY7Pbyzd9DbfYAHk73w4V5DipT2sRKfH/5Q", + "pdDmsGgAPHjDTCR7dKRpKI9fGGcK5TfVNHGeaI0RlZRREwQMBuoro5fqHkC6hvpNvMj11ZUfX2BNqO7b", + "FP5a/cXlLFNaDYJv5CxTSP8FU9ZLsPr+6i4MzR+hFxy+sTPtakFZMxxMc8yi8WK5ed3I6NgwEEGk4oJE", + "MJhlYEfoac60crZn2VxHEvtoeKkNl4JQsC1jVVsd3+5W0A0s1INuYEAYdAMHGf1oVghPMPmgG9iJeCMt", + "y3jj8xYTHAMPKyIxMkVj+sGQnJ46lYqGxs7CsJtNZGevs5FoZERo03mOOd63Yjb/yFH19TnqwOWDvyBr", + "hum/tvKznzIJ7e8+2n908HD30UGr0MViguu58QkEnyxPbi1rDtNs5FJFNSz95OIKhI8WbDJLTKykXXth", + "u2nGEWptjzJU5J4qBn/Uf1SO2Ix4No5L7gYbsg1hgW0ShTUcdryj8ZxOJuzdh/Bm9zdBk533B3J37DWO", + "8oH8muRZ2UW2ZHaRcc9cOvYH1QFCCdkYd/qKSFgBuiQKAf70NMPSEjWPGbEo56JTLcS9iLW/t7d3+PDB", + "biu8srMrEc4I7L/lWZ7bGZRIDFqizqvLS7RdQjjTpwukSwWRenHmKoWXztAwGwz2CBpUYuy07bHnw5IG", + "haXAGtv3PGkE+bXVWOyiLNAh9CXXZpao3Avtvb3Bw/0Hhw/akbG1eEbi/WoOY9vZI2NBQkLnlZ3vgFv1", + "9fEF0r2LCQ6rGv7O7t7+g4OHhxvNSm00KyUwkwlVaqOJHT48eLC/t7vTLoDa5zq1VwMqBFvlXR6i8yCF", + "Zzc8oFhmvd0maeHTEpfj7VaG+BUxg/UAsU0iQovrYFRCr7QUjIg6WokqK6SlK01bbfwMfhapx2lKQKnV", + "xbbBmqtjMy+wmp2xCV/2jW9i8NmIF3cSkWrFR0JqrogwSiLHu3LLz+pSEEMTS4KijFjIGd1IYAtwbM4H", + "UqxmoKzCh5RNq9HDSwO2McPMHFZf/oNxbcM2HiPpj9J4LTKAlfHqSoSLeI1WLmoqR36rYrljQaZZjAWq", + "BySvmLJcJDFlN216l4tkzGMaIv1B3Zyf8DjmtyP9Sv4V1rLVanX6g1FxNFkzz83k7MG02ZDauMUS/qpX", + "uVULdQHJv22+34YMw20ccN6Q3afaeDMxu1eMvi8hevUmzf7uoCmyqaHTSkzTcrz3przdoqyP4l0o9nGe", + "ccJzLmaObGoWbFUPrqzXt1o42loVx7WsCaCO8+m5m0pVuJZuDLUSxO1O1+reazebbUnC6uj7hw8eHrS8", + "svVFqvaKHKxfoFjPkxUKdcNOnbfR2g4fHD56tLf/4NHuRvqRO+ho2J+mw47y/tQSy9R0tgcD+N9GkzJH", + "Hf4pNRx3VCdUSRLz2RP6tIJ0i1sUDVb3qvznxU46M7+qgLdTcVdoS8cVlauUMKxDJhMCjqORgVuvmEwt", + "qqfVHEKc4pCqhccCxLcQ6IDyJrXbAC16r03WA1LbN8ITRQScRshsXFyH67jB0X8ay66GC4etb37KbNxk", + "Rb6sj2psSBMZFNU8FC0cBAYjfMfgtzkw0S2WFa++fg4VibqlhHD14x/Ton2+W4frecrb4mDbd6PFn962", + "vP217SxZHRUluQ7xVSK0mQS1RgBhR20c7B6J7LkmE64Po6jxBysAP++r0bh8J3vlpffKBe5C6m4+brsU", + "fMvfGQm2+XilE/xNPqxfTwV8tHOwIC/67lZQwodN5nylKfdJ4gqD1G6vUpNq3V5RQqXGqEOSVC1cGL6z", + "TLc2O+85zjv0IuNXDpwaPPoaodtXK2O1/4dk0ykfsblB1h6uLe1pY4CkX109rYevGJvQZhOohlvU7khL", + "taKiwKrqNaaMDBh8Njh5mtVvU21QsabJxC8ox5UKcCVr1lmuK/1ppZWVZtK8N+Z89QvL+1Dp6vp8Jsis", + "+bU+2tecUWkDuFdPN2EuqwoK9pwFkAGsBkFuoi/7AVaHfZzj9/kIYC1jiWoJ+sw6Shlvnz2GC+ivXNoB", + "OnFdwDTqqRYff1ndI4dVy5uxqhCSO8H3Ep7lPys4WhNt1ZCzGKO7utaSZl0kzARVi0stEGxwGsGCiOPM", + "oCFIClgE/FwMDhHvnz6BmTrxaKvPCCOChuj44gywJMEMT/WWXZ+jmE5IuAhjYgOWl8524b79y5Oznrlp", + "kWfEgzIGCgDiUlEdX5xBFhxbQCAY9Hf7kPWXp4ThlAZHwV5/B/L8aDDAErfhIhs8WkeUpkOQZGeRlbiP", + "TRMNWplyJg1wdgeDWkEKXGQa2f5NGg+LEa+tlUJT8Wc53mIpDtdpAnb6n7rB/mBno/msTQ7iG/aK4UzN", + "uKAfCEzzwYZA+KxBz5ixql1OYmIbFjgbHP1axdZf33x60w1kliRYq4gGXAWsUi6bVBgiEUaM3Nobjr/x", + "cR9dGpsEMoUUtdSMy4BEmiVhpLDoTz8gLMIZnZMhs5zYJHrBAq5zJEhzYBNMX0UzM7TZfUPCRKrHPFrU", + "oJt3t627A22kCuCNK3XkWQvThpIdPu5okiPJkHuzQhGGmSpy7ZisSDcEDjEn9L03IB7ie/3e7tP8navt", + "UuXtWt2lLIyzqBCA1Zoa3ovWpjaEzfN0Qzz6wjNoYedfDoV2kobxiJiw1nShZpyZ52ycMZWZ57Hgt5II", + "LY/svQwLFm025zW5TBI9msDdCHOTU4+5baa4/fGGLD71h+w4StzNW5vLFceS2wRYJkCBSpRnFB4yrwYt", + "R1j3Mxq74mI1RZVAV8NAi8phoJ+nAmuVLJMzhEMISNA/loHTMdjMBYi7rfpcQ8xQytMs1soDbI/JkFXp", + "A6644ThGCvDHfauFKMCkYT2ShIL4bKW/Xb58gYB/QpEVaFZEl8MaKNPSL88UqwfsD9kTHM6QEYyQQXEY", + "0GgYFMU0tkCIZZIY2dTrgWT9K1QZMsN0afTXfl93ZYT2Efr1o+nlSGNNmowUvyFsGHzqotKLKVWzbJy/", + "e9Ow4AZfzWUF5VHHMKQtdylYr7DEmw0zwyxC3DKAeIEwKmitbJKNKcNi0VSZhmeqOd7F3Jm2zYoLfQeD", + "wdb68wy7VI+6UmmoMfXTknTe/WqCyQrlZcFUqkKnxQCzF+IjI47vQDI+xpG7p/VTBVijAljbpSTc4Xur", + "AG5/pNEng74xMfGVNQkNxYqchE6xwAlRkKn6Vz/OQ2gp1X+700fwNRhLvoq83RJ46gr9myXE3m+sApXX", + "UwJc2L8D/INxizRlMO6juxoXxyZJbl6Z8l6hI2yWQ8Su3/p4RtSPgHGDu2KlLpvid8Tf+4I/z4hVkQqg", + "1bjZNqSnL5u29SsQguBE2l5MY23LXMKcepeEKQT1B2Xf/uvUbIgufxvz6dsjZEAY2+qL0ubHy33AWiha", + "WMJHJn1I/p3NqhPOMJsSiTpGfv7+r3+7CnK//+vftoLc7//6N5D7tq2HCt3ltQ/fHqG/E5L2cEznxC0G", + "IibJnIgF2hvYghzwypOjRw7ZkL0iKhNM5vFGel0AE9MhqOwM1kNZRiSSAELInj2xgTDGxeQx8RwtG1De", + "KUV3lyxdu4LSArRUdDgAJ5uUUUVxjHimTKJLmAdcyikmYtYclAeve8uW/Kfr+Ysi75XB3p6Z4IYMxtQO", + "9dCdKadp+kSdy8snW30E6r7BCgh2Aruh6MZaAv2fPGk9TzIcpcpQAMqGN5XSMzb62k5tm7twtjWlbmz2", + "tgnIM0+07eoW81PtbuF588PNeeF8rrBTl0682Rf2+ev1lRZtZVN+vX12uLcMc5srvwDZ97AmUcemOc6z", + "mVQS8n8vpL8TBlyq45BzYcRNDpU7s3BOOJvENFSo5+Ziy03mVk8VQe4LO3hlZ42wW1c9Qr8sKrYrAWeN", + "QiOPPbtL6VEbdBMxUtwiKHDtpyRZhzqnVIZcf1vCll6IUwCkBWJBp2UsWufbOYXfc5GzUjHPC8A6grw7", + "L48dOmN12XAHTPG0xhC/IyOspfko3bu5T9h8le+iK52ywgn0Y6Hm4O60oLt2CPnQ/D55hKIa2DQXnOXZ", + "xZvQy+Yf/4YbbUfwLPySCEfVZqImvUSxLPMpCmckvDELshV+VmkEZ64I0LfXA0wS9Q2kv53+T3HfwnAs", + "YLXKWDyzOUe+na0II2xkKn6940eLYB4gQ5TG2DlSTToPLBcs3PpDnUDeiWSoV+S5R5R0kcWxc8TPiVBF", + "KvkyP93+qPWDFnqyo7aVusjVq+c9wkIOMTkGdI0Kicsc/XW1ZbNhZik/0aSNfQWgcojRrIx+wf6b0CmU", + "Z3P80+5Tm8/xT7tPTUbHP+0dm5yOW98MWQZ3xZrvWnu9x8inlVdaBRqwJpPfeZ22l7e6E4XPJtLfROXL", + "J/hT62uj9ZXBtVLxy2safEPVz6aK/z7nBDmy+aANr1z82R9M5btb15PFyFL1v4ov3iY24aJIz25rh92/", + "ADmaY1yZ/7b0oRYEuVI7cKh7dtq1mfdNvvw8QPyOPKpuHneuJdpx796depyM6TTjmSwHtEOhBSKLqrQV", + "Bnzf9NdCPDdqsD8wlg7uUnTcuYL6E++/kepc31DDvG0h2zXKs2t1N8pzcVTTXnt2M/ypPbfSnkvgWq09", + "53lcv6X6bAb5bvqzwzcfwO0V5p8a9F1o0DKbTGhICVNFDqKlqBabwuwe3ith1glfOo2uMOHWGnSRXHm1", + "cmKR93tEIuSD373i7BKd3c/4WG4i4iOnqhbCsFlX/dHwYXC3zPnuddT7jGLPykXt/NqguRwS8+n6qyF5", + "T+4ehOduyJC5CnhvDVN/i3JERYojSWISKnQ7o+EM7ono36B/c40Ep+nb/GLo1hF6BvGn5auqMHhHEkFx", + "DBnTeWyS/b+dJ8nbo+WcEdfn5/CRuSJiskO8PUIuT0ROY1K3Kt/70KuIsVTohb3N0tEbLngcm+zMbzU8", + "S+vbsjdCiju0Q+a7HcLIre2QTtDb0kWRtw03RRwSPte79J0ov9ucHN+sRXEkAHDmzjphUcMtEQ01/x2R", + "nYE39VHL+ypmGt/4usrSZJ7zaZ5foILKOE3boq+dJmDxPElW4DDqlAoCSBXxTP1FqogIU7TWYncTcqMO", + "Ds0fCt+YEquVGnOmBIUPVPbutRdUgSkk7SpXmL/mSRKYgncJ9lWi+PJ7P/UOlw1GvTOlyz0/ZcYm13aq", + "zL50b6cmOWwJFMg24rUuX5kGf3jNxdWK+c5o+B0svWIWFErIsGi8gL0tivDcr0sLsJHFykDe2XV5acS9", + "a6QRW7vnD08jBX78wakk5AKqfktXgO/+RJeVLI4SuXeg4ldRSavrrN7r8/OtJqIxhaMbSUb8NIdtoOcf", + "XqZAEbT7Ry2m/ifOF7DKWagJQjXa6M5mrRRIHPNM976UPhUKg8iFVCQxBvski+HmHYTV2wQGuFz4pIuo", + "kpCGuwsuq1LRiyEbk4mWhykRemz9OaRnK2wPn1l7qXBOvheGBn8MuxYyqoIph1UT1GrVRdLUJVP12U55", + "/tfPntJTMFSrhVck6sT0xlQTRHOJYv2wtdLSNVVZvnZ6hs+nrLzukO/arcHZHJn/CBzurMbWXF3Ne8fW", + "npEysTj+AxvtZ2tyLV8TGxamdLArFajsD9k5UUK3wYKgkMcx1CMw+vt2Kni4DUXzwpRGpnoeTA4YXvPr", + "BEY8ubiCdiYFfHfI9B/LZdvqE3XV3862X67x/ZmCnf+D9RyzwFVk4d/wn26dzY8CGmlINpAoT1dp4jz9", + "qYjbOrw/zdZ7abbCWWy+ms5U4BCUYmkrLftNVFuebPujeThbd6KvcDi7dtUifgxt1yaXXzeMW+C9IEq7", + "poiYtAB3T5M8z/9/T69+acC5JYASU45N8EsBU1fkj4bdXz9OrgzHjaLk7pS2XMqNH4a27lry2Tm4QLUy", + "PO4LmRtMcyuBBOhl75MoFzhbaZu5+lNQbS9XLV3dtW65/J/J8Jn7kIq6MXmlsf6Q5aXVXIZRbV11nWmF", + "IipvTA/WeuojfwU8Y+fZMnhDpjgKcRyavPN5KThTvlE2WF+vSuURvxm9FYN4NjqvgSfzkmX3yeTw4wTs", + "XrkmGmCcVadWxqdf2zZ3EZ1uhdkGseluBT8j01tEppeA1aYCiyloZ7mVrUSWl8+AalD9hkIquVLy7eLa", + "P0Nefz30cHjaKK1/RrTfmUJQXAk9O73/Yexlmqvw6G1tFfRseaOya2gVBVsQpYL0XP2XyADMwsPYGvXq", + "Sf0hez0j7i9EXSgliWwF/XiBKIOCN64I3p8lEpyrosJ+c5UlQyJPBU+O7WrWGC+ty0H6DmI2zlfR9ZTA", + "o0mW5MXinz2G8tfCRPahCaYxxJU6kJL3ISGRBJzcqpeZ9Ib65fUk185yRYxmXkgqzKTiidv7s1PUwZni", + "vSlhei+Kmk2p4HMa1WsGV+p1+mYLFuJXMNKmH2haJb219W6WCa+KtygvUmUL7hT46XYn+Ckm6hmG9W5r", + "I88BUXGOYiymZOunKLnPoqTsTXJyoyJR2l2Iaudgaun3+RaXoXLn491ehbr+cXwipYys9zBhwDw3+pru", + "YP1YKDi4O/lw13evru+xD/0ZcQZu6d4VdKB79CHMcx7iGEVkTmKeQilq0zboBpmIbWHdo+3tWLebcamO", + "DgeHg+DTm0//PwAA//+YHpPWP+IAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/resources/cpu.go b/lib/resources/cpu.go index 883cbff7..edac6e50 100644 --- a/lib/resources/cpu.go +++ b/lib/resources/cpu.go @@ -1,12 +1,7 @@ package resources import ( - "bufio" "context" - "fmt" - "os" - "strconv" - "strings" ) // CPUResource implements Resource for CPU discovery and tracking. @@ -15,7 +10,7 @@ type CPUResource struct { instanceLister InstanceLister } -// NewCPUResource discovers host CPU capacity from /proc/cpuinfo. +// NewCPUResource discovers host CPU capacity. func NewCPUResource() (*CPUResource, error) { capacity, err := detectCPUCapacity() if err != nil { @@ -59,78 +54,6 @@ func (c *CPUResource) Allocated(ctx context.Context) (int64, error) { return total, nil } -// detectCPUCapacity reads /proc/cpuinfo to determine total vCPU count. -// Returns threads × cores × sockets. -func detectCPUCapacity() (int64, error) { - file, err := os.Open("/proc/cpuinfo") - if err != nil { - return 0, fmt.Errorf("open /proc/cpuinfo: %w", err) - } - defer file.Close() - - var ( - siblings int - physicalIDs = make(map[int]bool) - hasSiblings bool - hasPhysicalID bool - ) - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - - parts := strings.SplitN(line, ":", 2) - if len(parts) != 2 { - continue - } - - key := strings.TrimSpace(parts[0]) - value := strings.TrimSpace(parts[1]) - - switch key { - case "siblings": - if !hasSiblings { - siblings, _ = strconv.Atoi(value) - hasSiblings = true - } - case "physical id": - physicalID, _ := strconv.Atoi(value) - physicalIDs[physicalID] = true - hasPhysicalID = true - } - } - - if err := scanner.Err(); err != nil { - return 0, err - } - - // Calculate total vCPUs - if hasSiblings && hasPhysicalID { - // siblings = threads per socket, physicalIDs = number of sockets - sockets := len(physicalIDs) - if sockets < 1 { - sockets = 1 - } - return int64(siblings * sockets), nil - } - - // Fallback: count processor entries - file.Seek(0, 0) - scanner = bufio.NewScanner(file) - count := 0 - for scanner.Scan() { - if strings.HasPrefix(scanner.Text(), "processor") { - count++ - } - } - if count > 0 { - return int64(count), nil - } - - // Ultimate fallback - return 1, nil -} - // isActiveState returns true if the instance state indicates it's consuming resources. func isActiveState(state string) bool { switch state { diff --git a/lib/resources/cpu_darwin.go b/lib/resources/cpu_darwin.go new file mode 100644 index 00000000..8931af85 --- /dev/null +++ b/lib/resources/cpu_darwin.go @@ -0,0 +1,13 @@ +//go:build darwin + +package resources + +import ( + "runtime" +) + +// detectCPUCapacity returns the number of logical CPUs on macOS. +// Uses runtime.NumCPU() which calls sysctl on macOS. +func detectCPUCapacity() (int64, error) { + return int64(runtime.NumCPU()), nil +} diff --git a/lib/resources/cpu_linux.go b/lib/resources/cpu_linux.go new file mode 100644 index 00000000..606cd718 --- /dev/null +++ b/lib/resources/cpu_linux.go @@ -0,0 +1,83 @@ +//go:build linux + +package resources + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" +) + +// detectCPUCapacity reads /proc/cpuinfo to determine total vCPU count. +// Returns threads × cores × sockets. +func detectCPUCapacity() (int64, error) { + file, err := os.Open("/proc/cpuinfo") + if err != nil { + return 0, fmt.Errorf("open /proc/cpuinfo: %w", err) + } + defer file.Close() + + var ( + siblings int + physicalIDs = make(map[int]bool) + hasSiblings bool + hasPhysicalID bool + ) + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + + parts := strings.SplitN(line, ":", 2) + if len(parts) != 2 { + continue + } + + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + + switch key { + case "siblings": + if !hasSiblings { + siblings, _ = strconv.Atoi(value) + hasSiblings = true + } + case "physical id": + physicalID, _ := strconv.Atoi(value) + physicalIDs[physicalID] = true + hasPhysicalID = true + } + } + + if err := scanner.Err(); err != nil { + return 0, err + } + + // Calculate total vCPUs + if hasSiblings && hasPhysicalID { + // siblings = threads per socket, physicalIDs = number of sockets + sockets := len(physicalIDs) + if sockets < 1 { + sockets = 1 + } + return int64(siblings * sockets), nil + } + + // Fallback: count processor entries + file.Seek(0, 0) + scanner = bufio.NewScanner(file) + count := 0 + for scanner.Scan() { + if strings.HasPrefix(scanner.Text(), "processor") { + count++ + } + } + if count > 0 { + return int64(count), nil + } + + // Ultimate fallback + return 1, nil +} diff --git a/lib/resources/disk.go b/lib/resources/disk.go index 2b6bf76d..087438d9 100644 --- a/lib/resources/disk.go +++ b/lib/resources/disk.go @@ -3,11 +3,8 @@ package resources import ( "context" "strings" - "syscall" "github.com/c2h5oh/datasize" - "github.com/kernel/hypeman/cmd/api/config" - "github.com/kernel/hypeman/lib/paths" ) // DiskResource implements Resource for disk space discovery and tracking. @@ -19,48 +16,17 @@ type DiskResource struct { volumeLister VolumeLister } -// NewDiskResource discovers disk capacity for the data directory. -// If cfg.DiskLimit is set, uses that as capacity; otherwise auto-detects via statfs. -func NewDiskResource(cfg *config.Config, p *paths.Paths, instLister InstanceLister, imgLister ImageLister, volLister VolumeLister) (*DiskResource, error) { - var capacity int64 - - if cfg.DiskLimit != "" { - // Parse configured limit - var ds datasize.ByteSize - if err := ds.UnmarshalText([]byte(cfg.DiskLimit)); err != nil { - return nil, err - } - capacity = int64(ds.Bytes()) - } else { - // Auto-detect from filesystem - var stat syscall.Statfs_t - if err := syscall.Statfs(cfg.DataDir, &stat); err != nil { - return nil, err - } - // Total space = blocks * block size - capacity = int64(stat.Blocks) * int64(stat.Bsize) - } - - return &DiskResource{ - capacity: capacity, - dataDir: cfg.DataDir, - instanceLister: instLister, - imageLister: imgLister, - volumeLister: volLister, - }, nil -} - // Type returns the resource type. func (d *DiskResource) Type() ResourceType { return ResourceDisk } -// Capacity returns the total disk space in bytes. +// Capacity returns the disk capacity in bytes. func (d *DiskResource) Capacity() int64 { return d.capacity } -// Allocated returns total disk space used by images, OCI cache, volumes, and overlays. +// Allocated returns currently allocated disk space. func (d *DiskResource) Allocated(ctx context.Context) (int64, error) { breakdown, err := d.GetBreakdown(ctx) if err != nil { @@ -73,13 +39,12 @@ func (d *DiskResource) Allocated(ctx context.Context) (int64, error) { func (d *DiskResource) GetBreakdown(ctx context.Context) (*DiskBreakdown, error) { var breakdown DiskBreakdown - // Get image sizes (exported rootfs disks) + // Get image sizes if d.imageLister != nil { imageBytes, err := d.imageLister.TotalImageBytes(ctx) if err == nil { breakdown.Images = imageBytes } - // Get OCI layer cache size ociCacheBytes, err := d.imageLister.TotalOCICacheBytes(ctx) if err == nil { breakdown.OCICache = ociCacheBytes @@ -94,7 +59,7 @@ func (d *DiskResource) GetBreakdown(ctx context.Context) (*DiskBreakdown, error) } } - // Get overlay sizes from instances (rootfs overlays + volume overlays) + // Get overlay sizes from instances if d.instanceLister != nil { instances, err := d.instanceLister.ListInstanceAllocations(ctx) if err == nil { diff --git a/lib/resources/disk_darwin.go b/lib/resources/disk_darwin.go new file mode 100644 index 00000000..8a5d32f3 --- /dev/null +++ b/lib/resources/disk_darwin.go @@ -0,0 +1,49 @@ +//go:build darwin + +package resources + +import ( + "os" + + "github.com/c2h5oh/datasize" + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/paths" + "golang.org/x/sys/unix" +) + +// NewDiskResource discovers disk capacity on macOS. +func NewDiskResource(cfg *config.Config, p *paths.Paths, instLister InstanceLister, imgLister ImageLister, volLister VolumeLister) (*DiskResource, error) { + var capacity int64 + + if cfg.DiskLimit != "" { + // Parse configured limit + var ds datasize.ByteSize + if err := ds.UnmarshalText([]byte(cfg.DiskLimit)); err != nil { + return nil, err + } + capacity = int64(ds.Bytes()) + } else { + // Auto-detect from filesystem using statfs + var stat unix.Statfs_t + dataDir := cfg.DataDir + if err := unix.Statfs(dataDir, &stat); err != nil { + // Fallback: try to stat the root if data dir doesn't exist yet + if os.IsNotExist(err) { + if err := unix.Statfs("/", &stat); err != nil { + return nil, err + } + } else { + return nil, err + } + } + capacity = int64(stat.Blocks) * int64(stat.Bsize) + } + + return &DiskResource{ + capacity: capacity, + dataDir: cfg.DataDir, + instanceLister: instLister, + imageLister: imgLister, + volumeLister: volLister, + }, nil +} diff --git a/lib/resources/disk_linux.go b/lib/resources/disk_linux.go new file mode 100644 index 00000000..e6cc8fb1 --- /dev/null +++ b/lib/resources/disk_linux.go @@ -0,0 +1,42 @@ +//go:build linux + +package resources + +import ( + "syscall" + + "github.com/c2h5oh/datasize" + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/paths" +) + +// NewDiskResource discovers disk capacity for the data directory. +// If cfg.DiskLimit is set, uses that as capacity; otherwise auto-detects via statfs. +func NewDiskResource(cfg *config.Config, p *paths.Paths, instLister InstanceLister, imgLister ImageLister, volLister VolumeLister) (*DiskResource, error) { + var capacity int64 + + if cfg.DiskLimit != "" { + // Parse configured limit + var ds datasize.ByteSize + if err := ds.UnmarshalText([]byte(cfg.DiskLimit)); err != nil { + return nil, err + } + capacity = int64(ds.Bytes()) + } else { + // Auto-detect from filesystem + var stat syscall.Statfs_t + if err := syscall.Statfs(cfg.DataDir, &stat); err != nil { + return nil, err + } + // Total space = blocks * block size + capacity = int64(stat.Blocks) * int64(stat.Bsize) + } + + return &DiskResource{ + capacity: capacity, + dataDir: cfg.DataDir, + instanceLister: instLister, + imageLister: imgLister, + volumeLister: volLister, + }, nil +} diff --git a/lib/resources/memory.go b/lib/resources/memory.go index 52cebd78..0e334cff 100644 --- a/lib/resources/memory.go +++ b/lib/resources/memory.go @@ -1,12 +1,7 @@ package resources import ( - "bufio" "context" - "fmt" - "os" - "strconv" - "strings" ) // MemoryResource implements Resource for memory discovery and tracking. @@ -15,7 +10,7 @@ type MemoryResource struct { instanceLister InstanceLister } -// NewMemoryResource discovers host memory capacity from /proc/meminfo. +// NewMemoryResource discovers host memory capacity. func NewMemoryResource() (*MemoryResource, error) { capacity, err := detectMemoryCapacity() if err != nil { @@ -58,34 +53,3 @@ func (m *MemoryResource) Allocated(ctx context.Context) (int64, error) { } return total, nil } - -// detectMemoryCapacity reads /proc/meminfo to determine total memory. -func detectMemoryCapacity() (int64, error) { - file, err := os.Open("/proc/meminfo") - if err != nil { - return 0, err - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "MemTotal:") { - // Format: "MemTotal: 16384000 kB" - fields := strings.Fields(line) - if len(fields) >= 2 { - kb, err := strconv.ParseInt(fields[1], 10, 64) - if err != nil { - return 0, fmt.Errorf("parse MemTotal: %w", err) - } - return kb * 1024, nil // Convert KB to bytes - } - } - } - - if err := scanner.Err(); err != nil { - return 0, err - } - - return 0, fmt.Errorf("MemTotal not found in /proc/meminfo") -} diff --git a/lib/resources/memory_darwin.go b/lib/resources/memory_darwin.go new file mode 100644 index 00000000..01989aa9 --- /dev/null +++ b/lib/resources/memory_darwin.go @@ -0,0 +1,17 @@ +//go:build darwin + +package resources + +import ( + "golang.org/x/sys/unix" +) + +// detectMemoryCapacity returns total physical memory on macOS using sysctl. +func detectMemoryCapacity() (int64, error) { + // Use sysctl to get hw.memsize + memsize, err := unix.SysctlUint64("hw.memsize") + if err != nil { + return 0, err + } + return int64(memsize), nil +} diff --git a/lib/resources/memory_linux.go b/lib/resources/memory_linux.go new file mode 100644 index 00000000..1ed59d26 --- /dev/null +++ b/lib/resources/memory_linux.go @@ -0,0 +1,42 @@ +//go:build linux + +package resources + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" +) + +// detectMemoryCapacity reads /proc/meminfo to determine total memory. +func detectMemoryCapacity() (int64, error) { + file, err := os.Open("/proc/meminfo") + if err != nil { + return 0, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "MemTotal:") { + // Format: "MemTotal: 16384000 kB" + fields := strings.Fields(line) + if len(fields) >= 2 { + kb, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + return 0, fmt.Errorf("parse MemTotal: %w", err) + } + return kb * 1024, nil // Convert KB to bytes + } + } + } + + if err := scanner.Err(); err != nil { + return 0, err + } + + return 0, fmt.Errorf("MemTotal not found in /proc/meminfo") +} diff --git a/lib/resources/network_darwin.go b/lib/resources/network_darwin.go new file mode 100644 index 00000000..4e662975 --- /dev/null +++ b/lib/resources/network_darwin.go @@ -0,0 +1,49 @@ +//go:build darwin + +package resources + +import ( + "context" + + "github.com/kernel/hypeman/cmd/api/config" +) + +// NetworkResource implements Resource for network bandwidth discovery and tracking. +// On macOS, network rate limiting is not supported. +type NetworkResource struct { + capacity int64 // bytes per second (set to high value on macOS) + instanceLister InstanceLister +} + +// NewNetworkResource creates a network resource on macOS. +// Network capacity detection and rate limiting are not supported on macOS. +func NewNetworkResource(ctx context.Context, cfg *config.Config, instLister InstanceLister) (*NetworkResource, error) { + // Default to 10 Gbps as a reasonable high limit on macOS + // Network rate limiting is not enforced on macOS + return &NetworkResource{ + capacity: 10 * 1024 * 1024 * 1024 / 8, // 10 Gbps in bytes/sec + instanceLister: instLister, + }, nil +} + +// Type returns the resource type. +func (n *NetworkResource) Type() ResourceType { + return ResourceNetwork +} + +// Capacity returns the network capacity in bytes per second. +func (n *NetworkResource) Capacity() int64 { + return n.capacity +} + +// Allocated returns currently allocated network bandwidth. +// On macOS, this is always 0 as rate limiting is not supported. +func (n *NetworkResource) Allocated(ctx context.Context) (int64, error) { + return 0, nil +} + +// AvailableFor returns available network bandwidth. +// On macOS, this always returns the full capacity. +func (n *NetworkResource) AvailableFor(ctx context.Context, requested int64) (int64, error) { + return n.capacity, nil +} diff --git a/lib/resources/network.go b/lib/resources/network_linux.go similarity index 73% rename from lib/resources/network.go rename to lib/resources/network_linux.go index 41ba3d8e..6fa285f1 100644 --- a/lib/resources/network.go +++ b/lib/resources/network_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package resources import ( @@ -7,7 +9,6 @@ import ( "strconv" "strings" - "github.com/c2h5oh/datasize" "github.com/kernel/hypeman/cmd/api/config" "github.com/kernel/hypeman/lib/logger" "github.com/vishvananda/netlink" @@ -139,50 +140,3 @@ func getInterfaceSpeed(iface string) (int64, error) { return speed, nil } - -// ParseBandwidth parses a bandwidth string like "10Gbps", "1GB/s", "125MB/s". -// Handles both bit-based (bps) and byte-based (/s) formats. -// Returns bytes per second. -func ParseBandwidth(limit string) (int64, error) { - limit = strings.TrimSpace(limit) - limit = strings.ToLower(limit) - - // Handle bps variants (bits per second) - if strings.HasSuffix(limit, "bps") { - // Remove "bps" suffix - numPart := strings.TrimSuffix(limit, "bps") - numPart = strings.TrimSpace(numPart) - - // Check for multiplier prefix - var multiplier int64 = 1 - if strings.HasSuffix(numPart, "g") { - multiplier = 1000 * 1000 * 1000 - numPart = strings.TrimSuffix(numPart, "g") - } else if strings.HasSuffix(numPart, "m") { - multiplier = 1000 * 1000 - numPart = strings.TrimSuffix(numPart, "m") - } else if strings.HasSuffix(numPart, "k") { - multiplier = 1000 - numPart = strings.TrimSuffix(numPart, "k") - } - - bits, err := strconv.ParseInt(strings.TrimSpace(numPart), 10, 64) - if err != nil { - return 0, fmt.Errorf("invalid number: %s", numPart) - } - - // Convert bits to bytes - return (bits * multiplier) / 8, nil - } - - // Handle byte-based variants (e.g., "125MB/s", "1GB") - limit = strings.TrimSuffix(limit, "/s") - limit = strings.TrimSuffix(limit, "ps") - - var ds datasize.ByteSize - if err := ds.UnmarshalText([]byte(limit)); err != nil { - return 0, fmt.Errorf("parse as bytes: %w", err) - } - - return int64(ds.Bytes()), nil -} diff --git a/lib/resources/resource_test.go b/lib/resources/resource_test.go index 7868d9b3..3fb9d66a 100644 --- a/lib/resources/resource_test.go +++ b/lib/resources/resource_test.go @@ -2,6 +2,7 @@ package resources import ( "context" + "runtime" "testing" "github.com/kernel/hypeman/cmd/api/config" @@ -353,6 +354,9 @@ func TestGetFullStatus_ReturnsAllResourceAllocations(t *testing.T) { // TestNetworkResource_Allocated verifies network allocation tracking // uses max(download, upload) since they share the physical link. func TestNetworkResource_Allocated(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("network rate limiting not supported on this platform") + } cfg := &config.Config{ DataDir: t.TempDir(), NetworkLimit: "1Gbps", // 125MB/s diff --git a/lib/resources/util.go b/lib/resources/util.go new file mode 100644 index 00000000..619037c8 --- /dev/null +++ b/lib/resources/util.go @@ -0,0 +1,56 @@ +package resources + +import ( + "fmt" + "strconv" + "strings" + + "github.com/c2h5oh/datasize" +) + +// ParseBandwidth parses a bandwidth string like "10Gbps", "1GB/s", "125MB/s". +// Handles both bit-based (bps) and byte-based (/s) formats. +// Returns bytes per second. +func ParseBandwidth(limit string) (int64, error) { + limit = strings.TrimSpace(limit) + limit = strings.ToLower(limit) + + // Handle bps variants (bits per second) + if strings.HasSuffix(limit, "bps") { + // Remove "bps" suffix + numPart := strings.TrimSuffix(limit, "bps") + numPart = strings.TrimSpace(numPart) + + // Check for multiplier prefix + var multiplier int64 = 1 + if strings.HasSuffix(numPart, "g") { + multiplier = 1000 * 1000 * 1000 + numPart = strings.TrimSuffix(numPart, "g") + } else if strings.HasSuffix(numPart, "m") { + multiplier = 1000 * 1000 + numPart = strings.TrimSuffix(numPart, "m") + } else if strings.HasSuffix(numPart, "k") { + multiplier = 1000 + numPart = strings.TrimSuffix(numPart, "k") + } + + bits, err := strconv.ParseInt(strings.TrimSpace(numPart), 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid number: %s", numPart) + } + + // Convert bits to bytes + return (bits * multiplier) / 8, nil + } + + // Handle byte-based variants (e.g., "125MB/s", "1GB") + limit = strings.TrimSuffix(limit, "/s") + limit = strings.TrimSuffix(limit, "ps") + + var ds datasize.ByteSize + if err := ds.UnmarshalText([]byte(limit)); err != nil { + return 0, fmt.Errorf("parse as bytes: %w", err) + } + + return int64(ds.Bytes()), nil +} diff --git a/lib/system/init/logger.go b/lib/system/init/logger.go index 6d0a5217..588c8bfb 100644 --- a/lib/system/init/logger.go +++ b/lib/system/init/logger.go @@ -17,12 +17,17 @@ func NewLogger() *Logger { l := &Logger{} // Open serial console for output - // ttyS0 for x86_64, ttyAMA0 for ARM64 (PL011 UART) - if f, err := os.OpenFile("/dev/ttyAMA0", os.O_WRONLY, 0); err == nil { - l.console = f - } else if f, err := os.OpenFile("/dev/ttyS0", os.O_WRONLY, 0); err == nil { - l.console = f - } else { + // hvc0 for Virtualization.framework (vz) on macOS + // ttyAMA0 for ARM64 PL011 UART (cloud-hypervisor) + // ttyS0 for x86_64 (QEMU, cloud-hypervisor) + consoles := []string{"/dev/hvc0", "/dev/ttyAMA0", "/dev/ttyS0"} + for _, console := range consoles { + if f, err := os.OpenFile(console, os.O_WRONLY, 0); err == nil { + l.console = f + break + } + } + if l.console == nil { // Fallback to stdout l.console = os.Stdout } diff --git a/lib/system/init/mount.go b/lib/system/init/mount.go index 50ebc079..07894d01 100644 --- a/lib/system/init/mount.go +++ b/lib/system/init/mount.go @@ -49,16 +49,20 @@ func mountEssentials(log *Logger) error { log.Info("mount", "mounted devpts/shm") // Set up serial console now that /dev is mounted - // ttyS0 for x86_64, ttyAMA0 for ARM64 (PL011 UART) - if _, err := os.Stat("/dev/ttyAMA0"); err == nil { - log.SetConsole("/dev/ttyAMA0") - redirectToConsole("/dev/ttyAMA0") - } else if _, err := os.Stat("/dev/ttyS0"); err == nil { - log.SetConsole("/dev/ttyS0") - redirectToConsole("/dev/ttyS0") + // hvc0 for Virtualization.framework (vz) on macOS + // ttyAMA0 for ARM64 PL011 UART (cloud-hypervisor) + // ttyS0 for x86_64 (QEMU, cloud-hypervisor) + consoles := []string{"/dev/hvc0", "/dev/ttyAMA0", "/dev/ttyS0"} + for _, console := range consoles { + if _, err := os.Stat(console); err == nil { + log.SetConsole(console) + redirectToConsole(console) + log.Info("mount", "using console "+console) + break + } } - log.Info("mount", "redirected to serial console") + log.Info("mount", "console setup complete") return nil } diff --git a/lib/system/initrd.go b/lib/system/initrd.go index e3891bdf..3ef4c103 100644 --- a/lib/system/initrd.go +++ b/lib/system/initrd.go @@ -35,14 +35,14 @@ func (m *manager) buildInitrd(ctx context.Context, arch string) (string, error) return "", fmt.Errorf("create oci client: %w", err) } - // Inspect Alpine base to get digest - digest, err := ociClient.InspectManifest(ctx, alpineBaseImage) + // Inspect Alpine base to get digest (always use Linux platform since this is for guest VMs) + digest, err := ociClient.InspectManifestForLinux(ctx, alpineBaseImage) if err != nil { return "", fmt.Errorf("inspect alpine manifest: %w", err) } - // Pull and unpack Alpine base - if err := ociClient.PullAndUnpack(ctx, alpineBaseImage, digest, rootfsDir); err != nil { + // Pull and unpack Alpine base (always use Linux platform since this is for guest VMs) + if err := ociClient.PullAndUnpackForLinux(ctx, alpineBaseImage, digest, rootfsDir); err != nil { return "", fmt.Errorf("pull alpine base: %w", err) } @@ -71,7 +71,7 @@ func (m *manager) buildInitrd(ctx context.Context, arch string) (string, error) } // Download and add kernel headers tarball (for DKMS support) - if err := downloadKernelHeaders(arch, rootfsDir); err != nil { + if err := downloadKernelHeaders(ctx, arch, rootfsDir); err != nil { return "", fmt.Errorf("download kernel headers: %w", err) } @@ -162,7 +162,7 @@ func computeInitrdHash(arch string) string { } // downloadKernelHeaders downloads kernel headers tarball and adds it to the initrd rootfs -func downloadKernelHeaders(arch, rootfsDir string) error { +func downloadKernelHeaders(ctx context.Context, arch, rootfsDir string) error { url, ok := KernelHeaderURLs[DefaultKernelVersion][arch] if !ok { // No headers available for this arch, skip (non-fatal) @@ -178,7 +178,12 @@ func downloadKernelHeaders(arch, rootfsDir string) error { }, } - resp, err := client.Get(url) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("create request: %w", err) + } + + resp, err := client.Do(req) if err != nil { return fmt.Errorf("http get: %w", err) } diff --git a/lib/vm_metrics/collector_darwin.go b/lib/vm_metrics/collector_darwin.go new file mode 100644 index 00000000..1b2a1405 --- /dev/null +++ b/lib/vm_metrics/collector_darwin.go @@ -0,0 +1,20 @@ +//go:build darwin + +package vm_metrics + +import "fmt" + +// ReadProcStat is not available on macOS (/proc does not exist). +func ReadProcStat(pid int) (uint64, error) { + return 0, fmt.Errorf("read proc stat: not supported on macOS") +} + +// ReadProcStatm is not available on macOS (/proc does not exist). +func ReadProcStatm(pid int) (rssBytes, vmsBytes uint64, err error) { + return 0, 0, fmt.Errorf("read proc statm: not supported on macOS") +} + +// ReadTAPStats is not available on macOS (/sys does not exist). +func ReadTAPStats(tapName string) (rxBytes, txBytes uint64, err error) { + return 0, 0, fmt.Errorf("read TAP stats: not supported on macOS") +} diff --git a/lib/vm_metrics/collector.go b/lib/vm_metrics/collector_linux.go similarity index 99% rename from lib/vm_metrics/collector.go rename to lib/vm_metrics/collector_linux.go index f9ba89b9..507627d3 100644 --- a/lib/vm_metrics/collector.go +++ b/lib/vm_metrics/collector_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package vm_metrics import ( diff --git a/lib/vm_metrics/collector_test.go b/lib/vm_metrics/collector_test.go index 6905a35b..5c4ca688 100644 --- a/lib/vm_metrics/collector_test.go +++ b/lib/vm_metrics/collector_test.go @@ -1,3 +1,5 @@ +//go:build linux + package vm_metrics import ( diff --git a/lib/vm_metrics/manager_test.go b/lib/vm_metrics/manager_test.go index 851771e7..67a94c45 100644 --- a/lib/vm_metrics/manager_test.go +++ b/lib/vm_metrics/manager_test.go @@ -1,3 +1,5 @@ +//go:build linux + package vm_metrics import ( diff --git a/lib/vm_metrics/metrics_test.go b/lib/vm_metrics/metrics_test.go index 65bee0d3..c14b4a0b 100644 --- a/lib/vm_metrics/metrics_test.go +++ b/lib/vm_metrics/metrics_test.go @@ -1,3 +1,5 @@ +//go:build linux + package vm_metrics import ( diff --git a/lib/vmm/binaries_darwin.go b/lib/vmm/binaries_darwin.go new file mode 100644 index 00000000..370c027c --- /dev/null +++ b/lib/vmm/binaries_darwin.go @@ -0,0 +1,34 @@ +//go:build darwin + +package vmm + +import ( + "fmt" + + "github.com/kernel/hypeman/lib/paths" +) + +// CHVersion represents Cloud Hypervisor version +type CHVersion string + +const ( + V48_0 CHVersion = "v48.0" + V49_0 CHVersion = "v49.0" +) + +// SupportedVersions lists supported Cloud Hypervisor versions. +// On macOS, Cloud Hypervisor is not supported (use vz instead). +var SupportedVersions = []CHVersion{} + +// ErrNotSupportedOnMacOS indicates Cloud Hypervisor is not available on macOS +var ErrNotSupportedOnMacOS = fmt.Errorf("cloud-hypervisor is not supported on macOS; use vz hypervisor instead") + +// ExtractBinary is not supported on macOS +func ExtractBinary(p *paths.Paths, version CHVersion) (string, error) { + return "", ErrNotSupportedOnMacOS +} + +// GetBinaryPath is not supported on macOS +func GetBinaryPath(p *paths.Paths, version CHVersion) (string, error) { + return "", ErrNotSupportedOnMacOS +} diff --git a/lib/vmm/binaries.go b/lib/vmm/binaries_linux.go similarity index 98% rename from lib/vmm/binaries.go rename to lib/vmm/binaries_linux.go index 319884a2..73064a41 100644 --- a/lib/vmm/binaries.go +++ b/lib/vmm/binaries_linux.go @@ -1,3 +1,5 @@ +//go:build linux + package vmm import ( diff --git a/lib/vmm/client_test.go b/lib/vmm/client_test.go index 2162551a..33febedb 100644 --- a/lib/vmm/client_test.go +++ b/lib/vmm/client_test.go @@ -1,3 +1,5 @@ +//go:build linux + package vmm import ( diff --git a/openapi.yaml b/openapi.yaml index d87dedd5..4d13dcef 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -176,7 +176,7 @@ components: $ref: "#/components/schemas/VolumeMount" hypervisor: type: string - enum: [cloud-hypervisor, qemu] + enum: [cloud-hypervisor, qemu, vz] description: Hypervisor to use for this instance. Defaults to server configuration. example: cloud-hypervisor skip_kernel_headers: @@ -306,7 +306,7 @@ components: example: false hypervisor: type: string - enum: [cloud-hypervisor, qemu] + enum: [cloud-hypervisor, qemu, vz] description: Hypervisor running this instance example: cloud-hypervisor diff --git a/scripts/e2e-install-test.sh b/scripts/e2e-install-test.sh new file mode 100755 index 00000000..41a506dd --- /dev/null +++ b/scripts/e2e-install-test.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# +# Hypeman E2E Install Test +# +# Runs a full install → verify → uninstall cycle. +# Platform-agnostic: works on both Linux and macOS. +# + +set -e + +# Colors +RED='\033[38;2;255;110;110m' +GREEN='\033[38;2;92;190;83m' +YELLOW='\033[0;33m' +NC='\033[0m' + +info() { echo -e "${GREEN}[INFO]${NC} $1"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +error() { echo -e "${RED}[ERROR]${NC} $1"; exit 1; } +pass() { echo -e "${GREEN}[PASS]${NC} $1"; } +fail() { echo -e "${RED}[FAIL]${NC} $1"; exit 1; } + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +OS=$(uname -s | tr '[:upper:]' '[:lower:]') + +cd "$REPO_DIR" + +# ============================================================================= +# Phase 1: Clean slate +# ============================================================================= +info "Phase 1: Cleaning previous installation..." +bash scripts/uninstall.sh 2>/dev/null || true + +# ============================================================================= +# Phase 2: Install from source +# ============================================================================= +info "Phase 2: Installing from source..." +BRANCH=$(git rev-parse --abbrev-ref HEAD) +BRANCH="$BRANCH" bash scripts/install.sh + +# ============================================================================= +# Phase 3: Wait for service +# ============================================================================= +info "Phase 3: Waiting for service to be healthy..." + +PORT=8080 +TIMEOUT=60 +ELAPSED=0 + +while [ $ELAPSED -lt $TIMEOUT ]; do + if curl -sf "http://localhost:${PORT}/health" >/dev/null 2>&1; then + pass "Service is responding on port ${PORT}" + break + fi + sleep 2 + ELAPSED=$((ELAPSED + 2)) +done + +if [ $ELAPSED -ge $TIMEOUT ]; then + # Dump logs for debugging + if [ "$OS" = "darwin" ]; then + LOG_FILE="$HOME/Library/Application Support/hypeman/logs/hypeman.log" + if [ -f "$LOG_FILE" ]; then + warn "Service logs (last 50 lines):" + tail -50 "$LOG_FILE" || true + else + warn "No log file found at $LOG_FILE" + fi + warn "launchctl list:" + launchctl list | grep hypeman || true + fi + fail "Service did not become healthy within ${TIMEOUT}s" +fi + +# ============================================================================= +# Phase 4: Validate installation +# ============================================================================= +info "Phase 4: Validating installation..." + +# Check binaries +if [ "$OS" = "darwin" ]; then + [ -x /usr/local/bin/hypeman-api ] || fail "hypeman-api binary not found" + [ -x /usr/local/bin/vz-shim ] || fail "vz-shim binary not found" + pass "Binaries installed correctly" + + # Check launchd service + if launchctl list | grep -q com.kernel.hypeman; then + pass "launchd service is loaded" + else + fail "launchd service not loaded" + fi +else + [ -x /opt/hypeman/bin/hypeman-api ] || fail "hypeman-api binary not found" + pass "Binaries installed correctly" + + # Check systemd service + if systemctl is-active --quiet hypeman; then + pass "systemd service is running" + else + fail "systemd service not running" + fi +fi + +# Check config +if [ "$OS" = "darwin" ]; then + [ -f "$HOME/.config/hypeman/config" ] || fail "Config file not found" +else + [ -f /etc/hypeman/config ] || fail "Config file not found" +fi +pass "Config file exists" + +# ============================================================================= +# Phase 5: Cleanup +# ============================================================================= +info "Phase 5: Cleaning up..." +KEEP_DATA=false bash scripts/uninstall.sh + +# ============================================================================= +# Phase 6: Verify cleanup +# ============================================================================= +info "Phase 6: Verifying cleanup..." + +if [ "$OS" = "darwin" ]; then + [ ! -f /usr/local/bin/hypeman-api ] || fail "hypeman-api binary still exists after uninstall" + if launchctl list 2>/dev/null | grep -q com.kernel.hypeman; then + fail "launchd service still loaded after uninstall" + fi +else + [ ! -f /opt/hypeman/bin/hypeman-api ] || fail "hypeman-api binary still exists after uninstall" + if systemctl is-active --quiet hypeman 2>/dev/null; then + fail "systemd service still running after uninstall" + fi +fi +pass "Cleanup verified" + +# ============================================================================= +# Done +# ============================================================================= +echo "" +info "All E2E install tests passed!" diff --git a/scripts/install.sh b/scripts/install.sh index 063241f4..0959d66b 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -10,20 +10,15 @@ # CLI_VERSION - Install specific CLI version (default: latest) # BRANCH - Build from source using this branch (for development/testing) # BINARY_DIR - Use binaries from this directory instead of building/downloading -# INSTALL_DIR - Binary installation directory (default: /opt/hypeman/bin) -# DATA_DIR - Data directory (default: /var/lib/hypeman) -# CONFIG_DIR - Config directory (default: /etc/hypeman) +# INSTALL_DIR - Binary installation directory (default: /opt/hypeman/bin on Linux, /usr/local/bin on macOS) +# DATA_DIR - Data directory (default: /var/lib/hypeman on Linux, ~/Library/Application Support/hypeman on macOS) +# CONFIG_DIR - Config directory (default: /etc/hypeman on Linux, ~/.config/hypeman on macOS) # set -e REPO="kernel/hypeman" BINARY_NAME="hypeman-api" -INSTALL_DIR="${INSTALL_DIR:-/opt/hypeman/bin}" -DATA_DIR="${DATA_DIR:-/var/lib/hypeman}" -CONFIG_DIR="${CONFIG_DIR:-/etc/hypeman}" -CONFIG_FILE="${CONFIG_DIR}/config" -SYSTEMD_DIR="/etc/systemd/system" SERVICE_NAME="hypeman" # Colors for output (true color) @@ -45,57 +40,115 @@ find_release_with_artifact() { local archive_prefix="$2" local os="$3" local arch="$4" - + # Fetch recent release tags (up to 10) local tags tags=$(curl -fsSL "https://api.github.com/repos/${repo}/releases?per_page=10" 2>/dev/null | grep '"tag_name"' | cut -d'"' -f4) if [ -z "$tags" ]; then return 1 fi - + # Check each release for the artifact for tag in $tags; do local version_num="${tag#v}" local artifact_name="${archive_prefix}_${version_num}_${os}_${arch}.tar.gz" local artifact_url="https://github.com/${repo}/releases/download/${tag}/${artifact_name}" - + # Check if artifact exists (follow redirects, fail silently) if curl -fsSL --head "$artifact_url" >/dev/null 2>&1; then echo "$tag" return 0 fi done - + return 1 } +# ============================================================================= +# Detect OS and architecture (before pre-flight checks) +# ============================================================================= + +OS=$(uname -s | tr '[:upper:]' '[:lower:]') +ARCH=$(uname -m) +case $ARCH in + x86_64|amd64) + ARCH="amd64" + ;; + aarch64|arm64) + ARCH="arm64" + ;; + *) + error "Unsupported architecture: $ARCH (supported: amd64, arm64)" + ;; +esac + +if [ "$OS" != "linux" ] && [ "$OS" != "darwin" ]; then + error "Unsupported OS: $OS (supported: linux, darwin)" +fi + +# ============================================================================= +# OS-conditional defaults +# ============================================================================= + +if [ "$OS" = "darwin" ]; then + INSTALL_DIR="${INSTALL_DIR:-/usr/local/bin}" + DATA_DIR="${DATA_DIR:-$HOME/Library/Application Support/hypeman}" + CONFIG_DIR="${CONFIG_DIR:-$HOME/.config/hypeman}" +else + INSTALL_DIR="${INSTALL_DIR:-/opt/hypeman/bin}" + DATA_DIR="${DATA_DIR:-/var/lib/hypeman}" + CONFIG_DIR="${CONFIG_DIR:-/etc/hypeman}" +fi + +CONFIG_FILE="${CONFIG_DIR}/config" +SYSTEMD_DIR="/etc/systemd/system" + # ============================================================================= # Pre-flight checks - verify all requirements before doing anything # ============================================================================= info "Running pre-flight checks..." -# Check for root or sudo access SUDO="" -if [ "$EUID" -ne 0 ]; then - if ! command -v sudo >/dev/null 2>&1; then - error "This script requires root privileges. Please run as root or install sudo." +if [ "$OS" = "darwin" ]; then + # macOS pre-flight + if [ "$ARCH" != "arm64" ]; then + error "Intel Macs not supported" fi - # Try passwordless sudo first, then prompt from terminal if needed - if ! sudo -n true 2>/dev/null; then - info "Requesting sudo privileges..." - # Read password from /dev/tty (terminal) even when script is piped - if ! sudo -v < /dev/tty; then - error "Failed to obtain sudo privileges" + command -v codesign >/dev/null 2>&1 || error "codesign is required but not installed (install Xcode Command Line Tools)" + command -v docker >/dev/null 2>&1 || error "Docker CLI is required but not found. Install Docker via Colima or Docker Desktop." + # Check if we need sudo for INSTALL_DIR + if [ ! -w "$INSTALL_DIR" ] 2>/dev/null && [ ! -w "$(dirname "$INSTALL_DIR")" ] 2>/dev/null; then + if command -v sudo >/dev/null 2>&1; then + if ! sudo -n true 2>/dev/null; then + info "Requesting sudo privileges (needed for $INSTALL_DIR)..." + if ! sudo -v < /dev/tty; then + error "Failed to obtain sudo privileges" + fi + fi + SUDO="sudo" + fi + fi +else + # Linux pre-flight + if [ "$EUID" -ne 0 ]; then + if ! command -v sudo >/dev/null 2>&1; then + error "This script requires root privileges. Please run as root or install sudo." + fi + if ! sudo -n true 2>/dev/null; then + info "Requesting sudo privileges..." + if ! sudo -v < /dev/tty; then + error "Failed to obtain sudo privileges" + fi fi + SUDO="sudo" fi - SUDO="sudo" + command -v systemctl >/dev/null 2>&1 || error "systemctl is required but not installed (systemd not available?)" fi -# Check for required commands +# Common checks command -v curl >/dev/null 2>&1 || error "curl is required but not installed" command -v tar >/dev/null 2>&1 || error "tar is required but not installed" -command -v systemctl >/dev/null 2>&1 || error "systemctl is required but not installed (systemd not available?)" command -v openssl >/dev/null 2>&1 || error "openssl is required but not installed" # Count how many of BRANCH, VERSION, BINARY_DIR are set @@ -122,73 +175,56 @@ if [ -n "$BINARY_DIR" ]; then fi fi -# Detect OS -OS=$(uname -s | tr '[:upper:]' '[:lower:]') -if [ "$OS" != "linux" ]; then - error "Hypeman only supports Linux (detected: $OS)" -fi - -# Detect architecture -ARCH=$(uname -m) -case $ARCH in - x86_64|amd64) - ARCH="amd64" - ;; - aarch64|arm64) - ARCH="arm64" - ;; - *) - error "Unsupported architecture: $ARCH (supported: amd64, arm64)" - ;; -esac - info "Pre-flight checks passed" # ============================================================================= # System Configuration - KVM access and network capabilities # ============================================================================= -# Get the installing user (for adding to groups) INSTALL_USER="${SUDO_USER:-$(whoami)}" -# Ensure KVM access -if [ -e /dev/kvm ]; then - if getent group kvm &>/dev/null; then - if ! groups "$INSTALL_USER" 2>/dev/null | grep -qw kvm; then - info "Adding user ${INSTALL_USER} to kvm group..." - $SUDO usermod -aG kvm "$INSTALL_USER" - warn "You may need to log out and back in for kvm group membership to take effect" +if [ "$OS" = "darwin" ]; then + info "macOS uses NAT networking via Virtualization.framework, no system config needed" +else + # Ensure KVM access + if [ -e /dev/kvm ]; then + if getent group kvm &>/dev/null; then + if ! groups "$INSTALL_USER" 2>/dev/null | grep -qw kvm; then + info "Adding user ${INSTALL_USER} to kvm group..." + $SUDO usermod -aG kvm "$INSTALL_USER" + warn "You may need to log out and back in for kvm group membership to take effect" + fi fi + else + warn "/dev/kvm not found - KVM may not be available on this system" fi -else - warn "/dev/kvm not found - KVM may not be available on this system" -fi -# Enable IPv4 forwarding (required for VM networking) -CURRENT_IP_FORWARD=$(sysctl -n net.ipv4.ip_forward 2>/dev/null || echo "0") -if [ "$CURRENT_IP_FORWARD" != "1" ]; then - info "Enabling IPv4 forwarding..." - $SUDO sysctl -w net.ipv4.ip_forward=1 > /dev/null - - # Make it persistent across reboots - if [ -d /etc/sysctl.d ]; then - echo 'net.ipv4.ip_forward=1' | $SUDO tee /etc/sysctl.d/99-hypeman.conf > /dev/null - elif ! grep -q '^net.ipv4.ip_forward=1' /etc/sysctl.conf 2>/dev/null; then - echo 'net.ipv4.ip_forward=1' | $SUDO tee -a /etc/sysctl.conf > /dev/null + # Enable IPv4 forwarding (required for VM networking) + CURRENT_IP_FORWARD=$(sysctl -n net.ipv4.ip_forward 2>/dev/null || echo "0") + if [ "$CURRENT_IP_FORWARD" != "1" ]; then + info "Enabling IPv4 forwarding..." + $SUDO sysctl -w net.ipv4.ip_forward=1 > /dev/null + + # Make it persistent across reboots + if [ -d /etc/sysctl.d ]; then + echo 'net.ipv4.ip_forward=1' | $SUDO tee /etc/sysctl.d/99-hypeman.conf > /dev/null + elif ! grep -q '^net.ipv4.ip_forward=1' /etc/sysctl.conf 2>/dev/null; then + echo 'net.ipv4.ip_forward=1' | $SUDO tee -a /etc/sysctl.conf > /dev/null + fi fi -fi -# Increase file descriptor limit for Caddy (ingress) -if [ -d /etc/security/limits.d ]; then - if [ ! -f /etc/security/limits.d/99-hypeman.conf ]; then - info "Configuring file descriptor limits for ingress..." - $SUDO tee /etc/security/limits.d/99-hypeman.conf > /dev/null << 'LIMITS' + # Increase file descriptor limit for Caddy (ingress) + if [ -d /etc/security/limits.d ]; then + if [ ! -f /etc/security/limits.d/99-hypeman.conf ]; then + info "Configuring file descriptor limits for ingress..." + $SUDO tee /etc/security/limits.d/99-hypeman.conf > /dev/null << 'LIMITS' # Hypeman: Increased file descriptor limits for Caddy ingress * soft nofile 65536 * hard nofile 65536 root soft nofile 65536 root hard nofile 65536 LIMITS + fi fi fi @@ -210,13 +246,22 @@ if [ -n "$BINARY_DIR" ]; then # Copy binaries to TMP_DIR info "Copying binaries from ${BINARY_DIR}..." - for f in "${BINARY_NAME}" "hypeman-token" ".env.example"; do - [ -f "${BINARY_DIR}/${f}" ] || error "File ${f} not found in ${BINARY_DIR}" - done + if [ "$OS" = "darwin" ]; then + for f in "${BINARY_NAME}" "vz-shim" "hypeman-token" ".env.darwin.example"; do + [ -f "${BINARY_DIR}/${f}" ] || error "File ${f} not found in ${BINARY_DIR}" + done + cp "${BINARY_DIR}/vz-shim" "${TMP_DIR}/vz-shim" + cp "${BINARY_DIR}/.env.darwin.example" "${TMP_DIR}/.env.darwin.example" + chmod +x "${TMP_DIR}/vz-shim" + else + for f in "${BINARY_NAME}" "hypeman-token" ".env.example"; do + [ -f "${BINARY_DIR}/${f}" ] || error "File ${f} not found in ${BINARY_DIR}" + done + cp "${BINARY_DIR}/.env.example" "${TMP_DIR}/.env.example" + fi cp "${BINARY_DIR}/${BINARY_NAME}" "${TMP_DIR}/${BINARY_NAME}" cp "${BINARY_DIR}/hypeman-token" "${TMP_DIR}/hypeman-token" - cp "${BINARY_DIR}/.env.example" "${TMP_DIR}/.env.example" # Make binaries executable chmod +x "${TMP_DIR}/${BINARY_NAME}" @@ -226,27 +271,47 @@ if [ -n "$BINARY_DIR" ]; then elif [ -n "$BRANCH" ]; then # Build from source mode info "Building from source (branch: $BRANCH)..." - + BUILD_DIR="${TMP_DIR}/hypeman" BUILD_LOG="${TMP_DIR}/build.log" - + # Clone repo (quiet) if ! git clone --branch "$BRANCH" --depth 1 -q "https://github.com/${REPO}.git" "$BUILD_DIR" 2>&1 | tee -a "$BUILD_LOG"; then error "Failed to clone repository. Build log:\n$(cat "$BUILD_LOG")" fi - + info "Building binaries (this may take a few minutes)..." cd "$BUILD_DIR" - - # Build main binary (includes dependencies) - capture output, show on error - if ! make build >> "$BUILD_LOG" 2>&1; then - echo "" - echo -e "${RED}Build failed. Full build log:${NC}" - cat "$BUILD_LOG" - error "Build failed" + + if [ "$OS" = "darwin" ]; then + # macOS: build darwin targets and sign + if ! make build-darwin >> "$BUILD_LOG" 2>&1; then + echo "" + echo -e "${RED}Build failed. Full build log:${NC}" + cat "$BUILD_LOG" + error "Build failed" + fi + if ! make sign-darwin >> "$BUILD_LOG" 2>&1; then + echo "" + echo -e "${RED}Signing failed. Full build log:${NC}" + cat "$BUILD_LOG" + error "Signing failed" + fi + cp "bin/hypeman" "${TMP_DIR}/${BINARY_NAME}" + cp "bin/vz-shim" "${TMP_DIR}/vz-shim" + cp ".env.darwin.example" "${TMP_DIR}/.env.darwin.example" + else + # Linux: standard build + if ! make build >> "$BUILD_LOG" 2>&1; then + echo "" + echo -e "${RED}Build failed. Full build log:${NC}" + cat "$BUILD_LOG" + error "Build failed" + fi + cp "bin/hypeman" "${TMP_DIR}/${BINARY_NAME}" + cp ".env.example" "${TMP_DIR}/.env.example" fi - cp "bin/hypeman" "${TMP_DIR}/${BINARY_NAME}" - + # Build hypeman-token (not included in make build) if ! go build -o "${TMP_DIR}/hypeman-token" ./cmd/gen-jwt >> "$BUILD_LOG" 2>&1; then echo "" @@ -254,13 +319,10 @@ elif [ -n "$BRANCH" ]; then cat "$BUILD_LOG" error "Failed to build hypeman-token" fi - - # Copy .env.example for config template - cp ".env.example" "${TMP_DIR}/.env.example" - + VERSION="$BRANCH (source)" cd - > /dev/null - + info "Build complete" else # Download release mode @@ -285,15 +347,30 @@ else info "Extracting..." tar -xzf "${TMP_DIR}/${ARCHIVE_NAME}" -C "$TMP_DIR" + + # On macOS, codesign after extraction + if [ "$OS" = "darwin" ]; then + info "Signing binaries..." + codesign --force --sign - "${TMP_DIR}/${BINARY_NAME}" 2>/dev/null || true + [ -f "${TMP_DIR}/vz-shim" ] && codesign --force --sign - "${TMP_DIR}/vz-shim" 2>/dev/null || true + fi fi # ============================================================================= # Stop existing service if running # ============================================================================= -if $SUDO systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then - info "Stopping existing ${SERVICE_NAME} service..." - $SUDO systemctl stop "$SERVICE_NAME" +if [ "$OS" = "darwin" ]; then + PLIST_PATH="$HOME/Library/LaunchAgents/com.kernel.hypeman.plist" + if [ -f "$PLIST_PATH" ]; then + info "Stopping existing ${SERVICE_NAME} service..." + launchctl unload "$PLIST_PATH" 2>/dev/null || true + fi +else + if $SUDO systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then + info "Stopping existing ${SERVICE_NAME} service..." + $SUDO systemctl stop "$SERVICE_NAME" + fi fi # ============================================================================= @@ -308,70 +385,184 @@ $SUDO install -m 755 "${TMP_DIR}/${BINARY_NAME}" "${INSTALL_DIR}/${BINARY_NAME}" info "Installing hypeman-token to ${INSTALL_DIR}..." $SUDO install -m 755 "${TMP_DIR}/hypeman-token" "${INSTALL_DIR}/hypeman-token" -# Install wrapper script to /usr/local/bin for easy access -info "Installing hypeman-token wrapper to /usr/local/bin..." -$SUDO tee /usr/local/bin/hypeman-token > /dev/null << EOF +# Install vz-shim on macOS +if [ "$OS" = "darwin" ] && [ -f "${TMP_DIR}/vz-shim" ]; then + info "Installing vz-shim to ${INSTALL_DIR}..." + $SUDO install -m 755 "${TMP_DIR}/vz-shim" "${INSTALL_DIR}/vz-shim" +fi + +if [ "$OS" = "linux" ]; then + # Install wrapper script to /usr/local/bin for easy access + info "Installing hypeman-token wrapper to /usr/local/bin..." + $SUDO tee /usr/local/bin/hypeman-token > /dev/null << EOF #!/bin/bash -# Wrapper script for hypeman-token that loads config from /etc/hypeman/config +# Wrapper script for hypeman-token that loads config from ${CONFIG_FILE} set -a source ${CONFIG_FILE} set +a exec ${INSTALL_DIR}/hypeman-token "\$@" EOF -$SUDO chmod 755 /usr/local/bin/hypeman-token + $SUDO chmod 755 /usr/local/bin/hypeman-token +fi # ============================================================================= # Create directories # ============================================================================= info "Creating data directory at ${DATA_DIR}..." -$SUDO mkdir -p "$DATA_DIR" +if [ "$OS" = "darwin" ]; then + mkdir -p "$DATA_DIR" + mkdir -p "$DATA_DIR/logs" +else + $SUDO mkdir -p "$DATA_DIR" +fi info "Creating config directory at ${CONFIG_DIR}..." -$SUDO mkdir -p "$CONFIG_DIR" +if [ "$OS" = "darwin" ]; then + mkdir -p "$CONFIG_DIR" +else + $SUDO mkdir -p "$CONFIG_DIR" +fi # ============================================================================= # Create config file (if it doesn't exist) # ============================================================================= if [ ! -f "$CONFIG_FILE" ]; then - # Get config template (from local build or download from repo) - if [ -f "${TMP_DIR}/.env.example" ]; then - info "Using config template from source..." - cp "${TMP_DIR}/.env.example" "${TMP_DIR}/config" + if [ "$OS" = "darwin" ]; then + # macOS config + if [ -f "${TMP_DIR}/.env.darwin.example" ]; then + info "Using macOS config template from source..." + cp "${TMP_DIR}/.env.darwin.example" "${TMP_DIR}/config" + else + info "Downloading macOS config template..." + CONFIG_URL="https://raw.githubusercontent.com/${REPO}/${VERSION}/.env.darwin.example" + if ! curl -fsSL "$CONFIG_URL" -o "${TMP_DIR}/config"; then + error "Failed to download config template from ${CONFIG_URL}" + fi + fi + + # Expand ~ to $HOME (launchd doesn't do shell expansion) + sed -i '' "s|~/|${HOME}/|g" "${TMP_DIR}/config" + + # Generate random JWT secret + info "Generating JWT secret..." + JWT_SECRET=$(openssl rand -hex 32) + sed -i '' "s/^JWT_SECRET=.*/JWT_SECRET=${JWT_SECRET}/" "${TMP_DIR}/config" + + # Auto-detect Docker socket + DOCKER_SOCKET="" + if [ -n "$DOCKER_HOST" ]; then + DOCKER_SOCKET="${DOCKER_HOST#unix://}" + elif [ -S /var/run/docker.sock ]; then + DOCKER_SOCKET="/var/run/docker.sock" + elif [ -S "$HOME/.colima/default/docker.sock" ]; then + DOCKER_SOCKET="$HOME/.colima/default/docker.sock" + fi + if [ -n "$DOCKER_SOCKET" ]; then + info "Detected Docker socket: ${DOCKER_SOCKET}" + if grep -q '^DOCKER_SOCKET=' "${TMP_DIR}/config"; then + sed -i '' "s|^DOCKER_SOCKET=.*|DOCKER_SOCKET=${DOCKER_SOCKET}|" "${TMP_DIR}/config" + elif grep -q '^# DOCKER_SOCKET=' "${TMP_DIR}/config"; then + sed -i '' "s|^# DOCKER_SOCKET=.*|DOCKER_SOCKET=${DOCKER_SOCKET}|" "${TMP_DIR}/config" + else + echo "DOCKER_SOCKET=${DOCKER_SOCKET}" >> "${TMP_DIR}/config" + fi + fi + + info "Installing config file at ${CONFIG_FILE}..." + install -m 600 "${TMP_DIR}/config" "$CONFIG_FILE" else - info "Downloading config template..." - CONFIG_URL="https://raw.githubusercontent.com/${REPO}/${VERSION}/.env.example" - if ! curl -fsSL "$CONFIG_URL" -o "${TMP_DIR}/config"; then - error "Failed to download config template from ${CONFIG_URL}" + # Linux config + if [ -f "${TMP_DIR}/.env.example" ]; then + info "Using config template from source..." + cp "${TMP_DIR}/.env.example" "${TMP_DIR}/config" + else + info "Downloading config template..." + CONFIG_URL="https://raw.githubusercontent.com/${REPO}/${VERSION}/.env.example" + if ! curl -fsSL "$CONFIG_URL" -o "${TMP_DIR}/config"; then + error "Failed to download config template from ${CONFIG_URL}" + fi fi + + # Generate random JWT secret + info "Generating JWT secret..." + JWT_SECRET=$(openssl rand -hex 32) + sed -i "s/^JWT_SECRET=$/JWT_SECRET=${JWT_SECRET}/" "${TMP_DIR}/config" + + # Set fixed ports for production (instead of random ports used in dev) + sed -i "s/^# CADDY_ADMIN_PORT=.*/CADDY_ADMIN_PORT=2019/" "${TMP_DIR}/config" + sed -i "s/^# INTERNAL_DNS_PORT=.*/INTERNAL_DNS_PORT=5353/" "${TMP_DIR}/config" + + info "Installing config file at ${CONFIG_FILE}..." + $SUDO install -m 640 "${TMP_DIR}/config" "$CONFIG_FILE" + $SUDO chown root:root "$CONFIG_FILE" fi - - # Generate random JWT secret - info "Generating JWT secret..." - JWT_SECRET=$(openssl rand -hex 32) - sed -i "s/^JWT_SECRET=$/JWT_SECRET=${JWT_SECRET}/" "${TMP_DIR}/config" - - # Set fixed ports for production (instead of random ports used in dev) - # Replace entire line to avoid trailing comments being included in the value - sed -i "s/^# CADDY_ADMIN_PORT=.*/CADDY_ADMIN_PORT=2019/" "${TMP_DIR}/config" - sed -i "s/^# INTERNAL_DNS_PORT=.*/INTERNAL_DNS_PORT=5353/" "${TMP_DIR}/config" - - info "Installing config file at ${CONFIG_FILE}..." - # Config is 640 root:root - intentionally requires root/sudo to read since it contains JWT_SECRET. - # The hypeman service runs as root and the CLI wrapper uses sudo to source the config. - $SUDO install -m 640 "${TMP_DIR}/config" "$CONFIG_FILE" - $SUDO chown root:root "$CONFIG_FILE" else info "Config file already exists at ${CONFIG_FILE}, skipping..." fi # ============================================================================= -# Install systemd service +# Install service # ============================================================================= -info "Installing systemd service..." -$SUDO tee "${SYSTEMD_DIR}/${SERVICE_NAME}.service" > /dev/null << EOF +if [ "$OS" = "darwin" ]; then + # macOS: launchd plist + PLIST_DIR="$HOME/Library/LaunchAgents" + PLIST_PATH="${PLIST_DIR}/com.kernel.hypeman.plist" + mkdir -p "$PLIST_DIR" + + info "Installing launchd service..." + + # Build environment variables from config file + ENV_DICT="" + if [ -f "$CONFIG_FILE" ]; then + while IFS= read -r line; do + # Skip comments and empty lines + [[ "$line" =~ ^[[:space:]]*# ]] && continue + [[ -z "$line" ]] && continue + key="${line%%=*}" + value="${line#*=}" + ENV_DICT="${ENV_DICT} + ${key} + ${value}" + done < "$CONFIG_FILE" + fi + + cat > "$PLIST_PATH" << PLIST + + + + + Label + com.kernel.hypeman + ProgramArguments + + ${INSTALL_DIR}/${BINARY_NAME} + + EnvironmentVariables + + PATH + /opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin${ENV_DICT} + + KeepAlive + + RunAtLoad + + StandardOutPath + ${DATA_DIR}/logs/hypeman.log + StandardErrorPath + ${DATA_DIR}/logs/hypeman.log + + +PLIST + + info "Loading ${SERVICE_NAME} service..." + launchctl load "$PLIST_PATH" +else + # Linux: systemd + info "Installing systemd service..." + $SUDO tee "${SYSTEMD_DIR}/${SERVICE_NAME}.service" > /dev/null << EOF [Unit] Description=Hypeman API Server Documentation=https://github.com/kernel/hypeman @@ -396,17 +587,42 @@ ReadWritePaths=${DATA_DIR} WantedBy=multi-user.target EOF -# Reload systemd -info "Reloading systemd..." -$SUDO systemctl daemon-reload + info "Reloading systemd..." + $SUDO systemctl daemon-reload + + info "Enabling ${SERVICE_NAME} service..." + $SUDO systemctl enable "$SERVICE_NAME" + + info "Starting ${SERVICE_NAME} service..." + $SUDO systemctl start "$SERVICE_NAME" +fi + +# ============================================================================= +# Build builder image (macOS) +# ============================================================================= -# Enable service -info "Enabling ${SERVICE_NAME} service..." -$SUDO systemctl enable "$SERVICE_NAME" +if [ "$OS" = "darwin" ]; then + info "Attempting to build builder image..." + if command -v docker >/dev/null 2>&1; then + if [ -n "$BRANCH" ] && [ -d "${TMP_DIR}/hypeman" ]; then + BUILD_CONTEXT="${TMP_DIR}/hypeman" + else + BUILD_CONTEXT="" + fi -# Start service -info "Starting ${SERVICE_NAME} service..." -$SUDO systemctl start "$SERVICE_NAME" + if [ -n "$BUILD_CONTEXT" ] && [ -f "${BUILD_CONTEXT}/lib/builds/images/generic/Dockerfile" ]; then + if ! docker build -t hypeman/builder:latest -f "${BUILD_CONTEXT}/lib/builds/images/generic/Dockerfile" "$BUILD_CONTEXT" 2>/dev/null; then + warn "Failed to build builder image. You can build it later manually." + else + info "Builder image built successfully" + fi + else + warn "Builder image Dockerfile not available. Build it manually: docker build -t hypeman/builder:latest -f lib/builds/images/generic/Dockerfile ." + fi + else + warn "Docker not available, skipping builder image build" + fi +fi # ============================================================================= # Install Hypeman CLI @@ -416,7 +632,7 @@ CLI_REPO="kernel/hypeman-cli" if [ -z "$CLI_VERSION" ] || [ "$CLI_VERSION" == "latest" ]; then info "Fetching latest CLI version with available artifacts..." - CLI_VERSION=$(find_release_with_artifact "$CLI_REPO" "hypeman" "$OS" "$ARCH") + CLI_VERSION=$(find_release_with_artifact "$CLI_REPO" "hypeman" "$OS" "$ARCH" || true) if [ -z "$CLI_VERSION" ]; then warn "Failed to find a CLI release with artifacts for ${OS}/${ARCH}, skipping CLI installation" fi @@ -424,24 +640,28 @@ fi if [ -n "$CLI_VERSION" ]; then info "Installing Hypeman CLI version: $CLI_VERSION" - + CLI_VERSION_NUM="${CLI_VERSION#v}" CLI_ARCHIVE_NAME="hypeman_${CLI_VERSION_NUM}_${OS}_${ARCH}.tar.gz" CLI_DOWNLOAD_URL="https://github.com/${CLI_REPO}/releases/download/${CLI_VERSION}/${CLI_ARCHIVE_NAME}" - + info "Downloading CLI ${CLI_ARCHIVE_NAME}..." if curl -fsSL "$CLI_DOWNLOAD_URL" -o "${TMP_DIR}/${CLI_ARCHIVE_NAME}"; then info "Extracting CLI..." mkdir -p "${TMP_DIR}/cli" tar -xzf "${TMP_DIR}/${CLI_ARCHIVE_NAME}" -C "${TMP_DIR}/cli" - - # Install CLI binary - info "Installing hypeman CLI to ${INSTALL_DIR}..." - $SUDO install -m 755 "${TMP_DIR}/cli/hypeman" "${INSTALL_DIR}/hypeman-cli" - - # Install wrapper script to /usr/local/bin for PATH access - info "Installing hypeman wrapper to /usr/local/bin..." - $SUDO tee /usr/local/bin/hypeman > /dev/null << WRAPPER + + if [ "$OS" = "darwin" ]; then + info "Installing hypeman CLI to ${INSTALL_DIR}..." + install -m 755 "${TMP_DIR}/cli/hypeman" "${INSTALL_DIR}/hypeman" + else + # Install CLI binary + info "Installing hypeman CLI to ${INSTALL_DIR}..." + $SUDO install -m 755 "${TMP_DIR}/cli/hypeman" "${INSTALL_DIR}/hypeman-cli" + + # Install wrapper script to /usr/local/bin for PATH access + info "Installing hypeman wrapper to /usr/local/bin..." + $SUDO tee /usr/local/bin/hypeman > /dev/null << WRAPPER #!/bin/bash # Wrapper script for hypeman CLI that auto-generates API token set -a @@ -450,7 +670,8 @@ set +a export HYPEMAN_API_KEY=\$(${INSTALL_DIR}/hypeman-token -user-id "cli-user-\$(whoami)" 2>/dev/null) exec ${INSTALL_DIR}/hypeman-cli "\$@" WRAPPER - $SUDO chmod 755 /usr/local/bin/hypeman + $SUDO chmod 755 /usr/local/bin/hypeman + fi else warn "Failed to download CLI from ${CLI_DOWNLOAD_URL}, skipping CLI installation" fi @@ -473,12 +694,25 @@ EOF echo -e "${NC}" info "Hypeman installed successfully!" echo "" -echo " API Binary: ${INSTALL_DIR}/${BINARY_NAME}" -echo " CLI: /usr/local/bin/hypeman" -echo " Token tool: /usr/local/bin/hypeman-token" -echo " Config: ${CONFIG_FILE}" -echo " Data: ${DATA_DIR}" -echo " Service: ${SERVICE_NAME}.service" + +if [ "$OS" = "darwin" ]; then + echo " API Binary: ${INSTALL_DIR}/${BINARY_NAME}" + echo " VZ Shim: ${INSTALL_DIR}/vz-shim" + echo " CLI: ${INSTALL_DIR}/hypeman" + echo " Token tool: ${INSTALL_DIR}/hypeman-token" + echo " Config: ${CONFIG_FILE}" + echo " Data: ${DATA_DIR}" + echo " Service: ~/Library/LaunchAgents/com.kernel.hypeman.plist" + echo " Logs: ${DATA_DIR}/logs/hypeman.log" +else + echo " API Binary: ${INSTALL_DIR}/${BINARY_NAME}" + echo " CLI: /usr/local/bin/hypeman" + echo " Token tool: /usr/local/bin/hypeman-token" + echo " Config: ${CONFIG_FILE}" + echo " Data: ${DATA_DIR}" + echo " Service: ${SERVICE_NAME}.service" +fi + echo "" echo "" echo "Next steps:" diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh index ac45bb42..607b64ef 100755 --- a/scripts/uninstall.sh +++ b/scripts/uninstall.sh @@ -6,16 +6,12 @@ # curl -fsSL https://raw.githubusercontent.com/kernel/hypeman/main/scripts/uninstall.sh | bash # # Options (via environment variables): -# KEEP_DATA=false - Remove data directory (/var/lib/hypeman) - kept by default -# KEEP_CONFIG=true - Keep config directory (/etc/hypeman) +# KEEP_DATA=false - Remove data directory - kept by default +# KEEP_CONFIG=true - Keep config directory # set -e -INSTALL_DIR="/opt/hypeman" -DATA_DIR="/var/lib/hypeman" -CONFIG_DIR="/etc/hypeman" -SYSTEMD_DIR="/etc/systemd/system" SERVICE_NAME="hypeman" SERVICE_USER="hypeman" @@ -30,50 +26,100 @@ info() { echo -e "${GREEN}[INFO]${NC} $1"; } warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } error() { echo -e "${RED}[ERROR]${NC} $1"; exit 1; } +# ============================================================================= +# Detect OS +# ============================================================================= + +OS=$(uname -s | tr '[:upper:]' '[:lower:]') +if [ "$OS" != "linux" ] && [ "$OS" != "darwin" ]; then + error "Unsupported OS: $OS (supported: linux, darwin)" +fi + +# ============================================================================= +# OS-conditional defaults +# ============================================================================= + +if [ "$OS" = "darwin" ]; then + INSTALL_DIR="/usr/local/bin" + DATA_DIR="$HOME/Library/Application Support/hypeman" + CONFIG_DIR="$HOME/.config/hypeman" +else + INSTALL_DIR="/opt/hypeman" + DATA_DIR="/var/lib/hypeman" + CONFIG_DIR="/etc/hypeman" +fi + +SYSTEMD_DIR="/etc/systemd/system" + # ============================================================================= # Pre-flight checks # ============================================================================= info "Running pre-flight checks..." -# Check for root or sudo access SUDO="" -if [ "$EUID" -ne 0 ]; then - if ! command -v sudo >/dev/null 2>&1; then - error "This script requires root privileges. Please run as root or install sudo." +if [ "$OS" = "linux" ]; then + if [ "$EUID" -ne 0 ]; then + if ! command -v sudo >/dev/null 2>&1; then + error "This script requires root privileges. Please run as root or install sudo." + fi + if ! sudo -n true 2>/dev/null; then + info "Requesting sudo privileges..." + if ! sudo -v < /dev/tty; then + error "Failed to obtain sudo privileges" + fi + fi + SUDO="sudo" fi - # Try passwordless sudo first, then prompt from terminal if needed - if ! sudo -n true 2>/dev/null; then - info "Requesting sudo privileges..." - if ! sudo -v < /dev/tty; then - error "Failed to obtain sudo privileges" +elif [ "$OS" = "darwin" ]; then + if [ ! -w "$INSTALL_DIR" ] 2>/dev/null; then + if command -v sudo >/dev/null 2>&1; then + if ! sudo -n true 2>/dev/null; then + info "Requesting sudo privileges (needed for $INSTALL_DIR)..." + sudo -v < /dev/tty 2>/dev/null || true + fi + SUDO="sudo" fi fi - SUDO="sudo" fi # ============================================================================= # Stop and disable service # ============================================================================= -if $SUDO systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then - info "Stopping ${SERVICE_NAME} service..." - $SUDO systemctl stop "$SERVICE_NAME" -fi +if [ "$OS" = "darwin" ]; then + PLIST_PATH="$HOME/Library/LaunchAgents/com.kernel.hypeman.plist" + if [ -f "$PLIST_PATH" ]; then + info "Stopping ${SERVICE_NAME} service..." + launchctl unload "$PLIST_PATH" 2>/dev/null || true + fi +else + if $SUDO systemctl is-active --quiet "$SERVICE_NAME" 2>/dev/null; then + info "Stopping ${SERVICE_NAME} service..." + $SUDO systemctl stop "$SERVICE_NAME" + fi -if $SUDO systemctl is-enabled --quiet "$SERVICE_NAME" 2>/dev/null; then - info "Disabling ${SERVICE_NAME} service..." - $SUDO systemctl disable "$SERVICE_NAME" + if $SUDO systemctl is-enabled --quiet "$SERVICE_NAME" 2>/dev/null; then + info "Disabling ${SERVICE_NAME} service..." + $SUDO systemctl disable "$SERVICE_NAME" + fi fi # ============================================================================= -# Remove systemd service +# Remove service files # ============================================================================= -if [ -f "${SYSTEMD_DIR}/${SERVICE_NAME}.service" ]; then - info "Removing systemd service..." - $SUDO rm -f "${SYSTEMD_DIR}/${SERVICE_NAME}.service" - $SUDO systemctl daemon-reload +if [ "$OS" = "darwin" ]; then + if [ -f "$PLIST_PATH" ]; then + info "Removing launchd plist..." + rm -f "$PLIST_PATH" + fi +else + if [ -f "${SYSTEMD_DIR}/${SERVICE_NAME}.service" ]; then + info "Removing systemd service..." + $SUDO rm -f "${SYSTEMD_DIR}/${SERVICE_NAME}.service" + $SUDO systemctl daemon-reload + fi fi # ============================================================================= @@ -82,13 +128,31 @@ fi info "Removing binaries..." -# Remove wrapper scripts from /usr/local/bin -$SUDO rm -f /usr/local/bin/hypeman -$SUDO rm -f /usr/local/bin/hypeman-token +if [ "$OS" = "darwin" ]; then + $SUDO rm -f "${INSTALL_DIR}/hypeman-api" + $SUDO rm -f "${INSTALL_DIR}/vz-shim" + $SUDO rm -f "${INSTALL_DIR}/hypeman-token" + $SUDO rm -f "${INSTALL_DIR}/hypeman" +else + # Remove wrapper scripts from /usr/local/bin + $SUDO rm -f /usr/local/bin/hypeman + $SUDO rm -f /usr/local/bin/hypeman-token + + # Remove install directory + if [ -d "$INSTALL_DIR" ]; then + $SUDO rm -rf "$INSTALL_DIR" + fi +fi + +# ============================================================================= +# Kill orphan vz-shim processes (macOS) +# ============================================================================= -# Remove install directory -if [ -d "$INSTALL_DIR" ]; then - $SUDO rm -rf "$INSTALL_DIR" +if [ "$OS" = "darwin" ]; then + if pgrep -f vz-shim >/dev/null 2>&1; then + info "Killing orphan vz-shim processes..." + pkill -f vz-shim 2>/dev/null || true + fi fi # ============================================================================= @@ -100,7 +164,11 @@ if [ -d "$DATA_DIR" ]; then info "Keeping data directory: ${DATA_DIR}" else info "Removing data directory: ${DATA_DIR}" - $SUDO rm -rf "$DATA_DIR" + if [ "$OS" = "darwin" ]; then + rm -rf "$DATA_DIR" + else + $SUDO rm -rf "$DATA_DIR" + fi fi fi @@ -113,20 +181,26 @@ if [ -d "$CONFIG_DIR" ]; then warn "Keeping config directory: ${CONFIG_DIR}" else info "Removing config directory: ${CONFIG_DIR}" - $SUDO rm -rf "$CONFIG_DIR" + if [ "$OS" = "darwin" ]; then + rm -rf "$CONFIG_DIR" + else + $SUDO rm -rf "$CONFIG_DIR" + fi fi fi # ============================================================================= -# Remove hypeman user +# Remove hypeman user (Linux only) # ============================================================================= -if id "$SERVICE_USER" &>/dev/null; then - if [ "${KEEP_DATA:-true}" = "true" ]; then - info "Keeping system user: ${SERVICE_USER} (data is preserved)" - else - info "Removing system user: ${SERVICE_USER}" - $SUDO userdel "$SERVICE_USER" 2>/dev/null || true +if [ "$OS" = "linux" ]; then + if id "$SERVICE_USER" &>/dev/null; then + if [ "${KEEP_DATA:-true}" = "true" ]; then + info "Keeping system user: ${SERVICE_USER} (data is preserved)" + else + info "Removing system user: ${SERVICE_USER}" + $SUDO userdel "$SERVICE_USER" 2>/dev/null || true + fi fi fi @@ -150,19 +224,33 @@ echo "" if [ "${KEEP_DATA:-true}" = "true" ] && [ -d "$DATA_DIR" ]; then info "Data directory preserved: ${DATA_DIR}" - echo " To remove: sudo rm -rf ${DATA_DIR}" + if [ "$OS" = "darwin" ]; then + echo " To remove: rm -rf \"${DATA_DIR}\"" + else + echo " To remove: sudo rm -rf ${DATA_DIR}" + fi echo "" fi if [ "${KEEP_CONFIG:-false}" = "true" ] && [ -d "$CONFIG_DIR" ]; then info "Config directory preserved: ${CONFIG_DIR}" - echo " To remove: sudo rm -rf ${CONFIG_DIR}" + if [ "$OS" = "darwin" ]; then + echo " To remove: rm -rf \"${CONFIG_DIR}\"" + else + echo " To remove: sudo rm -rf ${CONFIG_DIR}" + fi echo "" fi -warn "Note: Caddy or Cloud Hypervisor processes may still be running." -echo " Check with: ps aux | grep -E 'caddy|cloud-h'" -echo " Kill all: sudo pkill -f caddy; sudo pkill -f cloud-h" +if [ "$OS" = "darwin" ]; then + warn "Note: vz-shim processes may still be running." + echo " Check with: ps aux | grep vz-shim" + echo " Kill all: pkill -f vz-shim" +else + warn "Note: Caddy or Cloud Hypervisor processes may still be running." + echo " Check with: ps aux | grep -E 'caddy|cloud-h'" + echo " Kill all: sudo pkill -f caddy; sudo pkill -f cloud-h" +fi echo "" echo "To reinstall:" diff --git a/vz.entitlements b/vz.entitlements new file mode 100644 index 00000000..41432913 --- /dev/null +++ b/vz.entitlements @@ -0,0 +1,14 @@ + + + + + + com.apple.security.virtualization + + + com.apple.security.network.server + + com.apple.security.network.client + + +